diff --git a/.gitignore b/.gitignore index 505a3b1ca2716907d5b292ab90c36df043c42bb6..22da39c5eb3a13e6224f436c108261ed958a1039 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ wheels/ # Virtual environments .venv + +*.png \ No newline at end of file diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/n_datasets_51 b/data/imputation_no/splits_all/tasks_all/datasets_all/n_datasets_51 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..efe6f4ec63c3610400ef5457242e2156c4d02a0f --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ab74bcd98d52b0516d8ece16fba2bf62d17355c3709f92d1176997deec7bdf9 +size 434350 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..0adb6fac03aa6f45e116483d94fa0784558cbbe6 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc338f509a3342554992e0ce9ba450fe38ef5b4c0434287cf6aff23593584051 +size 987831 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4a7e2cb7761873c8444607e3545b2c8e2502d19f --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b282bc5c113ba1ae79f044a2ce90c306ab0900f5301b949d50b532750fb4930b +size 108478 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_all/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..e7e4af8c567f04bb697103ff99d7625c5684a82c --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/website_leaderboard.csv @@ -0,0 +1,51 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1679.0,+64/-51,0.789,6.94,3.2,5.507,582.21,6.116,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1626.0,+89/-60,0.739,8.58,2.9,4.753,2040.22,8.92,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1580.0,+73/-54,0.68,10.27,3.97,5.726,2040.22,1.218,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1546.0,+72/-52,0.654,11.6,4.97,6.429,5.81,0.645,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1539.0,+55/-51,0.549,11.93,5.37,8.693,1735.72,2.559,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1536.0,+57/-48,0.556,12.03,6.93,8.739,2950.72,11.988,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1479.0,+65/-53,0.523,14.61,4.53,8.827,4907.64,286.652,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1466.0,+56/-43,0.428,15.26,8.29,10.223,3285.87,1.472,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1450.0,+36/-31,0.378,16.06,11.15,11.452,416.98,2.639,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1445.0,+46/-48,0.379,16.28,8.65,10.227,2950.72,0.663,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1432.0,+44/-43,0.36,16.98,10.14,10.958,1658.41,0.653,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1420.0,+60/-58,0.419,17.58,6.04,10.229,4907.64,39.96,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1419.0,+42/-44,0.345,17.6,9.69,11.142,1658.41,0.081,0.0,False,CPU +13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1409.0,+82/-54,0.385,18.12,7.61,11.442,4621.67,8.149,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1404.0,+56/-47,0.341,18.38,9.66,11.14,3285.87,0.173,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1400.0,+31/-30,0.28,18.63,13.38,12.143,416.98,0.334,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1389.0,+35/-36,0.296,19.21,12.62,12.22,693.49,1.69,0.0,False,CPU +17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1378.0,+41/-42,0.315,19.76,11.31,12.023,6.83,0.08,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1371.0,+45/-40,0.243,20.17,10.17,11.984,4621.67,0.475,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1366.0,+49/-41,0.267,20.45,11.91,12.584,846.88,2.552,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1363.0,+31/-33,0.241,20.59,14.97,12.54,693.49,0.308,0.0,False,CPU +21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1339.0,+70/-60,0.336,21.91,7.2,12.933,47.62,43.739,0.0,False,GPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1313.0,+46/-38,0.198,23.35,14.43,13.883,846.88,0.13,0.0,False,GPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1305.0,+54/-47,0.236,23.79,14.16,13.655,10.49,0.132,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1299.0,+50/-50,0.164,24.11,16.52,13.439,2874.67,1.952,0.0,False,CPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1282.0,+43/-40,0.171,25.07,16.13,15.191,2929.85,0.422,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1257.0,+44/-44,0.123,26.41,15.77,15.566,14.87,0.308,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1243.0,+47/-43,0.121,27.22,17.59,15.868,2929.85,0.051,0.0,False,CPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1240.0,+36/-41,0.086,27.36,19.31,14.957,10.06,1.689,0.0,False,GPU +29,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1219.0,+39/-38,0.091,28.49,20.6,15.422,1.94,0.123,0.0,False,CPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+52/-61,0.134,28.51,17.29,16.343,593.24,4.467,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1218.0,+41/-47,0.078,28.58,20.1,15.309,2874.67,0.131,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1212.0,+46/-42,0.101,28.89,19.17,16.88,183.02,0.761,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1208.0,+46/-48,0.115,29.06,19.52,17.03,7.32,0.05,0.0,False,CPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1190.0,+35/-29,0.048,30.03,26.04,16.049,1.96,0.142,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1179.0,+55/-48,0.076,30.61,20.12,17.756,373.18,0.771,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1172.0,+47/-52,0.076,30.96,19.36,17.853,183.02,0.091,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1153.0,+52/-55,0.057,31.96,21.29,17.842,593.24,0.306,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1141.0,+49/-46,0.052,32.55,19.95,18.528,373.18,0.085,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1073.0,+35/-48,0.012,35.84,28.74,19.694,9.99,0.126,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1043.0,+62/-61,0.036,37.16,29.18,23.269,3.23,0.919,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1005.0,+59/-60,0.013,38.74,28.39,22.719,2.86,0.373,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+45/-50,0.006,38.93,34.17,23.485,0.43,0.054,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,998.0,+63/-88,0.016,39.0,27.81,25.352,129.01,1.802,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,980.0,+59/-71,0.011,39.69,34.68,24.998,0.25,0.05,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,970.0,+64/-97,0.032,40.07,30.33,31.269,237.58,0.424,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,940.0,+71/-111,0.023,41.15,31.63,31.88,237.58,0.085,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,888.0,+68/-108,0.003,42.8,32.9,30.599,129.01,0.18,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,853.0,+76/-133,0.014,43.79,34.99,34.618,1.19,0.12,0.0,False,CPU +49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,636.0,+89/-113,0.0,47.95,43.9,44.269,0.19,0.036,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..b0fa597a376b42f522482b1694e0eba25d95eee6 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:348dfb8ae7ee428690bd12aed9771b32bcb4dfee97c8fc80db5696433d7b0ff4 +size 1739374 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/n_datasets_15 b/data/imputation_no/splits_all/tasks_all/datasets_medium/n_datasets_15 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..08de6f023ce8dc5c97e8a598ca1e99507b0dabef --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd03f6dff15fd9fcda1d0dbc2cc3f5bcc08cb41a0600762210985e0171c15373 +size 399354 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..807bebde5664d9cabee72b2663a8bf2808878dad --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b97ad68cbfd79a48e72703bb4d4b7d080493f752b2c1774faf646dac00a7c0d9 +size 981194 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..90197764f449b15e26a8aed93ca8955ded78bad2 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:701c5872555086215e2e04852b6a4f27bc9b7004114bf8d52735f904a99e98a6 +size 112567 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_all/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..b3b083c9ccee1c5ac6f18fe8d056525ec6194ca8 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/website_leaderboard.csv @@ -0,0 +1,51 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1825.0,+86/-46,0.836,5.78,2.85,2.566,289.53,3.187,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1804.0,+82/-49,0.808,6.29,3.08,2.669,432.35,4.085,0.0,False,CPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1676.0,+164/-124,0.66,10.21,3.09,5.382,735.58,11.736,0.0,False,GPU +3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1649.0,+109/-94,0.581,11.21,6.96,5.684,1719.82,1.675,0.0,False,GPU +4,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1635.0,+125/-116,0.598,11.74,6.46,5.75,2526.28,6.013,0.0,False,GPU +5,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1621.0,+86/-63,0.539,12.33,6.64,5.585,777.59,0.25,0.0,False,CPU +6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1620.0,+145/-112,0.591,12.34,5.04,6.097,735.58,1.386,0.0,False,GPU +7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1610.0,+94/-69,0.541,12.74,5.67,5.783,777.59,0.052,0.0,False,CPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1602.0,+76/-56,0.493,13.08,8.87,7.351,131.56,2.639,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1567.0,+105/-96,0.524,14.58,7.14,5.97,3.24,0.031,0.0,False,CPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1560.0,+87/-61,0.447,14.9,8.9,7.478,282.13,0.563,0.0,False,CPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1549.0,+116/-94,0.405,15.39,8.76,7.016,1719.82,0.081,0.0,False,GPU +12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1546.0,+155/-99,0.478,15.51,7.63,6.906,1.88,0.645,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1542.0,+155/-95,0.426,15.7,7.33,7.41,1993.14,0.619,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1534.0,+141/-125,0.48,16.05,4.17,7.024,4786.55,444.544,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1512.0,+80/-65,0.342,17.07,11.95,8.574,131.56,0.132,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1510.0,+69/-51,0.356,17.18,13.18,8.062,282.13,0.066,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1490.0,+152/-101,0.36,18.13,8.42,8.168,1993.14,0.061,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1481.0,+107/-91,0.32,18.51,12.5,8.151,2526.28,0.349,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1438.0,+108/-100,0.271,20.59,12.75,9.676,566.74,0.951,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1412.0,+122/-124,0.296,21.86,8.61,9.655,4786.55,42.641,0.0,False,GPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1387.0,+71/-71,0.179,23.12,16.95,9.424,1770.56,1.069,0.0,False,CPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1382.0,+97/-96,0.191,23.34,18.62,11.377,566.74,0.06,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1371.0,+104/-120,0.234,23.89,14.51,9.928,0.49,0.053,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1339.0,+130/-140,0.221,25.48,10.01,12.451,46.62,43.739,0.0,False,GPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1334.0,+119/-133,0.194,25.72,17.81,12.922,1517.65,0.134,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1328.0,+132/-111,0.177,26.01,17.36,11.007,5.16,0.074,0.0,False,GPU +27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1302.0,+77/-82,0.096,27.27,23.36,11.463,8.06,0.25,0.0,False,GPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1280.0,+84/-70,0.046,28.34,25.52,12.214,6.75,0.235,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1275.0,+78/-77,0.055,28.58,24.36,11.211,1770.56,0.051,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1271.0,+110/-134,0.125,28.79,22.32,13.812,1517.65,0.015,0.0,False,CPU +31,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1258.0,+62/-73,0.051,29.37,27.19,11.863,0.29,0.045,0.0,False,CPU +32,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1249.0,+86/-79,0.039,29.81,24.11,12.418,66.0,0.341,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1220.0,+97/-140,0.085,31.14,26.84,15.399,3.12,0.016,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1216.0,+97/-87,0.04,31.35,25.48,13.131,66.0,0.024,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1214.0,+108/-138,0.071,31.42,25.0,14.369,351.94,1.658,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1206.0,+71/-86,0.021,31.77,29.62,12.542,36.48,0.329,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1158.0,+63/-76,0.006,33.91,32.33,13.771,36.48,0.037,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1118.0,+67/-72,0.0,35.56,34.12,15.534,2.18,0.039,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1102.0,+85/-108,0.021,36.21,32.28,16.646,351.94,0.114,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1046.0,+145/-191,0.018,38.34,33.71,23.683,2.49,0.145,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1022.0,+138/-176,0.024,39.18,35.21,20.34,1.51,0.111,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+60/-87,0.0,39.92,39.15,18.698,0.2,0.021,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,998.0,+87/-126,0.006,39.99,34.2,22.252,150.3,10.071,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,950.0,+152/-228,0.005,41.48,38.36,31.109,43.64,0.081,0.0,False,CPU +45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,940.0,+106/-141,0.002,41.8,40.36,21.813,0.11,0.028,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,897.0,+156/-249,0.003,42.99,40.6,32.052,43.64,0.018,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,813.0,+58/-95,0.0,44.96,44.7,28.248,150.3,0.482,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,785.0,+139/-247,0.0,45.52,44.41,34.158,0.19,0.019,0.0,False,CPU +49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,566.0,+107/-159,0.0,48.53,48.44,42.214,0.1,0.072,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a2e79b1bdf2c2bcf9b96883dd445ff5925494c65 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b456446d2244486892edc8aadc235111c5ebd909560b2395fecbcc0dfc26278 +size 1634342 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/n_datasets_36 b/data/imputation_no/splits_all/tasks_all/datasets_small/n_datasets_36 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4ea02c88840a3a5ab3998b7dd826bda451461bb5 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc2daec9ff7c8fec95e11c76a7892200ba5b61fea726f612cf992675ade9ff04 +size 397284 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..cf47d9342444ac075c435373b6d92a6f03bb4717 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de7549430fdf91e2226031fe4d67a85f0153bc784a52430fd298040c8d6f8136 +size 981954 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..75d46ef2823987833d8af4dc44664957531dd3a3 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f30a940c53f43045c7a16b6bf14ce9c528cef2529410f3e61744ae4b604fdaf +size 107979 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_all/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..b4c4bbe3b79f567fcdc13f3b692e84f3177cd697 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/website_leaderboard.csv @@ -0,0 +1,51 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1640.0,+68/-54,0.769,7.42,3.38,6.733,659.63,7.444,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1625.0,+97/-63,0.772,7.9,2.83,4.491,2289.05,8.049,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1580.0,+79/-58,0.717,9.4,3.65,5.571,2289.05,1.143,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1565.0,+86/-62,0.728,9.98,4.34,6.23,6.76,0.636,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1506.0,+68/-49,0.546,12.38,6.91,10.012,3770.75,21.902,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1470.0,+83/-61,0.54,14.01,4.71,9.579,5119.36,218.711,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1464.0,+54/-48,0.441,14.28,7.78,11.203,3716.07,2.547,0.0,False,CPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1448.0,+59/-48,0.429,15.08,8.77,11.395,3553.12,1.741,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1433.0,+76/-59,0.471,15.79,5.38,10.468,5119.36,28.351,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1416.0,+56/-42,0.369,16.66,8.6,11.565,3770.75,1.778,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1404.0,+35/-36,0.33,17.29,12.49,13.16,892.41,2.574,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1381.0,+56/-51,0.333,18.49,10.3,12.378,3553.12,0.241,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1374.0,+43/-44,0.286,18.91,13.0,13.197,2476.51,0.814,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1367.0,+35/-28,0.254,19.29,14.08,13.63,892.41,0.346,0.0,False,CPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1360.0,+46/-36,0.263,19.62,13.76,13.375,2476.51,0.101,0.0,False,CPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1347.0,+47/-42,0.265,20.39,11.59,13.796,1126.71,2.941,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1346.0,+72/-67,0.384,20.43,6.44,13.133,50.32,43.709,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1340.0,+64/-61,0.296,20.78,8.22,13.813,6521.96,8.396,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1338.0,+46/-42,0.211,20.86,9.44,13.58,6521.96,0.506,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1335.0,+36/-38,0.233,21.01,15.28,14.196,884.18,2.368,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1319.0,+37/-41,0.227,21.92,14.94,14.545,9.64,0.128,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1317.0,+34/-37,0.193,22.02,15.86,14.406,884.18,0.393,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1302.0,+56/-46,0.261,22.86,13.15,14.758,13.18,0.17,0.0,False,GPU +23,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1294.0,+40/-44,0.201,23.35,13.19,14.927,1126.71,0.163,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1273.0,+47/-50,0.158,24.52,16.34,15.112,3865.95,2.158,0.0,False,CPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1268.0,+47/-45,0.162,24.79,15.52,16.137,3351.28,0.47,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1246.0,+48/-54,0.135,26.05,13.88,17.276,16.27,0.321,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1237.0,+48/-47,0.119,26.56,16.17,16.725,3351.28,0.056,0.0,False,CPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1230.0,+39/-42,0.103,26.95,17.53,16.1,15.69,4.691,0.0,False,GPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1224.0,+58/-62,0.16,27.29,15.32,17.165,1678.92,8.069,0.0,False,CPU +30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1217.0,+70/-64,0.135,27.68,16.72,18.687,499.14,1.466,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1208.0,+51/-51,0.128,28.2,17.53,17.71,9.08,0.057,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1201.0,+40/-47,0.087,28.58,18.73,17.016,3865.95,0.148,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1180.0,+64/-62,0.105,29.74,16.59,19.553,499.14,0.207,0.0,False,CPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1172.0,+54/-51,0.072,30.18,18.64,18.34,1678.92,0.826,0.0,False,CPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1169.0,+40/-39,0.046,30.3,25.59,17.793,3.61,0.17,0.0,False,CPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1168.0,+34/-37,0.031,30.4,24.97,17.712,3.29,0.253,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1158.0,+58/-70,0.091,30.94,18.82,19.98,590.04,1.461,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1117.0,+56/-70,0.057,33.05,18.3,20.776,590.04,0.15,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1058.0,+48/-47,0.017,35.95,26.97,21.427,13.69,0.155,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1042.0,+69/-90,0.043,36.67,27.64,23.096,5.44,1.098,0.0,False,GPU +41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+56/-75,0.008,38.52,32.45,25.48,0.51,0.089,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,999.0,+50/-60,0.008,38.56,26.27,23.71,5.48,0.685,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,998.0,+73/-96,0.019,38.58,25.8,26.644,111.84,1.215,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,993.0,+72/-72,0.015,38.81,32.76,26.326,0.47,0.092,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,976.0,+86/-100,0.043,39.48,27.89,31.336,520.95,0.647,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,953.0,+84/-107,0.031,40.38,28.96,31.808,520.95,0.113,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,910.0,+73/-92,0.004,41.9,29.64,31.579,111.84,0.111,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,872.0,+99/-124,0.02,43.07,32.15,34.809,2.59,0.138,0.0,False,CPU +49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,654.0,+110/-151,0.0,47.7,42.25,45.125,0.37,0.033,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..26e4df3291be85aa6833c266e09956111b2694ea --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd4f3b83906d55fa12c1647af4573b7c9816fc5a608ab11bfe4d0a5d5f10f5ce +size 1729917 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/n_datasets_33 b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/n_datasets_33 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..0e5d9ee29ccb1bc5ac6a7fabeabc5b6b6a39f8ae --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7745b6b4051e6c121203dddf9d74d5c8fd9f0f919e6eb82d895e1d89c25ab899 +size 428426 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..37a4cb77bb990d9a450ebc59474aad1c5d673b02 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12fc29ef59fbcd6e44a02905d150fd9379bd71a8c220b77a16e800c257acf463 +size 1048964 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ebd19d0e801a9bafb8b9ec68dc493b1ee141776e --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98d67d02c424a2064c561b33cd726834309a1e67f88ff45806d70266bb4fef6b +size 115758 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..0c25af8e6c53948d5f8fd5b83bab734bdf7c3cca --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,56 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1640.0,+95/-71,0.744,9.05,3.4,5.186,2166.18,7.945,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1637.0,+78/-54,0.72,9.17,4.23,7.606,611.55,7.01,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1602.0,+66/-63,0.723,10.54,5.18,6.822,6.14,0.586,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1597.0,+72/-62,0.693,10.73,4.24,6.284,2166.18,0.993,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1546.0,+74/-50,0.591,12.97,5.42,9.458,3899.42,55.833,0.0,False,GPU +5,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1516.0,+74/-47,0.559,14.46,5.84,10.623,136.59,3.391,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1514.0,+54/-45,0.504,14.56,8.05,11.014,3547.04,20.963,0.0,False,GPU +7,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1506.0,+74/-71,0.586,14.97,5.38,13.034,4.85,0.629,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1468.0,+75/-61,0.481,16.95,5.43,10.616,4980.83,207.372,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1459.0,+46/-52,0.384,17.46,9.83,12.287,3376.59,2.54,0.0,False,CPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1446.0,+55/-50,0.373,18.19,10.24,12.487,3372.56,1.658,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1442.0,+80/-69,0.445,18.39,5.94,11.415,4980.83,23.583,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1439.0,+82/-69,0.434,18.6,6.82,12.024,3899.42,0.983,0.0,False,GPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1422.0,+56/-47,0.337,19.55,10.33,12.593,3547.04,1.57,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1398.0,+36/-36,0.258,20.98,15.0,14.456,771.57,2.49,0.0,False,CPU +15,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1390.0,+71/-69,0.399,21.43,9.2,13.505,4.22,0.554,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1384.0,+58/-51,0.284,21.83,11.77,13.499,3372.56,0.208,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1382.0,+42/-37,0.24,21.91,15.08,14.384,2034.85,0.805,0.0,False,CPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1368.0,+37/-33,0.219,22.76,16.26,14.581,2034.85,0.097,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1366.0,+90/-66,0.381,22.88,6.96,14.141,48.9,43.804,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1363.0,+36/-30,0.189,23.09,16.97,14.932,771.57,0.322,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1342.0,+50/-45,0.205,24.36,14.6,15.104,1105.59,2.552,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1338.0,+36/-36,0.181,24.6,18.37,15.506,828.74,2.312,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1338.0,+73/-67,0.25,24.64,11.28,15.155,6147.69,8.149,0.0,False,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1338.0,+46/-49,0.17,24.64,11.71,14.737,6147.69,0.485,0.0,False,GPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1327.0,+48/-45,0.184,25.31,17.24,15.809,8.51,0.123,0.0,False,CPU +26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1320.0,+32/-34,0.142,25.71,18.6,15.718,828.74,0.336,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1313.0,+54/-53,0.226,26.15,14.69,15.881,12.24,0.154,0.0,False,GPU +28,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1296.0,+45/-40,0.151,27.25,16.29,16.243,1105.59,0.13,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1282.0,+48/-45,0.12,28.07,18.33,17.283,2929.85,0.433,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1275.0,+48/-54,0.124,28.54,20.3,16.303,3704.3,2.074,0.0,False,CPU +31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1267.0,+47/-42,0.118,28.99,15.86,17.906,16.16,0.307,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1252.0,+52/-49,0.084,29.95,18.24,17.885,2929.85,0.054,0.0,False,CPU +33,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1245.0,+42/-38,0.082,30.35,19.8,17.002,15.48,4.535,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1236.0,+51/-62,0.142,30.9,16.77,18.355,1459.62,8.056,0.0,False,CPU +35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1219.0,+49/-53,0.078,31.95,19.88,18.772,8.47,0.054,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1213.0,+60/-60,0.101,32.33,21.51,20.26,416.39,1.393,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1203.0,+47/-48,0.065,32.91,21.43,18.22,3704.3,0.143,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1181.0,+56/-61,0.063,34.18,20.76,19.544,1459.62,0.89,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1177.0,+57/-61,0.069,34.43,22.54,21.094,416.39,0.179,0.0,False,CPU +40,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1172.0,+37/-38,0.017,34.71,29.77,19.131,3.05,0.241,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1158.0,+33/-41,0.021,35.5,31.12,19.359,3.39,0.158,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1142.0,+56/-58,0.047,36.41,24.9,21.744,572.67,1.421,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1103.0,+47/-48,0.019,38.56,25.42,22.52,572.67,0.143,0.0,False,CPU +44,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1067.0,+50/-60,0.012,40.39,30.45,22.688,11.82,0.147,0.0,False,CPU +45,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1053.0,+71/-83,0.03,41.1,31.46,24.705,5.39,1.034,0.0,False,GPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1010.0,+55/-66,0.0,43.07,28.88,24.827,5.18,0.652,0.0,False,CPU +47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1004.0,+72/-90,0.012,43.34,31.16,28.179,100.58,1.106,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1003.0,+71/-108,0.044,43.36,30.05,31.292,504.17,0.629,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1002.0,+83/-87,0.008,43.4,36.68,27.374,0.42,0.078,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+55/-70,0.002,43.49,37.64,26.617,0.47,0.069,0.0,False,CPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,980.0,+74/-108,0.03,44.33,31.71,31.725,504.17,0.103,0.0,False,CPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,908.0,+78/-103,0.0,47.0,33.68,33.337,100.58,0.101,0.0,False,CPU +53,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,903.0,+82/-129,0.019,47.19,34.88,34.211,2.51,0.137,0.0,False,CPU +54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,680.0,+97/-169,0.0,52.44,45.72,45.686,0.29,0.028,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ab5fde715bb23f5b76c2dc50218d746567d19d6c --- /dev/null +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4eb4cf5b22ffe61c7126a39e9142cefcf53241d59bc19a5ba0daef6274f7377 +size 2212024 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/n_datasets_30 b/data/imputation_no/splits_all/tasks_binary/datasets_all/n_datasets_30 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..77a091b2d02c1b733b84291b78a4c726927a61d9 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8979e608672d5a0f32a339d07ff373d69dc10b01e1d2fa5cdbc9808720dd16c2 +size 429276 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d17d3d36fed03539daa2611d3312eba6d5ecd48e --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92fb1048a148173627e64602c7e3334ab49dcdc5eda005fd84595d81d8f26a17 +size 985265 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e4cb3ac95b85658a2268ec28fca38856280e424a --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2185545f1af5db251cf8b3acfd34ea001ca9e9e4c816eaf000981ccf7b09a8c0 +size 113807 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_binary/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..e840217cfd8bf4440b36d8485b8238fa6ca0ae4b --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/website_leaderboard.csv @@ -0,0 +1,53 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1657.0,+98/-71,0.803,7.38,2.99,5.749,442.09,6.381,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1568.0,+110/-63,0.681,10.55,3.6,6.02,1923.88,9.103,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1545.0,+101/-57,0.687,11.52,4.71,7.098,4.91,0.694,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1535.0,+101/-59,0.632,11.94,4.35,7.13,1923.88,1.329,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1515.0,+73/-65,0.555,12.85,5.23,9.038,1152.95,1.784,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1500.0,+53/-56,0.508,13.58,8.76,10.144,2775.44,8.849,0.0,False,GPU +6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1475.0,+76/-57,0.463,14.8,7.23,10.795,2229.7,1.387,0.0,False,GPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1438.0,+55/-38,0.382,16.76,10.85,12.3,328.96,1.102,0.0,False,CPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1422.0,+72/-62,0.436,17.64,5.97,9.134,4982.66,358.753,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1416.0,+81/-59,0.371,17.95,8.6,11.711,2229.7,0.133,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1408.0,+61/-52,0.353,18.37,9.9,12.08,1055.6,0.482,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1403.0,+59/-53,0.353,18.64,9.24,12.174,1055.6,0.063,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1403.0,+56/-50,0.374,18.67,9.35,12.048,4.27,0.064,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1400.0,+52/-56,0.298,18.83,9.7,11.944,2775.44,0.415,0.0,False,GPU +14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1385.0,+62/-53,0.314,19.7,11.17,13.208,472.99,0.621,0.0,False,CPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1384.0,+49/-31,0.268,19.73,13.42,13.159,328.96,0.126,0.0,False,CPU +16,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1380.0,+68/-68,0.371,19.99,9.43,14.06,250.36,1.708,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1367.0,+94/-94,0.374,20.73,8.42,12.88,3383.91,8.396,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1366.0,+58/-59,0.248,20.75,9.56,12.838,3383.91,0.51,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1355.0,+65/-58,0.301,21.39,7.03,10.825,4982.66,42.199,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1350.0,+49/-43,0.236,21.73,14.36,13.724,472.99,0.13,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1331.0,+58/-56,0.21,22.85,12.96,13.837,642.33,2.767,0.0,False,GPU +22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1318.0,+49/-41,0.197,23.61,15.72,13.457,2241.65,2.415,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1312.0,+69/-65,0.259,23.95,12.93,14.413,8.72,0.126,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1311.0,+52/-48,0.18,24.03,15.4,15.332,1560.02,0.404,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1302.0,+47/-58,0.178,24.57,13.95,15.116,642.33,0.146,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1283.0,+64/-72,0.243,25.68,9.36,14.197,49.67,43.824,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1274.0,+51/-46,0.121,26.2,16.49,16.045,1560.02,0.052,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1267.0,+50/-42,0.15,26.62,13.82,15.891,13.5,0.311,0.0,False,GPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1267.0,+60/-64,0.152,26.62,16.59,15.482,559.38,4.08,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1265.0,+44/-44,0.11,26.75,17.68,14.803,9.33,1.116,0.0,False,GPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1248.0,+49/-52,0.123,27.77,17.29,17.1,4.98,0.049,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1231.0,+66/-56,0.117,28.74,18.33,16.365,1.24,0.11,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1226.0,+46/-39,0.077,29.08,19.82,15.668,2241.65,0.136,0.0,False,CPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1191.0,+57/-63,0.061,31.05,19.79,17.521,559.38,0.276,0.0,False,CPU +35,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1188.0,+51/-53,0.061,31.26,22.1,18.0,125.38,0.674,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1179.0,+49/-34,0.047,31.73,27.14,17.148,0.81,0.093,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1154.0,+51/-57,0.035,33.12,22.93,19.362,169.75,0.603,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1146.0,+50/-53,0.038,33.56,23.89,19.162,125.38,0.069,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1119.0,+43/-54,0.018,35.01,23.64,20.091,169.75,0.059,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1093.0,+65/-48,0.019,36.39,27.59,20.071,4.95,0.091,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1057.0,+64/-82,0.021,38.16,24.5,21.46,2.56,0.267,0.0,False,CPU +42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1051.0,+84/-107,0.054,38.43,26.2,26.808,162.68,0.405,0.0,False,CPU +43,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1045.0,+109/-116,0.099,38.71,22.36,25.52,0.85,0.129,0.0,False,GPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1022.0,+61/-103,0.017,39.74,28.61,24.212,139.84,3.917,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1022.0,+91/-118,0.039,39.75,27.56,27.502,162.68,0.054,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+62/-79,0.013,40.7,33.84,24.287,0.25,0.029,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,983.0,+73/-101,0.026,41.38,33.73,26.581,2.8,0.564,0.0,False,GPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,959.0,+86/-119,0.024,42.33,30.48,29.448,0.84,0.094,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,955.0,+75/-92,0.01,42.47,36.5,25.759,0.2,0.037,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,915.0,+82/-136,0.0,43.91,31.65,29.325,139.84,0.193,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,602.0,+105/-199,0.0,50.36,46.66,43.863,0.15,0.023,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e09bac4ac78c6c9462ef01863fee3e21cd51228c --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:783344ba6c5f6c1d3bc94e50769f5330bbc21fb45bec22d445dd7f76d69ede86 +size 2071274 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/n_datasets_9 b/data/imputation_no/splits_all/tasks_binary/datasets_medium/n_datasets_9 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ad74532ae0c43305b13ce94b72a2641049023bce --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7db63d502490ea6f171e4158f07599e360bcbf896034b80539013dc4cf2669f8 +size 419894 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4ad2c8ac9f7e4fe6df034454bbc6e30b00baa280 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44ee0506dd2e7e004d3ad5fe55a48e72879dc48b4002bceb5592b4ebd3c996f5 +size 990457 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..311239e3010d7de2ef154431c11385fcdb0a3f47 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0b85edce9fc6e7fed4af0e845e4a7252b0e0c96601efb601290a81220d792ec +size 118425 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..8cfc57ca22cb179cf53215cdc5a783b81af95939 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv @@ -0,0 +1,54 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1966.0,+155/-60,0.904,4.95,2.42,2.212,275.07,2.448,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1904.0,+176/-64,0.851,6.35,2.7,2.49,325.91,2.156,0.0,False,CPU +2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1759.0,+141/-88,0.631,10.98,5.87,4.725,619.15,0.198,0.0,False,CPU +3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1754.0,+144/-81,0.636,11.17,4.87,5.018,619.15,0.016,0.0,False,CPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1751.0,+113/-38,0.575,11.28,8.0,5.44,1633.8,2.053,0.0,False,GPU +5,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1736.0,+126/-79,0.63,11.87,5.47,4.763,2.21,0.025,0.0,False,CPU +6,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1718.0,+152/-92,0.545,12.6,6.95,7.508,110.13,0.386,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1701.0,+149/-83,0.524,13.3,8.51,7.923,86.85,0.668,0.0,False,CPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1681.0,+192/-136,0.513,14.19,5.59,7.27,804.65,0.71,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1679.0,+154/-138,0.544,14.26,7.1,6.907,2139.17,6.41,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1660.0,+210/-122,0.489,15.1,5.39,8.381,842.24,15.119,0.0,False,GPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1652.0,+124/-66,0.376,15.51,8.78,7.028,1633.8,0.088,0.0,False,GPU +12,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1640.0,+124/-80,0.421,16.04,11.5,8.376,110.13,0.061,0.0,False,CPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1625.0,+197/-156,0.458,16.73,6.29,7.978,804.65,0.063,0.0,False,GPU +14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1610.0,+194/-117,0.419,17.44,6.83,9.033,842.24,1.465,0.0,False,GPU +15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1582.0,+158/-128,0.416,18.81,7.95,8.63,2.3,0.896,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1580.0,+163/-89,0.323,18.95,12.1,9.74,86.85,0.078,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1564.0,+170/-166,0.42,19.72,8.41,9.211,8.53,3.578,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1552.0,+126/-84,0.253,20.35,12.42,9.782,2139.17,0.349,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1536.0,+178/-158,0.347,21.12,7.53,9.019,4860.88,475.791,0.0,False,GPU +20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1527.0,+95/-112,0.248,21.62,14.48,9.12,1156.69,1.558,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1490.0,+141/-106,0.196,23.53,15.19,10.678,572.82,1.935,0.0,False,GPU +22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1478.0,+124/-111,0.174,24.2,17.96,12.87,270.2,0.192,0.0,False,CPU +23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1461.0,+126/-146,0.228,25.07,13.69,10.861,0.49,0.053,0.0,False,CPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1454.0,+132/-118,0.163,25.44,19.32,12.698,572.82,0.109,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1453.0,+156/-178,0.241,25.48,14.6,10.49,5.5,0.074,0.0,False,GPU +26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1431.0,+105/-129,0.123,26.67,20.62,11.432,371.59,2.386,0.0,False,CPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1430.0,+81/-65,0.077,26.75,23.68,10.923,6.41,0.247,0.0,False,GPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1417.0,+121/-138,0.136,27.44,20.16,13.96,96.61,0.754,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1404.0,+129/-107,0.091,28.11,24.03,13.997,270.2,0.015,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1380.0,+97/-126,0.069,29.38,23.47,11.531,1156.69,0.054,0.0,False,CPU +31,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1374.0,+134/-137,0.087,29.67,18.69,12.762,4860.88,42.641,0.0,False,GPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1361.0,+129/-110,0.05,30.35,26.41,15.831,1.07,0.018,0.0,False,CPU +33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1343.0,+98/-137,0.069,31.22,28.09,12.461,0.29,0.033,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1319.0,+52/-59,0.002,32.42,30.58,14.148,42.95,0.226,0.0,False,CPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1314.0,+88/-85,0.008,32.69,29.68,13.953,8.34,0.25,0.0,False,GPU +36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1284.0,+174/-186,0.035,34.12,20.85,17.019,47.03,45.876,0.0,False,GPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1280.0,+61/-90,0.0,34.32,32.02,13.457,40.36,0.318,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1279.0,+74/-74,0.005,34.35,30.6,15.128,42.95,0.021,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1278.0,+84/-119,0.036,34.41,29.27,15.031,371.59,0.102,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1224.0,+66/-92,0.0,36.86,35.23,15.173,1.86,0.046,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1218.0,+64/-73,0.0,37.12,35.45,15.174,40.36,0.028,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1206.0,+134/-218,0.042,37.63,32.31,18.046,1.52,0.115,0.0,False,CPU +43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1162.0,+147/-233,0.009,39.45,35.34,24.034,43.64,0.085,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1140.0,+101/-190,0.011,40.28,31.54,20.729,513.06,78.02,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1112.0,+166/-225,0.005,41.35,38.03,25.027,43.64,0.022,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+88/-208,0.0,44.96,44.23,22.275,0.12,0.014,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,982.0,+135/-219,0.0,45.44,43.9,26.73,0.22,0.024,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,972.0,+202/-455,0.025,45.72,39.86,28.513,2.62,0.186,0.0,False,GPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,918.0,+91/-188,0.0,47.05,46.26,30.352,0.24,0.036,0.0,False,GPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,916.0,+100/-247,0.0,47.09,46.63,24.142,0.1,0.021,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,883.0,+76/-126,0.0,47.81,47.49,27.794,513.06,2.568,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,514.0,+107/-398,0.0,52.27,52.24,45.591,0.11,0.087,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..69b8832e2e72d5de7d29b42399c9760a8bb72b96 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0179b2e0997e900c319a3a5cf43b29f396a61a3a97b23e2d07c337dce82192e3 +size 2102015 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/n_datasets_21 b/data/imputation_no/splits_all/tasks_binary/datasets_small/n_datasets_21 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..93cc3cbe3bcc99b383026c19fbdde2abd16f242e --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09d6cd5b871ef3135cbae865a4a9965b387a85614651c1074b1a453df0ea4cbe +size 430099 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..612fbd3d513e8028761884078a4a8fa8283bd1bf --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58ec2732e734b0b9acc52e0f938e161f18521a103ec56468d54f0d93292b6060 +size 957912 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..428986fbf1cde7359601845a8bed08b80569218e --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c86e4d19ab70052c9bb6456343c526867fef05756fefd94833876c5013162365 +size 124173 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_binary/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..b97772301012df819761b01a357ab62e2af30869 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/website_leaderboard.csv @@ -0,0 +1,53 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1582.0,+110/-77,0.748,8.48,3.33,7.287,593.57,8.084,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1579.0,+98/-56,0.789,8.57,4.04,6.466,5.96,0.554,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1574.0,+142/-69,0.753,8.74,3.16,5.03,2052.28,8.252,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1547.0,+128/-57,0.713,9.74,3.78,6.337,2052.28,1.266,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1435.0,+81/-50,0.471,14.72,9.21,12.183,2950.72,15.595,0.0,False,GPU +5,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1425.0,+85/-56,0.434,15.23,8.35,12.33,2457.54,1.658,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1415.0,+90/-69,0.417,15.75,8.92,11.866,1921.63,1.718,0.0,False,CPU +7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1402.0,+82/-72,0.465,16.4,5.52,9.204,5351.68,268.579,0.0,False,GPU +8,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1390.0,+81/-66,0.467,17.07,7.72,14.126,317.91,1.983,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1370.0,+83/-73,0.385,18.2,5.58,10.014,5351.68,37.621,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1366.0,+49/-38,0.31,18.4,12.57,14.198,512.57,1.18,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1361.0,+85/-59,0.327,18.67,10.34,13.333,2457.54,0.178,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1333.0,+44/-34,0.237,20.3,14.26,14.644,512.57,0.243,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1330.0,+80/-60,0.259,20.46,10.21,14.072,2950.72,1.225,0.0,False,GPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1319.0,+88/-68,0.243,21.12,8.79,14.168,4806.04,0.512,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1310.0,+52/-43,0.224,21.68,14.73,15.257,1096.5,0.584,0.0,False,CPU +16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1309.0,+64/-47,0.253,21.73,13.86,15.195,4.81,0.08,0.0,False,CPU +17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1305.0,+53/-43,0.222,21.97,15.14,15.264,1096.5,0.074,0.0,False,CPU +18,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1297.0,+88/-76,0.324,22.45,7.6,13.006,51.83,43.615,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1290.0,+51/-68,0.214,22.82,12.25,15.211,711.84,2.983,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1289.0,+47/-43,0.203,22.93,15.54,15.673,693.49,1.398,0.0,False,CPU +21,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1278.0,+68/-61,0.262,23.55,12.39,16.117,10.08,0.152,0.0,False,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1276.0,+114/-107,0.292,23.67,9.47,15.466,4806.04,8.461,0.0,False,GPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1267.0,+48/-48,0.176,24.26,14.62,16.409,1634.54,0.422,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1265.0,+62/-65,0.21,24.38,11.3,16.741,14.37,0.334,0.0,False,GPU +25,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1264.0,+42/-33,0.149,24.39,16.25,16.038,693.49,0.194,0.0,False,CPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1263.0,+53/-61,0.182,24.48,12.54,16.17,711.84,0.162,0.0,False,GPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1259.0,+81/-56,0.167,24.72,16.57,15.336,2665.36,2.619,0.0,False,CPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1243.0,+49/-50,0.134,25.68,14.64,16.945,1634.54,0.054,0.0,False,CPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1224.0,+62/-72,0.163,26.88,15.4,17.235,644.55,5.974,0.0,False,CPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1222.0,+52/-53,0.153,26.97,15.18,17.663,7.32,0.054,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1221.0,+56/-50,0.123,27.02,16.05,16.483,10.89,2.249,0.0,False,GPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1185.0,+72/-49,0.08,29.23,18.73,17.457,2665.36,0.143,0.0,False,CPU +33,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1174.0,+64/-77,0.071,29.9,17.46,18.602,644.55,0.398,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1167.0,+63/-83,0.087,30.28,19.64,19.966,203.15,0.968,0.0,False,CPU +35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1162.0,+61/-59,0.058,30.6,22.99,18.743,1.94,0.121,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1132.0,+45/-51,0.033,32.32,27.15,19.177,2.44,0.121,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1131.0,+59/-76,0.054,32.38,21.08,20.887,203.15,0.091,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1107.0,+54/-85,0.049,33.76,20.87,21.614,484.22,1.025,0.0,False,CPU +39,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1074.0,+108/-127,0.142,35.56,18.38,23.463,1.3,0.15,0.0,False,GPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1073.0,+51/-82,0.024,35.64,21.78,22.235,484.22,0.086,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1056.0,+61/-76,0.028,36.52,25.42,22.185,9.39,0.13,0.0,False,CPU +42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1021.0,+89/-132,0.073,38.3,23.69,28.012,278.97,0.873,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1012.0,+76/-85,0.012,38.71,22.33,22.938,2.97,0.388,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+58/-111,0.018,39.3,30.97,25.166,0.42,0.041,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,998.0,+96/-137,0.053,39.38,24.75,28.578,278.97,0.103,0.0,False,CPU +46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,988.0,+75/-112,0.019,39.87,27.69,25.72,90.71,1.623,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,986.0,+81/-105,0.027,39.92,31.85,25.768,3.71,0.919,0.0,False,GPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,964.0,+79/-130,0.015,40.92,33.63,26.469,0.33,0.043,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,952.0,+88/-162,0.034,41.4,27.1,30.627,1.48,0.139,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,921.0,+80/-144,0.001,42.66,27.85,29.993,90.71,0.125,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,616.0,+109/-176,0.0,49.97,44.93,43.126,0.22,0.02,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..fd87c7ee914de6db8defa5e91fba1e6ef61e1eb1 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78172565bb2582fdac1b2d466832f1a9afe26356d22bc415349ffaf92aca2f6f +size 2084457 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/n_datasets_20 b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/n_datasets_20 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..6860dde1bdb4ae69c419d83bf083e0bf26c67ca7 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33786548e598a47b40a21281a191ef04bfcbc56650372b56c5483527bb4cced6 +size 483576 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..871469bbcacbe460b2dbc9a3850e8c3ec089eb90 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:607718a2189859c597b085474b19b7604dd69b9214b9368fe8b78d4767e0c1e8 +size 1038997 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1a325393f01648fd856d7db66461049ba96d1939 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a98a00b37dd6733cac15b4511aff08f6d130dc899b4406659209decbb2889a1c +size 131179 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..f3f6c9e65b8ce5c20388b697463d899b2f1ab624 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1600.0,+91/-54,0.729,10.36,5.33,7.184,5.52,0.529,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1588.0,+113/-69,0.685,10.87,4.46,5.826,2046.25,8.114,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1584.0,+88/-56,0.662,11.04,4.67,8.183,592.15,8.082,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1569.0,+104/-64,0.657,11.74,4.79,7.089,2046.25,1.254,0.0,False,GPU +4,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1510.0,+92/-60,0.585,14.67,5.88,10.733,144.27,3.772,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1497.0,+79/-54,0.512,15.33,6.23,10.68,3445.6,51.274,0.0,False,GPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1486.0,+98/-79,0.565,15.96,5.68,11.297,3.76,0.527,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1479.0,+79/-67,0.493,16.35,5.93,10.213,8.06,1.714,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1442.0,+65/-50,0.385,18.49,11.6,13.145,2879.46,14.626,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1438.0,+75/-59,0.362,18.78,9.99,13.186,2379.18,1.595,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1430.0,+69/-59,0.406,19.25,9.32,14.423,316.0,1.931,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1419.0,+76/-58,0.339,19.93,11.82,12.9,1828.68,1.648,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1419.0,+84/-68,0.391,19.94,6.64,10.011,5319.2,249.314,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1415.0,+108/-75,0.407,20.2,6.42,13.416,3445.6,0.995,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1399.0,+104/-71,0.361,21.18,6.68,10.637,5319.2,29.488,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1381.0,+86/-58,0.273,22.36,12.32,14.163,2379.18,0.176,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1376.0,+45/-44,0.233,22.67,15.32,15.222,449.53,1.149,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1372.0,+96/-85,0.405,22.91,8.63,15.503,3.73,0.488,0.0,False,GPU +18,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1352.0,+65/-52,0.22,24.25,12.67,14.896,2879.46,0.944,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1347.0,+63/-58,0.225,24.59,10.35,14.824,4575.49,0.51,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1347.0,+38/-37,0.17,24.6,17.27,15.649,449.53,0.215,0.0,False,CPU +21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1326.0,+105/-79,0.303,25.96,8.98,13.65,51.44,43.709,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1326.0,+61/-49,0.189,25.97,16.69,16.157,4.7,0.08,0.0,False,CPU +23,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1323.0,+56/-53,0.151,26.23,17.99,16.287,1081.93,0.556,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1316.0,+56/-45,0.159,26.64,19.09,16.312,1081.93,0.074,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1308.0,+67/-67,0.219,27.19,14.07,16.789,9.81,0.146,0.0,False,GPU +26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1302.0,+53/-58,0.162,27.58,15.63,16.244,592.34,2.767,0.0,False,GPU +27,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1298.0,+52/-52,0.138,27.86,19.56,16.778,645.57,1.032,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1295.0,+66/-71,0.174,28.07,13.53,17.352,14.36,0.311,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1292.0,+50/-50,0.129,28.32,17.46,17.207,1618.46,0.415,0.0,False,CPU +30,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1290.0,+97/-103,0.254,28.39,11.16,16.484,4575.49,8.396,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1286.0,+69/-58,0.14,28.68,19.64,15.992,2576.18,2.415,0.0,False,CPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1280.0,+54/-58,0.127,29.14,15.16,17.188,592.34,0.146,0.0,False,GPU +33,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1275.0,+42/-42,0.084,29.47,19.6,17.137,645.57,0.172,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1271.0,+52/-55,0.1,29.74,16.79,17.71,1618.46,0.054,0.0,False,CPU +35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+54/-58,0.115,30.88,16.81,18.247,6.86,0.052,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1253.0,+67/-78,0.143,30.93,17.0,17.887,618.9,5.468,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1250.0,+55/-56,0.088,31.15,18.15,17.097,10.47,1.987,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1213.0,+63/-50,0.062,33.59,21.56,17.997,2576.18,0.136,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1206.0,+67/-74,0.058,34.09,19.4,19.212,618.9,0.352,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1183.0,+84/-72,0.067,35.56,23.19,21.088,199.83,0.865,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1172.0,+64/-60,0.025,36.23,29.08,19.888,1.77,0.119,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1150.0,+78/-78,0.04,37.6,25.09,21.933,199.83,0.086,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1144.0,+40/-43,0.018,38.0,32.13,20.296,2.2,0.12,0.0,False,CPU +44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1110.0,+95/-131,0.115,40.03,21.41,23.531,1.24,0.143,0.0,False,GPU +45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1108.0,+65/-85,0.02,40.14,26.17,22.968,440.56,0.933,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1086.0,+75/-74,0.018,41.44,29.26,22.66,8.39,0.127,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1078.0,+52/-79,0.004,41.88,24.91,23.518,440.56,0.085,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1057.0,+98/-123,0.071,43.04,26.02,28.113,273.27,0.715,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1043.0,+78/-87,0.0,43.76,24.97,23.546,2.84,0.381,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1034.0,+94/-126,0.047,44.19,27.79,28.584,273.27,0.098,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1020.0,+83/-108,0.02,44.93,30.67,26.329,89.68,1.55,0.0,False,CPU +52,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1002.0,+98/-112,0.011,45.79,36.55,26.937,2.96,0.796,0.0,False,GPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+63/-105,0.003,45.87,37.86,26.578,0.42,0.04,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,989.0,+102/-139,0.03,46.39,30.04,30.404,1.44,0.138,0.0,False,CPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,976.0,+93/-114,0.012,46.96,38.72,27.729,0.31,0.042,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,953.0,+86/-138,0.001,47.95,30.7,30.595,89.68,0.12,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,639.0,+130/-215,0.0,55.9,49.79,43.912,0.21,0.019,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..638dd53bb30bfe8715f63867ac7b3878ec984073 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f936028ebc0edda40cbeaa24cd8fa98995ab66cc1da7cc7a583e2ba79c96a503 +size 2690375 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/n_datasets_38 b/data/imputation_no/splits_all/tasks_classification/datasets_all/n_datasets_38 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d16e93ce4dee0ee950330748a3b999ba5486fe50 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41b2b68c08bf4c8b20b53c19cab3d3913cb96d16e8538fe0acd115cb8eae7eff +size 439786 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..b9cea6ba0bdd14469114dda09ac20e15282599a8 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92a50fc0f408d31414b0fa5d7c12d492410c765c812b7fbae452aa4a323e4984 +size 1001503 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7c8e1b33e973f467b37404544980d59d4a6940ed --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caeba71b3ca8c6b5e75d780089f20ceb917a3d1c9e4a0c76fa86bc6b8e83c1e3 +size 113343 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_classification/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..af4dbd30f40bf40a7d9134fbf4ffef2bd341430a --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/website_leaderboard.csv @@ -0,0 +1,53 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1673.0,+76/-63,0.809,7.05,3.08,6.447,545.23,6.469,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1593.0,+81/-76,0.709,9.74,3.16,5.921,2046.25,8.976,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1566.0,+79/-70,0.671,10.79,3.98,6.967,2046.25,1.329,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1550.0,+72/-69,0.683,11.47,4.59,7.392,5.76,0.794,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1516.0,+78/-58,0.533,13.01,5.2,10.128,1684.49,1.97,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1500.0,+62/-54,0.508,13.74,8.2,10.684,2879.46,12.49,0.0,False,GPU +6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1472.0,+76/-48,0.449,15.12,7.64,12.019,2466.21,1.502,0.0,False,GPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1440.0,+48/-36,0.374,16.81,11.12,13.132,382.05,1.488,0.0,False,CPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1421.0,+77/-56,0.428,17.83,6.22,11.076,4940.61,307.751,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1417.0,+73/-58,0.367,18.06,8.85,12.922,2466.21,0.176,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1416.0,+58/-47,0.328,18.11,9.33,12.344,2879.46,0.598,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1415.0,+54/-46,0.349,18.15,10.54,12.625,1372.94,0.556,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1406.0,+58/-44,0.338,18.65,10.04,12.832,1372.94,0.074,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1396.0,+41/-34,0.279,19.26,13.36,13.856,382.05,0.254,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1386.0,+53/-47,0.305,19.81,11.85,13.978,685.87,1.455,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1383.0,+50/-46,0.325,19.97,10.59,13.472,5.72,0.076,0.0,False,CPU +16,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1376.0,+57/-54,0.354,20.4,10.32,15.277,282.72,1.886,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1371.0,+89/-82,0.345,20.64,9.1,13.51,4879.89,8.744,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1370.0,+60/-41,0.238,20.74,9.81,13.555,4879.89,0.525,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1359.0,+47/-44,0.245,21.35,14.58,14.358,685.87,0.205,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1358.0,+67/-58,0.307,21.4,7.33,12.802,4940.61,41.606,0.0,False,GPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1337.0,+44/-45,0.226,22.61,13.17,14.984,934.1,3.063,0.0,False,GPU +22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1318.0,+43/-36,0.183,23.73,15.62,16.035,2686.11,0.47,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1315.0,+61/-54,0.254,23.93,13.32,15.709,10.21,0.138,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1314.0,+45/-43,0.184,23.95,16.27,14.908,2389.22,2.158,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1294.0,+51/-54,0.175,25.17,14.33,16.495,934.1,0.169,0.0,False,GPU +26,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1280.0,+45/-38,0.126,25.99,16.88,16.759,2686.11,0.054,0.0,False,CPU +27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1277.0,+67/-64,0.238,26.17,9.08,16.109,49.21,43.824,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1266.0,+61/-60,0.169,26.82,15.31,17.277,618.9,4.766,0.0,False,CPU +29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1252.0,+44/-46,0.128,27.62,18.04,17.998,6.86,0.057,0.0,False,CPU +30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1252.0,+46/-45,0.125,27.62,15.26,17.837,14.78,0.346,0.0,False,GPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1243.0,+40/-37,0.087,28.19,19.4,17.108,10.47,1.707,0.0,False,GPU +32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1232.0,+48/-52,0.106,28.81,19.48,17.195,1.77,0.117,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1228.0,+47/-37,0.081,29.06,20.0,17.147,2389.22,0.152,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1214.0,+52/-52,0.1,29.84,19.19,18.724,189.76,0.743,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1195.0,+53/-59,0.074,30.91,19.12,19.123,618.9,0.298,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1188.0,+64/-57,0.09,31.32,18.57,19.635,323.74,0.743,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1188.0,+48/-38,0.065,31.32,26.28,18.263,1.79,0.12,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1172.0,+54/-54,0.073,32.24,19.76,19.957,189.76,0.079,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1150.0,+60/-56,0.062,33.44,18.14,20.504,323.74,0.076,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1085.0,+45/-56,0.015,36.8,28.14,21.856,6.83,0.147,0.0,False,CPU +41,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1041.0,+86/-88,0.043,38.9,28.09,29.56,252.58,0.528,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1038.0,+63/-67,0.017,39.03,26.67,24.207,2.91,0.368,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1019.0,+67/-93,0.02,39.87,26.93,27.374,139.84,3.615,0.0,False,CPU +44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1013.0,+86/-103,0.079,40.12,25.02,28.483,1.11,0.189,0.0,False,GPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1010.0,+87/-92,0.031,40.23,29.46,30.319,252.58,0.089,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1005.0,+73/-85,0.032,40.44,33.02,27.121,3.61,0.939,0.0,False,GPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+58/-63,0.01,40.67,34.46,26.637,0.38,0.037,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,947.0,+68/-72,0.008,42.72,37.34,28.882,0.25,0.041,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,927.0,+94/-111,0.019,43.43,32.94,33.16,1.29,0.131,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,900.0,+84/-117,0.004,44.32,32.53,33.585,139.84,0.193,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,563.0,+111/-112,0.0,50.66,47.64,49.469,0.18,0.029,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e5d0d2249c56bac1c47aca87a817601f75c263d3 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4412cc4e4acfe3d0b1f43a8c92229e9de6f20809b050633dee55777f88f1411e +size 2041622 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/n_datasets_10 b/data/imputation_no/splits_all/tasks_classification/datasets_medium/n_datasets_10 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..0f594a75b803242960cb57b7b785cb7e85e60d62 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a8347b99901bafd5d829f273bb4bf3c1c34b0e0eef183a023e5463dc2f256a9 +size 420152 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7a388d6ab1d2a7b031b6722990f9d15319650978 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d29fc7770ed6ace6d699448c49540fff92665fe6862340dbb2c44074380f024 +size 1007878 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..159822f78ae5431bc0c125d63164f795524581da --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9706fe88435fe26d86b2b18584303159ab48ddcb5c3669d8a432028c24f3f583 +size 118664 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..dc0f99d32ca3dc3bcceb4ad363d94ff8522bb412 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv @@ -0,0 +1,54 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1929.0,+143/-60,0.907,4.76,2.46,2.178,275.97,2.818,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1873.0,+172/-66,0.859,5.97,2.64,2.411,314.17,2.249,0.0,False,CPU +2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1707.0,+129/-66,0.618,11.18,6.2,5.406,697.8,0.199,0.0,False,CPU +3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1705.0,+141/-72,0.623,11.24,5.16,5.651,697.8,0.023,0.0,False,CPU +4,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1676.0,+124/-64,0.609,12.42,5.87,5.593,2.44,0.027,0.0,False,CPU +5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1673.0,+138/-96,0.542,12.54,7.25,7.874,207.65,0.398,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1669.0,+144/-74,0.529,12.7,8.34,8.132,93.21,0.758,0.0,False,CPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1651.0,+120/-91,0.517,13.46,8.65,7.253,1624.54,1.763,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1646.0,+218/-129,0.54,13.7,3.82,7.545,783.29,13.79,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1626.0,+148/-132,0.53,14.59,7.55,7.556,2338.19,14.248,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1604.0,+180/-118,0.462,15.6,6.08,8.633,1398.9,0.645,0.0,False,GPU +11,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1599.0,+111/-77,0.429,15.82,11.69,8.689,207.65,0.056,0.0,False,CPU +12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1594.0,+205/-121,0.468,16.03,6.06,8.372,783.29,1.924,0.0,False,GPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1567.0,+129/-90,0.338,17.34,9.49,8.727,1624.54,0.083,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1554.0,+167/-78,0.346,17.97,11.64,9.811,93.21,0.087,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1554.0,+190/-148,0.412,17.98,6.83,9.345,1398.9,0.062,0.0,False,GPU +16,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1541.0,+154/-119,0.414,18.63,8.37,9.121,2.3,0.82,0.0,False,GPU +17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1518.0,+166/-148,0.413,19.76,8.93,9.743,7.51,3.146,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1508.0,+114/-82,0.263,20.28,12.89,10.245,2338.19,0.54,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1486.0,+156/-141,0.326,21.43,8.09,9.963,4805.66,461.797,0.0,False,GPU +20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1476.0,+89/-75,0.234,21.91,15.1,10.081,1169.3,1.442,0.0,False,CPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1466.0,+146/-106,0.216,22.47,15.25,12.538,893.93,0.205,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1438.0,+156/-129,0.255,23.92,13.66,10.925,0.47,0.052,0.0,False,CPU +23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1422.0,+131/-134,0.177,24.79,16.12,12.437,709.85,2.929,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1398.0,+160/-107,0.139,26.12,19.84,13.588,893.93,0.016,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1388.0,+130/-132,0.146,26.66,20.31,14.508,709.85,0.19,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1385.0,+149/-155,0.217,26.8,15.57,12.673,5.67,0.081,0.0,False,GPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1359.0,+84/-82,0.069,28.18,24.73,13.533,6.98,0.233,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1357.0,+96/-140,0.11,28.29,21.75,14.453,389.62,2.114,0.0,False,CPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1350.0,+138/-132,0.122,28.69,21.21,15.914,84.36,0.919,0.0,False,GPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1345.0,+81/-84,0.073,28.91,23.58,12.262,1169.3,0.053,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1342.0,+125/-102,0.084,29.11,25.2,15.622,2.09,0.019,0.0,False,CPU +32,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1329.0,+117/-121,0.078,29.75,19.44,13.696,4805.66,43.211,0.0,False,GPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1327.0,+118/-103,0.062,29.86,22.08,13.691,52.45,0.219,0.0,False,CPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1294.0,+94/-111,0.062,31.52,28.6,13.707,0.28,0.033,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1289.0,+127/-116,0.063,31.78,23.78,14.607,52.45,0.021,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1281.0,+79/-74,0.007,32.17,29.44,14.625,10.5,0.606,0.0,False,GPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1262.0,+71/-88,0.025,33.1,30.65,13.747,38.42,0.32,0.0,False,CPU +38,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1249.0,+123/-152,0.032,33.75,21.52,17.564,46.9,45.623,0.0,False,GPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1218.0,+79/-144,0.032,35.18,30.19,17.632,389.62,0.112,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1203.0,+77/-86,0.011,35.87,33.94,15.534,38.42,0.028,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1179.0,+62/-68,0.0,36.96,35.47,16.733,1.87,0.046,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1145.0,+110/-215,0.038,38.39,33.26,21.37,1.61,0.112,0.0,False,CPU +43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1103.0,+142/-227,0.008,40.08,36.17,26.799,43.46,0.087,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1070.0,+93/-195,0.01,41.31,32.77,24.389,331.68,44.947,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1051.0,+151/-241,0.005,41.96,38.8,28.022,43.46,0.021,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+102/-183,0.0,43.65,42.56,22.382,0.12,0.013,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,927.0,+140/-232,0.0,45.73,44.31,29.594,0.2,0.023,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,916.0,+205/-468,0.022,46.02,40.6,31.186,2.8,0.224,0.0,False,GPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,884.0,+91/-212,0.0,46.79,46.36,26.748,0.09,0.019,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,861.0,+107/-173,0.0,47.32,46.59,32.966,0.2,0.033,0.0,False,GPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,816.0,+64/-150,0.0,48.23,47.9,32.095,331.68,1.626,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,451.0,+130/-393,0.0,52.34,52.31,49.299,0.12,0.101,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..f0383e4fc917d94175f889dbb3c639c90230e776 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7844015ae3a527022ddb90b51b6317a42c30db6b9aae2d71a986c77a0d83615e +size 2104005 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/n_datasets_28 b/data/imputation_no/splits_all/tasks_classification/datasets_small/n_datasets_28 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5b57854b9c0419d9b4769f60abf1543b55975dfe --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5928d64f3b05875ae9a17f637423f6592cc4090a1b0830a5a42cbfa32fecf351 +size 420646 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..cb8aabbcf7c5c2de2ff9d34fb6d3a77e043410eb --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ca44dd09f35857d8e852eccd21840b24db35c83c0f75ed5bed58c6704670317 +size 989010 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..eeb589455f7b803ab12b337b164916bd189aecee --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64d1d15e597f7ec965fcad544391dfad317dff5ff0db2937b3a5fd914c429b1b +size 124086 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_classification/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..8daf7f37270cdc0ed63bb697ac3aa4295f93ce43 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/website_leaderboard.csv @@ -0,0 +1,53 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1621.0,+89/-68,0.765,7.91,3.4,7.988,610.76,8.082,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1605.0,+95/-83,0.761,8.43,2.98,5.358,2289.05,8.114,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1588.0,+82/-68,0.736,9.04,3.55,6.481,2289.05,1.254,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1587.0,+87/-84,0.769,9.05,3.97,6.793,6.07,0.762,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1470.0,+71/-56,0.499,13.99,8.09,11.927,3422.82,18.936,0.0,False,GPU +5,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1448.0,+70/-57,0.44,15.11,8.48,13.246,3170.35,1.694,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1438.0,+81/-56,0.408,15.62,8.07,12.901,3143.8,1.884,0.0,False,CPU +7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1416.0,+66/-62,0.457,16.78,5.78,11.489,5271.34,231.986,0.0,False,GPU +8,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1399.0,+74/-53,0.434,17.68,8.76,15.067,330.95,2.359,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1388.0,+62/-63,0.345,18.27,10.0,14.216,3170.35,0.241,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1386.0,+46/-33,0.31,18.39,12.83,14.934,735.1,1.901,0.0,False,CPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1383.0,+68/-54,0.32,18.57,9.32,13.652,3422.82,1.534,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1380.0,+70/-62,0.384,18.72,6.02,12.497,5271.34,33.986,0.0,False,GPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1360.0,+48/-29,0.249,19.89,14.24,15.316,735.1,0.311,0.0,False,CPU +14,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1345.0,+47/-39,0.246,20.75,14.59,15.222,1714.62,0.768,0.0,False,CPU +15,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1340.0,+55/-50,0.227,21.06,9.12,14.753,6047.72,0.525,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1334.0,+42/-38,0.228,21.39,15.26,15.413,1714.62,0.098,0.0,False,CPU +17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1322.0,+58/-58,0.242,22.07,12.42,15.908,1053.84,3.063,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1314.0,+49/-44,0.21,22.55,15.66,16.175,797.9,2.228,0.0,False,CPU +19,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1310.0,+53/-40,0.216,22.77,15.24,16.305,7.19,0.118,0.0,False,CPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1308.0,+81/-81,0.273,22.93,10.09,15.656,6047.72,8.744,0.0,False,GPU +21,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1304.0,+66/-53,0.265,23.13,12.73,16.81,10.78,0.17,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1298.0,+46/-42,0.173,23.49,16.12,16.399,797.9,0.353,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1293.0,+77/-78,0.306,23.79,7.55,15.602,51.44,43.709,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1283.0,+48/-40,0.166,24.41,15.85,17.3,3351.28,0.544,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1275.0,+57/-55,0.183,24.89,13.03,17.218,1053.84,0.169,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1275.0,+57/-53,0.16,24.91,16.94,16.647,2977.49,2.415,0.0,False,CPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+47/-42,0.121,26.17,16.12,17.908,3351.28,0.064,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1251.0,+61/-61,0.166,26.3,13.09,18.997,15.54,0.346,0.0,False,GPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1248.0,+71/-62,0.189,26.54,13.9,18.299,1423.89,9.208,0.0,False,CPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1234.0,+57/-48,0.142,27.33,16.49,18.862,8.03,0.059,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1216.0,+46/-41,0.092,28.43,18.13,18.399,14.89,4.433,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1207.0,+79/-69,0.126,28.96,17.02,20.517,399.55,1.466,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1200.0,+46/-51,0.084,29.36,19.11,18.904,2977.49,0.156,0.0,False,CPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1196.0,+61/-50,0.088,29.63,16.97,19.665,1423.89,0.749,0.0,False,CPU +35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1176.0,+43/-48,0.044,30.77,24.27,19.449,2.79,0.238,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1168.0,+80/-76,0.094,31.24,17.28,21.549,399.55,0.174,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1162.0,+48/-50,0.063,31.56,25.83,19.906,3.15,0.141,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1153.0,+71/-87,0.099,32.1,17.65,21.772,571.65,1.39,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1114.0,+72/-74,0.062,34.3,16.83,22.622,571.65,0.129,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1062.0,+62/-63,0.021,37.04,26.37,23.697,11.29,0.204,0.0,False,CPU +41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1044.0,+97/-100,0.107,37.89,21.55,26.892,2.59,0.533,0.0,False,GPU +42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1027.0,+75/-96,0.055,38.74,26.12,30.557,483.02,0.874,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1026.0,+72/-102,0.035,38.78,31.12,25.68,4.95,1.012,0.0,False,GPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1009.0,+65/-65,0.009,39.55,25.04,25.23,5.04,0.646,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1007.0,+84/-107,0.023,39.66,25.44,28.452,121.17,1.63,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1002.0,+76/-98,0.04,39.88,27.22,31.151,483.02,0.113,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+64/-71,0.014,39.96,32.48,28.169,0.46,0.068,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,960.0,+77/-98,0.011,41.63,35.13,29.657,0.42,0.074,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,926.0,+96/-133,0.026,42.94,30.32,34.444,2.39,0.143,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,917.0,+99/-112,0.005,43.28,29.33,34.126,121.17,0.136,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,582.0,+111/-168,0.0,50.41,46.46,49.533,0.28,0.023,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..55eedf0832ea812f5edc807630d79f8a72ae7a74 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7ceac32f0de3543928c9f5c1555e6a60a16e6b7f59a838abddf6ac201992b40 +size 2071503 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/n_datasets_26 b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/n_datasets_26 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..68a77ff1dd9e37f5f021012f30655dfea8e35705 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b349df431daea8a8bf5293741f622707a71953616c88725dd3816b4c0a32015a +size 471650 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e4b81761c96ed17bdba23ad15ae2d70c90b70f1b --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d4b0c21ddff96de33cf89933287f3d1cc6f7686e2ea2e0b5d529ef5a60a5bfc +size 1073311 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..2908de4b01e9703168804fccf2f4a3a2b6bfb90a --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da2018312c5fd5a48044b93dc3e4613742d285aa3708c8de104bcdbd792e735a +size 120230 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..3e24e20a24c4cda44ab4bc96fbb38217adfa91f4 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1634.0,+83/-79,0.728,9.78,3.8,6.234,2168.17,7.961,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1631.0,+70/-57,0.747,9.87,4.93,7.553,5.99,0.633,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1625.0,+85/-64,0.701,10.13,4.67,9.074,604.44,8.082,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1614.0,+81/-69,0.702,10.59,4.34,7.377,2168.17,1.23,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1544.0,+69/-56,0.576,13.78,5.67,11.21,3445.6,48.236,0.0,False,GPU +5,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1536.0,+80/-59,0.593,14.17,6.16,11.857,146.65,4.101,0.0,False,GPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1518.0,+87/-73,0.58,15.09,5.58,11.42,4.37,0.585,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1486.0,+61/-44,0.447,16.86,9.64,13.101,3169.9,16.793,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1459.0,+71/-61,0.447,18.45,6.6,12.877,8.89,1.743,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1454.0,+71/-46,0.373,18.73,10.01,14.494,2828.45,1.596,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1452.0,+82/-72,0.442,18.89,6.45,13.931,3445.6,0.995,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1444.0,+66/-55,0.356,19.32,10.43,14.093,2898.23,1.884,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1425.0,+77/-70,0.388,20.47,6.97,12.671,5119.36,231.986,0.0,False,GPU +13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1424.0,+60/-53,0.369,20.56,10.58,15.951,330.95,2.023,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1410.0,+84/-72,0.421,21.42,8.85,15.11,4.06,0.436,0.0,False,GPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1400.0,+56/-54,0.297,22.04,11.42,14.802,3169.9,1.446,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1400.0,+82/-72,0.355,22.07,7.06,13.572,5119.36,26.967,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1399.0,+72/-51,0.29,22.12,11.71,15.476,2828.45,0.216,0.0,False,GPU +18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1388.0,+42/-35,0.23,22.83,15.77,16.363,647.56,1.72,0.0,False,CPU +19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1362.0,+37/-30,0.175,24.46,17.6,16.741,647.56,0.284,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1356.0,+50/-36,0.186,24.9,17.76,16.602,1465.86,0.692,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1355.0,+56/-50,0.195,24.97,11.03,15.889,5944.88,0.516,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1342.0,+48/-35,0.173,25.77,18.98,16.823,1465.86,0.091,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1326.0,+64/-56,0.23,26.84,14.3,18.007,10.42,0.155,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1325.0,+46/-41,0.157,26.9,19.33,17.617,766.06,1.917,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1323.0,+95/-83,0.242,27.06,11.78,17.015,5944.88,8.396,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1320.0,+45/-44,0.159,27.23,18.32,17.726,5.72,0.11,0.0,False,CPU +27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1320.0,+87/-95,0.3,27.25,8.62,16.755,50.32,43.824,0.0,False,GPU +28,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1320.0,+50/-53,0.172,27.25,16.63,17.396,934.1,2.767,0.0,False,GPU +29,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1309.0,+39/-36,0.119,27.97,19.24,17.841,766.06,0.278,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1297.0,+46/-41,0.117,28.77,18.8,18.633,2686.11,0.47,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1294.0,+56/-54,0.135,28.96,20.03,17.813,2862.05,2.158,0.0,False,CPU +32,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1283.0,+45/-58,0.14,29.68,15.39,19.49,14.8,0.337,0.0,False,GPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1277.0,+50/-50,0.12,30.07,16.88,18.738,934.1,0.165,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1275.0,+77/-87,0.174,30.22,15.28,19.42,1358.63,8.067,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1269.0,+45/-48,0.083,30.63,18.46,19.24,2686.11,0.056,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1244.0,+50/-47,0.089,32.25,19.07,20.124,7.4,0.057,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1243.0,+44/-39,0.068,32.34,20.31,19.291,13.83,3.953,0.0,False,GPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+64/-78,0.076,33.88,19.08,20.796,1358.63,0.897,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1213.0,+53/-43,0.061,34.25,22.19,20.109,2862.05,0.152,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1199.0,+60/-67,0.076,35.15,24.0,22.25,370.85,1.466,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1184.0,+45/-43,0.019,36.07,30.03,21.0,2.4,0.218,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1160.0,+62/-70,0.046,37.55,25.98,23.266,370.85,0.161,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1153.0,+36/-44,0.027,37.95,32.6,21.599,2.9,0.133,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1133.0,+70/-69,0.043,39.17,25.14,23.712,527.42,1.39,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1093.0,+61/-65,0.011,41.45,24.97,24.551,527.42,0.123,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1081.0,+56/-63,0.014,42.1,30.05,24.892,10.38,0.188,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1071.0,+83/-130,0.088,42.63,24.72,27.865,2.43,0.495,0.0,False,GPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1058.0,+100/-110,0.054,43.29,28.54,30.113,436.81,0.874,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1034.0,+89/-103,0.024,44.5,35.91,27.438,4.4,0.974,0.0,False,GPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1033.0,+101/-105,0.036,44.53,30.27,30.656,436.81,0.106,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1032.0,+72/-81,0.0,44.59,27.79,26.107,4.73,0.623,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1008.0,+77/-109,0.015,45.72,32.67,30.107,113.26,1.55,0.0,False,CPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+61/-83,0.003,46.1,38.93,29.212,0.45,0.066,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,969.0,+75/-112,0.009,47.45,40.18,30.601,0.4,0.07,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,958.0,+91/-127,0.023,47.9,33.34,33.436,2.16,0.139,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,909.0,+87/-143,0.001,49.71,34.2,36.035,113.26,0.12,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,600.0,+121/-261,0.0,56.33,51.43,49.89,0.26,0.023,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a864fb99d67c662c46cd954f1500be6398f0eedb --- /dev/null +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2d108298b8807cf1f16cb60d3e435091a643eb653052340b0d92183ddad0857 +size 2801926 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/n_datasets_8 b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/n_datasets_8 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..53257fc1da22bf448e33e8d3c7d08322225ebb81 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e4a8a2c38b55f2aea496a5da37419f9919fcb2f5b55b7e3e532da6cd2940e43 +size 419969 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..859cc1eca0e4b1c5c31b8861f15cd7d78dbaa6be --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f2888952e914138db7784f0d08eda250ba66bcdb33413a2ef8bb1faeeb72dd6 +size 951519 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e35d5e84adb8894dd9cb6da94f949b5f1265338c --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81f535dc60f72929211eca71035956558c460b88f62b97a1358897c410d74f93 +size 117390 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..87063dd151ad25bc36895faee5218919b6448af2 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv @@ -0,0 +1,53 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1760.0,+178/-83,0.832,5.81,3.5,9.062,1045.67,7.086,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1734.0,+245/-102,0.819,6.48,3.01,6.353,2710.27,1.347,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1725.0,+327/-154,0.812,6.69,2.17,5.551,2710.27,8.099,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1588.0,+259/-161,0.668,11.28,4.18,8.494,10.23,0.844,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1534.0,+219/-124,0.45,13.62,5.1,14.217,6219.24,3.78,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1519.0,+190/-118,0.51,14.32,6.59,12.711,4150.25,19.131,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1496.0,+137/-88,0.441,15.41,8.18,13.842,4150.25,3.423,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1478.0,+125/-87,0.398,16.31,9.73,16.609,5885.87,1.632,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1465.0,+122/-75,0.344,17.0,12.25,16.251,1377.29,5.202,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1458.0,+101/-59,0.335,17.34,13.96,14.67,3104.39,1.095,0.0,False,CPU +10,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1456.0,+100/-70,0.322,17.47,13.14,16.47,1377.29,1.259,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1438.0,+123/-87,0.349,18.46,9.94,17.463,5885.87,0.299,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1435.0,+170/-123,0.397,18.57,7.39,18.358,4635.14,212.285,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1433.0,+102/-67,0.278,18.67,14.85,15.297,3104.39,0.212,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1410.0,+103/-98,0.276,19.91,15.48,16.733,2210.02,0.974,0.0,False,CPU +15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1404.0,+101/-101,0.269,20.24,15.39,16.868,2210.02,3.985,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1403.0,+113/-97,0.239,20.3,13.01,15.873,7127.77,9.214,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1396.0,+76/-45,0.201,20.68,10.91,16.245,7127.77,0.776,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1382.0,+179/-128,0.333,21.41,8.7,20.214,4635.14,34.77,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1376.0,+153/-120,0.284,21.74,14.05,19.283,1980.6,3.605,0.0,False,GPU +20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1373.0,+139/-108,0.293,21.93,15.97,19.839,507.73,4.428,0.0,False,GPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1361.0,+115/-90,0.197,22.61,16.5,18.672,36484.23,0.827,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1339.0,+128/-120,0.238,23.84,15.04,20.568,17.83,0.197,0.0,False,GPU +23,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1327.0,+185/-126,0.245,24.53,12.84,21.441,717.36,1.612,0.0,False,CPU +24,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1326.0,+185/-166,0.293,24.56,10.84,20.662,737.91,1.864,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1322.0,+77/-52,0.144,24.83,21.0,18.814,50.54,0.227,0.0,False,CPU +26,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1315.0,+106/-103,0.146,25.21,18.54,19.435,36484.23,0.127,0.0,False,CPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1314.0,+158/-113,0.136,25.25,18.72,20.348,3901.28,2.028,0.0,False,CPU +28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1282.0,+125/-114,0.145,27.06,21.54,21.368,19.31,0.108,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1278.0,+182/-138,0.203,27.3,11.99,22.94,717.36,0.326,0.0,False,CPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1276.0,+123/-114,0.163,27.43,15.98,21.665,1980.6,0.61,0.0,False,GPU +31,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1273.0,+194/-174,0.234,27.57,11.87,24.011,2644.99,12.294,0.0,False,CPU +32,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1273.0,+198/-187,0.227,27.57,9.69,22.052,737.91,0.316,0.0,False,CPU +33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1265.0,+210/-138,0.221,28.02,8.16,23.278,43.39,42.575,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1247.0,+118/-131,0.098,28.99,20.71,22.692,3901.28,0.223,0.0,False,CPU +35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1246.0,+115/-89,0.063,29.06,25.43,20.308,4.72,0.542,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1232.0,+137/-118,0.133,29.78,23.49,22.446,5.04,0.429,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1221.0,+151/-140,0.122,30.37,16.99,25.129,2644.99,1.148,0.0,False,CPU +38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1202.0,+94/-115,0.032,31.38,25.09,25.131,17.94,0.691,0.0,False,GPU +39,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1159.0,+68/-94,0.0,33.59,30.64,25.754,19.2,7.624,0.0,False,GPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1089.0,+135/-197,0.054,36.92,30.62,29.146,9.42,1.675,0.0,False,GPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1055.0,+75/-117,0.0,38.37,30.41,28.55,13.69,0.314,0.0,False,CPU +42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1006.0,+229/-348,0.031,40.34,22.09,39.232,151.69,2.346,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+118/-185,0.0,40.55,36.98,35.45,0.74,0.137,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,997.0,+100/-244,0.0,40.66,38.52,39.878,892.95,0.818,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,959.0,+92/-261,0.0,42.01,39.7,40.883,892.95,0.146,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,949.0,+131/-194,0.003,42.33,39.94,34.505,8.51,0.734,0.0,False,CPU +47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,906.0,+139/-185,0.0,43.69,40.86,40.594,0.69,0.137,0.0,False,CPU +48,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,844.0,+66/-133,0.0,45.37,45.16,39.593,5.05,1.013,0.0,False,GPU +49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,824.0,+265/-559,0.016,45.88,36.33,49.561,151.69,0.258,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,746.0,+93/-250,0.0,47.53,47.17,47.08,4.53,0.164,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,198.0,+134/-662,0.0,51.77,51.76,70.493,0.52,0.076,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5968b4b4d2d42727dcf627bbeee516b6dbfa91cf --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7044cc937df41448be482c3b01aa846da805e260cb93976dc7be8aa5af306c41 +size 1988199 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/n_datasets_1 b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/n_datasets_1 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..9f7559f2266d3ad347e58953f23210fefc944c4e --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40b62a18ea285b21e014dcd6fe5be66c90cb2cab020bc6ed54c8ed94c3e38986 +size 444887 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..aa4e688feba7e49be28c1e73b45f6fb9b991c7c5 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c5430a3114c31d695eba5195595e770a2c6a7e86dd6069b5f4969effa691e4e +size 977457 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..158486c5d429ddafe6f9209f53ecfb39f85147ac --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33cb433f2ada5e02999e95e87c16fac3485e9c6cf71aac38735029012243231a +size 106892 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..c4f7f7abf85b38dcd22115ae20c737de87eb39da --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv @@ -0,0 +1,54 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,9270.0,+0/-0,1.0,1.11,1.06,0.019,133.98,8.253,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,8803.0,+0/-0,0.934,2.56,2.2,1.7,276.87,4.662,0.0,False,CPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,8700.0,+0/-0,0.927,3.0,2.84,1.878,276.86,3.44,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,8621.0,+0/-0,0.905,3.33,3.0,2.424,133.98,2.816,0.0,False,GPU +4,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,6195.0,+0/-0,0.594,6.78,6.31,9.581,61.95,0.17,0.0,False,CPU +5,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,6182.0,+0/-0,0.595,6.89,6.45,9.545,7606.4,0.826,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,6134.0,+0/-0,0.573,7.33,7.13,10.011,157.72,3.649,0.0,False,CPU +7,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,6038.0,+0/-0,0.578,8.22,7.72,9.898,7606.4,0.048,0.0,False,CPU +8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,5997.0,+0/-0,0.578,8.67,7.91,9.921,61.95,0.016,0.0,False,CPU +9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,5959.0,+0/-0,0.552,9.11,8.66,10.441,157.72,0.374,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,5682.0,+0/-0,0.508,11.89,10.98,11.347,2688.02,0.24,0.0,False,CPU +11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,5671.0,+0/-0,0.518,12.0,11.9,11.168,466.47,1.036,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,5573.0,+0/-0,0.499,13.0,12.51,11.536,2688.02,0.726,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,5519.0,+0/-0,0.501,13.56,13.39,11.505,0.39,0.051,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,5486.0,+0/-0,0.501,13.89,13.77,11.505,466.47,0.051,0.0,False,CPU +15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,5145.0,+0/-0,0.399,17.0,16.0,13.544,1.48,0.53,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,5108.0,+0/-0,0.423,17.33,17.24,13.063,7.54,0.047,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,5082.0,+0/-0,0.407,17.56,17.52,13.396,4423.24,29.2,0.0,False,GPU +18,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,5032.0,+0/-0,0.389,18.0,17.84,13.739,17.7,0.086,0.0,False,CPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,4812.0,+0/-0,0.354,19.67,19.62,14.412,4423.24,2.237,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,4739.0,+0/-0,0.348,20.11,20.02,14.523,2.82,1.073,0.0,False,GPU +21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,3202.0,+0/-0,0.25,22.11,22.11,16.353,20.98,0.329,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,2532.0,+0/-0,0.132,24.22,24.17,18.457,4638.91,343.215,0.0,False,GPU +23,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,2475.0,+0/-0,0.115,24.56,24.47,18.732,2389.31,0.267,0.0,False,CPU +24,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,2475.0,+0/-0,0.113,24.56,24.51,18.777,20.98,0.027,0.0,False,CPU +25,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,2456.0,+0/-0,0.109,24.67,24.61,18.841,2389.31,0.023,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1738.0,+0/-0,0.0,27.44,27.43,20.674,14.87,1.855,0.0,False,GPU +27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1554.0,+0/-0,0.0,28.33,28.3,20.901,1993.14,0.452,0.0,False,GPU +28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1404.0,+0/-0,0.0,29.22,29.16,21.65,1993.14,0.059,0.0,False,GPU +29,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1218.0,+0/-0,0.0,30.44,30.4,22.097,4638.91,43.781,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1218.0,+0/-0,0.0,30.44,30.35,22.475,40.54,38.741,0.0,False,GPU +31,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+0/-0,0.0,31.89,31.77,23.341,0.18,0.008,0.0,False,CPU +32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,821.0,+0/-0,0.0,33.0,32.98,23.568,1294.93,0.417,0.0,False,GPU +33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,665.0,+0/-0,0.0,33.89,33.86,24.01,1294.93,0.06,0.0,False,GPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,599.0,+0/-0,0.0,34.22,34.18,24.919,0.18,0.04,0.0,False,CPU +35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,-215.0,+0/-0,0.0,36.11,36.11,28.274,1863.58,6.302,0.0,False,GPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,-734.0,+0/-0,0.0,37.56,37.54,30.79,1863.58,1.131,0.0,False,GPU +37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-788.0,+0/-0,0.0,37.78,37.75,30.768,3.96,0.025,0.0,False,CPU +38,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,-1010.0,+0/-0,0.0,38.67,38.66,32.321,6.85,0.088,0.0,False,GPU +39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,-1462.0,+0/-0,0.0,39.89,39.89,33.496,66.06,1.792,0.0,False,GPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,-2063.0,+0/-0,0.0,41.0,41.0,37.017,7.55,0.149,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,-2631.0,+0/-0,0.0,42.11,42.11,41.044,569.0,0.17,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,-2955.0,+0/-0,0.0,42.89,42.89,41.644,569.0,1.658,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,-3626.0,+0/-0,0.0,44.11,44.11,50.207,0.07,0.015,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-4076.0,+0/-0,0.0,45.22,45.21,51.289,2.86,0.085,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-4255.0,+0/-0,0.0,45.78,45.77,51.687,30.41,0.089,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-4823.0,+0/-0,0.0,47.44,47.44,54.976,30.41,0.015,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-5018.0,+0/-0,0.0,48.33,48.32,55.365,0.13,0.014,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,-5106.0,+0/-0,0.0,48.78,48.72,55.242,3.09,0.997,0.0,False,GPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,-5308.0,+0/-0,0.0,49.78,49.77,56.49,0.11,0.023,0.0,False,GPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-5500.0,+0/-0,0.0,50.56,50.54,57.332,76.27,3.054,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-6498.0,+0/-0,0.0,52.0,52.0,70.796,76.27,0.37,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-7418.0,+0/-0,0.0,53.0,53.0,82.671,0.14,0.136,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..52cec7ac1f22d5e4c918348b45e02c96f2af6d47 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61f905981f0117f3c6a4910114b1377423f4c83aa5cb4aad2b0a61fda9c5f907 +size 1124587 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/n_datasets_7 b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/n_datasets_7 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..800360a1fd3e8a00e53bb9e9145138cda8c844e2 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a30f09a5b5502336a6c235b7334d2ef5b990b30845b4d0c605b4514c4af75d32 +size 429560 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4d2fa973f758e8ba1b41d8d375dc9e8098666a7d --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8c6ab44c4c00f4c19a55698ecdf7a9ac4ffa5932c181961be54db61142d1c1a +size 964875 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..de97703f69efefa6f00202ed59fd8da2e9479339 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b533194d520b4d9f9b83923ed3eddb5aa381dffa6d3b75afbb8c207364a0c57 +size 116913 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..1c7c333a9032490222f8291a0c6935941fc9cd2c --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv @@ -0,0 +1,54 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1780.0,+190/-78,0.821,6.49,3.79,10.304,1154.67,7.879,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1755.0,+239/-99,0.812,7.18,3.12,7.171,2742.37,1.218,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1736.0,+281/-137,0.793,7.75,2.66,6.592,2742.37,7.945,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1648.0,+320/-180,0.703,10.83,3.96,8.027,11.21,0.845,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1613.0,+210/-135,0.58,12.31,6.11,11.362,4471.59,21.354,0.0,False,GPU +5,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1607.0,+176/-118,0.554,12.54,4.81,10.368,6.22,1.092,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1586.0,+139/-72,0.498,13.48,7.68,12.584,4471.59,3.76,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1547.0,+151/-75,0.456,15.33,9.15,16.178,5929.46,1.73,0.0,False,GPU +8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1538.0,+225/-90,0.37,15.8,7.22,16.193,6548.58,2.898,0.0,False,CPU +9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1500.0,+152/-80,0.399,17.71,9.33,17.039,5929.46,0.337,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1485.0,+193/-157,0.422,18.51,7.3,18.473,4631.38,190.647,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1481.0,+104/-65,0.293,18.73,15.09,15.318,3272.91,1.185,0.0,False,CPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1474.0,+153/-80,0.298,19.1,14.57,17.306,1502.45,6.33,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1469.0,+142/-72,0.274,19.4,15.11,17.496,1502.45,1.367,0.0,False,CPU +14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1454.0,+136/-100,0.328,20.23,15.27,18.046,684.78,4.816,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1450.0,+99/-57,0.224,20.42,16.47,16.053,3272.91,0.21,0.0,False,CPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1447.0,+112/-130,0.308,20.59,13.61,18.168,2097.62,3.142,0.0,False,GPU +17,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1441.0,+204/-163,0.384,20.91,8.16,20.039,4631.38,30.351,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1432.0,+139/-101,0.197,21.42,13.4,16.443,7359.32,9.026,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1429.0,+124/-86,0.232,21.59,16.53,17.647,2551.65,1.021,0.0,False,CPU +20,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1429.0,+84/-52,0.167,21.61,10.61,16.711,7359.32,0.596,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1418.0,+138/-86,0.22,22.21,16.65,17.847,2551.65,4.527,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1411.0,+177/-111,0.278,22.59,14.24,19.041,19.65,0.236,0.0,False,GPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1358.0,+80/-78,0.132,25.65,21.96,20.132,63121.14,0.827,0.0,False,CPU +24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1354.0,+196/-181,0.241,25.83,12.52,22.31,717.53,1.628,0.0,False,CPU +25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1347.0,+181/-110,0.118,26.24,18.68,20.717,4142.25,2.105,0.0,False,CPU +26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1346.0,+232/-170,0.252,26.32,11.2,21.631,2775.63,12.592,0.0,False,CPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1338.0,+83/-58,0.094,26.77,23.03,19.786,64.18,0.255,0.0,False,CPU +28,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1334.0,+100/-110,0.191,26.97,15.29,20.501,2097.62,0.515,0.0,False,GPU +29,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1318.0,+195/-204,0.249,27.92,12.51,22.379,755.05,2.155,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1308.0,+86/-80,0.076,28.47,23.93,20.964,63121.14,0.173,0.0,False,CPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1308.0,+227/-212,0.252,28.5,7.67,23.486,46.25,45.74,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1304.0,+202/-208,0.213,28.68,11.66,23.665,717.53,0.339,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1294.0,+144/-116,0.111,29.24,22.8,22.606,20.91,0.126,0.0,False,CPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1287.0,+213/-154,0.136,29.67,16.06,22.991,2775.63,1.431,0.0,False,CPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1278.0,+174/-162,0.154,30.14,23.14,22.23,5.32,0.431,0.0,False,CPU +36,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1270.0,+136/-128,0.105,30.56,20.74,23.361,4142.25,0.287,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1260.0,+190/-218,0.175,31.12,10.31,23.912,755.05,0.317,0.0,False,CPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1239.0,+52/-47,0.0,32.26,30.21,21.716,5.48,0.599,0.0,False,CPU +39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1224.0,+112/-120,0.035,33.05,26.03,25.903,20.96,0.482,0.0,False,GPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1213.0,+75/-81,0.0,33.64,30.7,24.273,19.63,7.686,0.0,False,GPU +41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1159.0,+107/-157,0.061,36.35,30.53,25.552,10.84,2.189,0.0,False,GPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1087.0,+76/-138,0.0,39.56,30.71,28.351,14.75,0.337,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1079.0,+217/-372,0.046,39.91,20.86,36.721,173.37,1.638,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1050.0,+107/-233,0.0,41.06,38.81,38.3,905.14,0.875,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1016.0,+105/-262,0.0,42.37,39.96,38.97,905.14,0.175,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+128/-228,0.0,42.93,39.25,37.253,0.85,0.148,0.0,False,CPU +47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,997.0,+163/-196,0.0,43.04,40.46,32.224,8.97,0.743,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,943.0,+161/-236,0.0,44.77,41.62,39.279,0.78,0.147,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,904.0,+45/-136,0.0,45.89,45.69,37.24,5.83,1.167,0.0,False,GPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,897.0,+264/-569,0.03,46.07,35.58,46.585,173.37,0.147,0.0,False,CPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,786.0,+98/-282,0.0,48.56,48.16,45.98,4.72,0.188,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,268.0,+117/-657,0.0,52.74,52.72,68.787,0.57,0.067,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..2156f75a0a4a0cc402f09864aaf97334150b2a95 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b9b0cb60880aa87d9b98a26d5141a2d042c0e1f1238379454e72345cded33ce +size 2155304 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/n_datasets_6 b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/n_datasets_6 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..228d094d15dd97439c7f8da4a003bfbe31c69b60 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b94285b128c76957a3efbb3a35732cd5e8b1346d7772a9f3ca6ceaa566af0eb +size 470825 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5df18ea584e78b74cd3709955baa605938e7004f --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16c46bd27ffddf88aa29a898b88f0b063724f2a5658dc80c4ac391b5bfd2be8f +size 1035459 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..15df7042483862528755d89ac375babb50464fcd --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06a34595139381a3b5d4bf7285bda5f1a8399d132ddf5a6057df4457303cb616 +size 122677 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..fe131ba2fc1d272b12013893e0a5463f60893637 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1876.0,+270/-100,0.87,6.14,2.55,7.591,2710.27,7.889,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1852.0,+240/-92,0.851,6.75,3.29,8.338,2710.27,1.076,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1838.0,+207/-77,0.833,7.12,4.7,12.046,1374.26,12.843,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1799.0,+306/-137,0.807,8.24,3.94,8.785,10.23,0.844,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1788.0,+296/-112,0.791,8.58,4.37,12.979,3585.96,32.815,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1708.0,+168/-44,0.653,11.4,6.17,12.954,4150.25,19.131,0.0,False,GPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1688.0,+166/-61,0.628,12.2,5.26,11.831,5.85,0.91,0.0,False,GPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1680.0,+197/-107,0.622,12.51,7.31,15.607,205.23,4.488,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1633.0,+220/-107,0.562,14.52,6.55,15.646,3585.96,1.167,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1630.0,+135/-43,0.553,14.68,8.59,14.49,4150.25,3.423,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1592.0,+203/-102,0.476,16.45,9.67,13.798,6.47,0.427,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1575.0,+284/-135,0.414,17.28,7.5,18.07,7307.18,4.205,0.0,False,CPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1549.0,+239/-110,0.412,18.59,10.08,18.854,5885.87,1.632,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1513.0,+108/-46,0.303,20.47,17.01,17.651,3104.39,1.095,0.0,False,CPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1497.0,+197/-114,0.346,21.34,10.08,19.852,5885.87,0.299,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1481.0,+240/-202,0.377,22.22,8.34,21.539,4513.63,201.505,0.0,False,GPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1474.0,+122/-64,0.203,22.61,14.46,18.788,7367.31,7.816,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1470.0,+95/-49,0.218,22.86,18.63,18.528,3104.39,0.18,0.0,False,CPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1468.0,+142/-63,0.236,22.95,18.11,20.186,2210.02,1.236,0.0,False,CPU +19,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1460.0,+224/-87,0.219,23.37,17.51,20.164,1377.29,6.652,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1455.0,+147/-63,0.221,23.69,18.61,20.414,2210.02,7.723,0.0,False,CPU +21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1450.0,+196/-70,0.188,23.97,18.81,20.379,1377.29,1.395,0.0,False,CPU +22,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1433.0,+202/-128,0.245,24.92,19.16,21.045,696.38,4.428,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1431.0,+237/-220,0.338,25.04,8.66,23.355,4513.63,26.967,0.0,False,GPU +24,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1424.0,+226/-151,0.294,25.44,10.59,21.754,12.57,2.243,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1420.0,+212/-145,0.267,25.68,15.1,22.067,17.83,0.197,0.0,False,GPU +26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1412.0,+144/-153,0.205,26.15,21.17,21.235,1880.81,2.744,0.0,False,GPU +27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1410.0,+86/-72,0.092,26.26,14.18,19.439,7367.31,0.587,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1381.0,+315/-274,0.276,27.89,11.42,24.53,2644.99,12.716,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1347.0,+200/-164,0.121,29.89,21.48,23.884,4654.35,2.028,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1340.0,+132/-106,0.076,30.27,25.3,23.387,36484.23,0.712,0.0,False,CPU +31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1320.0,+134/-78,0.06,31.45,27.13,22.957,264.83,0.227,0.0,False,CPU +32,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1318.0,+331/-279,0.291,31.55,7.6,27.104,40.51,46.546,0.0,False,GPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1289.0,+146/-150,0.099,33.19,27.21,23.905,1880.81,0.353,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1289.0,+262/-248,0.134,33.2,18.09,26.079,2644.99,1.47,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1282.0,+107/-99,0.027,33.59,27.61,24.339,36484.23,0.127,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1278.0,+176/-198,0.106,33.8,27.13,26.121,717.36,1.838,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1255.0,+115/-83,0.029,35.05,28.39,26.62,17.94,0.418,0.0,False,GPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1247.0,+64/-58,0.0,35.52,33.72,24.705,4.72,0.714,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1239.0,+180/-243,0.119,35.94,22.24,26.195,737.91,2.289,0.0,False,CPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1232.0,+70/-89,0.0,36.29,33.66,26.605,20.73,8.179,0.0,False,GPU +41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1230.0,+181/-140,0.06,36.45,24.6,27.149,4654.35,0.223,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1223.0,+101/-89,0.0,36.81,34.51,26.38,14.75,0.108,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1212.0,+159/-217,0.065,37.39,29.45,27.709,717.36,0.326,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1204.0,+196/-142,0.055,37.78,34.29,25.943,5.04,0.449,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1160.0,+160/-231,0.033,40.02,25.17,27.997,737.91,0.316,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1156.0,+183/-247,0.067,40.22,33.93,29.106,9.81,2.345,0.0,False,GPU +47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1069.0,+131/-286,0.0,44.15,42.1,36.78,892.95,0.895,0.0,False,CPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1066.0,+132/-195,0.0,44.29,33.03,32.333,13.69,0.314,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1031.0,+119/-328,0.0,45.67,43.14,37.562,892.95,0.146,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+170/-313,0.0,46.83,42.97,37.99,0.74,0.137,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,985.0,+174/-259,0.0,47.35,44.55,34.643,8.51,0.828,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,954.0,+212/-444,0.0,48.38,41.73,42.698,151.69,1.252,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,932.0,+195/-360,0.0,49.06,45.92,40.173,0.69,0.137,0.0,False,CPU +54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,851.0,+29/-142,0.0,51.27,51.09,42.312,5.05,1.013,0.0,False,GPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,775.0,+103/-312,0.0,52.93,52.56,43.542,4.53,0.164,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,580.0,+156/-543,0.0,55.58,55.16,54.165,151.69,0.099,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,178.0,+157/-1027,0.0,57.75,57.73,69.817,0.52,0.056,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3d9cb05d365111ebbcf4ca4bf7c1729fd581d7c3 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:338def1117e6a5f95eff7222b2bcbabb135b83c7c94b5554c62379a384092ff9 +size 2656649 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/n_datasets_13 b/data/imputation_no/splits_all/tasks_regression/datasets_all/n_datasets_13 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..43f5626eb1eaaf4e4730c5856f7dd1c781fd80ad --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4195800e25572a8cf53c95aa8c69184693b7a6059dce16df3ad7a7d3f969d5aa +size 426338 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..130ee7b391adc274659c9dfa7c66dfe4ab063b5a --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7be5821fb3d875758fefb2469588323f10272cbfde09df1f9dc09e595f006f05 +size 981158 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ac11590dcbfa97bb300b20442e4010a2f35fec89 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32f6b5437f704b5e45fe03193895778030c08a6a24725b2343dee80043c431fb +size 117379 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_regression/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..de82f05722078727620c0c6195bec24da34e9a98 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/website_leaderboard.csv @@ -0,0 +1,52 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1816.0,+189/-125,0.812,6.26,2.56,1.486,1709.05,8.122,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1783.0,+136/-84,0.791,7.1,2.64,2.397,4786.55,239.537,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1760.0,+117/-75,0.696,7.73,3.92,2.908,649.34,5.869,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1730.0,+107/-72,0.729,8.63,4.27,2.848,4786.55,38.501,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1728.0,+113/-66,0.677,8.69,5.41,3.196,3995.01,10.051,0.0,False,GPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1689.0,+174/-116,0.697,9.94,4.34,2.246,1709.05,0.812,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1675.0,+108/-89,0.567,10.44,6.37,4.64,1866.07,6.07,0.0,False,CPU +7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1632.0,+135/-71,0.601,11.98,4.77,3.791,46.62,39.212,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1606.0,+109/-79,0.517,12.99,7.92,4.177,3995.01,0.844,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1600.0,+132/-117,0.506,13.21,5.41,5.528,3779.52,7.69,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1600.0,+168/-119,0.557,13.21,7.34,3.758,7.04,0.508,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1535.0,+106/-61,0.369,15.79,9.82,6.221,3552.96,0.966,0.0,False,CPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1527.0,+93/-80,0.361,16.1,12.59,6.683,700.15,9.322,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1516.0,+120/-91,0.379,16.58,10.22,5.711,714.5,1.376,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1509.0,+110/-76,0.348,16.87,9.48,6.341,3552.96,0.105,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1496.0,+140/-90,0.349,17.46,12.61,5.114,4158.29,1.41,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1458.0,+87/-74,0.274,19.17,15.01,7.275,700.15,0.968,0.0,False,CPU +17,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1447.0,+191/-193,0.423,19.66,5.5,13.147,5.2,0.755,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1442.0,+56/-45,0.255,19.92,17.26,7.217,834.93,2.614,0.0,False,CPU +19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1426.0,+95/-84,0.262,20.63,16.37,6.387,714.5,0.099,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1420.0,+60/-49,0.221,20.92,17.74,7.365,834.93,0.388,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1419.0,+116/-86,0.265,20.96,12.86,7.527,3779.52,0.396,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1409.0,+128/-104,0.25,21.44,14.82,6.071,4158.29,0.167,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1404.0,+106/-103,0.262,21.68,15.62,7.922,10.89,0.092,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1307.0,+92/-77,0.118,26.06,20.62,9.067,15.5,0.299,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1304.0,+146/-119,0.161,26.23,20.14,7.789,13.32,0.131,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1275.0,+127/-103,0.116,27.53,19.36,9.279,4608.59,1.233,0.0,False,CPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1262.0,+132/-92,0.087,28.11,21.77,8.803,8.9,1.645,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1234.0,+125/-104,0.107,29.43,21.54,11.617,158.22,0.844,0.0,False,CPU +29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1226.0,+40/-39,0.005,29.76,28.4,9.708,2.11,0.275,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1211.0,+115/-100,0.081,30.44,22.91,10.071,4608.59,0.097,0.0,False,CPU +31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1206.0,+111/-97,0.082,30.68,20.81,11.83,158.22,0.151,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1204.0,+84/-82,0.061,30.79,27.39,10.368,2.24,0.242,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1179.0,+133/-168,0.117,31.89,23.15,12.864,2929.85,0.294,0.0,False,CPU +34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1172.0,+114/-119,0.039,32.19,24.29,12.134,2.45,0.743,0.0,False,GPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1170.0,+70/-88,0.035,32.28,29.71,12.388,515.73,0.771,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1133.0,+66/-94,0.025,33.8,31.32,12.876,515.73,0.124,0.0,False,CPU +37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1129.0,+136/-171,0.088,33.95,24.71,13.402,2929.85,0.03,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1084.0,+104/-126,0.014,35.69,31.52,13.769,0.47,0.055,0.0,False,CPU +39,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1060.0,+128/-181,0.049,36.6,31.21,14.338,8.47,0.035,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1057.0,+101/-120,0.02,36.71,34.25,13.746,540.06,2.672,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1043.0,+84/-102,0.0,37.2,35.18,13.502,20.48,0.08,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1000.0,+97/-105,0.01,38.71,37.01,14.233,540.06,0.325,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+54/-79,0.0,38.71,37.38,14.393,0.53,0.062,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,912.0,+150/-180,0.007,41.42,36.02,19.563,92.55,0.897,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,869.0,+106/-170,0.0,42.56,41.65,18.494,2.6,0.392,0.0,False,CPU +46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,806.0,+150/-188,0.0,43.99,39.8,21.987,92.55,0.052,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,672.0,+199/-297,0.0,46.39,39.43,29.17,0.19,0.04,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,506.0,+138/-315,0.0,48.39,48.21,36.388,193.95,0.173,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,459.0,+162/-378,0.0,48.83,48.63,36.561,193.95,0.074,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,248.0,+141/-390,0.0,50.31,50.29,38.996,0.95,0.097,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..fe2eda6839fc2998126bd6c7477fe1bb0a5e916d --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db3b4b0ec1158ab053e54d2eee5caf5681bea698ded84437184856322d3c5dea +size 1808653 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/n_datasets_5 b/data/imputation_no/splits_all/tasks_regression/datasets_medium/n_datasets_5 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e0d4cc2eb3cecb1cfd968589453176bf224e22c5 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d8541e458f7a4a5ba3b69544da2bf1b07b75cc99a9760f65cf588d306d4acc3 +size 429005 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a7f74017c978bd88f65210914d58ed4652b8d43a --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfffe6e1a4b8a2181b87380c90a9b0959aec4c13e1af4d035a3003f1f28273ae +size 984608 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..27849f9a35c081a2db43c506b9144674919ac336 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:103a335ae2472aee78c057a242a72bce572ac210c3c04320b4a599772c41ad85 +size 119720 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..d41a2046e31a460a63a19e7ed98b54b547edec91 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv @@ -0,0 +1,52 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,2069.0,+1045/-120,0.862,4.4,2.67,1.209,735.58,9.557,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1971.0,+783/-126,0.802,6.31,4.21,1.696,735.58,0.714,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1922.0,+681/-41,0.713,7.47,5.68,2.304,2526.28,3.717,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1917.0,+1210/-123,0.782,7.6,2.16,1.293,4786.55,361.217,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1909.0,+515/-11,0.661,7.8,4.92,3.331,473.01,8.73,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1900.0,+797/-114,0.688,8.04,5.61,2.7,2028.29,1.675,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1884.0,+532/-21,0.644,8.47,4.35,3.488,430.4,7.174,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1850.0,+1175/-128,0.717,9.42,4.22,1.707,4786.55,41.599,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1802.0,+856/-218,0.564,10.87,7.04,2.636,1.6,0.508,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1753.0,+1086/-143,0.591,12.42,4.95,2.355,45.77,39.212,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1721.0,+794/-134,0.535,13.47,8.53,3.741,2028.29,0.081,0.0,False,GPU +11,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1684.0,+503/-95,0.474,14.71,9.45,4.288,425.61,0.615,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1671.0,+376/-133,0.388,15.18,11.21,5.936,222.05,11.236,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1650.0,+347/-105,0.356,15.93,8.83,6.099,1658.41,0.742,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1623.0,+616/-142,0.443,16.89,13.4,4.108,2526.28,0.166,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1619.0,+361/-178,0.35,17.04,7.26,6.197,1658.41,0.059,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1618.0,+331/-121,0.326,17.09,13.63,6.24,222.05,0.968,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1602.0,+522/-214,0.341,17.64,14.24,5.117,3440.87,0.619,0.0,False,GPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1555.0,+327/-104,0.296,19.4,17.23,5.243,425.61,0.04,0.0,False,GPU +19,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1545.0,+667/-210,0.305,19.76,5.04,4.564,2.06,0.755,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1533.0,+425/-345,0.326,20.22,14.49,6.882,5.49,0.037,0.0,False,CPU +21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1526.0,+421/-182,0.24,20.44,18.38,5.96,3440.87,0.052,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1509.0,+268/-136,0.217,21.09,19.43,6.83,282.13,1.74,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1506.0,+647/-156,0.285,21.18,18.43,5.272,6.88,0.094,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1497.0,+267/-142,0.197,21.51,19.3,6.953,282.13,0.388,0.0,False,CPU +25,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1360.0,+317/-308,0.159,26.42,21.95,8.067,0.6,0.08,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1330.0,+330/-211,0.091,27.49,25.06,7.827,5.03,0.025,0.0,False,GPU +27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1305.0,+240/-126,0.011,28.33,27.32,8.79,2.28,0.13,0.0,False,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1301.0,+209/-114,0.061,28.49,26.91,8.242,4348.61,0.528,0.0,False,CPU +29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1292.0,+283/-160,0.013,28.78,27.91,8.309,0.55,0.233,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+204/-297,0.038,31.24,29.53,9.231,4348.61,0.034,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1199.0,+231/-316,0.0,31.89,30.73,9.7,6.75,0.235,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1176.0,+359/-419,0.022,32.62,30.66,10.264,35.67,0.529,0.0,False,CPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1156.0,+87/-156,0.0,33.27,33.0,9.993,79.08,0.546,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1144.0,+260/-394,0.006,33.64,32.41,10.361,35.67,0.05,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1127.0,+112/-248,0.0,34.16,33.79,10.297,79.08,0.057,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1102.0,+387/-661,0.143,34.91,29.95,13.832,2044.56,0.128,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1037.0,+193/-452,0.001,36.8,35.28,13.252,15.08,0.028,0.0,False,CPU +38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1029.0,+364/-709,0.114,37.02,33.51,14.402,2044.56,0.01,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1026.0,+358/-676,0.006,37.11,35.07,12.06,0.17,0.049,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+37/-248,0.0,37.82,37.67,11.446,0.43,0.053,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,976.0,+370/-780,0.102,38.47,35.09,15.083,3.66,0.011,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,848.0,+60/-540,0.0,41.4,41.16,14.323,318.98,0.827,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,794.0,+54/-529,0.0,42.44,42.26,14.781,318.98,0.114,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,789.0,+112/-640,0.0,42.53,42.33,18.087,142.58,1.184,0.0,False,CPU +45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,664.0,+83/-704,0.0,44.53,44.43,20.643,142.58,0.064,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,607.0,+116/-707,0.0,45.27,45.07,18.384,1.36,0.111,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,349.0,+100/-1185,0.0,47.51,47.41,28.1,0.06,0.034,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,81.0,+131/-1638,0.0,48.92,48.92,39.828,44.3,0.048,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-114.0,+182/-2167,0.0,49.77,49.76,40.21,44.3,0.011,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-414.0,+224/-2121,0.0,50.8,50.8,43.378,0.13,0.018,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..70bad7dfdde14268234e295d58f2033b14c688a3 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eefb189956d2e0f6a276df4b6eff034d61a4713f4fb7887a5d2605f89ed0dc8 +size 1489576 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/n_datasets_8 b/data/imputation_no/splits_all/tasks_regression/datasets_small/n_datasets_8 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..89a2e2bbb7a360298dbc27c4a1b1d99830ec6a1d --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebbf464ff08f97e962caf0d71ffb944e18a72cad6a1783ab6b10fda4cf765445 +size 419998 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e4bdf9559430650d131a7730b5ee051e1781d71a --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d57e80d26ed5be20540aa428a4fa13e103c0cdd3aa9c853334639d017c6baa0 +size 1002629 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..de49784416401acb83f3a4fe0d417e427f2c2344 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfa6b312790fa796a5e9dd65aa2fb36144a3b1dc90dbffdf4dd8692f01b484a3 +size 114452 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_regression/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..00c9eb1f3cc4032ce4a5f4acfe498e3e17482e7d --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/website_leaderboard.csv @@ -0,0 +1,52 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1763.0,+145/-63,0.797,6.79,3.07,3.087,4822.0,140.81,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1746.0,+229/-93,0.729,7.27,3.69,2.545,709.58,5.864,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1740.0,+293/-144,0.782,7.42,2.5,1.659,2306.87,7.231,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1716.0,+119/-54,0.737,8.13,4.31,3.561,4822.0,22.324,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1686.0,+133/-56,0.67,9.09,5.29,3.505,4558.61,33.853,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1612.0,+126/-52,0.606,11.7,4.66,4.689,48.13,42.001,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1602.0,+145/-76,0.509,12.08,7.82,5.458,5805.42,4.31,0.0,False,CPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1599.0,+238/-134,0.631,12.2,4.42,2.59,2306.87,0.902,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1587.0,+131/-72,0.506,12.69,7.58,4.449,4558.61,2.234,0.0,False,GPU +9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1539.0,+225/-156,0.553,14.68,7.53,4.46,7.67,0.547,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1516.0,+140/-75,0.377,15.7,10.55,6.298,4430.73,1.103,0.0,False,CPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1494.0,+121/-71,0.345,16.68,13.64,7.15,1203.24,7.106,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1492.0,+143/-72,0.347,16.76,11.72,6.43,4430.73,0.179,0.0,False,CPU +13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1491.0,+180/-93,0.376,16.79,5.25,7.543,8977.36,8.017,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1479.0,+203/-101,0.355,17.34,11.77,5.112,4438.37,1.981,0.0,False,GPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1471.0,+125/-94,0.321,17.74,10.77,6.601,2039.04,2.144,0.0,False,GPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1440.0,+102/-35,0.278,19.19,16.13,7.459,1265.82,3.15,0.0,False,CPU +17,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1431.0,+292/-243,0.497,19.6,5.84,18.512,6.44,0.709,0.0,False,GPU +18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1413.0,+102/-71,0.242,20.47,16.02,7.922,1203.24,0.956,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1412.0,+96/-39,0.237,20.55,16.89,7.622,1265.82,0.476,0.0,False,CPU +20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1394.0,+135/-83,0.241,21.4,15.88,7.101,2039.04,0.109,0.0,False,GPU +21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1381.0,+203/-117,0.257,22.06,13.23,6.14,4438.37,0.231,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1370.0,+154/-99,0.222,22.59,16.41,8.572,12.31,0.204,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1352.0,+157/-92,0.154,23.51,12.54,9.663,8977.36,0.421,0.0,False,GPU +24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1313.0,+187/-132,0.205,25.44,17.93,7.765,22.14,0.179,0.0,False,GPU +25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1307.0,+151/-99,0.142,25.74,18.41,8.242,21.51,8.356,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1284.0,+166/-166,0.15,26.92,16.47,9.927,5150.31,1.592,0.0,False,CPU +27,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1273.0,+116/-149,0.159,27.44,18.16,12.463,664.2,1.363,0.0,False,CPU +28,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1245.0,+114/-152,0.13,28.82,17.01,12.747,664.2,0.274,0.0,False,CPU +29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1239.0,+68/-67,0.013,29.12,22.27,11.438,26.36,0.302,0.0,False,GPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1222.0,+166/-151,0.107,29.94,20.1,10.595,5150.31,0.136,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1220.0,+186/-176,0.101,30.01,20.27,12.259,4333.58,0.422,0.0,False,CPU +32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1213.0,+58/-45,0.0,30.38,28.71,10.582,4.3,0.318,0.0,False,CPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1185.0,+94/-140,0.057,31.67,27.97,13.884,675.17,1.541,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1177.0,+185/-183,0.072,32.03,21.23,12.776,4333.58,0.037,0.0,False,CPU +35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1144.0,+53/-50,0.0,33.52,32.42,11.807,4.06,0.299,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1143.0,+98/-144,0.041,33.57,29.95,14.489,675.17,0.206,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1138.0,+99/-130,0.033,33.78,31.0,13.385,2214.9,7.795,0.0,False,CPU +38,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1120.0,+153/-223,0.056,34.61,22.72,14.224,10.2,2.718,0.0,False,GPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1115.0,+97/-173,0.018,34.79,29.64,14.837,0.5,0.116,0.0,False,CPU +40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1100.0,+152/-182,0.016,35.44,29.19,13.873,16.01,0.041,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1077.0,+95/-132,0.016,36.38,34.34,13.89,2214.9,0.838,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1050.0,+118/-144,0.0,37.45,35.11,13.659,22.39,0.134,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+87/-145,0.0,39.27,37.2,16.235,0.63,0.12,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,956.0,+163/-363,0.011,40.72,32.95,20.486,88.43,0.512,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,952.0,+90/-160,0.0,40.87,39.76,18.564,6.97,0.818,0.0,False,CPU +46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,852.0,+166/-381,0.0,43.65,37.37,22.826,88.43,0.051,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,756.0,+184/-548,0.0,45.69,35.68,29.839,0.42,0.047,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,600.0,+123/-419,0.0,48.06,47.79,34.237,777.57,0.312,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,585.0,+130/-439,0.0,48.25,47.95,34.281,777.57,0.108,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,399.0,+87/-397,0.0,50.0,49.97,36.257,3.91,0.126,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a3b19ef7950240184b2448b8a2492edbf91141b9 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d022054d916b1fb13939d83df8fcae625d5029d5ae4b0b9b579e78c379acd929 +size 1820517 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/n_datasets_7 b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/n_datasets_7 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..06173afb9960a414073c3b1f2e5ced673fee3953 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79ab9a6d01cf85e3dbda94e820c2827b65864c1bc65edf6bf9830fa693a1fa64 +size 453334 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7a267d5629d377cd5d8326072bdc42ffb52e4b0d --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea7b722927aea2823e62720196d93cc9a41a45b51e8ffc7d222acb4cbc6cdfb2 +size 1046899 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7f65ab86bdf3d398b2a9c86027135dfabbd936c7 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fba320f98ae4202658ec44d36911c52cae0e19c174b582a7cc018e5bd4a699c +size 119094 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..90377fef65b435ce248793b94ebd2127cb72e3d0 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,56 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1738.0,+227/-92,0.724,8.07,4.14,2.676,669.92,5.859,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1729.0,+168/-38,0.767,8.36,3.52,3.492,4663.16,132.911,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1723.0,+332/-141,0.764,8.56,2.6,1.797,2166.18,6.341,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1699.0,+128/-34,0.721,9.38,4.73,3.903,4663.16,18.295,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1682.0,+131/-57,0.676,10.01,5.56,3.736,4823.88,33.539,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1626.0,+176/-56,0.627,12.17,4.57,4.903,47.35,39.185,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1607.0,+195/-116,0.601,13.01,5.36,3.411,6577.99,56.795,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1592.0,+287/-164,0.619,13.66,4.39,2.725,2166.18,0.812,0.0,False,GPU +8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1568.0,+116/-54,0.451,14.73,9.53,6.052,6530.9,4.363,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1565.0,+155/-64,0.462,14.89,8.5,4.84,4823.88,1.952,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1557.0,+236/-169,0.583,15.25,7.94,4.606,7.15,0.501,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1540.0,+116/-42,0.392,16.08,10.73,6.578,3558.14,0.966,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1520.0,+129/-52,0.354,17.06,12.43,6.697,3558.14,0.105,0.0,False,CPU +13,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1503.0,+250/-242,0.546,17.88,5.52,19.511,5.94,0.652,0.0,False,GPU +14,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1492.0,+191/-91,0.36,18.42,6.11,6.52,99.14,2.48,0.0,False,GPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1476.0,+131/-107,0.309,19.23,11.16,7.027,2045.39,1.388,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1475.0,+112/-68,0.303,19.3,15.45,7.822,1003.11,4.891,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1459.0,+217/-133,0.328,20.15,13.53,5.499,4158.29,1.946,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1440.0,+231/-181,0.391,21.16,9.88,5.382,6577.99,0.498,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1437.0,+138/-67,0.266,21.28,11.17,8.679,9308.22,7.885,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1428.0,+88/-37,0.229,21.77,18.45,8.1,933.42,2.759,0.0,False,CPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1412.0,+127/-98,0.246,22.61,16.94,7.406,2045.39,0.104,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1401.0,+91/-46,0.192,23.23,19.21,8.265,933.42,0.388,0.0,False,CPU +23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1399.0,+114/-70,0.201,23.35,18.34,8.659,1003.11,0.894,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1385.0,+140/-87,0.219,24.07,16.92,9.129,10.91,0.197,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1362.0,+218/-156,0.219,25.36,14.47,6.622,4158.29,0.208,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1354.0,+232/-206,0.284,25.77,13.92,7.982,9.1,0.85,0.0,False,GPU +27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1299.0,+96/-60,0.073,28.81,20.91,10.902,9308.22,0.416,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1293.0,+194/-163,0.171,29.15,18.85,13.262,623.85,1.144,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1292.0,+215/-168,0.168,29.2,20.9,8.427,15.01,0.154,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1287.0,+170/-106,0.118,29.44,22.47,8.943,23.19,8.523,0.0,False,GPU +31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1272.0,+170/-162,0.135,30.3,17.5,13.412,623.85,0.291,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1250.0,+149/-168,0.091,31.47,21.51,12.707,2929.85,0.41,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1228.0,+90/-90,0.015,32.61,24.17,12.445,27.82,0.299,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+163/-152,0.07,33.04,27.68,11.15,5692.02,1.354,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1204.0,+166/-180,0.074,33.82,22.52,13.286,2929.85,0.037,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1202.0,+47/-44,0.0,33.94,32.18,11.45,4.57,0.275,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1198.0,+126/-178,0.06,34.14,29.95,14.807,633.57,1.512,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1187.0,+188/-175,0.077,34.7,22.5,11.641,5692.02,0.131,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1161.0,+120/-186,0.042,35.98,31.92,15.352,633.57,0.248,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1142.0,+154/-178,0.055,36.87,25.15,14.894,10.27,3.409,0.0,False,GPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1142.0,+46/-60,0.0,36.87,35.63,12.598,3.79,0.311,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1131.0,+130/-155,0.001,37.4,30.44,14.18,11.19,0.038,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1128.0,+134/-196,0.002,37.53,32.07,15.73,0.47,0.117,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1085.0,+62/-94,0.0,39.46,38.31,14.849,2248.07,7.53,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1025.0,+68/-114,0.0,41.94,41.02,15.336,2248.07,0.89,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1013.0,+116/-188,0.0,42.38,40.15,14.924,24.31,0.134,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+109/-207,0.0,42.87,40.52,17.325,0.53,0.118,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,982.0,+155/-419,0.0,43.54,34.61,21.384,84.3,0.438,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,902.0,+53/-155,0.0,46.1,45.46,20.488,7.09,0.843,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,877.0,+150/-428,0.0,46.82,39.94,23.664,84.3,0.05,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,801.0,+167/-577,0.0,48.7,36.63,30.315,0.39,0.044,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,607.0,+111/-430,0.0,52.0,51.7,36.055,779.18,0.247,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,594.0,+126/-470,0.0,52.17,51.85,36.078,779.18,0.089,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,421.0,+89/-422,0.0,53.9,53.88,37.463,3.97,0.122,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..67d13c61ff2a8acc88679d0f15f80d7672a7da83 --- /dev/null +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e507d90a9dcfd362b60a1916d0fa251f185125461583ddda9f451158cc0f42d +size 2122484 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/n_datasets_51 b/data/imputation_no/splits_lite/tasks_all/datasets_all/n_datasets_51 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..b2a2efdaa09236541bda3e813fb2e99cdddc9d0e --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0398d05006a861ff463a29080f79a072c4ce7697620ec5d5944ecbcb16ca9f10 +size 406340 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c430998c2493ec0cabf58fd44d82f1f2ed9459e4 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f9c279491f0b5d7b5792ce60983973b5a1d9971b78419aadef3912fb14e89d3 +size 991847 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..59d3043203ad1a64190709707c0e8600bdbe12e0 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:820a15afae04f81bcbfd042d66b17131ccf26177600152bf545291e099c7cc75 +size 108706 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_all/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..dbf3d054ddaf2bdcf4b6bc7acab261970cdcbb70 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/website_leaderboard.csv @@ -0,0 +1,51 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1652.0,+102/-67,0.686,7.61,2.54,3.671,2059.94,9.785,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1647.0,+88/-77,0.664,7.77,3.65,5.78,556.15,6.31,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1603.0,+92/-71,0.614,9.21,3.81,5.787,2059.94,1.03,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1583.0,+63/-68,0.569,9.92,4.53,7.556,1754.94,1.767,0.0,False,CPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1562.0,+83/-55,0.558,10.74,4.89,6.429,5.71,0.611,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1546.0,+58/-54,0.509,11.35,6.92,8.669,2791.97,13.886,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1462.0,+55/-60,0.416,15.05,7.54,10.194,2791.97,0.373,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1461.0,+74/-64,0.448,15.12,4.62,9.079,6154.73,386.167,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1444.0,+42/-42,0.318,15.93,12.3,11.616,416.56,2.236,0.0,False,CPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1441.0,+75/-47,0.376,16.06,8.26,10.656,3133.91,1.273,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1417.0,+57/-51,0.325,17.27,11.49,11.2,1665.53,0.559,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1416.0,+79/-61,0.4,17.34,5.58,10.724,6154.73,39.452,0.0,False,GPU +12,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1413.0,+85/-64,0.385,17.49,7.38,11.505,4618.5,7.737,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1392.0,+51/-47,0.293,18.55,12.16,11.63,1665.53,0.065,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1390.0,+42/-50,0.265,18.62,12.58,12.245,700.96,1.439,0.0,False,CPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1375.0,+56/-49,0.249,19.43,14.97,12.525,416.56,0.381,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1371.0,+73/-56,0.294,19.64,11.95,11.592,3133.91,0.13,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1370.0,+61/-61,0.296,19.69,10.44,12.118,4618.5,0.47,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1369.0,+42/-45,0.252,19.73,11.47,12.212,6.7,0.088,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1364.0,+46/-44,0.24,19.98,12.06,12.431,700.96,0.213,0.0,False,CPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1363.0,+58/-46,0.284,20.04,11.85,12.395,866.11,2.007,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1311.0,+58/-49,0.207,22.85,12.6,13.767,866.11,0.097,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1305.0,+76/-71,0.283,23.15,8.71,13.42,45.42,39.406,0.0,False,GPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1301.0,+56/-53,0.212,23.39,16.03,14.0,11.56,0.127,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1292.0,+53/-56,0.191,23.88,14.66,14.955,2961.52,0.482,0.0,False,CPU +25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1272.0,+47/-54,0.148,24.92,19.48,13.751,2832.8,1.801,0.0,False,CPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1272.0,+47/-53,0.146,24.92,17.63,14.37,10.44,1.714,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1235.0,+65/-59,0.146,26.94,14.02,15.791,2961.52,0.048,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1226.0,+62/-53,0.125,27.43,15.34,16.621,13.74,0.316,0.0,False,GPU +29,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1210.0,+58/-59,0.113,28.25,20.79,17.039,191.44,0.76,0.0,False,CPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1210.0,+59/-63,0.13,28.27,15.82,16.737,7.66,0.046,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1205.0,+49/-65,0.117,28.49,22.59,15.617,2832.8,0.112,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1195.0,+61/-57,0.108,29.04,17.06,15.665,2.06,0.122,0.0,False,CPU +33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1187.0,+67/-72,0.119,29.46,21.04,17.388,594.95,4.651,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1174.0,+70/-72,0.115,30.11,18.25,18.088,191.44,0.101,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1169.0,+61/-69,0.084,30.35,22.42,17.981,377.08,0.747,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1160.0,+58/-50,0.066,30.85,27.1,16.457,2.2,0.171,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1125.0,+53/-54,0.045,32.55,26.69,18.796,377.08,0.091,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1109.0,+66/-85,0.081,33.35,21.78,19.015,594.95,0.337,0.0,False,CPU +39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1033.0,+72/-74,0.058,36.76,25.12,23.637,3.14,0.741,0.0,False,GPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1027.0,+59/-69,0.026,37.0,33.69,20.862,8.96,0.129,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+69/-66,0.014,38.12,30.41,23.662,0.43,0.053,0.0,False,CPU +42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,972.0,+68/-86,0.023,39.2,35.14,25.503,129.1,1.627,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,970.0,+77/-89,0.014,39.27,35.71,25.257,0.26,0.054,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,969.0,+73/-80,0.029,39.31,36.05,22.958,3.12,0.312,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,908.0,+84/-111,0.022,41.43,22.85,32.143,240.73,0.308,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,871.0,+82/-112,0.017,42.56,30.24,32.729,240.73,0.068,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,808.0,+88/-128,0.01,44.25,41.39,35.299,1.23,0.115,0.0,False,CPU +48,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,804.0,+77/-92,0.01,44.33,42.72,31.641,129.1,0.103,0.0,False,CPU +49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,601.0,+82/-142,0.0,48.0,47.66,44.249,0.19,0.037,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ca6f0c9baec1f4999659ddc588d0ffd6804d8c2c --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a2c4d1562b9065f114d063a87eae87a1d23dbad4b09be6e3a6d0917c374c06e +size 1650238 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/n_datasets_15 b/data/imputation_no/splits_lite/tasks_all/datasets_medium/n_datasets_15 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..490c0681e21a977fbda40d575bd384b180fe3fdf --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9ec4506ec71725269c8706c526e12472ecbf61d0e2cac79eb499db0d58fb99e +size 412803 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..59b58dc955f745acce0bbdf2b919a9d3bd473039 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b02cb89e554cd8ce20c8cb6c832ebb6e9dd239e5967c382c6506278b51a4cb62 +size 958450 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..86d151cd52c823b81ca7f26ecb576f6f97eb550a --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07e8caca5c524cd021065ed087f58e3e6ad12469692e47c3ed6ecd1bec3950f3 +size 110829 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..d388f8e0e7be6be20dc26315fd6b192968365e70 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv @@ -0,0 +1,51 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1903.0,+146/-69,0.802,4.47,2.35,2.092,432.14,4.214,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1861.0,+154/-67,0.768,5.27,2.85,2.561,302.05,2.868,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1746.0,+184/-126,0.659,8.13,2.86,3.897,739.1,11.65,0.0,False,GPU +3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1705.0,+184/-133,0.633,9.4,4.51,4.31,2522.58,6.234,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1694.0,+166/-114,0.606,9.73,4.35,5.33,739.1,1.134,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1643.0,+104/-92,0.521,11.53,7.94,5.912,1714.29,1.743,0.0,False,GPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1600.0,+93/-65,0.422,13.2,9.88,7.706,129.25,2.621,0.0,False,CPU +7,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1596.0,+112/-75,0.446,13.33,8.51,6.757,763.16,0.246,0.0,False,CPU +8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1566.0,+103/-69,0.421,14.57,8.16,6.886,763.16,0.04,0.0,False,CPU +9,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1564.0,+147/-120,0.434,14.67,10.09,7.852,2522.58,0.318,0.0,False,GPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1560.0,+96/-65,0.371,14.8,10.43,7.44,289.35,0.706,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1552.0,+161/-99,0.397,15.13,6.74,7.607,1993.57,0.581,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1543.0,+109/-91,0.352,15.53,11.78,8.332,129.25,0.117,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1532.0,+138/-116,0.396,16.0,9.61,7.939,1714.29,0.079,0.0,False,GPU +14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1529.0,+164/-124,0.4,16.13,8.01,6.506,1.91,0.668,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1524.0,+111/-86,0.393,16.37,7.52,7.158,2.93,0.045,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1502.0,+106/-85,0.332,17.33,12.89,7.674,289.35,0.08,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1475.0,+149/-129,0.341,18.53,5.34,8.076,4532.14,387.927,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1465.0,+149/-93,0.296,19.0,10.82,8.336,1993.57,0.06,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1423.0,+116/-118,0.283,20.93,14.58,9.191,559.64,0.88,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1411.0,+152/-149,0.299,21.5,8.17,10.666,4532.14,39.452,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1382.0,+104/-97,0.179,22.87,17.76,11.539,559.64,0.051,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1378.0,+131/-134,0.235,23.07,8.97,9.53,0.48,0.047,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1375.0,+127/-114,0.191,23.2,19.49,10.925,5.42,0.078,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1346.0,+80/-69,0.125,24.6,21.11,10.049,1803.23,1.199,0.0,False,CPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1331.0,+130/-148,0.21,25.33,17.48,13.131,1535.55,0.182,0.0,False,CPU +26,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1290.0,+130/-142,0.188,27.23,13.1,13.594,41.5,39.095,0.0,False,GPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1288.0,+81/-70,0.055,27.33,25.1,12.893,6.54,0.294,0.0,False,GPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1276.0,+95/-102,0.099,27.87,23.83,11.609,9.07,0.224,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1239.0,+83/-96,0.057,29.6,27.08,12.534,1803.23,0.051,0.0,False,CPU +30,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1236.0,+103/-99,0.048,29.73,24.05,12.859,65.73,0.23,0.0,False,CPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1222.0,+98/-130,0.044,30.33,27.5,12.599,36.73,0.359,0.0,False,CPU +32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1218.0,+80/-93,0.03,30.53,28.9,12.025,0.34,0.07,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1216.0,+135/-138,0.121,30.6,24.81,14.424,1535.55,0.013,0.0,False,CPU +34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1204.0,+113/-137,0.109,31.13,27.61,15.52,2.99,0.016,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1203.0,+93/-86,0.037,31.17,27.59,13.195,65.73,0.025,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1189.0,+115/-139,0.063,31.8,26.56,15.027,354.91,1.636,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1148.0,+95/-121,0.021,33.53,31.16,14.978,36.73,0.038,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1074.0,+77/-88,0.005,36.47,35.3,16.346,2.58,0.039,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1049.0,+90/-96,0.011,37.4,35.95,17.593,354.91,0.094,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1005.0,+164/-230,0.017,38.93,33.65,25.288,2.55,0.157,0.0,False,GPU +41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+73/-113,0.0,39.1,38.22,18.839,0.24,0.022,0.0,False,CPU +42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,977.0,+96/-131,0.011,39.87,37.93,22.57,150.58,4.952,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,933.0,+106/-153,0.015,41.2,39.26,21.097,1.49,0.113,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,914.0,+167/-227,0.008,41.77,38.82,32.011,43.96,0.081,0.0,False,CPU +45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,908.0,+108/-152,0.001,41.93,40.6,22.415,0.13,0.022,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,856.0,+156/-254,0.0,43.3,41.29,32.774,43.96,0.017,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,787.0,+59/-118,0.0,44.87,44.64,29.026,150.58,0.546,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,716.0,+107/-199,0.0,46.2,45.88,34.827,0.2,0.02,0.0,False,CPU +49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,540.0,+101/-189,0.0,48.47,48.38,42.676,0.1,0.068,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..855d27292f2f5ebf35cb689bcb7be40fb2201782 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:518a6a4dbcad5381156ad0db009272679376d99b4922d59efff7a12cad879cfc +size 1388476 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/n_datasets_36 b/data/imputation_no/splits_lite/tasks_all/datasets_small/n_datasets_36 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d4894317f7644fb88b1a41c8d2e2e7cf15be16d1 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80b1418472ffa11a08598dc879a89413c116cd506fd3a29db14b742e851e8e5a +size 407490 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..21f8adb761d8be307c5bf904e5de941cb0a0e8a6 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6831e05eb05b9dd58e2f230ae6526a890a379dc2d2e73b5ab601f77443415492 +size 971065 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3c30eb94e21b49dc2cb39e4602a51c862acb1861 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43fcba54351b0d0b5f70196fa648c5b62fac67d5fe68f268b9d5c21e02f66b4d +size 114869 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_all/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..7f6a634a9368effd7419dcf0b09dd58ccebef717 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/website_leaderboard.csv @@ -0,0 +1,52 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1636.0,+133/-80,0.686,7.58,2.52,3.789,2319.48,8.957,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1601.0,+120/-70,0.613,8.71,4.3,6.609,6.58,0.586,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1586.0,+93/-71,0.607,9.25,4.24,7.334,657.21,7.989,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1583.0,+118/-73,0.605,9.35,3.72,6.191,2319.48,0.942,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1521.0,+89/-61,0.484,11.78,6.89,10.035,3855.22,24.935,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1501.0,+59/-54,0.459,12.64,7.65,10.041,3351.09,1.56,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1468.0,+97/-73,0.488,14.14,4.45,9.692,8059.77,364.288,0.0,False,GPU +7,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1456.0,+93/-77,0.474,14.71,5.45,12.275,5.11,0.624,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1446.0,+76/-75,0.402,15.21,7.8,11.353,3855.22,0.954,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1426.0,+92/-83,0.438,16.17,5.13,10.94,8059.77,39.456,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1410.0,+74/-63,0.35,17.03,9.34,12.139,3629.74,1.48,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1395.0,+49/-43,0.264,17.76,14.34,13.451,833.03,2.182,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1361.0,+67/-55,0.258,19.58,14.1,13.265,2448.79,0.814,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1347.0,+74/-65,0.271,20.36,12.18,13.932,1346.37,2.149,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1343.0,+73/-70,0.276,20.57,12.83,13.161,3629.74,0.189,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1337.0,+60/-54,0.23,20.9,15.67,13.815,2448.79,0.078,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1336.0,+60/-43,0.207,20.96,14.19,14.459,886.51,1.982,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1325.0,+71/-76,0.272,21.61,10.38,14.708,6341.76,8.556,0.0,False,GPU +18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1322.0,+52/-49,0.198,21.78,17.8,14.478,833.03,0.435,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1321.0,+62/-49,0.193,21.83,12.01,14.619,886.51,0.318,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1320.0,+50/-45,0.181,21.85,16.14,14.531,8.39,0.139,0.0,False,CPU +21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1317.0,+87/-85,0.321,22.03,7.85,13.538,51.78,39.927,0.0,False,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1309.0,+68/-67,0.231,22.5,10.84,14.095,6341.76,0.501,0.0,False,GPU +23,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1288.0,+68/-61,0.208,23.68,11.95,14.901,1346.37,0.103,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1282.0,+74/-56,0.175,24.03,14.51,15.925,3249.24,0.548,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1279.0,+71/-62,0.215,24.17,15.22,15.488,13.71,0.149,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1270.0,+62/-68,0.169,24.69,16.32,15.198,16.6,4.299,0.0,False,GPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1250.0,+50/-60,0.153,25.81,19.29,15.496,3693.32,2.14,0.0,False,CPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1244.0,+64/-60,0.15,26.17,12.07,16.566,3249.24,0.059,0.0,False,CPU +29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1213.0,+77/-61,0.127,27.92,14.0,17.458,9.93,0.052,0.0,False,CPU +30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1212.0,+73/-78,0.125,27.97,13.54,18.902,14.9,0.323,0.0,False,GPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1208.0,+82/-76,0.138,28.17,19.51,19.083,497.58,1.147,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1196.0,+64/-71,0.137,28.83,21.74,17.098,3693.32,0.131,0.0,False,CPU +33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1188.0,+83/-87,0.136,29.26,20.25,18.568,1666.78,7.827,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1185.0,+86/-87,0.15,29.46,16.02,19.577,497.58,0.163,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1148.0,+76/-74,0.093,31.44,23.1,20.32,575.62,1.155,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1140.0,+56/-61,0.079,31.85,27.22,18.504,3.63,0.218,0.0,False,CPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1130.0,+56/-64,0.052,32.36,28.08,18.427,3.1,0.207,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1128.0,+71/-93,0.106,32.47,20.92,19.793,1666.78,0.561,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1099.0,+66/-74,0.046,33.99,27.62,21.333,575.62,0.147,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1041.0,+93/-104,0.07,36.75,23.22,23.137,6.09,0.951,0.0,False,GPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1011.0,+60/-89,0.028,38.11,33.87,22.946,12.68,0.155,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+72/-86,0.02,38.57,28.53,25.864,0.51,0.085,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,988.0,+78/-86,0.018,39.08,35.07,26.623,0.43,0.081,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,979.0,+57/-96,0.03,39.44,35.96,23.922,5.83,0.571,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,970.0,+65/-114,0.028,39.81,34.88,26.92,116.3,1.243,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,904.0,+86/-105,0.028,42.24,19.72,32.392,520.62,0.491,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,874.0,+89/-114,0.024,43.19,27.61,32.903,520.62,0.097,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,834.0,+102/-146,0.014,44.39,40.65,35.691,2.59,0.129,0.0,False,CPU +49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,808.0,+77/-124,0.014,45.08,43.04,32.92,116.3,0.075,0.0,False,CPU +50,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,616.0,+96/-191,0.0,48.78,48.35,45.084,0.37,0.035,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..06175863fae6274eb2d7d30dd8ef2c944826cf27 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8080ebf1b937af12ee817a53b9dfc7c55193a86885889400f3d01df47b8d10fe +size 1755538 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/n_datasets_33 b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/n_datasets_33 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..dab9e9a82a807625edc4dfb378b49ad557ca1a9d --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b5b226efa578385cbd7c75214d056d74b2bea80633da9010082c5dadd60e1f0 +size 436370 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..accc773041d8689cc277a0abe27a68cbcb49951c --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ce1d227a5d5353d2bc3e7da959e4ce1358609b45716e941793690feb613372e +size 1028883 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a06945e8dc24dcc41d8e72799dea6e1f110f64f4 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a041aff989a737503ffb2d0b30d8fd2ba6c324e75102f666055c013e4476645e +size 116602 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..97101c0243328cfe99938a4a6ed039932d45f645 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,56 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1662.0,+123/-83,0.681,7.94,2.68,4.083,2200.35,8.314,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1641.0,+86/-70,0.624,8.62,4.92,6.963,6.21,0.561,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1608.0,+112/-89,0.599,9.85,4.13,6.6,2200.35,0.882,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1581.0,+100/-81,0.579,10.88,4.62,7.89,625.63,7.662,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1544.0,+104/-79,0.512,12.47,5.25,9.548,3650.56,52.333,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1542.0,+73/-57,0.484,12.55,7.17,10.583,3565.03,23.942,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1494.0,+64/-52,0.422,14.85,8.48,10.816,3035.75,1.498,0.0,False,CPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1492.0,+85/-71,0.426,14.95,7.23,10.796,137.76,3.046,0.0,False,GPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1478.0,+93/-77,0.462,15.65,5.54,12.913,4.66,0.611,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1469.0,+101/-74,0.458,16.12,4.82,10.352,7821.64,382.96,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1462.0,+66/-66,0.396,16.47,8.59,11.945,3565.03,0.706,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1430.0,+110/-75,0.415,18.24,5.58,11.612,7821.64,39.414,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1419.0,+91/-76,0.355,18.83,10.17,12.117,3650.56,0.543,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1405.0,+86/-73,0.325,19.64,10.49,12.789,3407.57,1.449,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1376.0,+57/-44,0.204,21.32,17.79,14.548,780.12,2.129,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1366.0,+66/-62,0.232,21.92,15.71,14.21,2234.83,0.653,0.0,False,CPU +16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1356.0,+87/-62,0.324,22.52,8.57,13.444,4.97,0.545,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1336.0,+83/-72,0.248,23.68,15.28,13.869,3407.57,0.173,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1334.0,+105/-86,0.327,23.82,7.82,14.157,51.52,40.033,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1333.0,+73/-58,0.222,23.85,16.69,14.964,1216.65,2.007,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1333.0,+59/-51,0.194,23.85,18.67,14.839,2234.83,0.071,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1327.0,+50/-54,0.165,24.2,19.28,15.598,848.99,1.866,0.0,False,CPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1326.0,+95/-77,0.244,24.3,16.0,15.646,6308.39,7.737,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1320.0,+58/-47,0.16,24.62,17.69,15.572,8.13,0.126,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1317.0,+78/-84,0.217,24.82,12.21,14.8,6308.39,0.477,0.0,False,GPU +25,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1313.0,+60/-51,0.151,25.06,13.64,15.742,848.99,0.298,0.0,False,CPU +26,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1304.0,+72/-62,0.151,25.61,20.96,15.595,780.12,0.381,0.0,False,CPU +27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1294.0,+73/-68,0.176,26.21,15.54,16.739,2961.52,0.521,0.0,False,CPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1290.0,+65/-57,0.164,26.45,16.92,15.72,16.29,4.082,0.0,False,GPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1284.0,+80/-82,0.207,26.82,17.19,16.14,13.3,0.146,0.0,False,GPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1274.0,+70/-67,0.157,27.44,19.34,15.938,1216.65,0.097,0.0,False,GPU +31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1250.0,+74/-66,0.142,28.85,12.92,17.471,2961.52,0.054,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1250.0,+68/-64,0.137,28.88,22.23,16.216,3460.09,2.057,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1224.0,+80/-78,0.116,30.42,14.17,18.085,9.77,0.046,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1223.0,+69/-63,0.113,30.52,13.73,19.271,14.5,0.3,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1203.0,+87/-86,0.138,31.68,21.35,19.33,1453.07,6.678,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1196.0,+73/-84,0.103,32.11,24.77,20.448,418.39,0.972,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1183.0,+69/-66,0.1,32.85,26.26,17.821,3460.09,0.121,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1174.0,+74/-87,0.126,33.33,19.28,20.873,418.39,0.147,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1141.0,+80/-84,0.104,35.24,21.86,20.613,1453.07,0.637,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1134.0,+78/-99,0.067,35.61,26.54,21.785,563.93,0.997,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1123.0,+53/-61,0.028,36.21,33.38,19.728,2.86,0.19,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1112.0,+53/-68,0.037,36.83,33.62,19.956,3.28,0.179,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1091.0,+71/-90,0.042,37.89,31.53,22.688,563.93,0.149,0.0,False,CPU +44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1048.0,+93/-107,0.073,40.09,23.76,24.424,5.55,0.867,0.0,False,GPU +45,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1011.0,+78/-85,0.028,41.82,36.56,23.923,9.39,0.15,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+86/-110,0.021,42.29,32.32,26.668,0.48,0.074,0.0,False,CPU +47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,988.0,+92/-122,0.02,42.79,38.51,27.3,0.38,0.075,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,984.0,+82/-97,0.027,42.97,39.0,24.547,5.2,0.522,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,958.0,+74/-86,0.007,44.06,41.64,28.135,101.2,1.135,0.0,False,CPU +50,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,919.0,+86/-112,0.03,45.55,19.46,32.177,497.63,0.481,0.0,False,CPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,891.0,+85/-107,0.025,46.5,27.98,32.668,497.63,0.084,0.0,False,CPU +52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,855.0,+97/-122,0.012,47.67,43.57,34.952,2.47,0.121,0.0,False,CPU +53,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,779.0,+85/-116,0.0,49.73,48.99,34.443,101.2,0.065,0.0,False,CPU +54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,631.0,+109/-168,0.0,52.58,52.11,45.322,0.28,0.026,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..faa2a793636c251865ab2b008fe72ec139bb89c3 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c1f706601fc4ac8b5895763a6a7cdf6331b39cd18e4c7f46613b74cd8ff0cb9 +size 1975031 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/n_datasets_30 b/data/imputation_no/splits_lite/tasks_binary/datasets_all/n_datasets_30 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4f922b4bf5ef66ada3ef0b45ac2526b0bf9a6897 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0893b8d9b7118066ddcbd449d173c9aeefa2bf4a8cd56f01b671977fdd7a109c +size 440584 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..2e1db656a942071988a75d9a277edc1c9983d601 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4b60d55c562818660f54351637dee3af17f83339b7e50d0e12c35b3134c6bfa +size 980762 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..bad542e5e4ce7dd94cfaec433c8c2b2386b9c67f --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfdf93aee84a27e96cbfc235152c7fc8e33c2229798e812cb802eac57f082785 +size 122415 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..8d49af7208550bec5bee20cd0ca3cfd35f80781e --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv @@ -0,0 +1,53 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1593.0,+118/-94,0.652,9.15,3.43,5.769,442.6,6.737,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1580.0,+109/-77,0.61,9.6,3.24,5.263,1877.76,10.745,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1580.0,+106/-77,0.598,9.63,4.44,7.063,1231.21,1.36,0.0,False,CPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1564.0,+103/-72,0.58,10.23,4.57,6.622,4.79,0.708,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1550.0,+111/-66,0.574,10.77,4.42,6.293,1877.76,1.035,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1528.0,+75/-64,0.507,11.7,7.56,8.807,2721.85,9.311,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1446.0,+86/-72,0.42,15.52,6.33,10.77,2721.85,0.265,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1437.0,+90/-66,0.415,16.0,6.81,10.645,2180.12,1.135,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1414.0,+62/-51,0.31,17.22,12.93,12.259,328.64,0.774,0.0,False,CPU +9,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1390.0,+80/-71,0.311,18.52,10.69,12.822,462.92,0.613,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1389.0,+65/-54,0.315,18.53,9.21,11.84,3.84,0.068,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1388.0,+81/-62,0.349,18.6,6.75,9.185,5890.6,409.986,0.0,False,GPU +12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1387.0,+104/-82,0.342,18.65,8.68,11.692,3436.74,0.405,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1375.0,+74/-75,0.324,19.3,11.38,11.971,1043.89,0.478,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1373.0,+104/-101,0.372,19.42,7.62,11.837,3436.74,8.478,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1363.0,+94/-75,0.328,19.98,10.69,11.784,2180.12,0.117,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1351.0,+62/-60,0.287,20.67,13.8,12.509,1043.89,0.041,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1350.0,+77/-81,0.265,20.7,9.89,13.208,462.92,0.108,0.0,False,CPU +18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1350.0,+102/-71,0.278,20.73,11.47,14.058,248.47,1.7,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1343.0,+84/-84,0.305,21.12,7.47,11.401,5890.6,39.565,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1338.0,+62/-58,0.238,21.4,17.03,13.439,328.64,0.089,0.0,False,CPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1332.0,+76/-56,0.23,21.7,13.18,14.224,1579.5,0.428,0.0,False,CPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1316.0,+78/-87,0.245,22.63,13.61,13.107,559.77,1.513,0.0,False,GPU +23,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1315.0,+56/-53,0.189,22.7,15.2,13.331,9.04,1.162,0.0,False,GPU +24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1301.0,+94/-79,0.245,23.5,15.1,14.33,8.0,0.124,0.0,False,GPU +25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1287.0,+72/-59,0.187,24.33,17.4,13.514,2206.58,2.301,0.0,False,CPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1272.0,+67/-73,0.177,25.18,17.14,14.761,559.77,0.086,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1269.0,+68/-58,0.158,25.33,12.79,14.999,1579.5,0.05,0.0,False,CPU +28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1266.0,+73/-64,0.167,25.53,12.12,15.996,4.79,0.045,0.0,False,CPU +29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1231.0,+74/-95,0.189,27.52,10.76,14.605,47.89,39.446,0.0,False,GPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1227.0,+75/-90,0.154,27.75,19.43,16.538,561.29,4.462,0.0,False,CPU +31,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1209.0,+88/-82,0.145,28.77,13.58,16.117,1.42,0.118,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1204.0,+76/-65,0.14,29.1,22.97,15.932,2206.58,0.107,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1195.0,+66/-70,0.12,29.6,12.96,16.944,11.15,0.312,0.0,False,GPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1173.0,+71/-64,0.086,30.82,25.14,17.891,122.9,0.583,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1144.0,+70/-84,0.064,32.37,26.56,19.294,171.61,0.552,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1142.0,+83/-115,0.111,32.5,18.23,18.653,561.29,0.236,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1140.0,+67/-71,0.074,32.62,28.52,17.125,0.93,0.094,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1131.0,+76/-90,0.093,33.07,19.56,19.504,122.9,0.072,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1108.0,+66/-74,0.022,34.28,30.56,19.946,171.61,0.053,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1041.0,+74/-76,0.044,37.6,32.81,21.201,4.97,0.092,0.0,False,CPU +41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1013.0,+111/-133,0.08,38.87,29.06,26.33,0.81,0.115,0.0,False,GPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+77/-100,0.025,39.45,27.55,24.112,0.29,0.031,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1000.0,+76/-108,0.042,39.47,35.71,21.671,2.79,0.259,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,984.0,+68/-95,0.006,40.13,38.17,24.306,140.66,3.209,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,974.0,+97/-117,0.038,40.53,17.35,27.936,155.53,0.277,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,938.0,+95/-130,0.029,41.95,25.15,28.597,155.53,0.05,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,938.0,+92/-121,0.047,41.97,36.6,26.856,2.71,0.392,0.0,False,GPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,932.0,+96/-116,0.017,42.17,37.91,25.941,0.18,0.035,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,902.0,+102/-109,0.017,43.27,39.53,30.347,0.8,0.088,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,816.0,+76/-102,0.0,45.87,45.13,30.586,140.66,0.188,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,600.0,+97/-211,0.0,50.0,49.65,43.693,0.15,0.024,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7af4d1791b31c2be9cc0c9f2e4f9e5c961a3698f --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2594cdd792059af937d31e9363f6a3d305209ab6c6900399450bc2f6d47e5c93 +size 1822134 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/n_datasets_9 b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/n_datasets_9 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4427220a9c1938cbf6be2aa3a4726d63d6f4e4bb --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85f32cacb3f4a35f820052ff92f31175f92f419c549e9baff27fef1392c634a1 +size 431992 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..dd0f5f80be319cd8a628fa131cb4140228eea878 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fffbc9d93552f2469ea3584acb346dcf3407cf00d192a8dd879f095747b5fd25 +size 994534 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..488575eebfdf086994283a700ca4c144e7622914 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56b287b209e5dd2042b0657e5b098681e6015d5396c7f307ffdb041063f07a9f +size 120976 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..f61a3bf6a31bef9e066a755c2a294347f2835805 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv @@ -0,0 +1,54 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,2079.0,+240/-68,0.866,3.33,2.13,1.537,310.55,1.824,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,2016.0,+291/-84,0.837,4.22,2.22,2.096,256.14,2.868,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1765.0,+277/-164,0.621,10.56,3.82,4.453,2149.06,6.373,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1743.0,+238/-173,0.527,11.33,4.11,5.875,841.84,11.882,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1734.0,+140/-56,0.515,11.67,8.77,5.562,1655.56,2.483,0.0,False,GPU +5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1725.0,+191/-108,0.458,12.0,8.28,7.345,114.48,0.404,0.0,False,CPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1705.0,+217/-120,0.517,12.78,7.57,6.556,618.23,0.184,0.0,False,CPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1688.0,+228/-141,0.45,13.44,7.6,7.872,841.84,1.263,0.0,False,GPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1686.0,+225/-118,0.477,13.56,5.11,7.382,807.72,0.575,0.0,False,GPU +9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1678.0,+210/-108,0.431,13.89,9.94,8.481,87.3,0.533,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1676.0,+166/-104,0.487,13.94,8.68,6.655,618.23,0.031,0.0,False,CPU +11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1656.0,+188/-119,0.422,14.78,10.05,9.295,2149.06,0.318,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1653.0,+159/-92,0.458,14.94,5.57,6.571,2.12,0.043,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1631.0,+170/-121,0.399,15.89,10.8,7.667,114.48,0.058,0.0,False,CPU +14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1628.0,+183/-107,0.39,16.0,9.31,8.174,1655.56,0.086,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1596.0,+183/-119,0.328,17.44,13.37,9.425,87.3,0.069,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1594.0,+229/-161,0.393,17.56,8.34,8.03,807.72,0.07,0.0,False,GPU +17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1562.0,+182/-145,0.326,19.0,8.97,7.783,2.32,0.889,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1548.0,+184/-161,0.295,19.67,9.57,9.48,9.96,4.847,0.0,False,GPU +19,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1488.0,+188/-198,0.248,22.56,18.43,10.588,5.77,0.078,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1475.0,+188/-199,0.267,23.22,6.55,10.147,0.48,0.047,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1473.0,+155/-148,0.247,23.33,15.26,9.606,559.89,1.156,0.0,False,GPU +22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1473.0,+120/-74,0.175,23.33,19.14,9.902,1128.19,1.674,0.0,False,CPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1457.0,+181/-128,0.212,24.11,20.3,13.211,262.92,0.205,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1453.0,+174/-154,0.181,24.33,18.53,10.102,4560.06,421.968,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1444.0,+165/-123,0.166,24.78,17.43,12.787,559.89,0.056,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1424.0,+89/-76,0.079,25.78,23.59,11.863,6.55,0.295,0.0,False,GPU +27,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1390.0,+114/-157,0.11,27.44,22.3,12.364,368.88,2.507,0.0,False,CPU +28,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1376.0,+211/-169,0.137,28.17,17.73,13.801,4560.06,39.452,0.0,False,GPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1347.0,+190/-192,0.13,29.56,19.19,13.488,92.9,0.786,0.0,False,GPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1333.0,+181/-120,0.073,30.22,27.34,16.174,0.92,0.018,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1324.0,+203/-161,0.093,30.67,25.33,14.896,262.92,0.015,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1317.0,+122/-138,0.079,31.0,27.5,13.481,1128.19,0.055,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1305.0,+130/-166,0.036,31.56,28.19,13.452,40.45,0.331,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1300.0,+57/-72,0.018,31.78,30.79,14.731,43.93,0.204,0.0,False,CPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1271.0,+112/-155,0.042,33.11,30.95,12.558,0.34,0.052,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1257.0,+53/-66,0.001,33.78,33.41,15.152,43.93,0.018,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1234.0,+108/-127,0.01,34.78,33.46,14.1,9.93,0.224,0.0,False,GPU +38,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1220.0,+141/-229,0.025,35.39,31.53,18.215,41.8,39.131,0.0,False,GPU +39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1198.0,+131/-157,0.022,36.33,33.39,17.05,40.45,0.028,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1185.0,+110/-188,0.018,36.89,34.91,16.514,368.88,0.084,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1144.0,+102/-133,0.01,38.56,37.02,16.247,1.81,0.048,0.0,False,CPU +42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1107.0,+174/-278,0.016,39.94,35.92,25.559,43.96,0.099,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1102.0,+121/-204,0.02,40.11,37.38,20.908,514.21,52.341,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1071.0,+139/-245,0.025,41.22,38.13,19.261,1.61,0.113,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1057.0,+170/-263,0.002,41.72,38.93,26.294,43.96,0.024,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+126/-288,0.0,43.56,42.51,22.447,0.11,0.012,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,898.0,+241/-628,0.027,46.33,38.97,30.801,2.77,0.157,0.0,False,GPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,884.0,+102/-218,0.0,46.67,46.31,28.037,0.22,0.022,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,860.0,+106/-244,0.0,47.22,46.92,25.027,0.1,0.02,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,850.0,+64/-159,0.0,47.44,47.16,28.826,514.21,2.49,0.0,False,CPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,823.0,+118/-263,0.0,48.0,47.49,33.173,0.2,0.035,0.0,False,GPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,491.0,+97/-381,0.0,52.11,52.08,46.122,0.11,0.089,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7a1481066ba52bd95038f1dd68e123c5ca6dbf8b --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4f3a4b13730e439be35e150444b8ab8964d84f0eda7fd9755d501314790d895 +size 1490980 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/n_datasets_21 b/data/imputation_no/splits_lite/tasks_binary/datasets_small/n_datasets_21 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d6bfcdd11a6466ae46505098275983f7ccdce699 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e5ff714838dadd692110df3542d134826a4b5c37a191fd57bb6ec3b6f5a595 +size 466124 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d387fe6da682dbf737dcf59b92cb47afdcd8e4de --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:214e39ac534cf6c8c4e7f3893a52eb70ecde041dcdeddfc718a01bbab271ff32 +size 962589 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..6f347e416c4b048bd9aee68239b585fac23f546a --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb1b2bf76a31a37f8a62248df6dfa0f3c5e2dcb13ed3ab8f7e253071b91551a5 +size 115824 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..d22508901ed91885d279572be2130babbc5e0544 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv @@ -0,0 +1,54 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1635.0,+100/-58,0.669,6.71,3.81,6.489,5.53,0.55,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1560.0,+146/-88,0.628,9.14,3.21,5.365,2125.17,9.785,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1535.0,+151/-108,0.607,10.05,3.84,5.981,2125.17,0.882,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1494.0,+147/-96,0.552,11.76,4.59,7.707,613.33,8.472,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1480.0,+91/-88,0.469,12.38,7.57,10.57,2791.97,23.942,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1470.0,+101/-81,0.465,12.81,9.12,9.79,1886.83,1.333,0.0,False,CPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1420.0,+121/-84,0.459,15.26,5.29,10.535,4.14,0.557,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1405.0,+110/-75,0.395,16.02,6.49,12.26,2791.97,0.544,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1386.0,+108/-84,0.413,17.05,5.48,9.125,6616.87,408.881,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1373.0,+114/-86,0.361,17.76,8.13,12.409,2237.77,1.273,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1372.0,+126/-89,0.33,17.81,10.09,14.65,298.33,2.036,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1350.0,+95/-92,0.371,19.0,6.14,10.701,6616.87,39.713,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1341.0,+69/-50,0.247,19.55,15.7,14.232,512.11,1.17,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1316.0,+82/-52,0.233,20.98,14.65,14.463,4.51,0.124,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1314.0,+154/-86,0.295,21.12,8.66,13.061,4618.5,0.426,0.0,False,GPU +15,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1304.0,+95/-85,0.223,21.67,12.32,15.018,1640.58,0.552,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1301.0,+114/-106,0.273,21.83,12.55,13.757,2237.77,0.147,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1296.0,+91/-80,0.229,22.17,12.6,15.531,700.96,0.743,0.0,False,CPU +18,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1291.0,+81/-60,0.211,22.43,13.73,14.324,11.09,2.46,0.0,False,GPU +19,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1283.0,+89/-81,0.218,22.95,15.45,14.659,1103.84,0.559,0.0,False,CPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1277.0,+88/-86,0.227,23.29,14.99,14.953,536.23,1.738,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1271.0,+78/-80,0.197,23.67,9.72,15.938,700.96,0.203,0.0,False,CPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1265.0,+125/-91,0.252,24.02,13.76,15.354,4618.5,9.218,0.0,False,GPU +23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1264.0,+69/-68,0.187,24.05,20.71,15.511,512.11,0.122,0.0,False,CPU +24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1264.0,+82/-66,0.176,24.1,10.77,15.396,1640.58,0.059,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1258.0,+84/-78,0.189,24.45,18.99,15.373,1103.84,0.051,0.0,False,CPU +26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1256.0,+90/-88,0.185,24.57,10.24,16.286,7.16,0.046,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1251.0,+103/-86,0.231,24.86,14.35,16.289,11.3,0.149,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1245.0,+106/-96,0.257,25.19,8.48,13.383,52.04,39.82,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1236.0,+100/-83,0.184,25.76,17.18,15.412,2535.0,2.546,0.0,False,CPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1223.0,+86/-93,0.167,26.55,18.75,15.96,536.23,0.097,0.0,False,GPU +31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1191.0,+106/-102,0.153,28.48,10.4,18.493,13.39,0.329,0.0,False,GPU +32,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1183.0,+100/-107,0.164,28.93,19.1,18.662,650.63,5.616,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1176.0,+98/-91,0.16,29.38,22.22,17.319,2535.0,0.112,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1138.0,+87/-95,0.106,31.6,24.84,20.126,206.04,0.946,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1136.0,+107/-144,0.145,31.71,17.48,19.888,650.63,0.442,0.0,False,CPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1127.0,+84/-79,0.089,32.24,26.22,19.028,1.98,0.122,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1117.0,+106/-113,0.122,32.81,16.91,20.888,206.04,0.098,0.0,False,CPU +38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1101.0,+75/-85,0.081,33.69,28.83,19.426,2.47,0.168,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1098.0,+92/-102,0.079,33.86,26.1,21.6,494.48,0.936,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1064.0,+84/-103,0.029,35.74,30.46,22.348,494.48,0.093,0.0,False,CPU +41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1053.0,+114/-152,0.105,36.29,25.95,23.728,1.29,0.149,0.0,False,GPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1011.0,+84/-133,0.049,38.43,32.24,23.67,7.76,0.147,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+105/-146,0.034,38.98,24.44,25.154,0.39,0.041,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,978.0,+79/-121,0.039,40.0,36.13,23.027,3.48,0.427,0.0,False,CPU +45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,947.0,+116/-143,0.023,41.38,36.36,26.643,0.33,0.044,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,947.0,+96/-134,0.05,41.38,36.81,25.487,2.66,0.702,0.0,False,GPU +47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,946.0,+68/-106,0.0,41.43,39.93,26.094,89.61,1.627,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,931.0,+108/-155,0.048,42.07,14.35,29.287,280.42,0.382,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,903.0,+115/-164,0.024,43.19,38.32,31.672,1.48,0.127,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,899.0,+113/-150,0.041,43.33,22.16,29.916,280.42,0.072,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,800.0,+87/-136,0.0,46.62,45.73,31.665,89.61,0.08,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,615.0,+107/-184,0.0,50.52,50.07,42.96,0.24,0.02,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a4d0d7be8d675781ce84bcde1bfe0aa6da3e1ac4 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a20bcd1caa820cbdbe9b800eeddde82e64653263e2db4cb100aa5b2ebcc2cfce +size 1849420 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/n_datasets_20 b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/n_datasets_20 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a075b356063975e1d2271c145a8bba070b3dbd07 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d84bd738950c43f964a4eb10aa64dd17ba0991b218e52b39501705c9f006d48d +size 495281 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..392052993d053110bedf3786622bfb02bf1f5b51 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b0ac3febadf712c2336a82cef7e39886a421463fc72993924634d88732caf34 +size 1035456 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c4e4d9c62ab0fc72e2cf5ba8cb4dbd46e44b13b5 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b67f94993552d63f41a9aec69e1862fcd32ccbc15bbcbf6aa86a9d1df3582a13 +size 118622 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..c424e67f04943ef0c170f2786ba8b9c0ddad8d85 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1644.0,+93/-60,0.637,7.8,5.36,6.865,5.25,0.517,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1567.0,+153/-98,0.586,10.65,4.0,5.801,2092.56,8.655,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1556.0,+140/-99,0.575,11.12,4.48,6.297,2092.56,0.867,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1493.0,+130/-78,0.506,14.1,5.36,8.203,598.6,8.421,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1492.0,+91/-82,0.441,14.2,8.25,10.934,2773.9,21.391,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1476.0,+124/-90,0.456,15.02,7.12,10.58,3361.32,65.579,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1471.0,+85/-74,0.413,15.3,10.76,10.37,1881.65,1.26,0.0,False,CPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1443.0,+115/-72,0.407,16.82,6.69,10.913,145.75,4.071,0.0,False,GPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1426.0,+123/-84,0.419,17.77,5.81,10.972,4.09,0.544,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1420.0,+120/-99,0.371,18.18,7.33,12.618,2773.9,0.534,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1402.0,+115/-88,0.33,19.25,11.6,14.276,292.15,1.882,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1401.0,+118/-76,0.371,19.27,6.01,10.185,8.05,1.864,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1399.0,+132/-82,0.38,19.4,6.06,9.42,6408.95,397.026,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1389.0,+117/-88,0.351,20.05,8.88,12.618,2217.3,1.258,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1384.0,+124/-88,0.331,20.32,10.16,13.232,3361.32,0.527,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1368.0,+131/-93,0.347,21.3,7.46,10.945,6408.95,39.606,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1334.0,+58/-48,0.186,23.52,20.03,15.024,457.0,1.064,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1333.0,+110/-76,0.293,23.58,9.54,13.112,4442.23,0.405,0.0,False,GPU +18,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1320.0,+84/-79,0.206,24.4,13.69,15.346,1618.79,0.502,0.0,False,CPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1315.0,+112/-89,0.269,24.72,14.55,13.982,2217.3,0.137,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1315.0,+77/-60,0.186,24.72,16.46,15.224,4.49,0.115,0.0,False,CPU +21,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1314.0,+80/-69,0.194,24.8,14.28,14.446,10.77,2.16,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1312.0,+104/-98,0.295,24.95,7.59,15.25,3.89,0.544,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1287.0,+83/-75,0.178,26.58,19.37,16.445,673.88,0.681,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1284.0,+85/-81,0.162,26.78,17.38,15.452,1103.74,0.549,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1281.0,+88/-92,0.186,27.0,18.24,15.662,516.54,1.513,0.0,False,GPU +26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1278.0,+83/-91,0.17,27.2,10.48,16.268,6.77,0.046,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1276.0,+99/-108,0.245,27.28,16.43,15.707,4442.23,8.478,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1275.0,+83/-79,0.154,27.4,11.18,15.789,1618.79,0.055,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1272.0,+120/-112,0.23,27.55,17.0,16.267,10.18,0.149,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1268.0,+136/-120,0.251,27.85,8.65,13.464,51.78,39.866,0.0,False,GPU +31,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1264.0,+74/-64,0.146,28.1,24.41,16.249,457.0,0.111,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1262.0,+76/-68,0.15,28.2,11.24,16.819,673.88,0.177,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1255.0,+98/-87,0.183,28.7,20.47,15.396,2533.55,2.301,0.0,False,CPU +34,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1252.0,+87/-54,0.133,28.9,24.07,16.217,1103.74,0.05,0.0,False,CPU +35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1227.0,+80/-86,0.12,30.58,21.9,16.637,516.54,0.086,0.0,False,GPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1212.0,+84/-118,0.139,31.52,10.49,18.467,11.78,0.312,0.0,False,GPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1204.0,+96/-119,0.154,32.02,20.34,18.721,622.79,5.389,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1196.0,+97/-94,0.153,32.6,24.44,17.128,2533.55,0.11,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1157.0,+98/-138,0.13,35.05,20.0,19.935,622.79,0.398,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1141.0,+94/-87,0.076,36.08,28.32,20.989,204.47,0.853,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1121.0,+117/-112,0.109,37.3,18.33,21.645,204.47,0.091,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1117.0,+81/-93,0.043,37.55,33.65,20.016,1.88,0.121,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1094.0,+98/-125,0.057,38.9,30.42,22.532,448.77,0.841,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1091.0,+80/-100,0.043,39.08,35.18,20.402,2.39,0.159,0.0,False,CPU +45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1078.0,+103/-162,0.089,39.85,28.74,23.197,1.17,0.148,0.0,False,GPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1065.0,+93/-110,0.03,40.55,34.95,23.044,448.77,0.092,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1033.0,+109/-126,0.046,42.3,34.74,23.554,7.36,0.138,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+98/-164,0.035,44.0,29.07,26.004,0.37,0.04,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,997.0,+107/-123,0.031,44.15,39.68,22.94,3.3,0.395,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,963.0,+65/-111,0.0,45.75,44.14,26.281,89.46,1.579,0.0,False,CPU +51,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,953.0,+121/-156,0.044,46.2,41.0,26.17,2.66,0.553,0.0,False,GPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,949.0,+114/-129,0.05,46.38,14.32,29.0,276.0,0.368,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,948.0,+121/-161,0.024,46.45,40.96,27.548,0.29,0.041,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,926.0,+106/-145,0.019,47.35,41.98,31.115,1.45,0.123,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,920.0,+122/-145,0.042,47.6,22.6,29.565,276.0,0.065,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,810.0,+84/-146,0.0,51.45,50.54,31.853,89.46,0.075,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,624.0,+110/-269,0.0,55.5,55.08,43.289,0.22,0.02,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4b3ce9b80b951a28613ed682675de2ecafa54f3e --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0483268e20186a3da29d0fbfaf0925938bb078bde1ebc90a02ef29676184250 +size 2250455 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/n_datasets_38 b/data/imputation_no/splits_lite/tasks_classification/datasets_all/n_datasets_38 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ef14fc114cbdfe7fc5d6f5c62c0360c06d06136f --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9db303e8bb6b504a8e841c2216edef7474ab321dcc65bed34b91964a7f8a2059 +size 440786 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..db366eb1918944048ac9aba539fe786ee9e287e0 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1d146f3660c5a23e6b3c365e0dca9cf9c7f0578c986ab495545d3ed6df002fe +size 990599 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..0f0e86391015353b7213ce5d17195673f3eee845 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:974ffb2ad6c1d0bc3fe2b63babe5d57ca926f337ff76059b538f1ac4754a50f0 +size 124439 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..a94f5357046d98b5326ddf967e870e06a3119a1a --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv @@ -0,0 +1,53 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1607.0,+92/-78,0.669,8.54,3.62,6.887,521.51,7.239,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1600.0,+107/-91,0.647,8.76,2.69,4.536,2063.62,9.938,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1569.0,+100/-83,0.595,9.91,4.18,7.055,2063.62,1.087,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1555.0,+99/-65,0.579,10.45,4.35,8.539,1815.7,1.456,0.0,False,CPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1550.0,+93/-73,0.571,10.62,4.56,7.411,5.62,0.771,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1508.0,+67/-58,0.49,12.42,6.88,10.213,2773.9,16.363,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1437.0,+57/-53,0.412,15.8,7.08,11.863,2773.9,0.39,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1429.0,+76/-65,0.399,16.24,7.49,12.341,2461.81,1.283,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1413.0,+61/-47,0.315,17.04,12.62,13.098,391.23,1.243,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1386.0,+73/-54,0.335,18.47,11.49,12.614,1326.58,0.531,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1386.0,+99/-64,0.355,18.5,7.28,11.346,6067.13,404.225,0.0,False,GPU +11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1380.0,+74/-69,0.299,18.8,11.7,13.691,682.62,0.971,0.0,False,CPU +12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1367.0,+70/-78,0.313,19.53,9.78,13.123,4853.07,0.473,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1367.0,+89/-64,0.33,19.57,10.85,13.311,2461.81,0.138,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1366.0,+70/-60,0.3,19.63,13.51,13.149,1326.58,0.051,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1365.0,+51/-53,0.279,19.66,10.49,13.258,5.74,0.107,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1362.0,+100/-92,0.346,19.84,8.61,13.165,4853.07,9.386,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1351.0,+66/-53,0.256,20.42,15.26,14.046,391.23,0.186,0.0,False,CPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1348.0,+71/-68,0.265,20.61,10.93,13.945,682.62,0.183,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1334.0,+77/-75,0.305,21.41,7.09,13.381,6067.13,39.565,0.0,False,GPU +20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1329.0,+71/-62,0.257,21.66,12.57,15.718,250.94,2.036,0.0,False,GPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1324.0,+68/-62,0.259,21.95,11.98,14.425,1053.72,2.067,0.0,False,GPU +22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1322.0,+66/-54,0.217,22.05,13.58,15.486,2399.23,0.558,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1299.0,+80/-64,0.243,23.39,14.93,15.974,10.31,0.136,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1277.0,+48/-53,0.174,24.68,18.36,15.239,2480.32,2.175,0.0,False,CPU +25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1274.0,+67/-49,0.168,24.82,16.77,15.84,10.77,1.787,0.0,False,GPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1272.0,+74/-59,0.191,24.93,12.06,16.074,1053.72,0.105,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1260.0,+61/-53,0.157,25.68,14.05,16.415,2399.23,0.056,0.0,False,CPU +28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1256.0,+63/-53,0.159,25.89,13.57,17.309,6.77,0.052,0.0,False,CPU +29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1220.0,+82/-74,0.195,27.93,10.63,16.595,47.89,39.446,0.0,False,GPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+71/-90,0.161,28.04,18.88,18.456,622.79,5.037,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1205.0,+67/-61,0.137,28.82,22.17,17.532,2480.32,0.117,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1203.0,+69/-70,0.129,28.92,15.16,17.049,1.88,0.121,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1196.0,+66/-58,0.119,29.29,20.81,18.674,197.16,0.721,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1172.0,+63/-66,0.099,30.66,14.84,19.122,13.35,0.353,0.0,False,GPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1172.0,+67/-77,0.106,30.68,21.08,19.605,327.31,0.708,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1159.0,+66/-81,0.118,31.38,17.52,19.975,197.16,0.083,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1150.0,+60/-62,0.087,31.88,27.07,18.358,2.25,0.159,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1131.0,+73/-82,0.108,32.89,19.44,20.473,622.79,0.346,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1128.0,+59/-63,0.055,33.04,25.79,20.441,327.31,0.085,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1028.0,+64/-78,0.035,38.03,33.9,23.172,5.73,0.148,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+68/-81,0.02,39.28,29.08,26.523,0.37,0.038,0.0,False,CPU +42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,990.0,+78/-108,0.033,39.71,34.47,27.289,140.66,2.876,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,986.0,+71/-68,0.04,39.87,35.96,24.378,3.12,0.334,0.0,False,CPU +44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,973.0,+91/-93,0.053,40.42,34.66,27.31,3.48,0.677,0.0,False,GPU +45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,972.0,+86/-141,0.063,40.45,31.52,29.506,1.04,0.192,0.0,False,GPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,965.0,+89/-97,0.03,40.75,19.72,30.755,256.15,0.36,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,932.0,+92/-106,0.023,42.03,27.39,31.478,256.15,0.07,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,926.0,+80/-103,0.016,42.24,38.26,29.05,0.24,0.037,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,871.0,+93/-114,0.013,44.11,40.88,34.085,1.32,0.117,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,809.0,+92/-127,0.013,45.89,43.66,34.791,140.66,0.188,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,553.0,+116/-146,0.0,50.42,50.13,49.426,0.18,0.028,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..cb5a4c72594701283ba7295dd693886aebe55ef2 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b89ecc0cbe9882f613adfae8c80280d7eb19c6519edf68ab7dbc81a01484b94f +size 1915089 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/n_datasets_10 b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/n_datasets_10 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..84a96d82d08c70daa0ff33b174446cb4948a6a90 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6090ddcc56ae88bb63938583bb61e4456475637b908970bdff58a15ccab8deba +size 425862 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..65a36815256ad1bc7d77aa05b8329b8d8bcd65c7 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71588239453cd962021aa88c7e8e91c20ec6fc2ea78d7603bb32f047dfd7a3ed +size 1011666 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..619f242c4b9305c5e043c946cf6565f3403587f0 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d302790329d0a0a6af88b80985623aa258d3c1906bf67fcb509a44e0d6794385 +size 118671 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..1fcb88a2b9a74b4cd5c09c788e1c15ff2f43d5d5 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv @@ -0,0 +1,54 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,2043.0,+165/-76,0.873,3.2,2.12,1.556,307.34,2.186,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1977.0,+218/-61,0.845,4.1,2.28,2.107,266.49,2.963,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1724.0,+267/-141,0.574,10.3,3.13,5.287,780.95,11.64,0.0,False,GPU +3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1699.0,+243/-163,0.601,11.2,4.14,5.346,2335.82,14.145,0.0,False,GPU +4,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1675.0,+165/-110,0.463,12.1,8.59,7.754,202.53,0.43,0.0,False,CPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1664.0,+245/-124,0.494,12.5,6.97,7.388,780.95,1.493,0.0,False,GPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1659.0,+175/-97,0.517,12.7,7.86,7.039,688.25,0.192,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1647.0,+189/-83,0.444,13.2,9.54,8.663,91.02,0.568,0.0,False,CPU +8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1638.0,+171/-87,0.491,13.55,8.8,7.113,688.25,0.036,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1632.0,+128/-91,0.463,13.8,9.46,7.465,1635.73,2.146,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1604.0,+180/-105,0.429,15.0,5.57,8.84,1400.65,0.571,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1603.0,+136/-69,0.455,15.05,5.96,7.228,2.2,0.044,0.0,False,CPU +12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1599.0,+175/-116,0.418,15.2,10.54,9.77,2335.82,0.502,0.0,False,GPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1588.0,+183/-107,0.409,15.7,11.05,8.072,202.53,0.055,0.0,False,CPU +14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1570.0,+182/-110,0.352,16.5,12.53,9.516,91.02,0.074,0.0,False,CPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1542.0,+159/-111,0.351,17.8,10.04,9.875,1635.73,0.082,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1522.0,+209/-168,0.353,18.7,8.98,9.432,1400.65,0.065,0.0,False,GPU +17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1514.0,+194/-127,0.329,19.1,9.49,8.469,2.3,0.818,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1499.0,+177/-140,0.294,19.8,10.12,10.117,8.46,3.879,0.0,False,GPU +19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1450.0,+204/-130,0.249,22.2,15.55,12.904,899.24,0.221,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1446.0,+233/-158,0.29,22.4,6.94,10.304,0.47,0.047,0.0,False,CPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1420.0,+113/-70,0.158,23.7,19.71,11.014,1128.73,1.573,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1412.0,+189/-172,0.223,24.1,19.43,12.709,5.79,0.084,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1408.0,+143/-135,0.169,24.3,18.96,11.095,4546.1,406.11,0.0,False,GPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1402.0,+158/-144,0.222,24.6,16.19,11.567,713.0,2.317,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1375.0,+166/-140,0.149,26.0,18.4,14.592,713.0,0.155,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1349.0,+96/-101,0.071,27.3,24.64,14.558,6.53,0.294,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1326.0,+165/-163,0.123,28.45,18.52,14.827,4546.1,39.323,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1321.0,+172/-131,0.136,28.7,22.41,14.536,899.24,0.015,0.0,False,CPU +29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1315.0,+157/-123,0.104,29.0,25.99,15.96,1.95,0.02,0.0,False,CPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1315.0,+106/-156,0.099,29.0,23.43,15.425,387.73,2.446,0.0,False,CPU +31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1311.0,+135/-106,0.073,29.2,21.79,14.282,52.7,0.202,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1287.0,+118/-103,0.073,30.4,27.23,14.195,1128.73,0.054,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1283.0,+145/-152,0.057,30.6,27.42,13.773,38.59,0.337,0.0,False,CPU +34,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1283.0,+183/-175,0.117,30.6,20.24,15.553,82.52,0.97,0.0,False,GPU +35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1268.0,+124/-101,0.056,31.3,26.28,14.704,52.7,0.017,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1225.0,+105/-129,0.038,33.3,31.31,13.946,0.33,0.05,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1212.0,+92/-115,0.009,33.9,32.53,14.785,11.03,0.489,0.0,False,GPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1187.0,+152/-168,0.031,35.0,31.95,17.259,38.59,0.028,0.0,False,CPU +39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1186.0,+136/-139,0.022,35.05,31.57,18.812,41.63,39.259,0.0,False,GPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1131.0,+103/-160,0.016,37.4,35.51,18.995,387.73,0.089,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1100.0,+94/-110,0.009,38.6,37.21,17.804,1.84,0.047,0.0,False,CPU +42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1048.0,+178/-260,0.014,40.55,36.73,28.29,43.41,0.111,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1036.0,+114/-212,0.018,41.0,38.29,24.539,332.4,33.481,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1018.0,+144/-200,0.022,41.6,38.72,22.564,1.63,0.11,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+117/-257,0.0,42.2,40.81,22.603,0.11,0.012,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,998.0,+166/-272,0.002,42.25,39.61,29.261,43.41,0.02,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,838.0,+248/-564,0.024,46.7,39.85,33.498,2.96,0.183,0.0,False,GPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,834.0,+103/-217,0.0,46.8,46.47,30.86,0.21,0.019,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,829.0,+93/-235,0.0,46.9,46.61,27.581,0.1,0.02,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,782.0,+66/-184,0.0,47.9,47.61,33.014,332.4,1.631,0.0,False,CPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,761.0,+125/-257,0.0,48.3,47.82,35.637,0.18,0.032,0.0,False,GPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,426.0,+114/-399,0.0,52.2,52.17,49.883,0.12,0.094,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..28b509e32fd196f36fe8d4f35754f7192712e0dd --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9f1c34da9b71ee151c8fd4d82a368efaad8540acc7987df5938ab016eb06710 +size 1687839 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/n_datasets_28 b/data/imputation_no/splits_lite/tasks_classification/datasets_small/n_datasets_28 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1025badef7a0aca774b15f059c17c4713f4d334e --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36a6c91dd9a78d85c00e8d8b0e330ac1035ac29a581864f99acc21dcd66b64e1 +size 437874 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..2573a0d877e5765f6aa675383718a60441b22456 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ace139da4a1e35dc36d52ef900bd9886308b277cf4e0ef38355cfd99fba033e +size 946898 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..0700a0f2a558845f0822f53b10430b993a2973db --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51a32b87534cb76286da6333129196bc4c0401afb15e9303db2ffd3ef09a6020 +size 128773 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..3526ce8863a4af890ab284d59d6547000702c35c --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv @@ -0,0 +1,54 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1610.0,+109/-82,0.641,7.88,3.92,7.306,6.03,0.703,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1592.0,+131/-96,0.66,8.46,2.69,4.54,2279.85,9.666,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1564.0,+130/-89,0.615,9.45,3.78,7.209,2279.85,0.968,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1535.0,+111/-88,0.589,10.57,4.69,8.868,635.11,8.569,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1487.0,+92/-69,0.473,12.61,6.55,11.474,3486.44,22.867,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1468.0,+96/-69,0.458,13.5,7.44,11.301,2993.17,1.401,0.0,False,CPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1456.0,+99/-63,0.476,14.09,5.46,10.565,4.53,0.611,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1421.0,+77/-75,0.404,15.84,7.35,12.856,3486.44,1.002,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1394.0,+85/-74,0.412,17.29,6.14,11.685,7432.95,404.225,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1391.0,+83/-79,0.367,17.43,8.78,13.865,3083.36,1.48,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1358.0,+62/-49,0.257,19.27,15.23,14.947,774.51,1.455,0.0,False,CPU +11,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1356.0,+104/-78,0.298,19.36,11.43,16.037,303.47,2.324,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1349.0,+89/-90,0.365,19.79,5.93,13.112,7432.95,39.797,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1333.0,+94/-70,0.3,20.7,12.07,14.97,3083.36,0.189,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1322.0,+76/-65,0.251,21.36,14.57,14.88,1612.87,0.724,0.0,False,CPU +15,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1313.0,+86/-74,0.264,21.89,10.05,14.576,6286.73,0.473,0.0,False,GPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1312.0,+82/-73,0.258,21.93,12.15,15.706,1165.19,2.067,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1310.0,+68/-72,0.223,22.05,13.9,16.083,829.32,1.642,0.0,False,CPU +18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1308.0,+58/-51,0.2,22.2,16.21,15.686,6.97,0.144,0.0,False,CPU +19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1300.0,+67/-46,0.221,22.66,17.19,15.572,1612.87,0.061,0.0,False,CPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1299.0,+68/-66,0.21,22.71,18.04,15.928,774.51,0.282,0.0,False,CPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1295.0,+71/-62,0.195,22.96,13.86,16.677,2989.36,0.609,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1290.0,+73/-69,0.202,23.25,11.14,16.308,829.32,0.241,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1281.0,+92/-81,0.242,23.75,14.62,16.222,6286.73,9.386,0.0,False,GPU +24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1275.0,+82/-83,0.24,24.11,14.13,17.407,11.86,0.149,0.0,False,GPU +25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1260.0,+70/-66,0.183,25.0,15.68,16.571,14.59,4.035,0.0,False,GPU +26,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1250.0,+62/-68,0.156,25.61,12.67,17.351,2989.36,0.063,0.0,False,CPU +27,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1249.0,+77/-84,0.194,25.7,11.39,16.867,1165.19,0.105,0.0,False,GPU +28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1247.0,+81/-65,0.161,25.82,12.12,18.065,8.12,0.053,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1243.0,+83/-70,0.172,26.04,18.42,17.009,2894.94,2.42,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1237.0,+82/-100,0.255,26.39,8.68,16.048,51.78,39.927,0.0,False,GPU +31,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1198.0,+100/-88,0.175,28.73,18.61,19.791,1425.28,7.355,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1188.0,+89/-73,0.153,29.32,21.46,18.976,2894.94,0.145,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1179.0,+81/-82,0.139,29.89,19.84,20.674,402.38,1.373,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1167.0,+74/-90,0.119,30.57,12.6,20.919,14.22,0.353,0.0,False,GPU +35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1156.0,+85/-85,0.147,31.2,15.48,21.194,402.38,0.163,0.0,False,CPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1137.0,+68/-74,0.068,32.32,27.34,19.723,2.85,0.19,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1136.0,+77/-96,0.114,32.36,22.36,21.769,562.0,1.155,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1136.0,+86/-93,0.135,32.39,19.06,21.239,1425.28,0.561,0.0,False,CPU +39,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1132.0,+66/-78,0.099,32.59,26.88,20.192,3.27,0.193,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1092.0,+71/-81,0.052,34.84,27.16,22.751,562.0,0.138,0.0,False,CPU +41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1012.0,+100/-117,0.079,38.93,29.19,27.565,2.45,0.519,0.0,False,GPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1010.0,+81/-101,0.037,39.04,33.86,25.349,9.06,0.196,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1001.0,+90/-124,0.058,39.43,34.33,25.341,5.43,0.836,0.0,False,GPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+83/-113,0.026,39.48,26.99,28.169,0.46,0.07,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,978.0,+88/-115,0.037,40.46,34.33,28.52,123.01,1.579,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,978.0,+86/-99,0.038,40.5,36.42,25.269,5.03,0.601,0.0,False,CPU +47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,945.0,+93/-118,0.02,41.89,37.28,29.809,0.38,0.072,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,941.0,+88/-110,0.036,42.07,17.14,31.884,479.28,0.495,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,912.0,+89/-121,0.031,43.2,25.1,32.518,479.28,0.098,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,877.0,+103/-132,0.018,44.46,40.34,35.489,2.34,0.132,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,811.0,+102/-144,0.018,46.54,43.84,35.669,123.01,0.092,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,572.0,+114/-198,0.0,51.14,50.77,49.495,0.28,0.024,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1fca51f24d8676405ddffc53f7ff83c91d1198ed --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5afcee3ae2b3df2ff09415fbe6a55cc5ab59ca1052b7a50c9788900523555fb1 +size 2005320 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/n_datasets_26 b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/n_datasets_26 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1066529811eaef6072ffdddad4df9a4e7d1b5961 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c76f2032f15bc5f3823418353bb630772d6e8c454891890127fdcdc5faf7683b +size 481575 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c882ce462312d2bc574402e1c4d851e59231e50b --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42e415ce939a495f49002e5905cef5bd038017462a815d5cfa07d9d3744e50b9 +size 1038842 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..df9c41b59e54e9bfa0952e60f46f3af41eda6049 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5ce48c8471515189a0ee09615c926602bb7c5eb214e2050bb6b85ede4fe18b3 +size 121062 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..c5cdb97720afb21c722cc84043868ea75819c1b0 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1648.0,+88/-68,0.635,8.1,5.12,7.839,5.78,0.586,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1624.0,+124/-100,0.653,8.92,2.96,4.968,2162.76,8.93,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1601.0,+124/-104,0.61,9.81,4.39,7.727,2162.76,0.894,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1533.0,+113/-82,0.548,12.73,5.43,9.618,620.85,8.569,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1510.0,+93/-63,0.463,13.81,6.88,12.159,3159.9,21.159,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1509.0,+107/-89,0.488,13.87,6.57,11.53,3361.32,51.669,0.0,False,GPU +6,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1472.0,+108/-68,0.432,15.79,7.34,12.029,150.4,4.403,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1467.0,+82/-70,0.41,16.08,8.22,12.218,2628.27,1.401,0.0,False,CPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1460.0,+100/-66,0.439,16.44,6.0,11.286,4.3,0.584,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1446.0,+89/-57,0.394,17.25,8.14,13.535,3159.9,0.646,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1403.0,+84/-80,0.341,19.75,10.41,14.036,3361.32,0.527,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1403.0,+103/-85,0.381,19.77,6.66,12.433,6883.9,405.331,0.0,False,GPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1392.0,+106/-73,0.335,20.42,9.82,14.604,2899.02,1.461,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1390.0,+84/-96,0.354,20.52,6.2,13.131,9.07,2.008,0.0,False,GPU +14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1377.0,+96/-83,0.298,21.31,12.92,16.382,303.47,2.098,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1360.0,+81/-99,0.341,22.42,6.69,13.867,6883.9,39.606,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1345.0,+83/-90,0.316,23.35,7.94,15.148,4.28,0.511,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1345.0,+53/-41,0.192,23.37,19.01,16.138,691.46,1.455,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1336.0,+73/-88,0.265,23.88,10.86,15.115,6020.89,0.454,0.0,False,GPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1330.0,+102/-70,0.27,24.33,14.64,15.756,2899.02,0.171,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1326.0,+85/-72,0.213,24.52,16.47,16.007,1394.45,0.618,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1308.0,+69/-57,0.182,25.71,19.95,17.318,785.9,1.281,0.0,False,CPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1306.0,+76/-53,0.169,25.83,18.07,16.868,5.74,0.139,0.0,False,CPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1305.0,+73/-64,0.185,25.92,15.16,17.587,2399.23,0.565,0.0,False,CPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1300.0,+79/-73,0.2,26.27,18.27,16.877,1053.72,1.872,0.0,False,GPU +25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1297.0,+91/-84,0.239,26.42,17.12,17.031,6020.89,8.478,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1293.0,+74/-51,0.173,26.69,21.51,16.785,1394.45,0.053,0.0,False,CPU +27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1290.0,+72/-60,0.166,26.88,12.61,17.51,785.9,0.213,0.0,False,CPU +28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1288.0,+71/-52,0.163,27.0,21.23,17.102,691.46,0.264,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1285.0,+102/-73,0.23,27.19,16.18,18.07,11.6,0.148,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1284.0,+83/-65,0.17,27.27,16.3,17.108,12.75,3.502,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1257.0,+98/-127,0.259,29.04,8.67,16.756,51.52,40.126,0.0,False,GPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+90/-93,0.139,29.23,12.47,18.783,7.41,0.052,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+60/-73,0.136,29.27,13.11,18.357,2399.23,0.061,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1253.0,+73/-74,0.169,29.31,21.75,17.668,2792.14,2.175,0.0,False,CPU +35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1234.0,+85/-76,0.126,30.56,21.73,18.071,1053.72,0.103,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1221.0,+93/-108,0.17,31.37,19.9,20.456,1350.04,6.57,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1191.0,+72/-113,0.109,33.29,12.56,21.041,13.67,0.339,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1183.0,+80/-82,0.119,33.81,26.42,19.614,2792.14,0.133,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1160.0,+84/-81,0.086,35.21,27.55,22.147,378.34,1.373,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1154.0,+96/-109,0.126,35.58,21.37,21.966,1350.04,0.613,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1137.0,+91/-92,0.107,36.62,19.58,22.595,378.34,0.154,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1128.0,+60/-59,0.033,37.19,33.98,21.116,2.66,0.17,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1118.0,+107/-106,0.078,37.77,26.65,23.308,527.27,1.155,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1103.0,+69/-69,0.047,38.63,34.62,21.73,3.01,0.177,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1079.0,+91/-89,0.045,40.04,32.03,24.138,527.27,0.128,0.0,False,CPU +46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1029.0,+86/-152,0.069,42.69,32.05,28.036,2.28,0.454,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1013.0,+91/-94,0.036,43.5,36.97,26.238,8.93,0.172,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+86/-108,0.027,44.15,31.42,28.762,0.44,0.065,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,999.0,+93/-122,0.051,44.19,38.38,26.76,5.0,0.753,0.0,False,GPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,990.0,+94/-109,0.031,44.62,40.08,25.606,4.68,0.547,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,959.0,+80/-107,0.009,46.04,43.24,29.843,118.4,1.479,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,956.0,+94/-99,0.038,46.17,16.98,31.197,429.17,0.495,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,946.0,+87/-146,0.02,46.62,41.57,30.28,0.36,0.068,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,930.0,+99/-111,0.032,47.27,25.54,31.807,429.17,0.081,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,897.0,+108/-120,0.015,48.58,43.99,34.214,2.16,0.129,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,766.0,+80/-151,0.0,52.58,51.76,37.314,118.4,0.082,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,576.0,+118/-291,0.0,56.08,55.73,49.46,0.27,0.024,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..330841195967124859a7bb07870ddcf4005f951f --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:144cce74b4d1987e758e0bd815195cf88ac08d5e25ccf6751a48e7580428d27c +size 2336052 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/n_datasets_8 b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/n_datasets_8 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..57003d563e2b8ad13a16cb7d0eec221a4ace58d2 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29448b9639b65b8d438c36d28f34d54d4838750997e49c9f7bb756df67f0393c +size 422738 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..b19427318370f75a74c6130ef65dd13589b8ca84 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4fe67b0531948eabf40de3116acf7d6c7be35bd7d3659852559a52a6d976bc5 +size 955458 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..6e3f254ee4f4747841063b792e2c58334bbd2d86 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3a4deb9312ba0e10f0311dd9ee15bc95913eeb6d3951f2286db9d53fd1cb438 +size 116890 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..a944d92a6b22188f4dac96da04c94dcd01da802d --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv @@ -0,0 +1,53 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1720.0,+746/-199,0.786,5.62,1.64,1.808,2793.65,8.93,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1694.0,+276/-91,0.731,6.25,4.56,11.082,1008.06,9.121,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1678.0,+370/-169,0.673,6.69,3.46,9.911,2793.65,1.513,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1521.0,+345/-142,0.537,12.06,4.52,10.367,9.88,0.825,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1488.0,+229/-105,0.507,13.5,4.06,14.071,5546.65,3.268,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1454.0,+218/-127,0.426,15.12,5.15,15.484,4334.8,21.159,0.0,False,GPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1448.0,+204/-96,0.378,15.38,11.91,15.024,3307.58,1.182,0.0,False,CPU +7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1441.0,+200/-70,0.351,15.75,12.54,15.552,3307.58,0.14,0.0,False,CPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1428.0,+142/-91,0.334,16.38,11.6,16.245,1280.01,4.078,0.0,False,CPU +9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1421.0,+136/-103,0.324,16.75,10.98,16.321,1280.01,1.051,0.0,False,CPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1418.0,+206/-131,0.381,16.88,12.61,15.962,4334.8,3.529,0.0,False,GPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1413.0,+174/-112,0.339,17.12,12.01,18.699,5568.31,1.781,0.0,False,GPU +12,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1396.0,+154/-116,0.337,18.0,11.47,19.038,5568.31,0.372,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1394.0,+223/-132,0.377,18.12,10.3,19.449,9411.86,344.009,0.0,False,GPU +14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1370.0,+216/-163,0.313,19.38,8.26,19.369,2079.21,3.347,0.0,False,GPU +15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1361.0,+134/-89,0.255,19.88,18.12,16.949,2029.77,4.113,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1354.0,+130/-108,0.262,20.25,18.02,16.708,2029.77,1.04,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1333.0,+163/-121,0.249,21.44,16.78,18.145,6684.65,9.59,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1314.0,+218/-144,0.304,22.5,5.96,20.809,9411.86,39.507,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1308.0,+161/-121,0.205,22.81,18.67,18.488,6684.65,0.752,0.0,False,GPU +20,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1305.0,+144/-127,0.236,23.0,14.3,22.14,17.09,0.145,0.0,False,GPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1298.0,+155/-114,0.172,23.38,15.32,20.217,27108.48,0.742,0.0,False,CPU +22,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1295.0,+210/-158,0.243,23.56,12.64,21.612,728.32,2.441,0.0,False,CPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1290.0,+104/-78,0.146,23.88,21.87,18.577,43.1,0.254,0.0,False,CPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1287.0,+249/-173,0.245,24.0,5.71,20.994,2079.21,0.319,0.0,False,GPU +25,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1280.0,+230/-206,0.265,24.38,11.89,20.771,729.17,1.833,0.0,False,CPU +26,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1268.0,+206/-145,0.213,25.06,12.59,21.742,728.32,0.356,0.0,False,CPU +27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1267.0,+129/-128,0.181,25.12,19.54,21.941,283.57,4.104,0.0,False,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1251.0,+131/-104,0.125,26.0,23.18,21.705,3646.83,2.162,0.0,False,CPU +29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1233.0,+119/-130,0.153,27.0,22.32,21.727,27108.48,0.066,0.0,False,CPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1228.0,+122/-124,0.131,27.25,24.61,22.231,17.86,0.099,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1220.0,+145/-166,0.124,27.75,19.6,23.531,3646.83,0.194,0.0,False,CPU +32,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1208.0,+209/-184,0.179,28.38,16.27,22.297,729.17,0.325,0.0,False,CPU +33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1195.0,+219/-198,0.186,29.12,17.06,25.651,2721.87,12.593,0.0,False,CPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1195.0,+155/-153,0.136,29.12,22.73,22.982,5.12,0.444,0.0,False,CPU +35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1188.0,+255/-173,0.216,29.5,10.16,24.06,46.36,39.812,0.0,False,GPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1188.0,+124/-124,0.069,29.5,26.9,20.547,4.93,0.592,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1126.0,+151/-163,0.09,32.75,27.3,25.251,19.12,6.922,0.0,False,GPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1094.0,+168/-176,0.095,34.38,25.91,27.296,2721.87,1.083,0.0,False,CPU +39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1088.0,+162/-219,0.074,34.62,28.92,29.013,8.99,1.372,0.0,False,GPU +40,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1088.0,+126/-157,0.022,34.62,32.43,27.29,17.24,0.565,0.0,False,GPU +41,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1012.0,+267/-384,0.13,38.12,25.29,38.476,155.52,2.046,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+105/-213,0.0,38.62,36.72,35.562,0.74,0.148,0.0,False,CPU +43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,975.0,+97/-144,0.0,39.62,38.73,30.565,14.37,0.363,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,928.0,+173/-268,0.031,41.38,36.93,34.532,8.37,0.659,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,923.0,+84/-249,0.0,41.56,40.55,41.326,909.67,0.609,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,901.0,+89/-241,0.0,42.31,41.21,42.282,909.67,0.107,0.0,False,CPU +47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,895.0,+167/-230,0.011,42.5,39.66,40.712,0.76,0.153,0.0,False,CPU +48,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,769.0,+281/-1308,0.063,46.0,38.91,50.559,155.52,0.191,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,752.0,+76/-209,0.0,46.38,46.17,41.42,4.67,0.855,0.0,False,GPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,710.0,+102/-321,0.0,47.25,46.92,48.104,4.62,0.135,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-292.0,+145/-1079,0.0,52.0,52.0,70.927,0.52,0.075,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ceb46e1d66dc1d3ab1a03bb3def9b25c3dc7981c --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f44c9215d321325a14ac6cf78025744cf0e741a1129e14ee12c9b24a32e46d1f +size 1446692 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/n_datasets_1 b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/n_datasets_1 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..eb960811c372a4320ec6aa5e015c13174c74be3c --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1812b98843d341304c501dd574fdfad807a23e9ee799aec728614df19c8b890b +size 451105 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d8fa81862f8c84571b8961b8df31068db4e68d1a --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7fee4d9b48c505eb387673c73890374048693708f19e8ea622267e8b6e84dcd +size 957364 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1bf01fae9dfd5c96cbef9b7b9f91540a430844b0 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96e37ecf79b85f239b72bcde25f28b50e7018d8817833f0fdea1290721e6337f +size 107975 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..ed900c3d3f2df0c96ed91727ee80cc17ac49c7f9 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv @@ -0,0 +1,54 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,25228.0,+0/-0,1.0,1.0,1.0,0.0,134.59,8.308,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,23917.0,+0/-0,0.934,2.0,2.0,1.729,276.98,4.897,0.0,False,CPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,22310.0,+0/-0,0.915,3.0,3.0,2.208,276.84,3.303,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,20483.0,+0/-0,0.883,4.0,4.0,3.024,134.59,2.819,0.0,False,GPU +4,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,19432.0,+0/-0,0.576,5.0,5.0,10.141,7855.1,0.565,0.0,False,CPU +5,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,18258.0,+0/-0,0.572,6.0,6.0,10.238,61.47,0.154,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,17510.0,+0/-0,0.569,7.0,7.0,10.303,157.4,4.579,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,16643.0,+0/-0,0.567,8.0,8.0,10.333,157.4,0.401,0.0,False,CPU +8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,15885.0,+0/-0,0.551,9.0,9.0,10.67,61.47,0.015,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,15137.0,+0/-0,0.525,10.0,10.0,11.231,2687.46,0.361,0.0,False,CPU +10,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,14345.0,+0/-0,0.522,11.0,11.0,11.295,7855.1,0.048,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,13528.0,+0/-0,0.517,12.0,12.0,11.389,2687.46,0.793,0.0,False,CPU +12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,12768.0,+0/-0,0.515,13.0,13.0,11.441,456.33,1.171,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,12036.0,+0/-0,0.501,14.0,14.0,11.722,456.33,0.047,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,11311.0,+0/-0,0.501,15.0,15.0,11.722,0.32,0.047,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,10518.0,+0/-0,0.432,16.0,16.0,13.143,8.22,0.049,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,9807.0,+0/-0,0.42,17.0,17.0,13.383,4378.82,25.595,0.0,False,GPU +17,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,9111.0,+0/-0,0.387,18.0,18.0,14.026,20.73,0.079,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,8390.0,+0/-0,0.386,19.0,19.0,14.047,4378.82,2.216,0.0,False,GPU +19,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,7635.0,+0/-0,0.356,20.0,20.0,14.639,1.48,0.535,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,6953.0,+0/-0,0.292,21.0,21.0,15.857,2.79,1.057,0.0,False,GPU +21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,6264.0,+0/-0,0.249,22.0,22.0,16.662,20.88,0.359,0.0,False,CPU +22,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,5562.0,+0/-0,0.111,23.0,23.0,19.143,20.88,0.026,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,5007.0,+0/-0,0.059,24.0,24.0,20.028,4390.96,275.957,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,4306.0,+0/-0,0.024,25.0,25.0,20.621,2341.91,0.027,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,3678.0,+0/-0,0.004,26.0,26.0,20.955,14.53,1.747,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,3004.0,+0/-0,0.0,27.0,27.0,21.025,2341.91,0.172,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,2329.0,+0/-0,0.0,28.0,28.0,21.961,1993.57,0.469,0.0,False,GPU +28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1701.0,+0/-0,0.0,29.0,29.0,22.049,1993.57,0.055,0.0,False,GPU +29,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+0/-0,0.0,30.0,30.0,24.001,0.18,0.007,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,445.0,+0/-0,0.0,31.0,31.0,24.06,4390.96,38.525,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,-256.0,+0/-0,0.0,32.0,32.0,24.187,41.21,39.406,0.0,False,GPU +32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,-946.0,+0/-0,0.0,33.0,33.0,24.591,1322.48,0.546,0.0,False,GPU +33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,-1628.0,+0/-0,0.0,34.0,34.0,25.183,1322.48,0.055,0.0,False,GPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,-2382.0,+0/-0,0.0,35.0,35.0,26.439,0.21,0.047,0.0,False,CPU +35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,-3104.0,+0/-0,0.0,36.0,36.0,29.215,2009.11,3.767,0.0,False,GPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,-3800.0,+0/-0,0.0,37.0,37.0,30.84,2009.11,0.974,0.0,False,GPU +37,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,-4511.0,+0/-0,0.0,38.0,38.0,31.796,6.84,0.09,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-5304.0,+0/-0,0.0,39.0,39.0,31.813,3.34,0.028,0.0,False,CPU +39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,-6029.0,+0/-0,0.0,40.0,40.0,34.136,66.06,1.783,0.0,False,GPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,-6760.0,+0/-0,0.0,41.0,41.0,38.819,6.51,0.162,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,-7521.0,+0/-0,0.0,42.0,42.0,41.321,553.48,0.174,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,-8338.0,+0/-0,0.0,43.0,43.0,42.976,553.48,1.636,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,-9130.0,+0/-0,0.0,44.0,44.0,50.562,0.08,0.017,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-9878.0,+0/-0,0.0,45.0,45.0,52.296,2.73,0.083,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-10636.0,+0/-0,0.0,46.0,46.0,52.871,30.19,0.134,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-11503.0,+0/-0,0.0,47.0,47.0,55.965,30.19,0.017,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-12250.0,+0/-0,0.0,48.0,48.0,56.26,0.13,0.015,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-13425.0,+0/-0,0.0,49.0,49.0,57.211,76.41,2.853,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,-14476.0,+0/-0,0.0,50.0,50.0,57.769,3.14,0.921,0.0,False,GPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,-16303.0,+0/-0,0.0,51.0,51.0,57.822,0.11,0.023,0.0,False,GPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-17910.0,+0/-0,0.0,52.0,52.0,70.703,76.41,0.258,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-19221.0,+0/-0,0.0,53.0,53.0,83.729,0.14,0.123,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5ebf7a0908d62c24eea90413a3d4a59af16f618a --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eae61392bfed037d20554aa6d1ceb8064a1145d57fdd5dbdc8a5bad0d955b073 +size 1057411 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/n_datasets_7 b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/n_datasets_7 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4903597b8356b7aca05e5d3ddfa3aaff62a7cf89 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81826499b7764ac326c05d7f3688e9420dcf17e68544f35787e82f2306de4e3f +size 420132 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..0edf73c2f6e8be017f8cd2efbf6347113c8a09b3 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b63aae85dfcb94f39861e8fef24396a481b7bff5b07141ce40782984093927e +size 954003 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..cc48aae2c232d5778c0f805bace307d1fa91c7a8 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5a7713cdce67dd6196975757021286317e9fa53b2847523ed235fc5dc137490 +size 120826 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..0e6a76b5edd0dc868d04ab23cea3ee4478d0213d --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv @@ -0,0 +1,54 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1735.0,+490/-208,0.754,6.43,1.8,2.066,2837.96,9.546,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1714.0,+270/-90,0.701,7.0,5.03,12.349,1060.41,10.928,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1692.0,+452/-185,0.639,7.64,3.61,10.895,2837.96,1.134,0.0,False,GPU +3,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1606.0,+237/-104,0.529,10.57,6.05,10.653,6.19,1.301,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1586.0,+384/-174,0.555,11.36,4.28,9.756,10.97,0.856,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1542.0,+286/-141,0.482,13.29,4.65,14.183,4655.15,21.791,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1499.0,+207/-149,0.431,15.29,12.16,14.644,4655.15,4.031,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1493.0,+283/-109,0.439,15.57,4.78,15.834,6318.92,3.195,0.0,False,CPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1476.0,+162/-116,0.383,16.43,11.56,18.233,5729.39,1.817,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1473.0,+246/-122,0.35,16.57,12.44,15.543,3375.11,1.208,0.0,False,CPU +10,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1459.0,+174/-119,0.381,17.29,10.81,18.608,5729.39,0.373,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1459.0,+194/-76,0.315,17.29,13.39,16.169,3375.11,0.132,0.0,False,CPU +12,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1448.0,+180/-192,0.351,17.86,7.75,17.962,2149.3,2.928,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1445.0,+228/-195,0.412,18.0,9.68,19.367,10182.26,399.569,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1437.0,+194/-90,0.287,18.43,13.96,17.093,1394.84,3.578,0.0,False,CPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1432.0,+228/-90,0.279,18.71,13.01,17.176,1394.84,1.087,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1376.0,+118/-82,0.206,21.71,20.11,17.736,2342.58,5.131,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1374.0,+204/-136,0.267,21.86,13.5,20.76,19.13,0.146,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1371.0,+119/-98,0.217,22.0,19.75,17.42,2342.58,1.065,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1369.0,+252/-210,0.345,22.14,5.38,20.345,10182.26,40.124,0.0,False,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1355.0,+170/-136,0.214,22.93,18.01,18.825,6994.17,9.554,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1351.0,+203/-188,0.277,23.14,5.23,19.587,2149.3,0.147,0.0,False,GPU +22,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1336.0,+148/-119,0.204,24.0,19.03,20.198,326.23,4.282,0.0,False,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1332.0,+178/-142,0.17,24.21,19.46,19.122,6994.17,0.583,0.0,False,GPU +24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1322.0,+208/-231,0.236,24.79,12.36,22.319,772.2,2.495,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1303.0,+128/-72,0.1,25.86,23.83,19.353,57.99,0.317,0.0,False,CPU +26,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1294.0,+198/-204,0.22,26.36,12.36,22.113,772.2,0.373,0.0,False,CPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1285.0,+151/-114,0.137,26.86,23.46,21.802,3833.57,2.294,0.0,False,CPU +28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1285.0,+127/-98,0.11,26.86,22.23,21.656,44054.5,0.843,0.0,False,CPU +29,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1268.0,+204/-281,0.216,27.86,15.62,22.276,731.01,2.021,0.0,False,CPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1263.0,+270/-230,0.206,28.14,17.27,23.176,2750.22,13.569,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1245.0,+189/-158,0.132,29.14,19.47,23.947,3833.57,0.22,0.0,False,CPU +32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1242.0,+194/-213,0.154,29.29,22.35,22.488,5.15,0.526,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1237.0,+117/-124,0.09,29.57,27.07,23.403,14.99,0.113,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1230.0,+256/-231,0.247,30.0,9.32,24.042,51.52,40.219,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1227.0,+129/-113,0.095,30.14,26.81,23.217,44054.5,0.068,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1190.0,+137/-224,0.122,32.14,20.49,23.958,731.01,0.334,0.0,False,CPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1182.0,+76/-92,0.004,32.57,31.38,21.807,5.38,0.624,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1179.0,+159/-187,0.1,32.71,27.33,23.313,20.63,7.532,0.0,False,GPU +39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1163.0,+154/-219,0.081,33.57,28.55,24.905,9.4,1.444,0.0,False,GPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1146.0,+234/-242,0.108,34.43,26.13,25.292,2750.22,1.239,0.0,False,CPU +41,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1096.0,+121/-168,0.015,36.86,34.67,28.196,19.64,0.478,0.0,False,GPU +42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1080.0,+257/-419,0.147,37.57,24.17,35.799,180.38,1.24,0.0,False,CPU +43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1004.0,+115/-178,0.0,40.86,39.86,30.387,14.87,0.391,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+112/-308,0.0,41.0,39.22,37.213,0.78,0.155,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,974.0,+204/-308,0.035,42.0,37.31,31.994,8.78,0.688,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,972.0,+82/-247,0.0,42.07,41.01,39.676,913.1,0.738,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,952.0,+100/-294,0.0,42.79,41.64,40.327,913.1,0.13,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,934.0,+190/-297,0.01,43.43,40.35,39.305,0.77,0.156,0.0,False,CPU +49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,836.0,+283/-1332,0.072,46.29,39.0,47.681,180.38,0.136,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,813.0,+47/-188,0.0,46.86,46.67,39.077,5.48,0.91,0.0,False,GPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,743.0,+101/-303,0.0,48.29,47.92,46.939,4.65,0.14,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-420.0,+334/-866,0.0,53.0,53.0,69.099,0.58,0.071,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..99737fefb923bf56f55481c7efa8a571d6891eae --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d79f0882c3673bddbf59fa975801a9b5738d58afa79d94afacd8da62d257867 +size 1435771 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/n_datasets_6 b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/n_datasets_6 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d3f437b39ade30f566216b7f3409472792a8c106 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0450671d94f605a39f56c84a18e00dfc6e05049a9d39d4fa7ab252223399b5be +size 456644 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3cbf67d18acbd9071304c5fcf41261ea1b4c9014 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b95440aa8b1ca7c2e94a4d13048ceb5de5d9978ceea4f3f797981bdff278c21 +size 1036337 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4a59fe4c4720204207629ad652826339cb880fd2 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c63e9680d6e6ae47a09e56b4b23f175296c0f4fe43ce347e00f7de2a91db0eb +size 125861 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..4f05fb5c88ee5a61d5225392ab49c2f9a4cf43ae --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1997.0,+809/-100,0.875,3.17,1.58,2.191,2793.65,8.93,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1857.0,+337/-82,0.729,5.42,4.12,12.495,2793.65,1.02,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1746.0,+356/-135,0.689,8.17,5.69,14.332,1295.16,14.354,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1716.0,+553/-144,0.632,9.08,4.46,11.083,9.88,0.825,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1688.0,+378/-116,0.596,10.0,5.22,14.698,3561.2,34.413,0.0,False,GPU +5,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1634.0,+358/-88,0.506,12.0,6.72,12.332,5.93,0.989,0.0,False,GPU +6,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1626.0,+204/-38,0.513,12.33,10.85,15.748,206.59,4.778,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1622.0,+272/-96,0.534,12.5,4.43,16.24,4334.8,21.159,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1583.0,+233/-63,0.472,14.17,12.85,16.59,4334.8,3.529,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1522.0,+224/-109,0.384,17.0,14.03,17.857,3307.58,1.182,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1506.0,+327/-108,0.373,17.83,11.35,16.716,3561.2,0.612,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1502.0,+371/-116,0.388,18.0,9.36,14.808,5.48,0.434,0.0,False,GPU +12,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1489.0,+378/-152,0.401,18.67,4.6,18.377,6700.69,3.268,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1476.0,+201/-74,0.307,19.33,15.87,18.68,3307.58,0.108,0.0,False,CPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1445.0,+344/-241,0.382,21.0,9.92,22.477,9411.86,433.416,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1433.0,+268/-147,0.283,21.67,15.18,21.223,5568.31,1.781,0.0,False,GPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1418.0,+175/-44,0.219,22.5,21.18,19.816,2029.77,1.073,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1412.0,+142/-39,0.197,22.83,22.15,20.228,2029.77,6.631,0.0,False,CPU +18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1412.0,+262/-84,0.213,22.83,16.26,19.85,1280.01,5.479,0.0,False,CPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1409.0,+244/-141,0.272,23.0,14.96,21.671,5568.31,0.372,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1403.0,+285/-92,0.219,23.33,14.81,19.946,1280.01,1.143,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1398.0,+205/-127,0.219,23.58,19.93,21.447,6829.34,7.818,0.0,False,GPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1394.0,+268/-198,0.246,23.83,18.35,20.929,2059.45,3.769,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1379.0,+437/-228,0.297,24.67,6.92,22.95,13.87,2.152,0.0,False,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1375.0,+264/-171,0.169,24.92,20.15,21.793,6829.34,0.53,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1356.0,+255/-176,0.229,26.0,13.94,24.08,17.09,0.145,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1353.0,+340/-277,0.32,26.17,4.97,23.608,9411.86,42.924,0.0,False,GPU +27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1319.0,+250/-164,0.191,28.17,20.83,23.403,410.08,4.104,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1302.0,+373/-354,0.223,29.17,18.56,26.238,2721.87,14.595,0.0,False,CPU +29,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1296.0,+161/-125,0.109,29.5,26.85,22.348,155.57,0.254,0.0,False,CPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1278.0,+292/-226,0.145,30.5,21.21,22.852,2059.45,0.127,0.0,False,GPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1270.0,+202/-155,0.115,31.0,23.64,25.058,27108.48,0.742,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1264.0,+222/-200,0.119,31.33,27.49,25.243,4370.77,2.162,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1246.0,+253/-279,0.12,32.33,25.27,26.005,728.32,2.548,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1235.0,+397/-370,0.284,33.0,8.74,27.727,42.75,45.46,0.0,False,GPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1217.0,+342/-353,0.148,34.0,18.85,25.893,729.17,2.154,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1211.0,+276/-254,0.102,34.33,25.35,25.765,728.32,0.356,0.0,False,CPU +37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1190.0,+175/-206,0.075,35.5,30.79,26.917,27108.48,0.066,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1190.0,+176/-212,0.088,35.5,30.85,25.981,19.2,6.922,0.0,False,GPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1181.0,+89/-95,0.0,36.0,35.1,24.783,4.93,0.671,0.0,False,CPU +40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1181.0,+170/-206,0.035,36.0,34.18,27.166,11.78,0.099,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1159.0,+197/-173,0.06,37.17,32.88,26.155,5.12,0.444,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1156.0,+276/-281,0.113,37.33,27.67,28.736,2721.87,1.334,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1153.0,+228/-281,0.078,37.5,31.64,28.727,8.99,1.498,0.0,False,GPU +44,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1146.0,+149/-137,0.006,37.83,36.17,27.899,4370.77,0.22,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1137.0,+257/-308,0.097,38.33,25.06,27.785,729.17,0.337,0.0,False,CPU +46,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1120.0,+154/-209,0.01,39.17,36.91,29.623,17.24,0.42,0.0,False,GPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+143/-432,0.0,44.67,42.99,37.954,0.74,0.148,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,979.0,+86/-362,0.0,45.5,44.67,38.521,909.67,0.893,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,962.0,+100/-303,0.0,46.17,45.13,39.28,909.67,0.107,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,962.0,+232/-508,0.032,46.17,41.47,34.491,8.37,0.685,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,939.0,+300/-564,0.037,47.0,40.48,41.716,155.52,1.033,0.0,False,CPU +52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,934.0,+277/-430,0.004,47.17,43.75,39.387,0.76,0.153,0.0,False,CPU +53,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,924.0,+96/-253,0.0,47.5,47.01,35.184,14.57,0.363,0.0,False,CPU +54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,748.0,+71/-303,0.0,52.17,51.98,44.164,4.67,0.855,0.0,False,GPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,723.0,+130/-462,0.0,52.67,52.34,44.545,4.62,0.135,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,372.0,+132/-1205,0.0,56.33,56.29,55.52,155.52,0.12,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-548.0,+343/-1220,0.0,58.0,58.0,70.032,0.52,0.058,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4195401ed2e3c411095ea85f129b36da9c298469 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9cc20b46186e388ec4f8e64098422e8c9a5863198eedea5cf9c6c3cfa75da69 +size 1681180 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/n_datasets_13 b/data/imputation_no/splits_lite/tasks_regression/datasets_all/n_datasets_13 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..8356129e13c41110fb5ab63217afee34c710a881 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20388c0baf853058e971237856e1383b9ac660890ac805957c3626bc24ea2fa3 +size 420357 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..f234b68253948c7b3193bf93f67fd5160ebf8b92 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3432fa60b73e63af1eb071844a8b53cfc7975d8ec716677ea9ae2ff31c22c397 +size 969672 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..da4d17838d0f838724630e575104e792d27a6fba --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10a7ddda84904aafba026cdc42b8a620d4ee2389d91a254598aee95555eaf54c +size 120084 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..0f8135132c4febae355137c293131396a8d62555 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv @@ -0,0 +1,52 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1932.0,+314/-174,0.799,5.15,2.41,1.161,2043.44,8.369,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1857.0,+142/-62,0.647,6.85,3.93,2.562,616.43,4.041,0.0,False,GPU +2,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1854.0,+258/-95,0.729,6.92,2.28,2.472,7125.58,336.907,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1819.0,+190/-85,0.677,7.85,3.79,2.973,7125.58,38.573,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1808.0,+228/-164,0.673,8.15,3.27,2.101,2043.44,0.781,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1760.0,+101/-62,0.537,9.62,5.51,4.699,1489.94,5.79,0.0,False,CPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1755.0,+118/-66,0.571,9.77,8.06,4.174,4057.12,10.478,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1692.0,+182/-93,0.541,11.85,6.26,4.157,44.65,38.298,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1690.0,+216/-178,0.532,11.92,6.71,3.576,6.94,0.519,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1663.0,+157/-141,0.505,12.92,5.46,6.669,3811.43,7.581,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1616.0,+147/-131,0.433,14.69,10.42,5.331,4057.12,0.337,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1611.0,+92/-59,0.328,14.92,12.75,7.3,686.46,5.477,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1582.0,+93/-70,0.29,16.08,12.31,7.083,2895.38,1.317,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1564.0,+127/-84,0.358,16.85,13.31,6.477,708.95,1.973,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1551.0,+132/-103,0.311,17.38,13.53,5.748,4228.53,1.188,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1542.0,+123/-97,0.271,17.77,9.88,7.207,2895.38,0.073,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1516.0,+104/-90,0.23,18.85,15.39,8.098,686.46,0.743,0.0,False,CPU +17,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1512.0,+232/-213,0.404,19.0,4.97,13.045,5.49,0.754,0.0,False,GPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1503.0,+118/-74,0.257,19.38,16.18,7.042,708.95,0.069,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1472.0,+71/-57,0.174,20.69,19.28,8.021,848.99,0.475,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1472.0,+60/-49,0.173,20.69,19.7,8.036,848.99,2.378,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1460.0,+109/-92,0.202,21.23,18.6,9.327,16.07,0.29,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1444.0,+134/-95,0.191,21.92,18.71,6.582,4228.53,0.125,0.0,False,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1434.0,+150/-147,0.259,22.38,13.93,9.198,3811.43,0.452,0.0,False,GPU +24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1430.0,+117/-136,0.173,22.54,18.16,9.171,8.35,0.087,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1357.0,+136/-132,0.129,25.77,22.76,8.245,13.9,0.123,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1309.0,+141/-114,0.087,27.85,23.61,10.088,9.17,1.416,0.0,False,GPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1296.0,+108/-102,0.078,28.38,25.93,9.416,4452.11,0.852,0.0,False,CPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1295.0,+134/-114,0.093,28.46,23.52,12.273,161.73,0.784,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1271.0,+135/-118,0.105,29.46,23.46,12.586,161.73,0.12,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1241.0,+130/-136,0.064,30.69,27.38,10.034,4452.11,0.089,0.0,False,CPU +31,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1241.0,+179/-168,0.076,30.69,14.6,12.917,2.47,0.755,0.0,False,GPU +32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1224.0,+76/-101,0.013,31.38,30.46,10.917,2.03,0.296,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1214.0,+159/-194,0.12,31.77,21.26,13.421,3017.28,0.248,0.0,False,CPU +34,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1191.0,+97/-118,0.053,32.69,30.4,11.634,2.15,0.18,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1182.0,+83/-122,0.018,33.08,31.12,13.249,526.17,0.77,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1166.0,+155/-206,0.119,33.69,15.28,13.982,3017.28,0.027,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1138.0,+95/-134,0.016,34.73,32.85,14.003,526.17,0.119,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1119.0,+104/-116,0.014,35.46,33.28,14.183,0.42,0.061,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1084.0,+75/-108,0.0,36.69,35.72,14.282,527.21,2.831,0.0,False,CPU +40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1039.0,+118/-167,0.049,38.15,35.84,15.082,10.04,0.03,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1037.0,+92/-138,0.0,38.23,36.79,14.125,20.5,0.081,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1034.0,+97/-114,0.0,38.31,37.3,14.768,527.21,0.311,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+73/-90,0.0,39.35,38.95,15.317,0.63,0.064,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,887.0,+94/-166,0.0,42.31,41.7,18.821,3.08,0.293,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,877.0,+134/-183,0.0,42.54,41.76,20.298,90.87,0.906,0.0,False,CPU +46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,713.0,+109/-192,0.0,45.54,45.39,22.447,90.87,0.055,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,572.0,+205/-344,0.0,47.31,46.87,29.125,0.25,0.044,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,439.0,+113/-319,0.0,48.58,48.52,36.211,177.04,0.14,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,351.0,+147/-403,0.0,49.27,49.19,36.396,177.04,0.059,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,199.0,+138/-328,0.0,50.23,50.22,38.856,0.88,0.109,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3605d8815e1f3db9d61adea12b7ff4158061bf12 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08be8afdd0f0e20bbeaa8eebb468ab0b46eb4440077e325ddc238397dd7422b8 +size 1471479 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/n_datasets_5 b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/n_datasets_5 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c545c74b873eb874e16fc2bcbd62c1825a8bd1ab --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:192d6f20c3bba8a089a3a2c83ec318ab168ac24a87e6057400e9b97750ada931 +size 430652 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..50b76ee5db1aed2c310523115b56412d78d347be --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0ed1a7b0aec7f79576fc6e35813f42a3c1c5a2eccf9907f5caa0f571a57e592 +size 984135 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d690b23148f0a4984f305cf26c6420ae62bd70c9 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc182a07eaa944d15afc1708dd15d412fd261840ba3fa70bf62871f5f947ce52 +size 119314 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..dbc54b0b2b4f8c3dd4bc8feaa1c2728697a1133a --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv @@ -0,0 +1,52 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,2061.0,+1014/-71,0.832,4.4,3.24,1.163,739.1,11.65,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,2027.0,+1086/-121,0.835,5.0,2.74,1.262,739.1,0.689,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1950.0,+850/-36,0.699,6.6,5.71,2.284,2552.28,4.522,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1916.0,+658/-8,0.655,7.4,3.4,3.209,474.94,11.837,0.0,False,CPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1885.0,+615/-13,0.612,8.2,6.09,3.513,400.37,2.16,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1870.0,+930/-101,0.637,8.6,6.91,2.851,2068.17,1.355,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1822.0,+1355/-177,0.687,10.0,2.25,2.083,4529.88,357.209,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1790.0,+1289/-174,0.65,11.0,4.14,2.39,4529.88,39.544,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1783.0,+1058/-258,0.548,11.2,6.42,2.625,1.59,0.506,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1688.0,+928/-198,0.485,14.4,10.08,4.112,2068.17,0.068,0.0,False,GPU +10,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1677.0,+780/-243,0.471,14.8,11.42,4.058,2552.28,0.201,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1677.0,+503/-139,0.377,14.8,11.88,5.836,223.05,9.707,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1671.0,+1088/-218,0.526,15.0,6.19,3.203,41.5,38.298,0.0,False,GPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1660.0,+548/-97,0.349,15.4,12.03,6.007,223.05,0.793,0.0,False,CPU +14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1649.0,+524/-91,0.418,15.8,13.49,4.481,438.55,0.52,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1644.0,+510/-102,0.3,16.0,10.62,6.235,1696.82,0.538,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1617.0,+754/-226,0.336,17.0,12.43,5.187,3456.17,0.611,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1590.0,+500/-169,0.28,18.0,7.25,6.475,1696.82,0.055,0.0,False,CPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1559.0,+428/-97,0.253,19.2,17.73,5.477,438.55,0.035,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1553.0,+728/-163,0.287,19.4,16.95,5.3,6.87,0.098,0.0,False,GPU +20,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1537.0,+750/-208,0.306,20.0,4.21,4.7,2.07,0.754,0.0,False,GPU +21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1521.0,+561/-275,0.27,20.6,16.66,7.061,4.99,0.047,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1505.0,+528/-187,0.183,21.2,19.63,6.188,3456.17,0.037,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1494.0,+356/-58,0.189,21.6,20.89,6.853,289.35,1.798,0.0,False,CPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1484.0,+386/-70,0.181,22.0,21.09,6.921,289.35,0.36,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1435.0,+648/-216,0.129,23.8,21.42,7.402,5.37,0.025,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1353.0,+354/-228,0.137,26.8,24.63,8.025,0.57,0.074,0.0,False,CPU +27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1302.0,+357/-139,0.002,28.6,27.72,8.909,2.32,0.093,0.0,False,GPU +28,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1297.0,+375/-186,0.018,28.8,28.08,8.225,0.66,0.213,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1274.0,+225/-245,0.064,29.6,27.65,8.161,4108.24,0.479,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1250.0,+508/-284,0.028,30.4,28.51,9.603,6.54,0.284,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1221.0,+204/-296,0.029,31.4,29.47,9.253,4108.24,0.037,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1172.0,+353/-451,0.023,33.0,31.22,10.288,35.47,0.502,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1147.0,+417/-890,0.144,33.8,25.8,13.625,1820.58,0.133,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1140.0,+314/-411,0.006,34.0,32.83,10.454,35.47,0.054,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1134.0,+107/-401,0.0,34.2,33.96,10.053,79.96,0.571,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1124.0,+128/-390,0.0,34.5,34.09,10.217,79.96,0.053,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1061.0,+165/-350,0.0,36.4,35.28,13.469,14.05,0.029,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1033.0,+387/-674,0.006,37.2,35.12,12.12,0.18,0.054,0.0,False,CPU +39,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1026.0,+316/-953,0.1,37.4,34.8,14.242,1820.58,0.012,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+78/-318,0.0,38.1,37.96,11.35,0.42,0.053,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,996.0,+363/-1057,0.12,38.2,34.58,14.681,3.47,0.01,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,871.0,+51/-752,0.0,41.2,40.95,14.269,317.48,0.719,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,842.0,+43/-769,0.0,41.8,41.54,14.827,317.48,0.127,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,800.0,+94/-889,0.0,42.6,42.42,18.666,142.18,1.027,0.0,False,CPU +45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,662.0,+74/-902,0.0,44.8,44.73,21.083,142.18,0.057,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,662.0,+107/-831,0.0,44.8,44.56,18.199,1.33,0.126,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,361.0,+132/-1452,0.0,47.6,47.49,28.294,0.06,0.036,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,145.0,+92/-1782,0.0,48.8,48.8,39.482,48.35,0.054,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-119.0,+197/-2129,0.0,50.0,49.99,39.829,48.35,0.013,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-276.0,+185/-2881,0.0,50.6,50.6,42.789,0.13,0.02,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e4459272d5fce82cd0bb6ce910b37184b692a967 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12f84f0e255219eee537db0bbdfacfbc33d4ea9a62c5bdbd4fb9ae959e328421 +size 1244143 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/n_datasets_8 b/data/imputation_no/splits_lite/tasks_regression/datasets_small/n_datasets_8 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..392f7e5eede4ec3a476d1160ddaa4cc4feb524d2 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afe4cf82d363d970a15d4718bb0f6eff0aba9ae74c699faa13cd6c4c8eb0b2a9 +size 423977 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..119b0ed1096e9deb67f08d09d959a1351202434d --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d6d6266d85db43f2f511fa3d2082e6336a9458060c1c3c45dab013c0774083a +size 992662 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4c22de815c91feebb6293253ba476d33c81d8319 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb9ccab10fe6092a3ab6687834dbe6002d793df2b0a8ab4cf8208746f2ca3165 +size 118758 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..c7312197bd2c110c2c16f513bce785df8b4b7818 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv @@ -0,0 +1,52 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1952.0,+273/-132,0.755,5.0,2.3,2.715,9830.47,232.266,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1921.0,+505/-228,0.779,5.62,2.08,1.16,2355.62,7.55,0.0,False,GPU +2,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1910.0,+208/-53,0.694,5.88,3.59,3.338,9830.47,22.221,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1904.0,+318/-109,0.67,6.0,3.22,1.968,716.17,4.181,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1759.0,+168/-71,0.55,9.88,6.3,4.753,51.27,43.085,0.0,False,GPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1751.0,+359/-200,0.571,10.12,3.72,2.625,2355.62,0.879,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1740.0,+162/-78,0.529,10.5,9.0,5.0,4918.07,35.607,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1725.0,+173/-65,0.463,11.0,8.97,5.631,4919.72,5.372,0.0,False,CPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1686.0,+347/-216,0.521,12.38,6.9,4.17,8.24,0.532,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1619.0,+208/-123,0.4,14.88,10.65,6.092,4918.07,0.949,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1616.0,+120/-74,0.297,15.0,13.37,8.215,1206.7,4.573,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1588.0,+151/-78,0.285,16.12,13.67,7.613,3495.99,1.462,0.0,False,CPU +12,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1569.0,+199/-136,0.384,16.88,5.32,9.41,9006.35,8.016,0.0,False,GPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1554.0,+166/-105,0.32,17.5,13.2,7.724,1672.82,2.447,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1551.0,+231/-103,0.265,17.62,12.79,7.664,3495.99,0.148,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1551.0,+184/-110,0.295,17.62,14.33,6.098,4322.16,1.598,0.0,False,GPU +16,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1534.0,+445/-282,0.466,18.38,5.6,18.26,5.86,0.71,0.0,False,GPU +17,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1509.0,+143/-86,0.259,19.5,15.34,8.02,1672.82,0.097,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1500.0,+108/-62,0.169,19.88,18.29,8.708,1277.27,0.648,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1495.0,+102/-52,0.163,20.12,19.02,8.776,1277.27,3.746,0.0,False,CPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1476.0,+129/-93,0.156,21.0,18.63,9.405,1206.7,0.689,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1446.0,+144/-126,0.149,22.38,19.81,11.844,25.13,0.294,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1446.0,+182/-115,0.196,22.38,18.18,6.829,4322.16,0.198,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1415.0,+158/-132,0.112,23.75,19.24,10.49,10.03,0.115,0.0,False,CPU +24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1372.0,+166/-175,0.136,25.62,20.38,13.514,644.11,0.953,0.0,False,CPU +25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1358.0,+178/-127,0.124,26.25,21.31,10.391,22.41,7.82,0.0,False,GPU +26,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1350.0,+189/-198,0.167,26.62,19.91,13.918,644.11,0.187,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1342.0,+194/-144,0.13,27.0,23.69,8.773,25.53,0.154,0.0,False,GPU +28,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1339.0,+195/-178,0.127,27.12,16.15,12.41,9006.35,0.527,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1328.0,+144/-144,0.087,27.62,24.95,10.201,5194.73,1.75,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1269.0,+207/-194,0.086,30.25,26.22,10.523,5194.73,0.121,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1263.0,+232/-223,0.105,30.5,19.16,13.293,3991.28,0.402,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1243.0,+248/-243,0.131,31.38,11.32,13.82,3991.28,0.034,0.0,False,CPU +33,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1228.0,+219/-289,0.121,32.0,11.27,15.422,7.0,1.362,0.0,False,GPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1219.0,+112/-184,0.029,32.38,29.57,15.247,657.99,1.135,0.0,False,CPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1204.0,+91/-98,0.009,33.0,32.16,12.599,4.54,0.374,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1182.0,+69/-127,0.0,33.88,33.07,14.291,2150.84,7.995,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1169.0,+101/-179,0.019,34.38,32.23,15.473,0.46,0.117,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1155.0,+114/-244,0.027,34.88,32.12,16.37,657.99,0.147,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1121.0,+95/-151,0.0,36.12,35.07,14.731,2150.84,0.566,0.0,False,CPU +40,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1114.0,+92/-103,0.0,36.38,35.62,13.89,4.08,0.255,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1064.0,+125/-209,0.005,38.12,36.68,15.332,12.71,0.039,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1025.0,+148/-218,0.0,39.38,37.8,14.535,22.25,0.131,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+88/-203,0.0,40.12,39.6,17.797,0.69,0.125,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,978.0,+89/-211,0.0,40.75,40.09,19.209,6.77,0.511,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,909.0,+148/-323,0.0,42.5,41.35,21.318,88.79,0.572,0.0,False,CPU +46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,724.0,+133/-310,0.0,46.0,45.81,23.299,88.79,0.052,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,642.0,+210/-603,0.0,47.12,46.49,29.645,0.43,0.047,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,525.0,+94/-492,0.0,48.44,48.34,34.167,742.99,0.364,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,485.0,+125/-572,0.0,48.81,48.71,34.251,742.99,0.097,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,334.0,+100/-501,0.0,50.0,49.99,36.397,3.98,0.12,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c000af8ee0fa595c822a87dd5b914f44ac50e4c1 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:760a144dc636ad9cbedacadb367fc542d053b2d3aabf5db118c774165a9d7d46 +size 1315140 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/n_datasets_7 b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/n_datasets_7 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..658b4d07378d5e822279a6541d3551fa7773f26e --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76f2750091c95fe15aa2a30e669a674d3b1d75c52e0f3398e4f07c0f900e083b +size 451328 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..6b3aa5e3350e472ba0452b48be92f3bbf19ace3b --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eae04c352a81993c5962bf6f2f5ab3cdc6b8c4a9dad5387f7ecae7050753c4f +size 1044019 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..b1d2618be1823f634605470d59c87d2937ef924d --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fdc6da47a0c36cda8163cf28974b2b607495d7413170b6bc7ed042598a0f372 +size 120889 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..a69a092ddc81da31abce4a88c97bc436b3de2008 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,56 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1906.0,+304/-113,0.726,6.43,2.55,3.095,10054.02,249.652,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1900.0,+513/-229,0.772,6.57,2.05,1.229,2279.61,6.732,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1884.0,+400/-86,0.683,7.0,3.48,1.908,669.82,4.041,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1873.0,+180/-50,0.671,7.29,4.98,3.69,10054.02,28.583,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1780.0,+499/-189,0.597,10.14,3.32,2.584,6601.21,73.561,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1772.0,+224/-44,0.565,10.43,6.19,4.944,47.81,37.998,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1756.0,+146/-30,0.551,11.0,9.95,5.167,4840.44,33.436,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1729.0,+403/-218,0.545,12.0,3.79,2.847,2279.61,0.781,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1714.0,+373/-213,0.554,12.57,7.37,4.191,7.82,0.519,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1693.0,+192/-49,0.444,13.43,10.94,6.056,5390.56,5.054,0.0,False,CPU +10,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1658.0,+220/-118,0.398,14.86,7.71,6.637,105.28,2.433,0.0,False,GPU +11,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1624.0,+410/-299,0.518,16.29,5.15,19.429,5.73,0.638,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1605.0,+159/-48,0.287,17.14,15.34,7.956,3301.07,1.317,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1605.0,+257/-127,0.39,17.14,12.18,6.481,4840.44,0.706,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1574.0,+114/-57,0.246,18.57,17.2,9.035,997.12,3.787,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1568.0,+226/-106,0.266,18.86,14.48,8.008,3301.07,0.125,0.0,False,CPU +16,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1558.0,+418/-227,0.406,19.29,10.76,5.384,6601.21,0.588,0.0,False,GPU +17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1544.0,+165/-105,0.295,20.0,14.25,8.299,1709.03,2.724,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1529.0,+188/-118,0.267,20.71,17.7,6.513,4228.53,1.447,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1515.0,+180/-138,0.267,21.43,15.3,10.909,9360.97,7.658,0.0,False,GPU +20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1512.0,+142/-77,0.251,21.57,16.24,8.482,1709.03,0.097,0.0,False,GPU +21,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1469.0,+405/-336,0.352,23.71,13.17,7.531,8.97,0.837,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1466.0,+83/-44,0.098,23.86,23.13,9.543,911.22,0.527,0.0,False,CPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1463.0,+78/-38,0.1,24.0,23.39,9.588,911.22,3.532,0.0,False,CPU +24,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1434.0,+126/-68,0.105,25.43,23.65,10.356,997.12,0.636,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1425.0,+196/-137,0.16,25.86,21.99,7.312,4228.53,0.173,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1422.0,+220/-139,0.117,26.0,21.17,11.174,9.39,0.105,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1407.0,+139/-152,0.121,26.71,24.25,13.106,27.26,0.29,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1392.0,+209/-236,0.158,27.43,21.57,14.555,613.95,0.952,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1384.0,+236/-250,0.193,27.86,20.58,14.881,613.95,0.135,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1366.0,+155/-135,0.129,28.71,23.97,11.015,22.96,8.006,0.0,False,GPU +31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1325.0,+195/-187,0.114,30.71,27.44,9.406,17.97,0.145,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1290.0,+275/-208,0.111,32.43,20.76,14.054,3778.28,0.323,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1282.0,+124/-160,0.026,32.86,31.52,11.205,5937.36,1.406,0.0,False,CPU +34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1276.0,+260/-286,0.138,33.14,10.38,16.185,7.12,1.138,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1272.0,+275/-228,0.14,33.29,14.11,14.636,3778.28,0.028,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1272.0,+146/-174,0.037,33.29,30.98,14.048,9360.97,0.522,0.0,False,GPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1229.0,+155/-260,0.03,35.29,32.39,16.471,628.51,0.812,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1222.0,+185/-280,0.031,35.57,32.35,11.535,5937.36,0.119,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1175.0,+138/-220,0.02,37.57,35.46,16.642,0.43,0.117,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1173.0,+176/-289,0.029,37.64,34.8,17.646,628.51,0.149,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1172.0,+59/-100,0.0,37.71,37.36,13.734,4.41,0.363,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1150.0,+55/-106,0.0,38.57,38.08,15.611,2163.39,7.622,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1114.0,+109/-169,0.0,40.0,39.21,14.895,3.97,0.24,0.0,False,CPU +44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1095.0,+89/-158,0.0,40.71,39.47,15.956,10.54,0.036,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1095.0,+106/-175,0.0,40.71,39.78,16.062,2163.39,0.713,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1002.0,+135/-301,0.0,43.86,42.36,15.699,20.5,0.13,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+131/-217,0.0,43.93,43.18,19.276,0.67,0.118,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,943.0,+77/-213,0.0,45.57,45.12,21.065,6.88,0.452,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,937.0,+178/-381,0.0,45.71,44.31,22.177,86.71,0.53,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,757.0,+168/-358,0.0,49.57,49.27,24.152,86.71,0.05,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,712.0,+245/-524,0.0,50.29,49.48,30.287,0.4,0.044,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,533.0,+85/-893,0.0,52.5,52.4,36.2,753.48,0.227,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,504.0,+117/-984,0.0,52.79,52.68,36.249,753.48,0.091,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,352.0,+97/-674,0.0,54.0,53.99,37.99,4.35,0.119,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..335f4191942278d85ffd727dfbdc8053305ace32 --- /dev/null +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b9c1483354240a504c08272434a27f08f7f0b455819c9efd8cb70f781c81736 +size 1396925 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/n_datasets_51 b/data/imputation_yes/splits_all/tasks_all/datasets_all/n_datasets_51 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4d93e73eb79a44adc38730e690a696d5c9c3c713 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:982bafdb0a62d7e0fc6a59d5ba4026375aaad014b56e5c39cae1285276f16fb3 +size 496056 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a95fa2732be5193eef2338f814be87c4c9a0cf7d --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6b35e58032a829038a7005caace50d3d0481328787ae5b37d20e3d5ef993660 +size 1083898 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..880f47d6d3445bdd99a64035889ad0a40c7d6fd5 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6643ee5c7740c444bfd741f033411ba831e09ea49b60c8f84dbb2f43ce36ae7f +size 177418 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_all/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..f159f35ad3210aaa1f9d4e8a6baf0c56b642d441 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1636.0,+60/-48,0.753,8.48,3.84,5.839,582.21,6.116,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1599.0,+78/-57,0.716,9.88,3.37,5.088,2040.22,8.92,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1554.0,+67/-50,0.658,11.83,4.65,6.062,2040.22,1.218,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1523.0,+66/-46,0.628,13.31,6.19,6.766,5.81,0.645,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1499.0,+56/-46,0.522,14.53,8.14,9.055,2950.72,11.988,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1499.0,+55/-47,0.517,14.56,5.99,9.005,1735.72,2.559,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1451.0,+62/-49,0.496,17.21,5.14,9.138,4907.64,286.652,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1435.0,+50/-38,0.399,18.15,9.57,10.535,3285.87,1.472,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1419.0,+32/-29,0.35,19.13,12.78,11.754,416.98,2.639,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1416.0,+47/-46,0.365,19.32,10.24,10.535,2950.72,0.663,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1403.0,+42/-39,0.334,20.12,11.5,11.264,1658.41,0.653,0.0,False,CPU +11,🧠🔁,Neural Network,[LimiX (default) [11.98% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1402.0,+67/-55,0.464,20.2,6.37,12.97,3.68,0.598,11.98,True,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1396.0,+57/-53,0.402,20.54,7.13,10.533,4907.64,39.96,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1392.0,+40/-41,0.324,20.8,10.82,11.447,1658.41,0.081,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1382.0,+75/-52,0.371,21.43,8.49,11.746,4621.67,8.149,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1379.0,+48/-44,0.316,21.65,11.04,11.448,3285.87,0.173,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1374.0,+31/-29,0.265,21.92,15.54,12.443,416.98,0.334,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1364.0,+33/-38,0.28,22.57,14.36,12.518,693.49,1.69,0.0,False,CPU +18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1354.0,+38/-43,0.29,23.24,12.84,12.323,6.83,0.08,0.0,False,CPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1348.0,+40/-37,0.245,23.61,12.01,12.287,4621.67,0.475,0.0,False,GPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1343.0,+51/-39,0.253,23.97,14.02,12.886,846.88,2.552,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1342.0,+32/-34,0.23,24.04,16.93,12.838,693.49,0.308,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1338.0,+65/-65,0.382,24.27,8.06,12.658,3030.15,21.444,35.29,True,GPU +23,🧠🔁,Neural Network,[Mitra (default) [35.29% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1326.0,+62/-62,0.36,25.11,8.78,13.408,87.65,2.505,35.29,True,GPU +24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1320.0,+68/-55,0.326,25.48,8.32,13.232,47.62,43.739,0.0,False,GPU +25,🧠⚡,Foundation Model,[TabICL (default) [29.41% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1317.0,+51/-57,0.318,25.66,9.42,12.932,6.63,1.48,29.41,True,GPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1295.0,+42/-37,0.188,27.14,16.97,14.176,846.88,0.13,0.0,False,GPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1287.0,+47/-44,0.219,27.69,16.45,13.952,10.49,0.132,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1281.0,+62/-58,0.285,28.06,9.86,14.316,3030.15,0.462,35.29,True,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1280.0,+47/-45,0.153,28.17,19.25,13.735,2874.67,1.952,0.0,False,CPU +30,🧠⚡,Foundation Model,[BetaTabPFN (default) [25.49% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1274.0,+56/-50,0.232,28.59,15.28,15.319,205.87,1.343,25.49,True,GPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1259.0,+40/-38,0.149,29.58,19.57,15.485,2929.85,0.422,0.0,False,CPU +32,🧠⚡,Foundation Model,[TabPFNv2 (default) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1254.0,+66/-61,0.259,29.86,13.15,15.267,3.36,0.313,35.29,True,GPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1246.0,+39/-40,0.122,30.46,19.1,15.853,14.87,0.308,0.0,False,GPU +34,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1226.0,+35/-38,0.082,31.77,22.77,15.245,10.06,1.689,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1223.0,+44/-45,0.106,31.95,20.88,16.161,2929.85,0.051,0.0,False,CPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1210.0,+38/-40,0.091,32.81,24.48,15.708,1.94,0.123,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1205.0,+43/-44,0.08,33.15,23.23,15.593,2874.67,0.131,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1205.0,+42/-41,0.094,33.16,22.18,17.165,183.02,0.761,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1202.0,+48/-50,0.12,33.37,20.04,16.636,593.24,4.467,0.0,False,CPU +40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1192.0,+44/-47,0.097,34.0,22.56,17.32,7.32,0.05,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1183.0,+31/-30,0.058,34.57,29.77,16.331,1.96,0.142,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1177.0,+50/-49,0.074,34.94,22.57,18.028,373.18,0.771,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1168.0,+43/-46,0.07,35.51,22.57,18.131,183.02,0.091,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1141.0,+48/-52,0.05,37.19,24.44,18.128,593.24,0.306,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1138.0,+49/-40,0.049,37.41,22.15,18.798,373.18,0.085,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1072.0,+36/-46,0.009,41.25,33.48,19.966,9.99,0.126,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1039.0,+59/-64,0.034,42.99,34.35,23.528,3.23,0.919,0.0,False,GPU +48,🧠⚡,Foundation Model,[TabFlex (default) [25.49% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,1008.0,+61/-75,0.045,44.54,31.75,25.129,0.79,0.123,25.49,True,GPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1001.0,+53/-65,0.009,44.88,33.0,22.986,2.86,0.373,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+44/-40,0.004,44.93,40.01,23.738,0.43,0.054,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,993.0,+62/-80,0.018,45.25,31.89,25.602,129.01,1.802,0.0,False,CPU +52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,978.0,+58/-62,0.011,45.94,40.23,25.248,0.25,0.05,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,961.0,+64/-98,0.031,46.69,35.2,31.513,237.58,0.424,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,933.0,+72/-114,0.02,47.87,37.08,32.121,237.58,0.085,0.0,False,CPU +55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,882.0,+62/-98,0.007,49.77,38.19,30.831,129.01,0.18,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,851.0,+70/-120,0.012,50.8,40.6,34.849,1.19,0.12,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,627.0,+89/-100,0.0,55.71,50.58,44.451,0.19,0.036,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d3671ed2f00756f7e528ecf1ff3d6fc1773b3065 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a83087f41cd3f2d63c07da50e88a4a1f2bb72da876688260a7d5b520458be1e7 +size 2772353 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/n_datasets_15 b/data/imputation_yes/splits_all/tasks_all/datasets_medium/n_datasets_15 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..0d152925db546fc2930aa020788ca71dafef3758 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8e1d1ef26fb63a3b6e6f8091565858b8c8022e852a8f37545521d1399e8fe83 +size 450547 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..f1896523e83a58c6c0b71bb8d74323dc78f48ae3 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d4de3950bd3972519bdf86de3563d6e37d70a460e34997d95ae9c68b613c21e +size 1020382 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..419fd1b694999f1f9d60061e7a8c38504eeb6465 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33325882d580d36b0dcd96c58e65741298e025ae4644afded040522c1d79cb85 +size 142000 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_all/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..101ae0c2851daed494536c592b2742064b15fae9 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/website_leaderboard.csv @@ -0,0 +1,55 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1809.0,+94/-49,0.822,6.04,2.88,2.62,289.53,3.187,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1784.0,+87/-53,0.795,6.69,3.13,2.723,432.35,4.085,0.0,False,CPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1662.0,+163/-119,0.649,10.68,3.35,5.438,735.58,11.736,0.0,False,GPU +3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1632.0,+104/-90,0.576,11.83,7.36,5.74,1719.82,1.675,0.0,False,GPU +4,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1620.0,+120/-113,0.593,12.33,6.83,5.81,2526.28,6.013,0.0,False,GPU +5,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1607.0,+90/-64,0.536,12.89,6.9,5.642,777.59,0.25,0.0,False,CPU +6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1606.0,+146/-108,0.582,12.92,5.31,6.152,735.58,1.386,0.0,False,GPU +7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1598.0,+97/-65,0.538,13.3,5.73,5.838,777.59,0.052,0.0,False,CPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1590.0,+75/-59,0.488,13.64,9.18,7.405,131.56,2.639,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1557.0,+110/-95,0.521,15.15,7.44,6.028,3.24,0.031,0.0,False,CPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1549.0,+87/-56,0.442,15.51,9.32,7.531,282.13,0.563,0.0,False,CPU +11,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1536.0,+146/-95,0.471,16.17,7.91,6.964,1.88,0.645,0.0,False,GPU +12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1533.0,+114/-88,0.406,16.29,9.19,7.07,1719.82,0.081,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1531.0,+149/-89,0.424,16.41,7.54,7.466,1993.14,0.619,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1519.0,+130/-121,0.482,16.97,4.24,7.078,4786.55,444.544,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1502.0,+78/-60,0.346,17.84,12.32,8.625,131.56,0.132,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1501.0,+70/-49,0.359,17.86,13.6,8.115,282.13,0.066,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1479.0,+145/-95,0.358,18.99,8.67,8.222,1993.14,0.061,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1473.0,+104/-87,0.33,19.32,13.86,8.204,2526.28,0.349,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1427.0,+112/-93,0.282,21.7,13.1,9.726,566.74,0.951,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1397.0,+123/-120,0.296,23.29,8.88,9.704,4786.55,42.641,0.0,False,GPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1375.0,+70/-65,0.177,24.49,17.88,9.473,1770.56,1.069,0.0,False,CPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1374.0,+96/-92,0.206,24.56,19.3,11.424,566.74,0.06,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1366.0,+101/-111,0.228,24.99,16.75,9.978,0.49,0.053,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabICL (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1341.0,+131/-109,0.276,26.37,12.29,10.315,3.97,1.596,33.33,True,GPU +25,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1328.0,+120/-136,0.225,27.08,10.23,12.499,46.62,43.739,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1322.0,+132/-105,0.181,27.38,17.94,11.062,5.16,0.074,0.0,False,GPU +27,🧠🔁,Neural Network,[LimiX (default) [40.00% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1322.0,+138/-128,0.25,27.39,8.86,14.019,1.08,0.32,40.0,True,GPU +28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1320.0,+118/-126,0.193,27.51,18.45,12.974,1517.65,0.134,0.0,False,CPU +29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1296.0,+76/-79,0.111,28.81,24.71,11.512,8.06,0.25,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1274.0,+85/-73,0.05,29.99,26.82,12.26,6.75,0.235,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1269.0,+78/-77,0.064,30.26,25.57,11.256,1770.56,0.051,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1259.0,+112/-138,0.136,30.76,23.35,13.864,1517.65,0.015,0.0,False,CPU +33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1257.0,+57/-67,0.045,30.9,28.62,11.913,0.29,0.045,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1249.0,+82/-72,0.041,31.29,25.01,12.464,66.0,0.341,0.0,False,CPU +35,🧠⚡,Foundation Model,[BetaTabPFN (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1225.0,+107/-93,0.081,32.56,25.25,14.429,66.06,0.652,33.33,True,GPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1213.0,+90/-83,0.042,33.14,26.81,13.175,66.0,0.024,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1212.0,+101/-142,0.096,33.22,28.28,15.447,3.12,0.016,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1208.0,+70/-83,0.025,33.41,30.98,12.591,36.48,0.329,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1200.0,+110/-138,0.075,33.83,26.33,14.414,351.94,1.658,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1163.0,+61/-79,0.011,35.65,33.8,13.815,36.48,0.037,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1118.0,+69/-72,0.001,37.78,36.08,15.577,2.18,0.039,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1093.0,+87/-116,0.022,38.9,34.23,16.687,351.94,0.114,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1049.0,+139/-182,0.02,40.74,35.28,23.725,2.49,0.145,0.0,False,GPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1014.0,+135/-175,0.026,42.13,37.5,20.379,1.51,0.111,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+58/-88,0.0,42.67,41.79,18.741,0.2,0.021,0.0,False,CPU +46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,989.0,+88/-119,0.006,43.08,36.37,22.293,150.3,10.071,0.0,False,CPU +47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,948.0,+151/-218,0.005,44.5,40.65,31.147,43.64,0.081,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,945.0,+111/-138,0.003,44.62,42.76,21.857,0.11,0.028,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default) [33.33% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,926.0,+72/-121,0.0,45.24,44.31,25.797,0.25,0.045,33.33,True,GPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,899.0,+155/-243,0.003,46.06,43.0,32.089,43.64,0.018,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,804.0,+62/-101,0.0,48.55,48.25,28.282,150.3,0.482,0.0,False,CPU +52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,787.0,+131/-231,0.0,48.93,47.58,34.193,0.19,0.019,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,560.0,+100/-154,0.0,52.4,52.31,42.236,0.1,0.072,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1eb665bf59487259ef7bbc7f6874c0ac35a32ff4 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e02322996075c246616c322456716eb4a3afbafd841188c04023641dbb4204b1 +size 2309578 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/n_datasets_36 b/data/imputation_yes/splits_all/tasks_all/datasets_small/n_datasets_36 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..222f946d89c203c74b5d408ad24e453330126aac --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ec2b217756c92ca695ebbaaf7715d86fd41c5b3713ceaa55f6948eb9420eaf6 +size 464248 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c12f78c5f2026f234a8236bc5ac70816afe237bc --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc17aa3a51db65b96144eba85fff725a3d07d62e55b6ba8bb8eeb8d91608d3d6 +size 1073532 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..aac6bef2da2044c94f7aef7873d6b1a9d0c3d041 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0862ad8e7e235e7e06d99016fd865cf0926b8d46b0b44e074d6ab4bc721cffec +size 193409 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_all/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..4acb141c6eaaa174f17ddc5eebabb529c726a2e8 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1612.0,+63/-54,0.715,9.5,4.46,7.181,659.63,7.444,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1610.0,+87/-62,0.738,9.55,3.38,4.943,2289.05,8.049,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1567.0,+71/-55,0.681,11.37,4.42,6.024,2289.05,1.143,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1551.0,+81/-54,0.676,12.09,5.68,6.684,6.76,0.636,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1482.0,+56/-48,0.487,15.58,8.52,10.436,3770.75,21.902,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1480.0,+89/-64,0.542,15.73,6.02,10.121,3527.3,48.6,8.33,True,GPU +6,🧠🔁,Neural Network,[LimiX (default) [0.31% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1459.0,+86/-69,0.544,16.89,5.69,12.533,5.08,0.645,0.31,True,GPU +7,🧠🔁,Neural Network,[Mitra (default) [8.33% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1458.0,+68/-56,0.51,16.97,6.59,11.186,126.84,3.094,8.33,True,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1452.0,+80/-59,0.486,17.3,5.65,9.997,5119.36,218.711,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1442.0,+59/-46,0.391,17.84,9.65,11.622,3716.07,2.547,0.0,False,CPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1426.0,+52/-46,0.373,18.86,10.78,11.814,3553.12,1.741,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1418.0,+73/-57,0.435,19.3,6.59,10.878,5119.36,28.351,0.0,False,GPU +12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1399.0,+52/-41,0.331,20.5,10.75,11.979,3770.75,1.778,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1389.0,+77/-62,0.403,21.1,7.45,12.469,3527.3,0.817,8.33,True,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1384.0,+33/-36,0.273,21.41,15.27,13.566,892.41,2.574,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1364.0,+51/-49,0.282,22.68,12.45,12.792,3553.12,0.241,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1358.0,+40/-44,0.23,23.13,15.92,13.606,2476.51,0.814,0.0,False,CPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1350.0,+34/-30,0.207,23.6,17.44,14.034,892.41,0.346,0.0,False,CPU +18,🧠⚡,Foundation Model,[TabPFNv2 (default) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1349.0,+76/-63,0.367,23.7,10.18,13.819,4.54,0.443,8.33,True,GPU +19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1346.0,+45/-37,0.215,23.91,17.16,13.784,2476.51,0.101,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1337.0,+72/-68,0.35,24.45,7.71,13.538,50.32,43.709,0.0,False,GPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1332.0,+44/-41,0.228,24.79,14.42,14.202,1126.71,2.941,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabICL (default) [27.78% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1328.0,+79/-106,0.327,25.08,8.57,14.023,7.15,1.47,27.78,True,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1326.0,+62/-62,0.266,25.22,9.46,14.22,6521.96,8.396,0.0,False,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1324.0,+41/-41,0.193,25.36,11.38,13.989,6521.96,0.506,0.0,False,GPU +25,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1321.0,+38/-39,0.19,25.51,18.55,14.596,884.18,2.368,0.0,False,CPU +26,🧠⚡,Foundation Model,[BetaTabPFN (default) [22.22% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1308.0,+58/-61,0.286,26.4,13.07,15.69,296.89,1.835,22.22,True,GPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1305.0,+37/-38,0.175,26.6,18.42,14.946,9.64,0.128,0.0,False,CPU +28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1305.0,+36/-40,0.151,26.61,18.85,14.805,884.18,0.393,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1290.0,+54/-48,0.219,27.59,15.87,15.156,13.18,0.17,0.0,False,GPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1284.0,+41/-41,0.164,27.99,16.12,15.323,1126.71,0.163,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1260.0,+42/-48,0.13,29.6,19.86,15.511,3865.95,2.158,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1255.0,+43/-43,0.117,29.96,19.99,16.532,3351.28,0.47,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1238.0,+44/-50,0.109,31.08,17.44,17.662,16.27,0.321,0.0,False,GPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1226.0,+44/-44,0.086,31.89,19.9,17.118,3351.28,0.056,0.0,False,CPU +35,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1221.0,+39/-42,0.082,32.22,21.35,16.489,15.69,4.691,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1216.0,+55/-59,0.134,32.51,18.12,17.561,1678.92,8.069,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1210.0,+65/-64,0.119,32.92,19.81,19.071,499.14,1.466,0.0,False,CPU +38,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1197.0,+48/-53,0.087,33.75,20.7,18.1,9.08,0.057,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1192.0,+43/-48,0.079,34.11,22.31,17.401,3865.95,0.148,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1174.0,+61/-64,0.091,35.22,19.78,19.929,499.14,0.207,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1167.0,+50/-51,0.06,35.7,21.68,18.728,1678.92,0.826,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1162.0,+40/-40,0.042,35.97,30.21,18.171,3.61,0.17,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1162.0,+30/-36,0.019,35.98,30.28,18.096,3.29,0.253,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1156.0,+57/-68,0.086,36.41,21.67,20.347,590.04,1.461,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1114.0,+55/-70,0.049,38.94,20.59,21.141,590.04,0.15,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1057.0,+44/-51,0.01,42.17,32.23,21.794,13.69,0.155,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1041.0,+70/-88,0.032,43.0,33.58,23.446,5.44,1.098,0.0,False,GPU +48,🧠⚡,Foundation Model,[TabFlex (default) [22.22% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,1035.0,+80/-81,0.064,43.32,28.07,24.85,1.58,0.15,22.22,True,GPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+58/-77,0.006,45.05,38.62,25.82,0.51,0.089,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1000.0,+51/-63,0.0,45.05,31.06,24.072,5.48,0.685,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,995.0,+71/-103,0.023,45.29,30.0,26.981,111.84,1.215,0.0,False,CPU +52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,995.0,+71/-73,0.013,45.29,38.52,26.662,0.47,0.092,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,967.0,+84/-94,0.039,46.55,32.92,31.665,520.95,0.647,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,944.0,+82/-100,0.026,47.53,34.6,32.134,520.95,0.113,0.0,False,CPU +55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,907.0,+74/-97,0.009,49.0,34.54,31.894,111.84,0.111,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,866.0,+98/-119,0.017,50.45,37.69,35.122,2.59,0.138,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,654.0,+112/-150,0.0,55.42,48.52,45.374,0.37,0.033,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e6c4037bd2ac41ca1af68e0086b37d916c64b16e --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c735852e7df29e1a141497f6979b5b60056595d6ed85043353ca3c7972118264 +size 2746120 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/n_datasets_33 b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/n_datasets_33 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7b379bf0d52703b2cf8f3000978208fb4fbaafb4 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f23b821023904354c3b2b82f2c3646bb1d36a6d02d839464f02c49eea670d91 +size 464205 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a5e633138abccdc67d7523a35ef1fe6f7c29e20c --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f3140f501f22776903203d8b1b05f4531d477070f85261de6441c97a3265a8f +size 1109426 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..29adf2838e145c9561aac98e7751b64cc787b976 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba620d1a5dd20e832c577c3be996153f35c34f0f676d17e115d80a6806239ce9 +size 145426 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..4ebf7f3adb7cf16d83ba4a97a4e4a8f3e2c782bd --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1632.0,+93/-70,0.74,9.53,3.46,5.292,2166.18,7.945,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1628.0,+76/-55,0.711,9.71,4.55,7.717,611.55,7.01,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1594.0,+66/-62,0.714,11.11,5.37,6.928,6.14,0.586,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1589.0,+71/-61,0.686,11.29,4.35,6.39,2166.18,0.993,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1540.0,+74/-48,0.586,13.62,5.6,9.556,3899.42,55.833,0.0,False,GPU +5,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1511.0,+72/-47,0.551,15.1,6.15,10.725,136.59,3.391,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1505.0,+56/-46,0.499,15.41,8.34,11.115,3547.04,20.963,0.0,False,GPU +7,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1498.0,+74/-70,0.576,15.78,5.57,13.136,4.85,0.629,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1461.0,+74/-61,0.47,17.9,5.77,10.724,4980.83,207.372,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1453.0,+48/-47,0.383,18.36,10.23,12.387,3376.59,2.54,0.0,False,CPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1441.0,+54/-50,0.369,19.08,10.6,12.586,3372.56,1.658,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1436.0,+78/-68,0.435,19.38,6.39,11.521,4980.83,23.583,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1434.0,+82/-68,0.435,19.47,6.97,12.117,3899.42,0.983,0.0,False,GPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1417.0,+57/-46,0.339,20.53,10.64,12.689,3547.04,1.57,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1392.0,+38/-36,0.252,22.1,15.71,14.551,771.57,2.49,0.0,False,CPU +15,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1386.0,+68/-69,0.396,22.44,9.6,13.598,4.22,0.554,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1379.0,+55/-51,0.278,22.92,12.22,13.598,3372.56,0.208,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1377.0,+43/-38,0.237,23.04,15.6,14.476,2034.85,0.805,0.0,False,CPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1363.0,+38/-34,0.219,23.94,17.08,14.675,2034.85,0.097,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabICL (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1363.0,+77/-84,0.352,23.95,8.04,13.82,7.27,1.64,21.21,True,GPU +20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1361.0,+91/-63,0.373,24.06,7.26,14.241,48.9,43.804,0.0,False,GPU +21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1358.0,+34/-30,0.186,24.26,17.76,15.026,771.57,0.322,0.0,False,CPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1338.0,+52/-46,0.208,25.57,15.07,15.196,1105.59,2.552,0.0,False,GPU +23,🧠⚡,Foundation Model,[BetaTabPFN (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1338.0,+64/-77,0.291,25.61,12.57,16.243,303.08,1.792,21.21,True,GPU +24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1334.0,+37/-36,0.182,25.82,19.14,15.598,828.74,2.312,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1334.0,+45/-48,0.172,25.85,12.27,14.832,6147.69,0.485,0.0,False,GPU +26,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1334.0,+73/-66,0.252,25.86,11.65,15.247,6147.69,8.149,0.0,False,GPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1322.0,+48/-44,0.177,26.64,18.03,15.903,8.51,0.123,0.0,False,CPU +28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1316.0,+34/-34,0.142,26.99,19.24,15.81,828.74,0.336,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1309.0,+55/-53,0.22,27.51,15.35,15.975,12.24,0.154,0.0,False,GPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1293.0,+45/-41,0.154,28.52,16.9,16.334,1105.59,0.13,0.0,False,GPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1278.0,+47/-44,0.114,29.53,19.36,17.376,2929.85,0.433,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1271.0,+45/-54,0.122,30.01,21.32,16.4,3704.3,2.074,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1266.0,+46/-41,0.117,30.36,16.68,17.996,16.16,0.307,0.0,False,GPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1248.0,+51/-48,0.081,31.51,19.23,17.977,2929.85,0.054,0.0,False,CPU +35,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1243.0,+40/-38,0.081,31.85,20.77,17.096,15.48,4.535,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1234.0,+51/-60,0.137,32.42,17.55,18.45,1459.62,8.056,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1216.0,+49/-51,0.072,33.57,20.75,18.863,8.47,0.054,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1211.0,+58/-55,0.1,33.91,22.7,20.343,416.39,1.393,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1201.0,+42/-46,0.065,34.56,22.3,18.313,3704.3,0.143,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1180.0,+54/-59,0.06,35.83,21.57,19.638,1459.62,0.89,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1177.0,+57/-56,0.069,36.06,23.58,21.176,416.39,0.179,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1172.0,+34/-37,0.015,36.37,31.15,19.218,3.05,0.241,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1158.0,+35/-41,0.021,37.24,32.62,19.446,3.39,0.158,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1143.0,+54/-53,0.051,38.11,26.04,21.823,572.67,1.421,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1103.0,+45/-45,0.018,40.42,26.25,22.6,572.67,0.143,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1067.0,+50/-59,0.011,42.4,31.83,22.778,11.82,0.147,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default) [21.21% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,1056.0,+82/-88,0.069,43.0,27.1,25.63,1.19,0.15,21.21,True,GPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1054.0,+70/-86,0.035,43.09,33.03,24.777,5.39,1.034,0.0,False,GPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1009.0,+51/-66,0.0,45.3,30.44,24.915,5.18,0.652,0.0,False,CPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1006.0,+82/-85,0.014,45.46,38.22,27.446,0.42,0.078,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1000.0,+72/-88,0.014,45.71,33.26,28.256,100.58,1.106,0.0,False,CPU +52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+53/-69,0.002,45.73,39.57,26.69,0.47,0.069,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1000.0,+71/-104,0.043,45.75,31.75,31.374,504.17,0.629,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,977.0,+72/-104,0.028,46.76,33.44,31.806,504.17,0.103,0.0,False,CPU +55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,904.0,+78/-101,0.0,49.65,35.57,33.41,100.58,0.101,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,899.0,+80/-125,0.018,49.8,36.54,34.29,2.51,0.137,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,677.0,+96/-161,0.0,55.29,47.88,45.738,0.29,0.028,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c6e00583dfd438eb74da387bc25b5e147919370d --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f59494c728d28acb933accda1130a6b956b207a31cfef8a288dd026817f0250b +size 2779156 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/n_datasets_30 b/data/imputation_yes/splits_all/tasks_binary/datasets_all/n_datasets_30 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4ea30a35f86dcf66d868b7729478dc04cae1b258 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a12cee638f020d19b1326743905f4ba5529c3a8d480d72b1037f9c5a532f2d1 +size 475350 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..125770bbc31eaf9384d5754030aa580a40a2e94f --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fafb3ed8788863cc447dfcbc24e9001df833f53b05dca56183099f8dd9d06ec0 +size 1066951 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..082324ec425eea51d3e5ab0825c8e47b9094aa20 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc4b850d6eaa2e2db758ecfc98ad7cc5ed4d941920627357bb1b246398a2e048 +size 176724 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_binary/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..95e32d1de558cee7178f6d93013845274b3f60f9 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1623.0,+89/-65,0.747,8.95,3.5,6.145,442.09,6.381,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1554.0,+100/-58,0.642,11.87,4.35,6.413,1923.88,9.103,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1529.0,+90/-51,0.639,13.04,6.02,7.496,4.91,0.694,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1521.0,+94/-54,0.594,13.46,5.24,7.523,1923.88,1.329,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1484.0,+71/-60,0.511,15.42,5.71,9.411,1152.95,1.784,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1472.0,+48/-54,0.455,16.12,10.09,10.519,2775.44,8.849,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabICL (default) [3.33% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1448.0,+65/-54,0.468,17.52,6.71,9.774,8.06,1.743,3.33,True,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1448.0,+66/-50,0.407,17.54,8.17,11.17,2229.7,1.387,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1414.0,+51/-37,0.337,19.61,12.21,12.663,328.96,1.102,0.0,False,CPU +9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1405.0,+66/-55,0.391,20.18,6.96,9.517,4982.66,358.753,0.0,False,GPU +10,🧠🔁,Neural Network,[LimiX (default) [17.04% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1398.0,+82/-76,0.469,20.62,6.79,13.11,1.92,0.489,17.04,True,GPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1393.0,+72/-56,0.326,20.99,9.69,12.081,2229.7,0.133,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1386.0,+60/-51,0.309,21.41,11.06,12.442,1055.6,0.482,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1382.0,+61/-52,0.317,21.68,10.08,12.536,1055.6,0.063,0.0,False,CPU +14,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1380.0,+54/-51,0.327,21.79,10.5,12.413,4.27,0.064,0.0,False,CPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1379.0,+49/-54,0.271,21.9,11.37,12.311,2775.44,0.415,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1365.0,+38/-30,0.234,22.8,15.34,13.52,328.96,0.126,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1364.0,+62/-51,0.28,22.87,12.42,13.566,472.99,0.621,0.0,False,CPU +18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1362.0,+62/-55,0.323,22.98,11.51,14.431,250.36,1.708,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1349.0,+46/-56,0.236,23.86,11.46,13.197,3383.91,0.51,0.0,False,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1349.0,+90/-92,0.348,23.88,9.54,13.238,3383.91,8.396,0.0,False,GPU +21,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1343.0,+59/-52,0.272,24.26,8.57,11.206,4982.66,42.199,0.0,False,GPU +22,🧠🔁,Neural Network,[Mitra (default) [33.33% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1336.0,+87/-81,0.395,24.76,8.22,14.039,103.86,2.616,33.33,True,GPU +23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1332.0,+47/-40,0.206,24.99,15.8,14.081,472.99,0.13,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1330.0,+86/-76,0.346,25.18,8.69,14.004,3031.01,27.044,33.33,True,GPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1315.0,+56/-53,0.181,26.15,15.32,14.2,642.33,2.767,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1302.0,+49/-40,0.175,27.06,18.1,13.82,2241.65,2.415,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1298.0,+64/-63,0.226,27.36,14.59,14.773,8.72,0.126,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1293.0,+48/-43,0.152,27.68,18.0,15.686,1560.02,0.404,0.0,False,CPU +29,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1288.0,+46/-56,0.142,28.06,16.42,15.473,642.33,0.146,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1282.0,+80/-75,0.276,28.42,8.94,15.828,3031.01,0.514,33.33,True,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1269.0,+62/-70,0.222,29.33,11.21,14.57,49.67,43.824,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1261.0,+48/-47,0.101,29.92,18.94,16.397,1560.02,0.052,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1257.0,+46/-40,0.124,30.14,16.66,16.239,13.5,0.311,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1256.0,+58/-56,0.138,30.25,18.4,15.841,559.38,4.08,0.0,False,CPU +35,🧠⚡,Foundation Model,[TabPFNv2 (default) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1256.0,+80/-80,0.275,30.26,11.7,17.22,2.71,0.276,33.33,True,GPU +36,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1253.0,+43/-43,0.094,30.44,20.02,15.156,9.33,1.116,0.0,False,GPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1236.0,+48/-49,0.102,31.6,19.41,17.45,4.98,0.049,0.0,False,CPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1221.0,+66/-56,0.096,32.61,22.15,16.712,1.24,0.11,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1215.0,+43/-37,0.066,32.99,22.6,16.017,2241.65,0.136,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1186.0,+57/-60,0.052,34.94,22.15,17.87,559.38,0.276,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1183.0,+48/-45,0.045,35.09,25.49,18.346,125.38,0.674,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1173.0,+46/-35,0.045,35.75,30.57,17.489,0.81,0.093,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1153.0,+50/-58,0.032,37.05,25.7,19.69,169.75,0.603,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1144.0,+50/-49,0.027,37.62,27.86,19.5,125.38,0.069,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1119.0,+42/-57,0.011,39.16,26.32,20.419,169.75,0.059,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1093.0,+53/-47,0.012,40.68,31.46,20.405,4.95,0.091,0.0,False,CPU +47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1056.0,+61/-77,0.015,42.74,27.51,21.791,2.56,0.267,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1050.0,+83/-99,0.053,43.03,29.05,27.109,162.68,0.405,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1041.0,+102/-102,0.076,43.53,26.39,25.867,0.85,0.129,0.0,False,GPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1028.0,+61/-93,0.017,44.19,31.56,24.522,139.84,3.917,0.0,False,CPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1022.0,+87/-118,0.034,44.49,31.08,27.802,162.68,0.054,0.0,False,CPU +52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+62/-69,0.007,45.57,38.74,24.603,0.25,0.029,0.0,False,CPU +53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,975.0,+72/-97,0.015,46.69,38.16,26.891,2.8,0.564,0.0,False,GPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,962.0,+77/-111,0.02,47.26,34.13,29.736,0.84,0.094,0.0,False,CPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,946.0,+72/-94,0.008,47.92,41.57,26.072,0.2,0.037,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,918.0,+76/-128,0.0,49.05,35.34,29.619,139.84,0.193,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,593.0,+109/-189,0.0,56.3,52.0,44.093,0.15,0.023,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5924697994483de3493806fe2b45026875c24d76 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b449687c2f1243b1440732d53a6a346bdb503d7e4ba27a67622c42e676faee9b +size 2760741 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/n_datasets_9 b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/n_datasets_9 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..dbadea366a0375a274c27ad08b1122827f183f00 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6ea72dde0d62e5ce043218bb592eda595ea2782a92310ac853e50152402be14 +size 430512 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7e4d53dc7adf17e93af26c3583c5b7df059f07b3 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de3c7513860b16aafc489a52272dea2dec3849db874fd96ada515da4b244302b +size 1001152 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3961b8a670fc5e5f11e4a150d951a83222a695ea --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f0153d85429faf4f0668bc0ab04de0abf1a1d06d62b0f7d739cdf10d994a072 +size 134896 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..de5864b86e9c7527b58e8fd133b851545edd1a45 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv @@ -0,0 +1,55 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1952.0,+156/-57,0.904,5.02,2.43,2.22,275.07,2.448,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1887.0,+175/-63,0.85,6.53,2.71,2.498,325.91,2.156,0.0,False,CPU +2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1744.0,+147/-88,0.63,11.19,5.89,4.733,619.15,0.198,0.0,False,CPU +3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1739.0,+142/-86,0.636,11.38,4.9,5.026,619.15,0.016,0.0,False,CPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1736.0,+118/-39,0.572,11.51,8.06,5.448,1633.8,2.053,0.0,False,GPU +5,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1722.0,+131/-79,0.63,12.09,5.58,4.771,2.21,0.025,0.0,False,CPU +6,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1705.0,+146/-89,0.544,12.8,7.1,7.516,110.13,0.386,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1688.0,+146/-81,0.524,13.49,8.59,7.932,86.85,0.668,0.0,False,CPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1668.0,+198/-135,0.512,14.38,5.61,7.278,804.65,0.71,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1667.0,+146/-138,0.543,14.46,7.14,6.915,2139.17,6.41,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1650.0,+204/-120,0.489,15.23,5.4,8.389,842.24,15.119,0.0,False,GPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1638.0,+126/-65,0.372,15.81,8.84,7.036,1633.8,0.088,0.0,False,GPU +12,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1628.0,+117/-76,0.419,16.27,11.67,8.384,110.13,0.061,0.0,False,CPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1612.0,+196/-154,0.456,17.02,6.32,7.986,804.65,0.063,0.0,False,GPU +14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1600.0,+193/-114,0.42,17.65,6.9,9.041,842.24,1.465,0.0,False,GPU +15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1572.0,+156/-126,0.417,19.02,8.0,8.638,2.3,0.896,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1568.0,+158/-86,0.323,19.23,12.24,9.749,86.85,0.078,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1556.0,+165/-162,0.42,19.88,8.63,9.22,8.53,3.578,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1542.0,+120/-80,0.254,20.59,13.66,9.79,2139.17,0.349,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1527.0,+170/-152,0.346,21.37,7.59,9.027,4860.88,475.791,0.0,False,GPU +20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1515.0,+96/-110,0.247,22.0,14.66,9.128,1156.69,1.558,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1480.0,+146/-104,0.195,23.86,15.28,10.686,572.82,1.935,0.0,False,GPU +22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1465.0,+127/-110,0.171,24.69,18.15,12.878,270.2,0.192,0.0,False,CPU +23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1451.0,+121/-141,0.227,25.47,15.18,10.869,0.49,0.053,0.0,False,CPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1444.0,+132/-115,0.161,25.86,19.53,12.706,572.82,0.109,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1443.0,+149/-178,0.243,25.88,14.7,10.498,5.5,0.074,0.0,False,GPU +26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1422.0,+105/-126,0.124,27.06,20.82,11.44,371.59,2.386,0.0,False,CPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1420.0,+83/-65,0.079,27.19,24.01,10.931,6.41,0.247,0.0,False,GPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1407.0,+118/-135,0.136,27.88,20.37,13.968,96.61,0.754,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1393.0,+126/-107,0.093,28.67,24.4,14.005,270.2,0.015,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1371.0,+97/-128,0.072,29.85,23.72,11.539,1156.69,0.054,0.0,False,CPU +31,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1365.0,+128/-130,0.085,30.14,19.07,12.77,4860.88,42.641,0.0,False,GPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1351.0,+127/-111,0.052,30.89,26.74,15.839,1.07,0.018,0.0,False,CPU +33,🧠🔁,Neural Network,[LimiX (default) [55.56% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1348.0,+200/-227,0.233,31.07,13.41,18.235,0.63,0.019,55.56,True,GPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1337.0,+97/-131,0.067,31.63,28.44,12.469,0.29,0.033,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1313.0,+52/-59,0.002,32.88,31.05,14.156,42.95,0.226,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1308.0,+86/-82,0.009,33.14,30.03,13.961,8.34,0.25,0.0,False,GPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1275.0,+59/-86,0.0,34.77,32.44,13.465,40.36,0.318,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1274.0,+76/-73,0.005,34.81,31.22,15.136,42.95,0.021,0.0,False,CPU +39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1274.0,+171/-184,0.032,34.82,21.22,17.027,47.03,45.876,0.0,False,GPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1272.0,+81/-116,0.037,34.91,29.62,15.039,371.59,0.102,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1220.0,+68/-96,0.0,37.36,35.66,15.181,1.86,0.046,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1214.0,+64/-68,0.0,37.63,35.91,15.182,40.36,0.028,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1199.0,+131/-210,0.043,38.31,32.82,18.054,1.52,0.115,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1157.0,+147/-229,0.008,40.12,35.81,24.041,43.64,0.085,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1138.0,+96/-184,0.011,40.89,31.9,20.737,513.06,78.02,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1107.0,+165/-222,0.005,42.07,38.55,25.035,43.64,0.022,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+83/-194,0.0,45.68,44.94,22.283,0.12,0.014,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,983.0,+131/-215,0.0,46.17,44.55,26.738,0.22,0.024,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,969.0,+198/-439,0.025,46.56,40.45,28.52,2.62,0.186,0.0,False,GPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,916.0,+95/-189,0.0,47.91,47.06,30.36,0.24,0.036,0.0,False,GPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,912.0,+93/-235,0.0,48.0,47.54,24.149,0.1,0.021,0.0,False,CPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,884.0,+82/-127,0.0,48.62,48.24,27.802,513.06,2.568,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,508.0,+104/-387,0.0,53.27,53.24,45.598,0.11,0.087,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..2323f41caf6eab6a8c6ffe2c219fedb06ae9a035 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48df8f16adb82f680a2411a7e25e4c514c07d25693c53ec233f93accb56e50ba +size 2264366 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/n_datasets_21 b/data/imputation_yes/splits_all/tasks_binary/datasets_small/n_datasets_21 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..aabdf1ae3bc40987980c87c9dc36e13c78e70554 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6677ec779d8216c9b5e173f90dcf1f917b1e61bb84d47d1bc411d0fb8e881c14 +size 482352 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..574b4fc49dc4c952928559a96b32c23999b3326a --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40402863f3ee036d58d994c79cfa7a80d5c4f21d8608d37571bda48046d5749c +size 1039896 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e4dbc32c40fbabd1dd32e7649761d94b55f672ab --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ee83e0e3ff5c7fd719bad2817bc4d7cb989b1b658ee43de6dc316a120c999ee +size 190540 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_binary/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..e7bb7fbb02b28eba762c21f291465146c1316c1d --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1573.0,+135/-64,0.7,10.43,4.02,5.566,2052.28,8.252,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1573.0,+93/-51,0.72,10.43,5.44,7.007,5.96,0.554,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1568.0,+103/-72,0.675,10.63,4.33,7.828,593.57,8.084,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1545.0,+122/-58,0.658,11.64,4.75,6.873,2052.28,1.266,0.0,False,GPU +4,🧠🔁,Neural Network,[Mitra (default) [4.76% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1479.0,+89/-54,0.564,14.91,6.08,10.506,143.16,3.761,4.76,True,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1466.0,+105/-56,0.494,15.6,6.44,10.456,3422.16,45.847,4.76,True,GPU +6,🧠🔁,Neural Network,[LimiX (default) [0.53% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1466.0,+94/-65,0.565,15.65,5.59,10.913,3.78,0.54,0.53,True,GPU +7,🧠⚡,Foundation Model,[TabICL (default) [4.76% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1450.0,+72/-70,0.477,16.52,6.13,10.012,7.78,1.706,4.76,True,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1423.0,+68/-46,0.391,18.1,11.32,12.692,2950.72,15.595,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1410.0,+74/-55,0.353,18.88,10.15,12.838,2457.54,1.658,0.0,False,GPU +10,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1404.0,+80/-70,0.36,19.23,10.86,12.373,1921.63,1.718,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1398.0,+79/-72,0.394,19.65,6.71,9.728,5351.68,268.579,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1388.0,+125/-63,0.394,20.24,6.64,13.062,3422.16,0.983,4.76,True,GPU +13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1382.0,+76/-66,0.387,20.62,9.69,14.629,317.91,1.983,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1366.0,+84/-73,0.344,21.63,6.93,10.535,5351.68,37.621,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1357.0,+44/-36,0.244,22.23,14.91,14.691,512.57,1.18,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1351.0,+73/-54,0.26,22.59,12.56,13.836,2457.54,0.178,0.0,False,GPU +17,🧠⚡,Foundation Model,[TabPFNv2 (default) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1349.0,+99/-81,0.393,22.76,8.85,15.05,4.05,0.421,4.76,True,GPU +18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1325.0,+37/-34,0.178,24.31,17.21,15.136,512.57,0.243,0.0,False,CPU +19,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1322.0,+74/-59,0.209,24.48,12.95,14.571,2950.72,1.225,0.0,False,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1312.0,+80/-64,0.215,25.21,10.71,14.657,4806.04,0.512,0.0,False,GPU +21,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1303.0,+47/-43,0.159,25.78,17.73,15.746,1096.5,0.584,0.0,False,CPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1301.0,+59/-44,0.185,25.92,16.88,15.688,4.81,0.08,0.0,False,CPU +23,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1299.0,+55/-42,0.168,26.08,18.41,15.755,1096.5,0.074,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1294.0,+86/-77,0.289,26.4,9.3,13.517,51.83,43.615,0.0,False,GPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1284.0,+46/-66,0.169,27.09,15.33,15.706,711.84,2.983,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1282.0,+44/-47,0.155,27.18,18.29,16.16,693.49,1.398,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1273.0,+67/-61,0.208,27.82,14.52,16.605,10.08,0.152,0.0,False,GPU +28,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1272.0,+102/-109,0.254,27.92,11.14,15.948,4806.04,8.461,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1260.0,+46/-48,0.123,28.71,17.9,16.89,1634.54,0.422,0.0,False,CPU +30,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1260.0,+43/-37,0.1,28.73,18.62,16.522,693.49,0.194,0.0,False,CPU +31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1259.0,+59/-64,0.166,28.77,13.98,17.215,14.37,0.334,0.0,False,GPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1258.0,+49/-61,0.127,28.81,15.35,16.659,711.84,0.162,0.0,False,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1252.0,+77/-56,0.133,29.21,20.12,15.831,2665.36,2.619,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1239.0,+47/-48,0.095,30.14,17.24,17.421,1634.54,0.054,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+60/-79,0.136,31.5,17.51,17.727,644.55,5.974,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1218.0,+46/-53,0.11,31.57,17.33,18.14,7.32,0.054,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1215.0,+52/-53,0.084,31.76,18.67,16.966,10.89,2.249,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1178.0,+69/-47,0.059,34.19,22.12,17.936,2665.36,0.143,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1170.0,+61/-84,0.055,34.69,19.95,19.083,644.55,0.398,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1163.0,+65/-80,0.064,35.18,23.33,20.437,203.15,0.968,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1158.0,+56/-58,0.033,35.52,27.53,19.215,1.94,0.121,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1127.0,+62/-79,0.038,37.43,25.36,21.351,203.15,0.091,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1127.0,+40/-53,0.02,37.45,31.55,19.64,2.44,0.121,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1106.0,+57/-92,0.042,38.77,23.91,22.062,484.22,1.025,0.0,False,CPU +45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1073.0,+106/-126,0.109,40.71,22.04,23.941,1.3,0.15,0.0,False,GPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1070.0,+52/-89,0.011,40.88,24.6,22.683,484.22,0.086,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1052.0,+62/-82,0.017,41.89,29.86,22.644,9.39,0.13,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1020.0,+92/-135,0.067,43.63,26.7,28.424,278.97,0.873,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1011.0,+74/-90,0.0,44.09,25.59,23.393,2.97,0.388,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+56/-113,0.01,44.63,36.01,25.597,0.42,0.041,0.0,False,CPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,997.0,+93/-135,0.044,44.79,28.48,28.987,278.97,0.103,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,988.0,+78/-117,0.019,45.2,31.26,26.144,90.71,1.623,0.0,False,CPU +53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,984.0,+80/-114,0.01,45.39,36.59,26.193,3.71,0.919,0.0,False,GPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,962.0,+82/-136,0.011,46.44,38.56,26.896,0.33,0.043,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,951.0,+89/-163,0.029,46.94,30.75,31.021,1.48,0.139,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,922.0,+83/-149,0.001,48.17,31.33,30.397,90.71,0.125,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,616.0,+111/-175,0.0,55.88,50.04,43.448,0.22,0.02,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e2519a06c163102ae8d06a7d31344079f12a0e57 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5650f2ac484e1d96f2825a0291ed23243ac70fe438a913bf3c7fbe640c6083d9 +size 2725579 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/n_datasets_20 b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/n_datasets_20 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1458b1d6553597fc28fca8eb22339f9fa181af02 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:607e71c3612163bf5be8f74d67bc57a128bcad60c555c4733f81cbac7bc63712 +size 483576 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5ef49fd18a5373d8e83ba08e66b60c4cfe17fc36 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82071b0ea6a221a8aac580dae2637db883e8a0a4a2ec23edf47d19dfd9fb9b7f +size 1038997 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..af31c3e7a27565e4c00476c1b27d74c47fb912c2 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f25ee4c9c0f0ca2e61ec55a72d03d0b00973ada673f4110bdcc3702dc58e268 +size 131179 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..f3f6c9e65b8ce5c20388b697463d899b2f1ab624 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1600.0,+91/-54,0.729,10.36,5.33,7.184,5.52,0.529,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1588.0,+113/-69,0.685,10.87,4.46,5.826,2046.25,8.114,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1584.0,+88/-56,0.662,11.04,4.67,8.183,592.15,8.082,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1569.0,+104/-64,0.657,11.74,4.79,7.089,2046.25,1.254,0.0,False,GPU +4,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1510.0,+92/-60,0.585,14.67,5.88,10.733,144.27,3.772,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1497.0,+79/-54,0.512,15.33,6.23,10.68,3445.6,51.274,0.0,False,GPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1486.0,+98/-79,0.565,15.96,5.68,11.297,3.76,0.527,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1479.0,+79/-67,0.493,16.35,5.93,10.213,8.06,1.714,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1442.0,+65/-50,0.385,18.49,11.6,13.145,2879.46,14.626,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1438.0,+75/-59,0.362,18.78,9.99,13.186,2379.18,1.595,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1430.0,+69/-59,0.406,19.25,9.32,14.423,316.0,1.931,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1419.0,+76/-58,0.339,19.93,11.82,12.9,1828.68,1.648,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1419.0,+84/-68,0.391,19.94,6.64,10.011,5319.2,249.314,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1415.0,+108/-75,0.407,20.2,6.42,13.416,3445.6,0.995,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1399.0,+104/-71,0.361,21.18,6.68,10.637,5319.2,29.488,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1381.0,+86/-58,0.273,22.36,12.32,14.163,2379.18,0.176,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1376.0,+45/-44,0.233,22.67,15.32,15.222,449.53,1.149,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1372.0,+96/-85,0.405,22.91,8.63,15.503,3.73,0.488,0.0,False,GPU +18,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1352.0,+65/-52,0.22,24.25,12.67,14.896,2879.46,0.944,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1347.0,+63/-58,0.225,24.59,10.35,14.824,4575.49,0.51,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1347.0,+38/-37,0.17,24.6,17.27,15.649,449.53,0.215,0.0,False,CPU +21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1326.0,+105/-79,0.303,25.96,8.98,13.65,51.44,43.709,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1326.0,+61/-49,0.189,25.97,16.69,16.157,4.7,0.08,0.0,False,CPU +23,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1323.0,+56/-53,0.151,26.23,17.99,16.287,1081.93,0.556,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1316.0,+56/-45,0.159,26.64,19.09,16.312,1081.93,0.074,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1308.0,+67/-67,0.219,27.19,14.07,16.789,9.81,0.146,0.0,False,GPU +26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1302.0,+53/-58,0.162,27.58,15.63,16.244,592.34,2.767,0.0,False,GPU +27,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1298.0,+52/-52,0.138,27.86,19.56,16.778,645.57,1.032,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1295.0,+66/-71,0.174,28.07,13.53,17.352,14.36,0.311,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1292.0,+50/-50,0.129,28.32,17.46,17.207,1618.46,0.415,0.0,False,CPU +30,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1290.0,+97/-103,0.254,28.39,11.16,16.484,4575.49,8.396,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1286.0,+69/-58,0.14,28.68,19.64,15.992,2576.18,2.415,0.0,False,CPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1280.0,+54/-58,0.127,29.14,15.16,17.188,592.34,0.146,0.0,False,GPU +33,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1275.0,+42/-42,0.084,29.47,19.6,17.137,645.57,0.172,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1271.0,+52/-55,0.1,29.74,16.79,17.71,1618.46,0.054,0.0,False,CPU +35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+54/-58,0.115,30.88,16.81,18.247,6.86,0.052,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1253.0,+67/-78,0.143,30.93,17.0,17.887,618.9,5.468,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1250.0,+55/-56,0.088,31.15,18.15,17.097,10.47,1.987,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1213.0,+63/-50,0.062,33.59,21.56,17.997,2576.18,0.136,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1206.0,+67/-74,0.058,34.09,19.4,19.212,618.9,0.352,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1183.0,+84/-72,0.067,35.56,23.19,21.088,199.83,0.865,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1172.0,+64/-60,0.025,36.23,29.08,19.888,1.77,0.119,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1150.0,+78/-78,0.04,37.6,25.09,21.933,199.83,0.086,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1144.0,+40/-43,0.018,38.0,32.13,20.296,2.2,0.12,0.0,False,CPU +44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1110.0,+95/-131,0.115,40.03,21.41,23.531,1.24,0.143,0.0,False,GPU +45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1108.0,+65/-85,0.02,40.14,26.17,22.968,440.56,0.933,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1086.0,+75/-74,0.018,41.44,29.26,22.66,8.39,0.127,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1078.0,+52/-79,0.004,41.88,24.91,23.518,440.56,0.085,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1057.0,+98/-123,0.071,43.04,26.02,28.113,273.27,0.715,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1043.0,+78/-87,0.0,43.76,24.97,23.546,2.84,0.381,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1034.0,+94/-126,0.047,44.19,27.79,28.584,273.27,0.098,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1020.0,+83/-108,0.02,44.93,30.67,26.329,89.68,1.55,0.0,False,CPU +52,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1002.0,+98/-112,0.011,45.79,36.55,26.937,2.96,0.796,0.0,False,GPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+63/-105,0.003,45.87,37.86,26.578,0.42,0.04,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,989.0,+102/-139,0.03,46.39,30.04,30.404,1.44,0.138,0.0,False,CPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,976.0,+93/-114,0.012,46.96,38.72,27.729,0.31,0.042,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,953.0,+86/-138,0.001,47.95,30.7,30.595,89.68,0.12,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,639.0,+130/-215,0.0,55.9,49.79,43.912,0.21,0.019,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..9a9597403be7612cf179c30fca0c70f875dd228b --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28826ea6797aca90c9e4190739a0fb902ff02c08ace5a831c651f8314a44b008 +size 2690375 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/n_datasets_38 b/data/imputation_yes/splits_all/tasks_classification/datasets_all/n_datasets_38 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..f3f370097269fe11540d0e3e38c2f5946b25852d --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e13d5a2ed7c6edb04d4c6ac684c420d81c5ae70f8a274fb2287e9e20ef65e31f +size 481972 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3191ab755af8386e22321129fe1159fecd554bed --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63e60df80df61a78e2dfac0636f42e297b9dc7f43e4e253d0f02a1edd8919805 +size 1075471 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5506aaab810afd4132e3678af06485fe5ade8817 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f353772dd7e83438ca05ec78338010e6013acb70a069b1e4f4a7119eaf0b4d5f +size 183321 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_classification/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..8bfb01a9702580b56d035045b482fd76def637de --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1638.0,+69/-56,0.765,8.56,3.69,6.824,545.23,6.469,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1577.0,+79/-72,0.681,10.99,3.71,6.302,2046.25,8.976,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1550.0,+72/-63,0.643,12.21,4.72,7.349,2046.25,1.329,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1535.0,+66/-61,0.643,12.96,5.76,7.777,5.76,0.794,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1485.0,+66/-53,0.493,15.62,5.79,10.48,1684.49,1.97,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1472.0,+56/-46,0.462,16.32,9.49,11.041,2879.46,12.49,0.0,False,GPU +6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1445.0,+61/-42,0.405,17.94,8.68,12.372,2466.21,1.502,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabICL (default) [5.26% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1417.0,+62/-60,0.427,19.7,7.44,12.417,8.68,1.743,5.26,True,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1416.0,+46/-33,0.333,19.76,12.62,13.471,382.05,1.488,0.0,False,CPU +9,🧠🔁,Neural Network,[LimiX (default) [16.08% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1415.0,+71/-72,0.47,19.82,6.63,12.898,3.45,0.504,16.08,True,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1404.0,+76/-50,0.39,20.47,7.33,11.426,4940.61,307.751,0.0,False,GPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1394.0,+57/-54,0.329,21.15,9.99,13.269,2466.21,0.176,0.0,False,GPU +12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1393.0,+49/-43,0.305,21.19,11.07,12.692,2879.46,0.598,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1392.0,+54/-41,0.308,21.23,11.95,12.971,1372.94,0.556,0.0,False,CPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1384.0,+54/-44,0.303,21.76,11.08,13.175,1372.94,0.074,0.0,False,CPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1375.0,+44/-31,0.246,22.4,15.39,14.193,382.05,0.254,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1365.0,+48/-44,0.271,23.04,13.33,14.314,685.87,1.455,0.0,False,CPU +17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1362.0,+46/-44,0.29,23.26,11.95,13.81,5.72,0.076,0.0,False,CPU +18,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1357.0,+79/-74,0.398,23.59,7.8,14.612,3008.22,20.849,31.58,True,GPU +19,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1356.0,+51/-48,0.312,23.63,12.55,15.62,282.72,1.886,0.0,False,GPU +20,🧠🔁,Neural Network,[Mitra (default) [31.58% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1355.0,+79/-79,0.41,23.72,8.41,15.045,121.91,2.776,31.58,True,GPU +21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1353.0,+81/-75,0.319,23.8,10.42,13.855,4879.89,8.744,0.0,False,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1352.0,+55/-37,0.226,23.92,11.59,13.898,4879.89,0.525,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1345.0,+61/-52,0.284,24.39,8.86,13.143,4940.61,41.606,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1341.0,+46/-40,0.214,24.64,16.3,14.692,685.87,0.205,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1319.0,+43/-45,0.197,26.1,15.62,15.322,934.1,3.063,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1305.0,+73/-76,0.307,27.09,8.8,16.473,3008.22,0.514,31.58,True,GPU +27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1300.0,+42/-36,0.16,27.4,18.06,16.365,2686.11,0.47,0.0,False,CPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1299.0,+44/-41,0.162,27.48,18.83,15.242,2389.22,2.158,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1299.0,+50/-48,0.229,27.49,15.1,16.043,10.21,0.138,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabPFNv2 (default) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1282.0,+72/-71,0.292,28.68,11.75,17.271,3.37,0.315,31.58,True,GPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1278.0,+53/-55,0.146,28.9,16.76,16.823,934.1,0.169,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1266.0,+43/-36,0.111,29.75,19.24,17.087,2686.11,0.054,0.0,False,CPU +33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1265.0,+62/-60,0.221,29.82,10.77,16.444,49.21,43.824,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1254.0,+60/-57,0.153,30.58,17.25,17.607,618.9,4.766,0.0,False,CPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1243.0,+41/-43,0.105,31.3,18.36,18.158,14.78,0.346,0.0,False,GPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1240.0,+44/-44,0.112,31.51,20.18,18.322,6.86,0.057,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1230.0,+40/-37,0.074,32.15,22.08,17.431,10.47,1.707,0.0,False,GPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1221.0,+49/-53,0.09,32.76,23.28,17.518,1.77,0.117,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1217.0,+48/-42,0.077,33.01,22.74,17.465,2389.22,0.152,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1206.0,+50/-47,0.086,33.72,21.84,19.045,189.76,0.743,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1188.0,+52/-58,0.062,34.94,21.46,19.443,618.9,0.298,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1183.0,+59/-58,0.085,35.25,20.57,19.941,323.74,0.743,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1179.0,+45/-38,0.064,35.49,29.61,18.579,1.79,0.12,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1166.0,+55/-51,0.062,36.33,22.54,20.27,189.76,0.079,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1146.0,+66/-57,0.056,37.59,19.77,20.807,323.74,0.076,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1084.0,+46/-55,0.01,41.26,32.08,22.16,6.83,0.147,0.0,False,CPU +47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1037.0,+84/-85,0.041,43.74,31.33,29.833,252.58,0.528,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1037.0,+68/-69,0.012,43.74,29.99,24.506,2.91,0.368,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1023.0,+66/-93,0.023,44.46,29.66,27.652,139.84,3.615,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1011.0,+85/-92,0.06,45.04,29.33,28.785,1.11,0.189,0.0,False,GPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1008.0,+78/-89,0.027,45.18,33.3,30.589,252.58,0.089,0.0,False,CPU +52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+56/-57,0.006,45.56,39.42,26.919,0.38,0.037,0.0,False,CPU +53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,998.0,+74/-88,0.022,45.68,37.75,27.409,3.61,0.939,0.0,False,GPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,938.0,+69/-71,0.006,48.2,42.58,29.159,0.25,0.041,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,929.0,+84/-111,0.016,48.58,36.92,33.418,1.29,0.131,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,904.0,+83/-104,0.009,49.5,36.12,33.842,139.84,0.193,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,555.0,+111/-114,0.0,56.61,53.11,49.665,0.18,0.029,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..fd4e6bea83453193a8a4b670ba988aa4f1e1292f --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b650769c5aa1d255c56cedf8be61aa4e83635bdce963675cda6452d927a0495 +size 2678386 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/n_datasets_10 b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/n_datasets_10 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..732b0103ce6981d44d2dc74fb1c1950cc5aa6973 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9f7bcea3025b254ae711555bc2ce966dd60bd434503e88722b792ebe0ac027d +size 437424 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5b5daf5c1ed82b954abf236db37b76fbc501de8b --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79e44af46288d2300b48c6c8a3cf75e275958cdec8c1aaf3074ee15fab5279c1 +size 1016226 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4c07131492a6f59cd4cf5b023153a833efda5e85 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3dac5dd0c19d57c6b52c8c140dcb9e43b85236b6844067b0b6ee92d3ea6772b +size 134430 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..36a517387216c3cfcb38c11b5b9eac5fc4566d6c --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv @@ -0,0 +1,55 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1917.0,+147/-60,0.906,4.82,2.46,2.185,275.97,2.818,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1858.0,+176/-74,0.858,6.13,2.65,2.418,314.17,2.249,0.0,False,CPU +2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1695.0,+135/-72,0.618,11.37,6.22,5.413,697.8,0.199,0.0,False,CPU +3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1694.0,+143/-70,0.624,11.43,5.19,5.658,697.8,0.023,0.0,False,CPU +4,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1665.0,+131/-67,0.609,12.62,5.98,5.6,2.44,0.027,0.0,False,CPU +5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1662.0,+128/-94,0.542,12.72,7.4,7.881,207.65,0.398,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1658.0,+143/-72,0.529,12.88,8.42,8.139,93.21,0.758,0.0,False,CPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1639.0,+120/-89,0.515,13.72,8.72,7.26,1624.54,1.763,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1637.0,+216/-124,0.54,13.82,3.83,7.552,783.29,13.79,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1616.0,+142/-128,0.529,14.77,7.59,7.563,2338.19,14.248,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1594.0,+182/-116,0.46,15.79,6.1,8.64,1398.9,0.645,0.0,False,GPU +11,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1589.0,+114/-75,0.427,16.03,11.85,8.696,207.65,0.056,0.0,False,CPU +12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1585.0,+204/-117,0.468,16.22,6.11,8.38,783.29,1.924,0.0,False,GPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1555.0,+126/-87,0.335,17.7,9.55,8.734,1624.54,0.083,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1544.0,+166/-78,0.346,18.22,11.76,9.818,93.21,0.087,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1544.0,+180/-146,0.41,18.26,6.86,9.353,1398.9,0.062,0.0,False,GPU +16,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1533.0,+146/-118,0.415,18.82,8.42,9.128,2.3,0.82,0.0,False,GPU +17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1512.0,+160/-142,0.413,19.9,9.15,9.75,7.51,3.146,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1500.0,+109/-79,0.264,20.5,14.09,10.252,2338.19,0.54,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1478.0,+154/-137,0.325,21.66,8.15,9.97,4805.66,461.797,0.0,False,GPU +20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1467.0,+86/-76,0.234,22.26,15.27,10.088,1169.3,1.442,0.0,False,CPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1455.0,+142/-105,0.214,22.91,15.36,12.545,893.93,0.205,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1430.0,+148/-126,0.255,24.28,14.98,10.933,0.47,0.052,0.0,False,CPU +23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1413.0,+135/-129,0.175,25.19,16.24,12.445,709.85,2.929,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1388.0,+158/-107,0.141,26.62,20.06,13.595,893.93,0.016,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1378.0,+132/-132,0.145,27.13,20.54,14.515,709.85,0.19,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1376.0,+145/-149,0.218,27.26,15.69,12.68,5.67,0.081,0.0,False,GPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1351.0,+84/-82,0.071,28.67,25.09,13.54,6.98,0.233,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1349.0,+97/-138,0.112,28.74,21.98,14.46,389.62,2.114,0.0,False,CPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1341.0,+136/-132,0.122,29.18,21.44,15.921,84.36,0.919,0.0,False,GPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1338.0,+83/-84,0.076,29.33,23.8,12.269,1169.3,0.053,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1334.0,+125/-102,0.086,29.6,25.47,15.629,2.09,0.019,0.0,False,CPU +32,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1322.0,+111/-117,0.077,30.22,19.83,13.703,4805.66,43.211,0.0,False,GPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1321.0,+116/-99,0.062,30.27,22.3,13.699,52.45,0.219,0.0,False,CPU +34,🧠🔁,Neural Network,[LimiX (default) [60.00% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1303.0,+188/-187,0.21,31.21,14.25,18.746,0.44,0.016,60.0,True,GPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1289.0,+90/-106,0.061,31.97,28.98,13.714,0.28,0.033,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1284.0,+124/-111,0.063,32.2,24.11,14.614,52.45,0.021,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1277.0,+79/-71,0.009,32.57,29.74,14.632,10.5,0.606,0.0,False,GPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1259.0,+71/-86,0.026,33.5,30.99,13.754,38.42,0.32,0.0,False,CPU +39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1241.0,+122/-150,0.029,34.41,21.89,17.572,46.9,45.623,0.0,False,GPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1214.0,+80/-141,0.033,35.73,30.58,17.64,389.62,0.112,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1201.0,+75/-84,0.012,36.32,34.32,15.542,38.42,0.028,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1176.0,+63/-70,0.0,37.5,35.95,16.74,1.87,0.046,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1139.0,+109/-210,0.038,39.1,33.8,21.377,1.61,0.112,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1099.0,+142/-227,0.007,40.78,36.67,26.806,43.46,0.087,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1068.0,+95/-197,0.009,41.96,33.16,24.396,331.68,44.947,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1048.0,+150/-236,0.004,42.71,39.35,28.029,43.46,0.021,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+99/-176,0.0,44.35,43.24,22.389,0.12,0.013,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,928.0,+135/-232,0.0,46.49,44.98,29.601,0.2,0.023,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,914.0,+201/-459,0.022,46.88,41.22,31.192,2.8,0.224,0.0,False,GPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,880.0,+87/-199,0.0,47.71,47.28,26.755,0.09,0.019,0.0,False,CPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,859.0,+109/-176,0.0,48.2,47.41,32.973,0.2,0.033,0.0,False,GPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,818.0,+67/-150,0.0,49.06,48.67,32.101,331.68,1.626,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,447.0,+126/-378,0.0,53.34,53.31,49.305,0.12,0.101,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..435ab5b02da7eba063be02372f645a2c5e461ebc --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8da259e29152fdee5ff682f90392202ada57ac3d15020a56cfe724911a4e438d +size 2273190 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/n_datasets_28 b/data/imputation_yes/splits_all/tasks_classification/datasets_small/n_datasets_28 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..31f68a600531270549a4e9315a31be48cacc06f2 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a2cd7b6d3bc2bf4bc5fe0ce7f0a3b1f41d42d467efc29590222a0104a5244e7 +size 471502 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a964b394c22b966a7aaeda8b5a94f84e1d773320 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:043be86e8746e0169982a8aaa9769291c0f8c46e5abf91ff3d06e8b8477fa6d6 +size 1059768 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..68a6d6c506517fbe86ee9ae9776cc9c9812b9d0b --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1426dbd5a5f0457aedbccd32c18bf18d86d4ed5e27158e59858c6509fbc9ce5 +size 190557 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_classification/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..783c8eca8668f7a8f76fb2770392eca7eee27a66 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1604.0,+78/-60,0.711,9.9,4.49,8.48,610.76,8.082,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1602.0,+88/-82,0.725,9.98,3.67,5.856,2289.05,8.114,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1583.0,+75/-67,0.697,10.77,4.36,6.98,2289.05,1.254,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1582.0,+78/-79,0.713,10.84,5.18,7.294,6.07,0.762,0.0,False,GPU +4,🧠🔁,Neural Network,[LimiX (default) [0.40% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1489.0,+85/-70,0.559,15.35,5.56,10.809,4.62,0.633,0.4,True,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1488.0,+85/-75,0.54,15.43,6.02,11.831,3302.86,45.811,7.14,True,GPU +6,🧠🔁,Neural Network,[Mitra (default) [7.14% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1484.0,+92/-78,0.556,15.65,6.52,12.423,144.27,3.772,7.14,True,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1457.0,+68/-52,0.433,17.16,9.79,12.392,3422.82,18.936,0.0,False,GPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1431.0,+68/-56,0.375,18.68,10.21,13.705,3170.35,1.694,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1426.0,+79/-52,0.359,19.01,10.04,13.359,3143.8,1.884,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabICL (default) [7.14% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1416.0,+82/-60,0.421,19.62,6.98,13.369,8.89,1.714,7.14,True,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1409.0,+67/-62,0.399,20.04,7.07,11.946,5271.34,231.986,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1407.0,+106/-83,0.416,20.18,6.83,14.357,3302.86,0.929,7.14,True,GPU +13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1389.0,+67/-50,0.368,21.32,10.91,15.512,330.95,2.359,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1376.0,+58/-60,0.291,22.09,11.93,14.668,3170.35,0.241,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1375.0,+72/-61,0.352,22.17,7.4,12.943,5271.34,33.986,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1374.0,+46/-36,0.252,22.22,15.36,15.375,735.1,1.901,0.0,False,CPU +17,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1373.0,+58/-52,0.28,22.32,11.72,14.106,3422.82,1.534,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabPFNv2 (default) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1372.0,+79/-73,0.396,22.38,9.29,15.443,4.15,0.42,7.14,True,GPU +19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1349.0,+44/-29,0.196,23.88,17.3,15.756,735.1,0.311,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1336.0,+41/-35,0.187,24.74,17.8,15.67,1714.62,0.768,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1330.0,+51/-49,0.2,25.11,10.89,15.2,6047.72,0.525,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1325.0,+39/-32,0.178,25.44,18.66,15.86,1714.62,0.098,0.0,False,CPU +23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1313.0,+55/-58,0.201,26.25,15.38,16.35,1053.84,3.063,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1306.0,+45/-47,0.163,26.73,18.68,16.611,797.9,2.228,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1302.0,+78/-77,0.234,27.03,12.02,16.103,6047.72,8.744,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1301.0,+44/-38,0.164,27.05,18.55,16.742,7.19,0.118,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1298.0,+59/-61,0.224,27.3,14.87,17.244,10.78,0.17,0.0,False,GPU +28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1291.0,+42/-42,0.126,27.72,18.81,16.833,797.9,0.353,0.0,False,CPU +29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1291.0,+77/-81,0.279,27.73,9.1,16.042,51.44,43.709,0.0,False,GPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1275.0,+41/-37,0.125,28.82,19.23,17.729,3351.28,0.544,0.0,False,CPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1268.0,+55/-60,0.141,29.25,15.68,17.648,1053.84,0.169,0.0,False,GPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1267.0,+54/-50,0.126,29.33,20.53,17.082,2977.49,2.415,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1248.0,+46/-42,0.092,30.63,18.92,18.335,3351.28,0.064,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1246.0,+54/-57,0.13,30.78,16.14,19.416,15.54,0.346,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1242.0,+70/-62,0.161,30.99,15.99,18.731,1423.89,9.208,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1228.0,+57/-49,0.109,31.94,18.75,19.284,8.03,0.059,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1209.0,+46/-41,0.063,33.18,21.12,18.821,14.89,4.433,0.0,False,GPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1200.0,+81/-70,0.105,33.77,19.75,20.935,399.55,1.466,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1194.0,+44/-56,0.072,34.22,22.35,19.321,2977.49,0.156,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1192.0,+58/-53,0.07,34.32,19.34,20.087,1423.89,0.749,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1171.0,+41/-48,0.025,35.67,28.99,19.87,2.79,0.238,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1163.0,+82/-76,0.078,36.19,20.05,21.958,399.55,0.174,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1156.0,+50/-53,0.054,36.6,29.76,20.316,3.15,0.141,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1150.0,+76/-89,0.091,36.98,19.99,22.17,571.65,1.39,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1110.0,+75/-74,0.051,39.41,18.55,23.019,571.65,0.129,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1059.0,+60/-66,0.013,42.3,30.74,24.096,11.29,0.204,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1044.0,+91/-103,0.082,43.07,25.61,27.29,2.59,0.533,0.0,False,GPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1025.0,+75/-110,0.022,44.09,36.08,26.058,4.95,1.012,0.0,False,GPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1023.0,+78/-96,0.051,44.16,29.56,30.914,483.02,0.874,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1009.0,+67/-72,0.0,44.85,28.65,25.623,5.04,0.646,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1008.0,+86/-111,0.027,44.91,28.44,28.815,121.17,1.63,0.0,False,CPU +52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+67/-78,0.008,45.29,37.69,28.537,0.46,0.068,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,998.0,+76/-98,0.033,45.36,31.3,31.504,483.02,0.113,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,959.0,+78/-102,0.009,47.15,40.28,30.018,0.42,0.074,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,923.0,+98/-136,0.022,48.6,34.38,34.781,2.39,0.143,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,920.0,+105/-116,0.012,48.72,32.71,34.463,121.17,0.136,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,582.0,+114/-167,0.0,56.35,51.76,49.793,0.28,0.023,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..53a70e51f324c5bf76de97ed0ca359bdadf8e096 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:200b8552c3bdee6f12c7e400ee9072a603a3c16e19bb2504a125a5470d1004ba +size 2759671 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/n_datasets_26 b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/n_datasets_26 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..14d00c02112a621975f53d4b7d91213220004358 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:125ea11c0f227c4777b562ba3bacf5e58787fec2d5882b38a4827a94dd64658f +size 471650 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..07ecdf64d41515fc7b88505d16f663cd393225ce --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d7aa972473c7332a2a51427ae7d348c7c8f3356ca7743f7d51666626d916e1c +size 1073311 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d0d93a3aac096f6d2eaa7c23456727c62f3c92eb --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e95d8250a0f919b54feb88a15237118d5bc6ca6b828a78c0d58a23e2ec57851 +size 120230 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..3e24e20a24c4cda44ab4bc96fbb38217adfa91f4 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1634.0,+83/-79,0.728,9.78,3.8,6.234,2168.17,7.961,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1631.0,+70/-57,0.747,9.87,4.93,7.553,5.99,0.633,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1625.0,+85/-64,0.701,10.13,4.67,9.074,604.44,8.082,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1614.0,+81/-69,0.702,10.59,4.34,7.377,2168.17,1.23,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1544.0,+69/-56,0.576,13.78,5.67,11.21,3445.6,48.236,0.0,False,GPU +5,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1536.0,+80/-59,0.593,14.17,6.16,11.857,146.65,4.101,0.0,False,GPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1518.0,+87/-73,0.58,15.09,5.58,11.42,4.37,0.585,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1486.0,+61/-44,0.447,16.86,9.64,13.101,3169.9,16.793,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1459.0,+71/-61,0.447,18.45,6.6,12.877,8.89,1.743,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1454.0,+71/-46,0.373,18.73,10.01,14.494,2828.45,1.596,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1452.0,+82/-72,0.442,18.89,6.45,13.931,3445.6,0.995,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1444.0,+66/-55,0.356,19.32,10.43,14.093,2898.23,1.884,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1425.0,+77/-70,0.388,20.47,6.97,12.671,5119.36,231.986,0.0,False,GPU +13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1424.0,+60/-53,0.369,20.56,10.58,15.951,330.95,2.023,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1410.0,+84/-72,0.421,21.42,8.85,15.11,4.06,0.436,0.0,False,GPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1400.0,+56/-54,0.297,22.04,11.42,14.802,3169.9,1.446,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1400.0,+82/-72,0.355,22.07,7.06,13.572,5119.36,26.967,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1399.0,+72/-51,0.29,22.12,11.71,15.476,2828.45,0.216,0.0,False,GPU +18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1388.0,+42/-35,0.23,22.83,15.77,16.363,647.56,1.72,0.0,False,CPU +19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1362.0,+37/-30,0.175,24.46,17.6,16.741,647.56,0.284,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1356.0,+50/-36,0.186,24.9,17.76,16.602,1465.86,0.692,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1355.0,+56/-50,0.195,24.97,11.03,15.889,5944.88,0.516,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1342.0,+48/-35,0.173,25.77,18.98,16.823,1465.86,0.091,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1326.0,+64/-56,0.23,26.84,14.3,18.007,10.42,0.155,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1325.0,+46/-41,0.157,26.9,19.33,17.617,766.06,1.917,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1323.0,+95/-83,0.242,27.06,11.78,17.015,5944.88,8.396,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1320.0,+45/-44,0.159,27.23,18.32,17.726,5.72,0.11,0.0,False,CPU +27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1320.0,+87/-95,0.3,27.25,8.62,16.755,50.32,43.824,0.0,False,GPU +28,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1320.0,+50/-53,0.172,27.25,16.63,17.396,934.1,2.767,0.0,False,GPU +29,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1309.0,+39/-36,0.119,27.97,19.24,17.841,766.06,0.278,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1297.0,+46/-41,0.117,28.77,18.8,18.633,2686.11,0.47,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1294.0,+56/-54,0.135,28.96,20.03,17.813,2862.05,2.158,0.0,False,CPU +32,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1283.0,+45/-58,0.14,29.68,15.39,19.49,14.8,0.337,0.0,False,GPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1277.0,+50/-50,0.12,30.07,16.88,18.738,934.1,0.165,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1275.0,+77/-87,0.174,30.22,15.28,19.42,1358.63,8.067,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1269.0,+45/-48,0.083,30.63,18.46,19.24,2686.11,0.056,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1244.0,+50/-47,0.089,32.25,19.07,20.124,7.4,0.057,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1243.0,+44/-39,0.068,32.34,20.31,19.291,13.83,3.953,0.0,False,GPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+64/-78,0.076,33.88,19.08,20.796,1358.63,0.897,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1213.0,+53/-43,0.061,34.25,22.19,20.109,2862.05,0.152,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1199.0,+60/-67,0.076,35.15,24.0,22.25,370.85,1.466,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1184.0,+45/-43,0.019,36.07,30.03,21.0,2.4,0.218,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1160.0,+62/-70,0.046,37.55,25.98,23.266,370.85,0.161,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1153.0,+36/-44,0.027,37.95,32.6,21.599,2.9,0.133,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1133.0,+70/-69,0.043,39.17,25.14,23.712,527.42,1.39,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1093.0,+61/-65,0.011,41.45,24.97,24.551,527.42,0.123,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1081.0,+56/-63,0.014,42.1,30.05,24.892,10.38,0.188,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1071.0,+83/-130,0.088,42.63,24.72,27.865,2.43,0.495,0.0,False,GPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1058.0,+100/-110,0.054,43.29,28.54,30.113,436.81,0.874,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1034.0,+89/-103,0.024,44.5,35.91,27.438,4.4,0.974,0.0,False,GPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1033.0,+101/-105,0.036,44.53,30.27,30.656,436.81,0.106,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1032.0,+72/-81,0.0,44.59,27.79,26.107,4.73,0.623,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1008.0,+77/-109,0.015,45.72,32.67,30.107,113.26,1.55,0.0,False,CPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+61/-83,0.003,46.1,38.93,29.212,0.45,0.066,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,969.0,+75/-112,0.009,47.45,40.18,30.601,0.4,0.07,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,958.0,+91/-127,0.023,47.9,33.34,33.436,2.16,0.139,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,909.0,+87/-143,0.001,49.71,34.2,36.035,113.26,0.12,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,600.0,+121/-261,0.0,56.33,51.43,49.89,0.26,0.023,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d95123a58ea83701b518cce39a14bf179497db75 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc0f448c09c36a25e04e35f41f96d68f04df436f0f0fd1ee65a44df5d6af2519 +size 2801926 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/n_datasets_8 b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/n_datasets_8 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..026ccc3d2e50968a4ecb229dc6a83aecc4cf4502 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad5e312dd06fafd0cf2724488f1f948d7da696f42e1ff4cc102477662390c2b0 +size 472876 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..f060ce78db3612e8a709a424fdb17a68f2cc947f --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6210edaa62ab3520e79558f5995e74429e8594969089932a89f062690443f3b2 +size 1019606 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..686463bd674296dbca0f3c1057b342174f3ae3ef --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3d957aebfa4fd233504ba7215a147cecf3c26084090132fba117661707e7f33 +size 172927 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..fdd8297dc3173f3f41147939737e658c0b626dd2 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1724.0,+151/-70,0.833,7.12,4.6,9.367,1045.67,7.086,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1710.0,+222/-98,0.825,7.55,3.44,6.694,2710.27,1.347,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1706.0,+284/-137,0.825,7.67,2.39,5.887,2710.27,8.099,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1578.0,+229/-151,0.66,12.67,4.96,8.831,10.23,0.844,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1504.0,+192/-99,0.428,16.41,6.12,14.488,6219.24,3.78,0.0,False,CPU +5,🧠🔁,Neural Network,[LimiX (default) [12.50% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1497.0,+174/-104,0.471,16.84,6.1,12.101,5.85,0.91,12.5,True,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1493.0,+163/-107,0.49,17.05,7.75,12.999,4150.25,19.131,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1482.0,+262/-213,0.593,17.66,5.63,16.891,2589.89,10.803,25.0,True,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1467.0,+128/-74,0.432,18.53,10.07,14.123,4150.25,3.423,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1452.0,+121/-82,0.395,19.44,11.32,16.878,5885.87,1.632,0.0,False,GPU +10,🧠🔁,Neural Network,[Mitra (default) [25.00% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1446.0,+200/-163,0.467,19.8,9.18,18.817,161.61,3.705,25.0,True,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1437.0,+120/-84,0.318,20.35,14.42,16.499,1377.29,5.202,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1434.0,+96/-58,0.303,20.56,17.07,14.953,3104.39,1.095,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1428.0,+103/-77,0.291,20.92,15.57,16.719,1377.29,1.259,0.0,False,CPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1417.0,+164/-123,0.387,21.57,9.17,18.584,4635.14,212.285,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1415.0,+113/-80,0.338,21.74,11.29,17.724,5885.87,0.299,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1409.0,+95/-59,0.248,22.06,17.73,15.572,3104.39,0.212,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1409.0,+198/-168,0.421,22.11,8.31,18.891,2589.89,0.54,25.0,True,GPU +18,🧠⚡,Foundation Model,[TabPFNv2 (default) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1398.0,+188/-143,0.357,22.75,11.93,17.461,6.47,0.363,25.0,True,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1389.0,+92/-81,0.244,23.32,18.49,16.984,2210.02,0.974,0.0,False,CPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1386.0,+106/-80,0.209,23.52,15.97,16.17,7127.77,9.214,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1383.0,+100/-84,0.235,23.69,18.39,17.116,2210.02,3.985,0.0,False,CPU +22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1376.0,+72/-52,0.187,24.16,12.11,16.529,7127.77,0.776,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1364.0,+176/-118,0.33,24.87,10.15,20.408,4635.14,34.77,0.0,False,GPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1348.0,+147/-123,0.258,25.92,16.87,19.531,1980.6,3.605,0.0,False,GPU +25,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1346.0,+132/-110,0.27,26.08,18.97,20.079,507.73,4.428,0.0,False,GPU +26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1341.0,+109/-97,0.191,26.36,18.26,18.908,36484.23,0.827,0.0,False,CPU +27,🧠⚡,Foundation Model,[TabICL (default) [12.50% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1319.0,+169/-167,0.269,27.84,12.57,22.325,10.98,1.898,12.5,True,GPU +28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1317.0,+124/-118,0.238,27.97,17.42,20.804,17.83,0.197,0.0,False,GPU +29,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1309.0,+180/-174,0.287,28.5,11.75,20.881,737.91,1.864,0.0,False,CPU +30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1307.0,+169/-141,0.238,28.59,14.2,21.669,717.36,1.612,0.0,False,CPU +31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1304.0,+86/-71,0.147,28.79,24.75,19.049,50.54,0.227,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1300.0,+136/-109,0.113,29.04,22.15,20.573,3901.28,2.028,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1299.0,+107/-113,0.147,29.14,20.44,19.677,36484.23,0.127,0.0,False,CPU +34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1267.0,+126/-136,0.148,31.18,23.71,21.594,19.31,0.108,0.0,False,CPU +35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1262.0,+163/-157,0.196,31.48,13.13,23.154,717.36,0.326,0.0,False,CPU +36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1259.0,+202/-138,0.219,31.68,9.38,23.472,43.39,42.575,0.0,False,GPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1259.0,+199/-203,0.226,31.69,10.23,22.263,737.91,0.316,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1257.0,+176/-164,0.207,31.79,13.98,24.229,2644.99,12.294,0.0,False,CPU +39,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1253.0,+134/-120,0.161,32.06,18.15,21.886,1980.6,0.61,0.0,False,GPU +40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1236.0,+119/-144,0.119,33.09,23.26,22.898,3901.28,0.223,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1233.0,+118/-80,0.067,33.31,28.8,20.542,4.72,0.542,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1212.0,+147/-122,0.134,34.53,26.5,22.667,5.04,0.429,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1206.0,+131/-146,0.1,34.95,19.21,25.342,2644.99,1.148,0.0,False,CPU +44,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1194.0,+79/-84,0.031,35.67,29.76,25.353,17.94,0.691,0.0,False,GPU +45,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1145.0,+58/-74,0.0,38.53,35.95,25.964,19.2,7.624,0.0,False,GPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1081.0,+118/-200,0.05,41.88,36.27,29.352,9.42,1.675,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1048.0,+72/-119,0.0,43.44,34.63,28.741,13.69,0.314,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1001.0,+214/-366,0.043,45.5,24.18,39.389,151.69,2.346,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+107/-182,0.0,45.54,42.17,35.605,0.74,0.137,0.0,False,CPU +50,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,979.0,+94/-231,0.0,46.4,44.39,40.046,892.95,0.818,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,950.0,+122/-199,0.0,47.5,45.33,34.685,8.51,0.734,0.0,False,CPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,943.0,+84/-237,0.0,47.74,45.41,41.043,892.95,0.146,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,899.0,+132/-177,0.0,49.25,46.82,40.735,0.69,0.137,0.0,False,CPU +54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,850.0,+110/-155,0.0,50.73,50.4,39.73,5.05,1.013,0.0,False,GPU +55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,833.0,+268/-604,0.04,51.18,39.36,49.678,151.69,0.258,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,731.0,+89/-254,0.0,53.53,53.21,47.224,4.53,0.164,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,188.0,+131/-663,0.0,57.77,57.76,70.56,0.52,0.076,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..9e3a4380d6f5fb5059290fb0a1f036a3028e0b85 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d6eadd414018d1f579c8c97dcf83ee1d0246d73eb94cdc815cfe99123fcfea +size 2634680 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/n_datasets_1 b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/n_datasets_1 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ea1a0801d6eb9497ee4cfb5098840b2741363401 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c47205bdca7aa36ca18a681f3208a8301fb66cac6e50cb2fe2db8dc8b94d5fb3 +size 444887 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3d874df419b12bff529f12338f841427910b5e3b --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfbe22e2d0b0e5e61981f999f52c0d0122cd3042e043406c10a193936f6e0d2a +size 977457 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..6a0351baa6eee60f21fe1c713c366298d4ccc924 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b731b62ff22925891023afe53607d0a7970d248ab5dc8eb1f3b2be2282b0979a +size 106892 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..c4f7f7abf85b38dcd22115ae20c737de87eb39da --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv @@ -0,0 +1,54 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,9270.0,+0/-0,1.0,1.11,1.06,0.019,133.98,8.253,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,8803.0,+0/-0,0.934,2.56,2.2,1.7,276.87,4.662,0.0,False,CPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,8700.0,+0/-0,0.927,3.0,2.84,1.878,276.86,3.44,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,8621.0,+0/-0,0.905,3.33,3.0,2.424,133.98,2.816,0.0,False,GPU +4,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,6195.0,+0/-0,0.594,6.78,6.31,9.581,61.95,0.17,0.0,False,CPU +5,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,6182.0,+0/-0,0.595,6.89,6.45,9.545,7606.4,0.826,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,6134.0,+0/-0,0.573,7.33,7.13,10.011,157.72,3.649,0.0,False,CPU +7,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,6038.0,+0/-0,0.578,8.22,7.72,9.898,7606.4,0.048,0.0,False,CPU +8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,5997.0,+0/-0,0.578,8.67,7.91,9.921,61.95,0.016,0.0,False,CPU +9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,5959.0,+0/-0,0.552,9.11,8.66,10.441,157.72,0.374,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,5682.0,+0/-0,0.508,11.89,10.98,11.347,2688.02,0.24,0.0,False,CPU +11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,5671.0,+0/-0,0.518,12.0,11.9,11.168,466.47,1.036,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,5573.0,+0/-0,0.499,13.0,12.51,11.536,2688.02,0.726,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,5519.0,+0/-0,0.501,13.56,13.39,11.505,0.39,0.051,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,5486.0,+0/-0,0.501,13.89,13.77,11.505,466.47,0.051,0.0,False,CPU +15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,5145.0,+0/-0,0.399,17.0,16.0,13.544,1.48,0.53,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,5108.0,+0/-0,0.423,17.33,17.24,13.063,7.54,0.047,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,5082.0,+0/-0,0.407,17.56,17.52,13.396,4423.24,29.2,0.0,False,GPU +18,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,5032.0,+0/-0,0.389,18.0,17.84,13.739,17.7,0.086,0.0,False,CPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,4812.0,+0/-0,0.354,19.67,19.62,14.412,4423.24,2.237,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,4739.0,+0/-0,0.348,20.11,20.02,14.523,2.82,1.073,0.0,False,GPU +21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,3202.0,+0/-0,0.25,22.11,22.11,16.353,20.98,0.329,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,2532.0,+0/-0,0.132,24.22,24.17,18.457,4638.91,343.215,0.0,False,GPU +23,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,2475.0,+0/-0,0.115,24.56,24.47,18.732,2389.31,0.267,0.0,False,CPU +24,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,2475.0,+0/-0,0.113,24.56,24.51,18.777,20.98,0.027,0.0,False,CPU +25,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,2456.0,+0/-0,0.109,24.67,24.61,18.841,2389.31,0.023,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1738.0,+0/-0,0.0,27.44,27.43,20.674,14.87,1.855,0.0,False,GPU +27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1554.0,+0/-0,0.0,28.33,28.3,20.901,1993.14,0.452,0.0,False,GPU +28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1404.0,+0/-0,0.0,29.22,29.16,21.65,1993.14,0.059,0.0,False,GPU +29,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1218.0,+0/-0,0.0,30.44,30.4,22.097,4638.91,43.781,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1218.0,+0/-0,0.0,30.44,30.35,22.475,40.54,38.741,0.0,False,GPU +31,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+0/-0,0.0,31.89,31.77,23.341,0.18,0.008,0.0,False,CPU +32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,821.0,+0/-0,0.0,33.0,32.98,23.568,1294.93,0.417,0.0,False,GPU +33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,665.0,+0/-0,0.0,33.89,33.86,24.01,1294.93,0.06,0.0,False,GPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,599.0,+0/-0,0.0,34.22,34.18,24.919,0.18,0.04,0.0,False,CPU +35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,-215.0,+0/-0,0.0,36.11,36.11,28.274,1863.58,6.302,0.0,False,GPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,-734.0,+0/-0,0.0,37.56,37.54,30.79,1863.58,1.131,0.0,False,GPU +37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-788.0,+0/-0,0.0,37.78,37.75,30.768,3.96,0.025,0.0,False,CPU +38,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,-1010.0,+0/-0,0.0,38.67,38.66,32.321,6.85,0.088,0.0,False,GPU +39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,-1462.0,+0/-0,0.0,39.89,39.89,33.496,66.06,1.792,0.0,False,GPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,-2063.0,+0/-0,0.0,41.0,41.0,37.017,7.55,0.149,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,-2631.0,+0/-0,0.0,42.11,42.11,41.044,569.0,0.17,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,-2955.0,+0/-0,0.0,42.89,42.89,41.644,569.0,1.658,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,-3626.0,+0/-0,0.0,44.11,44.11,50.207,0.07,0.015,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-4076.0,+0/-0,0.0,45.22,45.21,51.289,2.86,0.085,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-4255.0,+0/-0,0.0,45.78,45.77,51.687,30.41,0.089,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-4823.0,+0/-0,0.0,47.44,47.44,54.976,30.41,0.015,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-5018.0,+0/-0,0.0,48.33,48.32,55.365,0.13,0.014,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,-5106.0,+0/-0,0.0,48.78,48.72,55.242,3.09,0.997,0.0,False,GPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,-5308.0,+0/-0,0.0,49.78,49.77,56.49,0.11,0.023,0.0,False,GPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-5500.0,+0/-0,0.0,50.56,50.54,57.332,76.27,3.054,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-6498.0,+0/-0,0.0,52.0,52.0,70.796,76.27,0.37,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-7418.0,+0/-0,0.0,53.0,53.0,82.671,0.14,0.136,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..eae64c32497ce508043396d6517707de0418a1e0 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca98ad1f4ef11a968abbb5e1e69ddb5c96de4a27e8d9b49b8ba51e74cefba221 +size 1124587 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/n_datasets_7 b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/n_datasets_7 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7c001f0846bb12168fdb94774164df6fd9862553 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb49dfae0005dcef9e69ea2ff9f7cd13b7b3a87337e6eab3eb521d6d7fdeae28 +size 466173 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..47d97bc1e2ccb504c48f8369485d78a9401240ee --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23a01081f441da74c00b988b40d2dfe5b5533834b364c0122491d0b288ddd830 +size 1027072 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a3a07ebc6bcad222592cdc256fed8d65b7c166e8 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e473feba4e9bc3958b806f6fb5cea95d4b421b7f9197f5d915b1229cc18428b6 +size 167986 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..39db0b91a56bd83a7bc3fc7381c7ed4cdcd22ab2 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1751.0,+173/-64,0.819,7.7,5.05,10.436,1154.67,7.879,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1737.0,+229/-89,0.813,8.15,3.51,7.304,2742.37,1.218,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1724.0,+264/-127,0.8,8.61,2.91,6.725,2742.37,7.945,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1637.0,+311/-165,0.691,12.05,4.51,8.157,11.21,0.845,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1589.0,+185/-115,0.56,14.33,6.97,11.489,4471.59,21.354,0.0,False,GPU +5,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1586.0,+164/-99,0.539,14.47,5.46,10.495,6.22,1.092,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1578.0,+333/-218,0.678,14.92,5.03,15.957,2986.29,15.005,14.29,True,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1560.0,+120/-62,0.494,15.82,9.12,12.71,4471.59,3.76,0.0,False,GPU +8,🧠🔁,Neural Network,[Mitra (default) [14.29% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1523.0,+229/-174,0.533,17.86,8.32,18.171,200.23,4.42,14.29,True,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1519.0,+153/-74,0.443,18.09,10.41,16.304,5929.46,1.73,0.0,False,GPU +10,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1514.0,+213/-86,0.355,18.38,8.2,16.315,6548.58,2.898,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1487.0,+230/-196,0.482,20.01,7.48,18.242,2986.29,0.63,14.29,True,GPU +12,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1477.0,+144/-80,0.384,20.59,10.38,17.164,5929.46,0.337,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1467.0,+185/-157,0.415,21.19,8.42,18.602,4631.38,190.647,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (default) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1466.0,+203/-166,0.408,21.23,10.93,16.621,7.26,0.403,14.29,True,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1460.0,+92/-51,0.27,21.64,18.01,15.441,3272.91,1.185,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1451.0,+161/-83,0.278,22.2,16.88,17.426,1502.45,6.33,0.0,False,CPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1444.0,+136/-72,0.249,22.61,17.58,17.616,1502.45,1.367,0.0,False,CPU +18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1431.0,+130/-108,0.309,23.4,17.52,18.163,684.78,4.816,0.0,False,GPU +19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1429.0,+82/-46,0.206,23.52,19.44,16.176,3272.91,0.21,0.0,False,CPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1426.0,+107/-136,0.295,23.75,15.56,18.282,2097.62,3.142,0.0,False,GPU +21,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1425.0,+196/-162,0.377,23.79,9.25,20.167,4631.38,30.351,0.0,False,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1416.0,+127/-95,0.174,24.37,15.77,16.566,7359.32,9.026,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1411.0,+108/-73,0.202,24.67,19.44,17.767,2551.65,1.021,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1409.0,+78/-58,0.157,24.81,11.48,16.831,7359.32,0.596,0.0,False,GPU +25,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1400.0,+124/-73,0.189,25.36,19.94,17.966,2551.65,4.527,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1394.0,+158/-112,0.272,25.73,16.04,19.159,19.65,0.236,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabICL (default) [14.29% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1344.0,+200/-161,0.252,28.94,11.94,23.44,11.18,2.132,14.29,True,GPU +28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1341.0,+80/-84,0.129,29.14,24.71,20.246,63121.14,0.827,0.0,False,CPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1336.0,+217/-166,0.237,29.49,12.7,21.741,2775.63,12.592,0.0,False,CPU +30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1336.0,+190/-189,0.229,29.51,13.51,22.429,717.53,1.628,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1333.0,+164/-116,0.103,29.68,21.86,20.836,4142.25,2.105,0.0,False,CPU +32,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1321.0,+93/-70,0.102,30.43,26.4,19.905,64.18,0.255,0.0,False,CPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1319.0,+102/-118,0.184,30.56,16.78,20.614,2097.62,0.515,0.0,False,GPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1302.0,+185/-212,0.239,31.6,13.41,22.496,755.05,2.155,0.0,False,CPU +35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1301.0,+229/-201,0.25,31.69,8.53,23.615,46.25,45.74,0.0,False,GPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1294.0,+86/-94,0.081,32.12,26.74,21.074,63121.14,0.173,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1288.0,+190/-222,0.199,32.47,12.32,23.78,717.53,0.339,0.0,False,CPU +38,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1279.0,+146/-130,0.108,33.06,24.87,22.716,20.91,0.126,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1276.0,+199/-153,0.115,33.22,17.71,23.099,2775.63,1.431,0.0,False,CPU +40,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1263.0,+171/-174,0.153,34.05,25.43,22.345,5.32,0.431,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1259.0,+129/-141,0.112,34.29,23.07,23.478,4142.25,0.287,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1247.0,+184/-230,0.171,34.98,10.68,24.026,755.05,0.317,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1228.0,+38/-42,0.0,36.13,34.47,21.833,5.48,0.599,0.0,False,CPU +44,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1216.0,+98/-86,0.025,36.84,30.12,26.021,20.96,0.482,0.0,False,GPU +45,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1205.0,+52/-73,0.0,37.46,34.86,24.386,19.63,7.686,0.0,False,GPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1156.0,+98/-168,0.057,40.18,34.66,25.653,10.84,2.189,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1088.0,+73/-139,0.0,43.54,33.72,28.451,14.75,0.337,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1077.0,+206/-394,0.049,44.06,22.38,36.825,173.37,1.638,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1038.0,+99/-212,0.0,45.77,43.6,38.383,905.14,0.875,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1005.0,+97/-238,0.0,47.07,44.56,39.053,905.14,0.175,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1004.0,+153/-222,0.0,47.11,44.7,32.313,8.97,0.743,0.0,False,CPU +52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+121/-211,0.0,47.28,43.84,37.357,0.85,0.148,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,943.0,+142/-240,0.0,49.27,46.51,39.382,0.78,0.147,0.0,False,CPU +54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,915.0,+88/-170,0.0,50.15,49.83,37.336,5.83,1.167,0.0,False,GPU +55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,908.0,+270/-615,0.046,50.35,37.69,46.661,173.37,0.147,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,776.0,+96/-280,0.0,53.56,53.19,46.061,4.72,0.188,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,261.0,+123/-665,0.0,57.74,57.72,68.83,0.57,0.067,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5049ab9e852d249566e6cfb5340742ab6bd3b82b --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29cab2deb2da72071dea2ced83b3f2c8292d28cf3738ed27dcdfc3ecf29e1db3 +size 2647495 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/n_datasets_6 b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/n_datasets_6 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..490d61ee1c997046948b35ce8d13badf37cd4503 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93e27a9c51454bcf66318850fa618e22f7d7a9f539b1615f62efee62a3095252 +size 470825 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..063c551d022b9085fd6da874144145278f97a5e9 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afcc5018dab96739735269a5e491334464445c02b8bb54b2289b1e0fa0f68d1d +size 1035459 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..fe734021b8a95d9e3ec02c1fb22d6535e7e6a41c --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7861e72ce185c1c220153766d6afcdf9f7cd2c8a9c2f2a0462381d00ee6bcf00 +size 122677 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..fe131ba2fc1d272b12013893e0a5463f60893637 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1876.0,+270/-100,0.87,6.14,2.55,7.591,2710.27,7.889,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1852.0,+240/-92,0.851,6.75,3.29,8.338,2710.27,1.076,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1838.0,+207/-77,0.833,7.12,4.7,12.046,1374.26,12.843,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1799.0,+306/-137,0.807,8.24,3.94,8.785,10.23,0.844,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1788.0,+296/-112,0.791,8.58,4.37,12.979,3585.96,32.815,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1708.0,+168/-44,0.653,11.4,6.17,12.954,4150.25,19.131,0.0,False,GPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1688.0,+166/-61,0.628,12.2,5.26,11.831,5.85,0.91,0.0,False,GPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1680.0,+197/-107,0.622,12.51,7.31,15.607,205.23,4.488,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1633.0,+220/-107,0.562,14.52,6.55,15.646,3585.96,1.167,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1630.0,+135/-43,0.553,14.68,8.59,14.49,4150.25,3.423,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1592.0,+203/-102,0.476,16.45,9.67,13.798,6.47,0.427,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1575.0,+284/-135,0.414,17.28,7.5,18.07,7307.18,4.205,0.0,False,CPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1549.0,+239/-110,0.412,18.59,10.08,18.854,5885.87,1.632,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1513.0,+108/-46,0.303,20.47,17.01,17.651,3104.39,1.095,0.0,False,CPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1497.0,+197/-114,0.346,21.34,10.08,19.852,5885.87,0.299,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1481.0,+240/-202,0.377,22.22,8.34,21.539,4513.63,201.505,0.0,False,GPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1474.0,+122/-64,0.203,22.61,14.46,18.788,7367.31,7.816,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1470.0,+95/-49,0.218,22.86,18.63,18.528,3104.39,0.18,0.0,False,CPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1468.0,+142/-63,0.236,22.95,18.11,20.186,2210.02,1.236,0.0,False,CPU +19,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1460.0,+224/-87,0.219,23.37,17.51,20.164,1377.29,6.652,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1455.0,+147/-63,0.221,23.69,18.61,20.414,2210.02,7.723,0.0,False,CPU +21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1450.0,+196/-70,0.188,23.97,18.81,20.379,1377.29,1.395,0.0,False,CPU +22,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1433.0,+202/-128,0.245,24.92,19.16,21.045,696.38,4.428,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1431.0,+237/-220,0.338,25.04,8.66,23.355,4513.63,26.967,0.0,False,GPU +24,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1424.0,+226/-151,0.294,25.44,10.59,21.754,12.57,2.243,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1420.0,+212/-145,0.267,25.68,15.1,22.067,17.83,0.197,0.0,False,GPU +26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1412.0,+144/-153,0.205,26.15,21.17,21.235,1880.81,2.744,0.0,False,GPU +27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1410.0,+86/-72,0.092,26.26,14.18,19.439,7367.31,0.587,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1381.0,+315/-274,0.276,27.89,11.42,24.53,2644.99,12.716,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1347.0,+200/-164,0.121,29.89,21.48,23.884,4654.35,2.028,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1340.0,+132/-106,0.076,30.27,25.3,23.387,36484.23,0.712,0.0,False,CPU +31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1320.0,+134/-78,0.06,31.45,27.13,22.957,264.83,0.227,0.0,False,CPU +32,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1318.0,+331/-279,0.291,31.55,7.6,27.104,40.51,46.546,0.0,False,GPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1289.0,+146/-150,0.099,33.19,27.21,23.905,1880.81,0.353,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1289.0,+262/-248,0.134,33.2,18.09,26.079,2644.99,1.47,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1282.0,+107/-99,0.027,33.59,27.61,24.339,36484.23,0.127,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1278.0,+176/-198,0.106,33.8,27.13,26.121,717.36,1.838,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1255.0,+115/-83,0.029,35.05,28.39,26.62,17.94,0.418,0.0,False,GPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1247.0,+64/-58,0.0,35.52,33.72,24.705,4.72,0.714,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1239.0,+180/-243,0.119,35.94,22.24,26.195,737.91,2.289,0.0,False,CPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1232.0,+70/-89,0.0,36.29,33.66,26.605,20.73,8.179,0.0,False,GPU +41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1230.0,+181/-140,0.06,36.45,24.6,27.149,4654.35,0.223,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1223.0,+101/-89,0.0,36.81,34.51,26.38,14.75,0.108,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1212.0,+159/-217,0.065,37.39,29.45,27.709,717.36,0.326,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1204.0,+196/-142,0.055,37.78,34.29,25.943,5.04,0.449,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1160.0,+160/-231,0.033,40.02,25.17,27.997,737.91,0.316,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1156.0,+183/-247,0.067,40.22,33.93,29.106,9.81,2.345,0.0,False,GPU +47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1069.0,+131/-286,0.0,44.15,42.1,36.78,892.95,0.895,0.0,False,CPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1066.0,+132/-195,0.0,44.29,33.03,32.333,13.69,0.314,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1031.0,+119/-328,0.0,45.67,43.14,37.562,892.95,0.146,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+170/-313,0.0,46.83,42.97,37.99,0.74,0.137,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,985.0,+174/-259,0.0,47.35,44.55,34.643,8.51,0.828,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,954.0,+212/-444,0.0,48.38,41.73,42.698,151.69,1.252,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,932.0,+195/-360,0.0,49.06,45.92,40.173,0.69,0.137,0.0,False,CPU +54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,851.0,+29/-142,0.0,51.27,51.09,42.312,5.05,1.013,0.0,False,GPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,775.0,+103/-312,0.0,52.93,52.56,43.542,4.53,0.164,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,580.0,+156/-543,0.0,55.58,55.16,54.165,151.69,0.099,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,178.0,+157/-1027,0.0,57.75,57.73,69.817,0.52,0.056,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d6630dd27a82e3488e31f060d871b54649d2c4eb --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a69f25312265ff9537b977a8308ec208056de898285291041eb7d4cbfd46b4a6 +size 2656649 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/n_datasets_13 b/data/imputation_yes/splits_all/tasks_regression/datasets_all/n_datasets_13 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..9eaf7f1f340953ecf906da410d77e8a5b6ecfaff --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d7e8c50c6ce63284d5f95c16332b90947a79ba9940a24791aabbc42abdb297 +size 450891 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..367f39271545a7b159c2e2db12d262837b6531aa --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbd624640db18c22b903e51841e2de7dab9a4a1dbadbcc60f2e58093ac5387ec +size 1035802 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..da4963fdc167d37fc4428017d1ef5ac1bd10add1 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e3ae13c1cc306c69bd718739145f17cfd8a101c7dc3b9b0bb5fb8fb76aec1a5 +size 146665 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_regression/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..a6d1ab79311d902a0de1df3bb9b31d8bcc26f029 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/website_leaderboard.csv @@ -0,0 +1,56 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1779.0,+172/-123,0.805,6.65,2.66,1.541,1709.05,8.122,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1741.0,+134/-74,0.789,7.69,2.75,2.451,4786.55,239.537,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1723.0,+109/-80,0.689,8.24,4.37,2.962,649.34,5.869,0.0,False,GPU +3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1691.0,+101/-62,0.676,9.28,5.76,3.249,3995.01,10.051,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1690.0,+109/-63,0.727,9.3,4.54,2.902,4786.55,38.501,0.0,False,GPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1654.0,+164/-114,0.691,10.59,4.46,2.299,1709.05,0.812,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1632.0,+109/-89,0.553,11.42,6.63,4.693,1866.07,6.07,0.0,False,CPU +7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1598.0,+123/-63,0.603,12.77,4.99,3.844,46.62,39.212,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1572.0,+107/-73,0.506,13.85,8.41,4.229,3995.01,0.844,0.0,False,GPU +9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1567.0,+141/-106,0.56,14.06,7.91,3.812,7.04,0.508,0.0,False,GPU +10,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1558.0,+132/-115,0.499,14.44,5.51,5.58,3779.52,7.69,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1506.0,+102/-62,0.361,16.85,10.35,6.274,3552.96,0.966,0.0,False,CPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1498.0,+92/-84,0.356,17.22,13.25,6.736,700.15,9.322,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1488.0,+107/-82,0.376,17.72,10.78,5.764,714.5,1.376,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1483.0,+105/-68,0.335,17.95,10.1,6.394,3552.96,0.105,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1469.0,+125/-91,0.348,18.65,13.69,5.166,4158.29,1.41,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1434.0,+89/-74,0.274,20.42,15.98,7.328,700.15,0.968,0.0,False,CPU +17,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1426.0,+183/-165,0.426,20.84,5.7,13.181,5.2,0.755,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1419.0,+60/-46,0.248,21.16,18.57,7.27,834.93,2.614,0.0,False,CPU +19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1404.0,+94/-77,0.265,21.9,17.61,6.439,714.5,0.099,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1398.0,+63/-51,0.218,22.21,19.08,7.418,834.93,0.388,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1393.0,+108/-87,0.277,22.51,13.43,7.578,3779.52,0.396,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1387.0,+120/-99,0.247,22.8,15.86,6.122,4158.29,0.167,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1383.0,+91/-93,0.254,22.99,16.39,7.975,10.89,0.092,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1335.0,+170/-110,0.323,25.52,8.92,6.948,4223.87,27.543,46.15,True,GPU +25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1292.0,+87/-74,0.138,27.84,21.63,9.117,15.5,0.299,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1292.0,+127/-111,0.157,27.84,22.13,7.84,13.32,0.131,0.0,False,GPU +27,🧠🔁,Neural Network,[Mitra (default) [46.15% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1281.0,+136/-104,0.194,28.41,10.03,8.622,71.01,1.848,46.15,True,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1260.0,+117/-101,0.111,29.53,20.46,9.33,4608.59,1.233,0.0,False,CPU +29,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1253.0,+144/-139,0.21,29.91,15.14,8.01,4223.87,0.452,46.15,True,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1250.0,+117/-90,0.087,30.07,24.85,8.854,8.9,1.645,0.0,False,GPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1228.0,+106/-100,0.101,31.22,23.16,11.668,158.22,0.844,0.0,False,CPU +32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1222.0,+44/-43,0.006,31.49,30.02,9.759,2.11,0.275,0.0,False,CPU +33,🧠⚡,Foundation Model,[TabPFNv2 (default) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1205.0,+137/-137,0.153,32.37,19.91,9.409,2.8,0.313,46.15,True,GPU +34,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1200.0,+82/-82,0.071,32.61,28.64,10.418,2.24,0.242,0.0,False,CPU +35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1199.0,+95/-95,0.076,32.65,22.52,11.88,158.22,0.151,0.0,False,CPU +36,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1198.0,+111/-102,0.076,32.69,24.55,10.122,4608.59,0.097,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1172.0,+64/-71,0.033,33.99,31.5,12.437,515.73,0.771,0.0,False,CPU +38,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1165.0,+117/-116,0.038,34.34,26.93,12.184,2.45,0.743,0.0,False,GPU +39,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1159.0,+132/-162,0.108,34.62,25.51,12.916,2929.85,0.294,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1128.0,+68/-83,0.023,36.1,33.54,12.925,515.73,0.124,0.0,False,CPU +41,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1112.0,+129/-163,0.09,36.89,27.31,13.453,2929.85,0.03,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1078.0,+103/-121,0.004,38.39,34.16,13.818,0.47,0.055,0.0,False,CPU +43,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1048.0,+120/-182,0.046,39.7,33.61,14.389,8.47,0.035,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1042.0,+100/-116,0.022,39.93,36.96,13.796,540.06,2.672,0.0,False,CPU +45,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1040.0,+89/-111,0.001,40.01,37.65,13.552,20.48,0.08,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+49/-76,0.0,41.58,40.25,14.44,0.53,0.062,0.0,False,CPU +47,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,986.0,+100/-110,0.012,42.11,39.99,14.283,540.06,0.325,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,896.0,+144/-178,0.0,45.08,39.28,19.61,92.55,0.897,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,860.0,+113/-170,0.0,46.1,44.94,18.544,2.6,0.392,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,792.0,+148/-192,0.0,47.8,43.61,22.032,92.55,0.052,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,664.0,+192/-310,0.0,50.22,42.38,29.211,0.19,0.04,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,490.0,+135/-304,0.0,52.38,52.21,36.423,193.95,0.173,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,442.0,+158/-371,0.0,52.82,52.63,36.597,193.95,0.074,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,223.0,+141/-372,0.0,54.31,54.29,39.031,0.95,0.097,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..841b3874e16f65754a5be18d6ac201d1ffe91c2b --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85ce88243265981856e7c5b5cf865891c948ed319ad43bdc8bf5145f2a0a2a19 +size 2131657 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/n_datasets_5 b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/n_datasets_5 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e2ec593d4ba6b3c234d468ed2d1d81ca06503bf4 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1602d02e7601316d5c1127a5af5c804cbf555616fe1ffe6fe6c9a2782101b00a +size 429005 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..22d8a2edb056361ffc2ce7ba43fd8899301f6e3a --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f0151062424c02a1d161ee84eca4a76cb65b171253e10262409d438f75826b1 +size 984608 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..932ac8a4653da6f1fd49440732888c9e4383dbff --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:809c87da8e16e4d96a7d5026be38632861635f0d0437620d175451a89da53662 +size 119720 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..d41a2046e31a460a63a19e7ed98b54b547edec91 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv @@ -0,0 +1,52 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,2069.0,+1045/-120,0.862,4.4,2.67,1.209,735.58,9.557,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1971.0,+783/-126,0.802,6.31,4.21,1.696,735.58,0.714,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1922.0,+681/-41,0.713,7.47,5.68,2.304,2526.28,3.717,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1917.0,+1210/-123,0.782,7.6,2.16,1.293,4786.55,361.217,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1909.0,+515/-11,0.661,7.8,4.92,3.331,473.01,8.73,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1900.0,+797/-114,0.688,8.04,5.61,2.7,2028.29,1.675,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1884.0,+532/-21,0.644,8.47,4.35,3.488,430.4,7.174,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1850.0,+1175/-128,0.717,9.42,4.22,1.707,4786.55,41.599,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1802.0,+856/-218,0.564,10.87,7.04,2.636,1.6,0.508,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1753.0,+1086/-143,0.591,12.42,4.95,2.355,45.77,39.212,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1721.0,+794/-134,0.535,13.47,8.53,3.741,2028.29,0.081,0.0,False,GPU +11,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1684.0,+503/-95,0.474,14.71,9.45,4.288,425.61,0.615,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1671.0,+376/-133,0.388,15.18,11.21,5.936,222.05,11.236,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1650.0,+347/-105,0.356,15.93,8.83,6.099,1658.41,0.742,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1623.0,+616/-142,0.443,16.89,13.4,4.108,2526.28,0.166,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1619.0,+361/-178,0.35,17.04,7.26,6.197,1658.41,0.059,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1618.0,+331/-121,0.326,17.09,13.63,6.24,222.05,0.968,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1602.0,+522/-214,0.341,17.64,14.24,5.117,3440.87,0.619,0.0,False,GPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1555.0,+327/-104,0.296,19.4,17.23,5.243,425.61,0.04,0.0,False,GPU +19,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1545.0,+667/-210,0.305,19.76,5.04,4.564,2.06,0.755,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1533.0,+425/-345,0.326,20.22,14.49,6.882,5.49,0.037,0.0,False,CPU +21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1526.0,+421/-182,0.24,20.44,18.38,5.96,3440.87,0.052,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1509.0,+268/-136,0.217,21.09,19.43,6.83,282.13,1.74,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1506.0,+647/-156,0.285,21.18,18.43,5.272,6.88,0.094,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1497.0,+267/-142,0.197,21.51,19.3,6.953,282.13,0.388,0.0,False,CPU +25,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1360.0,+317/-308,0.159,26.42,21.95,8.067,0.6,0.08,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1330.0,+330/-211,0.091,27.49,25.06,7.827,5.03,0.025,0.0,False,GPU +27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1305.0,+240/-126,0.011,28.33,27.32,8.79,2.28,0.13,0.0,False,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1301.0,+209/-114,0.061,28.49,26.91,8.242,4348.61,0.528,0.0,False,CPU +29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1292.0,+283/-160,0.013,28.78,27.91,8.309,0.55,0.233,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+204/-297,0.038,31.24,29.53,9.231,4348.61,0.034,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1199.0,+231/-316,0.0,31.89,30.73,9.7,6.75,0.235,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1176.0,+359/-419,0.022,32.62,30.66,10.264,35.67,0.529,0.0,False,CPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1156.0,+87/-156,0.0,33.27,33.0,9.993,79.08,0.546,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1144.0,+260/-394,0.006,33.64,32.41,10.361,35.67,0.05,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1127.0,+112/-248,0.0,34.16,33.79,10.297,79.08,0.057,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1102.0,+387/-661,0.143,34.91,29.95,13.832,2044.56,0.128,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1037.0,+193/-452,0.001,36.8,35.28,13.252,15.08,0.028,0.0,False,CPU +38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1029.0,+364/-709,0.114,37.02,33.51,14.402,2044.56,0.01,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1026.0,+358/-676,0.006,37.11,35.07,12.06,0.17,0.049,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+37/-248,0.0,37.82,37.67,11.446,0.43,0.053,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,976.0,+370/-780,0.102,38.47,35.09,15.083,3.66,0.011,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,848.0,+60/-540,0.0,41.4,41.16,14.323,318.98,0.827,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,794.0,+54/-529,0.0,42.44,42.26,14.781,318.98,0.114,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,789.0,+112/-640,0.0,42.53,42.33,18.087,142.58,1.184,0.0,False,CPU +45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,664.0,+83/-704,0.0,44.53,44.43,20.643,142.58,0.064,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,607.0,+116/-707,0.0,45.27,45.07,18.384,1.36,0.111,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,349.0,+100/-1185,0.0,47.51,47.41,28.1,0.06,0.034,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,81.0,+131/-1638,0.0,48.92,48.92,39.828,44.3,0.048,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-114.0,+182/-2167,0.0,49.77,49.76,40.21,44.3,0.011,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-414.0,+224/-2121,0.0,50.8,50.8,43.378,0.13,0.018,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ef957bbcc4409333f1f292772059a25c42dda644 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff312f1b97b4a62e9b723c827e650657371d7f7845f952179911e893c04ff57c +size 1489576 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/n_datasets_8 b/data/imputation_yes/splits_all/tasks_regression/datasets_small/n_datasets_8 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..9637fe9797a805bc3880393fc2e964e90bba4b13 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2598c44942741229ee3625a943a4e2d38bb1cf4218857161f7523d616305b56 +size 449740 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5cdf55f92355a664329c88d0a16e3f24ccb877d1 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:789580861ff5a749789c72048d60709ab6891d47ebb6fcd6392ef75936e60687 +size 1066800 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e54a71e37a8be63038b9a34e797390de4195a62d --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:985f58682cb3eb29d070ea553d53182ecc1640815b0bc7db69a270d7e7440e7d +size 157524 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_regression/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..d9ad1d77f80774b121f60757ced973abed73872b --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/website_leaderboard.csv @@ -0,0 +1,56 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1732.0,+139/-53,0.779,7.74,3.31,3.175,4822.0,140.81,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1722.0,+271/-134,0.765,8.05,2.65,1.748,2306.87,7.231,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1721.0,+208/-83,0.707,8.09,4.39,2.633,709.58,5.864,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1687.0,+107/-42,0.716,9.22,4.77,3.649,4822.0,22.324,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1664.0,+125/-50,0.658,10.05,5.86,3.591,4558.61,33.853,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1591.0,+114/-48,0.584,12.98,5.02,4.775,48.13,42.001,0.0,False,GPU +6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1585.0,+224/-129,0.615,13.26,4.62,2.677,2306.87,0.902,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1576.0,+131/-73,0.476,13.68,8.49,5.544,5805.42,4.31,0.0,False,CPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1566.0,+119/-72,0.476,14.1,8.34,4.535,4558.61,2.234,0.0,False,GPU +9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1525.0,+209/-141,0.537,16.05,8.57,4.547,7.67,0.547,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1514.0,+258/-178,0.525,16.57,6.01,4.137,6559.81,56.314,12.5,True,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1498.0,+128/-64,0.346,17.42,11.6,6.384,4430.73,1.103,0.0,False,CPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1476.0,+120/-73,0.319,18.5,14.95,7.235,1203.24,7.106,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1476.0,+125/-64,0.31,18.52,13.37,6.518,4430.73,0.179,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1470.0,+177/-92,0.358,18.8,5.41,7.628,8977.36,8.017,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1461.0,+187/-96,0.342,19.27,13.36,5.197,4438.37,1.981,0.0,False,GPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1455.0,+118/-86,0.294,19.59,11.82,6.686,2039.04,2.144,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1425.0,+95/-43,0.241,21.2,18.07,7.544,1265.82,3.15,0.0,False,CPU +18,🧠🔁,Neural Network,[Mitra (default) [12.50% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1423.0,+172/-124,0.315,21.3,6.84,6.857,93.39,2.396,12.5,True,GPU +19,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1419.0,+279/-232,0.478,21.52,6.2,18.567,6.44,0.709,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1401.0,+106/-79,0.221,22.5,17.92,8.008,1203.24,0.956,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1399.0,+88/-44,0.204,22.65,18.94,7.708,1265.82,0.476,0.0,False,CPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1384.0,+125/-75,0.215,23.47,17.87,7.186,2039.04,0.109,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1380.0,+255/-186,0.342,23.69,10.92,5.862,6559.81,0.485,12.5,True,GPU +24,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1369.0,+174/-114,0.237,24.27,14.61,6.224,4438.37,0.231,0.0,False,GPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1361.0,+141/-86,0.192,24.72,17.85,8.658,12.31,0.204,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1339.0,+167/-97,0.155,25.96,13.44,9.748,8977.36,0.421,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabPFNv2 (default) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1308.0,+225/-226,0.248,27.73,15.18,8.136,8.74,0.773,12.5,True,GPU +28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1304.0,+171/-124,0.183,27.94,20.59,7.848,22.14,0.179,0.0,False,GPU +29,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1298.0,+135/-104,0.133,28.32,22.03,8.326,21.51,8.356,0.0,False,GPU +30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1271.0,+153/-168,0.141,29.79,17.71,10.01,5150.31,1.592,0.0,False,CPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1270.0,+110/-149,0.149,29.84,19.99,12.545,664.2,1.363,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1240.0,+109/-141,0.118,31.48,18.81,12.829,664.2,0.274,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1233.0,+75/-77,0.018,31.88,24.2,11.52,26.36,0.302,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1214.0,+152/-154,0.099,32.86,21.99,10.679,5150.31,0.136,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1213.0,+175/-164,0.079,32.94,22.94,12.343,4333.58,0.422,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1208.0,+59/-44,0.0,33.18,31.5,10.665,4.3,0.318,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1185.0,+95/-138,0.053,34.38,30.6,13.964,675.17,1.541,0.0,False,CPU +38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1170.0,+178/-174,0.065,35.15,23.99,12.86,4333.58,0.037,0.0,False,CPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1143.0,+50/-59,0.0,36.48,35.38,11.888,4.06,0.299,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1141.0,+88/-140,0.037,36.59,32.81,14.568,675.17,0.206,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1136.0,+107/-134,0.036,36.85,33.44,13.467,2214.9,7.795,0.0,False,CPU +42,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1112.0,+152/-226,0.048,37.98,26.64,14.305,10.2,2.718,0.0,False,GPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1110.0,+96/-178,0.001,38.08,33.03,14.916,0.5,0.116,0.0,False,CPU +44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1094.0,+148/-172,0.001,38.8,31.92,13.954,16.01,0.041,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1077.0,+103/-140,0.02,39.57,37.02,13.971,2214.9,0.838,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1049.0,+116/-145,0.0,40.74,38.28,13.739,22.39,0.134,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+87/-150,0.0,42.69,40.63,16.312,0.63,0.12,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,959.0,+106/-170,0.0,44.17,42.75,18.644,6.97,0.818,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,954.0,+157/-376,0.0,44.34,35.99,20.561,88.43,0.512,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,853.0,+158/-384,0.0,47.34,41.05,22.9,88.43,0.051,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,762.0,+178/-559,0.0,49.41,38.18,29.906,0.42,0.047,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,594.0,+118/-414,0.0,52.04,51.78,34.296,777.57,0.312,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,579.0,+128/-434,0.0,52.23,51.95,34.339,777.57,0.108,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,396.0,+82/-395,0.0,54.0,53.97,36.314,3.91,0.126,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..f0c12b2f18ab5d04e0c9f80c4da0501d4ff062db --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4113a666737bc59c010716a87d06d940d8bac923ac6b19e09cab8f01790dbe12 +size 2146599 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/n_datasets_7 b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/n_datasets_7 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..0b503294162f6e968e4eadb689c44d40cb813bca --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed5c282d4440e740116db018c0e6ce515fc77c3f78479e08e5d802138655a14e +size 453334 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..326387a2d746df0c301be8f746f7d975094de781 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c66c1389ce1248421f63c3339fd7264e2321c98a881cff87ef915b770df74faa +size 1046899 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a9da4f243560b39a29a88b8ce770d54fda14aa34 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82f091b46d1bd9488958a1d1f4ce0e034e01a582431cce998d54a1e02dbdf23d +size 119094 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..90377fef65b435ce248793b94ebd2127cb72e3d0 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,56 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1738.0,+227/-92,0.724,8.07,4.14,2.676,669.92,5.859,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1729.0,+168/-38,0.767,8.36,3.52,3.492,4663.16,132.911,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1723.0,+332/-141,0.764,8.56,2.6,1.797,2166.18,6.341,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1699.0,+128/-34,0.721,9.38,4.73,3.903,4663.16,18.295,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1682.0,+131/-57,0.676,10.01,5.56,3.736,4823.88,33.539,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1626.0,+176/-56,0.627,12.17,4.57,4.903,47.35,39.185,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1607.0,+195/-116,0.601,13.01,5.36,3.411,6577.99,56.795,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1592.0,+287/-164,0.619,13.66,4.39,2.725,2166.18,0.812,0.0,False,GPU +8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1568.0,+116/-54,0.451,14.73,9.53,6.052,6530.9,4.363,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1565.0,+155/-64,0.462,14.89,8.5,4.84,4823.88,1.952,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1557.0,+236/-169,0.583,15.25,7.94,4.606,7.15,0.501,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1540.0,+116/-42,0.392,16.08,10.73,6.578,3558.14,0.966,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1520.0,+129/-52,0.354,17.06,12.43,6.697,3558.14,0.105,0.0,False,CPU +13,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1503.0,+250/-242,0.546,17.88,5.52,19.511,5.94,0.652,0.0,False,GPU +14,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1492.0,+191/-91,0.36,18.42,6.11,6.52,99.14,2.48,0.0,False,GPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1476.0,+131/-107,0.309,19.23,11.16,7.027,2045.39,1.388,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1475.0,+112/-68,0.303,19.3,15.45,7.822,1003.11,4.891,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1459.0,+217/-133,0.328,20.15,13.53,5.499,4158.29,1.946,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1440.0,+231/-181,0.391,21.16,9.88,5.382,6577.99,0.498,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1437.0,+138/-67,0.266,21.28,11.17,8.679,9308.22,7.885,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1428.0,+88/-37,0.229,21.77,18.45,8.1,933.42,2.759,0.0,False,CPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1412.0,+127/-98,0.246,22.61,16.94,7.406,2045.39,0.104,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1401.0,+91/-46,0.192,23.23,19.21,8.265,933.42,0.388,0.0,False,CPU +23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1399.0,+114/-70,0.201,23.35,18.34,8.659,1003.11,0.894,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1385.0,+140/-87,0.219,24.07,16.92,9.129,10.91,0.197,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1362.0,+218/-156,0.219,25.36,14.47,6.622,4158.29,0.208,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1354.0,+232/-206,0.284,25.77,13.92,7.982,9.1,0.85,0.0,False,GPU +27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1299.0,+96/-60,0.073,28.81,20.91,10.902,9308.22,0.416,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1293.0,+194/-163,0.171,29.15,18.85,13.262,623.85,1.144,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1292.0,+215/-168,0.168,29.2,20.9,8.427,15.01,0.154,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1287.0,+170/-106,0.118,29.44,22.47,8.943,23.19,8.523,0.0,False,GPU +31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1272.0,+170/-162,0.135,30.3,17.5,13.412,623.85,0.291,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1250.0,+149/-168,0.091,31.47,21.51,12.707,2929.85,0.41,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1228.0,+90/-90,0.015,32.61,24.17,12.445,27.82,0.299,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+163/-152,0.07,33.04,27.68,11.15,5692.02,1.354,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1204.0,+166/-180,0.074,33.82,22.52,13.286,2929.85,0.037,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1202.0,+47/-44,0.0,33.94,32.18,11.45,4.57,0.275,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1198.0,+126/-178,0.06,34.14,29.95,14.807,633.57,1.512,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1187.0,+188/-175,0.077,34.7,22.5,11.641,5692.02,0.131,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1161.0,+120/-186,0.042,35.98,31.92,15.352,633.57,0.248,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1142.0,+154/-178,0.055,36.87,25.15,14.894,10.27,3.409,0.0,False,GPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1142.0,+46/-60,0.0,36.87,35.63,12.598,3.79,0.311,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1131.0,+130/-155,0.001,37.4,30.44,14.18,11.19,0.038,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1128.0,+134/-196,0.002,37.53,32.07,15.73,0.47,0.117,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1085.0,+62/-94,0.0,39.46,38.31,14.849,2248.07,7.53,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1025.0,+68/-114,0.0,41.94,41.02,15.336,2248.07,0.89,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1013.0,+116/-188,0.0,42.38,40.15,14.924,24.31,0.134,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+109/-207,0.0,42.87,40.52,17.325,0.53,0.118,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,982.0,+155/-419,0.0,43.54,34.61,21.384,84.3,0.438,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,902.0,+53/-155,0.0,46.1,45.46,20.488,7.09,0.843,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,877.0,+150/-428,0.0,46.82,39.94,23.664,84.3,0.05,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,801.0,+167/-577,0.0,48.7,36.63,30.315,0.39,0.044,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,607.0,+111/-430,0.0,52.0,51.7,36.055,779.18,0.247,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,594.0,+126/-470,0.0,52.17,51.85,36.078,779.18,0.089,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,421.0,+89/-422,0.0,53.9,53.88,37.463,3.97,0.122,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..0abb8e8d5d5b845c6382929c4fcfab4e55b7a5e7 --- /dev/null +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f585c63f5038a44eb83c2e5bbeb08389834c9dcc9d9404b58c0b4dec97477850 +size 2122484 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/n_datasets_51 b/data/imputation_yes/splits_lite/tasks_all/datasets_all/n_datasets_51 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..bd929c4237d8f88f0f02daeee088af5596918860 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1939f7264edc2ccde633fc0b7dac4b139e258e5e0212942f3eb65c9320c4004d +size 476815 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..9a1544fc424271f65b3e591ef1040911973a7e11 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcf74642b27f45732f55b52cca3d37769d7a620befd9da9c4b3a44e8276f36dd +size 1088958 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..663fdef44566d6beca448604a940675cf0398566 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57ac7325b5752691a829f383e68eb3e8efe192db40ade299ae333749744dd213 +size 165567 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_all/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..e2fc4c5d22965e135b7adca5f81003dd453b1905 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1628.0,+99/-65,0.679,8.55,2.79,3.917,2059.94,9.785,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1604.0,+80/-74,0.651,9.45,4.05,6.026,556.15,6.31,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1576.0,+84/-62,0.605,10.57,4.51,6.034,2059.94,1.03,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1543.0,+67/-62,0.556,12.0,4.98,7.8,1754.94,1.767,0.0,False,CPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1539.0,+70/-51,0.559,12.17,6.36,6.68,5.71,0.611,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1507.0,+58/-50,0.496,13.71,8.01,8.918,2791.97,13.886,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1436.0,+68/-58,0.449,17.55,5.1,9.317,6154.73,386.167,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1432.0,+54/-55,0.401,17.77,9.48,10.444,2791.97,0.373,0.0,False,GPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1411.0,+66/-44,0.365,19.02,9.32,10.905,3133.91,1.273,0.0,False,GPU +9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1410.0,+39/-40,0.317,19.03,14.04,11.851,416.56,2.236,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1391.0,+75/-59,0.39,20.23,6.99,10.959,6154.73,39.452,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1390.0,+58/-44,0.32,20.26,12.91,11.444,1665.53,0.559,0.0,False,CPU +12,🧠🔁,Neural Network,[LimiX (default) [11.76% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1389.0,+73/-61,0.396,20.35,6.09,12.972,3.88,0.594,11.76,True,GPU +13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1383.0,+82/-60,0.378,20.73,8.09,11.741,4618.5,7.737,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1366.0,+52/-46,0.296,21.79,13.57,11.867,1665.53,0.065,0.0,False,CPU +15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1363.0,+39/-50,0.272,21.93,14.17,12.481,700.96,1.439,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1350.0,+51/-47,0.258,22.76,17.23,12.755,416.56,0.381,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1347.0,+60/-58,0.298,22.96,11.99,12.351,4618.5,0.47,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1346.0,+62/-53,0.289,23.03,13.65,11.839,3133.91,0.13,0.0,False,GPU +19,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1344.0,+43/-45,0.256,23.18,13.07,12.455,6.7,0.088,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1342.0,+45/-46,0.251,23.29,13.19,12.662,700.96,0.213,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1340.0,+60/-46,0.279,23.39,14.52,12.633,866.11,2.007,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1334.0,+75/-72,0.334,23.77,7.85,12.812,2942.08,17.372,35.29,True,GPU +23,🧠🔁,Neural Network,[Mitra (default) [35.29% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1310.0,+56/-67,0.281,25.35,10.45,13.616,87.34,2.433,35.29,True,GPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1291.0,+56/-45,0.202,26.64,14.41,14.011,866.11,0.097,0.0,False,GPU +25,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1288.0,+71/-72,0.284,26.81,9.54,13.65,45.42,39.406,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabICL (default) [29.41% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1284.0,+60/-55,0.248,27.1,9.5,13.464,6.86,1.52,29.41,True,GPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1283.0,+51/-50,0.221,27.14,18.82,14.241,11.56,0.127,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1268.0,+70/-58,0.231,28.09,14.28,14.474,2942.08,0.262,35.29,True,GPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1266.0,+51/-55,0.186,28.25,17.26,15.201,2961.52,0.482,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1257.0,+46/-56,0.151,28.84,22.85,13.981,2832.8,1.801,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1256.0,+43/-48,0.136,28.9,20.14,14.615,10.44,1.714,0.0,False,GPU +32,🧠⚡,Foundation Model,[BetaTabPFN (default) [25.49% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1243.0,+68/-56,0.191,29.75,17.81,15.841,202.99,1.155,25.49,True,GPU +33,🧠⚡,Foundation Model,[TabPFNv2 (default) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1232.0,+67/-79,0.21,30.48,12.04,15.328,3.27,0.315,35.29,True,GPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1218.0,+62/-50,0.132,31.4,16.55,16.847,13.74,0.316,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1217.0,+60/-59,0.141,31.47,16.17,16.032,2961.52,0.048,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1205.0,+56/-60,0.116,32.22,23.68,17.265,191.44,0.76,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1193.0,+55/-58,0.124,33.02,17.8,16.984,7.66,0.046,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1189.0,+48/-60,0.117,33.25,25.96,15.841,2832.8,0.112,0.0,False,CPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1186.0,+56/-52,0.114,33.47,22.61,15.887,2.06,0.122,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1176.0,+63/-66,0.11,34.11,24.29,17.624,594.95,4.651,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1172.0,+64/-62,0.117,34.32,20.6,18.313,191.44,0.101,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1166.0,+56/-62,0.085,34.75,26.14,18.204,377.08,0.747,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1154.0,+51/-47,0.081,35.44,30.91,16.682,2.2,0.171,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1122.0,+47/-47,0.049,37.4,31.35,19.017,377.08,0.091,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1103.0,+59/-77,0.07,38.51,27.59,19.244,594.95,0.337,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1032.0,+55/-68,0.022,42.41,37.99,21.088,8.96,0.129,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1032.0,+71/-72,0.057,42.43,27.72,23.862,3.14,0.741,0.0,False,GPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+61/-56,0.014,44.01,36.11,23.881,0.43,0.053,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default) [25.49% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,979.0,+77/-88,0.035,45.0,37.8,26.116,0.8,0.119,25.49,True,GPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,970.0,+73/-77,0.019,45.39,41.48,25.472,0.26,0.054,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,969.0,+63/-82,0.024,45.47,40.12,25.723,129.1,1.627,0.0,False,CPU +52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,966.0,+67/-78,0.022,45.57,42.03,23.185,3.12,0.312,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,898.0,+81/-109,0.024,48.35,24.73,32.354,240.73,0.308,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,868.0,+82/-109,0.017,49.46,33.4,32.94,240.73,0.068,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,805.0,+91/-127,0.008,51.41,48.35,35.498,1.23,0.115,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,804.0,+69/-92,0.011,51.43,49.38,31.855,129.1,0.103,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,589.0,+81/-131,0.0,55.84,55.51,44.446,0.19,0.037,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3f654e8d14f56cd4f1dceda4215202d682e8a748 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fde2ce642997b850951dc514c346fab73d4295e1221de8e8b15e19cbb08526c9 +size 2541132 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/n_datasets_15 b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/n_datasets_15 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..9a66337ef74b7149ae3d9edd615b9018c1911502 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caa44c98322dc9a7661b7b42a54df9e8c6aca73d02cdc57a6164d25423490879 +size 459061 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..8c38275b4c62cd14a13141ca27fdfd12e2a199f3 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90b1af57de59828870a8623866abfcbbbd79bdb8f07a4c7d77b804b23ddc3f35 +size 1008317 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..571e5f03c4d1f0cfdb417749c27fe0104660af4a --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:804bea56cec315135fbc9f1ff8ec2b332ead9138db3152430d211b96eaf6859e +size 138318 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..c022edda34d3522856934c4ced369b3fdc212cf0 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv @@ -0,0 +1,55 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1884.0,+146/-62,0.804,4.67,2.45,2.12,432.14,4.214,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1841.0,+140/-68,0.774,5.53,2.88,2.589,302.05,2.868,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1732.0,+175/-116,0.666,8.4,3.18,3.925,739.1,11.65,0.0,False,GPU +3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1689.0,+167/-130,0.642,9.8,4.58,4.338,2522.58,6.234,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1681.0,+158/-108,0.613,10.07,4.61,5.359,739.1,1.134,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1621.0,+99/-84,0.532,12.33,8.51,5.94,1714.29,1.743,0.0,False,GPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1582.0,+118/-80,0.461,13.93,8.65,6.784,763.16,0.246,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1582.0,+86/-60,0.436,13.93,10.4,7.733,129.25,2.621,0.0,False,CPU +8,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1555.0,+139/-115,0.441,15.13,11.23,7.879,2522.58,0.318,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1554.0,+109/-69,0.436,15.17,8.23,6.913,763.16,0.04,0.0,False,CPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1549.0,+95/-57,0.393,15.4,10.77,7.467,289.35,0.706,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1540.0,+150/-93,0.408,15.8,6.85,7.635,1993.57,0.581,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1529.0,+100/-84,0.364,16.33,12.67,8.359,129.25,0.117,0.0,False,CPU +13,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1522.0,+152/-115,0.415,16.67,8.21,6.533,1.91,0.668,0.0,False,GPU +14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1514.0,+132/-117,0.404,17.0,10.11,7.967,1714.29,0.079,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1512.0,+109/-84,0.41,17.1,7.62,7.185,2.93,0.045,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1495.0,+104/-81,0.355,17.93,13.2,7.701,289.35,0.08,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1458.0,+133/-120,0.353,19.73,5.46,8.104,4532.14,387.927,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1458.0,+144/-86,0.313,19.73,11.0,8.363,1993.57,0.06,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1414.0,+112/-107,0.304,21.93,15.24,9.218,559.64,0.88,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1394.0,+152/-142,0.303,22.97,8.77,10.694,4532.14,39.452,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1373.0,+104/-94,0.204,24.07,18.29,11.567,559.64,0.051,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1372.0,+124/-124,0.241,24.13,13.15,9.557,0.48,0.047,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1368.0,+120/-107,0.205,24.33,20.34,10.953,5.42,0.078,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1335.0,+83/-66,0.133,26.07,21.96,10.076,1803.23,1.199,0.0,False,CPU +25,🧠⚡,Foundation Model,[TabICL (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1326.0,+149/-121,0.195,26.53,15.87,10.541,3.98,1.593,33.33,True,GPU +26,🧠🔁,Neural Network,[LimiX (default) [40.00% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1323.0,+159/-146,0.24,26.7,5.9,14.403,1.09,0.318,40.0,True,GPU +27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1319.0,+134/-147,0.219,26.93,18.11,13.157,1535.55,0.182,0.0,False,CPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1282.0,+80/-74,0.064,28.87,26.1,12.919,6.54,0.294,0.0,False,GPU +29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1281.0,+124/-138,0.197,28.9,13.45,13.622,41.5,39.095,0.0,False,GPU +30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1273.0,+94/-95,0.117,29.33,25.0,11.636,9.07,0.224,0.0,False,GPU +31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1234.0,+97/-92,0.049,31.33,25.01,12.885,65.73,0.23,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1231.0,+82/-95,0.065,31.47,28.31,12.561,1803.23,0.051,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1224.0,+95/-118,0.055,31.8,28.99,12.624,36.73,0.359,0.0,False,CPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1218.0,+78/-94,0.044,32.13,30.42,12.051,0.34,0.07,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1210.0,+134/-142,0.131,32.53,25.81,14.45,1535.55,0.013,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1201.0,+89/-83,0.038,32.97,28.86,13.221,65.73,0.025,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1200.0,+111/-142,0.116,33.0,28.81,15.546,2.99,0.016,0.0,False,CPU +38,🧠⚡,Foundation Model,[BetaTabPFN (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1181.0,+124/-115,0.078,33.93,24.31,14.165,66.06,0.61,33.33,True,GPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1175.0,+114/-140,0.067,34.2,27.87,15.053,354.91,1.636,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1154.0,+89/-111,0.031,35.2,32.74,15.004,36.73,0.038,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1078.0,+85/-91,0.008,38.67,37.11,16.371,2.58,0.039,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1042.0,+91/-111,0.012,40.13,38.37,17.618,354.91,0.094,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1014.0,+159/-215,0.022,41.27,35.06,25.313,2.55,0.157,0.0,False,GPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+65/-106,0.0,41.8,40.8,18.863,0.24,0.022,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,969.0,+92/-124,0.012,42.93,40.74,22.593,150.58,4.952,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,932.0,+109/-151,0.016,44.2,41.74,21.122,1.49,0.113,0.0,False,CPU +47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,918.0,+111/-145,0.01,44.67,42.84,22.439,0.13,0.022,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,915.0,+167/-225,0.011,44.77,41.01,32.033,43.96,0.081,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default) [33.33% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,874.0,+95/-141,0.0,46.0,45.16,27.553,0.27,0.043,33.33,True,GPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,864.0,+152/-239,0.001,46.3,43.62,32.796,43.96,0.017,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,785.0,+59/-123,0.0,48.33,48.08,29.049,150.58,0.546,0.0,False,CPU +52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,725.0,+107/-185,0.0,49.6,49.2,34.849,0.2,0.02,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,536.0,+93/-196,0.0,52.33,52.25,42.696,0.1,0.068,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c1bd338ff97883925f84f47dd3ecc35db55b746c --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab0531860fb5a6a1c7ac3f06f669015e9ea18aeab6ac2a97f9c68423fe7097eb +size 1860124 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/n_datasets_36 b/data/imputation_yes/splits_lite/tasks_all/datasets_small/n_datasets_36 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1157b8f972b95965c998ae0b1faeef5cccabc8e8 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cad364b8cc2dbab97aa740b921e9e194a41a91d29277db65667d1a4e092d7dac +size 471941 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..8de22d41f9b84bdbedc8046b218d8f7b27403126 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7ec3ccc63dc1e8d0825467df296264478a36290a83298c69bc1aad7ca3a58be +size 1074214 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..fe1d8ad24c4edb55b02aeb077797d7a7d6abea9c --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2aca1dfc405bc95c07620d9719cb02274d59b8e30e2df8f70063ff977e55979 +size 174670 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_all/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..9bdb1545545b76158be4b8b45cd4b0056e1090cc --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1628.0,+128/-83,0.678,8.61,2.65,3.914,2319.48,8.957,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1587.0,+101/-69,0.598,10.18,5.81,6.742,6.58,0.586,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1572.0,+108/-71,0.593,10.78,4.47,6.315,2319.48,0.942,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1565.0,+80/-67,0.591,11.08,4.88,7.459,657.21,7.989,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1500.0,+69/-57,0.466,14.17,7.81,10.159,3855.22,24.935,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1482.0,+57/-52,0.444,15.06,8.72,10.167,3351.09,1.56,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1475.0,+111/-81,0.474,15.44,5.86,10.288,3560.23,49.21,8.33,True,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1453.0,+94/-72,0.469,16.64,4.96,9.823,8059.77,364.288,0.0,False,GPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1440.0,+93/-72,0.454,17.38,6.16,12.376,5.11,0.624,0.0,False,GPU +9,🧠🔁,Neural Network,[Mitra (default) [8.33% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1435.0,+89/-50,0.398,17.67,7.95,11.429,128.8,2.999,8.33,True,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1432.0,+71/-72,0.384,17.88,9.22,11.477,3855.22,0.954,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1414.0,+89/-80,0.419,18.92,6.44,11.069,8059.77,39.456,0.0,False,GPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1390.0,+70/-56,0.336,20.36,10.97,12.268,3629.74,1.48,0.0,False,GPU +13,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1377.0,+46/-43,0.251,21.15,16.43,13.566,833.03,2.182,0.0,False,CPU +14,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1371.0,+88/-72,0.327,21.56,11.17,12.643,3560.23,0.519,8.33,True,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1350.0,+61/-58,0.244,22.9,16.25,13.386,2448.79,0.814,0.0,False,CPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1334.0,+72/-63,0.259,23.89,14.22,14.055,1346.37,2.149,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1328.0,+73/-72,0.265,24.29,15.16,13.287,3629.74,0.189,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1324.0,+63/-55,0.218,24.56,18.59,13.932,2448.79,0.078,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1322.0,+57/-45,0.202,24.65,16.31,14.57,886.51,1.982,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabPFNv2 (default) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1318.0,+79/-65,0.298,24.93,9.27,13.855,4.98,0.511,8.33,True,GPU +21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1313.0,+67/-71,0.26,25.28,11.87,14.826,6341.76,8.556,0.0,False,GPU +22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1310.0,+52/-46,0.196,25.44,20.28,14.586,833.03,0.435,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1310.0,+85/-82,0.312,25.5,8.49,13.661,51.78,39.927,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1309.0,+60/-52,0.188,25.53,13.18,14.729,886.51,0.318,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1306.0,+50/-44,0.172,25.71,18.61,14.65,8.39,0.139,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1298.0,+64/-64,0.224,26.22,12.34,14.214,6341.76,0.501,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabICL (default) [27.78% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1286.0,+92/-100,0.256,27.03,8.13,14.682,7.56,1.494,27.78,True,GPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default) [22.22% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1281.0,+77/-78,0.225,27.36,15.93,16.539,269.12,1.881,22.22,True,GPU +29,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1279.0,+65/-63,0.191,27.49,13.22,15.03,1346.37,0.103,0.0,False,GPU +30,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1270.0,+66/-61,0.211,28.08,18.21,15.611,13.71,0.149,0.0,False,GPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1268.0,+68/-58,0.162,28.25,16.85,16.053,3249.24,0.548,0.0,False,CPU +32,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1262.0,+57/-60,0.149,28.58,18.33,15.321,16.6,4.299,0.0,False,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1243.0,+51/-62,0.149,29.89,23.21,15.608,3693.32,2.14,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1234.0,+65/-61,0.136,30.47,13.95,16.691,3249.24,0.059,0.0,False,CPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1208.0,+72/-71,0.119,32.15,14.49,19.018,14.9,0.323,0.0,False,GPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1206.0,+79/-77,0.135,32.28,21.98,19.199,497.58,1.147,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1203.0,+71/-60,0.113,32.47,15.3,17.583,9.93,0.052,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1188.0,+76/-91,0.125,33.4,22.89,18.695,1666.78,7.827,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1186.0,+59/-70,0.13,33.56,24.95,17.208,3693.32,0.131,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1185.0,+82/-88,0.149,33.62,17.79,19.691,497.58,0.163,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1148.0,+77/-82,0.098,35.94,26.54,20.42,575.62,1.155,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1137.0,+57/-66,0.078,36.6,30.99,18.612,3.63,0.218,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1129.0,+69/-91,0.092,37.06,24.49,19.921,1666.78,0.561,0.0,False,CPU +44,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1126.0,+53/-63,0.047,37.25,32.25,18.524,3.1,0.207,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1097.0,+63/-68,0.052,38.97,32.32,21.432,575.62,0.147,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1043.0,+90/-101,0.067,41.92,25.27,23.257,6.09,0.951,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1014.0,+64/-97,0.026,43.42,38.0,23.054,12.68,0.155,0.0,False,CPU +48,🧠⚡,Foundation Model,[TabFlex (default) [22.22% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,1010.0,+75/-83,0.05,43.61,34.89,25.516,1.59,0.149,22.22,True,GPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+69/-89,0.02,44.1,33.9,25.971,0.51,0.085,0.0,False,CPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,992.0,+76/-81,0.021,44.47,40.13,26.736,0.43,0.081,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,980.0,+58/-83,0.023,45.03,41.34,24.045,5.83,0.571,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,967.0,+70/-121,0.029,45.64,39.24,27.027,116.3,1.243,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,890.0,+85/-106,0.028,48.74,21.03,32.488,520.62,0.491,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,864.0,+93/-116,0.023,49.67,30.07,33.0,520.62,0.097,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,824.0,+102/-147,0.011,50.94,46.99,35.769,2.59,0.129,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,801.0,+79/-133,0.016,51.61,48.91,33.024,116.3,0.075,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,608.0,+97/-193,0.0,55.64,55.2,45.176,0.37,0.035,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..107bfd04a4050b35b62c92c5e5c5a373d1eb5a28 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:718f6ffef5b3775c081b2649fd75a44f2ffc768cdc1029eec7298e4e7994c234 +size 2471898 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/n_datasets_33 b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/n_datasets_33 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..54f7e40b419baa0a0cb5a59840bca37c1401cd56 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:541522fa0fecd41069da4c83a3aedb37c7f9c698b779d4934605451bde66e4ef +size 470545 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5510e2572425d91e3a7eb9413f52bed9cb9b66b0 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61314dd30fcb4829f024fe83747e5f8b0d6b71c496e9f9825cc2d8715d5e4186 +size 1076521 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7a307c956acb2af1fb95bb3008d223bc498ceab0 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa0202a88554a0103f2ca0b92184cdb9c6bf98a9adc4737190f8bae625dd0b24 +size 143442 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..11a95648b532df4e405f1545d941e79afd4e1834 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1650.0,+124/-84,0.683,8.42,2.71,4.175,2200.35,8.314,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1630.0,+84/-70,0.622,9.14,5.48,7.065,6.21,0.561,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1598.0,+113/-87,0.6,10.36,4.25,6.692,2200.35,0.882,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1570.0,+96/-80,0.581,11.52,4.86,7.982,625.63,7.662,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1536.0,+102/-76,0.517,13.08,5.44,9.632,3650.56,52.333,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1533.0,+74/-57,0.488,13.21,7.36,10.676,3565.03,23.942,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1488.0,+62/-51,0.426,15.52,8.68,10.911,3035.75,1.498,0.0,False,CPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1486.0,+86/-68,0.434,15.59,7.42,10.885,137.76,3.046,0.0,False,GPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1469.0,+92/-79,0.461,16.5,5.8,13.013,4.66,0.611,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1462.0,+96/-75,0.458,16.94,4.96,10.452,7821.64,382.96,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1456.0,+69/-67,0.4,17.23,8.75,12.039,3565.03,0.706,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1422.0,+108/-73,0.415,19.21,6.24,11.708,7821.64,39.414,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1413.0,+88/-73,0.357,19.74,10.49,12.201,3650.56,0.543,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1401.0,+81/-72,0.33,20.48,10.84,12.887,3407.57,1.449,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1370.0,+60/-45,0.213,22.35,18.6,14.631,780.12,2.129,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1361.0,+70/-63,0.239,22.95,16.22,14.299,2234.83,0.653,0.0,False,CPU +16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1352.0,+86/-61,0.325,23.52,8.67,13.532,4.97,0.545,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1334.0,+82/-72,0.253,24.65,15.76,13.965,3407.57,0.173,0.0,False,GPU +18,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1329.0,+74/-58,0.227,24.94,17.24,15.058,1216.65,2.007,0.0,False,GPU +19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1328.0,+56/-52,0.202,25.03,19.5,14.923,2234.83,0.071,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1327.0,+103/-84,0.33,25.09,7.99,14.25,51.52,40.033,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1323.0,+51/-55,0.178,25.35,20.59,15.678,848.99,1.866,0.0,False,CPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1323.0,+91/-73,0.253,25.36,16.7,15.733,6308.39,7.737,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabICL (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1316.0,+94/-98,0.279,25.8,7.59,14.435,7.68,1.615,21.21,True,GPU +24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1313.0,+58/-46,0.161,25.95,18.68,15.66,8.13,0.126,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1312.0,+77/-83,0.223,26.06,12.6,14.889,6308.39,0.477,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1309.0,+59/-50,0.165,26.24,13.96,15.82,848.99,0.298,0.0,False,CPU +27,🧠⚡,Foundation Model,[BetaTabPFN (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1306.0,+84/-85,0.235,26.42,15.22,16.996,277.28,1.727,21.21,True,GPU +28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1302.0,+72/-63,0.16,26.67,21.7,15.671,780.12,0.381,0.0,False,CPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1290.0,+73/-68,0.172,27.48,16.1,16.838,2961.52,0.521,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1287.0,+62/-54,0.163,27.67,17.51,15.815,16.29,4.082,0.0,False,GPU +31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1281.0,+77/-76,0.211,28.03,17.74,16.232,13.3,0.146,0.0,False,GPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1272.0,+68/-66,0.16,28.65,20.28,16.037,1216.65,0.097,0.0,False,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1248.0,+64/-62,0.144,30.15,23.3,16.297,3460.09,2.057,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1246.0,+74/-66,0.138,30.3,13.32,17.567,2961.52,0.054,0.0,False,CPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1221.0,+66/-59,0.117,31.89,14.0,19.358,14.5,0.3,0.0,False,GPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1220.0,+78/-78,0.11,31.94,14.62,18.183,9.77,0.046,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1204.0,+86/-83,0.134,32.98,22.16,19.428,1453.07,6.678,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1195.0,+72/-79,0.107,33.56,26.02,20.536,418.39,0.972,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1180.0,+71/-61,0.106,34.45,27.58,17.9,3460.09,0.121,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1176.0,+75/-82,0.13,34.76,19.79,20.959,418.39,0.147,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1141.0,+77/-85,0.1,36.85,23.73,20.713,1453.07,0.637,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1134.0,+74/-98,0.073,37.24,27.69,21.857,563.93,0.997,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1120.0,+51/-60,0.026,38.06,35.14,19.796,2.86,0.19,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1112.0,+51/-65,0.037,38.53,35.23,20.033,3.28,0.179,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1092.0,+70/-84,0.046,39.67,32.65,22.761,563.93,0.149,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1050.0,+89/-100,0.073,41.94,24.43,24.517,5.55,0.867,0.0,False,GPU +47,🧠⚡,Foundation Model,[TabFlex (default) [21.21% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,1024.0,+82/-95,0.054,43.27,34.1,26.178,1.29,0.149,21.21,True,GPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1012.0,+79/-77,0.028,43.85,38.13,24.002,9.39,0.15,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+82/-107,0.021,44.42,33.54,26.75,0.48,0.074,0.0,False,CPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,994.0,+86/-117,0.023,44.7,40.11,27.387,0.38,0.075,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,984.0,+77/-97,0.025,45.18,41.29,24.643,5.2,0.522,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,955.0,+74/-83,0.007,46.42,43.79,28.217,101.2,1.135,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,912.0,+86/-109,0.03,48.15,19.91,32.259,497.63,0.481,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,887.0,+84/-108,0.026,49.08,28.85,32.749,497.63,0.084,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,849.0,+98/-121,0.012,50.36,46.23,35.015,2.47,0.121,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,771.0,+84/-107,0.0,52.58,51.87,34.522,101.2,0.065,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,625.0,+106/-165,0.0,55.48,55.02,45.393,0.28,0.026,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3e9f0a42a61bff7e25a22da5071d8d9fa4e14ce4 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f12e7a9e0d7458bb5569a6d568573d8e0b8ac3a5c31b95b77c248e3d59fb9771 +size 2416990 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/n_datasets_30 b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/n_datasets_30 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a2cca212f7d1a952f819750ab74749a9532d5dce --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:347ece1eab807333dda6cc4f437de798dbf715b69556ebe6dd3f51f39af0c99a +size 492020 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..10fe889affac1bcfe1b079afd9e98a40a92e566d --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7371fdfbb3a8c478a67a430f9b7ecb37635c048604130741c840d99b8623e64a +size 1062154 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1b4dc0fc1c29b632da50c130bdd9e8179a8ed593 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f045000dff91c57f8134ad8eb8bb3956854b487b51cf5a3c2b7cf1cbc2e1d2a +size 183658 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..9bcca32f6b2b8a324fb2f08ae31bebfd2ab5c4be --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1568.0,+105/-74,0.589,10.57,3.67,5.636,1877.76,10.745,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1562.0,+112/-85,0.623,10.8,3.7,6.142,442.6,6.737,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1547.0,+99/-70,0.567,11.47,4.88,7.435,1231.21,1.36,0.0,False,CPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1546.0,+95/-69,0.567,11.5,6.22,7.006,4.79,0.708,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1537.0,+102/-63,0.551,11.92,5.24,6.667,1877.76,1.035,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1500.0,+69/-60,0.477,13.73,8.61,9.187,2721.85,9.311,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1424.0,+76/-72,0.388,17.92,8.05,11.154,2721.85,0.265,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1412.0,+81/-64,0.382,18.67,7.44,11.026,2180.12,1.135,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabICL (default) [3.33% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1394.0,+82/-68,0.349,19.78,7.26,10.012,8.05,2.008,3.33,True,GPU +9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1388.0,+52/-50,0.292,20.12,14.44,12.615,328.64,0.774,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1374.0,+71/-61,0.34,21.03,7.84,9.545,5890.6,409.986,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1368.0,+62/-51,0.304,21.4,10.27,12.209,3.84,0.068,0.0,False,CPU +12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1367.0,+70/-70,0.302,21.45,11.5,13.18,462.92,0.613,0.0,False,CPU +13,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1366.0,+87/-76,0.331,21.52,10.11,12.046,3436.74,0.405,0.0,False,GPU +14,🧠🔁,Neural Network,[LimiX (default) [16.67% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1362.0,+93/-84,0.365,21.75,6.22,13.114,1.95,0.488,16.67,True,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1358.0,+69/-73,0.302,22.05,12.29,12.343,1043.89,0.478,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1350.0,+94/-95,0.353,22.52,8.36,12.196,3436.74,8.478,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1340.0,+85/-73,0.305,23.22,12.12,12.161,2180.12,0.117,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1332.0,+56/-61,0.276,23.72,14.78,12.867,1043.89,0.041,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1332.0,+76/-80,0.281,23.72,9.4,11.754,5890.6,39.565,0.0,False,GPU +20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1332.0,+95/-64,0.275,23.73,13.6,14.412,248.47,1.7,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1331.0,+75/-80,0.262,23.77,10.5,13.559,462.92,0.108,0.0,False,CPU +22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1320.0,+58/-62,0.23,24.53,19.55,13.786,328.64,0.089,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1315.0,+94/-79,0.304,24.85,9.85,14.164,2914.83,17.91,33.33,True,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1311.0,+68/-57,0.213,25.1,15.6,14.601,1579.5,0.428,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1304.0,+69/-82,0.224,25.6,17.15,13.468,559.77,1.513,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1299.0,+58/-53,0.175,25.9,16.67,13.707,9.04,1.162,0.0,False,GPU +27,🧠🔁,Neural Network,[Mitra (default) [33.33% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1296.0,+102/-77,0.272,26.07,9.31,14.387,106.33,2.667,33.33,True,GPU +28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1285.0,+89/-78,0.24,26.83,17.89,14.697,8.0,0.124,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1273.0,+73/-55,0.179,27.63,20.43,13.865,2206.58,2.301,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1262.0,+91/-73,0.221,28.38,13.63,15.933,2914.83,0.225,33.33,True,GPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1258.0,+63/-69,0.148,28.68,20.29,15.134,559.77,0.086,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1257.0,+67/-57,0.14,28.73,13.78,15.369,1579.5,0.05,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1251.0,+70/-60,0.15,29.1,13.32,16.377,4.79,0.045,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1222.0,+72/-90,0.179,31.08,11.5,14.951,47.89,39.446,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1218.0,+76/-88,0.139,31.35,21.38,16.897,561.29,4.462,0.0,False,CPU +36,🧠⚡,Foundation Model,[TabPFNv2 (default) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1216.0,+78/-78,0.196,31.48,10.46,17.278,2.61,0.256,33.33,True,GPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1200.0,+85/-80,0.141,32.53,18.88,16.455,1.42,0.118,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1191.0,+77/-69,0.131,33.07,25.9,16.272,2206.58,0.107,0.0,False,CPU +39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1188.0,+63/-62,0.111,33.25,13.68,17.287,11.15,0.312,0.0,False,GPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1173.0,+74/-68,0.081,34.25,27.98,18.237,122.9,0.583,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1140.0,+68/-85,0.061,36.3,29.95,19.635,171.61,0.552,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1138.0,+79/-101,0.096,36.47,23.6,19.002,561.29,0.236,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1135.0,+63/-66,0.082,36.65,32.0,17.467,0.93,0.094,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1130.0,+79/-75,0.086,36.97,21.52,19.849,122.9,0.072,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1106.0,+64/-69,0.021,38.42,34.53,20.285,171.61,0.053,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1044.0,+74/-69,0.035,41.93,36.06,21.546,4.97,0.092,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1010.0,+104/-121,0.059,43.7,33.56,26.685,0.81,0.115,0.0,False,GPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+76/-92,0.024,44.18,32.57,24.447,0.29,0.031,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,998.0,+75/-93,0.031,44.27,40.03,22.018,2.79,0.259,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,991.0,+63/-92,0.006,44.63,42.41,24.644,140.66,3.209,0.0,False,CPU +51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,973.0,+93/-126,0.041,45.47,18.11,28.274,155.53,0.277,0.0,False,CPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,941.0,+96/-133,0.03,46.88,26.66,28.935,155.53,0.05,0.0,False,CPU +53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,930.0,+91/-113,0.037,47.33,41.1,27.199,2.71,0.392,0.0,False,GPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,924.0,+92/-110,0.016,47.6,43.29,26.269,0.18,0.035,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,904.0,+98/-113,0.013,48.4,44.34,30.667,0.8,0.088,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,827.0,+62/-96,0.0,51.03,50.24,30.916,140.66,0.188,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,586.0,+96/-207,0.0,56.0,55.7,43.999,0.15,0.024,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..dd456d41fd4cecf92cabf16557d18bf051c996d2 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0981a5d5ab05f632ac0ae85de016bf0a95549ab0d5dfd2423f885cd6ad7bb030 +size 2329337 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/n_datasets_9 b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/n_datasets_9 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..b88c38a35d7a3bedcd9481cab8c455f9452c1376 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c4da7f7b53bb4f19ba91748131ea59211a4eaf3b7ac7a4506dacdf44a2665cb +size 448234 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..263d5c1ab0148cfdd44222c98d71c1c01e3030c4 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:576e294afa617e55c7e28cae2cea0eb3a304e0eb873e1dca1d2ca7514aefbfb3 +size 1001999 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e4277058d79cee575d51c48ea8af0db92abcf125 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9aa94612b23003628b74e8131f5e534773f14abc331137384b2d26f0591203d7 +size 134935 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..944f59a8f5fd947d06096b4b0069d6ceec6ab4c8 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv @@ -0,0 +1,55 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,2055.0,+219/-66,0.859,3.44,2.17,1.559,310.55,1.824,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1995.0,+270/-81,0.834,4.33,2.22,2.117,256.14,2.868,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1747.0,+274/-161,0.619,10.78,3.85,4.475,2149.06,6.373,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1729.0,+219/-166,0.527,11.44,4.12,5.896,841.84,11.882,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1714.0,+130/-48,0.517,12.0,8.92,5.584,1655.56,2.483,0.0,False,GPU +5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1708.0,+186/-105,0.46,12.22,8.36,7.366,114.48,0.404,0.0,False,CPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1689.0,+204/-112,0.517,13.0,7.63,6.578,618.23,0.184,0.0,False,CPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1675.0,+224/-137,0.452,13.56,7.63,7.894,841.84,1.263,0.0,False,GPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1670.0,+201/-118,0.478,13.78,5.14,7.403,807.72,0.575,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1661.0,+168/-101,0.489,14.17,8.72,6.676,618.23,0.031,0.0,False,CPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1660.0,+186/-100,0.43,14.22,10.24,8.502,87.3,0.533,0.0,False,CPU +11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1644.0,+181/-117,0.418,14.89,10.64,9.317,2149.06,0.318,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1635.0,+152/-84,0.46,15.28,5.61,6.592,2.12,0.043,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1617.0,+163/-118,0.403,16.11,10.87,7.688,114.48,0.058,0.0,False,CPU +14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1609.0,+189/-106,0.394,16.44,9.39,8.195,1655.56,0.086,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1581.0,+177/-114,0.326,17.78,13.98,9.447,87.3,0.069,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1578.0,+230/-158,0.396,17.89,8.37,8.051,807.72,0.07,0.0,False,GPU +17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1548.0,+182/-140,0.333,19.33,9.01,7.804,2.32,0.889,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1537.0,+176/-155,0.29,19.89,11.67,9.501,9.96,4.847,0.0,False,GPU +19,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1474.0,+173/-195,0.249,23.0,18.82,10.609,5.77,0.078,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1461.0,+174/-196,0.26,23.67,10.33,10.169,0.48,0.047,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1461.0,+158/-145,0.248,23.67,15.34,9.627,559.89,1.156,0.0,False,GPU +22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1461.0,+119/-70,0.179,23.67,19.26,9.923,1128.19,1.674,0.0,False,CPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1444.0,+183/-128,0.215,24.56,20.49,13.232,262.92,0.205,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1442.0,+169/-148,0.186,24.67,18.8,10.124,4560.06,421.968,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1431.0,+163/-120,0.166,25.22,17.58,12.808,559.89,0.056,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1412.0,+85/-76,0.087,26.22,23.87,11.884,6.55,0.295,0.0,False,GPU +27,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1380.0,+112/-153,0.112,27.89,22.51,12.385,368.88,2.507,0.0,False,CPU +28,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1366.0,+202/-171,0.132,28.61,19.08,13.822,4560.06,39.452,0.0,False,GPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1338.0,+181/-190,0.13,30.0,19.31,13.509,92.9,0.786,0.0,False,GPU +30,🧠🔁,Neural Network,[LimiX (default) [55.56% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1338.0,+244/-253,0.218,30.0,6.8,18.727,0.68,0.014,55.56,True,GPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1323.0,+165/-119,0.077,30.78,27.72,16.195,0.92,0.018,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1314.0,+200/-159,0.097,31.22,25.64,14.917,262.92,0.015,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1307.0,+122/-138,0.083,31.56,27.8,13.503,1128.19,0.055,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1300.0,+123/-154,0.036,31.89,28.72,13.474,40.45,0.331,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1294.0,+55/-72,0.018,32.22,31.24,14.752,43.93,0.204,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1266.0,+109/-148,0.047,33.56,31.32,12.579,0.34,0.052,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1251.0,+51/-62,0.0,34.28,33.94,15.173,43.93,0.018,0.0,False,CPU +38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1230.0,+108/-121,0.015,35.22,33.77,14.121,9.93,0.224,0.0,False,GPU +39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1210.0,+137/-213,0.022,36.17,32.49,18.236,41.8,39.131,0.0,False,GPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1194.0,+128/-148,0.02,36.89,34.1,17.071,40.45,0.028,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1178.0,+108/-183,0.02,37.56,35.52,16.535,368.88,0.084,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1142.0,+101/-139,0.013,39.11,37.42,16.268,1.81,0.048,0.0,False,CPU +43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1104.0,+173/-273,0.019,40.61,36.3,25.579,43.96,0.099,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1102.0,+114/-195,0.02,40.67,37.79,20.929,514.21,52.341,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1067.0,+137/-238,0.026,42.0,38.71,19.282,1.61,0.113,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1056.0,+170/-258,0.002,42.39,39.36,26.315,43.96,0.024,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+121/-272,0.0,44.28,43.24,22.467,0.11,0.012,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,896.0,+238/-618,0.027,47.22,39.51,30.82,2.77,0.157,0.0,False,GPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,887.0,+98/-215,0.0,47.44,47.08,28.058,0.22,0.022,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,859.0,+70/-166,0.0,48.11,47.78,28.846,514.21,2.49,0.0,False,CPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,859.0,+102/-228,0.0,48.11,47.82,25.047,0.1,0.02,0.0,False,CPU +52,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,823.0,+121/-262,0.0,48.89,48.33,33.192,0.2,0.035,0.0,False,GPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,488.0,+95/-357,0.0,53.11,53.08,46.138,0.11,0.089,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e56ea89150a6c6c33aeadc625bc3f1892c566b74 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6941472102817838bcb51720593592be79ef6a4ee9b400ca3c905e5cb43a25f6 +size 1568808 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/n_datasets_21 b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/n_datasets_21 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c33175fabb9ae8f8367e4e8b2827b1eaeea2ba1b --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0db8c3ca1b450190cc5a5fca59c91385075ab62dbe2a81a0a983b5fd64eb882 +size 508699 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5bd36e5f586d2f77cc688ab1c0d5c48ee8b9ee41 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b96111c03fc5ef9f581c67a6fc330810645ae97ebde9b220693457565914cd78 +size 1029384 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..6262d304a2840429e4463ec39039b9f5c92a1c1a --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4a53f90fe84061c32a5d9b6b538b5e214ec4af58add43f9309860f9c7f08718 +size 163970 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..a1f8028ab04dfc937cc06aa7fca357be2640081d --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1622.0,+93/-51,0.644,7.95,5.49,6.664,5.53,0.55,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1562.0,+146/-85,0.606,10.19,3.5,5.525,2125.17,9.785,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1538.0,+139/-103,0.583,11.21,4.62,6.141,2125.17,0.882,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1488.0,+128/-92,0.525,13.57,5.17,7.867,613.33,8.472,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1470.0,+81/-86,0.441,14.48,8.48,10.731,2791.97,23.942,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1462.0,+91/-82,0.435,14.9,10.49,9.953,1886.83,1.333,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1447.0,+142/-74,0.434,15.69,7.38,10.606,3275.74,52.333,4.76,True,GPU +7,🧠🔁,Neural Network,[Mitra (default) [4.76% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1417.0,+139/-66,0.388,17.4,6.94,10.924,145.41,3.958,4.76,True,GPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1412.0,+107/-82,0.425,17.69,5.99,10.708,4.14,0.557,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1401.0,+111/-76,0.365,18.36,7.57,12.422,2791.97,0.544,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1382.0,+100/-80,0.382,19.48,6.28,9.297,6616.87,408.881,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabICL (default) [4.76% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1378.0,+81/-87,0.353,19.74,6.25,10.231,7.68,1.811,4.76,True,GPU +12,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1363.0,+114/-83,0.314,20.67,12.04,14.798,298.33,2.036,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1362.0,+102/-68,0.315,20.74,10.49,13.132,3275.74,0.512,4.76,True,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1361.0,+104/-87,0.334,20.76,9.2,12.578,2237.77,1.273,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1351.0,+96/-92,0.339,21.43,7.72,10.868,6616.87,39.713,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1332.0,+66/-54,0.219,22.64,17.52,14.378,512.11,1.17,0.0,False,CPU +17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1310.0,+81/-60,0.216,24.02,15.97,14.616,4.51,0.124,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1305.0,+138/-85,0.279,24.36,9.9,13.216,4618.5,0.426,0.0,False,GPU +19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1296.0,+92/-85,0.196,24.95,14.11,15.188,1640.58,0.552,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabPFNv2 (default) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1294.0,+109/-93,0.28,25.14,7.87,15.054,4.16,0.543,4.76,True,GPU +21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1291.0,+114/-102,0.256,25.31,14.98,13.922,2237.77,0.147,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1290.0,+88/-79,0.216,25.4,13.7,15.671,700.96,0.743,0.0,False,CPU +23,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1287.0,+80/-61,0.185,25.57,14.74,14.488,11.09,2.46,0.0,False,GPU +24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1282.0,+89/-84,0.194,25.93,16.64,14.814,1103.84,0.559,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1274.0,+82/-84,0.208,26.43,18.06,15.114,536.23,1.738,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1265.0,+79/-77,0.186,27.05,10.35,16.075,700.96,0.203,0.0,False,CPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1261.0,+77/-68,0.147,27.29,11.48,15.562,1640.58,0.059,0.0,False,CPU +28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1259.0,+68/-70,0.173,27.43,23.57,15.646,512.11,0.122,0.0,False,CPU +29,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1257.0,+114/-91,0.233,27.55,16.83,15.505,4618.5,9.218,0.0,False,GPU +30,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1253.0,+85/-82,0.168,27.81,21.05,15.521,1103.84,0.051,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1250.0,+90/-86,0.162,28.0,10.87,16.455,7.16,0.046,0.0,False,CPU +32,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1246.0,+98/-87,0.219,28.29,17.5,16.449,11.3,0.149,0.0,False,GPU +33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1245.0,+106/-102,0.239,28.33,8.98,13.544,52.04,39.82,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1230.0,+98/-88,0.175,29.33,20.98,15.554,2535.0,2.546,0.0,False,CPU +35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1221.0,+86/-94,0.137,29.98,21.67,16.131,536.23,0.097,0.0,False,GPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1187.0,+108/-107,0.133,32.21,10.89,18.644,13.39,0.329,0.0,False,GPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1180.0,+101/-110,0.147,32.64,20.89,18.831,650.63,5.616,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1170.0,+94/-93,0.146,33.33,25.03,17.459,2535.0,0.112,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1139.0,+87/-99,0.096,35.26,27.67,20.279,206.04,0.946,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1134.0,+104/-151,0.124,35.62,20.56,20.059,650.63,0.442,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1125.0,+83/-84,0.081,36.14,29.19,19.149,1.98,0.122,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1118.0,+108/-111,0.114,36.62,18.51,21.039,206.04,0.098,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1098.0,+74/-90,0.078,37.79,32.21,19.563,2.47,0.168,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1097.0,+93/-108,0.079,37.86,29.32,21.728,494.48,0.936,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1061.0,+86/-101,0.028,40.0,34.61,22.476,494.48,0.093,0.0,False,CPU +46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1051.0,+115/-148,0.085,40.52,29.4,23.895,1.29,0.149,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1010.0,+91/-135,0.044,42.76,35.3,23.809,7.76,0.147,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+105/-148,0.034,43.29,29.07,25.296,0.39,0.041,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,976.0,+86/-127,0.029,44.48,40.11,23.191,3.48,0.427,0.0,False,CPU +50,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,947.0,+100/-143,0.041,45.86,40.9,25.646,2.66,0.702,0.0,False,GPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,947.0,+119/-146,0.023,45.86,40.57,26.793,0.33,0.044,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,944.0,+72/-114,0.0,45.95,44.39,26.237,89.61,1.627,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,926.0,+112/-152,0.048,46.79,14.84,29.429,280.42,0.382,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,900.0,+118/-175,0.018,47.86,42.54,31.785,1.48,0.127,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,895.0,+118/-156,0.04,48.05,23.27,30.058,280.42,0.072,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,793.0,+89/-139,0.0,51.52,50.65,31.804,89.61,0.08,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,608.0,+105/-184,0.0,55.52,55.12,43.082,0.24,0.02,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..115d16fdb282c66d79bdbe87e9e97c200ca1a79c --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ef3f834103fbcaaca88f44b2c8937811053039c448ccccf07ef905a7f1e56a1 +size 2224691 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/n_datasets_20 b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/n_datasets_20 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c5f786ed8931af32da1b1ceefd9a0f40cccfd79e --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54f2d456c5e55daf68dbb4d06fe05b21738409be336bff93efdc067ef59b02b9 +size 495281 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4f31edf0b96f16fb5caf66659deaa8395ab34e9e --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69e553175190835e8dbe1afecee9bf7afc9a9cb75a17a5cf4476e9e22333ca64 +size 1035456 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5294175492e4154e77b8ce6df9d901b36825dcae --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa24b7ecdb3434e6e7af68075c969e22d0f6891e4ce5b323cdc3cd862e13c644 +size 118622 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..c424e67f04943ef0c170f2786ba8b9c0ddad8d85 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1644.0,+93/-60,0.637,7.8,5.36,6.865,5.25,0.517,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1567.0,+153/-98,0.586,10.65,4.0,5.801,2092.56,8.655,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1556.0,+140/-99,0.575,11.12,4.48,6.297,2092.56,0.867,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1493.0,+130/-78,0.506,14.1,5.36,8.203,598.6,8.421,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1492.0,+91/-82,0.441,14.2,8.25,10.934,2773.9,21.391,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1476.0,+124/-90,0.456,15.02,7.12,10.58,3361.32,65.579,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1471.0,+85/-74,0.413,15.3,10.76,10.37,1881.65,1.26,0.0,False,CPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1443.0,+115/-72,0.407,16.82,6.69,10.913,145.75,4.071,0.0,False,GPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1426.0,+123/-84,0.419,17.77,5.81,10.972,4.09,0.544,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1420.0,+120/-99,0.371,18.18,7.33,12.618,2773.9,0.534,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1402.0,+115/-88,0.33,19.25,11.6,14.276,292.15,1.882,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1401.0,+118/-76,0.371,19.27,6.01,10.185,8.05,1.864,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1399.0,+132/-82,0.38,19.4,6.06,9.42,6408.95,397.026,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1389.0,+117/-88,0.351,20.05,8.88,12.618,2217.3,1.258,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1384.0,+124/-88,0.331,20.32,10.16,13.232,3361.32,0.527,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1368.0,+131/-93,0.347,21.3,7.46,10.945,6408.95,39.606,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1334.0,+58/-48,0.186,23.52,20.03,15.024,457.0,1.064,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1333.0,+110/-76,0.293,23.58,9.54,13.112,4442.23,0.405,0.0,False,GPU +18,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1320.0,+84/-79,0.206,24.4,13.69,15.346,1618.79,0.502,0.0,False,CPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1315.0,+112/-89,0.269,24.72,14.55,13.982,2217.3,0.137,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1315.0,+77/-60,0.186,24.72,16.46,15.224,4.49,0.115,0.0,False,CPU +21,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1314.0,+80/-69,0.194,24.8,14.28,14.446,10.77,2.16,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1312.0,+104/-98,0.295,24.95,7.59,15.25,3.89,0.544,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1287.0,+83/-75,0.178,26.58,19.37,16.445,673.88,0.681,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1284.0,+85/-81,0.162,26.78,17.38,15.452,1103.74,0.549,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1281.0,+88/-92,0.186,27.0,18.24,15.662,516.54,1.513,0.0,False,GPU +26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1278.0,+83/-91,0.17,27.2,10.48,16.268,6.77,0.046,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1276.0,+99/-108,0.245,27.28,16.43,15.707,4442.23,8.478,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1275.0,+83/-79,0.154,27.4,11.18,15.789,1618.79,0.055,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1272.0,+120/-112,0.23,27.55,17.0,16.267,10.18,0.149,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1268.0,+136/-120,0.251,27.85,8.65,13.464,51.78,39.866,0.0,False,GPU +31,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1264.0,+74/-64,0.146,28.1,24.41,16.249,457.0,0.111,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1262.0,+76/-68,0.15,28.2,11.24,16.819,673.88,0.177,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1255.0,+98/-87,0.183,28.7,20.47,15.396,2533.55,2.301,0.0,False,CPU +34,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1252.0,+87/-54,0.133,28.9,24.07,16.217,1103.74,0.05,0.0,False,CPU +35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1227.0,+80/-86,0.12,30.58,21.9,16.637,516.54,0.086,0.0,False,GPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1212.0,+84/-118,0.139,31.52,10.49,18.467,11.78,0.312,0.0,False,GPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1204.0,+96/-119,0.154,32.02,20.34,18.721,622.79,5.389,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1196.0,+97/-94,0.153,32.6,24.44,17.128,2533.55,0.11,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1157.0,+98/-138,0.13,35.05,20.0,19.935,622.79,0.398,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1141.0,+94/-87,0.076,36.08,28.32,20.989,204.47,0.853,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1121.0,+117/-112,0.109,37.3,18.33,21.645,204.47,0.091,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1117.0,+81/-93,0.043,37.55,33.65,20.016,1.88,0.121,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1094.0,+98/-125,0.057,38.9,30.42,22.532,448.77,0.841,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1091.0,+80/-100,0.043,39.08,35.18,20.402,2.39,0.159,0.0,False,CPU +45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1078.0,+103/-162,0.089,39.85,28.74,23.197,1.17,0.148,0.0,False,GPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1065.0,+93/-110,0.03,40.55,34.95,23.044,448.77,0.092,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1033.0,+109/-126,0.046,42.3,34.74,23.554,7.36,0.138,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+98/-164,0.035,44.0,29.07,26.004,0.37,0.04,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,997.0,+107/-123,0.031,44.15,39.68,22.94,3.3,0.395,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,963.0,+65/-111,0.0,45.75,44.14,26.281,89.46,1.579,0.0,False,CPU +51,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,953.0,+121/-156,0.044,46.2,41.0,26.17,2.66,0.553,0.0,False,GPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,949.0,+114/-129,0.05,46.38,14.32,29.0,276.0,0.368,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,948.0,+121/-161,0.024,46.45,40.96,27.548,0.29,0.041,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,926.0,+106/-145,0.019,47.35,41.98,31.115,1.45,0.123,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,920.0,+122/-145,0.042,47.6,22.6,29.565,276.0,0.065,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,810.0,+84/-146,0.0,51.45,50.54,31.853,89.46,0.075,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,624.0,+110/-269,0.0,55.5,55.08,43.289,0.22,0.02,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..711594a5a08091f45d1ddde524750d80cc59f30f --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9348f5b45a35aff4ebe40e59f58cbe2981206a9e9cd51cb980f4a2a0897d1ed8 +size 2250455 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/n_datasets_38 b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/n_datasets_38 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..aa766d692d52efca9b981957412542b8650dd1cc --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b130297b67c3e5f46ba1c4098b5ed92eac7adb86888b248ace20c3e67a672a2 +size 488004 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..9d2bd597eeb265e325bda157308cb1e54fedb008 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdd380009c6b6b95b7e3ecb69692e40263e270bdb21072b7d9e7598e08c64c3d +size 1059023 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e9d665c49a3f6edb4f26654db34e9fd4cf821918 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c669cc744b65e418aa7d2c48d99bbad906d3680ad754ac08d807cc8a049a4e9 +size 175033 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..852127de0d77c8b9725f0ada6d968b728cc86864 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1590.0,+102/-84,0.632,9.61,2.92,4.83,2063.62,9.938,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1577.0,+90/-78,0.643,10.13,3.95,7.182,521.51,7.239,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1555.0,+94/-77,0.577,11.08,5.09,7.35,2063.62,1.087,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1537.0,+80/-70,0.559,11.88,6.08,7.714,5.62,0.771,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1525.0,+94/-61,0.552,12.42,4.74,8.832,1815.7,1.456,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1480.0,+56/-49,0.461,14.68,7.76,10.513,2773.9,16.363,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1415.0,+53/-51,0.381,18.38,8.94,12.166,2773.9,0.39,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1404.0,+75/-60,0.369,19.08,8.25,12.641,2461.81,1.283,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1389.0,+56/-44,0.297,19.96,14.18,13.379,391.23,1.243,0.0,False,CPU +9,🧠🔁,Neural Network,[LimiX (default) [15.79% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1384.0,+73/-74,0.384,20.29,6.54,12.947,3.46,0.517,15.79,True,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1373.0,+95/-58,0.347,21.0,8.39,11.63,6067.13,404.225,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1369.0,+72/-56,0.314,21.22,12.73,12.907,1326.58,0.531,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabICL (default) [5.26% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1367.0,+79/-72,0.332,21.37,7.5,12.807,9.07,2.008,5.26,True,GPU +13,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1360.0,+69/-72,0.284,21.83,12.73,13.973,682.62,0.971,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1350.0,+64/-68,0.3,22.47,11.37,13.402,4853.07,0.473,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1347.0,+66/-56,0.289,22.67,14.9,13.432,1326.58,0.051,0.0,False,CPU +16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1346.0,+50/-54,0.272,22.74,11.73,13.55,5.74,0.107,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1344.0,+83/-62,0.307,22.86,12.22,13.609,2461.81,0.138,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1342.0,+93/-83,0.326,22.97,9.57,13.448,4853.07,9.386,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1334.0,+93/-80,0.334,23.51,8.96,15.007,2914.83,17.597,31.58,True,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1333.0,+64/-47,0.249,23.58,17.69,14.32,391.23,0.186,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1331.0,+63/-69,0.256,23.68,11.74,14.222,682.62,0.183,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1323.0,+75/-74,0.285,24.22,8.48,13.66,6067.13,39.565,0.0,False,GPU +23,🧠🔁,Neural Network,[Mitra (default) [31.58% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1316.0,+86/-83,0.295,24.66,9.93,15.339,118.41,2.667,31.58,True,GPU +24,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1312.0,+66/-58,0.256,24.92,14.82,15.997,250.94,2.036,0.0,False,GPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1310.0,+69/-58,0.239,25.11,14.61,14.71,1053.72,2.067,0.0,False,GPU +26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1303.0,+63/-51,0.207,25.53,15.73,15.784,2399.23,0.558,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1282.0,+72/-66,0.237,26.95,17.43,16.264,10.31,0.136,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1273.0,+76/-76,0.233,27.54,13.68,16.721,2914.83,0.271,31.58,True,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1264.0,+54/-51,0.169,28.13,21.55,15.515,2480.32,2.175,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1260.0,+67/-50,0.152,28.42,18.64,16.137,10.77,1.787,0.0,False,GPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1258.0,+70/-56,0.166,28.59,13.64,16.368,1053.72,0.105,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1249.0,+66/-51,0.146,29.18,15.17,16.707,2399.23,0.056,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1242.0,+59/-54,0.147,29.63,14.98,17.609,6.77,0.052,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabPFNv2 (default) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1239.0,+82/-80,0.216,29.83,10.67,17.473,3.3,0.305,31.58,True,GPU +35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1212.0,+81/-70,0.186,31.62,11.33,16.869,47.89,39.446,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1210.0,+68/-91,0.145,31.75,21.46,18.74,622.79,5.037,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1194.0,+64/-63,0.114,32.82,23.16,18.948,197.16,0.721,0.0,False,CPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1193.0,+65/-71,0.126,32.84,20.53,17.316,1.88,0.121,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1192.0,+69/-62,0.13,32.89,24.82,17.8,2480.32,0.117,0.0,False,CPU +40,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1170.0,+65/-60,0.092,34.36,15.73,19.393,13.35,0.353,0.0,False,GPU +41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1167.0,+67/-75,0.104,34.53,24.3,19.874,327.31,0.708,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1155.0,+66/-76,0.113,35.3,19.39,20.247,197.16,0.083,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1142.0,+57/-62,0.092,36.12,30.52,18.628,2.25,0.159,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1127.0,+68/-78,0.093,37.03,24.79,20.748,622.79,0.346,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1124.0,+58/-60,0.056,37.2,30.05,20.709,327.31,0.085,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1030.0,+62/-79,0.028,42.55,37.41,23.445,5.73,0.148,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+61/-86,0.019,44.07,34.16,26.787,0.37,0.038,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,995.0,+74/-102,0.032,44.29,38.03,27.556,140.66,2.876,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,986.0,+72/-69,0.029,44.76,40.61,24.653,3.12,0.334,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,972.0,+82/-138,0.047,45.39,36.21,29.787,1.04,0.192,0.0,False,GPU +51,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,966.0,+90/-91,0.042,45.66,39.33,27.581,3.48,0.677,0.0,False,GPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,961.0,+83/-99,0.032,45.88,20.78,31.022,256.15,0.36,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,932.0,+89/-110,0.023,47.13,29.33,31.745,256.15,0.07,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,917.0,+77/-108,0.014,47.74,43.73,29.31,0.24,0.037,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,872.0,+91/-108,0.01,49.42,45.92,34.338,1.32,0.117,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,819.0,+93/-118,0.015,51.11,48.48,35.052,140.66,0.188,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,540.0,+117/-145,0.0,56.42,56.17,49.668,0.18,0.028,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..75bfb93719a7bb3d8ede5fd6f77df6381a08aea6 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6eaeef65c81a4ffb8dcdaa9f12c8756cc18139f7e288c01c13699f3895c073a1 +size 2410880 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/n_datasets_10 b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/n_datasets_10 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..9a1a6bf053b00b49ca2112e6602b214f987e795d --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b8b5c0fa95416cd837832e3dd764fcb30899666cfa80da3612e60fd8bb5524f +size 444464 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..082a11eaa1c273be69588fb7ac5ba21f62cafbff --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35877a053515c3156deffd9b9d92b1bfedff657eb1d515c8161ea82c50e6faf4 +size 1021051 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..543421cebe4e5c3072378c19e85320258f045e4e --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7e3b95acfd3931bbbf4b71cc2b64849dde90be9d0968a7bd4e071d999bef443 +size 133309 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..ec5b948ba88c15e72207926674febfe2839aeec6 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv @@ -0,0 +1,55 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,2023.0,+176/-73,0.867,3.3,2.15,1.576,307.34,2.186,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1959.0,+218/-71,0.842,4.2,2.28,2.126,266.49,2.963,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1712.0,+263/-137,0.575,10.4,3.14,5.306,780.95,11.64,0.0,False,GPU +3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1684.0,+240/-162,0.601,11.4,4.17,5.366,2335.82,14.145,0.0,False,GPU +4,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1661.0,+156/-106,0.467,12.3,8.67,7.774,202.53,0.43,0.0,False,CPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1654.0,+240/-119,0.495,12.6,6.99,7.407,780.95,1.493,0.0,False,GPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1646.0,+177/-92,0.519,12.9,7.92,7.059,688.25,0.192,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1632.0,+173/-79,0.445,13.5,9.79,8.682,91.02,0.568,0.0,False,CPU +8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1626.0,+168/-86,0.494,13.75,8.83,7.132,688.25,0.036,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1615.0,+129/-89,0.466,14.2,9.63,7.485,1635.73,2.146,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1592.0,+183/-102,0.43,15.2,5.6,8.859,1400.65,0.571,0.0,False,GPU +11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1590.0,+170/-111,0.417,15.3,11.13,9.79,2335.82,0.502,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1589.0,+136/-69,0.459,15.35,6.0,7.247,2.2,0.044,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1576.0,+176/-103,0.414,15.9,11.12,8.091,202.53,0.055,0.0,False,CPU +14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1557.0,+174/-105,0.352,16.8,13.01,9.535,91.02,0.074,0.0,False,CPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1525.0,+159/-111,0.355,18.3,10.13,9.894,1635.73,0.082,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1511.0,+196/-166,0.356,19.0,9.01,9.451,1400.65,0.065,0.0,False,GPU +17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1502.0,+191/-126,0.337,19.4,9.54,8.488,2.3,0.818,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1490.0,+167/-135,0.292,20.0,12.21,10.137,8.46,3.879,0.0,False,GPU +19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1439.0,+205/-127,0.253,22.6,15.64,12.923,899.24,0.221,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1435.0,+228/-155,0.285,22.8,10.66,10.324,0.47,0.047,0.0,False,CPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1412.0,+111/-69,0.164,24.0,19.83,11.033,1128.73,1.573,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1400.0,+180/-172,0.224,24.6,19.84,12.728,5.79,0.084,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1400.0,+137/-131,0.176,24.6,19.22,11.114,4546.1,406.11,0.0,False,GPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1393.0,+156/-139,0.223,25.0,16.3,11.586,713.0,2.317,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1364.0,+159/-138,0.15,26.5,18.58,14.611,713.0,0.155,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1340.0,+93/-103,0.078,27.8,24.94,14.578,6.53,0.294,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1318.0,+162/-158,0.118,28.95,19.88,14.846,4546.1,39.323,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1313.0,+170/-130,0.141,29.2,22.63,14.555,899.24,0.015,0.0,False,CPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1307.0,+107/-154,0.101,29.5,23.67,15.444,387.73,2.446,0.0,False,CPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1307.0,+156/-121,0.11,29.5,26.3,15.979,1.95,0.02,0.0,False,CPU +31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1305.0,+130/-103,0.074,29.6,21.99,14.301,52.7,0.202,0.0,False,CPU +32,🧠🔁,Neural Network,[LimiX (default) [60.00% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1296.0,+236/-216,0.196,30.05,7.37,19.254,0.46,0.013,60.0,True,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1279.0,+119/-104,0.08,30.9,27.5,14.214,1128.73,0.054,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1279.0,+136/-144,0.06,30.9,27.87,13.793,38.59,0.337,0.0,False,CPU +35,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1276.0,+179/-173,0.117,31.1,20.39,15.572,82.52,0.97,0.0,False,GPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1262.0,+124/-96,0.057,31.75,26.58,14.723,52.7,0.017,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1220.0,+104/-127,0.042,33.8,31.74,13.965,0.33,0.05,0.0,False,CPU +38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1210.0,+91/-114,0.016,34.3,32.79,14.804,11.03,0.489,0.0,False,GPU +39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1184.0,+145/-162,0.031,35.5,32.53,17.278,38.59,0.028,0.0,False,CPU +40,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1176.0,+131/-135,0.02,35.85,32.54,18.831,41.63,39.259,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1125.0,+100/-144,0.018,38.1,36.15,19.014,387.73,0.089,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1099.0,+96/-108,0.011,39.2,37.66,17.823,1.84,0.047,0.0,False,CPU +43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1046.0,+175/-253,0.017,41.25,37.15,28.308,43.41,0.111,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1037.0,+113/-202,0.018,41.6,38.74,24.557,332.4,33.481,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1014.0,+145/-195,0.024,42.4,39.33,22.583,1.63,0.11,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+112/-243,0.0,42.9,41.5,22.62,0.11,0.012,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,998.0,+162/-268,0.002,42.95,40.08,29.28,43.41,0.02,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,837.0,+96/-216,0.0,47.6,47.26,30.878,0.21,0.019,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,837.0,+242/-553,0.024,47.6,40.42,33.515,2.96,0.183,0.0,False,GPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,828.0,+85/-214,0.0,47.8,47.52,27.598,0.1,0.02,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,792.0,+67/-191,0.0,48.6,48.26,33.031,332.4,1.631,0.0,False,CPU +52,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,762.0,+124/-263,0.0,49.2,48.67,35.655,0.18,0.032,0.0,False,GPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,424.0,+108/-391,0.0,53.2,53.17,49.897,0.12,0.094,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ce1e2f54f8b09beeb5c2add968e8eeff82b5acb4 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f8e9ad1973094f40746081f5f1223a893bbc0d209670aad6b6890a80d1e30b8 +size 1795058 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/n_datasets_28 b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/n_datasets_28 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d3d45d00858326bab4de9bc2261b5fec5e2588e1 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f3158320802329628380b20cf3983f8686e135140eaa0272de60d1478cef63a +size 473361 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1b3975015b422a573b8e87088f7e7f658314b630 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b4e05f4251199c1edfe9547e7eef11c4516e162b8ad0f774ec2a7cfb44ba414 +size 1036578 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..13af61e65915224f687e47c51502b3eee8feebbb --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fb9c854f9a6ea3a7638f7c5d77ee8c3172ccb1a5898573da47cf9e9108b19cf +size 179441 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..494483cf0a96d894192d94511cdb42327ee09829 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1603.0,+95/-74,0.618,9.05,5.38,7.437,6.03,0.703,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1596.0,+129/-95,0.646,9.32,2.85,4.66,2279.85,9.666,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1565.0,+125/-84,0.598,10.54,4.64,7.33,2279.85,0.968,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1526.0,+107/-86,0.567,12.25,5.35,8.987,635.11,8.569,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1476.0,+84/-64,0.445,14.71,7.26,11.594,3486.44,22.867,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1459.0,+89/-66,0.434,15.68,8.29,11.423,2993.17,1.401,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1457.0,+132/-93,0.454,15.79,6.98,12.285,3226.71,49.21,7.14,True,GPU +7,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1446.0,+87/-62,0.449,16.38,6.27,10.694,4.53,0.611,0.0,False,GPU +8,🧠🔁,Neural Network,[Mitra (default) [7.14% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1427.0,+107/-80,0.401,17.46,7.78,12.738,145.75,4.071,7.14,True,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1416.0,+74/-72,0.375,18.12,8.57,12.977,3486.44,1.002,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1389.0,+84/-72,0.387,19.71,6.98,11.814,7432.95,404.225,0.0,False,GPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1377.0,+76/-76,0.34,20.46,9.93,13.992,3083.36,1.48,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1365.0,+108/-81,0.316,21.25,10.98,14.612,3226.71,0.494,7.14,True,GPU +13,🧠⚡,Foundation Model,[TabICL (default) [7.14% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1355.0,+72/-64,0.329,21.86,6.59,13.761,9.07,1.864,7.14,True,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1349.0,+62/-52,0.233,22.27,16.88,15.057,774.51,1.455,0.0,False,CPU +15,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1348.0,+96/-74,0.289,22.29,13.46,16.148,303.47,2.324,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1348.0,+85/-89,0.341,22.32,7.03,13.237,7432.95,39.797,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1320.0,+89/-67,0.281,24.09,13.98,15.094,3083.36,0.189,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1319.0,+71/-61,0.228,24.2,16.26,14.996,1612.87,0.724,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabPFNv2 (default) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1314.0,+94/-71,0.293,24.48,8.41,15.634,4.32,0.469,7.14,True,GPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1307.0,+78/-74,0.241,25.0,14.07,15.826,1165.19,2.067,0.0,False,GPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1306.0,+82/-77,0.246,25.04,11.45,14.692,6286.73,0.473,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1303.0,+68/-70,0.205,25.23,15.29,16.187,829.32,1.642,0.0,False,CPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1301.0,+57/-56,0.188,25.38,17.8,15.801,6.97,0.144,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1294.0,+67/-47,0.202,25.86,19.76,15.683,1612.87,0.061,0.0,False,CPU +25,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1291.0,+70/-66,0.199,26.0,20.3,16.028,774.51,0.282,0.0,False,CPU +26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1287.0,+72/-64,0.178,26.29,15.72,16.805,2989.36,0.609,0.0,False,CPU +27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1284.0,+69/-67,0.186,26.46,11.97,16.412,829.32,0.241,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1275.0,+83/-79,0.222,27.11,17.78,16.335,6286.73,9.386,0.0,False,GPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1269.0,+81/-80,0.229,27.5,16.66,17.527,11.86,0.149,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1256.0,+70/-70,0.157,28.36,17.05,16.694,14.59,4.035,0.0,False,GPU +31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1248.0,+60/-67,0.138,28.89,13.54,17.476,2989.36,0.063,0.0,False,CPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1246.0,+78/-82,0.17,29.05,12.43,16.995,1165.19,0.105,0.0,False,GPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1240.0,+80/-64,0.145,29.39,12.97,18.192,8.12,0.053,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1238.0,+86/-106,0.24,29.54,9.17,16.168,51.78,39.927,0.0,False,GPU +35,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1237.0,+81/-73,0.165,29.61,22.23,17.116,2894.94,2.42,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1197.0,+102/-91,0.158,32.27,20.71,19.917,1425.28,7.355,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1180.0,+87/-76,0.141,33.32,23.9,19.081,2894.94,0.145,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1178.0,+83/-85,0.129,33.5,21.84,20.789,402.38,1.373,0.0,False,CPU +39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1166.0,+69/-85,0.102,34.23,13.26,21.032,14.22,0.353,0.0,False,GPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1155.0,+84/-85,0.139,34.95,16.9,21.307,402.38,0.163,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1136.0,+80/-98,0.113,36.14,25.19,21.865,562.0,1.155,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1135.0,+91/-92,0.117,36.21,22.19,21.368,1425.28,0.561,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1134.0,+67/-66,0.061,36.29,30.61,19.814,2.85,0.19,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1128.0,+64/-81,0.095,36.66,29.97,20.294,3.27,0.193,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1088.0,+72/-83,0.054,39.0,31.4,22.847,562.0,0.138,0.0,False,CPU +46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1012.0,+95/-122,0.064,43.18,32.86,27.691,2.45,0.519,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1010.0,+82/-100,0.033,43.32,37.04,25.453,9.06,0.196,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1002.0,+96/-132,0.048,43.68,38.27,25.461,5.43,0.836,0.0,False,GPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+85/-113,0.025,43.8,31.74,28.276,0.46,0.07,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,979.0,+92/-121,0.037,44.82,37.52,28.627,123.01,1.579,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,978.0,+84/-106,0.029,44.89,40.58,25.392,5.03,0.601,0.0,False,CPU +52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,944.0,+94/-123,0.018,46.43,41.59,29.921,0.38,0.072,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,934.0,+89/-116,0.036,46.82,17.87,31.991,479.28,0.495,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,908.0,+89/-126,0.03,47.91,26.58,32.625,479.28,0.098,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,873.0,+104/-131,0.014,49.21,44.74,35.574,2.34,0.132,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,809.0,+108/-156,0.02,51.29,47.97,35.773,123.01,0.092,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,565.0,+114/-194,0.0,56.14,55.81,49.586,0.28,0.024,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..60a71a7cf219376263f40e674247b710020bb94f --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f35a4ce1cdc4d296be798638f97b52e11e9bd34e90c56ec42fa4a2750d5e407 +size 2402746 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/n_datasets_26 b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/n_datasets_26 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7f62c7bd1c985d943e492b8f831d12355fa72bb9 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a77ba8244c373ba077f7fd8b3fc037bec886d67eb997f47375a4ca5ba374dcb +size 481575 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e6a35f19914097e935eed4fbfbd98675bf75b383 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caa0fbfc23a316e4ea64fe9d5bb4aade532ea12b0f6f084ecdaf58e75e135f5e +size 1038842 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ce4b8166d249bad8358abb84ef8445a274a1f9de --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa56c5b223191a4ea45d028c9f764cf9d11450c8b889d4b83927101c54682afb +size 121062 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..c5cdb97720afb21c722cc84043868ea75819c1b0 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1648.0,+88/-68,0.635,8.1,5.12,7.839,5.78,0.586,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1624.0,+124/-100,0.653,8.92,2.96,4.968,2162.76,8.93,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1601.0,+124/-104,0.61,9.81,4.39,7.727,2162.76,0.894,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1533.0,+113/-82,0.548,12.73,5.43,9.618,620.85,8.569,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1510.0,+93/-63,0.463,13.81,6.88,12.159,3159.9,21.159,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1509.0,+107/-89,0.488,13.87,6.57,11.53,3361.32,51.669,0.0,False,GPU +6,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1472.0,+108/-68,0.432,15.79,7.34,12.029,150.4,4.403,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1467.0,+82/-70,0.41,16.08,8.22,12.218,2628.27,1.401,0.0,False,CPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1460.0,+100/-66,0.439,16.44,6.0,11.286,4.3,0.584,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1446.0,+89/-57,0.394,17.25,8.14,13.535,3159.9,0.646,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1403.0,+84/-80,0.341,19.75,10.41,14.036,3361.32,0.527,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1403.0,+103/-85,0.381,19.77,6.66,12.433,6883.9,405.331,0.0,False,GPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1392.0,+106/-73,0.335,20.42,9.82,14.604,2899.02,1.461,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1390.0,+84/-96,0.354,20.52,6.2,13.131,9.07,2.008,0.0,False,GPU +14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1377.0,+96/-83,0.298,21.31,12.92,16.382,303.47,2.098,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1360.0,+81/-99,0.341,22.42,6.69,13.867,6883.9,39.606,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1345.0,+83/-90,0.316,23.35,7.94,15.148,4.28,0.511,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1345.0,+53/-41,0.192,23.37,19.01,16.138,691.46,1.455,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1336.0,+73/-88,0.265,23.88,10.86,15.115,6020.89,0.454,0.0,False,GPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1330.0,+102/-70,0.27,24.33,14.64,15.756,2899.02,0.171,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1326.0,+85/-72,0.213,24.52,16.47,16.007,1394.45,0.618,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1308.0,+69/-57,0.182,25.71,19.95,17.318,785.9,1.281,0.0,False,CPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1306.0,+76/-53,0.169,25.83,18.07,16.868,5.74,0.139,0.0,False,CPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1305.0,+73/-64,0.185,25.92,15.16,17.587,2399.23,0.565,0.0,False,CPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1300.0,+79/-73,0.2,26.27,18.27,16.877,1053.72,1.872,0.0,False,GPU +25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1297.0,+91/-84,0.239,26.42,17.12,17.031,6020.89,8.478,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1293.0,+74/-51,0.173,26.69,21.51,16.785,1394.45,0.053,0.0,False,CPU +27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1290.0,+72/-60,0.166,26.88,12.61,17.51,785.9,0.213,0.0,False,CPU +28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1288.0,+71/-52,0.163,27.0,21.23,17.102,691.46,0.264,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1285.0,+102/-73,0.23,27.19,16.18,18.07,11.6,0.148,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1284.0,+83/-65,0.17,27.27,16.3,17.108,12.75,3.502,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1257.0,+98/-127,0.259,29.04,8.67,16.756,51.52,40.126,0.0,False,GPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+90/-93,0.139,29.23,12.47,18.783,7.41,0.052,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+60/-73,0.136,29.27,13.11,18.357,2399.23,0.061,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1253.0,+73/-74,0.169,29.31,21.75,17.668,2792.14,2.175,0.0,False,CPU +35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1234.0,+85/-76,0.126,30.56,21.73,18.071,1053.72,0.103,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1221.0,+93/-108,0.17,31.37,19.9,20.456,1350.04,6.57,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1191.0,+72/-113,0.109,33.29,12.56,21.041,13.67,0.339,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1183.0,+80/-82,0.119,33.81,26.42,19.614,2792.14,0.133,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1160.0,+84/-81,0.086,35.21,27.55,22.147,378.34,1.373,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1154.0,+96/-109,0.126,35.58,21.37,21.966,1350.04,0.613,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1137.0,+91/-92,0.107,36.62,19.58,22.595,378.34,0.154,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1128.0,+60/-59,0.033,37.19,33.98,21.116,2.66,0.17,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1118.0,+107/-106,0.078,37.77,26.65,23.308,527.27,1.155,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1103.0,+69/-69,0.047,38.63,34.62,21.73,3.01,0.177,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1079.0,+91/-89,0.045,40.04,32.03,24.138,527.27,0.128,0.0,False,CPU +46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1029.0,+86/-152,0.069,42.69,32.05,28.036,2.28,0.454,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1013.0,+91/-94,0.036,43.5,36.97,26.238,8.93,0.172,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+86/-108,0.027,44.15,31.42,28.762,0.44,0.065,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,999.0,+93/-122,0.051,44.19,38.38,26.76,5.0,0.753,0.0,False,GPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,990.0,+94/-109,0.031,44.62,40.08,25.606,4.68,0.547,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,959.0,+80/-107,0.009,46.04,43.24,29.843,118.4,1.479,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,956.0,+94/-99,0.038,46.17,16.98,31.197,429.17,0.495,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,946.0,+87/-146,0.02,46.62,41.57,30.28,0.36,0.068,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,930.0,+99/-111,0.032,47.27,25.54,31.807,429.17,0.081,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,897.0,+108/-120,0.015,48.58,43.99,34.214,2.16,0.129,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,766.0,+80/-151,0.0,52.58,51.76,37.314,118.4,0.082,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,576.0,+118/-291,0.0,56.08,55.73,49.46,0.27,0.024,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3b9bc6c0dd70956c31f7ad678ce566558d97efff --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1ccaff088036697bfb009fbf65c0f4219c2bdc58f19ac73fe5f642de08f15e7 +size 2336052 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/n_datasets_8 b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/n_datasets_8 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..165cdf251befd47a2c0356804d9129c9d965c70b --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9beae876e2d8e4ddd581bcedfdb26cf9c2f4772ae1f7ec09c74184c00926e9ca +size 466924 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c86657c322f132d65120eec987e7ee40fc035291 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b95acc32536124ad75cd15f63cdbd2c8093b34c8c014cd6fefa8cc3374f7f3eb +size 1027006 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..952decdb436cb48489551aae752b7a523db0447c --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86f32f6a1dfa8c0b9af8304c47654e0b347cb6c75896a0b962b8b35232f3776c +size 169156 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..9d1bcb26db617ed948b4876d4ff97b16e8aa7b00 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1724.0,+707/-194,0.796,6.0,1.66,1.808,2793.65,8.93,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1666.0,+231/-99,0.72,7.62,5.34,11.082,1008.06,9.121,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1657.0,+269/-152,0.675,7.94,4.58,9.911,2793.65,1.513,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1523.0,+311/-140,0.527,13.31,5.61,10.367,9.88,0.825,0.0,False,GPU +4,🧠🔁,Neural Network,[LimiX (default) [12.50% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1492.0,+207/-87,0.456,14.81,8.07,12.322,5.93,0.989,12.5,True,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1470.0,+196/-107,0.496,16.0,4.26,14.071,5546.65,3.268,0.0,False,CPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1432.0,+176/-88,0.359,18.12,14.73,15.024,3307.58,1.182,0.0,False,CPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1430.0,+204/-111,0.401,18.25,5.67,15.484,4334.8,21.159,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1425.0,+318/-180,0.447,18.5,6.69,18.166,2584.13,12.372,25.0,True,GPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1421.0,+173/-68,0.336,18.75,15.39,15.552,3307.58,0.14,0.0,False,CPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1410.0,+136/-93,0.316,19.38,13.27,16.245,1280.01,4.078,0.0,False,CPU +11,🧠🔁,Neural Network,[Mitra (default) [25.00% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1410.0,+215/-171,0.384,19.38,13.23,18.908,159.43,3.545,25.0,True,GPU +12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1400.0,+134/-112,0.32,20.0,13.04,16.321,1280.01,1.051,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1398.0,+193/-119,0.354,20.12,15.39,15.962,4334.8,3.529,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1389.0,+142/-108,0.32,20.62,13.94,18.699,5568.31,1.781,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1385.0,+215/-130,0.374,20.88,11.34,19.449,9411.86,344.009,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1375.0,+148/-106,0.317,21.5,12.59,19.038,5568.31,0.372,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1347.0,+112/-85,0.219,23.25,21.3,16.949,2029.77,4.113,0.0,False,CPU +18,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1347.0,+193/-161,0.296,23.25,9.4,19.369,2079.21,3.347,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1345.0,+124/-90,0.234,23.38,21.0,16.708,2029.77,1.04,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabPFNv2 (default) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1341.0,+193/-143,0.291,23.62,11.54,18.203,5.48,0.354,25.0,True,GPU +21,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1330.0,+214/-150,0.28,24.38,13.89,19.679,2584.13,0.408,25.0,True,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1325.0,+143/-104,0.225,24.69,20.75,18.145,6684.65,9.59,0.0,False,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1303.0,+157/-112,0.184,26.06,21.31,18.488,6684.65,0.752,0.0,False,GPU +24,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1302.0,+213/-134,0.302,26.12,6.19,20.809,9411.86,39.507,0.0,False,GPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1287.0,+146/-134,0.185,27.12,16.25,20.217,27108.48,0.742,0.0,False,CPU +26,🧠⚡,Foundation Model,[TabICL (default) [12.50% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1284.0,+236/-185,0.269,27.31,8.56,23.291,11.51,1.948,12.5,True,GPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1283.0,+144/-138,0.227,27.38,15.88,22.14,17.09,0.145,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1282.0,+193/-166,0.24,27.44,14.07,21.612,728.32,2.441,0.0,False,CPU +29,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1277.0,+114/-76,0.153,27.75,24.97,18.577,43.1,0.254,0.0,False,CPU +30,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1275.0,+224/-214,0.266,27.88,14.24,20.771,729.17,1.833,0.0,False,CPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1269.0,+242/-179,0.234,28.25,6.12,20.994,2079.21,0.319,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1256.0,+205/-148,0.211,29.06,14.14,21.742,728.32,0.356,0.0,False,CPU +33,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1251.0,+123/-146,0.187,29.38,22.35,21.941,283.57,4.104,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1241.0,+123/-109,0.132,30.0,27.09,21.705,3646.83,2.162,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1228.0,+122/-158,0.169,30.88,24.3,21.727,27108.48,0.066,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1216.0,+122/-136,0.139,31.62,28.2,22.231,17.86,0.099,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1206.0,+148/-164,0.126,32.25,21.47,23.531,3646.83,0.194,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1200.0,+198/-186,0.191,32.62,20.2,22.297,729.17,0.325,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1190.0,+204/-185,0.168,33.25,21.78,25.651,2721.87,12.593,0.0,False,CPU +40,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1184.0,+252/-168,0.213,33.62,10.77,24.06,46.36,39.812,0.0,False,GPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1178.0,+130/-119,0.07,34.0,30.54,20.547,4.93,0.592,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1176.0,+155/-160,0.13,34.12,26.02,22.982,5.12,0.444,0.0,False,CPU +43,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1112.0,+125/-146,0.066,37.88,33.48,25.251,19.12,6.922,0.0,False,GPU +44,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1101.0,+125/-156,0.022,38.5,35.99,27.29,17.24,0.565,0.0,False,GPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1089.0,+164/-171,0.085,39.12,30.54,27.296,2721.87,1.083,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1085.0,+163/-237,0.058,39.38,33.84,29.013,8.99,1.372,0.0,False,GPU +47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1013.0,+264/-406,0.128,43.0,27.42,38.476,155.52,2.046,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+99/-219,0.0,43.62,41.77,35.562,0.74,0.148,0.0,False,CPU +49,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,972.0,+107/-162,0.0,44.88,43.55,30.565,14.37,0.363,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,929.0,+156/-270,0.024,46.62,42.92,34.532,8.37,0.659,0.0,False,CPU +51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,908.0,+73/-232,0.0,47.44,46.53,41.326,909.67,0.609,0.0,False,CPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,890.0,+83/-217,0.0,48.06,47.0,42.282,909.67,0.107,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,885.0,+164/-240,0.003,48.25,45.44,40.712,0.76,0.153,0.0,False,CPU +54,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,780.0,+283/-1302,0.071,51.38,42.87,50.559,155.52,0.191,0.0,False,CPU +55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,765.0,+104/-224,0.0,51.75,51.44,41.42,4.67,0.855,0.0,False,GPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,698.0,+93/-331,0.0,53.25,52.96,48.104,4.62,0.135,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-283.0,+165/-1047,0.0,58.0,58.0,70.927,0.52,0.075,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ffeaf6f1a5c0ae8c51a2c5267db381427c7c2b23 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb16a715a0f5f5dcfbbc7a0f4c65c94776aab8f68562b31d5f782689b08b143a +size 1788887 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/n_datasets_1 b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/n_datasets_1 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..032193a1de40318e74f2abb61794382a84bc4bb7 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:009b6cb81ef3de594a819fbf56a75afa7f313664c5bdc519192d3e9ec56fdacd +size 451105 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3a4ee0ea943bc5e8dfe9425f2de3082de817c8db --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83cb622eff48677f2ce2f9d550d58c9737f520bf60536e21a0a6465d3e3b8a39 +size 957364 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..cb8cf85c3ed1e4005aafef7196f4e8a1928f909e --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5028f09f92049a1c238298072cdba70e78de0446128528f17c6fa13c08379f4b +size 107975 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..ed900c3d3f2df0c96ed91727ee80cc17ac49c7f9 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv @@ -0,0 +1,54 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,25228.0,+0/-0,1.0,1.0,1.0,0.0,134.59,8.308,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,23917.0,+0/-0,0.934,2.0,2.0,1.729,276.98,4.897,0.0,False,CPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,22310.0,+0/-0,0.915,3.0,3.0,2.208,276.84,3.303,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,20483.0,+0/-0,0.883,4.0,4.0,3.024,134.59,2.819,0.0,False,GPU +4,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,19432.0,+0/-0,0.576,5.0,5.0,10.141,7855.1,0.565,0.0,False,CPU +5,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,18258.0,+0/-0,0.572,6.0,6.0,10.238,61.47,0.154,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,17510.0,+0/-0,0.569,7.0,7.0,10.303,157.4,4.579,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,16643.0,+0/-0,0.567,8.0,8.0,10.333,157.4,0.401,0.0,False,CPU +8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,15885.0,+0/-0,0.551,9.0,9.0,10.67,61.47,0.015,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,15137.0,+0/-0,0.525,10.0,10.0,11.231,2687.46,0.361,0.0,False,CPU +10,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,14345.0,+0/-0,0.522,11.0,11.0,11.295,7855.1,0.048,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,13528.0,+0/-0,0.517,12.0,12.0,11.389,2687.46,0.793,0.0,False,CPU +12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,12768.0,+0/-0,0.515,13.0,13.0,11.441,456.33,1.171,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,12036.0,+0/-0,0.501,14.0,14.0,11.722,456.33,0.047,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,11311.0,+0/-0,0.501,15.0,15.0,11.722,0.32,0.047,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,10518.0,+0/-0,0.432,16.0,16.0,13.143,8.22,0.049,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,9807.0,+0/-0,0.42,17.0,17.0,13.383,4378.82,25.595,0.0,False,GPU +17,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,9111.0,+0/-0,0.387,18.0,18.0,14.026,20.73,0.079,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,8390.0,+0/-0,0.386,19.0,19.0,14.047,4378.82,2.216,0.0,False,GPU +19,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,7635.0,+0/-0,0.356,20.0,20.0,14.639,1.48,0.535,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,6953.0,+0/-0,0.292,21.0,21.0,15.857,2.79,1.057,0.0,False,GPU +21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,6264.0,+0/-0,0.249,22.0,22.0,16.662,20.88,0.359,0.0,False,CPU +22,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,5562.0,+0/-0,0.111,23.0,23.0,19.143,20.88,0.026,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,5007.0,+0/-0,0.059,24.0,24.0,20.028,4390.96,275.957,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,4306.0,+0/-0,0.024,25.0,25.0,20.621,2341.91,0.027,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,3678.0,+0/-0,0.004,26.0,26.0,20.955,14.53,1.747,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,3004.0,+0/-0,0.0,27.0,27.0,21.025,2341.91,0.172,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,2329.0,+0/-0,0.0,28.0,28.0,21.961,1993.57,0.469,0.0,False,GPU +28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1701.0,+0/-0,0.0,29.0,29.0,22.049,1993.57,0.055,0.0,False,GPU +29,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+0/-0,0.0,30.0,30.0,24.001,0.18,0.007,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,445.0,+0/-0,0.0,31.0,31.0,24.06,4390.96,38.525,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,-256.0,+0/-0,0.0,32.0,32.0,24.187,41.21,39.406,0.0,False,GPU +32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,-946.0,+0/-0,0.0,33.0,33.0,24.591,1322.48,0.546,0.0,False,GPU +33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,-1628.0,+0/-0,0.0,34.0,34.0,25.183,1322.48,0.055,0.0,False,GPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,-2382.0,+0/-0,0.0,35.0,35.0,26.439,0.21,0.047,0.0,False,CPU +35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,-3104.0,+0/-0,0.0,36.0,36.0,29.215,2009.11,3.767,0.0,False,GPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,-3800.0,+0/-0,0.0,37.0,37.0,30.84,2009.11,0.974,0.0,False,GPU +37,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,-4511.0,+0/-0,0.0,38.0,38.0,31.796,6.84,0.09,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-5304.0,+0/-0,0.0,39.0,39.0,31.813,3.34,0.028,0.0,False,CPU +39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,-6029.0,+0/-0,0.0,40.0,40.0,34.136,66.06,1.783,0.0,False,GPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,-6760.0,+0/-0,0.0,41.0,41.0,38.819,6.51,0.162,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,-7521.0,+0/-0,0.0,42.0,42.0,41.321,553.48,0.174,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,-8338.0,+0/-0,0.0,43.0,43.0,42.976,553.48,1.636,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,-9130.0,+0/-0,0.0,44.0,44.0,50.562,0.08,0.017,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-9878.0,+0/-0,0.0,45.0,45.0,52.296,2.73,0.083,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-10636.0,+0/-0,0.0,46.0,46.0,52.871,30.19,0.134,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-11503.0,+0/-0,0.0,47.0,47.0,55.965,30.19,0.017,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-12250.0,+0/-0,0.0,48.0,48.0,56.26,0.13,0.015,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-13425.0,+0/-0,0.0,49.0,49.0,57.211,76.41,2.853,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,-14476.0,+0/-0,0.0,50.0,50.0,57.769,3.14,0.921,0.0,False,GPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,-16303.0,+0/-0,0.0,51.0,51.0,57.822,0.11,0.023,0.0,False,GPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-17910.0,+0/-0,0.0,52.0,52.0,70.703,76.41,0.258,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-19221.0,+0/-0,0.0,53.0,53.0,83.729,0.14,0.123,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a743b27742076b214468da0b918a64db6e626287 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3ad521dfab263587542afa8704e9433787dfa03aa2cd2dba62173808bf3c2a8 +size 1057411 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/n_datasets_7 b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/n_datasets_7 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a06bf256e58e8e0b859cd87422d4b7110f6f2273 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82a851656225549bb63e59517a3c1806a7caed200d4ff955d0c90a4a523ccc0d +size 456356 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e7c1e6c074cea47584507d90696f91223a1890b1 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc4220f2419fb9ed82a5d2820d1b8068b27c4e85ec87f8845783f9fd6d244669 +size 1022007 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1d6eba8878692769975533c916cf329b2d0a0cc9 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8523dea13361b5c5e2f1a86c00d162188f15794b2be61928ec934ccb9291571d +size 168480 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..cc8df56a8ff15d9cb8b60133646d09b4ad9d671a --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1738.0,+465/-200,0.767,6.71,1.83,2.066,2837.96,9.546,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1688.0,+223/-89,0.691,8.29,6.0,12.349,1060.41,10.928,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1681.0,+348/-165,0.644,8.5,4.68,10.895,2837.96,1.134,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1584.0,+371/-164,0.542,12.36,5.08,9.756,10.97,0.856,0.0,False,GPU +4,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1583.0,+198/-94,0.521,12.43,7.3,10.653,6.19,1.301,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1521.0,+248/-121,0.458,15.43,5.06,14.183,4655.15,21.791,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1509.0,+372/-201,0.511,16.07,5.99,17.32,2969.66,17.821,14.29,True,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1485.0,+187/-130,0.405,17.43,14.17,14.644,4655.15,4.031,0.0,False,GPU +8,🧠🔁,Neural Network,[Mitra (default) [14.29% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1481.0,+229/-184,0.439,17.64,12.22,18.18,202.6,4.622,14.29,True,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1474.0,+269/-114,0.432,18.0,5.08,15.834,6318.92,3.195,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1457.0,+221/-107,0.329,19.0,15.22,15.543,3375.11,1.208,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1447.0,+150/-114,0.357,19.57,13.01,18.233,5729.39,1.817,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1440.0,+174/-63,0.302,20.0,16.67,16.169,3375.11,0.132,0.0,False,CPU +13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1433.0,+218/-186,0.404,20.43,10.55,19.367,10182.26,399.569,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1433.0,+163/-119,0.354,20.43,11.64,18.608,5729.39,0.373,0.0,False,GPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1428.0,+171/-184,0.339,20.71,8.46,17.962,2149.3,2.928,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1421.0,+193/-93,0.273,21.14,15.22,17.093,1394.84,3.578,0.0,False,CPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1411.0,+214/-96,0.278,21.71,14.33,17.176,1394.84,1.087,0.0,False,CPU +18,🧠⚡,Foundation Model,[TabPFNv2 (default) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1398.0,+221/-189,0.332,22.5,10.58,17.374,6.0,0.388,14.29,True,GPU +19,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1394.0,+259/-194,0.32,22.79,12.78,19.05,2969.66,0.477,14.29,True,GPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1363.0,+102/-64,0.169,24.71,23.44,17.736,2342.58,5.131,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1363.0,+102/-82,0.187,24.71,22.61,17.42,2342.58,1.065,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1359.0,+245/-202,0.345,25.0,5.55,20.345,10182.26,40.124,0.0,False,GPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1357.0,+188/-136,0.26,25.14,14.57,20.76,19.13,0.146,0.0,False,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1347.0,+159/-119,0.188,25.79,21.43,18.825,6994.17,9.554,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1339.0,+191/-199,0.267,26.29,5.46,19.587,2149.3,0.147,0.0,False,GPU +26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1326.0,+160/-143,0.145,27.07,21.69,19.122,6994.17,0.583,0.0,False,GPU +27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1325.0,+160/-127,0.214,27.14,20.85,20.198,326.23,4.282,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1308.0,+189/-237,0.227,28.21,13.38,22.319,772.2,2.495,0.0,False,CPU +29,🧠⚡,Foundation Model,[TabICL (default) [14.29% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1308.0,+256/-194,0.254,28.21,7.89,24.353,11.7,2.132,14.29,True,GPU +30,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1289.0,+122/-76,0.104,29.43,27.14,19.353,57.99,0.317,0.0,False,CPU +31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1281.0,+180/-226,0.212,29.93,13.4,22.113,772.2,0.373,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1276.0,+135/-105,0.122,30.29,23.95,21.656,44054.5,0.843,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1273.0,+139/-119,0.135,30.43,27.11,21.802,3833.57,2.294,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1264.0,+194/-273,0.215,31.0,17.71,22.276,731.01,2.021,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1262.0,+252/-221,0.192,31.14,20.2,23.176,2750.22,13.569,0.0,False,CPU +36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1230.0,+249/-225,0.244,33.14,9.78,24.042,51.52,40.219,0.0,False,GPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1228.0,+182/-165,0.125,33.29,21.05,23.947,3833.57,0.22,0.0,False,CPU +38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1228.0,+182/-225,0.148,33.29,24.78,22.488,5.15,0.526,0.0,False,CPU +39,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1223.0,+115/-138,0.094,33.57,30.68,23.403,14.99,0.113,0.0,False,CPU +40,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1221.0,+124/-126,0.111,33.71,29.37,23.217,44054.5,0.068,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1183.0,+134/-218,0.132,36.0,24.57,23.958,731.01,0.334,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1170.0,+54/-72,0.0,36.71,35.84,21.807,5.38,0.624,0.0,False,CPU +43,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1170.0,+140/-177,0.076,36.71,32.22,23.313,20.63,7.532,0.0,False,GPU +44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1163.0,+150/-236,0.067,37.14,32.08,24.905,9.4,1.444,0.0,False,GPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1148.0,+220/-223,0.097,38.0,29.08,25.292,2750.22,1.239,0.0,False,CPU +46,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1106.0,+117/-141,0.008,40.29,38.08,28.196,19.64,0.478,0.0,False,GPU +47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1084.0,+250/-449,0.146,41.43,25.62,35.799,180.38,1.24,0.0,False,CPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1008.0,+129/-210,0.0,45.0,43.49,30.387,14.87,0.391,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+105/-290,0.0,45.36,43.81,37.213,0.78,0.155,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,981.0,+192/-316,0.028,46.14,42.07,31.994,8.78,0.688,0.0,False,CPU +51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,961.0,+74/-238,0.0,46.93,45.95,39.676,913.1,0.738,0.0,False,CPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,946.0,+82/-283,0.0,47.5,46.37,40.327,913.1,0.13,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,929.0,+180/-311,0.004,48.14,44.97,39.305,0.77,0.156,0.0,False,CPU +54,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,853.0,+284/-1300,0.081,50.57,41.4,47.681,180.38,0.136,0.0,False,CPU +55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,832.0,+80/-228,0.0,51.14,50.85,39.077,5.48,0.91,0.0,False,GPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,736.0,+100/-302,0.0,53.29,52.96,46.939,4.65,0.14,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-464.0,+388/-687,0.0,58.0,58.0,69.099,0.58,0.071,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..445d629d4ff752840a9859c740c4195647fca088 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e4bc54e85e022a6bb6c5ce09ee4a3f2aca1756bb98f4a49b0560677075644c3 +size 1680410 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/n_datasets_6 b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/n_datasets_6 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..dd29dbebbf56a3c1cfa9ca5eac382bb9efa18b74 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cb50ec84ff7772a71f417b1307f323b19aa82231e98600d2c63c8ae6e8b875e +size 456644 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ad260ae7b7a0c0d685e9e2efef2eccb1cf20e884 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05ef32025b5c1a53b6b4d023ae5608490df3acb167848b47e48fbcc33d754b3f +size 1036337 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..43c8294c6fd51e053ebddd9f0e9aac92551cc52f --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82cb5a176ada11a344c6b12f22b2bf17a6a42d4fb9ab369ee4726d3e819e75bc +size 125861 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..4f05fb5c88ee5a61d5225392ab49c2f9a4cf43ae --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,59 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1997.0,+809/-100,0.875,3.17,1.58,2.191,2793.65,8.93,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1857.0,+337/-82,0.729,5.42,4.12,12.495,2793.65,1.02,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1746.0,+356/-135,0.689,8.17,5.69,14.332,1295.16,14.354,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1716.0,+553/-144,0.632,9.08,4.46,11.083,9.88,0.825,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1688.0,+378/-116,0.596,10.0,5.22,14.698,3561.2,34.413,0.0,False,GPU +5,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1634.0,+358/-88,0.506,12.0,6.72,12.332,5.93,0.989,0.0,False,GPU +6,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1626.0,+204/-38,0.513,12.33,10.85,15.748,206.59,4.778,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1622.0,+272/-96,0.534,12.5,4.43,16.24,4334.8,21.159,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1583.0,+233/-63,0.472,14.17,12.85,16.59,4334.8,3.529,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1522.0,+224/-109,0.384,17.0,14.03,17.857,3307.58,1.182,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1506.0,+327/-108,0.373,17.83,11.35,16.716,3561.2,0.612,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1502.0,+371/-116,0.388,18.0,9.36,14.808,5.48,0.434,0.0,False,GPU +12,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1489.0,+378/-152,0.401,18.67,4.6,18.377,6700.69,3.268,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1476.0,+201/-74,0.307,19.33,15.87,18.68,3307.58,0.108,0.0,False,CPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1445.0,+344/-241,0.382,21.0,9.92,22.477,9411.86,433.416,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1433.0,+268/-147,0.283,21.67,15.18,21.223,5568.31,1.781,0.0,False,GPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1418.0,+175/-44,0.219,22.5,21.18,19.816,2029.77,1.073,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1412.0,+142/-39,0.197,22.83,22.15,20.228,2029.77,6.631,0.0,False,CPU +18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1412.0,+262/-84,0.213,22.83,16.26,19.85,1280.01,5.479,0.0,False,CPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1409.0,+244/-141,0.272,23.0,14.96,21.671,5568.31,0.372,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1403.0,+285/-92,0.219,23.33,14.81,19.946,1280.01,1.143,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1398.0,+205/-127,0.219,23.58,19.93,21.447,6829.34,7.818,0.0,False,GPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1394.0,+268/-198,0.246,23.83,18.35,20.929,2059.45,3.769,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1379.0,+437/-228,0.297,24.67,6.92,22.95,13.87,2.152,0.0,False,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1375.0,+264/-171,0.169,24.92,20.15,21.793,6829.34,0.53,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1356.0,+255/-176,0.229,26.0,13.94,24.08,17.09,0.145,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1353.0,+340/-277,0.32,26.17,4.97,23.608,9411.86,42.924,0.0,False,GPU +27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1319.0,+250/-164,0.191,28.17,20.83,23.403,410.08,4.104,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1302.0,+373/-354,0.223,29.17,18.56,26.238,2721.87,14.595,0.0,False,CPU +29,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1296.0,+161/-125,0.109,29.5,26.85,22.348,155.57,0.254,0.0,False,CPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1278.0,+292/-226,0.145,30.5,21.21,22.852,2059.45,0.127,0.0,False,GPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1270.0,+202/-155,0.115,31.0,23.64,25.058,27108.48,0.742,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1264.0,+222/-200,0.119,31.33,27.49,25.243,4370.77,2.162,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1246.0,+253/-279,0.12,32.33,25.27,26.005,728.32,2.548,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1235.0,+397/-370,0.284,33.0,8.74,27.727,42.75,45.46,0.0,False,GPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1217.0,+342/-353,0.148,34.0,18.85,25.893,729.17,2.154,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1211.0,+276/-254,0.102,34.33,25.35,25.765,728.32,0.356,0.0,False,CPU +37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1190.0,+175/-206,0.075,35.5,30.79,26.917,27108.48,0.066,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1190.0,+176/-212,0.088,35.5,30.85,25.981,19.2,6.922,0.0,False,GPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1181.0,+89/-95,0.0,36.0,35.1,24.783,4.93,0.671,0.0,False,CPU +40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1181.0,+170/-206,0.035,36.0,34.18,27.166,11.78,0.099,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1159.0,+197/-173,0.06,37.17,32.88,26.155,5.12,0.444,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1156.0,+276/-281,0.113,37.33,27.67,28.736,2721.87,1.334,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1153.0,+228/-281,0.078,37.5,31.64,28.727,8.99,1.498,0.0,False,GPU +44,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1146.0,+149/-137,0.006,37.83,36.17,27.899,4370.77,0.22,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1137.0,+257/-308,0.097,38.33,25.06,27.785,729.17,0.337,0.0,False,CPU +46,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1120.0,+154/-209,0.01,39.17,36.91,29.623,17.24,0.42,0.0,False,GPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+143/-432,0.0,44.67,42.99,37.954,0.74,0.148,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,979.0,+86/-362,0.0,45.5,44.67,38.521,909.67,0.893,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,962.0,+100/-303,0.0,46.17,45.13,39.28,909.67,0.107,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,962.0,+232/-508,0.032,46.17,41.47,34.491,8.37,0.685,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,939.0,+300/-564,0.037,47.0,40.48,41.716,155.52,1.033,0.0,False,CPU +52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,934.0,+277/-430,0.004,47.17,43.75,39.387,0.76,0.153,0.0,False,CPU +53,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,924.0,+96/-253,0.0,47.5,47.01,35.184,14.57,0.363,0.0,False,CPU +54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,748.0,+71/-303,0.0,52.17,51.98,44.164,4.67,0.855,0.0,False,GPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,723.0,+130/-462,0.0,52.67,52.34,44.545,4.62,0.135,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,372.0,+132/-1205,0.0,56.33,56.29,55.52,155.52,0.12,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-548.0,+343/-1220,0.0,58.0,58.0,70.032,0.52,0.058,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e914d639b577e9d0dcb98b4a1950066961a8c575 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfb2066c14637bcb9500b96764cc10c89af12f9bdf93c1a3c9ac51a3f7548333 +size 1681180 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/n_datasets_13 b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/n_datasets_13 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..fd2d2f1ef586b78ae3c9c8c739f85b7251f6748f --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b50ac480dcca750fbb098580041f64724a47a36f90d82e379b690813b343a643 +size 442857 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ad8ba1d965b6e66b444ddaa5520489e126f3e7b3 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d9bfb5f60e3e3973b0240dfd2851a66ac6b7eb86eef49947ee06f704de16399 +size 1008866 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d4e7923dc02d09682e2ec84ee323abeff328e9c4 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ac6848f4b0d7d0089cdcbd0dad246e47bf30b6035421265cd5aef579c978520 +size 147318 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..e227eaaabdd5c649aa1df699c196a864faebaefa --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv @@ -0,0 +1,56 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1878.0,+277/-162,0.798,5.46,2.46,1.248,2043.44,8.369,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1798.0,+204/-85,0.729,7.46,2.38,2.558,7125.58,336.907,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1798.0,+115/-60,0.652,7.46,4.37,2.649,616.43,4.041,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1761.0,+145/-59,0.677,8.54,4.62,3.061,7125.58,38.573,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1751.0,+194/-142,0.672,8.85,3.38,2.186,2043.44,0.781,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1695.0,+96/-70,0.535,10.77,5.86,4.784,1489.94,5.79,0.0,False,CPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1692.0,+106/-54,0.573,10.85,8.82,4.256,4057.12,10.478,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1642.0,+178/-151,0.535,12.77,7.33,3.66,6.94,0.519,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1642.0,+150/-73,0.542,12.77,6.52,4.24,44.65,38.298,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1608.0,+154/-138,0.501,14.15,5.57,6.753,3811.43,7.581,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1566.0,+126/-126,0.43,16.0,11.47,5.413,4057.12,0.337,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1560.0,+92/-63,0.332,16.31,13.64,7.383,686.46,5.477,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1535.0,+82/-66,0.293,17.46,13.48,7.168,2895.38,1.317,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1516.0,+109/-75,0.362,18.38,14.25,6.559,708.95,1.973,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1506.0,+135/-95,0.314,18.85,15.04,5.831,4228.53,1.188,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1498.0,+109/-88,0.272,19.23,10.75,7.292,2895.38,0.073,0.0,False,CPU +16,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1481.0,+208/-182,0.407,20.08,5.06,13.045,5.49,0.754,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1475.0,+108/-87,0.243,20.38,16.02,8.18,686.46,0.743,0.0,False,CPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1464.0,+95/-64,0.268,20.92,17.27,7.123,708.95,0.069,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1439.0,+58/-51,0.18,22.15,20.62,8.102,848.99,0.475,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1437.0,+53/-46,0.181,22.23,21.12,8.118,848.99,2.378,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1427.0,+106/-89,0.221,22.77,19.49,9.404,16.07,0.29,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1412.0,+113/-82,0.201,23.54,20.78,6.664,4228.53,0.125,0.0,False,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1404.0,+126/-138,0.265,23.92,14.23,9.279,3811.43,0.452,0.0,False,GPU +24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1403.0,+211/-142,0.322,23.96,5.75,6.396,3805.62,10.413,46.15,True,GPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1398.0,+116/-125,0.188,24.23,19.58,9.254,8.35,0.087,0.0,False,CPU +26,🧠🔁,Neural Network,[Mitra (default) [46.15% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1350.0,+167/-112,0.214,26.69,12.3,8.579,70.97,1.59,46.15,True,GPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1336.0,+123/-115,0.142,27.46,24.43,8.326,13.9,0.123,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1308.0,+180/-154,0.218,28.88,16.23,7.904,3805.62,0.262,46.15,True,GPU +29,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1289.0,+126/-104,0.085,29.85,26.13,10.165,9.17,1.416,0.0,False,GPU +30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1281.0,+113/-112,0.103,30.23,25.29,12.346,161.73,0.784,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1277.0,+89/-103,0.071,30.46,27.55,9.496,4452.11,0.852,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1261.0,+114/-110,0.116,31.23,25.13,12.658,161.73,0.12,0.0,False,CPU +33,🧠⚡,Foundation Model,[TabPFNv2 (default) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1256.0,+184/-178,0.19,31.46,19.12,9.06,2.78,0.315,46.15,True,GPU +34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1234.0,+173/-152,0.084,32.54,14.85,12.992,2.47,0.755,0.0,False,GPU +35,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1221.0,+114/-126,0.054,33.15,29.53,10.114,4452.11,0.089,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1220.0,+72/-95,0.031,33.23,31.93,10.993,2.03,0.296,0.0,False,CPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1191.0,+94/-114,0.062,34.62,31.76,11.709,2.15,0.18,0.0,False,CPU +38,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1186.0,+155/-179,0.118,34.85,23.77,13.499,3017.28,0.248,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1179.0,+69/-87,0.016,35.15,33.41,13.32,526.17,0.77,0.0,False,CPU +40,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1144.0,+148/-191,0.118,36.77,19.83,14.059,3017.28,0.027,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1134.0,+88/-107,0.015,37.19,35.35,14.071,526.17,0.119,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1113.0,+101/-107,0.023,38.08,35.85,14.254,0.42,0.061,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1069.0,+83/-106,0.004,39.85,38.7,14.361,527.21,2.831,0.0,False,CPU +44,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1042.0,+93/-142,0.0,40.85,39.11,14.199,20.5,0.081,0.0,False,CPU +45,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1030.0,+109/-172,0.05,41.31,38.59,15.156,10.04,0.03,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1026.0,+102/-118,0.0,41.46,40.27,14.845,527.21,0.311,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+66/-87,0.0,42.35,41.83,15.385,0.63,0.064,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,881.0,+101/-164,0.0,45.85,45.08,18.896,3.08,0.293,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,859.0,+134/-176,0.0,46.38,45.51,20.364,90.87,0.906,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,705.0,+107/-204,0.0,49.38,49.19,22.511,90.87,0.055,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,571.0,+216/-360,0.0,51.15,50.64,29.184,0.25,0.044,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,425.0,+111/-304,0.0,52.58,52.52,36.25,177.04,0.14,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,338.0,+142/-400,0.0,53.27,53.2,36.435,177.04,0.059,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,187.0,+134/-333,0.0,54.23,54.22,38.891,0.88,0.109,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..53ef079a19e785f8aa87b4989c6509cd75fc99c9 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a82e1d19971463577b44de7c5e892de47540ab8eb410c021ee45e38cf509158 +size 1701640 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/n_datasets_5 b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/n_datasets_5 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..34e3865de5058b187da287b6ef6a3623fca0f691 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc183c073380caee2a8d78e894b780b0397e83f82bb3124534c03daf683071bd +size 430652 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d3186d6a05168a73e7ac8c0a1edfbdf17d244e9e --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6617321e0cfec776451c064538587cc84c0a4b31724e9fd6f8f4abd53574ccc4 +size 984135 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3596f8595f447410176b1deabb1a3c30a70848ab --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c35f0fd3382e45b0a83b6ff9b9295b5be17648c447a527baafcb236f76eaae58 +size 119314 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..dbc54b0b2b4f8c3dd4bc8feaa1c2728697a1133a --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv @@ -0,0 +1,52 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,2061.0,+1014/-71,0.832,4.4,3.24,1.163,739.1,11.65,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,2027.0,+1086/-121,0.835,5.0,2.74,1.262,739.1,0.689,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1950.0,+850/-36,0.699,6.6,5.71,2.284,2552.28,4.522,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1916.0,+658/-8,0.655,7.4,3.4,3.209,474.94,11.837,0.0,False,CPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1885.0,+615/-13,0.612,8.2,6.09,3.513,400.37,2.16,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1870.0,+930/-101,0.637,8.6,6.91,2.851,2068.17,1.355,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1822.0,+1355/-177,0.687,10.0,2.25,2.083,4529.88,357.209,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1790.0,+1289/-174,0.65,11.0,4.14,2.39,4529.88,39.544,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1783.0,+1058/-258,0.548,11.2,6.42,2.625,1.59,0.506,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1688.0,+928/-198,0.485,14.4,10.08,4.112,2068.17,0.068,0.0,False,GPU +10,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1677.0,+780/-243,0.471,14.8,11.42,4.058,2552.28,0.201,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1677.0,+503/-139,0.377,14.8,11.88,5.836,223.05,9.707,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1671.0,+1088/-218,0.526,15.0,6.19,3.203,41.5,38.298,0.0,False,GPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1660.0,+548/-97,0.349,15.4,12.03,6.007,223.05,0.793,0.0,False,CPU +14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1649.0,+524/-91,0.418,15.8,13.49,4.481,438.55,0.52,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1644.0,+510/-102,0.3,16.0,10.62,6.235,1696.82,0.538,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1617.0,+754/-226,0.336,17.0,12.43,5.187,3456.17,0.611,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1590.0,+500/-169,0.28,18.0,7.25,6.475,1696.82,0.055,0.0,False,CPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1559.0,+428/-97,0.253,19.2,17.73,5.477,438.55,0.035,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1553.0,+728/-163,0.287,19.4,16.95,5.3,6.87,0.098,0.0,False,GPU +20,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1537.0,+750/-208,0.306,20.0,4.21,4.7,2.07,0.754,0.0,False,GPU +21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1521.0,+561/-275,0.27,20.6,16.66,7.061,4.99,0.047,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1505.0,+528/-187,0.183,21.2,19.63,6.188,3456.17,0.037,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1494.0,+356/-58,0.189,21.6,20.89,6.853,289.35,1.798,0.0,False,CPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1484.0,+386/-70,0.181,22.0,21.09,6.921,289.35,0.36,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1435.0,+648/-216,0.129,23.8,21.42,7.402,5.37,0.025,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1353.0,+354/-228,0.137,26.8,24.63,8.025,0.57,0.074,0.0,False,CPU +27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1302.0,+357/-139,0.002,28.6,27.72,8.909,2.32,0.093,0.0,False,GPU +28,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1297.0,+375/-186,0.018,28.8,28.08,8.225,0.66,0.213,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1274.0,+225/-245,0.064,29.6,27.65,8.161,4108.24,0.479,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1250.0,+508/-284,0.028,30.4,28.51,9.603,6.54,0.284,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1221.0,+204/-296,0.029,31.4,29.47,9.253,4108.24,0.037,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1172.0,+353/-451,0.023,33.0,31.22,10.288,35.47,0.502,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1147.0,+417/-890,0.144,33.8,25.8,13.625,1820.58,0.133,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1140.0,+314/-411,0.006,34.0,32.83,10.454,35.47,0.054,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1134.0,+107/-401,0.0,34.2,33.96,10.053,79.96,0.571,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1124.0,+128/-390,0.0,34.5,34.09,10.217,79.96,0.053,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1061.0,+165/-350,0.0,36.4,35.28,13.469,14.05,0.029,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1033.0,+387/-674,0.006,37.2,35.12,12.12,0.18,0.054,0.0,False,CPU +39,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1026.0,+316/-953,0.1,37.4,34.8,14.242,1820.58,0.012,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+78/-318,0.0,38.1,37.96,11.35,0.42,0.053,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,996.0,+363/-1057,0.12,38.2,34.58,14.681,3.47,0.01,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,871.0,+51/-752,0.0,41.2,40.95,14.269,317.48,0.719,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,842.0,+43/-769,0.0,41.8,41.54,14.827,317.48,0.127,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,800.0,+94/-889,0.0,42.6,42.42,18.666,142.18,1.027,0.0,False,CPU +45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,662.0,+74/-902,0.0,44.8,44.73,21.083,142.18,0.057,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,662.0,+107/-831,0.0,44.8,44.56,18.199,1.33,0.126,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,361.0,+132/-1452,0.0,47.6,47.49,28.294,0.06,0.036,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,145.0,+92/-1782,0.0,48.8,48.8,39.482,48.35,0.054,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-119.0,+197/-2129,0.0,50.0,49.99,39.829,48.35,0.013,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-276.0,+185/-2881,0.0,50.6,50.6,42.789,0.13,0.02,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4592db7088a316aea22b530972c07d31c944a8e8 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16545df5ba8fac5ff84297fb83742815f52924c8c210dbf293fec1eec1c6aee9 +size 1244143 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/n_datasets_8 b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/n_datasets_8 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..42fe7bbf0990bb2a2b2976973c024cb37278e120 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2329eab6fd3f94694a7e4df84c1a8da6122a5d25488fde802f0b9b242c76d96 +size 449846 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ad1754fc0f1cf3973d8427499dd79cd546aa433e --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cb2d1519d325164a2e9eefa8edacf5116dfc39f2aac7087b5bc2ea77d78fe26 +size 1058098 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ab8f67fe5a5f536ec2430ae26f09373f84408d54 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3ae294b248d5d62805ac74a762fac39a3b398eb8f9c3e09ecfab5dae16af7f7 +size 159497 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..c5f3cc5175bd0031c5bdfab6b5cd209538c58bb8 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv @@ -0,0 +1,56 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1891.0,+265/-104,0.738,5.88,2.47,2.855,9830.47,232.266,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1880.0,+464/-211,0.767,6.12,2.14,1.301,2355.62,7.55,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1846.0,+282/-80,0.656,7.0,3.72,2.109,716.17,4.181,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1846.0,+180/-36,0.674,7.0,4.99,3.48,9830.47,22.221,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1713.0,+345/-190,0.561,11.25,3.97,2.764,2355.62,0.879,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1710.0,+158/-52,0.533,11.38,6.74,4.888,51.27,43.085,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1687.0,+130/-51,0.512,12.25,10.65,5.134,4918.07,35.607,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1672.0,+142/-65,0.442,12.88,10.65,5.768,4919.72,5.372,0.0,False,CPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1651.0,+309/-190,0.51,13.75,8.04,4.306,8.24,0.532,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1644.0,+544/-239,0.523,14.06,3.75,3.301,6437.34,56.116,12.5,True,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1577.0,+165/-113,0.381,17.0,12.55,6.226,4918.07,0.949,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1572.0,+116/-77,0.272,17.25,15.04,8.35,1206.7,4.573,0.0,False,CPU +12,🧠🔁,Neural Network,[Mitra (default) [12.50% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1552.0,+246/-167,0.348,18.19,8.58,6.847,94.13,2.176,12.5,True,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1548.0,+124/-62,0.252,18.38,16.21,7.752,3495.99,1.462,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1538.0,+206/-133,0.358,18.88,5.49,9.546,9006.35,8.016,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1515.0,+188/-91,0.233,20.0,15.41,7.803,3495.99,0.148,0.0,False,CPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1515.0,+141/-90,0.291,20.0,14.78,7.857,1672.82,2.447,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1515.0,+153/-109,0.277,20.0,17.31,6.233,4322.16,1.598,0.0,False,GPU +18,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1513.0,+408/-258,0.453,20.12,5.79,18.26,5.86,0.71,0.0,False,GPU +19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1476.0,+150/-74,0.231,22.0,16.98,8.152,1672.82,0.097,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1475.0,+382/-238,0.355,22.06,11.85,5.751,6437.34,0.557,12.5,True,GPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1472.0,+94/-57,0.136,22.25,20.33,8.84,1277.27,0.648,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1465.0,+89/-51,0.133,22.62,21.26,8.909,1277.27,3.746,0.0,False,CPU +23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1448.0,+125/-102,0.144,23.5,20.2,9.538,1206.7,0.689,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1421.0,+141/-132,0.154,24.88,21.51,11.969,25.13,0.294,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1419.0,+162/-120,0.179,25.0,21.57,6.961,4322.16,0.198,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabPFNv2 (default) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1400.0,+328/-281,0.308,25.94,14.4,7.63,8.68,0.726,12.5,True,GPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1389.0,+147/-121,0.102,26.5,21.98,10.625,10.03,0.115,0.0,False,CPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1360.0,+148/-186,0.138,28.0,22.48,13.632,644.11,0.953,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1340.0,+168/-199,0.169,29.0,21.79,14.036,644.11,0.187,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1340.0,+150/-102,0.113,29.0,24.67,10.515,22.41,7.82,0.0,False,GPU +31,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1328.0,+188/-190,0.12,29.62,16.81,12.542,9006.35,0.527,0.0,False,GPU +32,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1326.0,+164/-137,0.12,29.75,26.79,8.903,25.53,0.154,0.0,False,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1311.0,+132/-150,0.067,30.5,27.26,10.331,5194.73,1.75,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1257.0,+190/-194,0.066,33.25,29.07,10.652,5194.73,0.121,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1241.0,+217/-212,0.097,34.0,22.26,13.421,3991.28,0.402,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1222.0,+233/-228,0.122,34.88,15.45,13.945,3991.28,0.034,0.0,False,CPU +37,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1220.0,+214/-280,0.121,35.0,11.51,15.544,7.0,1.362,0.0,False,GPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1214.0,+100/-178,0.026,35.25,32.69,15.363,657.99,1.135,0.0,False,CPU +39,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1198.0,+92/-125,0.015,36.0,34.93,12.723,4.54,0.374,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1175.0,+76/-124,0.006,37.0,35.99,14.418,2150.84,7.995,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1160.0,+97/-168,0.018,37.62,35.75,15.588,0.46,0.117,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1148.0,+114/-214,0.025,38.12,35.51,16.48,657.99,0.147,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1121.0,+107/-158,0.0,39.25,38.02,14.856,2150.84,0.566,0.0,False,CPU +44,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1114.0,+86/-123,0.0,39.5,38.78,14.012,4.08,0.255,0.0,False,CPU +45,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1055.0,+114/-185,0.0,41.75,40.45,15.454,12.71,0.039,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1030.0,+146/-217,0.0,42.62,41.09,14.656,22.25,0.131,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+92/-204,0.0,43.62,42.96,17.907,0.69,0.125,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,988.0,+106/-216,0.0,44.0,43.14,19.332,6.77,0.511,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,907.0,+147/-333,0.0,46.25,44.95,21.426,88.79,0.572,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,734.0,+153/-330,0.0,49.75,49.48,23.404,88.79,0.052,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,658.0,+224/-609,0.0,50.88,50.11,29.74,0.43,0.047,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,523.0,+100/-474,0.0,52.44,52.35,34.23,742.99,0.364,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,484.0,+130/-555,0.0,52.81,52.72,34.314,742.99,0.097,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,334.0,+99/-505,0.0,54.0,53.99,36.454,3.98,0.12,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..951af4ef779cf3c320d7a75c78b1705435fb2d36 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a88aa7db882b5775021c82fa4b41628af6c471fd32523be36727dffeca3e7038 +size 1493505 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/n_datasets_7 b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/n_datasets_7 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..9770d8ced84ec9f8078f354435f1a647dbd8ee6d --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4e9d30d6fe968d6fcfa038753f9a4f6c6620a372f1f0cafd6eadd9382db811b +size 451328 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..957bf041d72e21a94cae09122e8faa18484ae1fd --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16909d75cbd781c4b5082e06c67a8f8944286346175c7ae0d2269f001ab6720d +size 1044019 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..19129600e032cbfe31c52d2d66186d7a33791222 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f25e9c69fd6e3ccea2977599e19b42743c4969436e0810430cabc9180dc7ab9 +size 120889 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..a69a092ddc81da31abce4a88c97bc436b3de2008 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv @@ -0,0 +1,56 @@ +#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1906.0,+304/-113,0.726,6.43,2.55,3.095,10054.02,249.652,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1900.0,+513/-229,0.772,6.57,2.05,1.229,2279.61,6.732,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1884.0,+400/-86,0.683,7.0,3.48,1.908,669.82,4.041,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1873.0,+180/-50,0.671,7.29,4.98,3.69,10054.02,28.583,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1780.0,+499/-189,0.597,10.14,3.32,2.584,6601.21,73.561,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1772.0,+224/-44,0.565,10.43,6.19,4.944,47.81,37.998,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1756.0,+146/-30,0.551,11.0,9.95,5.167,4840.44,33.436,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1729.0,+403/-218,0.545,12.0,3.79,2.847,2279.61,0.781,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1714.0,+373/-213,0.554,12.57,7.37,4.191,7.82,0.519,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1693.0,+192/-49,0.444,13.43,10.94,6.056,5390.56,5.054,0.0,False,CPU +10,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1658.0,+220/-118,0.398,14.86,7.71,6.637,105.28,2.433,0.0,False,GPU +11,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1624.0,+410/-299,0.518,16.29,5.15,19.429,5.73,0.638,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1605.0,+159/-48,0.287,17.14,15.34,7.956,3301.07,1.317,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1605.0,+257/-127,0.39,17.14,12.18,6.481,4840.44,0.706,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1574.0,+114/-57,0.246,18.57,17.2,9.035,997.12,3.787,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1568.0,+226/-106,0.266,18.86,14.48,8.008,3301.07,0.125,0.0,False,CPU +16,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1558.0,+418/-227,0.406,19.29,10.76,5.384,6601.21,0.588,0.0,False,GPU +17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1544.0,+165/-105,0.295,20.0,14.25,8.299,1709.03,2.724,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1529.0,+188/-118,0.267,20.71,17.7,6.513,4228.53,1.447,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1515.0,+180/-138,0.267,21.43,15.3,10.909,9360.97,7.658,0.0,False,GPU +20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1512.0,+142/-77,0.251,21.57,16.24,8.482,1709.03,0.097,0.0,False,GPU +21,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1469.0,+405/-336,0.352,23.71,13.17,7.531,8.97,0.837,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1466.0,+83/-44,0.098,23.86,23.13,9.543,911.22,0.527,0.0,False,CPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1463.0,+78/-38,0.1,24.0,23.39,9.588,911.22,3.532,0.0,False,CPU +24,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1434.0,+126/-68,0.105,25.43,23.65,10.356,997.12,0.636,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1425.0,+196/-137,0.16,25.86,21.99,7.312,4228.53,0.173,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1422.0,+220/-139,0.117,26.0,21.17,11.174,9.39,0.105,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1407.0,+139/-152,0.121,26.71,24.25,13.106,27.26,0.29,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1392.0,+209/-236,0.158,27.43,21.57,14.555,613.95,0.952,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1384.0,+236/-250,0.193,27.86,20.58,14.881,613.95,0.135,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1366.0,+155/-135,0.129,28.71,23.97,11.015,22.96,8.006,0.0,False,GPU +31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1325.0,+195/-187,0.114,30.71,27.44,9.406,17.97,0.145,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1290.0,+275/-208,0.111,32.43,20.76,14.054,3778.28,0.323,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1282.0,+124/-160,0.026,32.86,31.52,11.205,5937.36,1.406,0.0,False,CPU +34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1276.0,+260/-286,0.138,33.14,10.38,16.185,7.12,1.138,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1272.0,+275/-228,0.14,33.29,14.11,14.636,3778.28,0.028,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1272.0,+146/-174,0.037,33.29,30.98,14.048,9360.97,0.522,0.0,False,GPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1229.0,+155/-260,0.03,35.29,32.39,16.471,628.51,0.812,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1222.0,+185/-280,0.031,35.57,32.35,11.535,5937.36,0.119,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1175.0,+138/-220,0.02,37.57,35.46,16.642,0.43,0.117,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1173.0,+176/-289,0.029,37.64,34.8,17.646,628.51,0.149,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1172.0,+59/-100,0.0,37.71,37.36,13.734,4.41,0.363,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1150.0,+55/-106,0.0,38.57,38.08,15.611,2163.39,7.622,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1114.0,+109/-169,0.0,40.0,39.21,14.895,3.97,0.24,0.0,False,CPU +44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1095.0,+89/-158,0.0,40.71,39.47,15.956,10.54,0.036,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1095.0,+106/-175,0.0,40.71,39.78,16.062,2163.39,0.713,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1002.0,+135/-301,0.0,43.86,42.36,15.699,20.5,0.13,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+131/-217,0.0,43.93,43.18,19.276,0.67,0.118,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,943.0,+77/-213,0.0,45.57,45.12,21.065,6.88,0.452,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,937.0,+178/-381,0.0,45.71,44.31,22.177,86.71,0.53,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,757.0,+168/-358,0.0,49.57,49.27,24.152,86.71,0.05,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,712.0,+245/-524,0.0,50.29,49.48,30.287,0.4,0.044,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,533.0,+85/-893,0.0,52.5,52.4,36.2,753.48,0.227,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,504.0,+117/-984,0.0,52.79,52.68,36.249,753.48,0.091,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,352.0,+97/-674,0.0,54.0,53.99,37.99,4.35,0.119,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..6c7afdc0fc2d3c05d69d554bc5e1e985175cfcb6 --- /dev/null +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad885aa3f6b8056563f89a79a12b7e4ed824df9776bacd8a1a556f5fd353b318 +size 1396925 diff --git a/data_pdfs_to_pngs.py b/data_pdfs_to_pngs.py index 4cf4463cdc67f8ac363fb5364a3f0f0f6b71ce07..6076b0b26ca88c2832d23390c0892d0de6acc143 100644 --- a/data_pdfs_to_pngs.py +++ b/data_pdfs_to_pngs.py @@ -2,30 +2,59 @@ from __future__ import annotations -import glob -import os import zipfile from pathlib import Path +from multiprocessing import Pool, cpu_count from pdf2image import convert_from_path -root_dir = "./data" -pdf_paths = glob.glob(os.path.join(root_dir, "**", "*.pdf"), recursive=True) +ROOT_DIR = Path("./data") +DPI = 800 # you can lower this if files are huge / too slow + + +def process_pdf(pdf_path_str: str) -> None: + pdf_path = Path(pdf_path_str).resolve() + zip_path = pdf_path.with_suffix(".png.zip") -for pdf_path in pdf_paths: - # Relative path to recreate folder structure - path_to_pdf = Path(pdf_path).resolve() - path_to_png = path_to_pdf.with_suffix(".png") - path_to_zip = path_to_pdf.with_suffix(".png.zip") print(f"Converting {pdf_path}...") - images = convert_from_path(pdf_path, dpi=800) - for _i, image in enumerate(images): - image.save(path_to_png, "PNG") + # Convert all pages of the PDF + images = convert_from_path(str(pdf_path), dpi=DPI) + + # Save pages as PNGs (multi-page safe naming) + png_path = pdf_path.with_suffix(".png") + images[0].save(png_path, "PNG") + png_paths = [png_path] + + + # Zip all PNGs into one archive + with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zipf: + for png_path in png_paths: + zipf.write(png_path, arcname=png_path.name) + + # Clean up PNGs and original PDF + for png_path in png_paths: + png_path.unlink(missing_ok=True) + pdf_path.unlink(missing_ok=True) + + +def main() -> None: + pdf_paths = [str(p) for p in ROOT_DIR.rglob("*.pdf")] + + if not pdf_paths: + print("No PDFs found.") + return + + # Use one process per CPU, but not more than number of PDFs + n_procs = min(cpu_count(), len(pdf_paths)) + print(f"Found {len(pdf_paths)} PDFs. Using {n_procs} processes.") + + with Pool(processes=n_procs) as pool: + # imap_unordered gives you streaming results + simple progress printing + for _ in pool.imap_unordered(process_pdf, pdf_paths): + pass - with zipfile.ZipFile(path_to_zip, "w") as zipf: - zipf.write(path_to_png, arcname=path_to_png.name) - path_to_png.unlink(missing_ok=True) - path_to_pdf.unlink(missing_ok=True) +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/main.py b/main.py index 35b3652ac60d1c0727336dc5ecfddb00bfc8b038..9feea251e249ec612b20ef92bfab9125189bf16c 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,11 @@ from dataclasses import dataclass from pathlib import Path import gradio as gr import pandas as pd +import numpy as np + import website_texts +import os +import re from apscheduler.schedulers.background import BackgroundScheduler from constants import Constants, model_type_emoji from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns @@ -19,27 +23,19 @@ from website_texts import ( ) -def get_model_family(model_name: str) -> str: - prefixes_mapping = { - Constants.reference: ["AutoGluon"], - Constants.neural_network: ["REALMLP", "TabM", "FASTAI", "MNCA", "NN_TORCH"], - Constants.tree: ["GBM", "CAT", "EBM", "XGB", "XT", "RF"], - Constants.foundational: ["TABDPT", "TABICL", "TABPFN"], - Constants.baseline: ["KNN", "LR"], - } - - for method_type, prefixes in prefixes_mapping.items(): - for prefix in prefixes: - if prefix.lower() in model_name.lower(): - return method_type - return Constants.other - - @dataclass class LBContainer: name: str base_path_to_results: str - blurb: str + blurb: str | None = None + n_datasets: int | None = None + + def __post_init__(self): + for fname in os.listdir(self._base_path): + match = re.match(r"n_datasets_(.+)", fname) + if match: + self.n_datasets = match.group(1) + break @property def _base_path(self): @@ -177,6 +173,7 @@ def make_overview_leaderboard(lbs: [LBContainer]): ) + def make_leaderboard(lb: LBContainer) -> Leaderboard: df_leaderboard = lb.load_df_leaderboard() @@ -186,17 +183,28 @@ def make_leaderboard(lb: LBContainer) -> Leaderboard: ) df_leaderboard["Only Default"] = df_leaderboard["Model"].str.endswith("(default)") df_leaderboard["Only Tuned"] = df_leaderboard["Model"].str.endswith("(tuned)") - df_leaderboard["Only Tuned + Ensemble"] = df_leaderboard["Model"].str.endswith( - "(tuned + ensemble)" - ) | df_leaderboard["Model"].str.endswith("(4h)") + df_leaderboard["Only Tuned + Ensembled"] = df_leaderboard["Model"].str.endswith( + "(tuned + ensembled)" + ) | df_leaderboard["Model"].str.endswith(", 4h)") + filter_columns = [ ColumnFilter("TypeFiler", type="checkboxgroup", label="🤖 Model Types"), ColumnFilter("Only Default", type="boolean", default=False), ColumnFilter("Only Tuned", type="boolean", default=False), - ColumnFilter("Only Tuned + Ensemble", type="boolean", default=False), + ColumnFilter("Only Tuned + Ensembled", type="boolean", default=False), ] + datatypes = [] + for s in df_leaderboard.T.values: + dt = s.dtype + if dt == bool: + datatypes.append("bool") + elif np.issubdtype(dt, np.number): + datatypes.append("number") + else: + datatypes.append("markdown") + # Add Imputed count postfix if any(df_leaderboard["Imputed"]): df_leaderboard["Imputed"] = df_leaderboard["Imputed"].replace( @@ -205,6 +213,7 @@ def make_leaderboard(lb: LBContainer) -> Leaderboard: False: "Not Imputed", } ) + datatypes.append("bool") filter_columns.append( ColumnFilter( "Imputed", @@ -220,9 +229,9 @@ def make_leaderboard(lb: LBContainer) -> Leaderboard: " being able to run on all datasets.", ) ) - return Leaderboard( value=df_leaderboard, + datatype=datatypes, select_columns=SelectColumns( default_selection=list(df_leaderboard.columns), cant_deselect=["Type", "Model"], @@ -234,13 +243,17 @@ def make_leaderboard(lb: LBContainer) -> Leaderboard: "RefModel", "Only Default", "Only Tuned", - "Only Tuned + Ensemble", + "Only Tuned + Ensembled", "Imputed", ], search_columns=["Model", "TypeName"], filter_columns=filter_columns, bool_checkboxgroup_label="Custom Views (exclusive, only toggle one at a time):", height=800, + column_widths={ + "Model": 200, + } + # interactive=True, ) @@ -267,32 +280,14 @@ class LBMatrix: tasks = ["all", "classification", "regression"] datasets = ["all", "small", "medium", "tabpfn"] - # TODO: get correct numbers - blurb_map_n_datasets = { - "all": { - "all": 51, - "small": 35, - "medium": 16, - "tabpfn": 33, - }, - "classification": { - "all": 30, - "small": 20, - "medium": 10, - "tabpfn": 20, - }, - "regression": { - "all": 21, - "small": 15, - "medium": 6, - "tabpfn": 13, - }, - } - @staticmethod def get_name_for_lb(lb_key, lb_value): if lb_key == "imputation": - return "Models (w/o imputation)" if lb_value == "no" else "Models (with imputation)" + return ( + "Models (w/o imputation)" + if lb_value == "no" + else "Models (with imputation)" + ) if lb_key == "splits": return "All Repeats" if lb_value == "all" else "Lite" if lb_key == "tasks": @@ -319,9 +314,7 @@ class LBMatrix: raise ValueError() raise ValueError() - def element_to_blurb(self, element: LBMatrixElement) -> str: - n_datasets = self.blurb_map_n_datasets[element.tasks][element.datasets] - + def element_to_blurb(self, element: LBMatrixElement, n_datasets: int) -> str: datasets_name = ( element.datasets if element.datasets != "tabpfn" else "TabPFNv2-compatible" ) @@ -375,7 +368,7 @@ def main(): gr.Markdown( website_texts.OVERVIEW_METRICS, elem_classes="markdown-text-box" ) - with gr.Column(), gr.Accordion("📊 Reference Pipeline", open=False): + with gr.Column(), gr.Accordion("📊 Reference Pipelines", open=False): gr.Markdown( website_texts.OVERVIEW_REF_PIPE, elem_classes="markdown-text-box" ) @@ -469,10 +462,13 @@ def main(): lb = LBContainer( name=f"{impute_t_name} | {splits_t} | {tasks_t_name} | {datasets_t_name}", base_path_to_results=lb_element.get_path_to_results(), - blurb=lb_matrix.element_to_blurb( - lb_element - ), ) + + lb.blurb = lb_matrix.element_to_blurb( + lb_element, + n_datasets=lb.n_datasets, + ) + gr.Markdown( lb.blurb, elem_classes="markdown-text", diff --git a/website_texts.py b/website_texts.py index 9540d37d124d5a1c72a0603e7390b17d83842422..f7be75ca03dd29852002da3afe29069a64b875c0 100644 --- a/website_texts.py +++ b/website_texts.py @@ -23,10 +23,12 @@ tuned configurations. Each model is implemented in a tested real-world pipeline optimized to get the most out of the model by the maintainers of TabArena, and where possible together with the authors of the model. -**Unverified Models:** Some models were contributed and evaluated, but have not been verified by the original authors or -maintainers of the model. We indicated whether a model was verified in an extra column in the leaderboard. Results for -unverified and recent models should be interpreted with more caution. Results for unverified but stable models (such as -XGBoost, LightGBM, CatBoost, Random Forests, or baselines) require less caution, even if unverified. +**Verified Models:** Some models were contributed and evaluated, but their implementations have +not been verified by the original authors or maintainers of the model. We indicate whether the implementation +of a model is verified in an extra column in the leaderboard. Results for unverified and recent models should +be interpreted with more caution. We count an older, stable model (such as XGBoost, LightGBM, +CatBoost, Random Forests, or baselines) as verified as long as the its implementation is verified by the +the maintainers of TabArena. """ OVERVIEW_METRICS = """ The leaderboards are ranked based on Elo. We present several additional @@ -42,12 +44,11 @@ less reliable than for `All Repeats` but often present a good proxy for the over cheaper to compute. """ OVERVIEW_REF_PIPE = """ -The leaderboard includes a reference pipeline, which is applied +The leaderboard includes reference pipelines, which are applied independently of the tuning protocol and constraints we constructed for models within TabArena. -The reference pipeline aims to represent the performance quickly achievable by a +A reference pipeline aims to represent the performance quickly achievable by a practitioner on a dataset. The current reference pipeline is the predictive machine -learning system AutoGluon (version 1.3, with the best_quality preset and -4 hours for training). AutoGluon represents an ensemble pipeline across various model +learning system AutoGluon. AutoGluon represents an ensemble pipeline across various model types and thus provides a reference for model-specific pipelines. """ @@ -142,10 +143,10 @@ The following details updates to the leaderboard (date format is YYYY/MM/DD): * 2025/12/01-v0.1.2: Add newest version of TabArena LB for NeurIPS 2025 * New UI and new leaderboard subsets for different dataset sizes, tasks, and imputation + general polish. - * Some metrics have been refacotred and made more stable (see GitHub for details). + * Some metrics have been refactored and made more stable (see GitHub for details). * Updated Reference Pipeline to include AutoGluon v1.4 with the extreme preset. * Updated existing models: RealMLP, TabDPT, EBM - * Add new verified models: Mitra, xRFM, TabPFN-2.5 + * Add new verified models: Mitra, xRFM, RealTabPFN-v2.5 * Add new unverified models: TabFlex, BetaTabPFN, LimiX * 2025/06/13-v0.1.1: Add data for all subsets and re-runs on GPU; Add leaderboards for subsets; new overview; add Figures to LBs.