diff --git a/data/full-cls/figures/critical-diagram.pdf b/data/full-cls/figures/critical-diagram.pdf index 1daefa080392772c3b1a2ed5670c4da458f152fc..77269746b34dfffee8b8ad752f0fd6c1f17bb42c 100644 Binary files a/data/full-cls/figures/critical-diagram.pdf and b/data/full-cls/figures/critical-diagram.pdf differ diff --git a/data/full-cls/figures/critical-diagram.png.zip b/data/full-cls/figures/critical-diagram.png.zip index 944b931e98d50e422b34fb2f61567f648bdbbe71..92897a64494f7b6718d47efa9e80122281e5c581 100644 --- a/data/full-cls/figures/critical-diagram.png.zip +++ b/data/full-cls/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:308a9d416aaaff64659e2ed5f56ed2cf782ca266a5ef0ded86ed835659327f4d -size 295101 +oid sha256:1a2fc785d9b294f00b9e38f8df53bb49dfbdbeb5abb55bdc2e92dec85cfadf37 +size 295239 diff --git a/data/full-cls/time_plot.pdf b/data/full-cls/time_plot.pdf index 87c552f8f53d3e24fe5b646b299638ba5a0269c0..4bcec06413283a7067523a7ac060ba12b8904d71 100644 Binary files a/data/full-cls/time_plot.pdf and b/data/full-cls/time_plot.pdf differ diff --git a/data/full-cls/time_plot.png.zip b/data/full-cls/time_plot.png.zip index 37122cdeb1b531b39b2d2530daa036e5ebf8752f..3ec0d8cd4c547485f23ac12f90574774d11b3c96 100644 --- a/data/full-cls/time_plot.png.zip +++ b/data/full-cls/time_plot.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a1425f58f7556e77814ca63062d6ffbc1d9f5f0b02dfed773d91d750d54675e9 -size 338592 +oid sha256:3191c6c11e4c1e2817a84a68264e8200e2ff1d416108ec10b6a0879fa2e37632 +size 338716 diff --git a/data/full-cls/tuning-impact-elo-horizontal.pdf b/data/full-cls/tuning-impact-elo-horizontal.pdf index 8673de4bd9a473b784efe27f2176b597f1eacb43..538320b6ef9d5568548b30d305791410c842db72 100644 Binary files a/data/full-cls/tuning-impact-elo-horizontal.pdf and b/data/full-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/full-cls/tuning-impact-elo-horizontal.png.zip b/data/full-cls/tuning-impact-elo-horizontal.png.zip index 5ef3fa53850b2d68b328dc51473147be47ff79e0..65c5f7db6ddba203ecc2d0857711d0d921b9f5e5 100644 --- a/data/full-cls/tuning-impact-elo-horizontal.png.zip +++ b/data/full-cls/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:37484f6a707cfe666b8b94eeb6f054fbe0d08aa807104a6d7ee3e03f4849e4ab -size 129171 +oid sha256:fce21001a221ae7a291aeec8471d3e6796a80e564359eb6fc24c11c08f18af3f +size 129333 diff --git a/data/full-cls/tuning-impact-elo.pdf b/data/full-cls/tuning-impact-elo.pdf index cc9b8ee783509145f7f040ecd35942ff9ffd0cf2..07093f4c37356be3b954a20b65211a23617a4631 100644 Binary files a/data/full-cls/tuning-impact-elo.pdf and b/data/full-cls/tuning-impact-elo.pdf differ diff --git a/data/full-cls/tuning-impact-elo.png.zip b/data/full-cls/tuning-impact-elo.png.zip index 19dfcaca01d3368e328477a6b19468a4ea73716f..3e8e0dda8975ad3c598017894e8d677ce26a6d61 100644 --- a/data/full-cls/tuning-impact-elo.png.zip +++ b/data/full-cls/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a3ecf402181f96fdf8623fbd8c1a34553df3cee6308ad17c0be48f153266a2a7 -size 126363 +oid sha256:d98c85c3cf718adbcf14f69f7fd3d0b09e6ba79e530d4754590e164c49490ee7 +size 126503 diff --git a/data/full-imputed-cls/figures/critical-diagram.pdf b/data/full-imputed-cls/figures/critical-diagram.pdf index e5bedda14e922e30e837de6f9751c4e999fc16d8..6dbc600ff7fcf1f3b04ad33d33bf7e13c97a7e01 100644 Binary files a/data/full-imputed-cls/figures/critical-diagram.pdf and b/data/full-imputed-cls/figures/critical-diagram.pdf differ diff --git a/data/full-imputed-cls/figures/critical-diagram.png.zip b/data/full-imputed-cls/figures/critical-diagram.png.zip index 05a97a2c8242d216adca63720b87600448ffba24..4280ed8fe4d8363696c24070325ba360806f4010 100644 --- a/data/full-imputed-cls/figures/critical-diagram.png.zip +++ b/data/full-imputed-cls/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3f39791709f73b5e54c7c8a889281edd9c1b1f4dedddaa62e35ea066760eb0f2 -size 319938 +oid sha256:fbcbd6e5d7a77f5bd3a1dfa8224f012d4c6d07db7845d9c3f8c3a5e33eb71193 +size 320076 diff --git a/data/full-imputed-cls/time_plot.pdf b/data/full-imputed-cls/time_plot.pdf index fb4dae9dbb8ee71eef058e44ff0e78395177e841..c3a8403a17057bca00d3717739f8f574ef93ef11 100644 Binary files a/data/full-imputed-cls/time_plot.pdf and b/data/full-imputed-cls/time_plot.pdf differ diff --git a/data/full-imputed-cls/time_plot.png.zip b/data/full-imputed-cls/time_plot.png.zip index 37122cdeb1b531b39b2d2530daa036e5ebf8752f..4ba3c5a0d90277b9d226dc045dd0508bbb0c64c0 100644 --- a/data/full-imputed-cls/time_plot.png.zip +++ b/data/full-imputed-cls/time_plot.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a1425f58f7556e77814ca63062d6ffbc1d9f5f0b02dfed773d91d750d54675e9 -size 338592 +oid sha256:dd4c29eb2eed3e361316b6519932deb32c082ebed4b5bcbe22a02a87f0cf946d +size 338716 diff --git a/data/full-imputed-cls/tuning-impact-elo-horizontal.pdf b/data/full-imputed-cls/tuning-impact-elo-horizontal.pdf index 6f150df54d2835f8244092073ecb06f54e8fc962..0a694ff7113b30cfb08e0482ac52f1d80570bc3f 100644 Binary files a/data/full-imputed-cls/tuning-impact-elo-horizontal.pdf and b/data/full-imputed-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/full-imputed-cls/tuning-impact-elo-horizontal.png.zip b/data/full-imputed-cls/tuning-impact-elo-horizontal.png.zip index b731c8016827e3f9e408a807292b2603f5ddf005..88bfb7fa7672582dfa59d80f10a0a036a5888570 100644 --- a/data/full-imputed-cls/tuning-impact-elo-horizontal.png.zip +++ b/data/full-imputed-cls/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a078f7afa2868485715760399c09bfafb613d965d83f6812f69cf97fffa1bf1a -size 157281 +oid sha256:51f643e891b93c1ffaf04714c8bd9db56eee2e5f845650e889e47a7309710c9f +size 157443 diff --git a/data/full-imputed-cls/tuning-impact-elo.pdf b/data/full-imputed-cls/tuning-impact-elo.pdf index 4e2cd4b8d4769790352e223f79112c3935dc2b61..f232f7b2cd7e3378adf403f78f7258672ee37e7c 100644 Binary files a/data/full-imputed-cls/tuning-impact-elo.pdf and b/data/full-imputed-cls/tuning-impact-elo.pdf differ diff --git a/data/full-imputed-cls/tuning-impact-elo.png.zip b/data/full-imputed-cls/tuning-impact-elo.png.zip index 4d334d34109b6667816c7fb12ee7ca4aca8b081a..a06bd0bb5c62922109876415ae1322573446eba4 100644 --- a/data/full-imputed-cls/tuning-impact-elo.png.zip +++ b/data/full-imputed-cls/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1c0d78234c9387f4f76d66f49e4864c2afceebe0a4eb87becefd80ea57425b17 -size 172675 +oid sha256:ac4f6ad459d909e85fa865d20f5eb97c8a498f831cf2d74b34bc08e02062bfe6 +size 172815 diff --git a/data/full-imputed-reg/figures/critical-diagram.pdf b/data/full-imputed-reg/figures/critical-diagram.pdf index 0359158a463986d53b8fd23927d617eeb054045f..4cc42f0d8e0c2b1c00b831bbd432c4393767f137 100644 Binary files a/data/full-imputed-reg/figures/critical-diagram.pdf and b/data/full-imputed-reg/figures/critical-diagram.pdf differ diff --git a/data/full-imputed-reg/figures/critical-diagram.png.zip b/data/full-imputed-reg/figures/critical-diagram.png.zip index 74782ea5e9693d23eb730d4cf71550b7a7f199aa..ab2e399d0cac26aff5d3c64cc191bb10c704389e 100644 --- a/data/full-imputed-reg/figures/critical-diagram.png.zip +++ b/data/full-imputed-reg/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1927615c20fa7401bab6e3cbd5a4ebb82562901a05a1efe9ad6f00ca041d364d -size 319836 +oid sha256:dd930723a963e7ec281f56107441e738a92b87022fd570cc34085c46dadffaf0 +size 319974 diff --git a/data/full-imputed-reg/time_plot.pdf b/data/full-imputed-reg/time_plot.pdf index dfd971ed46ac356a5a9c8d41072da5b8d3569fdd..872de41df5549d662484ac7d6e3b01ad04442605 100644 Binary files a/data/full-imputed-reg/time_plot.pdf and b/data/full-imputed-reg/time_plot.pdf differ diff --git a/data/full-imputed-reg/time_plot.png.zip b/data/full-imputed-reg/time_plot.png.zip index 263ba6a3c2cb850a1ba7da44ac69dda4162238c3..9b5af0ec63d4f8998ba92694088ed1cfd474c641 100644 --- a/data/full-imputed-reg/time_plot.png.zip +++ b/data/full-imputed-reg/time_plot.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3fe29fdec65f03c7d3bae994bc888e7c7ec498e3800a153239b08e6703208a4a -size 79911 +oid sha256:2618db1ac376bcf34b1360d1a174827ef25939497c3f481621db8f7f86c1da39 +size 80035 diff --git a/data/full-imputed-reg/tuning-impact-elo-horizontal.pdf b/data/full-imputed-reg/tuning-impact-elo-horizontal.pdf index fa99d13f1764f21347e4e3f84a9183689be4f8cf..723ccdc1bfdb36e8ebca50e789a80fd6ecab02f6 100644 Binary files a/data/full-imputed-reg/tuning-impact-elo-horizontal.pdf and b/data/full-imputed-reg/tuning-impact-elo-horizontal.pdf differ diff --git a/data/full-imputed-reg/tuning-impact-elo-horizontal.png.zip b/data/full-imputed-reg/tuning-impact-elo-horizontal.png.zip index 95a13229786bc6e9b4aaeb3b1925294acd57587c..29d479c31646d17dbe441979c6d18c13ce4812b4 100644 --- a/data/full-imputed-reg/tuning-impact-elo-horizontal.png.zip +++ b/data/full-imputed-reg/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7763698868699976cb6bc6983d84a21cb6289f9fdc372f4c53289e4f0c5818b6 -size 155490 +oid sha256:c598877093106b3daad003e4e2434bb27e9ab9b4af46df8e387ae9e9a024fe26 +size 155652 diff --git a/data/full-imputed-reg/tuning-impact-elo.pdf b/data/full-imputed-reg/tuning-impact-elo.pdf index 0641409dca583318a6381238781bc55a864424df..e1cac68f5b82cd89189a26585f7e6aa848e04b8f 100644 Binary files a/data/full-imputed-reg/tuning-impact-elo.pdf and b/data/full-imputed-reg/tuning-impact-elo.pdf differ diff --git a/data/full-imputed-reg/tuning-impact-elo.png.zip b/data/full-imputed-reg/tuning-impact-elo.png.zip index c0a79f745731d0b1db23454ae7a3111bdc8db8dc..efffaaece448ba695b2987e6f555993f6374441e 100644 --- a/data/full-imputed-reg/tuning-impact-elo.png.zip +++ b/data/full-imputed-reg/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c5b69c34917f2c565e7487c09d55cd3b936da19993feb16027c89ce94c1aa173 -size 162911 +oid sha256:971f39307de7b700a4e6ce60f96779e7c01f02a7597c26247544fc97fe17c64a +size 163051 diff --git a/data/full-imputed/figures/critical-diagram.pdf b/data/full-imputed/figures/critical-diagram.pdf index 654eb1203dc767fb7ff1e50aa55cbcad632c89f9..907c35d8ee39aae2ddff6520ed9a79ae8b9a0b8b 100644 Binary files a/data/full-imputed/figures/critical-diagram.pdf and b/data/full-imputed/figures/critical-diagram.pdf differ diff --git a/data/full-imputed/figures/critical-diagram.png.zip b/data/full-imputed/figures/critical-diagram.png.zip index 9002a5418eaf16c3ee195a7fcea6d7f783cecbf5..ecfca818b1ceedaa5f7079fa72b2faeb71ff83cb 100644 --- a/data/full-imputed/figures/critical-diagram.png.zip +++ b/data/full-imputed/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9f4333e1332194db0036e55bd21228c80b814ff384b793062c746bce92695b65 -size 317258 +oid sha256:f55f2080855d0ae5bf0e3fbaac11b72e4b9e6c0841ad668e79ac166fa2090dca +size 317396 diff --git a/data/full-imputed/time_plot.pdf b/data/full-imputed/time_plot.pdf index 18da66e6ac6e448a5dac1082f270ddec02d78472..9463e5619fd9b64835fa96c7969da327a882ee05 100644 Binary files a/data/full-imputed/time_plot.pdf and b/data/full-imputed/time_plot.pdf differ diff --git a/data/full-imputed/time_plot.png.zip b/data/full-imputed/time_plot.png.zip index 37122cdeb1b531b39b2d2530daa036e5ebf8752f..512f87eb1958bbe6844b2cf27bbd0026aae7e701 100644 --- a/data/full-imputed/time_plot.png.zip +++ b/data/full-imputed/time_plot.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a1425f58f7556e77814ca63062d6ffbc1d9f5f0b02dfed773d91d750d54675e9 -size 338592 +oid sha256:c2f5cfb249d16a440e4f90c1b0bc2eb8099b013654962f621d7b47389bd1c609 +size 338716 diff --git a/data/full-imputed/tuning-impact-elo-horizontal.pdf b/data/full-imputed/tuning-impact-elo-horizontal.pdf index 36584767122541fa34b98ee6d2af6cbd911f4a74..bb784c484a5e4117d0f9d361cea1b9a918671bdc 100644 Binary files a/data/full-imputed/tuning-impact-elo-horizontal.pdf and b/data/full-imputed/tuning-impact-elo-horizontal.pdf differ diff --git a/data/full-imputed/tuning-impact-elo-horizontal.png.zip b/data/full-imputed/tuning-impact-elo-horizontal.png.zip index 9bfad0537f036930805c1ae5c069683289557a01..e0760eafabea054a2f37218f3e9a3d6639e3a946 100644 --- a/data/full-imputed/tuning-impact-elo-horizontal.png.zip +++ b/data/full-imputed/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:40a9013b43f4ad5807a9a23a498f17e8688a0f507366819b883f193c85aa9616 -size 155996 +oid sha256:06d8023bc3849017443207624611fb05c3f3d4a3168d6edbf3302b7626265f95 +size 156158 diff --git a/data/full-imputed/tuning-impact-elo.pdf b/data/full-imputed/tuning-impact-elo.pdf index 23bf8d46a5c33637a6970116d70e9a87c602f3ed..0e06e80976ff03aeb71030398dc48dc79c6c1bf8 100644 Binary files a/data/full-imputed/tuning-impact-elo.pdf and b/data/full-imputed/tuning-impact-elo.pdf differ diff --git a/data/full-imputed/tuning-impact-elo.png.zip b/data/full-imputed/tuning-impact-elo.png.zip index 725b3a584d70c171066dc658e00a2321048bd8c7..1b7ec2af93c3b2f7dd9cb52f30f3d3430146694e 100644 --- a/data/full-imputed/tuning-impact-elo.png.zip +++ b/data/full-imputed/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b06bb49de3089f94392fa747be2a80ad66ae2e3ab15d416956834862b277f2dd -size 168849 +oid sha256:6f2a5632df7e3f3d696d9218a3dce5fead988826551f4f069c3283c52147aeaf +size 168989 diff --git a/data/full-reg/figures/critical-diagram.pdf b/data/full-reg/figures/critical-diagram.pdf index 05a32546ee98d768209b8dc3de52de24c8c34f4e..e1623a7dd5394ace418feb0595ea20b5f3ea46de 100644 Binary files a/data/full-reg/figures/critical-diagram.pdf and b/data/full-reg/figures/critical-diagram.pdf differ diff --git a/data/full-reg/figures/critical-diagram.png.zip b/data/full-reg/figures/critical-diagram.png.zip index 05bf6ba23e3c05ac8a74fe91271598efb7449c5f..7f6708f34881904afa6f72f88e55426a5f362e00 100644 --- a/data/full-reg/figures/critical-diagram.png.zip +++ b/data/full-reg/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ade5ce1686ebdc9790174260de63d2867ffd11045f93679eddae440176ce8e97 -size 294500 +oid sha256:a058b33764916a04b097ce875a3e8664cca6b4ef973f48f21788b67b5a806aea +size 294638 diff --git a/data/full-reg/time_plot.pdf b/data/full-reg/time_plot.pdf index 90922e6262ebb3eb971775e92bf721a1cf0a0a0a..78e04dcd9af5e3c3b8cdaa96e754dd58abf881ed 100644 Binary files a/data/full-reg/time_plot.pdf and b/data/full-reg/time_plot.pdf differ diff --git a/data/full-reg/time_plot.png.zip b/data/full-reg/time_plot.png.zip index 263ba6a3c2cb850a1ba7da44ac69dda4162238c3..ca22d4e135dfc69f447c597c2ce6c951a7928435 100644 --- a/data/full-reg/time_plot.png.zip +++ b/data/full-reg/time_plot.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3fe29fdec65f03c7d3bae994bc888e7c7ec498e3800a153239b08e6703208a4a -size 79911 +oid sha256:e36b4c76827646dd054d4f49b42dbb5d6f7c50f106e9ee89f04b71ec10bea689 +size 80035 diff --git a/data/full-reg/tuning-impact-elo-horizontal.pdf b/data/full-reg/tuning-impact-elo-horizontal.pdf index 8ab93305764ec10694e573706f10749d661d127d..4e94f6a7eb3b591fb4d436ec76e58f9361077c09 100644 Binary files a/data/full-reg/tuning-impact-elo-horizontal.pdf and b/data/full-reg/tuning-impact-elo-horizontal.pdf differ diff --git a/data/full-reg/tuning-impact-elo-horizontal.png.zip b/data/full-reg/tuning-impact-elo-horizontal.png.zip index afe3867d082612c2e91501f734d1dac3b920e392..57098b1b96c9d3e7a8fceb5e2a312881fa240ffe 100644 --- a/data/full-reg/tuning-impact-elo-horizontal.png.zip +++ b/data/full-reg/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e1d5f881271b5a00e3b778451ad67bace45bd811aab630061c78057f191972c3 -size 129887 +oid sha256:8303a217f846ea50091e856dc461e97847a8641faa7b4dbe6ae22c679e142e61 +size 130049 diff --git a/data/full-reg/tuning-impact-elo.pdf b/data/full-reg/tuning-impact-elo.pdf index 3bee6e8cdcd324aab499ef8c9d8694cd8a1120f2..b8764423e2bce174de6651e640d47c49a64991e4 100644 Binary files a/data/full-reg/tuning-impact-elo.pdf and b/data/full-reg/tuning-impact-elo.pdf differ diff --git a/data/full-reg/tuning-impact-elo.png.zip b/data/full-reg/tuning-impact-elo.png.zip index c20e3e0f66bfd81ed643341f4e1ced5c1d314601..876dde45c4afe844e396d98ed27bb65c4efc23c4 100644 --- a/data/full-reg/tuning-impact-elo.png.zip +++ b/data/full-reg/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:48a53db8aa38c1ed4c8785923113986e05d6b398ff01a02150bf521dbad9d5be -size 131368 +oid sha256:643b461960219aedf710844b03d9f4e13735bf0533e5c35c52358f1d08500b69 +size 131508 diff --git a/data/full/figures/critical-diagram.pdf b/data/full/figures/critical-diagram.pdf index d93ca7dd961de52243a3db2bbdf6d240ea15e668..87525c1533e0ca216e1fd4acbebe62b890dcbea5 100644 Binary files a/data/full/figures/critical-diagram.pdf and b/data/full/figures/critical-diagram.pdf differ diff --git a/data/full/figures/critical-diagram.png.zip b/data/full/figures/critical-diagram.png.zip index e9f3fa98dc88711314c65417b79f3997c3a6af49..e762749e91623d22d2dcc9bb4e27df6fec9ff86b 100644 --- a/data/full/figures/critical-diagram.png.zip +++ b/data/full/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:15ef6dc5b13370dc816b3db4e65ebfd7e1bbffda30630b5beed17cf780e6443a -size 294416 +oid sha256:89e96ea5d3556565ae161982375990a620e51862ee8651296159f479349970ab +size 294554 diff --git a/data/full/time_plot.pdf b/data/full/time_plot.pdf index c861a227698ca3f02bb107756bdde8c1852197b6..57883b5ff38b77bd92ac2fd8389d8e1d895db5bb 100644 Binary files a/data/full/time_plot.pdf and b/data/full/time_plot.pdf differ diff --git a/data/full/time_plot.png.zip b/data/full/time_plot.png.zip index 37122cdeb1b531b39b2d2530daa036e5ebf8752f..f102ccd615b0ba16998c69dec65b33d5387685b8 100644 --- a/data/full/time_plot.png.zip +++ b/data/full/time_plot.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a1425f58f7556e77814ca63062d6ffbc1d9f5f0b02dfed773d91d750d54675e9 -size 338592 +oid sha256:b6231ebab7e8185c187c0d235aec014a5a75d2cfea91eac51177af59a238508d +size 338716 diff --git a/data/full/tuning-impact-elo-horizontal.pdf b/data/full/tuning-impact-elo-horizontal.pdf index 0e1072b99e29499c07680838b06d84e029b29788..d7de76e6e4ffb4f9b0c3d0a28912a2be18dfcb16 100644 Binary files a/data/full/tuning-impact-elo-horizontal.pdf and b/data/full/tuning-impact-elo-horizontal.pdf differ diff --git a/data/full/tuning-impact-elo-horizontal.png.zip b/data/full/tuning-impact-elo-horizontal.png.zip index c433d9669a016f04e47cd229da721e21f36d6163..3fa3543a6884da66a3ae20d59584d3fde5aaaea7 100644 --- a/data/full/tuning-impact-elo-horizontal.png.zip +++ b/data/full/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e8ca383234264edb2f91b7ceb983dbec4663cf7525f4cf1fc75a12cfef8b89dd -size 129065 +oid sha256:580b87c2ed20b862b92fdcf02c922b29d47c65515be423ef60a17d87523944bb +size 129227 diff --git a/data/full/tuning-impact-elo.pdf b/data/full/tuning-impact-elo.pdf index fe71f5b2d86e8b7712e7104eec0e385ef17aff02..a4096ba1b1d50b72cc9794372d481c6d8338637a 100644 Binary files a/data/full/tuning-impact-elo.pdf and b/data/full/tuning-impact-elo.pdf differ diff --git a/data/full/tuning-impact-elo.png.zip b/data/full/tuning-impact-elo.png.zip index fb1a0019a4401584b3f58c50723737e5b6d23601..d025b8d159062d1c1c185a8fa1a4875c9e093866 100644 --- a/data/full/tuning-impact-elo.png.zip +++ b/data/full/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8e85834f7ee7086c3ffdd269028fb734dab4ae9488a5e4600a610c3bd5390736 -size 126040 +oid sha256:58ea2363c280e561a4b299713b32426911d315036e31b727391d0fd1774a22f1 +size 126180 diff --git a/data/lite/full-cls/figures/critical-diagram.pdf b/data/lite/full-cls/figures/critical-diagram.pdf index fcba36cf5f60298f6cc49fa20f954977818ca780..112675ec3d8b0f88d11245005e025b8ad9f4f7fb 100644 Binary files a/data/lite/full-cls/figures/critical-diagram.pdf and b/data/lite/full-cls/figures/critical-diagram.pdf differ diff --git a/data/lite/full-cls/figures/critical-diagram.png.zip b/data/lite/full-cls/figures/critical-diagram.png.zip index fb1c72a0f47700992a4bbc21723ed521c2ed270c..0eec83210d4f1c9eedebe21fce9d5f177bebf32b 100644 --- a/data/lite/full-cls/figures/critical-diagram.png.zip +++ b/data/lite/full-cls/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a4f696da207039843ab87b576f7f02cf853c814c3b21cac719bfaf87a11d8558 -size 295197 +oid sha256:105dd142265ffe7e8f43759b3effd1a0bec3f931f72541d642fde463505936c4 +size 295335 diff --git a/data/lite/full-cls/time_plot.pdf b/data/lite/full-cls/time_plot.pdf index 19364a1a41aca9a0e5c3d5172acf719e33a175a7..4ee9615ec169c83fdcf36960a80713821220e5b9 100644 Binary files a/data/lite/full-cls/time_plot.pdf and b/data/lite/full-cls/time_plot.pdf differ diff --git a/data/lite/full-cls/time_plot.png.zip b/data/lite/full-cls/time_plot.png.zip index 8b42c4b6e2232a76fd9e3d1f5ee7ee2f360f2d09..2174750c5d32abfb078a141a0359c81b7c31bbf5 100644 --- a/data/lite/full-cls/time_plot.png.zip +++ b/data/lite/full-cls/time_plot.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4f0389b7f203eb4d39d6b6a3da311337a46c01621e12fc206d904eff8ce58527 -size 337899 +oid sha256:fb63f537ae37db0baa7dd2aced15f9bf3914872657f3236780f18785f244136b +size 338023 diff --git a/data/lite/full-cls/tuning-impact-elo-horizontal.pdf b/data/lite/full-cls/tuning-impact-elo-horizontal.pdf index bd2bbd6c2d88e3c17f415c27abdbd1920b5aa370..b28273094b0be8ffe177bc7074da7a05b2aabaee 100644 Binary files a/data/lite/full-cls/tuning-impact-elo-horizontal.pdf and b/data/lite/full-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/full-cls/tuning-impact-elo-horizontal.png.zip b/data/lite/full-cls/tuning-impact-elo-horizontal.png.zip index fe0ff9a7960de4149eaee744b8c19171d2bd2ff5..921bb427d35953172e023e0bbbf3e457447f1595 100644 --- a/data/lite/full-cls/tuning-impact-elo-horizontal.png.zip +++ b/data/lite/full-cls/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4a737dcd6d82a8040b8eb82b9d90664185427b808cdef322a20c51796c88f15c -size 128533 +oid sha256:7e386e77eb8f920b86b602ade74b8b8ec897a5364e4b2997360871921a6352e1 +size 128695 diff --git a/data/lite/full-cls/tuning-impact-elo.pdf b/data/lite/full-cls/tuning-impact-elo.pdf index 628097f297eb220c9f6f25adced9cfdf98a316f9..e0e4ead02f12161e7a9dbf0b888aad2155c2bbca 100644 Binary files a/data/lite/full-cls/tuning-impact-elo.pdf and b/data/lite/full-cls/tuning-impact-elo.pdf differ diff --git a/data/lite/full-cls/tuning-impact-elo.png.zip b/data/lite/full-cls/tuning-impact-elo.png.zip index 548c5b62d0aa0ac9670259f66fb36291c4cef5ed..b5e405ec8f825b88dceb0d0a5d94dea73e6fb918 100644 --- a/data/lite/full-cls/tuning-impact-elo.png.zip +++ b/data/lite/full-cls/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2b7f6bd0146921670f8452f5a59e6970208f4dbb60786b6ec8b535284d5fb72e -size 124422 +oid sha256:eb37b1f71f5af16a61a859dfbf47817ee535c09d0f2ddbbda92a0c5d1fbd092b +size 124562 diff --git a/data/lite/full-imputed-cls/figures/critical-diagram.pdf b/data/lite/full-imputed-cls/figures/critical-diagram.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ee6edfea8cc46b34ed5680163b14dc6886139a37 Binary files /dev/null and b/data/lite/full-imputed-cls/figures/critical-diagram.pdf differ diff --git a/data/lite/full-imputed-cls/figures/critical-diagram.png.zip b/data/lite/full-imputed-cls/figures/critical-diagram.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d2b055498a93e0962764957f3d39b9662da13a92 --- /dev/null +++ b/data/lite/full-imputed-cls/figures/critical-diagram.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0441a6a1f482b1d6300d3ac5aa850102ad66e105d4a8e81ce23cd03cc9f52ec +size 323319 diff --git a/data/lite/full-imputed-cls/leaderboard.tex b/data/lite/full-imputed-cls/leaderboard.tex new file mode 100644 index 0000000000000000000000000000000000000000..e3dbdb98f308c378de9891682ed99a4895c54386 --- /dev/null +++ b/data/lite/full-imputed-cls/leaderboard.tex @@ -0,0 +1,53 @@ +\begin{tabular}{llcccccrr} +\toprule +\textbf{Model} & \textbf{Elo ($\uparrow$)} & \textbf{Norm.} & \textbf{Avg.} & \textbf{Harm.} & \textbf{\#wins ($\uparrow$)} & \textbf{Improva-} & \textbf{Train time} & \textbf{Predict time} \\ + & & \textbf{score ($\uparrow$)} & \textbf{rank ($\downarrow$)} & \textbf{mean} & & \textbf{bility ($\downarrow$)} & \textbf{per 1K [s]} & \textbf{per 1K [s]} \\ + & & & & \textbf{rank ($\downarrow$)} & & & & \\ +\midrule +AutoGluon 1.3 (4h) & \textcolor{gold}{\textbf{1509${}_{-30,+33}$}} & \textcolor{gold}{\textbf{0.564}} & \textcolor{gold}{\textbf{8.6}} & \textcolor{gold}{\textbf{3.4}} & \textcolor{gold}{\textbf{6}} & \textcolor{gold}{\textbf{7.6\%}} & 1355.34 & 1.98 \\ +RealMLP (T+E) & \textcolor{silver}{\textbf{1413${}_{-31,+32}$}} & 0.419 & \textcolor{silver}{\textbf{12.4}} & 7.8 & 0 & 11.2\% & 5463.43 & 3.35 \\ +TabM (T+E) & \textcolor{bronze}{\textbf{1409${}_{-29,+27}$}} & \textcolor{bronze}{\textbf{0.492}} & \textcolor{silver}{\textbf{12.4}} & 5.7 & 2 & \textcolor{silver}{\textbf{10.7\%}} & 2461.81 & 1.28 \\ +LightGBM (T+E) & 1403${}_{-28,+27}$ & 0.404 & 12.7 & 8.7 & 0 & 11.9\% & 391.23 & 1.24 \\ +CatBoost (T+E) & 1376${}_{-33,+37}$ & 0.379 & 13.9 & 8.1 & 0 & 11.1\% & 1326.58 & 0.53 \\ +XGBoost (T+E) & 1375${}_{-27,+21}$ & 0.321 & 14.1 & 7.1 & 1 & 12.4\% & 682.62 & 0.97 \\ +TabICL (D) & 1370${}_{-27,+26}$ & \textcolor{silver}{\textbf{0.501}} & 14.3 & \textcolor{bronze}{\textbf{4.8}} & \textcolor{bronze}{\textbf{4}} & \textcolor{bronze}{\textbf{10.9\%}} & 9.07 & 2.01 \\ +CatBoost (T) & 1355${}_{-23,+29}$ & 0.365 & 14.9 & 9.7 & 0 & 11.7\% & 1326.58 & 0.05 \\ +CatBoost (D) & 1354${}_{-32,+27}$ & 0.345 & 15.1 & 7.9 & 2 & 12.0\% & 5.74 & 0.11 \\ +TabM (T) & 1353${}_{-32,+25}$ & 0.407 & 15.2 & 7.3 & 0 & 11.7\% & 2461.81 & 0.14 \\ +ModernNCA (T) & 1352${}_{-30,+26}$ & 0.278 & 15.3 & 7.0 & 2 & 11.6\% & 4853.07 & 0.47 \\ +LightGBM (T) & 1345${}_{-36,+30}$ & 0.318 & 15.7 & 10.6 & 0 & 12.9\% & 391.23 & 0.19 \\ +ModernNCA (T+E) & 1344${}_{-25,+28}$ & 0.373 & 15.6 & 6.1 & 2 & 11.6\% & 4853.07 & 9.39 \\ +XGBoost (T) & 1342${}_{-25,+27}$ & 0.266 & 15.7 & 8.0 & 1 & 12.7\% & 682.62 & 0.18 \\ +TabPFNv2 (T+E) & 1329${}_{-31,+24}$ & 0.486 & 16.2 & \textcolor{silver}{\textbf{3.9}} & \textcolor{gold}{\textbf{6}} & 12.5\% & 2914.83 & 17.60 \\ +TabM (D) & 1286${}_{-26,+22}$ & 0.271 & 18.6 & 9.4 & 1 & 14.9\% & 10.31 & 0.14 \\ +TabPFNv2 (T) & 1279${}_{-25,+31}$ & 0.379 & 18.8 & 6.2 & 1 & 14.3\% & 2914.83 & 0.27 \\ +RealMLP (T) & 1269${}_{-27,+24}$ & 0.202 & 19.4 & 13.0 & 0 & 14.8\% & 5463.43 & 0.19 \\ +EBM (T+E) & 1268${}_{-28,+23}$ & 0.180 & 19.3 & 9.9 & 0 & 16.5\% & 1011.59 & 0.22 \\ +TorchMLP (T+E) & 1268${}_{-30,+31}$ & 0.222 & 19.5 & 13.3 & 0 & 14.1\% & 2480.32 & 2.18 \\ +TabPFNv2 (D) & 1239${}_{-32,+28}$ & 0.348 & 20.9 & 6.1 & 3 & 15.3\% & 3.30 & 0.31 \\ +EBM (T) & 1236${}_{-35,+22}$ & 0.122 & 21.2 & 12.6 & 0 & 17.1\% & 1011.59 & 0.02 \\ +FastaiMLP (T+E) & 1204${}_{-33,+28}$ & 0.196 & 22.9 & 13.9 & 0 & 17.8\% & 622.79 & 5.04 \\ +EBM (D) & 1199${}_{-31,+23}$ & 0.136 & 23.0 & 11.5 & 1 & 18.1\% & 5.44 & 0.06 \\ +XGBoost (D) & 1195${}_{-33,+26}$ & 0.105 & 23.2 & 12.4 & 1 & 15.9\% & 1.88 & 0.12 \\ +ExtraTrees (T+E) & 1194${}_{-29,+20}$ & 0.104 & 23.3 & 13.5 & 0 & 17.6\% & 197.16 & 0.72 \\ +TorchMLP (T) & 1188${}_{-27,+24}$ & 0.096 & 23.7 & 17.1 & 0 & 16.4\% & 2480.32 & 0.12 \\ +RealMLP (D) & 1180${}_{-23,+26}$ & 0.090 & 24.0 & 13.0 & 1 & 16.6\% & 34.76 & 0.19 \\ +ModernNCA (D) & 1175${}_{-29,+27}$ & 0.134 & 24.2 & 12.2 & 1 & 18.1\% & 13.35 & 0.35 \\ +RandomForest (T+E) & 1161${}_{-24,+28}$ & 0.107 & 24.9 & 13.4 & 0 & 18.2\% & 327.31 & 0.71 \\ +ExtraTrees (T) & 1157${}_{-32,+29}$ & 0.079 & 25.2 & 12.5 & 0 & 18.8\% & 197.16 & 0.08 \\ +LightGBM (D) & 1145${}_{-30,+26}$ & 0.076 & 25.9 & 21.3 & 0 & 17.4\% & 2.25 & 0.16 \\ +TabDPT (D) & 1136${}_{-31,+26}$ & 0.176 & 26.1 & 9.4 & 1 & 18.7\% & 22.39 & 8.59 \\ +FastaiMLP (T) & 1126${}_{-25,+27}$ & 0.089 & 26.7 & 17.2 & 0 & 19.8\% & 622.79 & 0.35 \\ +RandomForest (T) & 1118${}_{-31,+26}$ & 0.076 & 27.1 & 20.5 & 0 & 19.1\% & 327.31 & 0.08 \\ +TorchMLP (D) & 1023${}_{-26,+26}$ & 0.021 & 31.4 & 27.4 & 0 & 22.3\% & 5.73 & 0.15 \\ +RandomForest (D) & 1000${}_{-0,+0}$ & 0.011 & 32.3 & 26.3 & 0 & 25.7\% & 0.37 & 0.04 \\ +FastaiMLP (D) & 995${}_{-34,+24}$ & 0.026 & 32.7 & 28.7 & 0 & 23.9\% & 3.12 & 0.33 \\ +Linear (T+E) & 929${}_{-33,+24}$ & 0.042 & 35.0 & 18.3 & 1 & 32.0\% & 51.96 & 0.17 \\ +ExtraTrees (D) & 922${}_{-26,+29}$ & 0.010 & 35.3 & 31.2 & 0 & 28.6\% & 0.24 & 0.04 \\ +Linear (T) & 900${}_{-33,+24}$ & 0.026 & 36.1 & 27.8 & 0 & 32.7\% & 51.96 & 0.06 \\ +Linear (D) & 872${}_{-30,+28}$ & 0.019 & 36.9 & 34.6 & 0 & 33.6\% & 1.62 & 0.09 \\ +KNN (T+E) & 735${}_{-35,+48}$ & 0.000 & 40.2 & 35.0 & 0 & 48.7\% & 3.51 & 0.16 \\ +KNN (T) & 675${}_{-43,+47}$ & 0.000 & 41.4 & 31.3 & 0 & 50.8\% & 3.51 & 0.04 \\ +KNN (D) & 482${}_{-60,+54}$ & 0.000 & 43.7 & 43.2 & 0 & 59.2\% & 0.07 & 0.02 \\ +\bottomrule +\end{tabular} \ No newline at end of file diff --git a/data/lite/full-imputed-cls/tabarena_leaderboard.csv b/data/lite/full-imputed-cls/tabarena_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..d0d29ac01118bbee20d7d5544ccd4c0b2b36f034 --- /dev/null +++ b/data/lite/full-imputed-cls/tabarena_leaderboard.csv @@ -0,0 +1,46 @@ +method,time_train_s,time_infer_s,time_train_s_per_1K,time_infer_s_per_1K,normalized-error,normalized-error-task,champ_delta,loss_rescaled,time_train_s_rescaled,time_infer_s_rescaled,rank,median_metric_error,median_time_train_s,median_time_infer_s,median_time_train_s_per_1K,median_time_infer_s_per_1K,median_normalized-error,median_normalized-error-task,median_champ_delta,median_loss_rescaled,median_time_train_s_rescaled,median_time_infer_s_rescaled,median_rank,rank=1_count,rank=2_count,rank=3_count,rank>3_count,elo,elo+,elo-,winrate,mrr +AutoGluon 1.3 (4h),7774.991528053033,19.875839289865997,2811.0795427294815,3.0481623340146933,0.4357260444094882,0.4148361283414099,0.07629365001402376,0.04227584471563422,29846.24559964093,265.3700814985468,8.56578947368421,0.157955,7014.845859646797,3.7256141901016235,1355.3359335244184,1.9841131308583402,0.3593825434326649,0.36790766468039304,0.04084708943083559,0.018367490316605305,21384.37692728419,133.53161235412554,7.0,6,3,2,27,1508.7,32.2,29.9,0.8280502392344498,0.2960718604294456 +TABM (tuned + ensemble),34310.62852196317,8.826874965115598,4300.923574958667,3.470904002174896,0.5077918677314276,0.5397861128157506,0.10732779626894647,0.08292330393026169,45131.79918122951,135.34700738700144,12.368421052631579,0.174405,8260.626611948013,3.27972948551178,2461.8116026347016,1.282862647131695,0.47136645123500964,0.5343939211655224,0.07139416682385408,0.03201326752328171,37327.19260107514,113.73588992793104,9.5,2,0,3,33,1409.2,26.5,28.8,0.7416267942583732,0.17514469468110425 +REALMLP (tuned + ensemble),75157.8044012283,16.487929639063385,5934.025599797772,7.762023296294149,0.5811880944579212,0.5771850324578947,0.11170245391763495,0.06751478344816422,134658.591064252,301.4009053371881,12.394736842105264,0.17028,21574.945648550987,6.0496333837509155,5463.431997434496,3.3505522191847916,0.5453504649932683,0.5676318375241935,0.06341551053522204,0.038644322872270584,90262.54064674574,233.90844576360837,12.0,0,1,2,35,1413.4,31.7,30.7,0.7410287081339713,0.12831564219626773 +GBM (tuned + ensemble),2920.3429353990055,11.208824527891059,755.2524337234863,2.3454352129897833,0.5957286671834613,0.6188878533536122,0.11902465875946111,0.08143933293433607,8248.756938263266,205.60882328189754,12.736842105263158,0.169945,1594.5504041910172,3.0867438316345215,391.23230441146256,1.2433237336531138,0.6251845339504845,0.6651492045932685,0.07096598380907965,0.02467304271820576,6721.07699522549,94.37802750757459,13.0,0,0,4,34,1403.3,26.8,27.9,0.7332535885167464,0.11444448956145252 +CAT (tuned + ensemble),16302.859700416264,2.6250121342508415,3325.427412489986,0.969863744839929,0.6209138650557025,0.6012798381645563,0.11073081767243305,0.06744303583601241,25676.604134391517,50.871466139342466,13.947368421052632,0.16073500000000002,4791.775886774063,1.3303372859954834,1326.5813740920516,0.5313769242211359,0.6153632608570143,0.5995830509750808,0.05451486125001109,0.031433267034335854,19895.263297992577,42.031537823549,13.0,0,1,4,33,1376.3,36.3,33.0,0.7057416267942583,0.12340182350966956 +XGB (tuned + ensemble),5799.34833753109,5.967870285636501,1147.820903083116,2.46393255534894,0.6794057655741552,0.6446090386925779,0.12424768982222067,0.08852863039271704,10132.57248871615,131.35779311288925,14.118421052631579,0.17348,1593.032392859459,2.3060801029205322,682.6224605815719,0.9713359475135803,0.7126419091535116,0.6563884597908853,0.0820999234905303,0.03265793856149843,7276.753308183146,76.46662354595179,12.5,1,2,2,33,1375.1,21.0,26.4,0.7018540669856459,0.14090383005978221 +TABICL (default),117.48550770157262,21.35919719620755,9.848013920703847,2.2947164283388504,0.49900795570726975,0.5915798274957776,0.10905021189648662,0.07891972794319439,170.36713500512033,270.6328952906184,14.276315789473685,0.17301,25.354220509529114,3.3787535429000854,9.073519792448254,2.007502492806867,0.5497919378000328,0.6652418475567421,0.05597714417034555,0.024738647104901798,133.3927784174379,132.9127572018825,14.0,4,2,2,30,1370.3,25.2,26.9,0.6982655502392344,0.2064602032416652 +CAT (tuned),16302.859700416264,0.5146497425280119,3325.427412489986,0.11950448243427039,0.6346801229162512,0.6385490550731812,0.11685920690054714,0.0718088688715085,25676.604134391517,8.18618115589542,14.868421052631579,0.16238999999999998,4791.775886774063,0.08605468273162842,1326.5813740920516,0.051041753580035654,0.6314852452199684,0.6931108990698708,0.06761434367525188,0.03776034888457011,19895.263297992577,3.2560454821039713,13.75,0,1,0,37,1354.6,28.6,22.6,0.6848086124401914,0.10319970806930756 +CAT (default),218.94677375492296,0.43937695026397705,108.24085958784511,0.1671626095074056,0.6551829671744798,0.6594240859585678,0.12000325153636153,0.07271915532555678,424.08805051709817,10.569617726030572,15.092105263157896,0.161695,18.44840371608734,0.411457896232605,5.736216485552211,0.10729215986107409,0.6872314742824219,0.7131246390179387,0.07956212107816629,0.02945410818367972,91.81803735744086,6.75924675825947,15.0,2,0,0,36,1353.7,26.2,31.8,0.6797248803827751,0.126524140784773 +TABM (tuned),34310.62852196317,1.0356075324510272,4300.923574958667,0.38276682295841274,0.5928893193157706,0.6007491246657526,0.1168573411078604,0.09752336795906187,45131.79918122951,14.823042284101287,15.197368421052632,0.175585,8260.626611948013,0.24344730377197266,2461.8116026347016,0.13835550204254063,0.5877464635288101,0.6244854782324665,0.08334874690385935,0.04117730891431355,37327.19260107514,9.329646701651438,14.0,0,3,3,32,1352.8,24.9,31.5,0.6773325358851675,0.13715971824475107 +MNCA (tuned),56939.09770153698,19.318985493559587,6212.23493998028,1.9628894017927039,0.721681429841744,0.625842107367332,0.11613801810420087,0.09229473197826113,79415.11574896757,126.43602273733568,15.302631578947368,0.175715,14329.4857878685,0.6404991149902344,4853.0743321238315,0.4731015689224996,0.7744612480983963,0.6505383469786996,0.08668289690877401,0.03213571989092236,67721.7850080898,32.352773289608024,13.0,2,0,0,36,1351.5,25.2,29.8,0.6749401913875598,0.142342524659497 +MNCA (tuned + ensemble),56939.09770153698,351.628879396539,6212.23493998028,38.90413717422724,0.6269167185273185,0.5984531354976621,0.11573636754109493,0.09313892703335769,79415.11574896757,2408.3540246144216,15.578947368421053,0.18282500000000002,14329.4857878685,13.981043934822083,4853.0743321238315,9.386068319220197,0.6265388929939845,0.5699786298554819,0.06347089264029215,0.0430523430616711,67721.7850080898,659.6644218615952,12.5,2,1,2,33,1343.7,27.1,24.5,0.6686602870813397,0.16394468484752917 +XGB (tuned),5799.34833753109,1.27206594065616,1147.820903083116,0.6531124582306008,0.7344435643624297,0.6866009006886469,0.12705232557945453,0.09426550030491268,10132.57248871615,32.25362424926425,15.68421052631579,0.176915,1593.032392859459,0.36533260345458984,682.6224605815719,0.18295137338644407,0.7444722954009122,0.7353840158409348,0.08553173078110604,0.03469633441846848,7276.753308183146,12.253516308447622,14.5,1,2,0,35,1341.5,26.7,24.7,0.666267942583732,0.12466050435587736 +GBM (tuned),2920.3429353990055,1.8159009845633256,755.2524337234863,0.5236582654503446,0.6821201235665096,0.6795255137349172,0.12859160917332496,0.09636240870350794,8248.756938263266,42.22279271596955,15.736842105263158,0.16955,1594.5504041910172,0.5379931926727295,391.23230441146256,0.18570879781926358,0.7168421399910502,0.7237674445143973,0.07807806844489962,0.03627576854264193,6721.07699522549,11.394220763848338,15.0,0,1,0,37,1344.9,29.5,35.9,0.6650717703349283,0.09415249407771492 +TABPFNV2 (tuned + ensemble),10671.813733621648,114.13957280861705,2717.754849558356,48.747249045567855,0.5138103973952525,0.5736551796907117,0.12518126985862188,0.10096148528564704,50731.962016917816,4329.537285428793,16.223684210526315,0.17508,3350.918938755989,9.716811895370483,2914.8333213375054,17.596511061575676,0.5072451025993939,0.623606465997792,0.10124565298109173,0.043994374641617755,26184.067235259834,926.3134038480441,12.0,6,1,3,28,1329.4,23.1,30.2,0.6540071770334929,0.25509079179329364 +TABM (default),151.95178728354605,1.2483709925099422,21.512717277386358,0.45562303344686134,0.7289050360420961,0.7095114237312439,0.1491348082432036,0.12147454771537929,207.28000362311982,15.083359148300596,18.55263157894737,0.17709,33.152939319610596,0.19442057609558105,10.309423174430956,0.13586267957403653,0.8404844924976476,0.7655355529381327,0.09137336521284833,0.05094980825106373,135.49680849639785,10.343117540463151,17.0,1,0,1,36,1286.2,22.0,25.3,0.6010765550239234,0.10671155275919082 +TABPFNV2 (tuned),10671.813733621648,4.116424629562779,2717.754849558356,2.010495247924752,0.6207514015330685,0.6835508214944778,0.14300784416015525,0.11546900554343373,50731.962016917816,143.74411000718078,18.81578947368421,0.19132,3350.918938755989,0.3354600667953491,2914.8333213375054,0.2713678077236172,0.7315480103218319,0.9059143811887247,0.1215657105488207,0.058112493187242,26184.067235259834,15.110476058190766,18.75,1,5,2,30,1278.6,30.7,24.8,0.5950956937799043,0.16250460654768312 +EBM (tuned + ensemble),36486.59858077451,1.3923115416576988,7650.518716752531,0.5440226127306762,0.8200760362636922,0.7691165482685514,0.16463001854426568,0.1279756133779474,21740.48293152139,22.192423380997223,19.276315789473685,0.177435,2273.2198663949966,0.35086679458618164,1011.590888307647,0.2249400302454423,0.9326307523783561,0.8842854370429472,0.07547386841907339,0.04276402555620501,14796.589958532411,12.4343949892923,19.75,0,3,0,35,1268.3,22.3,27.2,0.5846291866028708,0.10124857161300282 +REALMLP (tuned),75157.8044012283,0.8101433327323512,5934.025599797772,0.42781608188450077,0.7977234575542793,0.7669539874246226,0.14830098147293377,0.10754379040274052,134658.591064252,14.869687448366182,19.42105263157895,0.16913499999999998,21574.945648550987,0.25513756275177,5463.431997434496,0.18623669559057404,0.8787741747777617,0.8265671653596647,0.1283441158885112,0.05736216794816598,90262.54064674574,11.894743904291648,18.0,0,1,0,37,1268.8,23.6,26.6,0.5813397129186603,0.07706249998982605 +NN_TORCH (tuned + ensemble),24155.06064368549,16.430658089487178,3018.869066392961,4.171284511368181,0.77831117981961,0.790906523513234,0.14101067792448846,0.09752188412518956,54165.934988934685,250.8131490307278,19.5,0.174655,9535.178918361664,4.419585108757019,2480.3150361602966,2.175346818589723,0.9031626632034823,0.917687560600783,0.0823266490663982,0.05626679601192999,43976.90578112008,186.31979424186875,20.5,0,0,1,37,1267.5,30.3,29.3,0.5795454545454546,0.07546603253162487 +TABPFNV2 (default),11.52735658068406,0.9154732164583708,4.096023954678057,0.45171050576219374,0.6515996652210705,0.7301122747015034,0.15260072650305895,0.12460000602465107,52.507166370564995,35.58086285535624,20.907894736842106,0.194575,8.611271619796753,0.30159199237823486,3.2954189873915243,0.305340033795448,0.782945687586422,0.9946213191350618,0.10102667031111617,0.0836253292155732,33.35727994934,18.648722781582272,22.5,3,2,2,31,1239.4,27.5,31.1,0.5475478468899522,0.16387890635230992 +EBM (tuned),36486.59858077451,0.1713965378309551,7650.518716752531,0.07548087023444203,0.8777961641049835,0.8195306909003875,0.1706589803939669,0.13332632757009968,21740.48293152139,2.6567815809050077,21.19736842105263,0.176925,2273.2198663949966,0.040004611015319824,1011.590888307647,0.024237136044788353,1.0,0.9915876241236676,0.08520888190451481,0.04977403056592767,14796.589958532411,1.267048812472469,22.0,0,1,1,36,1235.6,21.7,34.1,0.5409688995215312,0.07952519297189753 +FASTAI (tuned + ensemble),7277.4807652172285,19.036358243540715,1381.3381577435234,8.418367504420003,0.8036000089866165,0.8211863274453,0.17758551251392274,0.10916114346814769,18515.61407347416,513.5182779565009,22.855263157894736,0.17479,3045.3940094709396,12.611860513687134,622.7909689289006,5.036877045866174,1.0,1.0,0.10375041141751795,0.07514022518705465,12734.142743886789,430.3713585782513,24.0,0,0,0,38,1204.1,27.4,32.8,0.5032894736842105,0.07202894642560889 +EBM (default),91.60202875890229,0.20198602425424675,10.86984880564699,0.10636637956102298,0.8643875945345197,0.8262044713307948,0.18089017670927535,0.14126635043494726,91.71524858307365,4.255062874198898,23.039473684210527,0.17785499999999999,9.752015590667725,0.06734633445739746,5.438852341621796,0.06019285478263442,1.0,0.9961748430942867,0.10192934818149862,0.05017166802118532,55.554769096641394,3.385160223476314,23.25,1,0,0,37,1199.1,22.7,30.3,0.4991028708133971,0.08724123679200738 +XGB (default),13.091729760169983,0.5841734095623619,3.2452155420082907,0.31884078802225646,0.8951319461246384,0.8364633795855838,0.15856430986262096,0.14044471792922789,30.28280029264466,15.70638363121535,23.19736842105263,0.174975,6.213568687438965,0.346899151802063,1.8757318201110196,0.1209323363837647,1.0,1.0,0.10819500111945202,0.061412007010831625,23.70039322753218,9.284870866458713,23.5,1,0,0,37,1195.2,25.5,32.8,0.49551435406698563,0.08072840317264102 +XT (tuned + ensemble),1323.525801639808,3.044984867698268,475.9687294303007,1.5051105818150412,0.8963117961824938,0.8436648219553781,0.17616082985940248,0.1373394981787012,4483.031571431591,90.45918158334129,23.342105263157894,0.17722,771.266873717308,2.041845917701721,197.16355296740528,0.7207715417972143,1.0,1.0,0.11915769010932858,0.07354043401742626,2772.464639120071,76.33017880937523,26.25,0,1,0,37,1194.3,19.4,28.9,0.4922248803827751,0.07405538531908194 +NN_TORCH (tuned),24155.06064368549,0.7120582053535863,3018.869066392961,0.21113726214708323,0.903659699715494,0.8380014862892633,0.1642912879261434,0.11987126119881306,54165.934988934685,11.64610038179527,23.657894736842106,0.170275,9535.178918361664,0.22691142559051514,2480.3150361602966,0.11665950752975736,1.0,1.0,0.10704543711737907,0.06675514797021173,43976.90578112008,8.64392602078908,25.0,0,0,0,38,1187.9,23.6,26.1,0.48504784688995217,0.05845269413383195 +REALMLP (default),476.9483523494319,0.8381578796788266,36.30909370748047,0.4350989512151319,0.9101816949003365,0.8436348954009102,0.16572189164021864,0.11694440364221069,843.0795281741405,15.31190920332024,24.039473684210527,0.17567,137.81327652931213,0.256045937538147,34.76442448369694,0.19322861602793034,1.0,1.0,0.10725973083151863,0.07415873564674558,579.0193625765228,12.615190387657702,26.0,1,0,0,37,1180.2,25.1,22.1,0.4763755980861244,0.0767481650874326 +MNCA (default),295.14466257471787,10.148893080259624,18.71986495055019,1.275643167953242,0.8657835470756838,0.8640539680687833,0.1814538347442414,0.12057099426044675,239.59173183900742,74.09701288388523,24.210526315789473,0.190215,30.572218418121338,0.5992807149887085,13.346578494858026,0.35250805744744207,1.0,1.0,0.15473484110666214,0.0863058709177113,166.7604765821143,25.21201207586398,24.5,1,1,0,36,1175.3,26.8,28.4,0.47248803827751196,0.08215531376099204 +RF (tuned + ensemble),2283.87174061725,2.1008308561224687,539.2243606533888,1.2856745856516028,0.8931944011226663,0.8714068606623216,0.1816871032462093,0.15303114050221425,5202.357004362883,79.46832861690719,24.86842105263158,0.17431000000000002,864.6687717437744,1.8380836248397827,327.30532028870516,0.7083122934636021,1.0,1.0,0.1288372065082003,0.07467224490228473,4181.556008332933,65.49937121638476,27.0,0,2,0,36,1161.3,27.1,23.8,0.45753588516746413,0.07450591784011229 +XT (tuned),1323.525801639808,0.31659403600190816,475.9687294303007,0.1798597078129874,0.9205594165466939,0.8568702086248896,0.18796363064191732,0.14444173355267456,4483.031571431591,9.660217818491914,25.25,0.18019000000000002,771.266873717308,0.16697990894317627,197.16355296740528,0.08311920181240409,1.0,1.0,0.13578917697796827,0.08371989745070943,2772.464639120071,7.999561250297406,29.0,0,2,0,36,1156.7,28.5,31.3,0.44886363636363635,0.08018399939811821 +GBM (default),8.557482913920754,0.7352644769768966,2.999430927452211,0.212636163127602,0.923947709053949,0.8822127594650478,0.17417727593132074,0.146880576355245,33.42403528282587,16.15506317327496,25.855263157894736,0.17846,6.559425115585327,0.615138053894043,2.2516979473740024,0.15858503371213617,1.0,1.0,0.14236978569535247,0.06852491939171679,20.494723229907102,10.269446661494042,26.5,0,0,0,38,1145.4,25.4,29.9,0.43510765550239233,0.04701040168223337 +TABDPT (default),170.86559940639296,65.69096382668144,26.71478740353263,21.258106281632944,0.8235947911156236,0.8244708887822227,0.18653930504677302,0.13851779761834887,467.1692516605129,1539.0473534455273,26.144736842105264,0.191185,97.51883804798126,28.29759955406189,22.386440735738876,8.590693574547014,1.0,1.0,0.13114122736509654,0.07269651025672075,386.2317213587028,1143.1582415263433,30.5,1,3,0,34,1135.8,25.7,30.7,0.4285287081339713,0.1069267946292278 +FASTAI (tuned),7277.4807652172285,1.090680348245721,1381.3381577435234,0.6851415707626749,0.9108193738616642,0.8800228525603003,0.1979067944283506,0.14024783714427164,18515.61407347416,41.17026917318611,26.736842105263158,0.175545,3045.3940094709396,0.8673808574676514,622.7909689289006,0.34554260532273706,1.0,1.0,0.1660331096790038,0.0823205592657475,12734.142743886789,28.552621407710575,29.0,0,0,1,37,1125.6,26.9,24.2,0.4150717703349282,0.05822430872642212 +RF (tuned),2283.87174061725,0.22623545872537712,539.2243606533888,0.14783605660070437,0.9239206793690314,0.9258040590207852,0.19054836036376613,0.1642290415502217,5202.357004362883,7.803307308994161,27.092105263157894,0.17563499999999999,864.6687717437744,0.15117239952087402,327.30532028870516,0.08452519779888767,1.0,1.0,0.1250152487815977,0.09963978217702693,4181.556008332933,6.996946602769185,28.5,0,0,0,38,1118.5,25.5,30.5,0.4069976076555024,0.048784570129945914 +NN_TORCH (default),47.78759841542495,0.6881854220440513,11.127495716627488,0.2952049566930359,0.9794017223291384,0.9554831244445245,0.22349455186019088,0.17360188725262388,136.11141421046577,15.027014536020943,31.43421052631579,0.180125,24.229564905166626,0.31156718730926514,5.726755660627104,0.14843028917594223,1.0,1.0,0.1627665585076386,0.09575304821587584,131.897186986728,9.730207537012728,33.0,0,0,0,38,1022.8,26.0,25.7,0.3083133971291866,0.03655529558591806 +RF (default),4.080644030320017,0.13749531068299947,0.8126703096595244,0.06965281933938666,0.9894800211663467,0.9754696131854965,0.25712948756835613,0.2419361775158654,6.74726637377606,4.2459113559317,32.31578947368421,0.20493,1.552122712135315,0.09239459037780762,0.37380590277783665,0.037675282187559014,1.0,1.0,0.22217921675183,0.13407694191427183,5.06536582970867,3.354866759305418,35.25,0,0,0,38,1000.0,0.0,0.0,0.28827751196172247,0.03803581191225408 +FASTAI (default),30.567647507316188,1.0886577556007786,5.142705410747499,0.514750514881177,0.9739593024362323,0.9460250227474908,0.2387906491559458,0.18694462294220246,72.5941463445461,31.969331116236056,32.6578947368421,0.187165,14.332883596420288,0.6891474723815918,3.1247662358290373,0.33408521032303745,1.0,1.0,0.16594179717402546,0.12823511377411606,51.9702290983138,26.158132978083096,35.0,0,0,0,38,995.2,23.3,33.3,0.2805023923444976,0.034794347320152505 +LR (tuned + ensemble),310.8921214467601,1.6511000143854242,112.04387665947259,0.6061180299813982,0.95841567098672,0.9708437365552002,0.3197185107629301,0.2850714510079638,1051.5597600046694,27.942607605611126,35.026315789473685,0.20664,172.39098751544952,0.3049362897872925,51.96211560156496,0.1713220944305931,1.0,1.0,0.257712918071613,0.1682426614669617,652.0216614396725,8.851961834833686,38.5,1,0,0,37,929.4,24.0,32.9,0.22667464114832536,0.05459065334803458 +XT (default),2.8151443882992395,0.17746856965516744,0.7494419232366839,0.0764609601145254,0.9898367767783371,0.9811026074610522,0.2855577072213641,0.27528478156506314,4.797654893597667,4.664132317562142,35.26315789473684,0.20700000000000002,0.9193735122680664,0.08453762531280518,0.24344244584214264,0.037069730458521875,1.0,1.0,0.21784002712442435,0.1545266336174043,3.5050999047562326,4.1617612391934,38.0,0,0,0,38,922.1,28.5,25.6,0.22129186602870812,0.032099006640318124 +LR (tuned),310.8921214467601,0.5151690432899877,112.04387665947259,0.17414856714912683,0.9738366106666893,0.9798927122573593,0.32733777684539805,0.29873711892817967,1051.5597600046694,7.373474320542152,36.078947368421055,0.206755,172.39098751544952,0.14061975479125977,51.96211560156496,0.06357160024321462,1.0,1.0,0.2713987757406089,0.1825065086765385,652.0216614396725,5.062717123136234,39.5,0,0,1,37,899.6,23.4,32.1,0.2027511961722488,0.0359180947607498 +LR (default),7.7264530345013265,0.5259399602287694,2.6686237618793442,0.19925514469367503,0.9806670457343031,0.9871680738219274,0.33640822817706334,0.3255039345142868,27.432135439923183,8.872452564577243,36.89473684210526,0.21439,5.452793717384338,0.1415010690689087,1.6174976146916054,0.09387864407044556,1.0,1.0,0.27139753987310355,0.1831054339424733,15.972794808418932,5.719143257720329,40.0,0,0,0,38,871.8,27.9,29.6,0.18421052631578946,0.028931497254828412 +KNN (tuned + ensemble),212.22012870562705,8.11043459490726,28.850708739248347,0.6076026284446762,1.0,0.9874344918422748,0.48740395612015763,0.5934795732877101,79.58293822496161,51.24023018425664,40.19736842105263,0.320445,18.52320122718811,0.22250914573669434,3.5145795148857952,0.1617064055034519,1.0,1.0,0.4081435434925469,0.6881430164840905,56.97784214902664,12.39378247919743,43.0,0,0,0,38,734.8,47.2,34.7,0.10915071770334928,0.028601475026556994 +KNN (tuned),212.22012870562705,1.883336155038131,28.850708739248347,0.13054448296641816,1.0,0.9845999552108321,0.5083956060275397,0.643418972329782,79.58293822496161,11.122407039906847,41.36842105263158,0.32193499999999997,18.52320122718811,0.07584631443023682,3.5145795148857952,0.03818510847996826,1.0,1.0,0.48517378670484473,0.7165338507811103,56.97784214902664,2.1231636318644824,44.0,0,0,1,37,675.0,46.9,43.0,0.08253588516746412,0.031988194812603496 +KNN (default),1.74290458779586,0.21126952296809146,0.5001806322977429,0.036905984979792666,1.0,1.0,0.5920487199552541,0.941375697812162,1.0024208997554083,2.3365493404640363,43.71052631578947,0.37738,0.3047138452529907,0.03128814697265625,0.06855464474199147,0.019133871606833427,1.0,1.0,0.5282941392408126,1.0,1.0,1.2278781340930796,45.0,0,0,0,38,481.5,54.0,59.7,0.02930622009569378,0.023149934679664256 diff --git a/data/lite/full-imputed-cls/time_plot.pdf b/data/lite/full-imputed-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d77ef160dca7462a00c3d312bf4370123b96c0cd Binary files /dev/null and b/data/lite/full-imputed-cls/time_plot.pdf differ diff --git a/data/lite/full-imputed-cls/time_plot.png.zip b/data/lite/full-imputed-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4b09da65c6b40cebd3e140dfd590bcc56dda4ee1 --- /dev/null +++ b/data/lite/full-imputed-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:945c261e1f18fa414902ffb7bf3f3489b38f72b63d1aa368c5e21a4caebdd3f7 +size 338023 diff --git a/data/lite/full-imputed-cls/tuning-impact-elo-horizontal.pdf b/data/lite/full-imputed-cls/tuning-impact-elo-horizontal.pdf new file mode 100644 index 0000000000000000000000000000000000000000..11e7f30f9841240535c555d0c68e94549523ec62 Binary files /dev/null and b/data/lite/full-imputed-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/full-imputed-cls/tuning-impact-elo-horizontal.png.zip b/data/lite/full-imputed-cls/tuning-impact-elo-horizontal.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..94da4284c5fbc74bf40acb160762e5e2c61298c2 --- /dev/null +++ b/data/lite/full-imputed-cls/tuning-impact-elo-horizontal.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fda0306dee056e3a70377a09df4466cb9ed914464464776e2db0c3fffd17263 +size 156429 diff --git a/data/lite/full-imputed-cls/tuning-impact-elo.pdf b/data/lite/full-imputed-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6b29c4e20344f90d2dda95ef2be27db115cbbdd4 Binary files /dev/null and b/data/lite/full-imputed-cls/tuning-impact-elo.pdf differ diff --git a/data/lite/full-imputed-cls/tuning-impact-elo.png.zip b/data/lite/full-imputed-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..2b174d302d8a0f726611eda38b71d12ae577fd6a --- /dev/null +++ b/data/lite/full-imputed-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4caba2ad7696f84750a0608f643e9b0b789d9c6de30295b3c481e7166013f47e +size 168153 diff --git a/data/lite/full-imputed-reg/figures/critical-diagram.pdf b/data/lite/full-imputed-reg/figures/critical-diagram.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b2ae6edcb13ac44b7fdca1766bedbc05ee5b5ac3 Binary files /dev/null and b/data/lite/full-imputed-reg/figures/critical-diagram.pdf differ diff --git a/data/lite/full-imputed-reg/figures/critical-diagram.png.zip b/data/lite/full-imputed-reg/figures/critical-diagram.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..b544ef08c237f67c310b5b1db3f65969d7be06c4 --- /dev/null +++ b/data/lite/full-imputed-reg/figures/critical-diagram.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9cccf72f97aed21fb1dca813822d2c3d414fa542ba79bca722d35dae0bf5f38 +size 317563 diff --git a/data/lite/full-imputed-reg/leaderboard.tex b/data/lite/full-imputed-reg/leaderboard.tex new file mode 100644 index 0000000000000000000000000000000000000000..9a241175eb3a72eb2ba3bfb914f6d3f92994d14e --- /dev/null +++ b/data/lite/full-imputed-reg/leaderboard.tex @@ -0,0 +1,53 @@ +\begin{tabular}{llcccccrr} +\toprule +\textbf{Model} & \textbf{Elo ($\uparrow$)} & \textbf{Norm.} & \textbf{Avg.} & \textbf{Harm.} & \textbf{\#wins ($\uparrow$)} & \textbf{Improva-} & \textbf{Train time} & \textbf{Predict time} \\ + & & \textbf{score ($\uparrow$)} & \textbf{rank ($\downarrow$)} & \textbf{mean} & & \textbf{bility ($\downarrow$)} & \textbf{per 1K [s]} & \textbf{per 1K [s]} \\ + & & & & \textbf{rank ($\downarrow$)} & & & & \\ +\midrule +AutoGluon 1.3 (4h) & \textcolor{gold}{\textbf{1779${}_{-54,+67}$}} & \textcolor{gold}{\textbf{0.735}} & \textcolor{gold}{\textbf{4.7}} & \textcolor{bronze}{\textbf{3.0}} & 1 & \textcolor{silver}{\textbf{3.5\%}} & 1734.20 & 7.06 \\ +RealMLP (T+E) & \textcolor{silver}{\textbf{1684${}_{-52,+66}$}} & \textcolor{silver}{\textbf{0.734}} & \textcolor{silver}{\textbf{6.6}} & 5.0 & 0 & \textcolor{gold}{\textbf{3.3\%}} & 6534.34 & 2.30 \\ +ModernNCA (T+E) & \textcolor{bronze}{\textbf{1617${}_{-58,+66}$}} & 0.603 & \textcolor{bronze}{\textbf{8.4}} & \textcolor{silver}{\textbf{2.6}} & \textcolor{silver}{\textbf{3}} & 5.3\% & 3811.43 & 7.58 \\ +LightGBM (T+E) & 1608${}_{-66,+56}$ & 0.531 & 8.9 & 7.0 & 0 & 6.0\% & 686.46 & 5.48 \\ +TabDPT (D) & 1569${}_{-56,+66}$ & \textcolor{bronze}{\textbf{0.635}} & 10.0 & 3.2 & \textcolor{silver}{\textbf{3}} & \textcolor{bronze}{\textbf{3.9\%}} & 16.97 & 8.70 \\ +CatBoost (T+E) & 1561${}_{-55,+57}$ & 0.536 & 10.2 & 8.2 & 0 & 5.7\% & 2895.38 & 1.32 \\ +TabM (T+E) & 1540${}_{-48,+53}$ & 0.488 & 11.2 & 7.8 & 0 & 4.3\% & 4228.53 & 1.19 \\ +CatBoost (T) & 1524${}_{-58,+43}$ & 0.512 & 11.8 & 6.5 & 0 & 5.8\% & 2895.38 & 0.07 \\ +LightGBM (T) & 1504${}_{-60,+50}$ & 0.432 & 12.5 & 9.5 & 0 & 6.8\% & 686.46 & 0.74 \\ +XGBoost (T) & 1473${}_{-53,+39}$ & 0.398 & 13.7 & 12.4 & 0 & 6.7\% & 848.99 & 0.47 \\ +XGBoost (T+E) & 1464${}_{-42,+44}$ & 0.432 & 14.0 & 13.1 & 0 & 6.7\% & 848.99 & 2.38 \\ +ModernNCA (D) & 1464${}_{-56,+43}$ & 0.209 & 14.1 & 9.9 & 0 & 8.0\% & 16.07 & 0.29 \\ +TabM (T) & 1449${}_{-44,+56}$ & 0.380 & 14.5 & 11.7 & 0 & 5.1\% & 4228.53 & 0.13 \\ +RealMLP (T) & 1424${}_{-51,+54}$ & 0.344 & 15.5 & 12.2 & 0 & 6.1\% & 6534.34 & 0.07 \\ +CatBoost (D) & 1419${}_{-42,+46}$ & 0.389 & 15.8 & 11.6 & 0 & 7.9\% & 8.35 & 0.09 \\ +ModernNCA (T) & 1400${}_{-59,+46}$ & 0.351 & 16.5 & 6.5 & 0 & 7.9\% & 3811.43 & 0.45 \\ +TabPFNv2 (T+E) & 1366${}_{-50,+62}$ & 0.408 & 17.7 & \textcolor{gold}{\textbf{2.5}} & \textcolor{gold}{\textbf{5}} & 4.8\% & 3805.62 & 10.41 \\ +TabM (D) & 1354${}_{-48,+51}$ & 0.287 & 18.4 & 15.4 & 0 & 6.9\% & 13.90 & 0.12 \\ +TabPFNv2 (T) & 1309${}_{-43,+54}$ & 0.282 & 20.2 & 6.6 & 0 & 6.4\% & 3805.62 & 0.26 \\ +TorchMLP (T+E) & 1302${}_{-50,+48}$ & 0.171 & 20.5 & 17.2 & 0 & 8.1\% & 4452.11 & 0.85 \\ +RealMLP (D) & 1291${}_{-51,+55}$ & 0.137 & 20.9 & 15.4 & 0 & 8.2\% & 32.24 & 0.08 \\ +ExtraTrees (T+E) & 1289${}_{-49,+48}$ & 0.157 & 21.0 & 14.7 & 0 & 11.1\% & 161.73 & 0.78 \\ +ExtraTrees (T) & 1268${}_{-37,+48}$ & 0.129 & 21.8 & 14.2 & 0 & 11.4\% & 161.73 & 0.12 \\ +TabPFNv2 (D) & 1261${}_{-63,+41}$ & 0.228 & 22.4 & 6.6 & 0 & 7.5\% & 2.78 & 0.32 \\ +TorchMLP (T) & 1236${}_{-45,+42}$ & 0.123 & 23.2 & 19.0 & 0 & 8.7\% & 4452.11 & 0.09 \\ +LightGBM (D) & 1227${}_{-43,+51}$ & 0.070 & 23.5 & 22.3 & 0 & 9.6\% & 2.03 & 0.30 \\ +XGBoost (D) & 1194${}_{-51,+44}$ & 0.111 & 25.3 & 23.2 & 0 & 10.4\% & 2.15 & 0.18 \\ +RandomForest (T+E) & 1194${}_{-46,+43}$ & 0.066 & 25.1 & 22.8 & 0 & 12.1\% & 526.17 & 0.77 \\ +RandomForest (T) & 1131${}_{-36,+44}$ & 0.048 & 27.3 & 25.0 & 0 & 12.8\% & 526.17 & 0.12 \\ +EBM (T+E) & 1121${}_{-54,+55}$ & 0.160 & 27.9 & 16.3 & 0 & 14.7\% & 2124.78 & 0.12 \\ +ExtraTrees (D) & 1118${}_{-54,+56}$ & 0.061 & 28.1 & 25.1 & 0 & 13.0\% & 0.42 & 0.06 \\ +FastaiMLP (T+E) & 1086${}_{-56,+49}$ & 0.023 & 29.2 & 28.0 & 0 & 13.1\% & 527.21 & 2.83 \\ +EBM (T) & 1070${}_{-59,+43}$ & 0.137 & 29.9 & 9.3 & 1 & 15.3\% & 2124.78 & 0.01 \\ +TorchMLP (D) & 1069${}_{-50,+50}$ & 0.016 & 29.7 & 27.5 & 0 & 12.9\% & 20.50 & 0.08 \\ +FastaiMLP (T) & 1040${}_{-43,+36}$ & 0.013 & 30.7 & 29.7 & 0 & 13.6\% & 527.21 & 0.31 \\ +RandomForest (D) & 1000${}_{-0,+0}$ & 0.000 & 32.0 & 31.3 & 0 & 14.1\% & 0.63 & 0.06 \\ +TabICL (D) & 999${}_{-49,+47}$ & 0.000 & 32.0 & 31.3 & 0 & 14.1\% & 0.63 & 0.06 \\ +EBM (D) & 984${}_{-52,+60}$ & 0.093 & 32.5 & 29.4 & 0 & 16.0\% & 7.25 & 0.04 \\ +FastaiMLP (D) & 917${}_{-48,+43}$ & 0.000 & 34.4 & 33.7 & 0 & 17.8\% & 3.08 & 0.29 \\ +KNN (T+E) & 518${}_{-81,+70}$ & 0.000 & 40.8 & 40.6 & 0 & 37.3\% & 2.25 & 0.15 \\ +Linear (T+E) & 478${}_{-85,+45}$ & 0.000 & 41.3 & 41.2 & 0 & 35.4\% & 46.50 & 0.14 \\ +Linear (T) & 414${}_{-92,+84}$ & 0.000 & 41.9 & 41.8 & 0 & 35.6\% & 46.50 & 0.04 \\ +KNN (T) & 408${}_{-77,+86}$ & 0.000 & 42.0 & 41.8 & 0 & 37.9\% & 2.25 & 0.03 \\ +Linear (D) & 303${}_{-83,+79}$ & 0.000 & 43.0 & 43.0 & 0 & 38.1\% & 1.16 & 0.08 \\ +KNN (D) & 226${}_{-106,+113}$ & 0.000 & 43.5 & 43.3 & 0 & 41.6\% & 0.04 & 0.02 \\ +\bottomrule +\end{tabular} \ No newline at end of file diff --git a/data/lite/full-imputed-reg/tabarena_leaderboard.csv b/data/lite/full-imputed-reg/tabarena_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..a8ce60f0b2d2f954785506c4e5ff351af637c930 --- /dev/null +++ b/data/lite/full-imputed-reg/tabarena_leaderboard.csv @@ -0,0 +1,46 @@ +method,time_train_s,time_infer_s,time_train_s_per_1K,time_infer_s_per_1K,normalized-error,normalized-error-task,champ_delta,loss_rescaled,time_train_s_rescaled,time_infer_s_rescaled,rank,median_metric_error,median_time_train_s,median_time_infer_s,median_time_train_s_per_1K,median_time_infer_s_per_1K,median_normalized-error,median_normalized-error-task,median_champ_delta,median_loss_rescaled,median_time_train_s_rescaled,median_time_infer_s_rescaled,median_rank,rank=1_count,rank=2_count,rank=3_count,rank>3_count,elo,elo+,elo-,winrate,mrr +AutoGluon 1.3 (4h),10051.83785378016,70.68019696382376,3153.0175193360524,8.747021690702612,0.26461131829744206,0.29115842981137285,0.03549166493402033,0.03920014558976489,77389.85490429384,1642.993680810643,4.6923076923076925,4.10013,12458.540483474731,10.641454219818115,1734.201033855079,7.062674608341483,0.23133371369265832,0.27408623724891273,0.016430119497990225,0.02479264320162475,76123.2860686676,448.5777143640419,4.0,1,2,3,7,1778.9,66.7,53.2,0.916083916083916,0.3317182817182817 +REALMLP (tuned + ensemble),67720.63958248726,12.596411228179932,7060.621324893922,7.808561874535024,0.26596078198184286,0.34232721258440973,0.03285005050218543,0.04401282593134286,300508.62324682355,250.14082076713328,6.615384615384615,4.1572,28985.167714595795,4.047658205032349,6534.342759080719,2.2990216834440518,0.1939333280527935,0.324574783076754,0.02253995800399078,0.021718159121346815,215284.61589654352,243.1358386691401,6.0,0,1,2,10,1684.2,65.9,51.5,0.8723776223776224,0.20188785573400958 +MNCA (tuned + ensemble),32095.7095044026,50.336935831950264,6167.970103891367,17.67280087259302,0.39661026996847265,0.3581239674749241,0.05341582328399139,0.050042393373680265,183828.00911751532,735.9936636946994,8.384615384615385,5.17584,16410.6605322361,9.919520378112793,3811.431210311487,7.580550796782489,0.35096562881515253,0.15878663570211426,0.013723617424411305,0.01836048171796887,215831.1755427445,518.4548592505826,4.0,3,2,0,8,1617.3,66.0,57.9,0.8321678321678322,0.38649715095711557 +GBM (tuned + ensemble),3436.114175393031,58.23533050830547,795.9694443599602,8.418828059060083,0.4687885136784251,0.4763742989836305,0.059558887741216864,0.060464652873721096,19731.156544048128,1434.220040412032,8.923076923076923,4.23184,3069.166275501251,10.259183883666992,686.4649311851701,5.476648655048637,0.41525976638432377,0.4634748152061456,0.025882078477524217,0.050453337192894875,19944.502036932092,444.7715795428446,7.0,0,0,1,12,1607.9,55.5,65.5,0.8199300699300699,0.14327324722061563 +TABDPT (default),148.7485225384052,53.04140243163476,27.664670515034977,22.751733349883246,0.36524485788247774,0.4383354661860421,0.03895918068763415,0.054308826459819506,812.9269128730391,1906.3906090967916,10.0,4.24006,138.3429434299469,23.635958433151245,16.972548083255166,8.69507121915186,0.28825222902504594,0.47549658590127725,0.028230517433738833,0.03713676774968525,862.9680595038594,1818.2713070390416,8.0,3,0,0,10,1569.2,65.4,55.1,0.7954545454545454,0.3097215425150648 +CAT (tuned + ensemble),28869.75754873569,4.897776438639714,6008.917386152554,1.2106725402691727,0.46394270088709505,0.519590289227365,0.05714402452817158,0.06548691214488393,133179.3418491014,149.6309053684063,10.23076923076923,4.34931,19659.746299266815,2.423489809036255,2895.38206857098,1.3169555135467037,0.4150140849407447,0.5182244539994575,0.03827463657502361,0.05389421229497224,136008.98155550173,63.945909864284914,10.0,0,0,1,12,1561.4,56.9,54.1,0.7902097902097902,0.12143066672418898 +TABM (tuned + ensemble),43863.87365627289,7.196907648673425,7630.0436857309,2.7973145627637854,0.5116630985629478,0.5198043493598762,0.04290685479251263,0.07822064428199164,178405.92798162796,122.01127915897794,11.23076923076923,4.37433,20695.145789146423,2.30704665184021,4228.525932170142,1.1878318177487799,0.43090407292838107,0.5335750739105611,0.0396200504203853,0.06046439799308363,167683.34522914552,99.64653279572039,10.0,0,0,1,12,1540.1,52.2,47.4,0.7674825174825175,0.1289168239168239 +CAT (tuned),28869.75754873569,0.37941277944124663,6008.917386152554,0.13317195365011947,0.48768370669415817,0.5463479499812743,0.05839445479116519,0.07231684828907538,133179.3418491014,12.25994177013758,11.807692307692308,4.33359,19659.746299266815,0.3871791362762451,2895.38206857098,0.07259850925684806,0.5120020111687084,0.5133061965308594,0.03706248382495947,0.053204471612172954,136008.98155550173,9.36829586948511,12.0,0,2,0,11,1523.9,42.1,57.7,0.7543706293706294,0.1537026326500011 +GBM (tuned),3436.114175393031,7.708258518805871,795.9694443599602,1.1322853845142717,0.5677497712335507,0.6014392970416428,0.06776685133908343,0.07552236395185406,19731.156544048128,203.98211974063437,12.538461538461538,4.36733,3069.166275501251,2.4075584411621094,686.4649311851701,0.7428209858877407,0.5552965190223861,0.5756896517205545,0.03596218031983012,0.05207436999842263,19944.502036932092,125.83374247654176,12.0,0,0,0,13,1504.0,49.2,59.2,0.7377622377622378,0.10512436281667051 +XGB (tuned),6347.206248815243,2.662817588219276,1403.828603839992,0.830905023108991,0.6023900941776976,0.6250779743377591,0.0667918478116874,0.07170009024260994,25057.523759861193,88.40573298970912,13.73076923076923,4.27126,2595.2154359817505,1.905684232711792,848.994191986563,0.4748247737108275,0.6620816436761664,0.6582919744938843,0.03917366894901786,0.060160294207553616,28391.44761906215,33.71919899313387,14.0,0,0,0,13,1473.3,38.4,52.3,0.7106643356643356,0.08067601202216587 +XGB (tuned + ensemble),6347.206248815243,13.31823913867657,1403.828603839992,4.96016839745584,0.567688570586379,0.6227280227839246,0.06698153788595129,0.0725859063185028,25057.523759861193,462.0201618770655,14.0,4.28931,2595.2154359817505,11.448588609695435,848.994191986563,2.377886790782213,0.6256631137377601,0.6107024069682365,0.04155329070129643,0.06320753111560178,28391.44761906215,165.90428541165622,15.0,0,0,0,13,1463.6,43.3,41.1,0.7045454545454546,0.07613468733378236 +MNCA (default),99.03008703085092,2.04661226272583,17.95183203493229,0.696529515302073,0.7905734390801098,0.6498732173588466,0.08041637581694962,0.08390389437461304,536.9851811460794,28.619994682567945,14.076923076923077,5.11399,53.54817509651184,0.404191255569458,16.074996445495383,0.28976407064988385,0.9481480616823943,0.5396637738217239,0.03164135806644719,0.09743889944848891,512.2293777300066,19.36698671755419,11.0,0,0,0,13,1463.6,43.0,55.8,0.7027972027972028,0.10143543204924278 +TABM (tuned),43863.87365627289,0.6320198682638315,7630.0436857309,0.26286272507788355,0.6195303355392976,0.6496850655443615,0.051372265876231,0.09148999172880691,178405.92798162796,12.46052452929135,14.461538461538462,4.38922,20695.145789146423,0.3208010196685791,4228.525932170142,0.1251857412799802,0.5658117450288995,0.6483290168515105,0.03964170904471853,0.08006597714469382,167683.34522914552,9.015021272319185,14.0,0,0,0,13,1449.4,55.3,43.5,0.6940559440559441,0.08548065182680567 +REALMLP (tuned),67720.63958248726,0.544457747386052,7060.621324893922,0.3613057143261696,0.655870427194709,0.6797373834227193,0.060505986157313056,0.08316728065693908,300508.62324682355,13.279279128405507,15.538461538461538,4.49659,28985.167714595795,0.20874404907226562,6534.342759080719,0.07393883227200762,0.67949922992921,0.7357067842152395,0.04525760113385646,0.06129841644554749,215284.61589654352,8.366633852135228,17.0,0,0,0,13,1424.2,53.8,50.1,0.6695804195804196,0.08208437073762151 +CAT (default),91.84739507161655,0.4293701098515437,19.94101966337271,0.1772958974443293,0.6107641551712003,0.6846055390379693,0.07881653818408804,0.09152830573044982,430.2934190313639,15.190031737648651,15.807692307692308,4.47999,56.422000885009766,0.48146557807922363,8.349037958570749,0.08723198972790447,0.5847419575275037,0.7875385337046236,0.045915518799006794,0.09538359624838474,318.4902885443583,10.727539533685711,17.0,0,0,0,13,1418.8,46.0,41.1,0.6634615384615384,0.08639569595687512 +MNCA (tuned),32095.7095044026,2.2542913326850305,6167.970103891367,0.8125346038667304,0.6494157477338965,0.6307262787897645,0.07928979753685937,0.09781240992471517,183828.00911751532,39.103427878901556,16.53846153846154,5.75005,16410.6605322361,0.6251811981201172,3811.431210311487,0.45193767318844,0.686862000079896,0.6969334761199919,0.03527696549346959,0.05891870917494727,215831.1755427445,27.98327705018131,16.0,0,2,1,10,1399.7,45.2,58.2,0.6468531468531469,0.15343961252032015 +TABPFNV2 (tuned + ensemble),8899.38509618319,62.91845811330355,3633.4558137211893,46.64108205495555,0.5916274908500255,0.5849838864936402,0.04839446613326732,0.08944393967137762,99443.62878221263,3679.6982603537876,17.653846153846153,3.91474,5595.9298593997955,5.2166831493377686,3805.619968588136,10.412541216243051,0.7821381201281035,1.0,0.025874273100668255,0.03630310268727526,98726.78828565427,873.5021358138049,25.0,5,0,0,8,1366.5,61.4,49.5,0.6215034965034965,0.4077406869543342 +TABM (default),115.03358120184679,0.551762947669396,30.805250424733348,0.23944113770930792,0.7132806572850755,0.7340372112580631,0.06854579934686048,0.12088875293111297,591.7176035490969,9.427562415305996,18.384615384615383,4.49975,77.85998892784119,0.17319130897521973,13.902122931630087,0.12286842671210456,0.6984407490367052,0.7811252019030427,0.05381146736356224,0.09871836929846278,376.39915054227686,7.849330306811178,18.0,0,0,0,13,1353.6,50.6,47.6,0.6048951048951049,0.06505300599593403 +TABPFNV2 (tuned),8899.38509618319,3.9829243880051832,3633.4558137211893,1.951163103246529,0.7178550513336984,0.7120821834319193,0.06387860143600042,0.11435494833471192,99443.62878221263,205.40878557856996,20.192307692307693,4.06636,5595.9298593997955,0.3870081901550293,3805.619968588136,0.2619314172957417,1.0,1.0,0.05440739873667044,0.06240298084549091,98726.78828565427,24.143642847427156,28.5,0,2,1,10,1309.2,54.0,43.0,0.5638111888111889,0.1517848528366536 +NN_TORCH (tuned + ensemble),44566.48122985546,3.9406251907348633,5669.080821343483,1.3428149317173896,0.8294336223884518,0.8237288714596172,0.08088108079156034,0.11213994758491941,212623.11870515408,100.05462140638757,20.53846153846154,4.45415,15782.205469369888,3.293855905532837,4452.108570637395,0.8515826242412635,0.9419042422995464,0.7884085019971289,0.08148151246451829,0.09102294293315928,180041.55346239166,102.16386050718569,21.0,0,0,0,13,1301.5,47.7,49.2,0.5559440559440559,0.05801588862969937 +REALMLP (default),351.8482951934521,0.5515201458564172,35.503559222074486,0.3453931510429455,0.8625329178435776,0.83683724224921,0.08164018029407737,0.12907740722461616,1536.8505495640943,11.94254014053474,20.923076923076923,4.72117,142.23176908493042,0.1978776454925537,32.236338542328866,0.07855930724815739,0.9645369038929025,0.9243572012327576,0.06218045116441895,0.09045562976994424,969.8530395782191,8.889469027651435,22.0,0,0,0,13,1290.6,54.5,50.3,0.5472027972027972,0.06491914962738762 +XT (tuned + ensemble),1203.8377161942994,3.407132093723004,436.75693766012125,1.1793024261894192,0.8432098887288747,0.8049811407971149,0.11060763264847792,0.12718490393522525,7927.278685066689,97.16356713108524,21.0,5.19051,775.9451651573181,4.3725128173828125,161.7254597218303,0.7839402487111646,1.0,1.0,0.05459719178375644,0.12151685876412094,4965.162637285519,114.56582435147743,24.0,0,0,0,13,1289.0,47.5,48.2,0.5454545454545454,0.06781235624248329 +XT (tuned),1203.8377161942994,0.42836998059199405,436.75693766012125,0.15974479700708394,0.8708387105454042,0.800557685904906,0.11386210725946759,0.13000093959060843,7927.278685066689,12.996585928646535,21.846153846153847,5.21222,775.9451651573181,0.3841361999511719,161.7254597218303,0.1196965961875078,1.0,1.0,0.059483366316838704,0.12062350958135247,4965.162637285519,11.747884708843724,23.0,0,0,0,13,1268.3,47.2,36.8,0.5262237762237763,0.07032908719674844 +TABPFNV2 (default),14.410667272714468,0.6207188092745267,5.29412356125824,0.43658793734220885,0.7716885347996363,0.7324942589285312,0.07542019349562212,0.13507197823314007,114.90906903053514,31.157310825104744,22.423076923076923,3.96968,10.626041173934937,0.44887351989746094,2.7779123281248537,0.3151157987035174,1.0,1.0,0.06293455289431216,0.06240298084549091,106.36256863695091,24.143642847427156,28.5,0,2,2,9,1260.8,40.4,62.4,0.5131118881118881,0.15112174356709815 +NN_TORCH (tuned),44566.48122985546,0.300307035446167,5669.080821343483,0.10884087554478102,0.8767833734464541,0.8424822836677806,0.08707948843325523,0.13260712389377693,212623.11870515408,7.435699914505718,23.23076923076923,4.35043,15782.205469369888,0.15927791595458984,4452.108570637395,0.08851260312676379,1.0,0.8224160256691871,0.09275938505548831,0.07439215953480095,180041.55346239166,7.021581615200423,20.0,0,0,0,13,1235.6,41.1,44.8,0.49475524475524474,0.05267127440710437 +GBM (default),11.557473989633413,2.3389387864332933,3.7830693017108263,0.6131180352644026,0.9303753212721771,0.8977865383859813,0.09641293232462503,0.12631547899331944,86.42318767453573,78.28800186182016,23.53846153846154,4.60997,7.728894472122192,1.0601136684417725,2.0295221938774763,0.2962542098282052,0.9919628610131755,1.0,0.056468228310221136,0.12652549891600978,79.9077479005168,21.84399881745161,24.0,0,0,0,13,1227.4,50.6,42.2,0.48776223776223776,0.044802849386652986 +RF (tuned + ensemble),2061.32346032216,3.516887463056124,498.80047788723755,1.0331010904881306,0.9341058521704168,0.9236047758809486,0.12061494844984984,0.13420467042903117,10568.524251535091,92.6089977104098,25.076923076923077,5.35575,1100.1833860874176,1.5810585021972656,526.1674899414485,0.7701151896148807,1.0,1.0,0.07253345631610286,0.16236919097501226,11284.229205513133,73.76895458081503,27.0,0,0,0,13,1193.5,43.0,45.9,0.4527972027972028,0.04380255277296115 +XGB (default),12.744810801285963,0.8711951879354624,3.3672477352464436,0.269627728742662,0.8890754852284094,0.864950613607708,0.10371851834711493,0.13896612670151764,67.4237557433486,29.92998850138266,25.26923076923077,5.03782,8.503239631652832,0.5015158653259277,2.1458947295259945,0.17973603002282956,1.0,1.0,0.06048068144140273,0.11472972098019232,79.4223078814837,14.470859449961994,26.0,0,0,0,13,1193.7,44.0,50.2,0.44842657342657344,0.04315970587940105 +RF (tuned),2061.32346032216,0.35773491859436035,498.80047788723755,0.13352674389299507,0.9521440935358421,0.9329830611610441,0.12823071216367044,0.1450753182102409,10568.524251535091,11.403218373596392,27.346153846153847,5.50373,1100.1833860874176,0.2583794593811035,526.1674899414485,0.11934386114600624,1.0,1.0,0.07200067617043104,0.17146504159205414,11284.229205513133,12.018495346547116,28.5,0,0,0,13,1130.7,43.5,35.9,0.4012237762237762,0.04002937207271572 +EBM (tuned + ensemble),26006.099437456865,0.9856858620276818,3356.515573705329,0.34153237266067143,0.8401103872067855,0.8714235609142195,0.14667419727189587,0.19519827065979006,98028.229877245,12.716029695527471,27.923076923076923,4.4992,6071.96945476532,0.23789739608764648,2124.784934448033,0.12211837692413026,1.0,1.0,0.12651110207479377,0.12909316505911625,95277.20874636172,10.244440895844145,34.0,0,0,1,12,1120.6,54.7,53.6,0.3881118881118881,0.061303030752746124 +XT (default),4.47036992586576,0.27958295895503116,0.8320963666478268,0.08259457701463509,0.9388750296989964,0.9200841340233729,0.12995089831738704,0.18146467541992853,12.3176265493243,7.870904574627616,28.076923076923077,5.21222,1.8384346961975098,0.15966486930847168,0.424311872114215,0.060998774854278015,1.0,1.0,0.08211867366377534,0.16206852516277276,10.170662473104239,6.482341208432202,30.0,0,0,0,13,1118.2,55.1,53.1,0.38461538461538464,0.039842149657870064 +FASTAI (tuned + ensemble),4587.4864926521595,8.088074555763832,1228.3496620568533,4.774504314817022,0.976857214707512,0.9882135422000573,0.13089947326898535,0.19661133685827967,31416.410399557783,300.5500778279919,29.23076923076923,5.22937,3495.172525405884,6.275083303451538,527.2149841541068,2.8305834490487536,1.0,1.0,0.09544576825940065,0.17798536123127906,32094.70665313662,284.14659267835043,28.0,0,0,0,13,1085.7,48.4,55.3,0.3583916083916084,0.03572335866660936 +NN_TORCH (default),149.93631285887497,0.22603647525493914,24.53274626856709,0.08820389410472865,0.9842745385355711,0.977905433234048,0.1288760723805274,0.20021556593244114,789.2682924687753,6.761490179412745,29.692307692307693,4.48918,79.02296853065491,0.1783456802368164,20.501721600246,0.08139732073405338,1.0,1.0,0.12094953272557762,0.12823638255534808,464.3796175172265,6.690027290059689,29.0,0,0,0,13,1069.1,49.9,49.7,0.3479020979020979,0.036411074251365284 +EBM (tuned),26006.099437456865,0.08786681982187125,3356.515573705329,0.03542955498326605,0.8629348413551933,0.8593961706935105,0.1533023278019875,0.22324193551187432,98028.229877245,1.247331364408003,29.923076923076923,4.53788,6071.96945476532,0.03220105171203613,2124.784934448033,0.011920453069690697,1.0,1.0,0.12684760095556957,0.1518090762784355,95277.20874636172,1.0,36.0,1,0,0,12,1070.2,43.0,58.5,0.34265734265734266,0.10782856962302997 +FASTAI (tuned),4587.4864926521595,0.8590086423433744,1228.3496620568533,0.4517426116430779,0.9865659937937807,0.9955510902736445,0.13574860944781228,0.21723167691030648,31416.410399557783,33.24632209827138,30.692307692307693,5.20475,3495.172525405884,0.7054669857025146,527.2149841541068,0.31148362904787064,1.0,1.0,0.11250145730924221,0.19828317849031726,32094.70665313662,33.069549879991385,30.0,0,0,0,13,1040.1,35.7,42.3,0.32517482517482516,0.033628393696266995 +RF (default),9.625078421372633,0.3119058792407696,1.3105120203011107,0.08833919573142503,1.0,1.0,0.1414698659996216,0.19127372198208098,28.977692368539365,8.6618037567804,31.96153846153846,5.50373,5.83245849609375,0.15849757194519043,0.6330948821924387,0.06385546519532179,1.0,1.0,0.08318481827952429,0.20419806328614365,24.89266133507776,6.5528732074693075,32.0,0,0,0,13,1000.0,0.0,0.0,0.29632867132867136,0.03196784925042353 +TABICL (default),9.625078421372633,0.3119058792407696,1.3105120203011107,0.08833919573142503,1.0,1.0,0.1414698659996216,0.19127372198208098,28.977692368539365,8.6618037567804,31.96153846153846,5.50373,5.83245849609375,0.15849757194519043,0.6330948821924387,0.06385546519532179,1.0,1.0,0.08318481827952429,0.20419806328614365,24.89266133507776,6.5528732074693075,32.0,0,0,0,13,999.2,46.1,48.8,0.29632867132867136,0.03196784925042353 +EBM (default),59.33740012462322,0.1286466121673584,9.101757338940997,0.07531792942152694,0.9074728958123806,0.929284582659214,0.16014805544094196,0.22817038573542853,235.3944936698285,3.0829910156446747,32.53846153846154,4.59891,16.830075979232788,0.058573246002197266,7.250631064699407,0.04116066154478904,1.0,1.0,0.13870824947029292,0.14578267826230468,199.27951947649242,2.3994193074057,37.0,0,0,0,13,984.1,59.9,51.1,0.28321678321678323,0.0339891038491637 +FASTAI (default),19.02570968407851,0.9372756664569561,4.5150492334878205,0.3838557095432917,1.0,1.0,0.17777881555460698,0.2710114034885087,122.59124416864653,31.37127570550211,34.38461538461539,6.27121,14.524733304977417,1.090883731842041,3.0836050750366097,0.29293149190497314,1.0,1.0,0.11103713173333196,0.29927743293530584,144.4018162786092,35.82998039822552,35.0,0,0,0,13,917.4,42.4,47.6,0.24125874125874125,0.029694724737649423 +KNN (tuned + ensemble),130.0447974755214,2.3221586850973277,31.413578991094717,0.3839981934688547,1.0,1.0,0.37277765130751944,0.6767426072657229,86.62047407939465,27.45502773232453,40.84615384615385,8.16815,12.739671230316162,0.2766997814178467,2.253836358499806,0.14740763433303566,1.0,1.0,0.42992491292377444,0.797964481423868,68.17976708247701,13.891874242000187,42.0,0,0,0,13,518.1,69.6,80.3,0.0944055944055944,0.024604263435352786 +LR (tuned + ensemble),259.4397970529703,0.5564901095170242,90.58997126979446,0.25975101191673244,1.0,1.0,0.35449036529297795,0.6326802829127195,1967.1950270790398,11.602054705435593,41.26923076923077,7.90301,156.73520398139954,0.17154169082641602,46.49516183285896,0.14113267629856877,1.0,1.0,0.3712901272900253,0.6730086523874259,1676.6799766486058,9.252055102805237,40.0,0,0,0,13,478.0,44.4,84.2,0.08479020979020979,0.024263628776410674 +LR (tuned),259.4397970529703,0.12883753042954665,90.58997126979446,0.058721919597199,1.0,1.0,0.3563435955227629,0.6421042284092535,1967.1950270790398,2.9663690915463836,41.92307692307692,7.95008,156.73520398139954,0.04031491279602051,46.49516183285896,0.04248926416039467,1.0,1.0,0.3720471503072048,0.6809514133120513,1676.6799766486058,3.0754592783196695,42.0,0,0,0,13,413.9,83.1,91.5,0.06993006993006994,0.023904081877045964 +KNN (tuned),130.0447974755214,0.2600700121659499,31.413578991094717,0.054814694748458494,1.0,1.0,0.3793529569651058,0.7088035547752614,86.62047407939465,3.596722365416154,42.0,8.28075,12.739671230316162,0.11144423484802246,2.253836358499806,0.029449351131916046,1.0,1.0,0.4343387863461754,0.8372013840030864,68.17976708247701,2.325030564010288,43.0,0,0,0,13,407.7,85.6,76.1,0.06818181818181818,0.02390428629793349 +LR (default),4.9569381567148065,0.18706706854013297,2.04240300519074,0.0952681059723661,1.0,1.0,0.3811678789837085,0.7484824808959079,46.37076890234856,6.575927550408862,43.03846153846154,8.07907,5.157731294631958,0.10304999351501465,1.1643471259929907,0.0800294014177239,1.0,1.0,0.3720269301046141,0.7403523158287414,40.21502356299305,3.589990624937666,42.0,0,0,0,13,302.7,78.1,82.3,0.044580419580419584,0.023274085270852826 +KNN (default),2.3537445801955004,0.07271528244018555,0.6011442901693675,0.029873606599490884,1.0,1.0,0.4161497596650681,0.8500786493003942,1.1881523781374654,1.663589934493969,43.53846153846154,8.5423,0.15627789497375488,0.04499936103820801,0.03787513578449361,0.02182507289554364,1.0,1.0,0.4698333732046267,1.0,1.0,1.6403093647746019,45.0,0,0,0,13,225.7,112.5,105.6,0.033216783216783216,0.023071373071373073 diff --git a/data/lite/full-imputed-reg/time_plot.pdf b/data/lite/full-imputed-reg/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2f1d5a87e897824a64b9caf20cfac7c9df50b407 Binary files /dev/null and b/data/lite/full-imputed-reg/time_plot.pdf differ diff --git a/data/lite/full-imputed-reg/time_plot.png.zip b/data/lite/full-imputed-reg/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..41365658dcbe1039b6439999c1bb7cf1f8653719 --- /dev/null +++ b/data/lite/full-imputed-reg/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fccff3eca1df1c8b03a23190e0e55e0f0216d7b93fbd63db371d9d3919c226e +size 80035 diff --git a/data/lite/full-imputed-reg/tuning-impact-elo-horizontal.pdf b/data/lite/full-imputed-reg/tuning-impact-elo-horizontal.pdf new file mode 100644 index 0000000000000000000000000000000000000000..16ea002430e4519b72149df7370741c2b2c43bb3 Binary files /dev/null and b/data/lite/full-imputed-reg/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/full-imputed-reg/tuning-impact-elo-horizontal.png.zip b/data/lite/full-imputed-reg/tuning-impact-elo-horizontal.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1a70229f7784114f125b2b0df487522dfa714102 --- /dev/null +++ b/data/lite/full-imputed-reg/tuning-impact-elo-horizontal.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9af67055d12b01ae9ba7d4a82de311d0b3b106ce061cb9388b76a027951673e +size 154956 diff --git a/data/lite/full-imputed-reg/tuning-impact-elo.pdf b/data/lite/full-imputed-reg/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d612c07b6c6a0a9e79b20398dff04ab0d45f0619 Binary files /dev/null and b/data/lite/full-imputed-reg/tuning-impact-elo.pdf differ diff --git a/data/lite/full-imputed-reg/tuning-impact-elo.png.zip b/data/lite/full-imputed-reg/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..760f2e8e397f183813027a9a82f6bb363520253a --- /dev/null +++ b/data/lite/full-imputed-reg/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:931bea198ddd5a69708097df0b9588e0032178ce6e2aa72b10a7ac0cc8030945 +size 158236 diff --git a/data/lite/full-imputed/figures/critical-diagram.pdf b/data/lite/full-imputed/figures/critical-diagram.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7ffff52a017c1d16256e691909dd47ac7859f82d Binary files /dev/null and b/data/lite/full-imputed/figures/critical-diagram.pdf differ diff --git a/data/lite/full-imputed/figures/critical-diagram.png.zip b/data/lite/full-imputed/figures/critical-diagram.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..53f1e1913708c58e423666b9e7987a32b602b48c --- /dev/null +++ b/data/lite/full-imputed/figures/critical-diagram.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e47c65c429b468f6cb27770d2b1e48acd17a2fb6a3fbff0c529a5072327d159e +size 324111 diff --git a/data/lite/full-imputed/leaderboard.tex b/data/lite/full-imputed/leaderboard.tex new file mode 100644 index 0000000000000000000000000000000000000000..787142558770a15cc824cc037cd91db6e9e533f3 --- /dev/null +++ b/data/lite/full-imputed/leaderboard.tex @@ -0,0 +1,53 @@ +\begin{tabular}{llcccccrr} +\toprule +\textbf{Model} & \textbf{Elo ($\uparrow$)} & \textbf{Norm.} & \textbf{Avg.} & \textbf{Harm.} & \textbf{\#wins ($\uparrow$)} & \textbf{Improva-} & \textbf{Train time} & \textbf{Predict time} \\ + & & \textbf{score ($\uparrow$)} & \textbf{rank ($\downarrow$)} & \textbf{mean} & & \textbf{bility ($\downarrow$)} & \textbf{per 1K [s]} & \textbf{per 1K [s]} \\ + & & & & \textbf{rank ($\downarrow$)} & & & & \\ +\midrule +AutoGluon 1.3 (4h) & \textcolor{gold}{\textbf{1544${}_{-33,+30}$}} & \textcolor{gold}{\textbf{0.608}} & \textcolor{gold}{\textbf{7.6}} & \textcolor{gold}{\textbf{3.3}} & \textcolor{silver}{\textbf{7}} & \textcolor{gold}{\textbf{6.6\%}} & 1453.27 & 3.15 \\ +RealMLP (T+E) & \textcolor{silver}{\textbf{1447${}_{-25,+21}$}} & \textcolor{silver}{\textbf{0.499}} & \textcolor{silver}{\textbf{10.9}} & 6.8 & 0 & \textcolor{bronze}{\textbf{9.2\%}} & 6026.47 & 3.13 \\ +LightGBM (T+E) & \textcolor{bronze}{\textbf{1428${}_{-22,+30}$}} & 0.437 & \textcolor{bronze}{\textbf{11.8}} & 8.2 & 0 & 10.4\% & 416.56 & 2.24 \\ +TabM (T+E) & 1421${}_{-27,+31}$ & \textcolor{bronze}{\textbf{0.491}} & 12.1 & 6.1 & 2 & \textcolor{silver}{\textbf{9.1\%}} & 3133.91 & 1.27 \\ +CatBoost (T+E) & 1400${}_{-25,+27}$ & 0.419 & 13.0 & 8.1 & 0 & 9.7\% & 1665.53 & 0.56 \\ +ModernNCA (T+E) & 1382${}_{-23,+24}$ & 0.432 & 13.7 & \textcolor{bronze}{\textbf{4.5}} & \textcolor{bronze}{\textbf{5}} & 10.0\% & 4618.50 & 7.74 \\ +CatBoost (T) & 1377${}_{-26,+19}$ & 0.403 & 14.1 & 8.6 & 0 & 10.2\% & 1665.53 & 0.07 \\ +XGBoost (T+E) & 1376${}_{-23,+22}$ & 0.349 & 14.1 & 8.0 & 1 & 11.0\% & 700.96 & 1.44 \\ +LightGBM (T) & 1358${}_{-23,+25}$ & 0.347 & 14.9 & 10.3 & 0 & 11.3\% & 416.56 & 0.38 \\ +TabM (T) & 1356${}_{-24,+27}$ & 0.400 & 15.0 & 8.1 & 0 & 10.0\% & 3133.91 & 0.13 \\ +XGBoost (T) & 1353${}_{-24,+23}$ & 0.299 & 15.2 & 8.8 & 1 & 11.2\% & 700.96 & 0.21 \\ +CatBoost (D) & 1353${}_{-24,+21}$ & 0.356 & 15.3 & 8.6 & 2 & 11.0\% & 6.70 & 0.09 \\ +ModernNCA (T) & 1344${}_{-22,+25}$ & 0.297 & 15.6 & 6.9 & 2 & 10.7\% & 4618.50 & 0.47 \\ +TabPFNv2 (T+E) & 1324${}_{-25,+25}$ & 0.466 & 16.6 & \textcolor{silver}{\textbf{3.4}} & \textcolor{gold}{\textbf{11}} & 10.6\% & 2942.08 & 17.37 \\ +RealMLP (T) & 1287${}_{-20,+25}$ & 0.238 & 18.4 & 12.8 & 0 & 12.6\% & 6026.47 & 0.17 \\ +TabM (D) & 1286${}_{-18,+23}$ & 0.275 & 18.5 & 10.4 & 1 & 12.9\% & 11.56 & 0.13 \\ +TabICL (D) & 1280${}_{-23,+25}$ & 0.373 & 18.8 & 6.2 & 4 & 11.7\% & 6.86 & 1.52 \\ +TabPFNv2 (T) & 1271${}_{-18,+27}$ & 0.354 & 19.2 & 6.3 & 1 & 12.3\% & 2942.08 & 0.26 \\ +TorchMLP (T+E) & 1261${}_{-25,+23}$ & 0.209 & 19.8 & 14.1 & 0 & 12.6\% & 2832.80 & 1.80 \\ +TabPFNv2 (D) & 1233${}_{-23,+22}$ & 0.318 & 21.3 & 6.2 & 3 & 13.3\% & 3.27 & 0.32 \\ +EBM (T+E) & 1230${}_{-24,+21}$ & 0.175 & 21.5 & 11.0 & 0 & 16.0\% & 1323.39 & 0.18 \\ +ModernNCA (D) & 1227${}_{-24,+25}$ & 0.153 & 21.6 & 11.5 & 1 & 15.6\% & 13.74 & 0.32 \\ +TabDPT (D) & 1217${}_{-27,+24}$ & 0.293 & 22.0 & 6.3 & 4 & 14.9\% & 20.56 & 8.62 \\ +ExtraTrees (T+E) & 1205${}_{-24,+22}$ & 0.117 & 22.7 & 13.8 & 0 & 15.9\% & 191.44 & 0.76 \\ +RealMLP (D) & 1193${}_{-21,+24}$ & 0.102 & 23.2 & 13.6 & 1 & 14.4\% & 34.70 & 0.18 \\ +EBM (T) & 1190${}_{-24,+20}$ & 0.126 & 23.4 & 11.5 & 1 & 16.6\% & 1323.39 & 0.02 \\ +TorchMLP (T) & 1189${}_{-21,+25}$ & 0.103 & 23.5 & 17.6 & 0 & 14.5\% & 2832.80 & 0.11 \\ +XGBoost (D) & 1186${}_{-25,+20}$ & 0.106 & 23.7 & 14.1 & 1 & 14.5\% & 2.06 & 0.12 \\ +ExtraTrees (T) & 1171${}_{-25,+25}$ & 0.092 & 24.4 & 12.9 & 0 & 16.9\% & 191.44 & 0.10 \\ +FastaiMLP (T+E) & 1168${}_{-23,+26}$ & 0.152 & 24.5 & 15.9 & 0 & 16.6\% & 594.95 & 4.65 \\ +RandomForest (T+E) & 1160${}_{-19,+22}$ & 0.096 & 24.9 & 15.0 & 0 & 16.6\% & 377.08 & 0.75 \\ +LightGBM (D) & 1151${}_{-21,+26}$ & 0.074 & 25.3 & 21.5 & 0 & 15.4\% & 2.20 & 0.17 \\ +EBM (D) & 1150${}_{-26,+26}$ & 0.125 & 25.5 & 13.6 & 1 & 17.6\% & 5.48 & 0.06 \\ +RandomForest (T) & 1115${}_{-17,+22}$ & 0.069 & 27.2 & 21.5 & 0 & 17.5\% & 377.08 & 0.09 \\ +FastaiMLP (T) & 1102${}_{-20,+22}$ & 0.070 & 27.7 & 19.2 & 0 & 18.2\% & 594.95 & 0.34 \\ +TorchMLP (D) & 1029${}_{-32,+21}$ & 0.019 & 31.0 & 27.4 & 0 & 19.9\% & 8.96 & 0.13 \\ +RandomForest (D) & 1000${}_{-0,+0}$ & 0.008 & 32.2 & 27.4 & 0 & 22.8\% & 0.43 & 0.05 \\ +FastaiMLP (D) & 977${}_{-24,+22}$ & 0.019 & 33.1 & 29.9 & 0 & 22.3\% & 3.12 & 0.31 \\ +ExtraTrees (D) & 969${}_{-25,+25}$ & 0.023 & 33.4 & 29.3 & 0 & 24.6\% & 0.26 & 0.05 \\ +Linear (T+E) & 874${}_{-29,+21}$ & 0.031 & 36.6 & 21.3 & 1 & 32.9\% & 47.11 & 0.16 \\ +Linear (T) & 838${}_{-23,+28}$ & 0.019 & 37.6 & 30.4 & 0 & 33.5\% & 47.11 & 0.06 \\ +Linear (D) & 804${}_{-33,+26}$ & 0.014 & 38.5 & 36.4 & 0 & 34.8\% & 1.53 & 0.09 \\ +KNN (T+E) & 718${}_{-37,+23}$ & 0.000 & 40.4 & 36.3 & 0 & 45.8\% & 3.13 & 0.16 \\ +KNN (T) & 652${}_{-36,+35}$ & 0.000 & 41.5 & 33.4 & 0 & 47.6\% & 3.13 & 0.03 \\ +KNN (D) & 468${}_{-50,+41}$ & 0.000 & 43.7 & 43.2 & 0 & 54.7\% & 0.05 & 0.02 \\ +\bottomrule +\end{tabular} \ No newline at end of file diff --git a/data/lite/full-imputed/tabarena_leaderboard.csv b/data/lite/full-imputed/tabarena_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..9369a515e930bdee0d33f05cc6a13bec93997807 --- /dev/null +++ b/data/lite/full-imputed/tabarena_leaderboard.csv @@ -0,0 +1,46 @@ +method,time_train_s,time_infer_s,time_train_s_per_1K,time_infer_s_per_1K,normalized-error,normalized-error-task,champ_delta,loss_rescaled,time_train_s_rescaled,time_infer_s_rescaled,rank,median_metric_error,median_time_train_s,median_time_infer_s,median_time_train_s_per_1K,median_time_infer_s_per_1K,median_normalized-error,median_normalized-error-task,median_champ_delta,median_loss_rescaled,median_time_train_s_rescaled,median_time_infer_s_rescaled,median_rank,rank=1_count,rank=2_count,rank=3_count,rank>3_count,elo,elo+,elo-,winrate,mrr +AutoGluon 1.3 (4h),8355.364120885437,32.82596967734543,2898.240203433117,4.500812758268476,0.39210856520445686,0.38331044048081225,0.06589314401323856,0.041491842977667524,41965.2048341603,616.5290381859439,7.578431372549019,0.20731,7964.962425470352,4.336968660354614,1453.2679445956664,3.1473262800156863,0.33725328327072535,0.31950727747004454,0.03622044739680108,0.020690397957805647,27194.492186888157,175.7696866507318,6.0,7,5,5,34,1543.5,29.9,32.1,0.8504901960784313,0.30515820311091363 +REALMLP (tuned + ensemble),73262.05650625509,15.495973965700935,6221.197059136006,7.773886071139863,0.5008360344149994,0.5173193136666141,0.09160282167448115,0.06152408839485682,176934.089463731,288.3346092703114,10.92156862745098,0.20743,24379.27822804451,5.209537744522095,6026.466890881138,3.134808014069114,0.48215030896347355,0.503279028427124,0.042954914649413034,0.03764395740492137,123158.45385063179,235.70660025917869,9.0,0,2,4,45,1447.3,20.9,24.5,0.7745098039215687,0.14706934368628036 +GBM (tuned + ensemble),3051.8140357896395,23.19597311113395,765.6312795719994,3.893554958066526,0.5633713731527658,0.5825608689063619,0.10386671712735962,0.07609284586006167,11175.64311228686,518.7842315699711,11.764705882352942,0.2094,1670.2684843540192,3.933397054672241,416.5640684155986,2.2357814013957977,0.5860060403960241,0.5969263542300586,0.05473531792199926,0.03541201174403787,9062.280137688022,129.84563236620332,12.0,0,0,5,46,1428.4,29.6,21.9,0.7553475935828877,0.121792996415749 +TABM (tuned + ensemble),36745.76943855192,8.411393100140142,5149.522818881001,3.299204733305397,0.5087786520610308,0.5346927221309199,0.09090677197103196,0.08172458676501637,79103.63593427226,131.94770411319155,12.07843137254902,0.20702,8818.62617468834,3.054008960723877,3133.9115732644937,1.2727714166408632,0.47020083976979304,0.5335750739105611,0.04867875413017375,0.03910323253388955,49987.707956643724,112.26280083003438,10.0,2,0,4,45,1421.1,30.7,26.1,0.7482174688057041,0.16336111978040535 +CAT (tuned + ensemble),19506.186602929058,3.204344211840162,4009.454268521621,1.0312463789689519,0.580901607522528,0.5804570119648801,0.09707143902781738,0.0669444160716071,53079.26276755286,76.04544084479011,13.0,0.20994,7158.277770042419,1.5355591773986816,1665.5315644683697,0.5594151152585728,0.5700946601892775,0.559944059731688,0.04848404795913319,0.035016679200169744,22918.764355268395,47.71552750484114,11.0,0,1,5,45,1400.1,26.2,24.2,0.7272727272727273,0.12289937178003728 +MNCA (tuned + ensemble),50606.46933756155,274.8289722134085,6200.95174646742,33.492227920869496,0.568211153208397,0.5371927593350034,0.09985073861281364,0.08215353610049873,106030.16699977385,1982.0660894780217,13.745098039215685,0.2069,14787.927460432053,13.568246603012085,4618.502341732761,7.737494511886756,0.5565369335340675,0.49725904468662846,0.04682898488900977,0.026551744085150493,92410.46095689958,607.7040280210158,12.0,5,3,2,41,1382.2,23.9,22.1,0.7103386809269162,0.22067374483624724 +XGB (tuned + ensemble),5938.998393348619,7.841493718764362,1213.0777679819278,3.1002279660820715,0.6509288335184475,0.6390315248335094,0.10965043540709318,0.08446479876595654,13936.971832341356,215.6442792684636,14.088235294117647,0.20933,2063.2708826065063,2.946662425994873,700.9607998746845,1.4386570273653,0.6793464009669915,0.6335945885076538,0.06457358773793653,0.040500031069913546,11430.712273277264,98.30128646762262,13.0,1,2,2,46,1376.3,21.6,22.5,0.7025401069518716,0.12439404858060579 +CAT (tuned),19506.186602929058,0.4801775754666796,4009.454268521621,0.12298834764615349,0.5972104481929726,0.6150468125987735,0.10195642695109683,0.07193835382108436,53079.26276755286,9.224590724231657,14.088235294117647,0.2099,7158.277770042419,0.14083647727966309,1665.5315644683697,0.06528969211261772,0.601029599608834,0.653061982396853,0.052825231831448494,0.03923415834907945,22918.764355268395,4.080719588963731,13.5,0,3,0,48,1377.4,18.7,25.3,0.7025401069518716,0.11607300257026866 +GBM (tuned),3051.8140357896395,3.3178744736839745,765.6312795719994,0.6787985114862476,0.6529668965012456,0.6596211839895727,0.11308725913714573,0.09105024043347852,11175.64311228686,83.45556234970765,14.92156862745098,0.20979,1670.2684843540192,0.7316880226135254,416.5640684155986,0.3811195826851199,0.6262832446301,0.7102844021186877,0.05983857821112848,0.04176567096628159,9062.280137688022,21.854692683376335,14.0,0,1,0,50,1358.3,24.2,22.5,0.6836007130124777,0.09694924493274282 +TABM (tuned),36745.76943855192,0.9327322455013499,5149.522818881001,0.3522030333025916,0.5996801665884344,0.6132229919485352,0.10016506702920977,0.09598544852782041,79103.63593427226,14.22083187601248,15.009803921568627,0.20774,8818.62617468834,0.25295114517211914,3133.9115732644937,0.13006170213303378,0.5745991505979262,0.6374982297455444,0.04950878682353699,0.05438833715883013,49987.707956643724,9.039602142310635,14.0,0,3,3,45,1356.0,26.8,23.1,0.6815953654188949,0.12398662288331401 +XGB (tuned),5938.998393348619,1.6265712625840132,1213.0777679819278,0.6984321316309747,0.7007828758839686,0.6709185861286167,0.11169181163865116,0.08851353303412964,13936.971832341356,46.56690686937765,15.186274509803921,0.20927,2063.2708826065063,0.5078930854797363,700.9607998746845,0.21305428930075773,0.7223432295573234,0.7206450880553464,0.06599393337115189,0.04761154585613394,11430.712273277264,15.141274536165636,14.0,1,2,0,48,1352.7,22.5,23.9,0.6775846702317291,0.11344877101591168 +CAT (default),186.54889291408014,0.4368261870215921,85.73305725415607,0.1697456044717195,0.643860524899134,0.6658428877239055,0.10950467754460554,0.07751364464445108,425.6698111187737,11.747370317227338,15.27450980392157,0.20726,25.447056531906128,0.4332277774810791,6.700347814449044,0.08827101352602937,0.6746502933037238,0.726310142308562,0.06252447723989962,0.03478958020223492,120.27803545982526,7.036160265837067,16.0,2,0,0,49,1352.6,20.5,24.0,0.6755793226381461,0.11629532151491669 +MNCA (tuned),50606.46933756155,14.969161491768032,6200.95174646742,1.6696617082037302,0.7032607657750378,0.6270870922397168,0.10674533443017264,0.09370119890578864,106030.16699977385,104.17477306753874,15.617647058823529,0.20447,14787.927460432053,0.6251811981201172,4618.502341732761,0.46952933073043823,0.7724401098237351,0.6695251937984512,0.06144927536231881,0.05466681384890431,92410.46095689958,31.933384765725453,13.0,2,2,1,46,1343.5,24.5,22.0,0.6677807486631016,0.14517119411421664 +TABPFNV2 (tuned + ensemble),10220.018198588315,101.0832102392234,2951.1688208155488,48.21038294992157,0.5336461270994103,0.5765428892679287,0.10560816302666877,0.09802564032514699,63148.66138846355,4163.892043743008,16.58823529411765,0.21666,4395.466120958328,7.814479351043701,2942.0790635597914,17.37173979844504,0.6071274732571645,0.6318283166109684,0.08318481827952429,0.03857004834586989,26428.31699976118,914.2037538981667,15.0,11,1,3,36,1324.4,24.3,24.2,0.6457219251336899,0.2940015493833628 +REALMLP (tuned),73262.05650625509,0.7424195560754514,6221.197059136006,0.41086245878139677,0.7615648419724279,0.7447223040515885,0.12592186501993244,0.1013301702714578,176934.089463731,14.464289249160522,18.431372549019606,0.21789,24379.27822804451,0.25364184379577637,6026.466890881138,0.1692739561746046,0.8260416027840659,0.825363549862922,0.08741748349669953,0.06129841644554749,123158.45385063179,11.197261335744736,18.0,0,1,0,50,1287.3,24.6,19.4,0.6038324420677362,0.0783425846902445 +TABM (default),142.54126416468154,1.0708042359819598,23.88140219729834,0.4005178443372889,0.7249223512608948,0.7157630950616095,0.12859251185786122,0.12132522747625257,305.2739016434669,13.641685471262758,18.50980392156863,0.2124,38.16493201255798,0.17726969718933105,11.555005609392405,0.12732556548192603,0.8325407781946508,0.7714285714285748,0.07455117151840984,0.05717855281816143,191.5780246731349,9.78619298588067,18.0,1,0,1,49,1285.6,22.9,17.9,0.6020499108734403,0.09609270750581164 +TABICL (default),89.99167278701184,15.994201370314055,7.671787946091385,1.7323065455173496,0.6267118101348285,0.6956869302909715,0.11731404529532494,0.10755898132565567,134.32668884285462,203.85595038983618,18.784313725490197,0.20899,19.30116367340088,1.4308764934539795,6.856698312130071,1.5204305848178132,0.7253160785960966,0.822617740397896,0.06412908564335906,0.048797809240279176,96.03934388429276,99.16403825333815,18.0,4,2,2,43,1280.3,24.9,22.3,0.5958110516934046,0.16198176006742712 +TABPFNV2 (tuned),10220.018198588315,4.082395156224568,2951.1688208155488,1.9953713679087344,0.6455033122665624,0.6908235215961785,0.1228376450343903,0.11518503017650464,63148.66138846355,159.46255672145648,19.166666666666668,0.22795,4395.466120958328,0.3721911907196045,2942.0790635597914,0.2619314172957417,0.7572752170216083,0.9420289855072642,0.09073865766779232,0.05918640451838397,26428.31699976118,18.72396576888414,21.0,1,7,3,40,1271.2,26.7,17.1,0.5871212121212122,0.15977212030761678 +NN_TORCH (tuned + ensemble),29357.971773493522,13.246924213334626,3694.4132392234865,3.45030206949641,0.7913423906704912,0.7992730043623121,0.12568352571413424,0.10124805716394422,94556.98181855923,212.38450473432735,19.764705882352942,0.21375,10620.000384569168,3.6842262744903564,2832.7960851387297,1.800873875617981,0.9196144626274988,0.9071618037135285,0.08148151246451829,0.06654975288816589,57294.450258986595,164.18272158737435,21.0,0,0,1,50,1261.3,22.4,24.4,0.5735294117647058,0.07101795663505563 +TABPFNV2 (default),12.262318129632988,0.840339740117391,4.40142189361026,0.44785573341984464,0.6822105535450186,0.73071944715153,0.13292725730508487,0.12726933227387377,68.41353371526327,34.453290769213694,21.294117647058822,0.22782,9.203422546386719,0.3870081901550293,3.265341403173851,0.3151157987035174,1.0,1.0,0.08973027820339718,0.08167997085223988,43.42507231078893,20.360866660523406,24.5,3,4,4,40,1232.7,21.3,22.3,0.5387700534759359,0.16062708054431477 +EBM (tuned + ensemble),33815.09879914452,1.2886618586147534,6555.968895975794,0.4924074534971456,0.8251828316021278,0.7951948063939178,0.16005304449444593,0.14511080052822103,41186.37921180388,19.776872049406894,21.480392156862745,0.21048,2774.86767077446,0.33999180793762207,1323.3940540554784,0.18426990509033203,0.9668458098028044,0.966938614382516,0.11314433759685272,0.06365614798694313,17200.38772644666,11.089686715499157,22.0,0,3,1,47,1230.3,20.4,23.9,0.5345365418894831,0.09106637492313346 +MNCA (default),245.15467273020278,8.083605813045128,18.524091854412294,1.1280259623754931,0.8466123430768121,0.8094588747505642,0.15569918835101015,0.11122447860327346,315.39790519179047,62.504831773745536,21.627450980392158,0.21297,40.294944524765015,0.5322227478027344,13.739717303881024,0.31576028319217697,1.0,0.9805816199463006,0.07931711197987557,0.08713555135923144,220.32243700895648,24.025524470708156,21.0,1,1,0,49,1226.7,24.4,23.9,0.5311942959001783,0.08706985371682067 +TABDPT (default),165.2279131459255,62.46656582402248,26.956914471170485,21.63883475001047,0.7067604944091354,0.7260442124341767,0.14892084197483565,0.11705276575441004,555.3035574597842,1632.683869591928,22.029411764705884,0.21757,99.63741898536682,27.656949520111084,20.559618245438234,8.61670085798515,1.0,1.0,0.09477956973445767,0.05489679334224334,445.8805538455051,1247.4102886322382,24.5,4,3,0,44,1216.6,23.6,26.9,0.5220588235294118,0.1586195735020882 +XT (tuned + ensemble),1293.0170739772273,3.137296905704573,465.97356682221573,1.4220614441065493,0.8827760158511792,0.8338042757777816,0.15945119135465702,0.13475107219507007,5360.97691412289,92.16814260531523,22.745098039215687,0.21292,775.4602122306824,2.0695900917053223,191.43562446750065,0.7604422990939085,1.0,1.0,0.10874752469158522,0.0792857142857151,3382.254534218774,83.98689534392173,26.0,0,1,0,50,1204.6,21.1,23.7,0.5057932263814616,0.07246402496622346 +REALMLP (default),445.0601024861429,0.7650933592927223,36.1037613876711,0.41223276685751575,0.8980359281995941,0.84190216028381,0.14428929855198652,0.12003713004556894,1019.9231218617758,14.453050422610211,23.245098039215687,0.22192,142.23176908493042,0.21722769737243652,34.70485605272871,0.17513688577887485,1.0,1.0,0.10505836520885536,0.07552484491529415,689.4249262469413,12.24685971375083,24.0,1,0,0,50,1192.7,23.7,20.2,0.4944295900178253,0.07373292585251917 +EBM (tuned),33815.09879914452,0.15010464892667882,6555.968895975794,0.06527171144492658,0.8740079837962136,0.829692479867262,0.1662347356156192,0.15624599233957168,41186.37921180388,2.297509957092046,23.42156862745098,0.21528,2774.86767077446,0.03941011428833008,1323.3940540554784,0.01867789003336745,1.0,1.0,0.11378284728865762,0.0787717680247385,17200.38772644666,1.1641602772099668,24.5,1,1,1,48,1189.5,19.5,23.6,0.49041889483065954,0.08673977917708815 +NN_TORCH (tuned),29357.971773493522,0.60710202478895,3694.4132392234865,0.18506171262100615,0.8968088714508368,0.8391436503269246,0.14460984883972092,0.12311765757203913,94556.98181855923,10.572861046995973,23.54901960784314,0.21109,10620.000384569168,0.22349023818969727,2832.7960851387297,0.11152923107147217,1.0,1.0,0.09669761975708657,0.07345521618835778,57294.450258986595,8.23840338809871,23.0,0,0,0,51,1188.6,25.0,20.4,0.4875222816399287,0.056978998909371976 +XGB (default),13.003299437317194,0.6573358236574659,3.2763217873435058,0.3062962827156932,0.8935881423667761,0.8437248313951449,0.1445840100645508,0.14006782212608607,39.750102662431935,19.332008402042312,23.725490196078432,0.21746,6.575343370437622,0.39388179779052734,2.0589472404367264,0.12183857766160212,1.0,1.0,0.09723220704529156,0.06256071497960061,32.87694416794321,9.490148793740056,24.0,1,0,0,50,1185.8,20.0,24.7,0.48351158645276293,0.07115206856848182 +XT (tuned),1293.0170739772273,0.34508594344643984,465.97356682221573,0.17473237760756105,0.9078855110953847,0.8425160361668544,0.16907500703462622,0.14076074685646162,5360.97691412289,10.510664591668581,24.38235294117647,0.21281,775.4602122306824,0.17961668968200684,191.43562446750065,0.10098353169828128,1.0,1.0,0.12217194570135836,0.08661653630785078,3382.254534218774,9.053845119655822,29.0,0,2,0,49,1171.1,24.4,24.2,0.46858288770053474,0.0776719629546318 +FASTAI (tuned + ensemble),6591.795950641819,16.245619264303468,1342.3410902155488,7.489539632560419,0.847763610444884,0.8637618919906302,0.16568514956913477,0.13145236923425976,21804.052352671944,459.2322661590378,24.480392156862745,0.22665,3142.961499929428,10.290910243988037,594.9528585230638,4.650812904660155,1.0,1.0,0.10135260329812934,0.08681294440231768,17514.935036133815,369.5983579935041,27.0,0,0,0,51,1168.1,26.0,22.4,0.4663547237076649,0.06277458091841293 +RF (tuned + ensemble),2227.1437476008546,2.46178646181144,528.9202336737816,1.2212931064922863,0.9036228102132694,0.884712211600403,0.16611969123929415,0.14823223636591268,6570.203557563642,82.81791093485884,24.92156862745098,0.21651,876.37784075737,1.7210140228271484,377.08301133934634,0.7469802601991967,1.0,1.0,0.08806938150372812,0.0845151199165799,5196.472002023166,66.34473572972412,27.0,0,2,0,49,1160.0,21.6,18.1,0.4563279857397504,0.06667956988181886 +GBM (default),9.322186521455354,1.1440442029167623,3.199181885596564,0.3147197775938453,0.9255861200115366,0.8861825462488151,0.15435499226686888,0.1416384927139699,46.93362314737936,31.99287107427668,25.264705882352942,0.21883,6.97411584854126,0.6232860088348389,2.2021467310523017,0.17114277689525664,1.0,1.0,0.10001855631842682,0.07712613158180785,26.977037743860404,12.72044936701149,26.0,0,0,0,51,1151.3,25.8,20.3,0.4485294117647059,0.04644769227355601 +EBM (default),83.37771165604686,0.18329166431053012,10.41915882393762,0.09845206874115144,0.8753701223112293,0.8524797938262743,0.17560296932715116,0.16341835943310914,128.33936987969741,3.956299459273312,25.46078431372549,0.21098,11.009112119674683,0.06519913673400879,5.48102419993599,0.05915899401478989,1.0,1.0,0.1212644465105922,0.07181719260065285,66.69318849626488,3.0422808378588053,25.0,1,0,0,50,1149.7,25.6,25.6,0.44407308377896615,0.07366716368892957 +RF (tuned),2227.1437476008546,0.2597549288880591,528.9202336737816,0.14418858473403337,0.9311148829801792,0.9276340007428121,0.17466346964609464,0.1593467199145403,6570.203557563642,8.720931698010416,27.15686274509804,0.21762,876.37784075737,0.15849757194519043,377.08301133934634,0.09141294871228287,1.0,1.0,0.11332046274795926,0.10944640481286505,5196.472002023166,7.735832448019426,28.5,0,0,0,51,1115.1,21.3,16.4,0.4055258467023173,0.046552852978102924 +FASTAI (tuned),6591.795950641819,1.031626776152966,1342.3410902155488,0.6256477184380718,0.930127335805145,0.9094712268793881,0.18206255119801731,0.15987116884933936,21804.052352671944,39.15043952663921,27.745098039215687,0.23007,3142.961499929428,0.8062961101531982,594.9528585230638,0.33651872811068495,1.0,1.0,0.13059041706547048,0.09939086593924433,17514.935036133815,29.379474918801876,29.0,0,0,1,50,1101.5,21.9,19.3,0.39215686274509803,0.05195476175795121 +NN_TORCH (default),73.82550601865731,0.5703827493331012,14.544520367121898,0.24243997995483998,0.9806438127346997,0.9611986149202854,0.1993761159143943,0.180385766131793,302.60238317827014,12.920116170611008,30.99019607843137,0.21872,30.76021695137024,0.2736239433288574,8.95763915486452,0.12885630130767822,1.0,1.0,0.1279603352562162,0.12220480897522226,159.50005812712556,8.385666110468058,33.0,0,0,0,51,1029.1,20.6,31.5,0.3184046345811052,0.03651853328495362 +RF (default),5.493931228039312,0.18195290658988206,0.9395711378622816,0.07441601332167096,0.9921615843984544,0.9817224568833112,0.22764762324691398,0.22902221826215566,12.413845548911803,5.3715309875205834,32.22549019607843,0.23701,1.763962745666504,0.09250092506408691,0.4316611380184793,0.05251745318464858,1.0,1.0,0.17048329819109986,0.15585831773749825,7.440479734614894,4.454035040702154,34.0,0,0,0,51,1000.0,0.0,0.0,0.29032976827094475,0.03648907633178747 +FASTAI (default),27.62558492492227,1.0500701642503925,4.982714620465621,0.48138517234406897,0.980597127305428,0.9597833502824441,0.2232386131399183,0.20837341014028052,85.33850461343444,31.816885619382305,33.09803921568628,0.23658,14.524733304977417,0.6894059181213379,3.1153293528650394,0.31195542895805195,1.0,1.0,0.15646970534797477,0.132224835215793,80.02261780548187,26.24796931235529,35.0,0,0,0,51,977.2,21.2,23.9,0.27049910873440286,0.033494443524612505 +XT (default),3.237064623365215,0.20349772771199545,0.7705107029297205,0.07802443108906316,0.976846527522819,0.9655488789377221,0.2458932265203503,0.2513698525476759,6.714510413684847,5.481544853676871,33.431372549019606,0.24149,0.9253263473510742,0.08609175682067871,0.25976606260770435,0.0542029349444295,1.0,1.0,0.17695109438477097,0.1548442777574803,5.482206453649765,4.486837961877324,37.0,0,0,0,51,969.0,24.2,24.8,0.2629233511586453,0.03407274897812548 +LR (tuned + ensemble),297.7768230718725,1.372081803340538,106.57523410916248,0.517828397925699,0.9690155979901051,0.9782757252764236,0.3285819246627462,0.373677623846431,1284.9569849451952,23.77736863105658,36.61764705882353,0.24535,167.935129404068,0.30171775817871094,47.11049555587997,0.15632276260139144,1.0,1.0,0.2711660718812774,0.2552683896620274,878.5162567525001,9.250054132082282,40.0,1,0,0,50,873.8,20.3,28.7,0.19050802139037434,0.04686023531997359 +LR (tuned),297.7768230718725,0.41669238314909096,106.57523410916248,0.1447260883613805,0.9805057099085136,0.9850180993290127,0.33473141690041264,0.3862620684037475,1284.9569849451952,6.250094556288328,37.568627450980394,0.25329,167.935129404068,0.11377191543579102,47.11049555587997,0.0626428060263515,1.0,1.0,0.2768258727778956,0.2552683896620274,878.5162567525001,4.008480462038967,40.5,0,0,1,50,837.7,27.3,22.4,0.16889483065953653,0.03285569931980569 +LR (default),7.020498261732214,0.4395605956806856,2.5089988631155826,0.17274864462745904,0.9855950536843827,0.9904389569653577,0.3478175509316984,0.43332199535666077,32.25963024407083,8.287063835475498,38.46078431372549,0.25334,5.345212697982788,0.13631606101989746,1.534023110424607,0.08993275118189932,1.0,1.0,0.2768258727778956,0.31807101339614474,17.290213667932342,5.167879066497201,41.0,0,0,0,51,804.0,25.4,32.3,0.14861853832442068,0.02748941184714836 +KNN (tuned + ensemble),191.27347564697266,6.6349917159361,29.50398939167977,0.5506054195292707,1.0,0.9906374645099302,0.4581854862659557,0.6147034839095564,81.3768199133465,45.17733544160728,40.36274509803921,0.34601,13.195080518722534,0.23362517356872559,3.133583813905716,0.15550157631257452,1.0,1.0,0.42992491292377444,0.6963553049981372,62.764963980484396,12.85103075181078,43.0,0,0,0,51,718.3,22.4,36.8,0.1053921568627451,0.027582577954289256 +KNN (tuned),191.27347564697266,1.469562432345222,29.50398939167977,0.11124081145987941,1.0,0.9885254568237573,0.4755023817567232,0.6600856306001983,81.3768199133465,9.204095260134709,41.529411764705884,0.34503,13.195080518722534,0.07799005508422852,3.133583813905716,0.030069828033447266,1.0,1.0,0.456326948689309,0.7494456544265635,62.764963980484396,2.2192320534223704,44.0,0,0,1,50,651.7,34.5,35.9,0.07887700534759358,0.029927590681413103 +KNN (default),1.8986088995840036,0.1759517753825468,0.5259164666571766,0.03511341794167653,1.0,1.0,0.5472117300773635,0.9181039011326919,1.049764217774364,2.1650106683540193,43.666666666666664,0.42464,0.2273859977722168,0.03137516975402832,0.0509044812778989,0.01926569938659668,1.0,1.0,0.5196494204127793,1.0,1.0,1.4495271718572689,45.0,0,0,0,51,468.3,40.2,49.3,0.030303030303030304,0.023129909171668465 diff --git a/data/lite/full-imputed/time_plot.pdf b/data/lite/full-imputed/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e726cea2ef230568231dd44e7bfe4ae43efd3ad8 Binary files /dev/null and b/data/lite/full-imputed/time_plot.pdf differ diff --git a/data/lite/full-imputed/time_plot.png.zip b/data/lite/full-imputed/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..0c4122df4eefdf2f0dc7de455901d673acb0aa85 --- /dev/null +++ b/data/lite/full-imputed/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56da6adb10ca2b086eabc4c876fe7df45eae13d50babe26234aee71f4c30d18c +size 338023 diff --git a/data/lite/full-imputed/tuning-impact-elo-horizontal.pdf b/data/lite/full-imputed/tuning-impact-elo-horizontal.pdf new file mode 100644 index 0000000000000000000000000000000000000000..56683368bad0a597a9d14f66823e81003a21f604 Binary files /dev/null and b/data/lite/full-imputed/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/full-imputed/tuning-impact-elo-horizontal.png.zip b/data/lite/full-imputed/tuning-impact-elo-horizontal.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..99681f4d639308f0e9c284c3c302ac60f1ba457a --- /dev/null +++ b/data/lite/full-imputed/tuning-impact-elo-horizontal.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97b601d38f5b7d8354869571499195e911b5b6cdf9c761567fc32bab24536c0a +size 155660 diff --git a/data/lite/full-imputed/tuning-impact-elo.pdf b/data/lite/full-imputed/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5ae443720ae08f92b669bb4c0f2de71e9923d223 Binary files /dev/null and b/data/lite/full-imputed/tuning-impact-elo.pdf differ diff --git a/data/lite/full-imputed/tuning-impact-elo.png.zip b/data/lite/full-imputed/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..d9a0891633b38517706a8e5ec2143c1f8c5c5eb2 --- /dev/null +++ b/data/lite/full-imputed/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e102b5fdf4047114770b3b4891f4c253cbb5f3c90f93a1a01ea9b709bf3535af +size 163953 diff --git a/data/lite/full-reg/figures/critical-diagram.pdf b/data/lite/full-reg/figures/critical-diagram.pdf index 8c8c71bb64483b45ec7d27bc8e8d7f74f516d4dc..f9c9146f9a76e5ee7b5a80225b987337f7ca2678 100644 Binary files a/data/lite/full-reg/figures/critical-diagram.pdf and b/data/lite/full-reg/figures/critical-diagram.pdf differ diff --git a/data/lite/full-reg/figures/critical-diagram.png.zip b/data/lite/full-reg/figures/critical-diagram.png.zip index 806ac9e814e9ff517048964639c9a84a47722c59..8878f8f4f5d647dbd2ac839838854910e868699e 100644 --- a/data/lite/full-reg/figures/critical-diagram.png.zip +++ b/data/lite/full-reg/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c820f0dacaec7458f05c9bf9b281cd3eefcc5514fb205adfe75336408a7e14a2 -size 291039 +oid sha256:78c1491819e0ba4900508b27a10fd8510bc971b81dfad652f1d58c08a761904e +size 291177 diff --git a/data/lite/full-reg/time_plot.pdf b/data/lite/full-reg/time_plot.pdf index b4802f771861d719bdf5520effd5bbac1b5146f7..04c06743adcc24222b57bcce0282a742e2f42f61 100644 Binary files a/data/lite/full-reg/time_plot.pdf and b/data/lite/full-reg/time_plot.pdf differ diff --git a/data/lite/full-reg/time_plot.png.zip b/data/lite/full-reg/time_plot.png.zip index 263ba6a3c2cb850a1ba7da44ac69dda4162238c3..04c4bbd00046934f882720b6b768477f3233dce8 100644 --- a/data/lite/full-reg/time_plot.png.zip +++ b/data/lite/full-reg/time_plot.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3fe29fdec65f03c7d3bae994bc888e7c7ec498e3800a153239b08e6703208a4a -size 79911 +oid sha256:cc5824fd61e5ebafb792bbf24c9f760eb5b11fd1d7b64f79a94713e7b4f072da +size 80035 diff --git a/data/lite/full-reg/tuning-impact-elo-horizontal.pdf b/data/lite/full-reg/tuning-impact-elo-horizontal.pdf index 8427817b6fb94a00fb9d33028da42717551415ac..e800459619bbaeb28a807f653c36219f029e9fe0 100644 Binary files a/data/lite/full-reg/tuning-impact-elo-horizontal.pdf and b/data/lite/full-reg/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/full-reg/tuning-impact-elo-horizontal.png.zip b/data/lite/full-reg/tuning-impact-elo-horizontal.png.zip index 1e01391e7e232af509f3fb9d964758fb24628af6..ba3d394513cdc5058d0b817f7dff80970fafc25f 100644 --- a/data/lite/full-reg/tuning-impact-elo-horizontal.png.zip +++ b/data/lite/full-reg/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7c6c1fe19f28c3138bf15b5a4b8ba0daa68f702c17056dc4cb9ec570d9e18105 -size 129444 +oid sha256:cae5ae2b5f5d3b044a13ac2d95bb50fafb4475a7fdbcbf08d09d4f5b261b1591 +size 129606 diff --git a/data/lite/full-reg/tuning-impact-elo.pdf b/data/lite/full-reg/tuning-impact-elo.pdf index a780a95e469545c019f40be2b37631144e2e54cc..b9b1cc839e36ecd08800b42f421d53961d9f5b52 100644 Binary files a/data/lite/full-reg/tuning-impact-elo.pdf and b/data/lite/full-reg/tuning-impact-elo.pdf differ diff --git a/data/lite/full-reg/tuning-impact-elo.png.zip b/data/lite/full-reg/tuning-impact-elo.png.zip index 78690e07dfc9a688eac9bdf64ef6e234840e997c..e0885fb5ac57057f80bd50e36cba6b69eb1c517b 100644 --- a/data/lite/full-reg/tuning-impact-elo.png.zip +++ b/data/lite/full-reg/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:48e44fb1167bb8a9344c3a7b3bf8f4b48876308965ed7c1d23cae687e763f871 -size 129280 +oid sha256:84fe2ac84347bcb80c9f5653957d45b118b12db3165913fe2b70c5107fcc275f +size 129420 diff --git a/data/lite/full/figures/critical-diagram.pdf b/data/lite/full/figures/critical-diagram.pdf index 603a4f6d89b2f1615ca5dee7bdb3f9eb18f8c611..533daa633fcce80ab8cd5144576047269e6038e2 100644 Binary files a/data/lite/full/figures/critical-diagram.pdf and b/data/lite/full/figures/critical-diagram.pdf differ diff --git a/data/lite/full/figures/critical-diagram.png.zip b/data/lite/full/figures/critical-diagram.png.zip index 7c6b7f6cf494dea9d0918f3fa500b978623344b4..65a5bb071c39977a3a158f4850d5c24769df520e 100644 --- a/data/lite/full/figures/critical-diagram.png.zip +++ b/data/lite/full/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8ff960f4ae97033ab8ca8cd56ce1f4d54e7f49be4b3f0cf96b17e2cb81321c5a -size 296265 +oid sha256:6b230f71d3e3336b1f36ab7fcb05cad4e2455c96a9c5b2f3aa45402b8ae4b1cf +size 296403 diff --git a/data/lite/full/time_plot.pdf b/data/lite/full/time_plot.pdf index 969dfadbd4236ef005d0ee18494eb219ce9112a4..55ee045de12d9ad38566cd4998c2d7346309e06f 100644 Binary files a/data/lite/full/time_plot.pdf and b/data/lite/full/time_plot.pdf differ diff --git a/data/lite/full/time_plot.png.zip b/data/lite/full/time_plot.png.zip index 8b42c4b6e2232a76fd9e3d1f5ee7ee2f360f2d09..a0b178da5b57a5a39b1aaa74bf683fb3f7d47cfe 100644 --- a/data/lite/full/time_plot.png.zip +++ b/data/lite/full/time_plot.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4f0389b7f203eb4d39d6b6a3da311337a46c01621e12fc206d904eff8ce58527 -size 337899 +oid sha256:b69a3b1d35aa95801b955ff4f60448cd991359909cbad8a05b036404900973c0 +size 338023 diff --git a/data/lite/full/tuning-impact-elo-horizontal.pdf b/data/lite/full/tuning-impact-elo-horizontal.pdf index 0d17bd96b8ec531addaf466b220f5684d2c39680..b17695c9c892f0502ba4bd0e4dc35c3db7e67de8 100644 Binary files a/data/lite/full/tuning-impact-elo-horizontal.pdf and b/data/lite/full/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/full/tuning-impact-elo-horizontal.png.zip b/data/lite/full/tuning-impact-elo-horizontal.png.zip index b65ab41d5845c5dcf678a9654d6135af282cba29..6747d2be29941f30e726ff3165b410434b6f7153 100644 --- a/data/lite/full/tuning-impact-elo-horizontal.png.zip +++ b/data/lite/full/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:717887ad1b2151bca8f71879dd6dc9ea0173eebf51e41d1f983b03864a6be794 -size 128322 +oid sha256:67455bf468e8eb465998b1c4c47b79c96d43913d90dbb5abc3b85e0529dee50f +size 128484 diff --git a/data/lite/full/tuning-impact-elo.pdf b/data/lite/full/tuning-impact-elo.pdf index 912f03361cb9baf857f7b1e25c537adcc1b1bcbe..cdcb5c728e75fa9025f96c9a0e8ca755f0543147 100644 Binary files a/data/lite/full/tuning-impact-elo.pdf and b/data/lite/full/tuning-impact-elo.pdf differ diff --git a/data/lite/full/tuning-impact-elo.png.zip b/data/lite/full/tuning-impact-elo.png.zip index ee51afa28d858e85d8a16f6aeaf3dc7e86aef758..3c167e442dd6d8db5ad185d7d43cc1e2378b63f1 100644 --- a/data/lite/full/tuning-impact-elo.png.zip +++ b/data/lite/full/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2a76c62115b0ed7fe8d657e2a60f9345f3b722b683797f51beddab991413d1c0 -size 122269 +oid sha256:21ad94e758e184f203869669f510a5b2c193ba602443b77d9b3fb9582b3f7c18 +size 122409 diff --git a/data/lite/tabicl-cls/figures/critical-diagram.pdf b/data/lite/tabicl-cls/figures/critical-diagram.pdf index 230e5237b8ba9e27c899ddb6d270c0ef3939635a..6950deb9637f10926a4d2404e8ae674e212b90a8 100644 Binary files a/data/lite/tabicl-cls/figures/critical-diagram.pdf and b/data/lite/tabicl-cls/figures/critical-diagram.pdf differ diff --git a/data/lite/tabicl-cls/figures/critical-diagram.png.zip b/data/lite/tabicl-cls/figures/critical-diagram.png.zip index 5473e45838602b2408c048ae6f15679b205af04e..81a0033b4dbff7777ce194a9e5c0b22dbb867bc0 100644 --- a/data/lite/tabicl-cls/figures/critical-diagram.png.zip +++ b/data/lite/tabicl-cls/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9b62643ca543a5b490148fb916ed1fdf862546abf3d6ecb1a20ed75b16e37b3f -size 312252 +oid sha256:124044fd7b23cbd430bc19b4e2e1a3fbd5e6b8807f46e0176d952b12077db8bf +size 312390 diff --git a/data/lite/tabicl-cls/time_plot.pdf b/data/lite/tabicl-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..aa97036b10cb8cb0231366f7b19c1966830dede6 Binary files /dev/null and b/data/lite/tabicl-cls/time_plot.pdf differ diff --git a/data/lite/tabicl-cls/time_plot.png.zip b/data/lite/tabicl-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..976b584da16d1bb663bdda31ebb092404ab21662 --- /dev/null +++ b/data/lite/tabicl-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6de0c7d779ec267a6404599faa961c02cebc5dfc3085abd3b68926a43013eff +size 338023 diff --git a/data/lite/tabicl-cls/tuning-impact-elo-horizontal.pdf b/data/lite/tabicl-cls/tuning-impact-elo-horizontal.pdf index c80bd4877fff9df6358355cf4e013022404bc8be..26c4107f5370328ecb3e09e8ea55206b00fdea2a 100644 Binary files a/data/lite/tabicl-cls/tuning-impact-elo-horizontal.pdf and b/data/lite/tabicl-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabicl-cls/tuning-impact-elo-horizontal.png.zip b/data/lite/tabicl-cls/tuning-impact-elo-horizontal.png.zip index a82e75a5f4e77ce83a36fe4ab772e61e785bbf53..c695beacaa8f34401985d73d2ef799fc1c86761f 100644 --- a/data/lite/tabicl-cls/tuning-impact-elo-horizontal.png.zip +++ b/data/lite/tabicl-cls/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:90a0e767745b3612add5803b4bb49d6445f96161eea7a533f54f123ac9244056 -size 131668 +oid sha256:31a468f3bfb822d843e8095e24c3b6ca0aa5e3aea40647028a7d3bc0d593bca1 +size 131830 diff --git a/data/lite/tabicl-cls/tuning-impact-elo.pdf b/data/lite/tabicl-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e68f544912a84212f1680b7ef7850c6565801748 Binary files /dev/null and b/data/lite/tabicl-cls/tuning-impact-elo.pdf differ diff --git a/data/lite/tabicl-cls/tuning-impact-elo.png.zip b/data/lite/tabicl-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..460a1e3f46bb9c73163190024c16acb03b4c1c5c --- /dev/null +++ b/data/lite/tabicl-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e4f5d8c3fc95ac7b69b4b91c95470a43de69d7b70aeed2f52a67fba612cea71 +size 124842 diff --git a/data/lite/tabicl-imputed-cls/figures/critical-diagram.pdf b/data/lite/tabicl-imputed-cls/figures/critical-diagram.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1f8f4db718c258df23dfa285651f41db6f323695 Binary files /dev/null and b/data/lite/tabicl-imputed-cls/figures/critical-diagram.pdf differ diff --git a/data/lite/tabicl-imputed-cls/figures/critical-diagram.png.zip b/data/lite/tabicl-imputed-cls/figures/critical-diagram.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..8b3e81b0ecec9d9c094415ad90a80294be7b6a25 --- /dev/null +++ b/data/lite/tabicl-imputed-cls/figures/critical-diagram.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45f41a0447e3a605567d48ea6e120a8b9f5ceb96fea12b8fdcd5091ed80e19b9 +size 319696 diff --git a/data/lite/tabicl-imputed-cls/leaderboard.tex b/data/lite/tabicl-imputed-cls/leaderboard.tex new file mode 100644 index 0000000000000000000000000000000000000000..dbce3cd5240d4d60bdbbe051e4a2a62dce853121 --- /dev/null +++ b/data/lite/tabicl-imputed-cls/leaderboard.tex @@ -0,0 +1,53 @@ +\begin{tabular}{llcccccrr} +\toprule +\textbf{Model} & \textbf{Elo ($\uparrow$)} & \textbf{Norm.} & \textbf{Avg.} & \textbf{Harm.} & \textbf{\#wins ($\uparrow$)} & \textbf{Improva-} & \textbf{Train time} & \textbf{Predict time} \\ + & & \textbf{score ($\uparrow$)} & \textbf{rank ($\downarrow$)} & \textbf{mean} & & \textbf{bility ($\downarrow$)} & \textbf{per 1K [s]} & \textbf{per 1K [s]} \\ + & & & & \textbf{rank ($\downarrow$)} & & & & \\ +\midrule +AutoGluon 1.3 (4h) & \textcolor{gold}{\textbf{1520${}_{-34,+33}$}} & \textcolor{gold}{\textbf{0.569}} & \textcolor{gold}{\textbf{8.5}} & \textcolor{gold}{\textbf{3.3}} & \textcolor{gold}{\textbf{6}} & \textcolor{gold}{\textbf{8.0\%}} & 1239.48 & 1.98 \\ +RealMLP (T+E) & \textcolor{silver}{\textbf{1424${}_{-30,+28}$}} & 0.418 & \textcolor{bronze}{\textbf{12.3}} & 7.6 & 0 & 11.5\% & 5429.82 & 2.94 \\ +TabM (T+E) & \textcolor{bronze}{\textbf{1422${}_{-27,+31}$}} & 0.491 & \textcolor{silver}{\textbf{12.2}} & 5.6 & 2 & \textcolor{bronze}{\textbf{11.0\%}} & 2261.57 & 1.26 \\ +LightGBM (T+E) & 1404${}_{-32,+27}$ & 0.389 & 13.1 & 9.3 & 0 & 12.5\% & 368.47 & 1.22 \\ +TabICL (D) & 1392${}_{-31,+30}$ & \textcolor{silver}{\textbf{0.521}} & 13.6 & \textcolor{bronze}{\textbf{4.6}} & \textcolor{bronze}{\textbf{4}} & \textcolor{silver}{\textbf{10.3\%}} & 9.07 & 2.11 \\ +CatBoost (T+E) & 1387${}_{-34,+29}$ & 0.377 & 13.9 & 8.0 & 0 & 11.6\% & 1207.49 & 0.52 \\ +XGBoost (T+E) & 1380${}_{-28,+25}$ & 0.320 & 14.2 & 8.3 & 0 & 13.0\% & 655.54 & 0.78 \\ +ModernNCA (T) & 1371${}_{-29,+31}$ & 0.275 & 14.7 & 6.7 & 2 & 11.8\% & 4498.66 & 0.45 \\ +ModernNCA (T+E) & 1363${}_{-34,+18}$ & 0.383 & 15.1 & 5.9 & 2 & 11.8\% & 4498.66 & 8.48 \\ +CatBoost (T) & 1361${}_{-29,+33}$ & 0.360 & 15.1 & 10.0 & 0 & 12.3\% & 1207.49 & 0.05 \\ +CatBoost (D) & 1360${}_{-23,+29}$ & 0.343 & 15.2 & 7.8 & 2 & 12.6\% & 5.18 & 0.09 \\ +TabM (T) & 1358${}_{-28,+28}$ & 0.409 & 15.3 & 7.4 & 0 & 12.0\% & 2261.57 & 0.13 \\ +TabPFNv2 (T+E) & 1355${}_{-35,+25}$ & \textcolor{bronze}{\textbf{0.506}} & 15.5 & \textcolor{silver}{\textbf{3.7}} & \textcolor{gold}{\textbf{6}} & 12.0\% & 2955.87 & 18.13 \\ +XGBoost (T) & 1348${}_{-25,+33}$ & 0.264 & 15.8 & 8.6 & 1 & 13.3\% & 655.54 & 0.16 \\ +LightGBM (T) & 1340${}_{-27,+33}$ & 0.302 & 16.1 & 10.7 & 0 & 13.5\% & 368.47 & 0.12 \\ +TabPFNv2 (T) & 1297${}_{-28,+26}$ & 0.393 & 18.2 & 5.9 & 1 & 13.9\% & 2955.87 & 0.32 \\ +TabM (D) & 1297${}_{-25,+27}$ & 0.272 & 18.4 & 9.1 & 1 & 15.2\% & 9.30 & 0.13 \\ +EBM (T+E) & 1282${}_{-32,+25}$ & 0.170 & 19.1 & 9.6 & 0 & 17.0\% & 852.29 & 0.21 \\ +RealMLP (T) & 1279${}_{-28,+29}$ & 0.198 & 19.3 & 12.8 & 0 & 15.1\% & 5429.82 & 0.17 \\ +TorchMLP (T+E) & 1278${}_{-30,+26}$ & 0.228 & 19.3 & 13.0 & 0 & 14.3\% & 2406.10 & 2.04 \\ +TabPFNv2 (D) & 1254${}_{-32,+23}$ & 0.360 & 20.6 & 5.9 & 3 & 14.9\% & 3.12 & 0.32 \\ +EBM (T) & 1239${}_{-30,+30}$ & 0.110 & 21.4 & 12.4 & 0 & 17.7\% & 852.29 & 0.02 \\ +FastaiMLP (T+E) & 1218${}_{-25,+28}$ & 0.207 & 22.4 & 13.5 & 0 & 18.0\% & 581.37 & 4.78 \\ +EBM (D) & 1202${}_{-27,+23}$ & 0.119 & 23.2 & 11.5 & 1 & 18.5\% & 4.40 & 0.05 \\ +XGBoost (D) & 1196${}_{-29,+24}$ & 0.102 & 23.6 & 12.7 & 1 & 16.6\% & 1.66 & 0.12 \\ +RealMLP (D) & 1193${}_{-22,+28}$ & 0.095 & 23.7 & 12.6 & 1 & 16.7\% & 34.62 & 0.17 \\ +ModernNCA (D) & 1193${}_{-35,+26}$ & 0.142 & 23.7 & 11.8 & 1 & 18.0\% & 12.99 & 0.34 \\ +TorchMLP (T) & 1190${}_{-31,+28}$ & 0.085 & 23.8 & 18.5 & 0 & 16.7\% & 2406.10 & 0.11 \\ +ExtraTrees (T+E) & 1184${}_{-31,+26}$ & 0.082 & 24.3 & 17.4 & 0 & 18.4\% & 187.87 & 0.72 \\ +RandomForest (T+E) & 1153${}_{-27,+28}$ & 0.067 & 25.7 & 13.6 & 0 & 19.0\% & 266.25 & 0.64 \\ +TabDPT (D) & 1144${}_{-35,+29}$ & 0.186 & 26.2 & 9.0 & 1 & 19.2\% & 20.26 & 8.55 \\ +ExtraTrees (T) & 1143${}_{-35,+27}$ & 0.056 & 26.2 & 15.6 & 0 & 19.6\% & 187.87 & 0.08 \\ +FastaiMLP (T) & 1139${}_{-27,+31}$ & 0.094 & 26.4 & 16.7 & 0 & 20.1\% & 581.37 & 0.33 \\ +LightGBM (D) & 1131${}_{-32,+21}$ & 0.053 & 26.8 & 23.5 & 0 & 18.3\% & 1.64 & 0.15 \\ +RandomForest (T) & 1112${}_{-31,+34}$ & 0.045 & 27.7 & 20.8 & 0 & 19.8\% & 266.25 & 0.07 \\ +TorchMLP (D) & 1032${}_{-36,+23}$ & 0.022 & 31.5 & 27.3 & 0 & 22.8\% & 5.44 & 0.14 \\ +FastaiMLP (D) & 1003${}_{-32,+30}$ & 0.027 & 32.5 & 28.5 & 0 & 24.0\% & 3.04 & 0.30 \\ +RandomForest (D) & 1000${}_{-0,+0}$ & 0.003 & 32.6 & 26.4 & 0 & 25.9\% & 0.35 & 0.03 \\ +Linear (T+E) & 949${}_{-36,+31}$ & 0.044 & 34.6 & 17.8 & 1 & 30.9\% & 44.56 & 0.16 \\ +ExtraTrees (D) & 925${}_{-24,+31}$ & 0.011 & 35.4 & 31.2 & 0 & 28.7\% & 0.24 & 0.04 \\ +Linear (T) & 919${}_{-29,+22}$ & 0.028 & 35.7 & 27.3 & 0 & 31.6\% & 44.56 & 0.06 \\ +Linear (D) & 892${}_{-33,+30}$ & 0.020 & 36.5 & 34.2 & 0 & 32.6\% & 1.42 & 0.09 \\ +KNN (T+E) & 741${}_{-43,+34}$ & 0.000 & 40.3 & 34.8 & 0 & 49.8\% & 3.50 & 0.16 \\ +KNN (T) & 677${}_{-34,+33}$ & 0.000 & 41.4 & 30.9 & 0 & 51.9\% & 3.50 & 0.03 \\ +KNN (D) & 453${}_{-48,+60}$ & 0.000 & 43.9 & 43.4 & 0 & 60.3\% & 0.07 & 0.02 \\ +\bottomrule +\end{tabular} \ No newline at end of file diff --git a/data/lite/tabicl-imputed-cls/tabarena_leaderboard.csv b/data/lite/tabicl-imputed-cls/tabarena_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..e13116b1208d97ac64ca44348b29027617a2fe5f --- /dev/null +++ b/data/lite/tabicl-imputed-cls/tabarena_leaderboard.csv @@ -0,0 +1,46 @@ +method,time_train_s,time_infer_s,time_train_s_per_1K,time_infer_s_per_1K,normalized-error,normalized-error-task,champ_delta,loss_rescaled,time_train_s_rescaled,time_infer_s_rescaled,rank,median_metric_error,median_time_train_s,median_time_infer_s,median_time_train_s_per_1K,median_time_infer_s_per_1K,median_normalized-error,median_normalized-error-task,median_champ_delta,median_loss_rescaled,median_time_train_s_rescaled,median_time_infer_s_rescaled,median_rank,rank=1_count,rank=2_count,rank=3_count,rank>3_count,elo,elo+,elo-,winrate,mrr +AutoGluon 1.3 (4h),7748.990037812127,20.667584995428722,2786.367822030483,2.9686980518587505,0.4305752039617605,0.42476091339704225,0.07976050283602283,0.043752904437839156,31469.589611836727,278.09709780834015,8.51388888888889,0.157955,6468.6270185709,3.7256141901016235,1239.480006130907,1.9841131308583402,0.3593825434326649,0.4192074348867768,0.042609438846846426,0.018367490316605305,23460.49845320325,142.30518626073365,6.5,6,3,2,25,1520.2,32.2,33.9,0.829229797979798,0.30652282237249556 +TABM (tuned + ensemble),32822.329592857095,6.718376735846202,3197.494145595591,1.5968882956786812,0.5092283925039975,0.5402494336923277,0.11003749697557409,0.08316701214454297,47368.16918956869,126.38148088436719,12.25,0.179425,8087.822376012802,2.9963064193725586,2261.5736774780025,1.258481528676095,0.47136645123500964,0.5343939211655224,0.07139416682385408,0.03201326752328171,38436.80992013264,113.73588992793104,9.5,2,0,3,31,1421.5,30.4,26.5,0.7443181818181818,0.1781619925337582 +REALMLP (tuned + ensemble),78236.15068201887,12.508955962128109,5830.1148293797,4.304066546062194,0.5822217846671746,0.5699729885500302,0.11493960724568313,0.06750738371665936,142054.31198219568,287.5042196718009,12.277777777777779,0.172085,23325.48352110386,5.581948518753052,5429.819541400706,2.9376822787963643,0.5258237901453162,0.5449004654198817,0.06341551053522204,0.038644322872270584,93177.39784523446,229.06556046190957,12.0,0,1,2,33,1423.8,27.2,29.8,0.7436868686868687,0.13160831014897043 +GBM (tuned + ensemble),2716.464209013515,11.650436507331001,652.7081231589766,2.3316901429136103,0.6113071815088934,0.6439211879568594,0.12519012390225173,0.08548112327588184,8680.115930381902,215.8804964577198,13.13888888888889,0.17347,1527.4595284461975,3.0867438316345215,368.4672845277482,1.2233637093106904,0.6534872634143774,0.6903859786744668,0.07633428287025418,0.02975866915001093,6991.1315945904835,101.20971107477635,13.5,0,0,3,33,1403.7,26.8,31.1,0.7241161616161617,0.10807103527782952 +TABICL (default),122.91359008020825,22.535752693812054,9.963658601166212,2.414261101064252,0.4788051732855598,0.5704204973919814,0.10299986379617171,0.07074836814764993,179.76964423723584,285.61149302454663,13.569444444444445,0.17301,25.354220509529114,3.4422082901000977,9.073519792448254,2.1146179912287675,0.4992564187967412,0.645094881252605,0.054495038504871085,0.023780759612610794,144.31099154558206,133.6960355402875,13.5,4,2,2,28,1392.1,29.3,30.5,0.7143308080808081,0.21564624041790878 +CAT (tuned + ensemble),14495.012520273527,2.4179346097840204,2435.7734425933513,0.7431584046715475,0.6226459246322733,0.6029024005124854,0.11596376482505977,0.07030865757782191,26875.894999800756,51.46768059921607,13.916666666666666,0.16073500000000002,4554.886538267136,1.2364250421524048,1207.4876040106037,0.5178688292431316,0.6153632608570143,0.5995830509750808,0.057394530346010664,0.03646442390646657,20500.32079395515,42.031537823549,13.0,0,1,4,31,1387.4,28.3,33.5,0.7064393939393939,0.12442012127308923 +XGB (tuned + ensemble),5342.747825033135,6.078431023491754,903.1395181375539,2.4252994962785444,0.6804014135907539,0.6526428741754988,0.1304313253929671,0.09298138567686971,10629.483194714896,137.278973628425,14.208333333333334,0.17348,1566.1410616636276,2.2440043687820435,655.5367556867432,0.7811829897319023,0.7126419091535116,0.6563884597908853,0.09321543161193863,0.03538700202822707,7509.936420425425,78.99719729544988,12.5,0,2,2,32,1380.1,24.8,27.2,0.6998106060606061,0.11979663543347382 +MNCA (tuned),57242.54890214073,19.57283937268787,5418.777655567625,1.42346301375637,0.7251534847407544,0.6050555577766282,0.11758188821167392,0.09100501949740235,83536.32309061421,128.61468675559033,14.652777777777779,0.17875999999999997,13889.081128954887,0.5763018131256104,4498.662765534045,0.4537708228509102,0.7744612480983963,0.6064333426206642,0.08668289690877401,0.03213571989092236,68177.28399410362,31.783037323749284,12.5,2,0,0,34,1371.3,30.5,28.1,0.6897095959595959,0.14819282952741145 +MNCA (tuned + ensemble),57242.54890214073,352.26783115996255,5418.777655567625,26.036546991587294,0.6174214208973879,0.5767607960831209,0.11822427150348405,0.0935800489452592,83536.32309061421,2422.151854876793,15.11111111111111,0.18984499999999999,13889.081128954887,13.45927095413208,4498.662765534045,8.47783782257851,0.6265388929939845,0.5298466298371136,0.06347089264029215,0.0430523430616711,68177.28399410362,597.4215710672797,12.0,2,1,2,31,1362.6,17.9,34.0,0.6792929292929293,0.17072172056361182 +CAT (tuned),14495.012520273527,0.51534503698349,2435.7734425933513,0.10399437886071032,0.6397756322608608,0.6539747950956394,0.12271147625619921,0.07516879263695758,26875.894999800756,8.46801242481876,15.13888888888889,0.16238999999999998,4554.886538267136,0.0815126895904541,1207.4876040106037,0.0500282797740022,0.6663747291561838,0.7094216618957403,0.07123019460271723,0.03992016835270228,20500.32079395515,3.2560454821039713,13.75,0,1,0,35,1360.8,32.4,28.9,0.6786616161616161,0.10025246962871354 +CAT (default),224.62928151422076,0.4154767062928941,111.70337568642236,0.13827596961428462,0.6569717560090025,0.6686247539310815,0.12570904101689606,0.075779879204702,447.23099485428673,10.877554497735453,15.180555555555555,0.161695,17.568647384643555,0.40540146827697754,5.180417923757872,0.0935400303065973,0.6872314742824219,0.7131246390179387,0.08465206654494872,0.03255107567592877,99.87500924350084,7.319630010463586,15.0,2,0,0,34,1359.8,28.3,22.3,0.6777146464646465,0.12819611686011753 +TABM (tuned),32822.329592857095,0.8157637053065829,3197.494145595591,0.18466029782681834,0.5912555585094392,0.6057894002037636,0.1198534770661128,0.098187833170796,47368.16918956869,14.021720853154184,15.26388888888889,0.180525,8087.822376012802,0.23180127143859863,2261.5736774780025,0.12874845474998858,0.5877464635288101,0.6244854782324665,0.08334874690385935,0.04117730891431355,38436.80992013264,9.17800659262771,14.0,0,3,2,31,1358.5,27.3,27.4,0.6758207070707071,0.13440933222131132 +TABPFNV2 (tuned + ensemble),11043.815329816607,120.47059361802206,2782.016322081119,51.447489974805976,0.49442997284509693,0.5515000358199672,0.11994790652266277,0.09390415474321508,53537.875699289056,4570.010571503732,15.51388888888889,0.17508,3350.918938755989,13.188844680786133,2955.8707825485667,18.13473173852503,0.46131723763599985,0.5586983584129163,0.09642260474701764,0.034714937246957414,26502.64853496402,960.5639334598642,9.5,6,1,3,26,1354.8,24.1,34.9,0.6701388888888888,0.26705776606652504 +XGB (tuned),5342.747825033135,1.2801591290367975,903.1395181375539,0.639985957335127,0.7363747567107989,0.6947932415762801,0.1330389375064511,0.09865153659229181,10629.483194714896,33.68776538848152,15.805555555555555,0.176915,1566.1410616636276,0.3626103401184082,655.5367556867432,0.15681948639835536,0.7444722954009122,0.7353840158409348,0.10245426870569208,0.04170721330284183,7509.936420425425,13.242651903663184,14.5,1,1,0,34,1347.9,32.4,24.6,0.663510101010101,0.11658608793120388 +GBM (tuned),2716.464209013515,1.8839820755852594,652.7081231589766,0.5269472540571296,0.6982904100452502,0.7030339130629332,0.13469397321752158,0.10049233526761625,8680.115930381902,44.39170809497161,16.083333333333332,0.17537,1527.4595284461975,0.5379931926727295,368.4672845277482,0.11972777459115919,0.7332165833714938,0.7506557591253837,0.07955273182646189,0.03627576854264193,6991.1315945904835,14.435205317774814,16.5,0,1,0,35,1339.7,32.1,26.6,0.6571969696969697,0.09351899066227934 +TABPFNV2 (tuned),11043.815329816607,4.335048317909241,2782.016322081119,2.1142498550715927,0.6073121438794584,0.6675009910572761,0.1387648460631702,0.10921764834865659,53537.875699289056,151.67333078091804,18.25,0.19132,3350.918938755989,0.4039733409881592,2955.8707825485667,0.31535477718886346,0.67505591577672,0.8867319417132491,0.1215657105488207,0.05534847843426033,26502.64853496402,17.12188918934154,18.5,1,5,2,28,1297.1,25.4,27.4,0.6079545454545454,0.16932790386282506 +TABM (default),140.8987251520157,0.9393433663580153,14.983795328833262,0.18062205057314412,0.7280741356398281,0.7091275685597331,0.15170581117942186,0.11987810433907341,217.11963453230365,13.588062902891982,18.36111111111111,0.18336999999999998,29.720080256462097,0.17920684814453125,9.303649390343331,0.12625846443862987,0.8404844924976476,0.7655355529381327,0.09137336521284833,0.05094980825106373,136.86363214905396,9.928949591639647,17.0,1,0,1,34,1297.1,26.6,24.3,0.6054292929292929,0.10945710198655328 +EBM (tuned + ensemble),23286.664896190166,1.091496765613556,2120.3766035360445,0.27276807805390907,0.8303965578080257,0.7636192894218304,0.16995597325515008,0.130095847234682,22277.212479524587,20.94625622434944,19.125,0.181755,2063.8647414445877,0.34018075466156006,852.2924105567174,0.2147416876316487,0.9326307523783561,0.8842854370429472,0.07547386841907339,0.04276402555620501,14796.589958532411,12.4343949892923,19.75,0,3,0,33,1282.1,24.3,31.6,0.5880681818181818,0.10426190631325036 +NN_TORCH (tuned + ensemble),24905.959744539527,15.398705760637919,2952.9414864535283,2.849701545461187,0.7723767872546564,0.7889657455799273,0.1434257766741491,0.09523940893062431,57128.74445660633,251.675577261934,19.305555555555557,0.174655,8823.152364969254,4.023689031600952,2406.0981194606593,2.04339307286537,0.9031626632034823,0.917687560600783,0.0823266490663982,0.05626679601192999,44532.50472025028,186.31979424186875,20.5,0,0,1,35,1277.9,25.4,29.5,0.5839646464646465,0.07706674856025808 +REALMLP (tuned),78236.15068201887,0.5794834030999078,5830.1148293797,0.2343189481249011,0.8023310570642167,0.7674452443175672,0.15116645540948104,0.1058360844611002,142054.31198219568,14.164064071067262,19.305555555555557,0.17396499999999998,23325.48352110386,0.2515758275985718,5429.819541400706,0.1667243421007542,0.8787741747777617,0.8265671653596647,0.1283441158885112,0.05736216794816598,93177.39784523446,11.047988768711498,18.0,0,1,0,35,1278.9,28.8,28.0,0.5839646464646465,0.07828107192658276 +TABPFNV2 (default),11.068875008159214,0.956266270743476,3.892113637027881,0.46886596056667007,0.6398741999945717,0.716649191664692,0.1489698514364425,0.11896643945585421,55.3618995674274,37.50101434288101,20.569444444444443,0.194575,8.10049295425415,0.3354600667953491,3.1215870763701354,0.3243599381046226,0.7746032035996981,0.9946213191350618,0.09135918614016814,0.08171589783487912,39.33829805767891,20.266440186992128,22.5,3,2,2,29,1253.7,22.3,31.5,0.555239898989899,0.17069931592358933 +EBM (tuned),23286.664896190166,0.12520347701178658,2120.3766035360445,0.03546501889806546,0.8902973079447266,0.8183190158063525,0.1768940086859969,0.1366334760994699,22277.212479524587,2.4619900653558613,21.430555555555557,0.181165,2063.8647414445877,0.03867542743682861,852.2924105567174,0.019590150951601572,1.0,0.9995069033530574,0.08520888190451481,0.04977403056592767,14796.589958532411,1.2081964035400725,22.5,0,1,1,34,1239.3,29.2,29.5,0.5356691919191919,0.08062941909216864 +FASTAI (tuned + ensemble),7289.840611391597,19.15695581171248,1303.7963220052682,8.14447528633287,0.7926888983747619,0.8112522345255945,0.18037445757593507,0.10566026790546856,19518.83385555907,536.4983396151715,22.375,0.17774,2935.3402432203293,12.184540390968323,581.3698825429726,4.7809972969941885,0.9972327674497973,0.9991527416588244,0.10375041141751795,0.07514022518705465,12902.237935802625,480.50623521802163,22.5,0,0,0,36,1217.5,27.8,24.6,0.5142045454545454,0.07426287779268817 +EBM (default),88.55436639653311,0.15727221965789795,8.239269939196605,0.06798490198186677,0.8813849391122723,0.8406296522618981,0.18481255142167125,0.13982840973848604,96.02410546318727,4.157976350403235,23.208333333333332,0.185075,9.609492063522339,0.06619513034820557,4.39583132423955,0.04826106459931703,1.0,0.9961748430942867,0.10192934818149862,0.05017166802118532,56.77501308270701,3.212150379842276,23.25,1,0,0,35,1202.3,23.0,26.3,0.4952651515151515,0.08664134907565921 +XGB (default),11.691550334294638,0.582956936624315,2.5863360706414102,0.3100034135849125,0.8979377675030106,0.8516433363192434,0.16587880009201375,0.14701810809105467,31.813142204435906,16.390368754261612,23.569444444444443,0.175445,5.739119291305542,0.3213019371032715,1.66399954826743,0.12057611518072474,1.0,1.0,0.12284267447258906,0.062110541237017966,27.152708445757774,9.436867401020109,23.5,1,0,0,35,1196.0,23.5,28.8,0.48705808080808083,0.07866569541239092 +REALMLP (default),497.1762997441822,0.6067886617448595,35.85201952385433,0.2401830912617534,0.9051917890614664,0.834947945145405,0.1667961005773081,0.11376693833237551,889.4575438184943,14.614115185514347,23.65277777777778,0.17567,146.45613551139832,0.23558390140533447,34.61854805295221,0.16884822006186367,1.0,1.0,0.10725973083151863,0.073097488329855,602.5345759525801,11.596995428149068,25.5,1,0,0,35,1193.3,27.7,21.8,0.48516414141414144,0.0792198373413939 +MNCA (default),306.5998376806577,9.98978877067566,17.805999463280088,0.7747951629896741,0.8583270774687776,0.8565014107392712,0.18000709978418794,0.1136914706057266,252.51128046279433,73.98229627081949,23.666666666666668,0.190215,27.28103244304657,0.5536909103393555,12.987204209764974,0.33894040253174296,1.0,0.99770932414154,0.13240284359564197,0.07989597658322067,169.66480804983473,24.653041834920074,22.5,1,1,0,34,1193.0,25.5,34.6,0.48484848484848486,0.0850840962234233 +NN_TORCH (tuned),24905.959744539527,0.6660838723182678,2952.9414864535283,0.15459242992627908,0.9148909672592942,0.8560839131795723,0.16676662859321792,0.11613273190983585,57128.74445660633,11.722847631461935,23.833333333333332,0.17361500000000002,8823.152364969254,0.2007770538330078,2406.0981194606593,0.11202253911830268,1.0,1.0,0.10704543711737907,0.06675514797021173,44532.50472025028,8.64392602078908,25.0,0,0,0,36,1189.5,27.1,30.8,0.4810606060606061,0.05400487083496075 +XT (tuned + ensemble),1037.372560845481,3.071019298500485,360.7132393580917,1.4744924910300816,0.9176899118580594,0.8772100688076164,0.1843143961543318,0.14290373773078288,4707.816009404905,94.53215065372743,24.26388888888889,0.183685,752.3897469043732,2.041845917701721,187.8732763026905,0.7207715417972143,1.0,1.0,0.1315762721206637,0.07600010491905307,3200.195519746224,83.245630982298,26.75,0,0,0,36,1184.0,25.7,30.1,0.47127525252525254,0.05733624005903094 +RF (tuned + ensemble),2020.4760966300964,2.134269985887739,415.1583564929086,1.2910427322323617,0.932518287859348,0.9005533897562101,0.190161624036382,0.15955834513704611,5462.814816714475,83.37351619032778,25.666666666666668,0.180965,851.531911611557,1.8475559949874878,266.2539346689623,0.6395657388264155,1.0,1.0,0.1352038852066204,0.0822503414133858,4305.508903299853,67.08260239114765,28.5,0,2,0,34,1153.4,27.6,26.5,0.4393939393939394,0.073342105194866 +TABDPT (default),175.0330073899693,68.71396222379472,26.097389455873152,21.943944030413235,0.8137945017331583,0.8179614287037148,0.19222382286116146,0.1398951815244204,492.7334429125401,1620.9148184577618,26.23611111111111,0.198215,97.51883804798126,29.261866688728333,20.255521876560408,8.552781426594889,1.0,1.0,0.13114122736509654,0.07269651025672075,407.04692448648916,1198.388331768314,31.0,1,3,0,32,1144.2,28.1,34.3,0.4264520202020202,0.11055235729381453 +XT (tuned),1037.372560845481,0.3051542573504978,360.7132393580917,0.16689901565151677,0.9439228284777492,0.8840380175065681,0.19597755438216247,0.14929813671791142,4707.816009404905,10.02739657365076,26.25,0.18688500000000002,752.3897469043732,0.1615079641342163,187.8732763026905,0.07985268047821353,1.0,1.0,0.1505192879837034,0.08371989745070943,3200.195519746224,8.479842721007039,29.0,0,1,0,35,1143.3,26.7,35.0,0.42613636363636365,0.06398339537607635 +FASTAI (tuned),7289.840611391597,1.1152561836772494,1303.7963220052682,0.6947586912123508,0.9058648946317567,0.873357455480317,0.20144752798465024,0.13768196892163254,19518.83385555907,43.25082874084665,26.416666666666668,0.18023,2935.3402432203293,0.9242111444473267,581.3698825429726,0.334061010051534,1.0,1.0,0.1660331096790038,0.0823205592657475,12902.237935802625,29.54949110419716,28.5,0,0,1,35,1139.3,30.8,26.5,0.42234848484848486,0.059729533200154804 +GBM (default),7.365004950099522,0.7322817908393012,2.509446514786595,0.18984168667689244,0.9471769132881858,0.9201699708464274,0.1830567677779008,0.1541138325172775,35.17208033262076,16.8021003794707,26.84722222222222,0.18529,6.238969683647156,0.6098566055297852,1.6407068695252,0.14810769896236448,1.0,1.0,0.14940090168048453,0.07094283568464387,22.04913243774461,11.229856118149964,27.5,0,0,0,36,1131.2,21.0,31.4,0.4125631313131313,0.042567416943239275 +RF (tuned),2020.4760966300964,0.22272791465123495,415.1583564929086,0.1434182534422469,0.9551019371507523,0.9360306919709499,0.19787032871599097,0.16914360068649603,5462.814816714475,8.151832659415685,27.694444444444443,0.18201499999999998,851.531911611557,0.14735674858093262,266.2539346689623,0.0714840059306155,1.0,1.0,0.13792533210297697,0.09963978217702693,4305.508903299853,7.364430017401964,29.0,0,0,0,36,1112.2,33.5,30.6,0.39330808080808083,0.048009195540425535 +NN_TORCH (default),47.65417258607017,0.6302466657426622,10.641224850569223,0.23501087478485314,0.978257373569646,0.954102548699942,0.2275331100570714,0.169651058381061,143.43581528304946,15.239602876479248,31.458333333333332,0.18706,22.84017276763916,0.28995954990386963,5.436935257156724,0.1377204507988835,1.0,1.0,0.1627665585076386,0.09575304821587584,137.2254146054324,9.730207537012728,33.0,0,0,0,36,1031.6,22.4,35.3,0.3077651515151515,0.03662907463362058 +FASTAI (default),30.521977769003975,1.0843763152758281,4.742334418927322,0.49201194647444324,0.972512597016023,0.9430264129001292,0.23958288237304964,0.18088306715708347,76.51706745050629,33.35318180861937,32.5,0.187165,13.653389573097229,0.6859067678451538,3.0439688517356123,0.3001709716511216,1.0,1.0,0.16007809952594787,0.12381628056074112,56.69169806857949,28.502895225657927,35.0,0,0,0,36,1003.4,29.2,31.6,0.2840909090909091,0.03514706034791027 +RF (default),3.2084562049971685,0.13506737020280626,0.4263514561749847,0.06558284712037367,0.9965256868256966,0.9756374933977958,0.25930576589425613,0.2428212871410249,7.059782904150193,4.425232204599554,32.611111111111114,0.20493,1.2789119482040405,0.09202027320861816,0.34809529918260984,0.03472589740080406,1.0,1.0,0.22217921675183,0.11658001888686689,5.376229465336108,3.6309495335392015,35.75,0,0,0,36,1000.0,0.0,0.0,0.2815656565656566,0.037864938459085944 +LR (tuned + ensemble),246.97270359595618,1.3169628845320807,86.24001785898447,0.299396801053486,0.9561054304859822,0.9692239441416002,0.30863610039561684,0.2486725924427303,1104.1489965774379,26.605204102831735,34.56944444444444,0.20664,169.60151755809784,0.2824718952178955,44.56368864780743,0.15651788282525403,1.0,1.0,0.2502384210107675,0.1550098957472666,675.130801866569,8.851961834833686,37.0,1,0,0,35,949.2,30.1,35.1,0.2370580808080808,0.05633787203620641 +XT (default),1.8672458993064032,0.17208321889241537,0.35727892050479015,0.06872435387895405,0.9892721532660226,0.9800527523199994,0.2874747079845823,0.2752176157333544,4.999840482001418,4.841906560022478,35.416666666666664,0.20700000000000002,0.8974905014038086,0.08268320560455322,0.23619265983619955,0.036597834553610306,1.0,1.0,0.21784002712442435,0.1482624508295911,3.672313260209499,4.261733262225922,38.0,0,0,0,36,925.3,30.7,23.6,0.2178030303030303,0.032013179657027775 +LR (tuned),246.97270359595618,0.43901119629542035,86.24001785898447,0.1002982819551287,0.9723830890370608,0.9787756407161015,0.3159674455247929,0.2604157048141723,1104.1489965774379,7.096822360818471,35.65277777777778,0.206755,169.60151755809784,0.12632572650909424,44.56368864780743,0.062293080471580324,1.0,1.0,0.25818261683105737,0.1688557029748925,675.130801866569,5.062717123136234,38.25,0,0,1,35,919.4,21.5,28.9,0.21243686868686867,0.036643329912295686 +LR (default),6.324005815717909,0.45038049750857884,2.0933349209116243,0.12679966936326295,0.9795929927195421,0.9864551890342567,0.3255669034156274,0.28877566195163695,28.81662586680685,8.679077173966622,36.52777777777778,0.21439,5.391173958778381,0.12632572650909424,1.4157886113918772,0.09164922519151064,1.0,1.0,0.2581813809635515,0.16885460955121118,16.81966393701253,5.719143257720329,39.0,0,0,0,36,891.6,29.9,32.3,0.19255050505050506,0.029261494694519958 +KNN (tuned + ensemble),168.93849890761905,8.359918620851305,8.837249824100754,0.4805954727942703,1.0,0.9867364080557345,0.4982079360481384,0.6064348609957761,80.93770906535752,52.719394620298914,40.27777777777778,0.321975,13.986609816551208,0.22182130813598633,3.4970040049929065,0.1592449370444793,1.0,1.0,0.44534583678290146,0.6925990746528932,56.97784214902664,12.39378247919743,43.0,0,0,0,36,741.1,33.5,42.1,0.10732323232323232,0.028756216828868864 +KNN (tuned),168.93849890761905,1.9504973888397217,8.837249824100754,0.10793579395855045,1.0,0.9837443971669894,0.5193096900773991,0.6567505669495036,80.93770906535752,11.495707208766925,41.43055555555556,0.32563,13.986609816551208,0.06376731395721436,3.4970040049929065,0.03322185201916636,1.0,1.0,0.5143946348760557,0.7341673886277578,56.97784214902664,2.1231636318644824,44.0,0,0,1,35,676.8,32.2,33.4,0.08112373737373738,0.0323845750911192 +KNN (default),0.7941402528021071,0.21308696932262844,0.1178118784481507,0.0311341997736934,1.0,1.0,0.6031940607224022,0.9574097789264635,1.001944530363665,2.4108020816009272,43.90277777777778,0.38312,0.23321354389190674,0.030547380447387695,0.06547009031681006,0.018147653154379814,1.0,1.0,0.6027815264295836,1.0,1.0,1.409117664778126,45.0,0,0,0,36,452.7,59.1,47.9,0.024936868686868688,0.02303628559779201 diff --git a/data/lite/tabicl-imputed-cls/time_plot.pdf b/data/lite/tabicl-imputed-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5134ea984c85f0f586dccfb56b5b8d14696fae39 Binary files /dev/null and b/data/lite/tabicl-imputed-cls/time_plot.pdf differ diff --git a/data/lite/tabicl-imputed-cls/time_plot.png.zip b/data/lite/tabicl-imputed-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..63d868682c40991e6428fec23b3a62a1305b2565 --- /dev/null +++ b/data/lite/tabicl-imputed-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:928ccdc9706cb14bf13236ffe4c8733c85269574a2e92ffd7dec3a0a9a59466b +size 338023 diff --git a/data/lite/tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf b/data/lite/tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fac1d44b5442ee4fba6a4c14036cecec50329444 Binary files /dev/null and b/data/lite/tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip b/data/lite/tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..448f6f7fed11bae2a1ad025a3ca7e5eb75951086 --- /dev/null +++ b/data/lite/tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d3df4ffd2e3efbaeb6b3c0054cbe657c153b4aa39496c102211a5cc2df6f47d +size 152245 diff --git a/data/lite/tabicl-imputed-cls/tuning-impact-elo.pdf b/data/lite/tabicl-imputed-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e3e50542136b1a8ce74885e0bf80deffd86b1f38 Binary files /dev/null and b/data/lite/tabicl-imputed-cls/tuning-impact-elo.pdf differ diff --git a/data/lite/tabicl-imputed-cls/tuning-impact-elo.png.zip b/data/lite/tabicl-imputed-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c0a6cc3d9f82b9bd651c267cd1b21e2d203e70af --- /dev/null +++ b/data/lite/tabicl-imputed-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f823b5852e6e8d4fca265c4023fe9512ed2b4d78c9a4a4f2d167b59ddbe8d00 +size 154208 diff --git a/data/lite/tabicl-imputed/figures/critical-diagram.pdf b/data/lite/tabicl-imputed/figures/critical-diagram.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b2b4e5d3d507fa4b0200712987a664dd9ee8398d Binary files /dev/null and b/data/lite/tabicl-imputed/figures/critical-diagram.pdf differ diff --git a/data/lite/tabicl-imputed/figures/critical-diagram.png.zip b/data/lite/tabicl-imputed/figures/critical-diagram.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..73733b22278a23f0be4c5021d6fc85c3b9f7d489 --- /dev/null +++ b/data/lite/tabicl-imputed/figures/critical-diagram.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d100ede054db613210b8f9298878dc14e754ab4c6c0546efa551436138c5bea +size 319696 diff --git a/data/lite/tabicl-imputed/leaderboard.tex b/data/lite/tabicl-imputed/leaderboard.tex new file mode 100644 index 0000000000000000000000000000000000000000..dbce3cd5240d4d60bdbbe051e4a2a62dce853121 --- /dev/null +++ b/data/lite/tabicl-imputed/leaderboard.tex @@ -0,0 +1,53 @@ +\begin{tabular}{llcccccrr} +\toprule +\textbf{Model} & \textbf{Elo ($\uparrow$)} & \textbf{Norm.} & \textbf{Avg.} & \textbf{Harm.} & \textbf{\#wins ($\uparrow$)} & \textbf{Improva-} & \textbf{Train time} & \textbf{Predict time} \\ + & & \textbf{score ($\uparrow$)} & \textbf{rank ($\downarrow$)} & \textbf{mean} & & \textbf{bility ($\downarrow$)} & \textbf{per 1K [s]} & \textbf{per 1K [s]} \\ + & & & & \textbf{rank ($\downarrow$)} & & & & \\ +\midrule +AutoGluon 1.3 (4h) & \textcolor{gold}{\textbf{1520${}_{-34,+33}$}} & \textcolor{gold}{\textbf{0.569}} & \textcolor{gold}{\textbf{8.5}} & \textcolor{gold}{\textbf{3.3}} & \textcolor{gold}{\textbf{6}} & \textcolor{gold}{\textbf{8.0\%}} & 1239.48 & 1.98 \\ +RealMLP (T+E) & \textcolor{silver}{\textbf{1424${}_{-30,+28}$}} & 0.418 & \textcolor{bronze}{\textbf{12.3}} & 7.6 & 0 & 11.5\% & 5429.82 & 2.94 \\ +TabM (T+E) & \textcolor{bronze}{\textbf{1422${}_{-27,+31}$}} & 0.491 & \textcolor{silver}{\textbf{12.2}} & 5.6 & 2 & \textcolor{bronze}{\textbf{11.0\%}} & 2261.57 & 1.26 \\ +LightGBM (T+E) & 1404${}_{-32,+27}$ & 0.389 & 13.1 & 9.3 & 0 & 12.5\% & 368.47 & 1.22 \\ +TabICL (D) & 1392${}_{-31,+30}$ & \textcolor{silver}{\textbf{0.521}} & 13.6 & \textcolor{bronze}{\textbf{4.6}} & \textcolor{bronze}{\textbf{4}} & \textcolor{silver}{\textbf{10.3\%}} & 9.07 & 2.11 \\ +CatBoost (T+E) & 1387${}_{-34,+29}$ & 0.377 & 13.9 & 8.0 & 0 & 11.6\% & 1207.49 & 0.52 \\ +XGBoost (T+E) & 1380${}_{-28,+25}$ & 0.320 & 14.2 & 8.3 & 0 & 13.0\% & 655.54 & 0.78 \\ +ModernNCA (T) & 1371${}_{-29,+31}$ & 0.275 & 14.7 & 6.7 & 2 & 11.8\% & 4498.66 & 0.45 \\ +ModernNCA (T+E) & 1363${}_{-34,+18}$ & 0.383 & 15.1 & 5.9 & 2 & 11.8\% & 4498.66 & 8.48 \\ +CatBoost (T) & 1361${}_{-29,+33}$ & 0.360 & 15.1 & 10.0 & 0 & 12.3\% & 1207.49 & 0.05 \\ +CatBoost (D) & 1360${}_{-23,+29}$ & 0.343 & 15.2 & 7.8 & 2 & 12.6\% & 5.18 & 0.09 \\ +TabM (T) & 1358${}_{-28,+28}$ & 0.409 & 15.3 & 7.4 & 0 & 12.0\% & 2261.57 & 0.13 \\ +TabPFNv2 (T+E) & 1355${}_{-35,+25}$ & \textcolor{bronze}{\textbf{0.506}} & 15.5 & \textcolor{silver}{\textbf{3.7}} & \textcolor{gold}{\textbf{6}} & 12.0\% & 2955.87 & 18.13 \\ +XGBoost (T) & 1348${}_{-25,+33}$ & 0.264 & 15.8 & 8.6 & 1 & 13.3\% & 655.54 & 0.16 \\ +LightGBM (T) & 1340${}_{-27,+33}$ & 0.302 & 16.1 & 10.7 & 0 & 13.5\% & 368.47 & 0.12 \\ +TabPFNv2 (T) & 1297${}_{-28,+26}$ & 0.393 & 18.2 & 5.9 & 1 & 13.9\% & 2955.87 & 0.32 \\ +TabM (D) & 1297${}_{-25,+27}$ & 0.272 & 18.4 & 9.1 & 1 & 15.2\% & 9.30 & 0.13 \\ +EBM (T+E) & 1282${}_{-32,+25}$ & 0.170 & 19.1 & 9.6 & 0 & 17.0\% & 852.29 & 0.21 \\ +RealMLP (T) & 1279${}_{-28,+29}$ & 0.198 & 19.3 & 12.8 & 0 & 15.1\% & 5429.82 & 0.17 \\ +TorchMLP (T+E) & 1278${}_{-30,+26}$ & 0.228 & 19.3 & 13.0 & 0 & 14.3\% & 2406.10 & 2.04 \\ +TabPFNv2 (D) & 1254${}_{-32,+23}$ & 0.360 & 20.6 & 5.9 & 3 & 14.9\% & 3.12 & 0.32 \\ +EBM (T) & 1239${}_{-30,+30}$ & 0.110 & 21.4 & 12.4 & 0 & 17.7\% & 852.29 & 0.02 \\ +FastaiMLP (T+E) & 1218${}_{-25,+28}$ & 0.207 & 22.4 & 13.5 & 0 & 18.0\% & 581.37 & 4.78 \\ +EBM (D) & 1202${}_{-27,+23}$ & 0.119 & 23.2 & 11.5 & 1 & 18.5\% & 4.40 & 0.05 \\ +XGBoost (D) & 1196${}_{-29,+24}$ & 0.102 & 23.6 & 12.7 & 1 & 16.6\% & 1.66 & 0.12 \\ +RealMLP (D) & 1193${}_{-22,+28}$ & 0.095 & 23.7 & 12.6 & 1 & 16.7\% & 34.62 & 0.17 \\ +ModernNCA (D) & 1193${}_{-35,+26}$ & 0.142 & 23.7 & 11.8 & 1 & 18.0\% & 12.99 & 0.34 \\ +TorchMLP (T) & 1190${}_{-31,+28}$ & 0.085 & 23.8 & 18.5 & 0 & 16.7\% & 2406.10 & 0.11 \\ +ExtraTrees (T+E) & 1184${}_{-31,+26}$ & 0.082 & 24.3 & 17.4 & 0 & 18.4\% & 187.87 & 0.72 \\ +RandomForest (T+E) & 1153${}_{-27,+28}$ & 0.067 & 25.7 & 13.6 & 0 & 19.0\% & 266.25 & 0.64 \\ +TabDPT (D) & 1144${}_{-35,+29}$ & 0.186 & 26.2 & 9.0 & 1 & 19.2\% & 20.26 & 8.55 \\ +ExtraTrees (T) & 1143${}_{-35,+27}$ & 0.056 & 26.2 & 15.6 & 0 & 19.6\% & 187.87 & 0.08 \\ +FastaiMLP (T) & 1139${}_{-27,+31}$ & 0.094 & 26.4 & 16.7 & 0 & 20.1\% & 581.37 & 0.33 \\ +LightGBM (D) & 1131${}_{-32,+21}$ & 0.053 & 26.8 & 23.5 & 0 & 18.3\% & 1.64 & 0.15 \\ +RandomForest (T) & 1112${}_{-31,+34}$ & 0.045 & 27.7 & 20.8 & 0 & 19.8\% & 266.25 & 0.07 \\ +TorchMLP (D) & 1032${}_{-36,+23}$ & 0.022 & 31.5 & 27.3 & 0 & 22.8\% & 5.44 & 0.14 \\ +FastaiMLP (D) & 1003${}_{-32,+30}$ & 0.027 & 32.5 & 28.5 & 0 & 24.0\% & 3.04 & 0.30 \\ +RandomForest (D) & 1000${}_{-0,+0}$ & 0.003 & 32.6 & 26.4 & 0 & 25.9\% & 0.35 & 0.03 \\ +Linear (T+E) & 949${}_{-36,+31}$ & 0.044 & 34.6 & 17.8 & 1 & 30.9\% & 44.56 & 0.16 \\ +ExtraTrees (D) & 925${}_{-24,+31}$ & 0.011 & 35.4 & 31.2 & 0 & 28.7\% & 0.24 & 0.04 \\ +Linear (T) & 919${}_{-29,+22}$ & 0.028 & 35.7 & 27.3 & 0 & 31.6\% & 44.56 & 0.06 \\ +Linear (D) & 892${}_{-33,+30}$ & 0.020 & 36.5 & 34.2 & 0 & 32.6\% & 1.42 & 0.09 \\ +KNN (T+E) & 741${}_{-43,+34}$ & 0.000 & 40.3 & 34.8 & 0 & 49.8\% & 3.50 & 0.16 \\ +KNN (T) & 677${}_{-34,+33}$ & 0.000 & 41.4 & 30.9 & 0 & 51.9\% & 3.50 & 0.03 \\ +KNN (D) & 453${}_{-48,+60}$ & 0.000 & 43.9 & 43.4 & 0 & 60.3\% & 0.07 & 0.02 \\ +\bottomrule +\end{tabular} \ No newline at end of file diff --git a/data/lite/tabicl-imputed/tabarena_leaderboard.csv b/data/lite/tabicl-imputed/tabarena_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..e13116b1208d97ac64ca44348b29027617a2fe5f --- /dev/null +++ b/data/lite/tabicl-imputed/tabarena_leaderboard.csv @@ -0,0 +1,46 @@ +method,time_train_s,time_infer_s,time_train_s_per_1K,time_infer_s_per_1K,normalized-error,normalized-error-task,champ_delta,loss_rescaled,time_train_s_rescaled,time_infer_s_rescaled,rank,median_metric_error,median_time_train_s,median_time_infer_s,median_time_train_s_per_1K,median_time_infer_s_per_1K,median_normalized-error,median_normalized-error-task,median_champ_delta,median_loss_rescaled,median_time_train_s_rescaled,median_time_infer_s_rescaled,median_rank,rank=1_count,rank=2_count,rank=3_count,rank>3_count,elo,elo+,elo-,winrate,mrr +AutoGluon 1.3 (4h),7748.990037812127,20.667584995428722,2786.367822030483,2.9686980518587505,0.4305752039617605,0.42476091339704225,0.07976050283602283,0.043752904437839156,31469.589611836727,278.09709780834015,8.51388888888889,0.157955,6468.6270185709,3.7256141901016235,1239.480006130907,1.9841131308583402,0.3593825434326649,0.4192074348867768,0.042609438846846426,0.018367490316605305,23460.49845320325,142.30518626073365,6.5,6,3,2,25,1520.2,32.2,33.9,0.829229797979798,0.30652282237249556 +TABM (tuned + ensemble),32822.329592857095,6.718376735846202,3197.494145595591,1.5968882956786812,0.5092283925039975,0.5402494336923277,0.11003749697557409,0.08316701214454297,47368.16918956869,126.38148088436719,12.25,0.179425,8087.822376012802,2.9963064193725586,2261.5736774780025,1.258481528676095,0.47136645123500964,0.5343939211655224,0.07139416682385408,0.03201326752328171,38436.80992013264,113.73588992793104,9.5,2,0,3,31,1421.5,30.4,26.5,0.7443181818181818,0.1781619925337582 +REALMLP (tuned + ensemble),78236.15068201887,12.508955962128109,5830.1148293797,4.304066546062194,0.5822217846671746,0.5699729885500302,0.11493960724568313,0.06750738371665936,142054.31198219568,287.5042196718009,12.277777777777779,0.172085,23325.48352110386,5.581948518753052,5429.819541400706,2.9376822787963643,0.5258237901453162,0.5449004654198817,0.06341551053522204,0.038644322872270584,93177.39784523446,229.06556046190957,12.0,0,1,2,33,1423.8,27.2,29.8,0.7436868686868687,0.13160831014897043 +GBM (tuned + ensemble),2716.464209013515,11.650436507331001,652.7081231589766,2.3316901429136103,0.6113071815088934,0.6439211879568594,0.12519012390225173,0.08548112327588184,8680.115930381902,215.8804964577198,13.13888888888889,0.17347,1527.4595284461975,3.0867438316345215,368.4672845277482,1.2233637093106904,0.6534872634143774,0.6903859786744668,0.07633428287025418,0.02975866915001093,6991.1315945904835,101.20971107477635,13.5,0,0,3,33,1403.7,26.8,31.1,0.7241161616161617,0.10807103527782952 +TABICL (default),122.91359008020825,22.535752693812054,9.963658601166212,2.414261101064252,0.4788051732855598,0.5704204973919814,0.10299986379617171,0.07074836814764993,179.76964423723584,285.61149302454663,13.569444444444445,0.17301,25.354220509529114,3.4422082901000977,9.073519792448254,2.1146179912287675,0.4992564187967412,0.645094881252605,0.054495038504871085,0.023780759612610794,144.31099154558206,133.6960355402875,13.5,4,2,2,28,1392.1,29.3,30.5,0.7143308080808081,0.21564624041790878 +CAT (tuned + ensemble),14495.012520273527,2.4179346097840204,2435.7734425933513,0.7431584046715475,0.6226459246322733,0.6029024005124854,0.11596376482505977,0.07030865757782191,26875.894999800756,51.46768059921607,13.916666666666666,0.16073500000000002,4554.886538267136,1.2364250421524048,1207.4876040106037,0.5178688292431316,0.6153632608570143,0.5995830509750808,0.057394530346010664,0.03646442390646657,20500.32079395515,42.031537823549,13.0,0,1,4,31,1387.4,28.3,33.5,0.7064393939393939,0.12442012127308923 +XGB (tuned + ensemble),5342.747825033135,6.078431023491754,903.1395181375539,2.4252994962785444,0.6804014135907539,0.6526428741754988,0.1304313253929671,0.09298138567686971,10629.483194714896,137.278973628425,14.208333333333334,0.17348,1566.1410616636276,2.2440043687820435,655.5367556867432,0.7811829897319023,0.7126419091535116,0.6563884597908853,0.09321543161193863,0.03538700202822707,7509.936420425425,78.99719729544988,12.5,0,2,2,32,1380.1,24.8,27.2,0.6998106060606061,0.11979663543347382 +MNCA (tuned),57242.54890214073,19.57283937268787,5418.777655567625,1.42346301375637,0.7251534847407544,0.6050555577766282,0.11758188821167392,0.09100501949740235,83536.32309061421,128.61468675559033,14.652777777777779,0.17875999999999997,13889.081128954887,0.5763018131256104,4498.662765534045,0.4537708228509102,0.7744612480983963,0.6064333426206642,0.08668289690877401,0.03213571989092236,68177.28399410362,31.783037323749284,12.5,2,0,0,34,1371.3,30.5,28.1,0.6897095959595959,0.14819282952741145 +MNCA (tuned + ensemble),57242.54890214073,352.26783115996255,5418.777655567625,26.036546991587294,0.6174214208973879,0.5767607960831209,0.11822427150348405,0.0935800489452592,83536.32309061421,2422.151854876793,15.11111111111111,0.18984499999999999,13889.081128954887,13.45927095413208,4498.662765534045,8.47783782257851,0.6265388929939845,0.5298466298371136,0.06347089264029215,0.0430523430616711,68177.28399410362,597.4215710672797,12.0,2,1,2,31,1362.6,17.9,34.0,0.6792929292929293,0.17072172056361182 +CAT (tuned),14495.012520273527,0.51534503698349,2435.7734425933513,0.10399437886071032,0.6397756322608608,0.6539747950956394,0.12271147625619921,0.07516879263695758,26875.894999800756,8.46801242481876,15.13888888888889,0.16238999999999998,4554.886538267136,0.0815126895904541,1207.4876040106037,0.0500282797740022,0.6663747291561838,0.7094216618957403,0.07123019460271723,0.03992016835270228,20500.32079395515,3.2560454821039713,13.75,0,1,0,35,1360.8,32.4,28.9,0.6786616161616161,0.10025246962871354 +CAT (default),224.62928151422076,0.4154767062928941,111.70337568642236,0.13827596961428462,0.6569717560090025,0.6686247539310815,0.12570904101689606,0.075779879204702,447.23099485428673,10.877554497735453,15.180555555555555,0.161695,17.568647384643555,0.40540146827697754,5.180417923757872,0.0935400303065973,0.6872314742824219,0.7131246390179387,0.08465206654494872,0.03255107567592877,99.87500924350084,7.319630010463586,15.0,2,0,0,34,1359.8,28.3,22.3,0.6777146464646465,0.12819611686011753 +TABM (tuned),32822.329592857095,0.8157637053065829,3197.494145595591,0.18466029782681834,0.5912555585094392,0.6057894002037636,0.1198534770661128,0.098187833170796,47368.16918956869,14.021720853154184,15.26388888888889,0.180525,8087.822376012802,0.23180127143859863,2261.5736774780025,0.12874845474998858,0.5877464635288101,0.6244854782324665,0.08334874690385935,0.04117730891431355,38436.80992013264,9.17800659262771,14.0,0,3,2,31,1358.5,27.3,27.4,0.6758207070707071,0.13440933222131132 +TABPFNV2 (tuned + ensemble),11043.815329816607,120.47059361802206,2782.016322081119,51.447489974805976,0.49442997284509693,0.5515000358199672,0.11994790652266277,0.09390415474321508,53537.875699289056,4570.010571503732,15.51388888888889,0.17508,3350.918938755989,13.188844680786133,2955.8707825485667,18.13473173852503,0.46131723763599985,0.5586983584129163,0.09642260474701764,0.034714937246957414,26502.64853496402,960.5639334598642,9.5,6,1,3,26,1354.8,24.1,34.9,0.6701388888888888,0.26705776606652504 +XGB (tuned),5342.747825033135,1.2801591290367975,903.1395181375539,0.639985957335127,0.7363747567107989,0.6947932415762801,0.1330389375064511,0.09865153659229181,10629.483194714896,33.68776538848152,15.805555555555555,0.176915,1566.1410616636276,0.3626103401184082,655.5367556867432,0.15681948639835536,0.7444722954009122,0.7353840158409348,0.10245426870569208,0.04170721330284183,7509.936420425425,13.242651903663184,14.5,1,1,0,34,1347.9,32.4,24.6,0.663510101010101,0.11658608793120388 +GBM (tuned),2716.464209013515,1.8839820755852594,652.7081231589766,0.5269472540571296,0.6982904100452502,0.7030339130629332,0.13469397321752158,0.10049233526761625,8680.115930381902,44.39170809497161,16.083333333333332,0.17537,1527.4595284461975,0.5379931926727295,368.4672845277482,0.11972777459115919,0.7332165833714938,0.7506557591253837,0.07955273182646189,0.03627576854264193,6991.1315945904835,14.435205317774814,16.5,0,1,0,35,1339.7,32.1,26.6,0.6571969696969697,0.09351899066227934 +TABPFNV2 (tuned),11043.815329816607,4.335048317909241,2782.016322081119,2.1142498550715927,0.6073121438794584,0.6675009910572761,0.1387648460631702,0.10921764834865659,53537.875699289056,151.67333078091804,18.25,0.19132,3350.918938755989,0.4039733409881592,2955.8707825485667,0.31535477718886346,0.67505591577672,0.8867319417132491,0.1215657105488207,0.05534847843426033,26502.64853496402,17.12188918934154,18.5,1,5,2,28,1297.1,25.4,27.4,0.6079545454545454,0.16932790386282506 +TABM (default),140.8987251520157,0.9393433663580153,14.983795328833262,0.18062205057314412,0.7280741356398281,0.7091275685597331,0.15170581117942186,0.11987810433907341,217.11963453230365,13.588062902891982,18.36111111111111,0.18336999999999998,29.720080256462097,0.17920684814453125,9.303649390343331,0.12625846443862987,0.8404844924976476,0.7655355529381327,0.09137336521284833,0.05094980825106373,136.86363214905396,9.928949591639647,17.0,1,0,1,34,1297.1,26.6,24.3,0.6054292929292929,0.10945710198655328 +EBM (tuned + ensemble),23286.664896190166,1.091496765613556,2120.3766035360445,0.27276807805390907,0.8303965578080257,0.7636192894218304,0.16995597325515008,0.130095847234682,22277.212479524587,20.94625622434944,19.125,0.181755,2063.8647414445877,0.34018075466156006,852.2924105567174,0.2147416876316487,0.9326307523783561,0.8842854370429472,0.07547386841907339,0.04276402555620501,14796.589958532411,12.4343949892923,19.75,0,3,0,33,1282.1,24.3,31.6,0.5880681818181818,0.10426190631325036 +NN_TORCH (tuned + ensemble),24905.959744539527,15.398705760637919,2952.9414864535283,2.849701545461187,0.7723767872546564,0.7889657455799273,0.1434257766741491,0.09523940893062431,57128.74445660633,251.675577261934,19.305555555555557,0.174655,8823.152364969254,4.023689031600952,2406.0981194606593,2.04339307286537,0.9031626632034823,0.917687560600783,0.0823266490663982,0.05626679601192999,44532.50472025028,186.31979424186875,20.5,0,0,1,35,1277.9,25.4,29.5,0.5839646464646465,0.07706674856025808 +REALMLP (tuned),78236.15068201887,0.5794834030999078,5830.1148293797,0.2343189481249011,0.8023310570642167,0.7674452443175672,0.15116645540948104,0.1058360844611002,142054.31198219568,14.164064071067262,19.305555555555557,0.17396499999999998,23325.48352110386,0.2515758275985718,5429.819541400706,0.1667243421007542,0.8787741747777617,0.8265671653596647,0.1283441158885112,0.05736216794816598,93177.39784523446,11.047988768711498,18.0,0,1,0,35,1278.9,28.8,28.0,0.5839646464646465,0.07828107192658276 +TABPFNV2 (default),11.068875008159214,0.956266270743476,3.892113637027881,0.46886596056667007,0.6398741999945717,0.716649191664692,0.1489698514364425,0.11896643945585421,55.3618995674274,37.50101434288101,20.569444444444443,0.194575,8.10049295425415,0.3354600667953491,3.1215870763701354,0.3243599381046226,0.7746032035996981,0.9946213191350618,0.09135918614016814,0.08171589783487912,39.33829805767891,20.266440186992128,22.5,3,2,2,29,1253.7,22.3,31.5,0.555239898989899,0.17069931592358933 +EBM (tuned),23286.664896190166,0.12520347701178658,2120.3766035360445,0.03546501889806546,0.8902973079447266,0.8183190158063525,0.1768940086859969,0.1366334760994699,22277.212479524587,2.4619900653558613,21.430555555555557,0.181165,2063.8647414445877,0.03867542743682861,852.2924105567174,0.019590150951601572,1.0,0.9995069033530574,0.08520888190451481,0.04977403056592767,14796.589958532411,1.2081964035400725,22.5,0,1,1,34,1239.3,29.2,29.5,0.5356691919191919,0.08062941909216864 +FASTAI (tuned + ensemble),7289.840611391597,19.15695581171248,1303.7963220052682,8.14447528633287,0.7926888983747619,0.8112522345255945,0.18037445757593507,0.10566026790546856,19518.83385555907,536.4983396151715,22.375,0.17774,2935.3402432203293,12.184540390968323,581.3698825429726,4.7809972969941885,0.9972327674497973,0.9991527416588244,0.10375041141751795,0.07514022518705465,12902.237935802625,480.50623521802163,22.5,0,0,0,36,1217.5,27.8,24.6,0.5142045454545454,0.07426287779268817 +EBM (default),88.55436639653311,0.15727221965789795,8.239269939196605,0.06798490198186677,0.8813849391122723,0.8406296522618981,0.18481255142167125,0.13982840973848604,96.02410546318727,4.157976350403235,23.208333333333332,0.185075,9.609492063522339,0.06619513034820557,4.39583132423955,0.04826106459931703,1.0,0.9961748430942867,0.10192934818149862,0.05017166802118532,56.77501308270701,3.212150379842276,23.25,1,0,0,35,1202.3,23.0,26.3,0.4952651515151515,0.08664134907565921 +XGB (default),11.691550334294638,0.582956936624315,2.5863360706414102,0.3100034135849125,0.8979377675030106,0.8516433363192434,0.16587880009201375,0.14701810809105467,31.813142204435906,16.390368754261612,23.569444444444443,0.175445,5.739119291305542,0.3213019371032715,1.66399954826743,0.12057611518072474,1.0,1.0,0.12284267447258906,0.062110541237017966,27.152708445757774,9.436867401020109,23.5,1,0,0,35,1196.0,23.5,28.8,0.48705808080808083,0.07866569541239092 +REALMLP (default),497.1762997441822,0.6067886617448595,35.85201952385433,0.2401830912617534,0.9051917890614664,0.834947945145405,0.1667961005773081,0.11376693833237551,889.4575438184943,14.614115185514347,23.65277777777778,0.17567,146.45613551139832,0.23558390140533447,34.61854805295221,0.16884822006186367,1.0,1.0,0.10725973083151863,0.073097488329855,602.5345759525801,11.596995428149068,25.5,1,0,0,35,1193.3,27.7,21.8,0.48516414141414144,0.0792198373413939 +MNCA (default),306.5998376806577,9.98978877067566,17.805999463280088,0.7747951629896741,0.8583270774687776,0.8565014107392712,0.18000709978418794,0.1136914706057266,252.51128046279433,73.98229627081949,23.666666666666668,0.190215,27.28103244304657,0.5536909103393555,12.987204209764974,0.33894040253174296,1.0,0.99770932414154,0.13240284359564197,0.07989597658322067,169.66480804983473,24.653041834920074,22.5,1,1,0,34,1193.0,25.5,34.6,0.48484848484848486,0.0850840962234233 +NN_TORCH (tuned),24905.959744539527,0.6660838723182678,2952.9414864535283,0.15459242992627908,0.9148909672592942,0.8560839131795723,0.16676662859321792,0.11613273190983585,57128.74445660633,11.722847631461935,23.833333333333332,0.17361500000000002,8823.152364969254,0.2007770538330078,2406.0981194606593,0.11202253911830268,1.0,1.0,0.10704543711737907,0.06675514797021173,44532.50472025028,8.64392602078908,25.0,0,0,0,36,1189.5,27.1,30.8,0.4810606060606061,0.05400487083496075 +XT (tuned + ensemble),1037.372560845481,3.071019298500485,360.7132393580917,1.4744924910300816,0.9176899118580594,0.8772100688076164,0.1843143961543318,0.14290373773078288,4707.816009404905,94.53215065372743,24.26388888888889,0.183685,752.3897469043732,2.041845917701721,187.8732763026905,0.7207715417972143,1.0,1.0,0.1315762721206637,0.07600010491905307,3200.195519746224,83.245630982298,26.75,0,0,0,36,1184.0,25.7,30.1,0.47127525252525254,0.05733624005903094 +RF (tuned + ensemble),2020.4760966300964,2.134269985887739,415.1583564929086,1.2910427322323617,0.932518287859348,0.9005533897562101,0.190161624036382,0.15955834513704611,5462.814816714475,83.37351619032778,25.666666666666668,0.180965,851.531911611557,1.8475559949874878,266.2539346689623,0.6395657388264155,1.0,1.0,0.1352038852066204,0.0822503414133858,4305.508903299853,67.08260239114765,28.5,0,2,0,34,1153.4,27.6,26.5,0.4393939393939394,0.073342105194866 +TABDPT (default),175.0330073899693,68.71396222379472,26.097389455873152,21.943944030413235,0.8137945017331583,0.8179614287037148,0.19222382286116146,0.1398951815244204,492.7334429125401,1620.9148184577618,26.23611111111111,0.198215,97.51883804798126,29.261866688728333,20.255521876560408,8.552781426594889,1.0,1.0,0.13114122736509654,0.07269651025672075,407.04692448648916,1198.388331768314,31.0,1,3,0,32,1144.2,28.1,34.3,0.4264520202020202,0.11055235729381453 +XT (tuned),1037.372560845481,0.3051542573504978,360.7132393580917,0.16689901565151677,0.9439228284777492,0.8840380175065681,0.19597755438216247,0.14929813671791142,4707.816009404905,10.02739657365076,26.25,0.18688500000000002,752.3897469043732,0.1615079641342163,187.8732763026905,0.07985268047821353,1.0,1.0,0.1505192879837034,0.08371989745070943,3200.195519746224,8.479842721007039,29.0,0,1,0,35,1143.3,26.7,35.0,0.42613636363636365,0.06398339537607635 +FASTAI (tuned),7289.840611391597,1.1152561836772494,1303.7963220052682,0.6947586912123508,0.9058648946317567,0.873357455480317,0.20144752798465024,0.13768196892163254,19518.83385555907,43.25082874084665,26.416666666666668,0.18023,2935.3402432203293,0.9242111444473267,581.3698825429726,0.334061010051534,1.0,1.0,0.1660331096790038,0.0823205592657475,12902.237935802625,29.54949110419716,28.5,0,0,1,35,1139.3,30.8,26.5,0.42234848484848486,0.059729533200154804 +GBM (default),7.365004950099522,0.7322817908393012,2.509446514786595,0.18984168667689244,0.9471769132881858,0.9201699708464274,0.1830567677779008,0.1541138325172775,35.17208033262076,16.8021003794707,26.84722222222222,0.18529,6.238969683647156,0.6098566055297852,1.6407068695252,0.14810769896236448,1.0,1.0,0.14940090168048453,0.07094283568464387,22.04913243774461,11.229856118149964,27.5,0,0,0,36,1131.2,21.0,31.4,0.4125631313131313,0.042567416943239275 +RF (tuned),2020.4760966300964,0.22272791465123495,415.1583564929086,0.1434182534422469,0.9551019371507523,0.9360306919709499,0.19787032871599097,0.16914360068649603,5462.814816714475,8.151832659415685,27.694444444444443,0.18201499999999998,851.531911611557,0.14735674858093262,266.2539346689623,0.0714840059306155,1.0,1.0,0.13792533210297697,0.09963978217702693,4305.508903299853,7.364430017401964,29.0,0,0,0,36,1112.2,33.5,30.6,0.39330808080808083,0.048009195540425535 +NN_TORCH (default),47.65417258607017,0.6302466657426622,10.641224850569223,0.23501087478485314,0.978257373569646,0.954102548699942,0.2275331100570714,0.169651058381061,143.43581528304946,15.239602876479248,31.458333333333332,0.18706,22.84017276763916,0.28995954990386963,5.436935257156724,0.1377204507988835,1.0,1.0,0.1627665585076386,0.09575304821587584,137.2254146054324,9.730207537012728,33.0,0,0,0,36,1031.6,22.4,35.3,0.3077651515151515,0.03662907463362058 +FASTAI (default),30.521977769003975,1.0843763152758281,4.742334418927322,0.49201194647444324,0.972512597016023,0.9430264129001292,0.23958288237304964,0.18088306715708347,76.51706745050629,33.35318180861937,32.5,0.187165,13.653389573097229,0.6859067678451538,3.0439688517356123,0.3001709716511216,1.0,1.0,0.16007809952594787,0.12381628056074112,56.69169806857949,28.502895225657927,35.0,0,0,0,36,1003.4,29.2,31.6,0.2840909090909091,0.03514706034791027 +RF (default),3.2084562049971685,0.13506737020280626,0.4263514561749847,0.06558284712037367,0.9965256868256966,0.9756374933977958,0.25930576589425613,0.2428212871410249,7.059782904150193,4.425232204599554,32.611111111111114,0.20493,1.2789119482040405,0.09202027320861816,0.34809529918260984,0.03472589740080406,1.0,1.0,0.22217921675183,0.11658001888686689,5.376229465336108,3.6309495335392015,35.75,0,0,0,36,1000.0,0.0,0.0,0.2815656565656566,0.037864938459085944 +LR (tuned + ensemble),246.97270359595618,1.3169628845320807,86.24001785898447,0.299396801053486,0.9561054304859822,0.9692239441416002,0.30863610039561684,0.2486725924427303,1104.1489965774379,26.605204102831735,34.56944444444444,0.20664,169.60151755809784,0.2824718952178955,44.56368864780743,0.15651788282525403,1.0,1.0,0.2502384210107675,0.1550098957472666,675.130801866569,8.851961834833686,37.0,1,0,0,35,949.2,30.1,35.1,0.2370580808080808,0.05633787203620641 +XT (default),1.8672458993064032,0.17208321889241537,0.35727892050479015,0.06872435387895405,0.9892721532660226,0.9800527523199994,0.2874747079845823,0.2752176157333544,4.999840482001418,4.841906560022478,35.416666666666664,0.20700000000000002,0.8974905014038086,0.08268320560455322,0.23619265983619955,0.036597834553610306,1.0,1.0,0.21784002712442435,0.1482624508295911,3.672313260209499,4.261733262225922,38.0,0,0,0,36,925.3,30.7,23.6,0.2178030303030303,0.032013179657027775 +LR (tuned),246.97270359595618,0.43901119629542035,86.24001785898447,0.1002982819551287,0.9723830890370608,0.9787756407161015,0.3159674455247929,0.2604157048141723,1104.1489965774379,7.096822360818471,35.65277777777778,0.206755,169.60151755809784,0.12632572650909424,44.56368864780743,0.062293080471580324,1.0,1.0,0.25818261683105737,0.1688557029748925,675.130801866569,5.062717123136234,38.25,0,0,1,35,919.4,21.5,28.9,0.21243686868686867,0.036643329912295686 +LR (default),6.324005815717909,0.45038049750857884,2.0933349209116243,0.12679966936326295,0.9795929927195421,0.9864551890342567,0.3255669034156274,0.28877566195163695,28.81662586680685,8.679077173966622,36.52777777777778,0.21439,5.391173958778381,0.12632572650909424,1.4157886113918772,0.09164922519151064,1.0,1.0,0.2581813809635515,0.16885460955121118,16.81966393701253,5.719143257720329,39.0,0,0,0,36,891.6,29.9,32.3,0.19255050505050506,0.029261494694519958 +KNN (tuned + ensemble),168.93849890761905,8.359918620851305,8.837249824100754,0.4805954727942703,1.0,0.9867364080557345,0.4982079360481384,0.6064348609957761,80.93770906535752,52.719394620298914,40.27777777777778,0.321975,13.986609816551208,0.22182130813598633,3.4970040049929065,0.1592449370444793,1.0,1.0,0.44534583678290146,0.6925990746528932,56.97784214902664,12.39378247919743,43.0,0,0,0,36,741.1,33.5,42.1,0.10732323232323232,0.028756216828868864 +KNN (tuned),168.93849890761905,1.9504973888397217,8.837249824100754,0.10793579395855045,1.0,0.9837443971669894,0.5193096900773991,0.6567505669495036,80.93770906535752,11.495707208766925,41.43055555555556,0.32563,13.986609816551208,0.06376731395721436,3.4970040049929065,0.03322185201916636,1.0,1.0,0.5143946348760557,0.7341673886277578,56.97784214902664,2.1231636318644824,44.0,0,0,1,35,676.8,32.2,33.4,0.08112373737373738,0.0323845750911192 +KNN (default),0.7941402528021071,0.21308696932262844,0.1178118784481507,0.0311341997736934,1.0,1.0,0.6031940607224022,0.9574097789264635,1.001944530363665,2.4108020816009272,43.90277777777778,0.38312,0.23321354389190674,0.030547380447387695,0.06547009031681006,0.018147653154379814,1.0,1.0,0.6027815264295836,1.0,1.0,1.409117664778126,45.0,0,0,0,36,452.7,59.1,47.9,0.024936868686868688,0.02303628559779201 diff --git a/data/lite/tabicl-imputed/time_plot.pdf b/data/lite/tabicl-imputed/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..afc156a0151d1994b993807676e6ebcb70d8b778 Binary files /dev/null and b/data/lite/tabicl-imputed/time_plot.pdf differ diff --git a/data/lite/tabicl-imputed/time_plot.png.zip b/data/lite/tabicl-imputed/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..b2e3ce1fee6d3e2b73511304b5a7a0bc134abb8f --- /dev/null +++ b/data/lite/tabicl-imputed/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c80d6f0bf2cc93053e01f71416b082143ace078ba31e86293a004cf6a08c3691 +size 338023 diff --git a/data/lite/tabicl-imputed/tuning-impact-elo-horizontal.pdf b/data/lite/tabicl-imputed/tuning-impact-elo-horizontal.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b6df5ce9d06614bd6c3853411fc044399ae2406c Binary files /dev/null and b/data/lite/tabicl-imputed/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabicl-imputed/tuning-impact-elo-horizontal.png.zip b/data/lite/tabicl-imputed/tuning-impact-elo-horizontal.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1a54bc5afef5d43bbabe94795aeb0446b170dbdf --- /dev/null +++ b/data/lite/tabicl-imputed/tuning-impact-elo-horizontal.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13832999264243934ee90d1a7979be9ee22b03801036c7c61fe792da53afcd3e +size 152245 diff --git a/data/lite/tabicl-imputed/tuning-impact-elo.pdf b/data/lite/tabicl-imputed/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..be31c43b7a369541946fd13c7d9f19e706cee3ac Binary files /dev/null and b/data/lite/tabicl-imputed/tuning-impact-elo.pdf differ diff --git a/data/lite/tabicl-imputed/tuning-impact-elo.png.zip b/data/lite/tabicl-imputed/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..28d205f29159e556a3f6695f767ebb95bc34effd --- /dev/null +++ b/data/lite/tabicl-imputed/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c6443b42af3803ddbd71bbc64632369da1bf2277f929bcf8f9254ea1e91308a +size 154208 diff --git a/data/lite/tabicl/figures/critical-diagram.pdf b/data/lite/tabicl/figures/critical-diagram.pdf index 5ee043ee0b4807541c52617dc2f11eeadc46960e..9c24185fcd9c091dfed3bbf03ac8ec77022a627e 100644 Binary files a/data/lite/tabicl/figures/critical-diagram.pdf and b/data/lite/tabicl/figures/critical-diagram.pdf differ diff --git a/data/lite/tabicl/figures/critical-diagram.png.zip b/data/lite/tabicl/figures/critical-diagram.png.zip index 5473e45838602b2408c048ae6f15679b205af04e..e9945818d685b1c3e2147a3da2092ceb6b9d214a 100644 --- a/data/lite/tabicl/figures/critical-diagram.png.zip +++ b/data/lite/tabicl/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9b62643ca543a5b490148fb916ed1fdf862546abf3d6ecb1a20ed75b16e37b3f -size 312252 +oid sha256:01e6e7e7d4a159bf10db72a5884ce6c959138dbb69c0d0d08f6eff3343c04e8e +size 312390 diff --git a/data/lite/tabicl/time_plot.pdf b/data/lite/tabicl/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1469bf1b746f793d225b6a9b37fa5926acb34072 Binary files /dev/null and b/data/lite/tabicl/time_plot.pdf differ diff --git a/data/lite/tabicl/time_plot.png.zip b/data/lite/tabicl/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..62a0fc562fb849e6d9bdbda848609fa20aa479c1 --- /dev/null +++ b/data/lite/tabicl/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c845f7bd99b29da34a99cb053d4eacf4caa98d84a5067143afb8efafab624f8f +size 338023 diff --git a/data/lite/tabicl/tuning-impact-elo-horizontal.pdf b/data/lite/tabicl/tuning-impact-elo-horizontal.pdf index 06be05d6dbe301b4226b2f0253c06c6a1f715a13..37506acb22b96bc3a2cc42f038d2be85a6e238de 100644 Binary files a/data/lite/tabicl/tuning-impact-elo-horizontal.pdf and b/data/lite/tabicl/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabicl/tuning-impact-elo-horizontal.png.zip b/data/lite/tabicl/tuning-impact-elo-horizontal.png.zip index a82e75a5f4e77ce83a36fe4ab772e61e785bbf53..afb485fa738cb133bc0d4e551905c1fa23a1a9b8 100644 --- a/data/lite/tabicl/tuning-impact-elo-horizontal.png.zip +++ b/data/lite/tabicl/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:90a0e767745b3612add5803b4bb49d6445f96161eea7a533f54f123ac9244056 -size 131668 +oid sha256:b61f67196ecc2f156ead630aecb543fdefa71868a68f14d44bfd8960864fd726 +size 131830 diff --git a/data/lite/tabicl/tuning-impact-elo.pdf b/data/lite/tabicl/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7793b97e42df164edd7bc5908a84c2cc3940d0da Binary files /dev/null and b/data/lite/tabicl/tuning-impact-elo.pdf differ diff --git a/data/lite/tabicl/tuning-impact-elo.png.zip b/data/lite/tabicl/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..24eabbb35ca624c3d35ee64a7c692bbed222a9eb --- /dev/null +++ b/data/lite/tabicl/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbe30bd8bfca5bb87f6cfb30b63729c2b8ea58afb9414f47eff4e68001cc5981 +size 124842 diff --git a/data/lite/tabpfn-cls/figures/critical-diagram.pdf b/data/lite/tabpfn-cls/figures/critical-diagram.pdf index 6bfd1724fd13f4e074ba791a9eb80a0e721fd27a..cf36f3a7c411b933e0eddca986870d845636799b 100644 Binary files a/data/lite/tabpfn-cls/figures/critical-diagram.pdf and b/data/lite/tabpfn-cls/figures/critical-diagram.pdf differ diff --git a/data/lite/tabpfn-cls/figures/critical-diagram.png.zip b/data/lite/tabpfn-cls/figures/critical-diagram.png.zip index 45ee2eb3b7d9536c80d1b09ccd1aaa64ed42fb2f..bb7d7dce863aede40a891261f486899520b8405a 100644 --- a/data/lite/tabpfn-cls/figures/critical-diagram.png.zip +++ b/data/lite/tabpfn-cls/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5fb8945a9b227c0087f10717cfe7d833f84cbbff633c8937aa161f906278af17 -size 315033 +oid sha256:338c39446521d6ae59f72bb721b64261c9453828639a627632e41533511ed816 +size 315171 diff --git a/data/lite/tabpfn-cls/time_plot.pdf b/data/lite/tabpfn-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..25f7ba5a1f331365db65a5da5c32c3302a279eb0 Binary files /dev/null and b/data/lite/tabpfn-cls/time_plot.pdf differ diff --git a/data/lite/tabpfn-cls/time_plot.png.zip b/data/lite/tabpfn-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..105f86abb910b768e419b11643b3ed14672e4086 --- /dev/null +++ b/data/lite/tabpfn-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaa18b7899747a9dddbc7a501311833c2c172677922d96d04e09fd4d9a010bd4 +size 338023 diff --git a/data/lite/tabpfn-cls/tuning-impact-elo-horizontal.pdf b/data/lite/tabpfn-cls/tuning-impact-elo-horizontal.pdf index 8211e352713b379cf3d86693384562f2d82a8887..914b87c9c81987efa1c80332d00f95743a96c75a 100644 Binary files a/data/lite/tabpfn-cls/tuning-impact-elo-horizontal.pdf and b/data/lite/tabpfn-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabpfn-cls/tuning-impact-elo-horizontal.png.zip b/data/lite/tabpfn-cls/tuning-impact-elo-horizontal.png.zip index c0548fd33e9fc1629dd65491542324387b1b97eb..ed5166dea3aed2f74c63d035f66baabd73508859 100644 --- a/data/lite/tabpfn-cls/tuning-impact-elo-horizontal.png.zip +++ b/data/lite/tabpfn-cls/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4de8849fd2a503912b139318e210f7f6b400766abd9b62db02078904f750a579 -size 134767 +oid sha256:c651c2fe30679b967e562552e48b70809964a3f5283cca9483f7a032c9044bf6 +size 134929 diff --git a/data/lite/tabpfn-cls/tuning-impact-elo.pdf b/data/lite/tabpfn-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6bb91fe0effef7b4372c6209496d590630f12f21 Binary files /dev/null and b/data/lite/tabpfn-cls/tuning-impact-elo.pdf differ diff --git a/data/lite/tabpfn-cls/tuning-impact-elo.png.zip b/data/lite/tabpfn-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ec3523b378e17b65edea34faeca1f4e6c9ba283f --- /dev/null +++ b/data/lite/tabpfn-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db291309355ab0c7be5fb36c5b01300875c6a5d7ac1c10b4531d1cba41b2f84c +size 127424 diff --git a/data/lite/tabpfn-imputed-cls/figures/critical-diagram.pdf b/data/lite/tabpfn-imputed-cls/figures/critical-diagram.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e4b9232afc564ed5513693318310deac0a7da23e Binary files /dev/null and b/data/lite/tabpfn-imputed-cls/figures/critical-diagram.pdf differ diff --git a/data/lite/tabpfn-imputed-cls/figures/critical-diagram.png.zip b/data/lite/tabpfn-imputed-cls/figures/critical-diagram.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4f8d50fa3b57138bb04c1aff08118f133bc9ff62 --- /dev/null +++ b/data/lite/tabpfn-imputed-cls/figures/critical-diagram.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ded7236739d75930a2da4f3d2ea144c2f16991dc50c6f62ffe15700b98e468ba +size 319138 diff --git a/data/lite/tabpfn-imputed-cls/leaderboard.tex b/data/lite/tabpfn-imputed-cls/leaderboard.tex new file mode 100644 index 0000000000000000000000000000000000000000..470e5847143808188c07aa37c1d5e5b76d097272 --- /dev/null +++ b/data/lite/tabpfn-imputed-cls/leaderboard.tex @@ -0,0 +1,53 @@ +\begin{tabular}{llcccccrr} +\toprule +\textbf{Model} & \textbf{Elo ($\uparrow$)} & \textbf{Norm.} & \textbf{Avg.} & \textbf{Harm.} & \textbf{\#wins ($\uparrow$)} & \textbf{Improva-} & \textbf{Train time} & \textbf{Predict time} \\ + & & \textbf{score ($\uparrow$)} & \textbf{rank ($\downarrow$)} & \textbf{mean} & & \textbf{bility ($\downarrow$)} & \textbf{per 1K [s]} & \textbf{per 1K [s]} \\ + & & & & \textbf{rank ($\downarrow$)} & & & & \\ +\midrule +TabPFNv2 (T+E) & \textcolor{gold}{\textbf{1507${}_{-45,+32}$}} & \textcolor{gold}{\textbf{0.700}} & \textcolor{gold}{\textbf{8.5}} & \textcolor{gold}{\textbf{2.8}} & \textcolor{gold}{\textbf{6}} & \textcolor{gold}{\textbf{8.0\%}} & 3361.32 & 51.67 \\ +AutoGluon 1.3 (4h) & \textcolor{silver}{\textbf{1434${}_{-31,+33}$}} & 0.440 & \textcolor{silver}{\textbf{11.0}} & 6.4 & 1 & \textcolor{silver}{\textbf{10.4\%}} & 2085.73 & 2.16 \\ +RealMLP (T+E) & \textcolor{bronze}{\textbf{1408${}_{-30,+28}$}} & 0.402 & \textcolor{bronze}{\textbf{12.1}} & 6.9 & 0 & 12.3\% & 5575.69 & 4.58 \\ +TabPFNv2 (T) & 1402${}_{-33,+32}$ & \textcolor{silver}{\textbf{0.544}} & 12.2 & \textcolor{bronze}{\textbf{4.5}} & 1 & 10.6\% & 3361.32 & 0.53 \\ +TabM (T+E) & 1392${}_{-40,+33}$ & 0.483 & 12.8 & 6.2 & 1 & 12.0\% & 2899.02 & 1.46 \\ +TabICL (D) & 1378${}_{-31,+37}$ & \textcolor{bronze}{\textbf{0.543}} & 13.4 & \textcolor{silver}{\textbf{3.9}} & \textcolor{silver}{\textbf{4}} & \textcolor{bronze}{\textbf{10.5\%}} & 9.07 & 2.01 \\ +LightGBM (T+E) & 1349${}_{-33,+29}$ & 0.319 & 14.8 & 11.6 & 0 & 14.1\% & 691.46 & 1.46 \\ +TabPFNv2 (D) & 1334${}_{-35,+35}$ & 0.499 & 15.5 & \textcolor{bronze}{\textbf{4.5}} & \textcolor{bronze}{\textbf{3}} & 12.1\% & 4.28 & 0.51 \\ +CatBoost (T+E) & 1332${}_{-36,+33}$ & 0.271 & 15.8 & 9.8 & 0 & 13.5\% & 1394.45 & 0.62 \\ +TabM (T) & 1330${}_{-31,+33}$ & 0.387 & 15.7 & 7.7 & 0 & 13.1\% & 2899.02 & 0.17 \\ +ModernNCA (T) & 1326${}_{-31,+31}$ & 0.255 & 16.0 & 6.4 & 2 & 12.7\% & 6020.89 & 0.45 \\ +XGBoost (T+E) & 1314${}_{-33,+29}$ & 0.222 & 16.6 & 11.4 & 0 & 15.2\% & 785.90 & 1.28 \\ +CatBoost (D) & 1308${}_{-32,+30}$ & 0.228 & 16.8 & 9.6 & 1 & 14.8\% & 5.74 & 0.14 \\ +CatBoost (T) & 1299${}_{-28,+42}$ & 0.247 & 17.2 & 13.3 & 0 & 14.4\% & 1394.45 & 0.05 \\ +ModernNCA (T+E) & 1292${}_{-35,+27}$ & 0.307 & 17.7 & 9.4 & 0 & 14.4\% & 6020.89 & 8.48 \\ +XGBoost (T) & 1290${}_{-26,+30}$ & 0.181 & 17.8 & 9.8 & 1 & 15.4\% & 785.90 & 0.21 \\ +LightGBM (T) & 1288${}_{-29,+30}$ & 0.262 & 17.6 & 11.9 & 0 & 15.1\% & 691.46 & 0.26 \\ +TabM (D) & 1281${}_{-25,+30}$ & 0.288 & 18.2 & 8.1 & 1 & 16.2\% & 11.60 & 0.15 \\ +EBM (T+E) & 1277${}_{-29,+37}$ & 0.186 & 18.4 & 8.2 & 0 & 17.2\% & 1142.31 & 0.24 \\ +TorchMLP (T+E) & 1247${}_{-28,+31}$ & 0.199 & 19.9 & 12.7 & 0 & 15.7\% & 2792.14 & 2.18 \\ +RealMLP (T) & 1231${}_{-29,+34}$ & 0.201 & 20.8 & 13.3 & 0 & 16.5\% & 5575.69 & 0.24 \\ +EBM (T) & 1227${}_{-32,+37}$ & 0.120 & 20.9 & 11.1 & 0 & 18.0\% & 1142.31 & 0.03 \\ +EBM (D) & 1207${}_{-35,+29}$ & 0.153 & 22.2 & 9.7 & 1 & 18.7\% & 5.57 & 0.07 \\ +FastaiMLP (T+E) & 1202${}_{-34,+29}$ & 0.230 & 22.3 & 12.4 & 0 & 19.1\% & 1350.04 & 6.57 \\ +ModernNCA (D) & 1188${}_{-28,+29}$ & 0.183 & 22.8 & 9.8 & 1 & 19.4\% & 13.67 & 0.34 \\ +TabDPT (D) & 1179${}_{-36,+29}$ & 0.258 & 23.5 & 7.1 & 1 & 18.4\% & 28.08 & 8.76 \\ +RealMLP (D) & 1179${}_{-31,+24}$ & 0.083 & 23.6 & 11.0 & 1 & 17.5\% & 35.27 & 0.25 \\ +TorchMLP (T) & 1168${}_{-36,+38}$ & 0.086 & 24.0 & 17.9 & 0 & 17.8\% & 2792.14 & 0.13 \\ +ExtraTrees (T+E) & 1155${}_{-36,+29}$ & 0.102 & 24.9 & 16.7 & 0 & 20.4\% & 378.34 & 1.37 \\ +FastaiMLP (T) & 1140${}_{-32,+30}$ & 0.113 & 25.5 & 14.7 & 0 & 20.7\% & 1350.04 & 0.61 \\ +XGBoost (D) & 1130${}_{-32,+28}$ & 0.032 & 26.0 & 21.9 & 0 & 19.1\% & 2.66 & 0.17 \\ +ExtraTrees (T) & 1128${}_{-26,+31}$ & 0.072 & 26.0 & 13.8 & 0 & 20.6\% & 378.34 & 0.15 \\ +RandomForest (T+E) & 1114${}_{-34,+28}$ & 0.057 & 27.0 & 14.8 & 0 & 21.0\% & 527.27 & 1.16 \\ +LightGBM (D) & 1098${}_{-30,+32}$ & 0.030 & 27.6 & 24.0 & 0 & 20.1\% & 3.01 & 0.18 \\ +RandomForest (T) & 1064${}_{-35,+30}$ & 0.023 & 29.2 & 22.2 & 0 & 21.9\% & 527.27 & 0.13 \\ +TorchMLP (D) & 1006${}_{-30,+39}$ & 0.030 & 31.7 & 26.7 & 0 & 24.8\% & 8.93 & 0.17 \\ +RandomForest (D) & 1000${}_{-0,+0}$ & 0.005 & 32.2 & 24.6 & 0 & 27.3\% & 0.44 & 0.06 \\ +FastaiMLP (D) & 1000${}_{-32,+30}$ & 0.019 & 32.2 & 27.7 & 0 & 24.6\% & 4.68 & 0.55 \\ +ExtraTrees (D) & 949${}_{-32,+36}$ & 0.015 & 34.1 & 29.1 & 0 & 29.3\% & 0.36 & 0.07 \\ +Linear (T+E) & 946${}_{-35,+33}$ & 0.055 & 34.2 & 15.0 & 1 & 31.3\% & 89.12 & 0.20 \\ +Linear (T) & 925${}_{-44,+35}$ & 0.035 & 34.9 & 24.7 & 0 & 32.0\% & 89.12 & 0.08 \\ +Linear (D) & 892${}_{-29,+36}$ & 0.025 & 36.0 & 33.0 & 0 & 33.3\% & 2.43 & 0.10 \\ +KNN (T+E) & 746${}_{-31,+45}$ & 0.000 & 39.9 & 33.1 & 0 & 47.7\% & 3.50 & 0.16 \\ +KNN (T) & 676${}_{-48,+42}$ & 0.000 & 41.2 & 28.3 & 0 & 50.2\% & 3.50 & 0.03 \\ +KNN (D) & 384${}_{-116,+61}$ & 0.000 & 44.3 & 44.2 & 0 & 59.5\% & 0.07 & 0.02 \\ +\bottomrule +\end{tabular} \ No newline at end of file diff --git a/data/lite/tabpfn-imputed-cls/tabarena_leaderboard.csv b/data/lite/tabpfn-imputed-cls/tabarena_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..df059e4121634d1f0f2b668e430505bc14507b44 --- /dev/null +++ b/data/lite/tabpfn-imputed-cls/tabarena_leaderboard.csv @@ -0,0 +1,46 @@ +method,time_train_s,time_infer_s,time_train_s_per_1K,time_infer_s_per_1K,normalized-error,normalized-error-task,champ_delta,loss_rescaled,time_train_s_rescaled,time_infer_s_rescaled,rank,median_metric_error,median_time_train_s,median_time_infer_s,median_time_train_s_per_1K,median_time_infer_s_per_1K,median_normalized-error,median_normalized-error-task,median_champ_delta,median_loss_rescaled,median_time_train_s_rescaled,median_time_infer_s_rescaled,median_rank,rank=1_count,rank=2_count,rank=3_count,rank>3_count,elo,elo+,elo-,winrate,mrr +TABPFNV2 (tuned + ensemble),14624.10810300937,166.6936253951146,3837.093570770289,71.22922446033486,0.2999799624009034,0.37900004959687766,0.08031051995518385,0.05455935044977521,73712.17169915051,6326.444952096131,8.461538461538462,0.1749,8447.90291428566,59.9476752281189,3361.317280362242,51.668880171806535,0.21441671773452015,0.3480554663987826,0.03548854933740464,0.027745461982937078,32774.70384527536,4094.471326370238,5.5,6,1,3,16,1506.7,31.8,44.7,0.8304195804195804,0.358094093982103 +AutoGluon 1.3 (4h),5811.540137483524,3.0737958137805643,3715.2599271484114,3.044362316385498,0.5599854226187498,0.5617828950350577,0.10445421398736063,0.0599680756233326,37983.87924037632,165.48830983683717,11.01923076923077,0.17751,5213.819254875183,2.0895845890045166,2085.7270446537464,2.161349128608518,0.5600808158121042,0.5574000428410242,0.06078825021132733,0.06659288302785793,30345.10100676535,118.3021811054546,9.0,1,0,1,24,1433.5,32.8,30.7,0.7722902097902098,0.15518544636191695 +REALMLP (tuned + ensemble),17853.666671056013,5.060677555891184,6069.468633023759,5.352972929535142,0.5983187605703805,0.5697719962134024,0.1225244655490201,0.0697158337547381,114488.59335090281,312.9299756303972,12.115384615384615,0.180765,10652.850141763687,4.7910391092300415,5575.693006637695,4.577857176513236,0.5696681015084324,0.5408431020221902,0.06341551053522204,0.04576778429435464,88499.22644289241,286.2007603461078,11.5,0,1,2,23,1407.7,28.0,30.0,0.7473776223776224,0.1450476689017345 +TABPFNV2 (tuned),14624.10810300937,5.890562671881455,3837.093570770289,2.9216612176257093,0.4562783530638654,0.539616756848536,0.10636474393434797,0.07576264928807884,73712.17169915051,208.74723417223507,12.25,0.19132,8447.90291428566,0.5261704921722412,3361.317280362242,0.52729181819818,0.34109768962419973,0.6143846899224816,0.08017059655741715,0.050216477642017796,32774.70384527536,36.401677545743524,11.0,1,5,2,18,1402.3,31.6,32.9,0.7443181818181818,0.22277582323851844 +TABM (tuned + ensemble),7183.65333878994,2.169150572556716,3570.712955098323,1.861754182648195,0.5166525238887076,0.5534887980632361,0.11956349986158774,0.08309123144106954,42989.483218042995,121.47528176372664,12.846153846153847,0.18310500000000002,6058.230099320412,1.6269241571426392,2899.0220559668596,1.460880880345262,0.47136645123500964,0.5213443194889227,0.07336602828104005,0.040297742432766875,37327.19260107514,109.27107326295308,9.5,1,0,1,24,1392.2,32.7,39.4,0.7307692307692307,0.16156485134544094 +TABICL (default),21.54241045621725,2.6437206268310547,10.219926189931016,1.9528462364644343,0.45732829959408833,0.5557361534084676,0.10499055070178752,0.06310686388674132,140.84761892371782,137.0823191069963,13.365384615384615,0.177865,18.648964643478394,2.1483960151672363,9.073519792448254,2.007502492806867,0.46959481282562443,0.6652418475567421,0.05302697309380272,0.028777901101857722,117.57303754113096,107.76328227971135,13.5,4,2,1,19,1378.5,36.4,30.5,0.7189685314685315,0.2556026489858902 +GBM (tuned + ensemble),1386.6633568268555,2.647538212629465,855.5354588276103,2.6861264004396594,0.6806482110513491,0.7106186510085695,0.141362130997881,0.08782930779014528,10078.423717398238,182.57724180654643,14.788461538461538,0.18084,1317.1929507255554,2.008246660232544,691.4641967411458,1.4553717527055694,0.7005535661954161,0.7652961068739543,0.07965351598262033,0.06332006129013433,9059.644744264673,90.72358893879789,14.75,0,0,1,25,1349.4,28.6,33.0,0.6866258741258742,0.08652272628643233 +TABPFNV2 (default),12.006092300781837,1.2122490681134737,5.3148066442099084,0.6434373636958165,0.5013642769155608,0.6076681115357273,0.12052806276021125,0.08926430649670257,74.57934257049592,50.66248833495304,15.538461538461538,0.194575,9.273962259292603,0.4255213737487793,4.283948847917442,0.5110342254172261,0.3771347187714357,0.8078260737500734,0.07359963710195983,0.07618105155863322,62.366726819804335,28.80821414198816,14.5,3,2,2,19,1333.5,34.9,34.7,0.6695804195804196,0.22455622157365326 +TABM (tuned),7183.65333878994,0.2019361532651461,3570.712955098323,0.2168923191387062,0.6127855914191946,0.6153990479247885,0.130856425990661,0.09537399493661194,42989.483218042995,13.062348029072668,15.692307692307692,0.18589499999999998,6058.230099320412,0.1839456558227539,2899.0220559668596,0.171251855223312,0.5915500043256848,0.6288985041617685,0.09148128983327741,0.05588464476989181,37327.19260107514,9.17800659262771,12.0,0,2,2,22,1329.7,32.2,30.3,0.666083916083916,0.13000506341067208 +CAT (tuned + ensemble),4861.749095384891,1.0365588940106905,3035.7630585976663,0.9271931106606893,0.728920075896456,0.6786642048762535,0.13487992339677388,0.0909217961956513,26561.38119334446,53.29220521181529,15.76923076923077,0.178855,3259.4502482414246,0.9432240724563599,1394.4481643221718,0.6180738220288715,0.7992439551459185,0.7007978686240202,0.07405507598059591,0.051124014208117546,20607.594190900367,47.175332427485074,15.0,0,1,1,24,1331.9,32.8,35.8,0.6643356643356644,0.10243020698237862 +MNCA (tuned),11104.727954295966,0.5286179322462815,5977.187221364461,0.4814995680859316,0.7448490525012423,0.6406839211091188,0.12655502884420408,0.10326715793661159,78433.1838326286,30.191390941699975,16.01923076923077,0.18580999999999998,11212.920736551285,0.4423438310623169,6020.892835479641,0.4537708228509102,0.8177816514691232,0.7061138774961084,0.08668289690877401,0.05913687649157323,67721.7850080898,25.373417518222958,14.25,2,0,0,24,1325.6,30.3,30.3,0.6586538461538461,0.15586362718013336 +XGB (tuned + ensemble),1868.2429302564035,2.459977333362286,1131.3403541498637,3.0908212073715786,0.7784275988369416,0.7326751610286906,0.1521183469738161,0.10487866753851346,11151.77336671448,133.9681639809692,16.576923076923077,0.184395,1441.2213016748428,1.317580223083496,785.8975953484313,1.2809013038285242,0.8316341855574652,0.7341106406501618,0.1144861696628025,0.054690456125919155,11226.485943771408,74.70529109142498,13.5,0,0,2,24,1313.5,28.5,32.1,0.6459790209790209,0.08752950287187947 +CAT (default),173.12264615755814,0.2530022859573364,152.19981148640542,0.1746769705615129,0.7721520925382083,0.7508782741378199,0.14762383159906592,0.09673912909834526,503.5985284620293,11.812145121206303,16.75,0.18153,12.185660362243652,0.2613861560821533,5.736216485552211,0.13945330874004275,0.8708469956136012,0.8764795848440701,0.09472627265148731,0.048694739189140704,101.89620933712393,7.710770185087604,19.25,1,0,0,25,1308.5,29.6,31.7,0.6420454545454546,0.10408056855425275 +CAT (tuned),4861.749095384891,0.10756405500265268,3035.7630585976663,0.11792367197674246,0.7533425274831995,0.7418580085557809,0.14394572275280534,0.09786470048169181,26561.38119334446,7.02054598025311,17.192307692307693,0.179485,3259.4502482414246,0.05912292003631592,1394.4481643221718,0.05275041900206211,0.8504867849422679,0.7781645153943832,0.09050829480547845,0.05413571715045103,20607.594190900367,2.886943526422101,17.0,0,0,0,26,1299.2,41.1,27.8,0.631993006993007,0.07539691390572127 +GBM (tuned),1386.6633568268555,0.6239433655372033,855.5354588276103,0.6651164927916418,0.737683616453633,0.7596688264282848,0.15123309377717353,0.10299158861394081,10078.423717398238,42.45419816933193,17.634615384615383,0.18125,1317.1929507255554,0.32417428493499756,691.4641967411458,0.2643910314838089,0.7826548575969626,0.7981479775650631,0.08157588989660797,0.06910038121186715,9059.644744264673,20.34411810847653,18.5,0,1,0,25,1288.1,29.9,29.0,0.6219405594405595,0.08417253395400366 +MNCA (tuned + ensemble),11104.727954295966,11.626816547833956,5977.187221364461,9.077822725159232,0.6933896363411076,0.6660665157088731,0.1444822450466191,0.11574155496848519,78433.1838326286,625.8156905319904,17.653846153846153,0.196755,11212.920736551285,9.569536089897156,6020.892835479641,8.47783782257851,0.722368050936999,0.6622270828972778,0.09664772425546841,0.06590198674037494,67721.7850080898,462.66839224080115,15.75,0,0,2,24,1292.0,26.5,34.9,0.6215034965034965,0.10668541415880203 +XGB (tuned),1868.2429302564035,0.6344939470291138,1131.3403541498637,0.845858557757576,0.8187393857188691,0.7667685498472205,0.15449551107917392,0.10860550617326425,11151.77336671448,35.07178353015141,17.807692307692307,0.18658,1441.2213016748428,0.2507840394973755,785.8975953484313,0.21279594317419753,0.9017943832788667,0.7453177957281472,0.12200656150282413,0.06269228828209442,11226.485943771408,15.348416723363933,16.5,1,0,0,25,1289.8,29.4,25.3,0.618006993006993,0.10251976073586389 +TABM (default),26.77778566800631,0.20130640726823074,16.983704596070464,0.1991903141757489,0.7120650416184634,0.7086289666607134,0.1622519047697801,0.11661283756574645,212.38711719883958,11.49822311803021,18.153846153846153,0.18572,20.835053324699402,0.1561201810836792,11.598826273977398,0.14759862387674094,0.7417274447397321,0.7810887350534926,0.09137336521284833,0.06275561870896407,109.84095896574044,8.743598466674035,18.0,1,0,1,24,1281.0,29.5,24.9,0.6101398601398601,0.12406889112890061 +EBM (tuned + ensemble),2866.551095375648,0.3536289380146907,1716.6253639310917,0.31694700723366953,0.8144475987903695,0.7364009363908267,0.17180309869898544,0.13289820608093436,16005.580219570395,21.915829440477204,18.423076923076923,0.18934,1826.563945055008,0.30398499965667725,1142.3094021828929,0.2383204663966832,0.9368603191709515,0.9207311313375326,0.07547386841907339,0.05109125672741019,15335.47890204938,12.950523458443941,21.5,0,3,0,23,1276.6,36.7,28.2,0.6040209790209791,0.12124171082532008 +NN_TORCH (tuned + ensemble),8269.36125588417,3.6064914281551657,3548.397093003735,3.1069146452932506,0.8005534385996745,0.7867221786490863,0.15745609955540044,0.10096884964933617,54687.42205511106,206.78432079512532,19.884615384615383,0.186795,7021.069797754288,2.925987720489502,2792.139673756541,2.175346818589723,0.9508009219506811,0.917687560600783,0.0823266490663982,0.0717701052690009,44532.50472025028,168.68500762383525,20.0,0,0,1,25,1247.3,30.8,27.1,0.5708041958041958,0.07895497881555723 +REALMLP (tuned),17853.666671056013,0.2349767226439256,6069.468633023759,0.2967994896112342,0.7992732991174807,0.8048895624469216,0.16530717153664265,0.11169886093388302,114488.59335090281,15.66992951032636,20.76923076923077,0.179705,10652.850141763687,0.22168636322021484,5575.693006637695,0.23576453293502733,0.895201963654964,0.9670251309899789,0.1283441158885112,0.08568734793187416,88499.22644289241,12.792606929662101,20.5,0,1,0,25,1231.0,33.9,28.5,0.5506993006993007,0.07503622635930923 +EBM (tuned),2866.551095375648,0.04391795855302077,1716.6253639310917,0.042622423373582956,0.8796773591220505,0.7989370532749394,0.18031150115586747,0.13991074423102637,16005.580219570395,2.6200464262605005,20.865384615384617,0.19075999999999999,1826.563945055008,0.031216979026794434,1142.3094021828929,0.02993136666804911,1.0,1.0,0.08520888190451481,0.06958046400412875,15335.47890204938,1.3258183958395588,23.75,0,1,1,24,1227.1,36.6,31.5,0.548513986013986,0.08978323529643507 +EBM (default),11.305762657752403,0.07217896901644193,6.708906519026521,0.08642030214738769,0.8468070324259693,0.8110280220039858,0.18686874456979283,0.14250462388903842,76.4056328740172,4.809914969274192,22.23076923076923,0.19205,8.478393912315369,0.060648202896118164,5.571264059656597,0.07129895504487138,1.0,1.0,0.10192934818149862,0.06122058058323697,66.99828845613939,3.601780880785588,24.75,1,0,0,25,1206.9,28.3,34.8,0.5174825174825175,0.10315367705023558 +FASTAI (tuned + ensemble),2663.3156125178702,9.71036501114185,1626.2333617037298,10.35375213848753,0.7698173487890432,0.7888707552608327,0.19143489263209343,0.11424870338763603,20608.72361153851,575.9084247062348,22.326923076923077,0.18179499999999998,2361.516105532646,9.277374625205994,1350.0449662692658,6.569572430045136,0.9823907390988809,1.0,0.10018841132165368,0.07677125815183776,13001.178549428518,549.1440903510916,24.0,0,0,0,26,1202.1,28.7,33.1,0.5152972027972028,0.08032410389081089 +MNCA (default),29.761946293023918,0.45968284056736874,15.939524477151517,0.3666958147944243,0.8169283513779425,0.8227126311705688,0.19359838536157442,0.10940904253610519,187.22763978272167,23.803143758716253,22.826923076923077,0.19357,23.457369804382324,0.36945486068725586,13.667502577653629,0.33894040253174296,0.9808195818002508,0.9797952351969426,0.13240284359564197,0.08931940966654422,154.11579909730304,19.118442709424343,21.5,1,1,0,24,1187.8,28.8,27.9,0.5039335664335665,0.1019829541933546 +TABDPT (default),72.55373939184042,21.63674608560709,32.1753170626944,27.14316288922542,0.7421770023997575,0.7479465935897589,0.18444900328904643,0.12119989176698268,513.8148563269098,1550.6457261531152,23.48076923076923,0.20388,69.82611036300659,21.28597331047058,28.076698775708525,8.760817737139146,1.0,1.0,0.1080149448719816,0.07851722429401914,429.0853998300959,1143.1582415263433,29.5,1,3,0,22,1178.7,28.4,35.5,0.48907342657342656,0.14118801082755247 +REALMLP (default),110.65054762363434,0.2512630499326266,36.51262336985926,0.304026477962315,0.9170988508971886,0.8257233846816412,0.17480849590114966,0.10965875529591303,716.9172586317429,16.106317028225142,23.634615384615383,0.179885,61.38514173030853,0.19377470016479492,35.26610472383891,0.25260632993704635,1.0,1.0,0.10725973083151863,0.09589271240176966,562.1682174661271,13.084740916081461,26.25,1,0,0,25,1178.7,23.8,30.2,0.4855769230769231,0.09086138282523847 +NN_TORCH (tuned),8269.36125588417,0.1732868873156034,3548.397093003735,0.17387289280012472,0.9141063957909149,0.842287949515361,0.1775257283605692,0.11814031211601564,54687.42205511106,10.378521059679809,24.0,0.184685,7021.069797754288,0.11897110939025879,2792.139673756541,0.13285542411599796,1.0,1.0,0.10704543711737907,0.07835117203069997,44532.50472025028,7.793728816499673,25.0,0,0,0,26,1167.5,37.1,35.1,0.4772727272727273,0.055970780933021255 +XT (tuned + ensemble),723.008487197069,1.6600552705618052,480.9309873883464,1.918436680526551,0.8984673764067701,0.8698563213577479,0.20350469181458591,0.1535920856593196,5917.229990900821,105.19348400260608,24.884615384615383,0.188425,702.5076073408127,1.3369704484939575,378.34316304246136,1.3734038445009942,1.0,1.0,0.1315762721206637,0.09230140454586463,3777.3315625688056,84.73782880976256,28.0,0,0,0,26,1154.8,28.9,35.7,0.4571678321678322,0.059958692919405074 +FASTAI (tuned),2663.3156125178702,0.7872952864720271,1626.2333617037298,0.9244876829128756,0.8865326975991655,0.8432447700175844,0.206866728565722,0.1387333836548531,20608.72361153851,52.86743788138416,25.53846153846154,0.18316,2361.516105532646,0.6897614002227783,1350.0449662692658,0.612778202858813,1.0,1.0,0.1372818526486348,0.08670903152513522,13001.178549428518,36.29076458115673,27.5,0,0,1,25,1140.3,29.4,31.8,0.4423076923076923,0.06820649009803559 +XT (tuned),723.008487197069,0.16853698400350717,480.9309873883464,0.21789332007450063,0.9275522820640911,0.8584728603742081,0.20621433350726176,0.1564465527864028,5917.229990900821,11.278027514695932,26.0,0.189205,702.5076073408127,0.14003515243530273,378.34316304246136,0.15364019245636173,1.0,1.0,0.1505192879837034,0.09210625451126611,3777.3315625688056,8.882695855316769,29.5,0,1,0,25,1128.1,31.0,25.7,0.4318181818181818,0.07221148701643333 +XGB (default),5.406896040989802,0.29052331814399135,3.2395402815762044,0.40571388124316665,0.967663807523489,0.9208621475953046,0.1913484639295636,0.1583121152662077,36.135643016538445,18.079059576557942,26.01923076923077,0.18918000000000001,4.358284831047058,0.2170184850692749,2.659926162500852,0.1702856674842303,1.0,1.0,0.13622845464672223,0.08590682453074228,32.63170368298432,10.069746654781287,24.25,0,0,0,26,1129.9,27.4,31.1,0.4313811188811189,0.04570122330728964 +RF (tuned + ensemble),911.769657428448,1.55073488675631,531.6502580092293,1.7081982422422,0.9428934399339288,0.9018568954164017,0.20973262545633878,0.18169828635432628,6577.080710276883,100.32600883678194,26.96153846153846,0.18825,798.9582680463791,1.2464795112609863,527.2734284753022,1.1554577826899797,1.0,1.0,0.15587829515575163,0.09511037806823772,5029.2193365439925,80.30745598154598,30.0,0,1,0,25,1114.2,27.3,33.7,0.40996503496503495,0.06740230643960834 +GBM (default),5.678787359824548,0.359198652780973,3.357556396858338,0.23593366680622332,0.9701181894525962,0.9244030054522838,0.2009750941665907,0.16096135302268902,43.778662055665606,16.224185451530264,27.557692307692307,0.19459,5.089803218841553,0.34710729122161865,3.0120719035705887,0.17665561056592852,1.0,1.0,0.14940090168048453,0.08991864451874558,30.743592100025186,14.007062707115537,28.0,0,0,0,26,1097.5,31.5,29.5,0.3964160839160839,0.04169946841702806 +RF (tuned),911.769657428448,0.15133640399345985,531.6502580092293,0.19063529883897276,0.9772506936041817,0.9414350435106434,0.21876669694055456,0.18885525418574936,6577.080710276883,9.67186030508054,29.25,0.18775,798.9582680463791,0.13221728801727295,527.2734284753022,0.1275233993660041,1.0,1.0,0.16269556435701765,0.11206528012392108,5029.2193365439925,7.908293435451007,31.5,0,0,0,26,1064.0,29.3,35.0,0.35795454545454547,0.04504401000608687 +NN_TORCH (default),23.867143649321335,0.24185629991384652,13.87033788200038,0.29581404152824126,0.9698948249425869,0.9457939759123197,0.24766534415358382,0.1805126846920292,159.99336582070887,16.359564345871572,31.71153846153846,0.19833499999999998,19.20789933204651,0.18236374855041504,8.931817801759234,0.17241159209016133,1.0,1.0,0.1806992032072448,0.1282818632772945,162.04458259531137,9.195674192084631,33.0,0,0,0,26,1005.9,38.8,30.0,0.30201048951048953,0.03748733782316879 +FASTAI (default),10.92040595641503,0.607048539014963,5.789474679574097,0.6364629667404095,0.9807465319111015,0.9340218157343317,0.2458147993858494,0.18433902398032298,78.41470283692,37.58331718018816,32.15384615384615,0.19146000000000002,9.578287720680237,0.6503585577011108,4.679521651364521,0.5473457672031852,1.0,1.0,0.15595107304948763,0.13206479856027808,61.09577848128312,35.985779724786894,35.0,0,0,0,26,999.9,29.7,31.1,0.291958041958042,0.036046519543766274 +RF (default),1.1224354964036207,0.07520443659562331,0.5160590091828214,0.08504536046248315,0.9951894125278877,0.9662672985507942,0.2733008673941071,0.2607556340607851,7.699488729035172,4.8652515281017985,32.21153846153846,0.213715,0.8405723571777344,0.06134629249572754,0.44070860935344924,0.06492877134601137,1.0,1.0,0.22304594727380544,0.1387641342747614,5.263965350091326,4.042169058695611,35.75,0,0,0,26,1000.0,0.0,0.0,0.2906468531468531,0.04063169893049472 +XT (default),0.7937821791722224,0.076584870998676,0.45161836217627827,0.08730195757590078,0.9851460583683391,0.9723807339815378,0.2928638810006597,0.2925631030122461,5.4850410746785805,4.998284751448414,34.11538461538461,0.22224,0.7121238708496094,0.07150018215179443,0.3597485800421042,0.06793435113493712,1.0,1.0,0.21784002712442435,0.1545266336174043,3.777771179368423,4.378376796908203,38.0,0,0,0,26,949.4,35.6,31.6,0.24737762237762237,0.034356084606725006 +LR (tuned + ensemble),175.34701893879816,0.4311328667860765,114.81546568819135,0.35152858974096185,0.9447815704031849,0.9613573808489063,0.31304194938194396,0.27486950659356457,1394.89696800973,28.50704727597215,34.21153846153846,0.219115,157.3445065021515,0.20991504192352295,89.11614089252916,0.1966045265631563,1.0,1.0,0.260779813420035,0.18354757952857997,1033.8951605639145,9.91761243845308,37.5,1,0,0,25,946.4,32.2,34.1,0.24519230769230768,0.06657835051950235 +LR (tuned),175.34701893879816,0.13578540545243484,114.81546568819135,0.11644774040693313,0.9645430999513104,0.9706124256069096,0.3197153495116481,0.2866149774984877,1394.89696800973,6.834352988282761,34.92307692307692,0.21990500000000002,157.3445065021515,0.06698489189147949,89.11614089252916,0.08340922333653941,1.0,1.0,0.2839064101424172,0.1973963943493853,1033.8951605639145,4.726037355894221,38.75,0,0,1,25,925.0,34.9,43.3,0.229020979020979,0.04041420009021431 +LR (default),4.7979896068573,0.14980417031508225,2.784461238483506,0.15310137710602748,0.9745259999639975,0.9812456463551247,0.3329427914389331,0.32581993219914024,36.357934158034695,9.005429323015887,35.98076923076923,0.227395,4.980341196060181,0.0988316535949707,2.4349667711867333,0.10027059217560387,1.0,1.0,0.283906387231373,0.21846589850078024,26.549049340848278,5.855361941404224,39.0,0,0,0,26,891.8,35.4,28.6,0.20498251748251747,0.030292514887996096 +KNN (tuned + ensemble),16.958975911140442,0.2818152537712684,7.194345160625263,0.20399826229470497,1.0,0.9816350265387093,0.47672042775227436,0.5833708185909443,56.76087640724927,13.193730328032911,39.92307692307692,0.297435,6.915013313293457,0.11628735065460205,3.5025132784274384,0.1592449370444793,1.0,1.0,0.4130987117462378,0.6739246116612398,52.78507887384432,10.328630925855668,43.0,0,0,0,26,745.5,44.9,30.5,0.11538461538461539,0.030206633151293592 +KNN (tuned),16.958975911140442,0.05680100734417255,7.194345160625263,0.04235176068048229,1.0,0.9774922422312161,0.5019983015456773,0.6443397962703555,56.76087640724927,2.623837177847182,41.21153846153846,0.301285,6.915013313293457,0.03549385070800781,3.5025132784274384,0.03322185201916636,1.0,1.0,0.5143946348760557,0.7015107115204589,52.78507887384432,1.9257403822827532,44.0,0,0,1,25,676.5,41.9,47.1,0.08610139860139861,0.035393141891706566 +KNN (default),0.30434812949253964,0.03149266426379864,0.14343197725298135,0.03163346219935531,1.0,1.0,0.5949762374279601,0.9484018378342699,1.002692426657382,1.6405188884998085,44.26923076923077,0.34604,0.17687928676605225,0.01871800422668457,0.0745054444441075,0.021103033819676938,1.0,1.0,0.6151111085811185,1.0,1.0,1.0188814221661033,45.0,0,0,0,26,383.7,60.2,115.9,0.016608391608391608,0.02263422903807308 diff --git a/data/lite/tabpfn-imputed-cls/time_plot.pdf b/data/lite/tabpfn-imputed-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..504ed6aada926990836ea314ce30d7d5c2830d59 Binary files /dev/null and b/data/lite/tabpfn-imputed-cls/time_plot.pdf differ diff --git a/data/lite/tabpfn-imputed-cls/time_plot.png.zip b/data/lite/tabpfn-imputed-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..cfe1e204c0910d73a2eba40482eaaf963e6658a2 --- /dev/null +++ b/data/lite/tabpfn-imputed-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b50180b966916e8d8fab5000ffbfabad56b04f113a38f14cf698237f8e18a7f0 +size 338023 diff --git a/data/lite/tabpfn-imputed-cls/tuning-impact-elo-horizontal.pdf b/data/lite/tabpfn-imputed-cls/tuning-impact-elo-horizontal.pdf new file mode 100644 index 0000000000000000000000000000000000000000..dc2e708417f19c16853f290c7e9e3545d486e4af Binary files /dev/null and b/data/lite/tabpfn-imputed-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabpfn-imputed-cls/tuning-impact-elo-horizontal.png.zip b/data/lite/tabpfn-imputed-cls/tuning-impact-elo-horizontal.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..47e71a24002ead01c3ea45dd3059e619577a41fc --- /dev/null +++ b/data/lite/tabpfn-imputed-cls/tuning-impact-elo-horizontal.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:832aab95761043ef7611a81366907b0a56b2084d7b362c2a4a359cbeb5561ceb +size 152023 diff --git a/data/lite/tabpfn-imputed-cls/tuning-impact-elo.pdf b/data/lite/tabpfn-imputed-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3d05b517cfd0e574df2382d94fc7f78260e609c1 Binary files /dev/null and b/data/lite/tabpfn-imputed-cls/tuning-impact-elo.pdf differ diff --git a/data/lite/tabpfn-imputed-cls/tuning-impact-elo.png.zip b/data/lite/tabpfn-imputed-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4971df0aa196df6e26c029ebb2f818aa79757de3 --- /dev/null +++ b/data/lite/tabpfn-imputed-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:628561cc9f0b51afd9b10bf6fa6e41bbfee70972e9de5047a420b81996beb10b +size 146982 diff --git a/data/lite/tabpfn-imputed-reg/figures/critical-diagram.pdf b/data/lite/tabpfn-imputed-reg/figures/critical-diagram.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a2aa05956645c126dbbd44c044d9855816cd4265 Binary files /dev/null and b/data/lite/tabpfn-imputed-reg/figures/critical-diagram.pdf differ diff --git a/data/lite/tabpfn-imputed-reg/figures/critical-diagram.png.zip b/data/lite/tabpfn-imputed-reg/figures/critical-diagram.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..88b04556f39c27ffacef706e26258a851bdaf50f --- /dev/null +++ b/data/lite/tabpfn-imputed-reg/figures/critical-diagram.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e49213359d35d743124226f0f33dfa6af615161504709e8c640033d782ad76a5 +size 317500 diff --git a/data/lite/tabpfn-imputed-reg/leaderboard.tex b/data/lite/tabpfn-imputed-reg/leaderboard.tex new file mode 100644 index 0000000000000000000000000000000000000000..560217472bca4a02ceae781898e1cfcb21d8df7a --- /dev/null +++ b/data/lite/tabpfn-imputed-reg/leaderboard.tex @@ -0,0 +1,53 @@ +\begin{tabular}{llcccccrr} +\toprule +\textbf{Model} & \textbf{Elo ($\uparrow$)} & \textbf{Norm.} & \textbf{Avg.} & \textbf{Harm.} & \textbf{\#wins ($\uparrow$)} & \textbf{Improva-} & \textbf{Train time} & \textbf{Predict time} \\ + & & \textbf{score ($\uparrow$)} & \textbf{rank ($\downarrow$)} & \textbf{mean} & & \textbf{bility ($\downarrow$)} & \textbf{per 1K [s]} & \textbf{per 1K [s]} \\ + & & & & \textbf{rank ($\downarrow$)} & & & & \\ +\midrule +RealMLP (T+E) & \textcolor{gold}{\textbf{1754${}_{-93,+92}$}} & \textcolor{silver}{\textbf{0.758}} & \textcolor{silver}{\textbf{6.6}} & 5.1 & 0 & \textcolor{bronze}{\textbf{3.9\%}} & 7718.29 & 4.52 \\ +AutoGluon 1.3 (4h) & \textcolor{silver}{\textbf{1753${}_{-54,+96}$}} & 0.691 & \textcolor{gold}{\textbf{6.3}} & 4.3 & 0 & 4.9\% & 4226.02 & 5.16 \\ +TabPFNv2 (T+E) & \textcolor{bronze}{\textbf{1747${}_{-91,+88}$}} & \textcolor{silver}{\textbf{0.758}} & \textcolor{silver}{\textbf{6.6}} & \textcolor{gold}{\textbf{1.4}} & \textcolor{gold}{\textbf{5}} & \textcolor{gold}{\textbf{0.7\%}} & 6601.21 & 73.56 \\ +TabDPT (D) & 1694${}_{-77,+72}$ & \textcolor{gold}{\textbf{0.787}} & 8.1 & \textcolor{bronze}{\textbf{4.0}} & \textcolor{silver}{\textbf{1}} & 4.5\% & 36.28 & 33.43 \\ +CatBoost (T+E) & 1651${}_{-68,+68}$ & 0.605 & 9.4 & 8.6 & 0 & 6.3\% & 3301.07 & 1.32 \\ +LightGBM (T+E) & 1621${}_{-64,+72}$ & 0.515 & 10.1 & 8.7 & 0 & 7.5\% & 997.12 & 3.79 \\ +CatBoost (T) & 1606${}_{-68,+89}$ & 0.572 & 10.9 & 6.1 & 0 & 6.4\% & 3301.07 & 0.13 \\ +TabPFNv2 (T) & 1591${}_{-82,+98}$ & 0.524 & 11.3 & \textcolor{silver}{\textbf{3.9}} & 0 & \textcolor{silver}{\textbf{3.6\%}} & 6601.21 & 0.59 \\ +TabM (T+E) & 1548${}_{-65,+78}$ & 0.487 & 12.6 & 9.5 & 0 & 4.8\% & 4228.53 & 1.45 \\ +ModernNCA (T+E) & 1537${}_{-72,+84}$ & 0.371 & 13.3 & 6.2 & 0 & 9.4\% & 9360.97 & 7.66 \\ +XGBoost (T) & 1505${}_{-66,+65}$ & 0.407 & 14.4 & 13.4 & 0 & 8.0\% & 911.22 & 0.53 \\ +XGBoost (T+E) & 1492${}_{-65,+75}$ & 0.453 & 14.9 & 13.9 & 0 & 8.0\% & 911.22 & 3.53 \\ +TabPFNv2 (D) & 1486${}_{-72,+94}$ & 0.424 & 15.4 & \textcolor{bronze}{\textbf{4.0}} & 0 & 5.7\% & 8.97 & 0.84 \\ +LightGBM (T) & 1467${}_{-60,+72}$ & 0.381 & 16.0 & 14.0 & 0 & 8.8\% & 997.12 & 0.64 \\ +TabM (T) & 1467${}_{-68,+77}$ & 0.361 & 16.0 & 12.5 & 0 & 5.6\% & 4228.53 & 0.17 \\ +CatBoost (D) & 1458${}_{-71,+73}$ & 0.376 & 16.3 & 10.9 & 0 & 9.7\% & 9.39 & 0.10 \\ +RealMLP (T) & 1451${}_{-67,+84}$ & 0.295 & 16.6 & 14.9 & 0 & 6.6\% & 7718.29 & 0.42 \\ +ModernNCA (D) & 1437${}_{-75,+87}$ & 0.061 & 17.3 & 13.7 & 0 & 11.7\% & 27.26 & 0.29 \\ +ExtraTrees (T+E) & 1416${}_{-91,+68}$ & 0.241 & 18.0 & 11.4 & 0 & 13.2\% & 613.95 & 0.95 \\ +ExtraTrees (T) & 1406${}_{-66,+64}$ & 0.210 & 18.3 & 10.4 & 0 & 13.5\% & 613.95 & 0.13 \\ +RealMLP (D) & 1362${}_{-72,+71}$ & 0.157 & 20.3 & 12.7 & 0 & 8.5\% & 41.85 & 0.30 \\ +TabM (D) & 1348${}_{-63,+62}$ & 0.319 & 20.7 & 17.3 & 0 & 7.7\% & 17.97 & 0.15 \\ +TorchMLP (T+E) & 1319${}_{-67,+63}$ & 0.136 & 22.0 & 20.4 & 0 & 9.6\% & 5937.36 & 1.41 \\ +ModernNCA (T) & 1270${}_{-79,+63}$ & 0.125 & 24.0 & 21.3 & 0 & 12.7\% & 9360.97 & 0.52 \\ +TorchMLP (T) & 1250${}_{-81,+72}$ & 0.139 & 25.0 & 21.1 & 0 & 10.0\% & 5937.36 & 0.12 \\ +RandomForest (T+E) & 1246${}_{-76,+68}$ & 0.122 & 25.0 & 21.4 & 0 & 15.2\% & 628.51 & 0.81 \\ +EBM (T+E) & 1219${}_{-91,+82}$ & 0.186 & 25.9 & 12.8 & 0 & 15.2\% & 2317.66 & 0.14 \\ +ExtraTrees (D) & 1191${}_{-69,+57}$ & 0.085 & 27.0 & 23.8 & 0 & 15.3\% & 0.43 & 0.12 \\ +LightGBM (D) & 1188${}_{-69,+61}$ & 0.016 & 27.1 & 26.7 & 0 & 12.3\% & 4.41 & 0.36 \\ +RandomForest (T) & 1175${}_{-58,+87}$ & 0.087 & 27.6 & 24.1 & 0 & 16.3\% & 628.51 & 0.15 \\ +FastaiMLP (T+E) & 1166${}_{-65,+75}$ & 0.000 & 28.0 & 27.6 & 0 & 14.2\% & 2163.39 & 7.62 \\ +EBM (T) & 1155${}_{-93,+81}$ & 0.151 & 28.3 & 5.9 & \textcolor{silver}{\textbf{1}} & 16.2\% & 2317.66 & 0.02 \\ +XGBoost (D) & 1120${}_{-82,+79}$ & 0.000 & 29.4 & 28.5 & 0 & 13.5\% & 3.97 & 0.24 \\ +FastaiMLP (T) & 1109${}_{-60,+98}$ & 0.000 & 29.7 & 29.2 & 0 & 14.7\% & 2163.39 & 0.71 \\ +EBM (D) & 1072${}_{-83,+88}$ & 0.074 & 31.1 & 28.8 & 0 & 16.9\% & 9.65 & 0.06 \\ +TorchMLP (D) & 1048${}_{-73,+96}$ & 0.010 & 31.7 & 29.4 & 0 & 14.2\% & 20.50 & 0.13 \\ +RandomForest (D) & 1000${}_{-0,+0}$ & 0.000 & 33.1 & 32.2 & 0 & 18.0\% & 0.67 & 0.12 \\ +TabICL (D) & 996${}_{-65,+81}$ & 0.000 & 33.1 & 32.2 & 0 & 18.0\% & 0.67 & 0.12 \\ +FastaiMLP (D) & 982${}_{-79,+82}$ & 0.000 & 33.6 & 33.1 & 0 & 19.9\% & 6.88 & 0.45 \\ +Linear (T) & 440${}_{-160,+118}$ & 0.000 & 41.3 & 41.2 & 0 & 35.4\% & 154.23 & 0.08 \\ +Linear (T+E) & 440${}_{-129,+114}$ & 0.000 & 41.2 & 41.2 & 0 & 35.4\% & 154.23 & 0.16 \\ +KNN (T+E) & 363${}_{-154,+127}$ & 0.000 & 41.9 & 41.8 & 0 & 41.0\% & 2.26 & 0.15 \\ +Linear (D) & 278${}_{-138,+143}$ & 0.000 & 42.5 & 42.4 & 0 & 37.2\% & 3.09 & 0.09 \\ +KNN (T) & 243${}_{-137,+136}$ & 0.000 & 42.7 & 42.7 & 0 & 41.6\% & 2.26 & 0.03 \\ +KNN (D) & -9${}_{-200,+184}$ & 0.000 & 44.4 & 44.4 & 0 & 45.3\% & 0.05 & 0.03 \\ +\bottomrule +\end{tabular} \ No newline at end of file diff --git a/data/lite/tabpfn-imputed-reg/tabarena_leaderboard.csv b/data/lite/tabpfn-imputed-reg/tabarena_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..43a6ce8bd9cf3532888f65ff17e7eb40907bafbf --- /dev/null +++ b/data/lite/tabpfn-imputed-reg/tabarena_leaderboard.csv @@ -0,0 +1,46 @@ +method,time_train_s,time_infer_s,time_train_s_per_1K,time_infer_s_per_1K,normalized-error,normalized-error-task,champ_delta,loss_rescaled,time_train_s_rescaled,time_infer_s_rescaled,rank,median_metric_error,median_time_train_s,median_time_infer_s,median_time_train_s_per_1K,median_time_infer_s_per_1K,median_normalized-error,median_normalized-error-task,median_champ_delta,median_loss_rescaled,median_time_train_s_rescaled,median_time_infer_s_rescaled,median_rank,rank=1_count,rank=2_count,rank=3_count,rank>3_count,elo,elo+,elo-,winrate,mrr +AutoGluon 1.3 (4h),6507.599508047104,9.172560078757149,4878.0794801313095,6.788904878407515,0.3085910704411772,0.3694003733668835,0.0486082452707653,0.04602529164641145,107683.46631145268,541.3270644129536,6.285714285714286,4.10013,6398.424601554871,2.726226568222046,4226.0222111993835,5.158970598131418,0.23133371369265832,0.28619237255944013,0.02087460906626326,0.03128273627544052,97437.24755684618,393.2261425771175,5.0,0,1,1,5,1753.1,95.8,53.3,0.8798701298701299,0.23509585652442794 +REALMLP (tuned + ensemble),17294.968933514185,3.506439651761736,7783.542231376436,5.804296668854893,0.24215826947341315,0.3865600665781051,0.038845120274675446,0.04229126594047559,213038.26238584044,318.83634333585195,6.571428571428571,4.1572,7049.241462230682,3.1589651107788086,7718.289864704748,4.516099027530876,0.1939333280527935,0.35437858599764244,0.02624438674143237,0.040913538766118675,158412.06608302437,260.1484504479807,6.0,0,0,2,5,1753.9,91.4,92.3,0.8733766233766234,0.1967032967032967 +TABPFNV2 (tuned + ensemble),13265.934764044625,116.34418719155448,6406.189740569957,86.56063966774829,0.2415939115786189,0.22925578920247455,0.007012905956065296,0.011864420363929354,177767.73115525587,6824.09057332562,6.571428571428571,3.91474,7449.2451322078705,31.52299928665161,6601.20821444942,73.56061810307183,0.06610997682360657,0.0,0.0,0.0,161815.97140652352,4111.6099760549805,1.0,5,0,0,2,1746.9,87.1,90.2,0.8733766233766234,0.7289285714285715 +TABDPT (default),58.233478920800344,21.44670605659485,36.772525520044425,34.985783087632136,0.21328360616453232,0.4083362888860263,0.044633371588008654,0.04493187524163665,864.7561387477256,2038.6761504521721,8.142857142857142,4.24006,45.445554971694946,18.623436212539673,36.28271871618091,33.43014439125232,0.18149452492987356,0.47549658590127725,0.01506955690923395,0.0493188827159149,780.9167218453919,1672.754347061846,7.0,1,0,0,6,1693.7,71.6,76.6,0.8376623376623377,0.24873194891991884 +CAT (tuned + ensemble),10219.439358098167,1.2273011548178536,6237.092384383964,1.3764643405546468,0.3954429862751284,0.515897173396756,0.06330676638407992,0.06060158351003724,151904.27519951403,99.66578971371487,9.428571428571429,4.34931,10386.55150604248,0.8227088451385498,3301.0682722746046,1.3169555135467037,0.36849553490903875,0.51898568789136,0.02223613591863527,0.06198224787320109,136008.98155550173,66.23708934234935,10.0,0,0,0,7,1650.6,67.6,67.2,0.8084415584415584,0.11581275866990152 +GBM (tuned + ensemble),1519.5935083457402,15.220876693725586,1075.2207780922756,8.071953392174887,0.48543215706848314,0.549854188587539,0.07463717866296053,0.06555042697877127,24050.082073960144,732.0577620840215,10.142857142857142,4.23184,1280.6412847042084,2.1158533096313477,997.1187061258496,3.7868055514991283,0.3027896779514977,0.497435935403833,0.021001080160577668,0.053508861901668286,21944.50900109299,305.1869734172427,8.0,0,0,0,7,1621.4,71.1,63.6,0.7922077922077922,0.1153986155865855 +CAT (tuned),10219.439358098167,0.20652290752955846,6237.092384383964,0.17520631671032216,0.42772839942623186,0.5470017074213716,0.06381219816795006,0.0719249798648832,151904.27519951403,14.407039225588733,10.857142857142858,4.33359,10386.55150604248,0.06416535377502441,3301.0682722746046,0.12532295659184456,0.5120020111687084,0.5133061965308594,0.03215961957864544,0.06130394854912813,136008.98155550173,8.436775680044018,10.0,0,1,0,6,1605.9,89.0,68.0,0.775974025974026,0.16386800334168755 +TABPFNV2 (tuned),13265.934764044625,6.892481701714652,6406.189740569957,3.565075900288693,0.4760165239054399,0.46529548351642175,0.03576915723256963,0.05812772216726451,177767.73115525587,371.838691600217,11.285714285714286,4.06636,7449.2451322078705,0.3009166717529297,6601.20821444942,0.5877278745174408,0.3837541852536315,0.2729549830348484,0.037284762407548744,0.02674626900933208,161815.97140652352,33.800294797494224,4.0,0,2,1,4,1591.2,97.5,81.9,0.7662337662337663,0.25358202235287886 +TABM (tuned + ensemble),10587.073945794788,1.1654579980032784,6469.982531368069,1.4219714334130276,0.5128718562532983,0.5583733644695366,0.04752740423249023,0.07247049692594254,159898.94570827132,89.13274144984146,12.571428571428571,4.37433,7112.7270860672,0.7802078723907471,4228.525932170142,1.4474226149502178,0.35916463746804045,0.5413070946293884,0.04867875413017375,0.06046439799308363,130257.5121002907,89.7509542969153,11.0,0,0,0,7,1547.9,77.7,64.4,0.737012987012987,0.1049611763897478 +MNCA (tuned + ensemble),11347.046411412102,5.150112356458392,8870.126225994847,7.01101400475912,0.629033079050954,0.5998500852863168,0.09420521732745024,0.08630788018224085,198709.11442978613,432.41908111150343,13.285714285714286,5.17584,9893.872790813446,3.8368866443634033,9360.97195459853,7.658456375974857,0.5415267425434339,0.7083182140911696,0.028945351266741137,0.04011651282919255,219316.22015007408,507.3929797993456,13.0,0,1,0,6,1537.1,84.0,71.6,0.7207792207792207,0.16165797422647987 +XGB (tuned),1964.623684678759,0.8363437311989921,1258.1348891579723,1.0162165524968663,0.5931661199323147,0.6958134106377959,0.07964264245312702,0.07605759147806708,29068.33225029261,76.75307718831554,14.428571428571429,4.27126,1646.6080114841461,0.44248437881469727,911.2186578282138,0.526588345238556,0.6620816436761664,0.6645140691864171,0.028796944630651078,0.066243759828395,28408.724850613697,33.71919899313387,14.0,0,0,0,7,1505.3,65.0,65.1,0.6948051948051948,0.0748299319727891 +XGB (tuned + ensemble),1964.623684678759,5.225094931466239,1258.1348891579723,6.231431906809619,0.5469310959609862,0.6925603174568474,0.080152008776187,0.07682133754513054,29068.33225029261,469.8791112336729,14.857142857142858,4.28931,1646.6080114841461,2.946662425994873,911.2186578282138,3.5316545373471357,0.5452119463463313,0.6991349623935156,0.026750868930337512,0.07131817083383857,28408.724850613697,165.90428541165622,15.0,0,0,0,7,1491.9,74.4,64.3,0.685064935064935,0.07177332471450118 +TABPFNV2 (default),10.536325999668666,0.6483856269291469,8.132158828132207,0.7522934493235269,0.5759929931993246,0.503203623724415,0.057203539607642365,0.09660218627950926,179.00806062175934,48.22880991521019,15.428571428571429,3.96968,9.203422546386719,0.44887351989746094,8.969435810057584,0.8365089530186938,0.4230930305474229,0.27795975146146573,0.05552265207828644,0.034900636525002166,193.52031139495512,52.107250060845345,3.0,0,2,2,3,1485.9,93.6,72.0,0.672077922077922,0.25235053370941873 +TABM (tuned),10587.073945794788,0.15967791421072824,6469.982531368069,0.17435700805504006,0.6392492711824916,0.6958983232461057,0.05563034885495726,0.08940263902914376,159898.94570827132,12.098100035047194,16.0,4.38922,7112.7270860672,0.1314229965209961,4228.525932170142,0.17298539846653752,0.4652267330424026,0.7342864421373895,0.05191534338451931,0.08006597714469382,130257.5121002907,9.015021272319185,15.0,0,0,0,7,1466.7,76.4,67.5,0.6590909090909091,0.08012166405023548 +GBM (tuned),1519.5935083457402,1.4627540792737688,1075.2207780922756,1.050105430857932,0.6186778290078211,0.7594090697512602,0.08826070516525375,0.09138656595801721,24050.082073960144,90.68047060933284,16.0,4.36733,1280.6412847042084,0.19193363189697266,997.1187061258496,0.6355418274734194,0.5552965190223861,0.7665861707914319,0.03499422009754971,0.07637313803793273,21944.50900109299,25.03430046335168,15.0,0,0,0,7,1467.3,71.1,59.6,0.6590909090909091,0.07125493554064984 +CAT (default),30.187910761151993,0.21431371143886022,21.23474421835824,0.22233607116297832,0.6237900244125646,0.7729777121066085,0.09689377152933412,0.09771959149576696,487.8800741332821,18.478250211890124,16.285714285714285,4.47999,43.05822467803955,0.04317522048950195,9.390118294059649,0.10484877571411665,0.5847419575275037,0.826178936979772,0.045915518799006794,0.11727889352322712,318.4902885443583,6.227518140238661,18.0,0,0,0,7,1458.3,72.8,70.8,0.6525974025974026,0.09164778191978547 +REALMLP (tuned),17294.968933514185,0.1830120086669922,7783.542231376436,0.3259796662543578,0.7047967586156892,0.7379416720239468,0.06611928626088935,0.08706588082885992,213038.26238584044,18.10152693439293,16.571428571428573,4.49659,7049.241462230682,0.18346905708312988,7718.289864704748,0.41665478856739646,0.7895709633169843,0.8504489620516743,0.03133979505568851,0.06965496351407507,158412.06608302437,16.898709432908465,17.0,0,0,0,7,1451.3,83.8,66.2,0.6461038961038961,0.06694456730060445 +MNCA (default),32.55938308579581,0.19486018589564733,24.957505841642334,0.25196370179633515,0.9394939857609429,0.7704421861569392,0.1167268604145739,0.1003795869866352,579.4685759941773,15.594763357138524,17.285714285714285,5.11399,27.96918511390686,0.14082908630371094,27.260414341039827,0.28976407064988385,1.0,0.8935735614971166,0.03164135806644719,0.10122870662762316,549.2641916961603,14.335706179717322,18.0,0,0,0,7,1437.2,87.0,74.3,0.6298701298701299,0.07283386960405594 +XT (tuned + ensemble),653.1345855508532,1.3570618970053536,577.6942803326062,1.4658800281842608,0.7590185126625372,0.7349105833120505,0.13182107507552596,0.10321816124521692,11988.325985095387,91.31150878008751,18.0,5.19051,619.4238421916962,1.1150686740875244,613.9510258467136,0.9522022685938181,1.0,0.8768784886512998,0.019723175236079915,0.0969336948409988,13486.685259241041,88.45578138045335,18.0,0,0,0,7,1416.5,67.6,90.6,0.6136363636363636,0.08802681992337165 +XT (tuned),653.1345855508532,0.19052549770900182,577.6942803326062,0.20596209279824323,0.7900820708578837,0.702913875464232,0.13531068870954055,0.10206565880892562,11988.325985095387,12.425140461101606,18.285714285714285,5.21222,619.4238421916962,0.12293267250061035,613.9510258467136,0.13470561454520408,1.0,0.8589430479942018,0.019404407055781392,0.07760478346112754,13486.685259241041,12.271457921927672,16.0,0,0,0,7,1406.5,63.5,65.6,0.6071428571428571,0.0957717275170248 +REALMLP (default),85.97223772321429,0.1575484275817871,39.12093352642527,0.2743856804074748,0.8426262409401332,0.8070277176824382,0.08492984112267965,0.09970894106924409,1070.0293170023945,15.38773372916076,20.285714285714285,4.72117,38.86532187461853,0.15464305877685547,41.84851132799618,0.30457759331800266,0.9775778014564183,1.0,0.05630051064737385,0.09045562976994424,871.512756220395,17.475355288117672,23.0,0,0,0,7,1362.5,70.7,71.6,0.5616883116883117,0.07846226237530586 +TABM (default),44.222292866025654,0.09508374759129115,33.52655463742946,0.14459166718033462,0.6807517607831785,0.7899466223209402,0.07746676233499121,0.11907088719148319,799.8890873088621,8.677317571690375,20.714285714285715,4.49975,33.95261740684509,0.09252333641052246,17.97321997410923,0.1450319163846654,0.6869868540300672,0.8550635603526185,0.05541802374036453,0.09871836929846278,614.4361778763692,5.7128774450352955,21.0,0,0,0,7,1348.3,61.9,63.0,0.551948051948052,0.05770731231560724 +NN_TORCH (tuned + ensemble),12457.980165379387,1.2596015930175781,7008.335221238185,1.51976815453829,0.8644212313341298,0.870894728824954,0.09635544815863366,0.13054872917279642,168590.23153378774,91.87300057791074,22.0,4.45415,7507.96986413002,0.7180907726287842,5937.361191915086,1.4061158013450725,1.0,0.9478676404035375,0.08724572872591152,0.09102294293315928,150677.82359954235,86.08980937276488,21.0,0,0,0,7,1318.7,62.1,66.7,0.5227272727272727,0.04909583398576822 +MNCA (tuned),11347.046411412102,0.3390509401048933,8870.126225994847,0.4537320246881022,0.8745416984197177,0.8793074065061627,0.12651248212292465,0.15226349694705085,198709.11442978613,28.22969633543519,24.0,5.75005,9893.872790813446,0.2758302688598633,9360.97195459853,0.5220087902191429,1.0,1.0,0.04948509220058683,0.10805583782920748,219316.22015007408,25.82571615254995,23.0,0,0,0,7,1270.1,62.3,78.6,0.4772727272727273,0.04697226740786727 +RF (tuned + ensemble),801.2955491883414,1.11563321522304,613.3607596500245,1.26624268506503,0.877625154030774,0.8743960460555773,0.15150038451410816,0.13535733856884374,13220.038476594822,74.24085143974197,25.0,5.35575,630.387446641922,0.864675760269165,628.5083029798404,0.811527051576754,1.0,1.0,0.04140035298054934,0.16236919097501226,13750.799241897716,73.76895458081503,29.0,0,0,0,7,1246.4,67.3,75.9,0.45454545454545453,0.04683025041397519 +NN_TORCH (tuned),12457.980165379387,0.08400774002075195,7008.335221238185,0.11080666811510796,0.8608314951688506,0.868559627738094,0.09957021763907885,0.15815621944343286,168590.23153378774,6.839931733747739,25.0,4.35043,7507.96986413002,0.06095480918884277,5937.361191915086,0.11905236169695854,1.0,1.0,0.09669761975708657,0.07439215953480095,150677.82359954235,6.913734822466806,26.0,0,0,0,7,1250.5,71.2,80.2,0.45454545454545453,0.04733778126635269 +EBM (tuned + ensemble),5561.30316393716,0.11157175472804479,3175.063314528853,0.13502700552021676,0.8141909618196094,0.8825163712446145,0.15236144498178752,0.1898206774797642,73928.60900997189,8.51769019764669,25.857142857142858,4.4992,4016.161784172058,0.0706477165222168,2317.663761142634,0.13798382133245468,1.0,1.0,0.11314433759685272,0.19494069658306334,53673.002392900955,8.013142594445496,31.0,0,0,1,6,1219.1,81.2,90.5,0.43506493506493504,0.07811527347987092 +XT (default),0.6613446303776332,0.07277822494506836,0.42786099680297607,0.10136865529358885,0.9145279857275386,0.9248915648270989,0.15302309586297927,0.14325676648751842,9.71399832220294,6.093015438403495,27.0,5.21222,0.43534398078918457,0.059967994689941406,0.42798109873088297,0.1173228956758976,1.0,1.0,0.07346763789813626,0.16206852516277276,9.783143310259103,5.904437603010231,30.5,0,0,0,7,1191.1,56.5,68.4,0.4090909090909091,0.04208640225911888 +GBM (default),7.295478275844029,0.6803168909890311,5.774057228808984,0.7873917849760669,0.9842293100404953,0.996574843691844,0.12276167672393257,0.1375507325867256,123.55828625698926,63.969416972987915,27.142857142857142,4.60997,5.699137926101685,0.33568358421325684,4.411330137552167,0.3631429914514132,1.0,1.0,0.05861977343818392,0.12652549891600978,89.74197938321082,41.091864073072564,26.0,0,0,0,7,1188.4,60.3,68.8,0.40584415584415584,0.03750052468539864 +RF (tuned),801.2955491883414,0.1265991755894252,613.3607596500245,0.187274000354846,0.9129471887632264,0.8964409965220048,0.16343200728271487,0.15127595329353374,13220.038476594822,10.741823760393732,27.571428571428573,5.50373,630.387446641922,0.11566352844238281,628.5083029798404,0.1492024419788353,1.0,1.0,0.05545969936477957,0.19024547228638602,13750.799241897716,12.51650764501577,34.0,0,0,0,7,1174.7,86.2,57.9,0.3961038961038961,0.04150326797385621 +FASTAI (tuned + ensemble),2255.801782812391,4.754738875797817,1874.5041683637385,7.02561873695556,1.0,0.9942470841751901,0.14237984020932382,0.17188628389751054,39821.805599866035,393.3305755649491,28.0,5.22937,2167.720710515976,4.366992950439453,2163.3939226706348,7.6216664328171975,1.0,1.0,0.09544576825940065,0.16368526688049675,44341.68565486565,365.3580675403427,28.0,0,0,0,7,1166.3,74.8,64.4,0.38636363636363635,0.036231433696871485 +EBM (tuned),5561.30316393716,0.015260968889508928,3175.063314528853,0.017210140514576817,0.8494479068291391,0.8443988855236891,0.16192837219447462,0.23515542970497189,73928.60900997189,1.0432270754270017,28.285714285714285,4.53788,4016.161784172058,0.00881648063659668,2317.663761142634,0.01721968874335289,1.0,1.0,0.11467541102152934,0.2156961514785443,53673.002392900955,1.0,36.0,1,0,0,6,1154.9,80.8,92.7,0.37987012987012986,0.17031659505343716 +XGB (default),5.3146242414202005,0.2511120523725237,3.9458596806008757,0.36393582726260965,1.0,0.9930075180483696,0.13461597663145097,0.17323981254075252,88.641308375137,25.06965036903791,29.428571428571427,5.03782,4.206391334533691,0.25308847427368164,3.970691708705885,0.24026284837936607,1.0,1.0,0.06048068144140273,0.13909193858299207,83.06864309112483,14.470859449961994,27.0,0,0,0,7,1119.9,78.1,81.1,0.3538961038961039,0.035063606271036614 +FASTAI (tuned),2255.801782812391,0.49124997002737864,1874.5041683637385,0.6913652874014505,1.0,1.0,0.14682535076785339,0.20225951117254543,39821.805599866035,40.61138754197303,29.714285714285715,5.20475,2167.720710515976,0.4167633056640625,2163.3939226706348,0.7125644494366172,1.0,1.0,0.11250145730924221,0.2028343782558529,44341.68565486565,41.0657519171911,30.0,0,0,0,7,1109.3,97.5,59.3,0.3474025974025974,0.03422813569872393 +EBM (default),13.620444434029716,0.04176497459411621,8.849664427822175,0.0725423696868711,0.9258962131926525,0.9841694566740207,0.16880716280386437,0.23207162894393898,199.98853409474154,3.960829194431375,31.142857142857142,4.59891,10.388706684112549,0.03829312324523926,9.65114625032283,0.0636073167690498,1.0,1.0,0.1212644465105922,0.22580812405457285,199.27951947649242,3.64412233970632,37.0,0,0,0,7,1072.4,87.7,82.6,0.31493506493506496,0.03472516258557449 +NN_TORCH (default),43.06239519800459,0.08627057075500488,31.428014747890156,0.11590329450401095,0.9903234435493359,0.9770876258639524,0.14240631859026293,0.22471185375084687,708.3643874980628,7.238187531162345,31.714285714285715,4.48918,41.744338512420654,0.06643533706665039,20.501721600246,0.12975651770830154,1.0,1.0,0.1279603352562162,0.12823638255534808,464.3796175172265,6.690027290059689,34.0,0,0,0,7,1047.9,95.4,72.6,0.30194805194805197,0.034025006017040844 +TABICL (default),1.6488038471766882,0.07487589972359794,0.7340231092118259,0.1055457863320713,1.0,1.0,0.17986721711507,0.20097685324182807,19.421218249481452,6.45143964546498,33.142857142857146,5.50373,0.5004377365112305,0.060329437255859375,0.6685572767188281,0.11783093214035034,1.0,1.0,0.10385640724520351,0.20419806328614365,11.221774322893834,5.842398233666076,35.5,0,0,0,7,996.5,80.1,65.0,0.2694805194805195,0.03106472997845159 +RF (default),1.6488038471766882,0.07487589972359794,0.7340231092118259,0.1055457863320713,1.0,1.0,0.17986721711507,0.20097685324182807,19.421218249481452,6.45143964546498,33.142857142857146,5.50373,0.5004377365112305,0.060329437255859375,0.6685572767188281,0.11783093214035034,1.0,1.0,0.10385640724520351,0.20419806328614365,11.221774322893834,5.842398233666076,35.5,0,0,0,7,1000.0,0.0,0.0,0.2694805194805195,0.03106472997845159 +FASTAI (default),8.895496198109218,0.4506887027195522,6.745758004706619,0.5305722397771327,1.0,1.0,0.1992892524760812,0.2546651622823661,149.59972016881048,35.25519751231355,33.57142857142857,6.27121,7.010618448257446,0.3927772045135498,6.882870030736257,0.45197373548430714,1.0,1.0,0.11103713173333196,0.30980814568049436,155.75215591877054,35.82998039822552,33.0,0,0,0,7,982.5,81.3,78.8,0.2597402597402597,0.03022907921306936 +LR (tuned + ensemble),151.49413922854833,0.12385766846793038,140.2291087325694,0.2034921921233621,1.0,1.0,0.3536775263323216,0.5794878220593526,2871.3444089692503,12.750390729258724,41.214285714285715,7.90301,151.6921091079712,0.07966971397399902,154.22778905270818,0.15536207236991023,1.0,1.0,0.27398044406077504,0.6617005914731261,3356.9082636752537,7.635360610075989,40.0,0,0,0,7,440.1,113.1,128.2,0.08603896103896104,0.024300860339548314 +LR (tuned),151.49413922854833,0.0357013429914202,140.2291087325694,0.05877661513349659,1.0,1.0,0.3541042830323525,0.5805259668922418,2871.3444089692503,3.4075401142248447,41.285714285714285,7.95008,151.6921091079712,0.03558516502380371,154.22778905270818,0.07586279315148999,1.0,1.0,0.274940797173291,0.6653082983995465,3356.9082636752537,3.222681902578225,41.5,0,0,0,7,440.4,117.1,159.5,0.08441558441558442,0.024279677697744622 +KNN (tuned + ensemble),4.365503992353167,0.1759951114654541,2.503089425947226,0.17631688214986382,1.0,1.0,0.4095786597069403,0.743043049481299,60.321121177196595,11.281300197479359,41.857142857142854,8.84074,2.312436103820801,0.1065065860748291,2.2580906778514502,0.14868705810425048,1.0,1.0,0.42992491292377444,0.8312484819939865,51.96556010372688,12.473272386123199,43.0,0,0,0,7,363.1,126.4,153.9,0.07142857142857142,0.023945718267277982 +LR (default),3.9944423266819546,0.07546016148158483,3.291819429311604,0.12482651745436109,1.0,1.0,0.371612807074338,0.658400915114039,71.50131231042234,8.781090197151686,42.5,8.07907,2.9657371044158936,0.0455625057220459,3.0870405547822,0.0889892689883709,1.0,1.0,0.27343206074359794,0.6653082514030221,67.40864758081219,3.589990624937666,42.0,0,0,0,7,277.7,142.2,137.5,0.056818181818181816,0.023567057345026364 +KNN (tuned),4.365503992353167,0.03152152470179966,2.503089425947226,0.031078100848287667,1.0,1.0,0.41647808374429435,0.7824790590876856,60.321121177196595,2.033314994156082,42.714285714285715,9.11406,2.312436103820801,0.014968395233154297,2.2580906778514502,0.02987703639352155,1.0,1.0,0.4343387863461754,0.8773703541691247,51.96556010372688,1.9337322466384677,43.0,0,0,0,7,242.8,135.7,137.0,0.05194805194805195,0.023440527141604852 +KNN (default),0.06725828988211495,0.02155137062072754,0.043712475995924135,0.02795156394158391,1.0,1.0,0.45340288690514835,0.9405484140280839,1.0,1.6852534934851076,44.42857142857143,9.84077,0.04459524154663086,0.014516353607177734,0.045943402958487325,0.029731415464968502,1.0,1.0,0.4698333732046267,1.0,1.0,1.6403093647746019,45.0,0,0,0,7,-8.9,183.8,199.7,0.012987012987012988,0.022521129663986807 diff --git a/data/lite/tabpfn-imputed-reg/time_plot.pdf b/data/lite/tabpfn-imputed-reg/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6d0755741a504ff46af4a7ffebce6ee9d87056e2 Binary files /dev/null and b/data/lite/tabpfn-imputed-reg/time_plot.pdf differ diff --git a/data/lite/tabpfn-imputed-reg/time_plot.png.zip b/data/lite/tabpfn-imputed-reg/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..c6a012b629b41257e26a055af4d8ce498797c174 --- /dev/null +++ b/data/lite/tabpfn-imputed-reg/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1950ba06e9623eedba5034901a69d9cc1f8ad53731de8a83636fbb1ec5befa3 +size 80035 diff --git a/data/lite/tabpfn-imputed-reg/tuning-impact-elo-horizontal.pdf b/data/lite/tabpfn-imputed-reg/tuning-impact-elo-horizontal.pdf new file mode 100644 index 0000000000000000000000000000000000000000..15c478d97adb57ebd1f93ec9ed71216433154fe3 Binary files /dev/null and b/data/lite/tabpfn-imputed-reg/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabpfn-imputed-reg/tuning-impact-elo-horizontal.png.zip b/data/lite/tabpfn-imputed-reg/tuning-impact-elo-horizontal.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..cc9fbebda2b8cb86823eac0c7ec40c32a6fe63fb --- /dev/null +++ b/data/lite/tabpfn-imputed-reg/tuning-impact-elo-horizontal.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c59a026a0fc86cf7ed5c81eda4a670c73a68974828abaf447c21e33e511ab87 +size 151116 diff --git a/data/lite/tabpfn-imputed-reg/tuning-impact-elo.pdf b/data/lite/tabpfn-imputed-reg/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b9412802baa9e80a200471abd9360a43dbf83b5c Binary files /dev/null and b/data/lite/tabpfn-imputed-reg/tuning-impact-elo.pdf differ diff --git a/data/lite/tabpfn-imputed-reg/tuning-impact-elo.png.zip b/data/lite/tabpfn-imputed-reg/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..34e4e6db05d6d32a9fe727bed6d5a6fceb4b19a5 --- /dev/null +++ b/data/lite/tabpfn-imputed-reg/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caed12add58a593b8ddb2ca70bb60959eb14d27f7a81d9ae61df5a0e4bcb731f +size 143614 diff --git a/data/lite/tabpfn-imputed/figures/critical-diagram.pdf b/data/lite/tabpfn-imputed/figures/critical-diagram.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8e955730650adee6d9d1e930e95c6810d0c57e1c Binary files /dev/null and b/data/lite/tabpfn-imputed/figures/critical-diagram.pdf differ diff --git a/data/lite/tabpfn-imputed/figures/critical-diagram.png.zip b/data/lite/tabpfn-imputed/figures/critical-diagram.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1ea6d908e2774d637a7879503d8d1fd0d6f18bf6 --- /dev/null +++ b/data/lite/tabpfn-imputed/figures/critical-diagram.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba7a2bbee95d32c851473dfc5398c0814c7c2283639e7100d67a2348de25cf1b +size 314086 diff --git a/data/lite/tabpfn-imputed/leaderboard.tex b/data/lite/tabpfn-imputed/leaderboard.tex new file mode 100644 index 0000000000000000000000000000000000000000..ef93c9d4d04646e0ebf4e3d8fe7e39f2991072a1 --- /dev/null +++ b/data/lite/tabpfn-imputed/leaderboard.tex @@ -0,0 +1,53 @@ +\begin{tabular}{llcccccrr} +\toprule +\textbf{Model} & \textbf{Elo ($\uparrow$)} & \textbf{Norm.} & \textbf{Avg.} & \textbf{Harm.} & \textbf{\#wins ($\uparrow$)} & \textbf{Improva-} & \textbf{Train time} & \textbf{Predict time} \\ + & & \textbf{score ($\uparrow$)} & \textbf{rank ($\downarrow$)} & \textbf{mean} & & \textbf{bility ($\downarrow$)} & \textbf{per 1K [s]} & \textbf{per 1K [s]} \\ + & & & & \textbf{rank ($\downarrow$)} & & & & \\ +\midrule +TabPFNv2 (T+E) & \textcolor{gold}{\textbf{1530${}_{-38,+30}$}} & \textcolor{gold}{\textbf{0.712}} & \textcolor{gold}{\textbf{8.1}} & \textcolor{gold}{\textbf{2.3}} & \textcolor{gold}{\textbf{11}} & \textcolor{gold}{\textbf{6.5\%}} & 3650.56 & 52.33 \\ +AutoGluon 1.3 (4h) & \textcolor{silver}{\textbf{1474${}_{-24,+32}$}} & \textcolor{bronze}{\textbf{0.493}} & \textcolor{silver}{\textbf{10.0}} & 5.8 & 1 & \textcolor{bronze}{\textbf{9.3\%}} & 2518.14 & 2.33 \\ +RealMLP (T+E) & \textcolor{bronze}{\textbf{1452${}_{-36,+31}$}} & 0.477 & \textcolor{bronze}{\textbf{10.9}} & 6.4 & 0 & 10.5\% & 6142.88 & 4.52 \\ +TabPFNv2 (T) & 1426${}_{-36,+28}$ & \textcolor{silver}{\textbf{0.540}} & 12.0 & \textcolor{bronze}{\textbf{4.4}} & 1 & \textcolor{silver}{\textbf{9.1\%}} & 3650.56 & 0.54 \\ +TabM (T+E) & 1410${}_{-31,+31}$ & 0.484 & 12.8 & 6.7 & 1 & 10.4\% & 3407.57 & 1.45 \\ +LightGBM (T+E) & 1388${}_{-30,+30}$ & 0.361 & 13.8 & 10.8 & 0 & 12.7\% & 780.12 & 2.13 \\ +CatBoost (T+E) & 1373${}_{-29,+32}$ & 0.342 & 14.4 & 9.5 & 0 & 12.0\% & 2234.83 & 0.65 \\ +TabPFNv2 (D) & 1349${}_{-35,+41}$ & 0.483 & 15.5 & \textcolor{silver}{\textbf{4.3}} & \textcolor{bronze}{\textbf{3}} & 10.7\% & 4.97 & 0.55 \\ +CatBoost (T) & 1346${}_{-34,+24}$ & 0.316 & 15.8 & 10.6 & 0 & 12.7\% & 2234.83 & 0.07 \\ +TabM (T) & 1344${}_{-28,+31}$ & 0.382 & 15.8 & 8.4 & 0 & 11.5\% & 3407.57 & 0.17 \\ +XGBoost (T+E) & 1336${}_{-32,+25}$ & 0.271 & 16.2 & 11.9 & 0 & 13.7\% & 848.99 & 1.87 \\ +CatBoost (D) & 1329${}_{-37,+26}$ & 0.259 & 16.7 & 9.9 & 1 & 13.7\% & 8.13 & 0.13 \\ +ModernNCA (T+E) & 1327${}_{-35,+27}$ & 0.320 & 16.7 & 8.4 & 0 & 13.4\% & 6308.39 & 7.74 \\ +XGBoost (T) & 1317${}_{-30,+31}$ & 0.229 & 17.1 & 10.3 & 1 & 13.9\% & 848.99 & 0.30 \\ +LightGBM (T) & 1314${}_{-30,+27}$ & 0.288 & 17.3 & 12.3 & 0 & 13.8\% & 780.12 & 0.38 \\ +TabICL (D) & 1307${}_{-30,+31}$ & 0.428 & 17.6 & 4.8 & \textcolor{silver}{\textbf{4}} & 12.1\% & 7.68 & 1.62 \\ +ModernNCA (T) & 1306${}_{-36,+29}$ & 0.228 & 17.7 & 7.5 & 2 & 12.7\% & 6308.39 & 0.48 \\ +TabM (D) & 1287${}_{-34,+24}$ & 0.295 & 18.7 & 9.1 & 1 & 14.4\% & 13.30 & 0.15 \\ +RealMLP (T) & 1262${}_{-28,+26}$ & 0.221 & 19.9 & 13.6 & 0 & 14.4\% & 6142.88 & 0.25 \\ +EBM (T+E) & 1260${}_{-26,+34}$ & 0.186 & 20.0 & 8.9 & 0 & 16.8\% & 1323.39 & 0.21 \\ +TorchMLP (T+E) & 1256${}_{-29,+29}$ & 0.186 & 20.3 & 13.8 & 0 & 14.4\% & 3460.09 & 2.06 \\ +TabDPT (D) & 1256${}_{-30,+30}$ & 0.370 & 20.2 & 6.1 & 2 & 15.5\% & 29.28 & 8.88 \\ +ModernNCA (D) & 1229${}_{-27,+27}$ & 0.157 & 21.7 & 10.4 & 1 & 17.7\% & 14.50 & 0.30 \\ +EBM (T) & 1214${}_{-25,+31}$ & 0.127 & 22.4 & 9.4 & 1 & 17.6\% & 1323.39 & 0.03 \\ +RealMLP (D) & 1210${}_{-36,+25}$ & 0.099 & 22.9 & 11.3 & 1 & 15.6\% & 35.65 & 0.30 \\ +ExtraTrees (T+E) & 1196${}_{-31,+32}$ & 0.131 & 23.4 & 15.2 & 0 & 18.8\% & 418.39 & 0.97 \\ +FastaiMLP (T+E) & 1196${}_{-35,+27}$ & 0.181 & 23.5 & 14.1 & 0 & 18.1\% & 1453.07 & 6.68 \\ +EBM (D) & 1182${}_{-28,+20}$ & 0.136 & 24.1 & 11.3 & 1 & 18.3\% & 5.90 & 0.07 \\ +TorchMLP (T) & 1181${}_{-33,+30}$ & 0.097 & 24.2 & 18.5 & 0 & 16.1\% & 3460.09 & 0.12 \\ +ExtraTrees (T) & 1178${}_{-34,+32}$ & 0.102 & 24.4 & 13.0 & 0 & 19.1\% & 418.39 & 0.15 \\ +FastaiMLP (T) & 1137${}_{-28,+24}$ & 0.089 & 26.4 & 16.4 & 0 & 19.4\% & 1453.07 & 0.64 \\ +RandomForest (T+E) & 1133${}_{-29,+31}$ & 0.071 & 26.5 & 15.9 & 0 & 19.7\% & 563.93 & 1.00 \\ +XGBoost (D) & 1127${}_{-32,+34}$ & 0.025 & 26.7 & 23.0 & 0 & 17.9\% & 2.86 & 0.19 \\ +LightGBM (D) & 1112${}_{-26,+29}$ & 0.027 & 27.5 & 24.5 & 0 & 18.4\% & 3.28 & 0.18 \\ +RandomForest (T) & 1082${}_{-33,+27}$ & 0.036 & 28.9 & 22.6 & 0 & 20.7\% & 563.93 & 0.15 \\ +TorchMLP (D) & 1021${}_{-34,+28}$ & 0.026 & 31.7 & 27.2 & 0 & 22.5\% & 9.39 & 0.15 \\ +FastaiMLP (D) & 1002${}_{-35,+30}$ & 0.015 & 32.5 & 28.7 & 0 & 23.6\% & 5.20 & 0.52 \\ +RandomForest (D) & 1000${}_{-0,+0}$ & 0.004 & 32.4 & 25.9 & 0 & 25.3\% & 0.48 & 0.07 \\ +ExtraTrees (D) & 995${}_{-30,+34}$ & 0.030 & 32.6 & 27.8 & 0 & 26.3\% & 0.38 & 0.07 \\ +Linear (T+E) & 911${}_{-28,+32}$ & 0.044 & 35.7 & 17.4 & 1 & 32.2\% & 97.93 & 0.19 \\ +Linear (T) & 892${}_{-40,+42}$ & 0.028 & 36.3 & 27.0 & 0 & 32.7\% & 97.93 & 0.08 \\ +Linear (D) & 853${}_{-42,+34}$ & 0.020 & 37.4 & 34.6 & 0 & 34.1\% & 2.75 & 0.09 \\ +KNN (T+E) & 730${}_{-53,+33}$ & 0.000 & 40.3 & 34.6 & 0 & 46.2\% & 3.13 & 0.16 \\ +KNN (T) & 656${}_{-42,+49}$ & 0.000 & 41.5 & 30.4 & 0 & 48.4\% & 3.13 & 0.03 \\ +KNN (D) & 360${}_{-80,+61}$ & 0.000 & 44.3 & 44.2 & 0 & 56.5\% & 0.06 & 0.02 \\ +\bottomrule +\end{tabular} \ No newline at end of file diff --git a/data/lite/tabpfn-imputed/tabarena_leaderboard.csv b/data/lite/tabpfn-imputed/tabarena_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..63885eb6f89c4268e420672e78cf4862278342a5 --- /dev/null +++ b/data/lite/tabpfn-imputed/tabarena_leaderboard.csv @@ -0,0 +1,46 @@ +method,time_train_s,time_infer_s,time_train_s_per_1K,time_infer_s_per_1K,normalized-error,normalized-error-task,champ_delta,loss_rescaled,time_train_s_rescaled,time_infer_s_rescaled,rank,median_metric_error,median_time_train_s,median_time_infer_s,median_time_train_s_per_1K,median_time_infer_s_per_1K,median_normalized-error,median_normalized-error-task,median_champ_delta,median_loss_rescaled,median_time_train_s_rescaled,median_time_infer_s_rescaled,median_rank,rank=1_count,rank=2_count,rank=3_count,rank>3_count,elo,elo+,elo-,winrate,mrr +TABPFNV2 (tuned + ensemble),14336.010728077455,156.01344153375337,4382.0533643641575,74.48134283766498,0.28759504252950974,0.34723611557382245,0.0647625412280981,0.04550285012853517,95784.56309893043,6432.006144478144,8.06060606060606,0.21046,7449.2451322078705,56.3117241859436,3650.5588495238276,52.332938254416526,0.16748308582785468,0.25336927223721006,0.02321606859178882,0.02238095238095369,71633.35015738624,4111.6099760549805,5.0,11,1,3,18,1530.5,29.3,37.4,0.8395316804407713,0.43675595283438423 +AutoGluon 1.3 (4h),5959.189094875798,4.367473082108931,3961.9186202053897,3.838659223481077,0.5066593479144164,0.5209744813478692,0.09260809941111314,0.05701051538580387,52768.640134241,245.21168201964974,10.015151515151516,0.22034,5402.2047798633575,2.42671799659729,2518.1361196041107,2.329453593213223,0.4597995521540763,0.47022286821705533,0.0452138062110049,0.048218303297502624,39027.82746427101,131.32895431780238,8.0,1,1,2,29,1473.5,31.1,23.4,0.7951101928374655,0.172136139426692 +REALMLP (tuned + ensemble),17735.155029759262,4.730990727742513,6433.060002371296,5.4487082681787244,0.5227695654892056,0.5309088596240968,0.1047743013999167,0.06389850118807634,135393.06860073804,314.1828415073118,10.93939393939394,0.21073,10195.217458248138,4.599701642990112,6142.876617706428,4.516099027530876,0.48215030896347355,0.503279028427124,0.058321654172537096,0.04108446298227332,111998.40741080203,285.1562934805996,9.0,0,1,4,28,1452.5,30.3,35.4,0.7741046831955923,0.15600492328388404 +TABPFNV2 (tuned),14336.010728077455,6.103090950936982,4382.0533643641575,3.0581431200087663,0.46046523778783577,0.523851638262936,0.09138992251275863,0.07202190717154246,95784.56309893043,243.34239180847368,12.045454545454545,0.21386,7449.2451322078705,0.4960660934448242,3650.5588495238276,0.5430449730566292,0.34176855083335494,0.5025334148392213,0.0565608520318478,0.0423413312816546,71633.35015738624,33.800294797494224,8.0,1,7,3,22,1425.8,27.8,35.5,0.7489669421487604,0.229310471535504 +TABM (tuned + ensemble),7905.59104330612,1.9562460870453806,4185.709531882815,1.768466932810432,0.5158505640872573,0.5545249182100271,0.10428311594026402,0.08083834836210321,67788.46010990962,114.61474290926614,12.787878787878787,0.21175,6126.394609212875,1.3961477279663086,3407.56780996161,1.4485538005828857,0.47020083976979304,0.5413070946293884,0.06402183372573667,0.04512415883221451,49987.707956643724,99.64653279572039,11.0,1,0,1,31,1409.8,30.5,30.2,0.7320936639118457,0.1495580112033242 +GBM (tuned + ensemble),1414.8606616944978,5.3146100116498545,902.1353750352665,3.8285745502016777,0.6392387450549837,0.6765170983738055,0.12720835322986757,0.0831034845877326,13042.108823335611,299.1337158047987,13.803030303030303,0.22026,1314.2295961380005,2.1158533096313477,780.1150358026446,2.128600501785187,0.6693216689945842,0.7196548678438859,0.06757639196359866,0.05353102776704557,13234.598330219944,119.30370400980756,14.0,0,0,1,32,1387.5,29.7,29.3,0.7090220385674931,0.09264791492585878 +CAT (tuned + ensemble),5998.228848081647,1.0770193735758464,3714.832915582639,1.0224930685169833,0.6581825114313259,0.644137864865451,0.11969773857589941,0.0844902359290059,53149.267800713766,63.12902616676368,14.424242424242424,0.21068,3471.3497779369354,0.9156572818756104,2234.828925740415,0.6529485519008803,0.6112565392764306,0.6219232105505488,0.05828476269775196,0.05387373518333481,22918.764355268395,48.89284806833719,13.0,0,1,1,31,1372.8,32.0,28.7,0.6949035812672176,0.10526893006761076 +TABPFNV2 (default),11.6943236914548,1.0926416714986165,5.91242680443585,0.666528048525937,0.5171946106727229,0.5855089777575702,0.10709558815209057,0.09082082645063125,96.73088882379422,50.146253518643945,15.515151515151516,0.21791,9.203422546386719,0.43659281730651855,4.965901903461194,0.5452976226806641,0.3939035196311629,0.7419962335216581,0.0629030449183201,0.06015293769345357,75.26381057033151,30.54843958112344,12.0,3,4,4,22,1348.9,40.9,34.9,0.6701101928374655,0.23045198475396717 +TABM (tuned),7905.59104330612,0.19297228437481503,4185.709531882815,0.20786967739368611,0.6183990992477728,0.6324746517808255,0.11489937932551171,0.09410734368351265,67788.46010990962,12.857810575794538,15.757575757575758,0.21145,6126.394609212875,0.16823339462280273,3407.56780996161,0.17298539846653752,0.5745991505979262,0.6495956873315449,0.07671356236690308,0.057665303848857825,49987.707956643724,9.039602142310635,14.0,0,2,2,29,1343.7,30.5,27.2,0.6646005509641874,0.11942373627360978 +CAT (tuned),5998.228848081647,0.1285553267507842,3714.832915582639,0.13007453601113816,0.6842728639559639,0.7005248537696941,0.1269477023863209,0.09236233550236878,53149.267800713766,8.587377880778849,15.848484848484848,0.21493,3471.3497779369354,0.06416535377502441,2234.828925740415,0.07115136708644841,0.6577077283695499,0.7192982456140343,0.07483269113770064,0.05609349712267564,22918.764355268395,3.6064895871728715,14.0,0,1,0,32,1346.5,23.3,33.6,0.662534435261708,0.09416350863456262 +XGB (tuned + ensemble),1888.6873327096303,3.046517429929791,1158.2361646061293,3.757011355737224,0.7293222800450723,0.7241659517861783,0.13685276008340994,0.09892711269143224,14952.255554140145,205.22200127699725,16.21212121212121,0.2153,1542.8871898651123,1.3964767456054688,848.994191986563,1.8656665541560775,0.7623744964555048,0.7071025284346675,0.07724634161829114,0.06320753111560178,12569.741812840497,126.36735681634848,14.0,0,0,2,31,1336.3,24.4,31.3,0.6542699724517906,0.08418728326273862 +CAT (default),142.80315683104774,0.2447956186352354,124.41934267197115,0.18478647674970256,0.740681350814587,0.755566033706957,0.1368629097660925,0.09694710597052561,500.26431087714354,13.226167413169538,16.651515151515152,0.20726,12.813451290130615,0.1487717628479004,8.126988439617271,0.12587527839504942,0.8332509641822385,0.8470451911935142,0.08897742363877803,0.08404761904761912,120.27803545982526,7.6030997550901045,19.0,1,0,0,32,1329.4,26.0,36.5,0.6442837465564738,0.10144331078330515 +MNCA (tuned + ensemble),11156.128839138782,10.252970204208836,6590.840949619391,8.639408754165267,0.679738245400772,0.6520206062253006,0.13381742098497723,0.10949804819564549,103946.26001990445,584.7921673215841,16.727272727272727,0.2215,10821.423727989197,7.313557863235474,6308.3910004173395,7.737494511886756,0.7185505017354659,0.6929026656356078,0.06424850472699273,0.0646978822615122,85886.71528316107,464.6138316092033,15.5,0,1,2,30,1326.8,26.2,34.2,0.6425619834710744,0.118346260233764 +XGB (tuned),1888.6873327096303,0.6773105679136334,1158.2361646061293,0.8819951020962133,0.7708905111580848,0.7517174597118881,0.138617629855467,0.10170140305610122,14952.255554140145,43.913270063701376,17.09090909090909,0.21833,1542.8871898651123,0.2559163570404053,848.994191986563,0.2979882044274592,0.8207592710887125,0.7400000000000069,0.0827897084298348,0.066243759828395,12569.741812840497,16.170469837088785,16.0,1,0,0,32,1317.4,30.7,29.6,0.6342975206611571,0.09664616069521167 +GBM (tuned),1414.8606616944978,0.8018729108752627,902.1353750352665,0.7467808129875216,0.7124399645711881,0.7596137265270978,0.13787531437464512,0.10052991714147216,13042.108823335611,52.68401353539273,17.28787878787879,0.22222,1314.2295961380005,0.2889828681945801,780.1150358026446,0.3811195826851199,0.745365207761696,0.7865756433465442,0.07753847811492465,0.0701999852719947,13234.598330219944,23.12721292618487,18.0,0,1,0,32,1313.5,27.0,30.0,0.6298209366391184,0.08143243732086801 +TABICL (default),17.322554508845013,2.0988141695658364,8.20776493038452,1.5609946258302967,0.5724404784680696,0.6499739390490956,0.12087347994096866,0.09235201314388092,115.09050362918282,109.37273861515632,17.560606060606062,0.20556,12.313645839691162,1.0587520599365234,7.682199329197478,1.615037441253662,0.5810932894452219,0.7571428571428533,0.05474453746925345,0.033505911727987066,91.21421124241789,96.97308672373808,16.0,4,2,1,26,1307.0,30.2,29.8,0.6236225895316805,0.20797339343885776 +MNCA (tuned),11156.128839138782,0.488406752095078,6590.840949619391,0.47560948312275564,0.7723596137566766,0.691301024072128,0.12654600378211453,0.1136603207570078,103946.26001990445,29.77527390400744,17.71212121212121,0.22855,10821.423727989197,0.40321969985961914,6308.3910004173395,0.476673807114561,0.8745571721847745,0.7663043478260984,0.07201319051941868,0.06463247386035899,85886.71528316107,25.730108223021155,16.0,2,0,0,31,1305.9,28.8,35.2,0.6201790633608816,0.13276545995571326 +TABM (default),30.478135679707382,0.1787743279428193,20.49279399878298,0.18760878299490344,0.7054228305321909,0.725878166346216,0.14426717758664306,0.11713424203181182,337.0087472221777,10.899849214261154,18.696969696969695,0.2188,24.11390233039856,0.13432693481445312,13.298290032000581,0.1461421244243193,0.6869868540300672,0.7907488986784105,0.08633751970306247,0.06957745058149634,137.0322141490484,7.861401175434497,18.0,1,0,1,31,1287.4,23.7,33.4,0.5977961432506887,0.10999219259274746 +REALMLP (tuned),17735.155029759262,0.22395390452760638,6433.060002371296,0.3029892240506847,0.779232820829222,0.790688494781442,0.14426731708421014,0.10647368333584784,135393.06860073804,16.18572290331018,19.87878787878788,0.22093,10195.217458248138,0.20874404907226562,6142.876617706428,0.2460546194468641,0.885134051632072,0.8847816803657822,0.09411587923715536,0.08023809523809645,111998.40741080203,13.170123583655588,18.0,0,1,0,32,1261.5,25.3,27.5,0.5709366391184573,0.07331981383170519 +EBM (tuned + ensemble),3438.1651705250597,0.30228347489328095,2025.9909898154654,0.278357915961119,0.8143931606450567,0.7673951195416302,0.16767911154685253,0.14497266971098918,28292.283296322228,19.073799904119216,20.0,0.21701,2006.7757859230042,0.22147798538208008,1323.3940540554784,0.21026261459440876,0.9460248644293228,0.966938614382516,0.08652042406281557,0.06365614798694313,20828.647605338916,11.089686715499157,22.0,0,3,1,29,1260.4,33.6,25.8,0.5681818181818182,0.1120936786611339 +TABDPT (default),69.51610838283192,21.59643456430146,33.15048249304137,28.806748991917754,0.629987494107437,0.675908044107149,0.15479114201912933,0.10502182765554564,588.2569465373858,1654.1673313074607,20.227272727272727,0.21757,65.70441889762878,21.056044578552246,29.27648859508967,8.8838207817589,0.7203280108909103,0.7839162896838312,0.09511484956324834,0.070909641670269,498.25427608514167,1247.4102886322382,19.0,2,3,0,28,1255.7,29.7,29.3,0.5630165289256198,0.16400036133199383 +NN_TORCH (tuned + ensemble),9157.856176080126,3.1086663116108286,4282.323362629224,2.77024720786038,0.8141011522100134,0.804576962019725,0.14449535531972263,0.10724336954825198,78848.62406573945,182.409192264201,20.333333333333332,0.21425,7053.366072177887,2.677239179611206,3460.094702877122,2.056839412206837,0.9757817342652233,0.9282133174880375,0.08543218616764037,0.07227319399821126,57294.450258986595,134.63673457491765,20.0,0,0,1,32,1256.2,28.2,28.1,0.5606060606060606,0.07262122082135955 +MNCA (default),30.355341976339165,0.40350833806124603,17.852429615073813,0.34235869991604173,0.8429271223076698,0.8116249610161627,0.17729230431221069,0.10749370348015701,270.4302626154547,22.061972158381582,21.651515151515152,0.23345,24.494569301605225,0.2764425277709961,14.495319221168756,0.3000060482138007,1.0,0.967654986522913,0.1256505576208179,0.09743889944848891,179.96393060938846,16.251218083714438,21.0,1,1,0,31,1229.4,26.3,26.8,0.5306473829201102,0.09579981503804881 +EBM (tuned),3438.1651705250597,0.03783920316985159,2025.9909898154654,0.037231939130763475,0.8732650510599177,0.8085804722367954,0.17641204955799628,0.1601141623618633,28292.283296322228,2.285569594265516,22.439393939393938,0.22761,2006.7757859230042,0.027222394943237305,1323.3940540554784,0.025386841881354122,1.0,1.0,0.0893728877206168,0.08235501040640528,20828.647605338916,1.1835017506360512,25.0,1,1,1,30,1214.4,30.3,24.8,0.5127410468319559,0.106866069184284 +REALMLP (default),105.41575461445433,0.23138419064608487,37.06590128185811,0.29773903605674285,0.9013016306032677,0.8217576371363557,0.1557433267057166,0.10754818864177114,791.8198164679417,15.953890267817545,22.924242424242426,0.22649,57.062673807144165,0.18715763092041016,35.649430762175584,0.30133283927465565,1.0,1.0,0.1063829787234043,0.09045562976994424,625.7008793991246,13.341950411904335,25.0,1,0,0,32,1210.1,24.6,35.7,0.5017217630853994,0.08823126636616184 +XT (tuned + ensemble),708.1867504842354,1.5957839488983154,501.4565343765227,1.8224398148781864,0.8688873144004177,0.841231467832903,0.1882990761426641,0.1429067077532978,7205.038232093607,102.24882259176883,23.424242424242426,0.22616,686.9523606300354,1.3263120651245117,418.38755533099174,0.9717198690479619,1.0,1.0,0.12222279775100575,0.0969336948409988,5299.380855419872,85.4887622756034,27.0,0,0,0,33,1196.1,31.5,30.2,0.4903581267217631,0.0659125380414586 +FASTAI (tuned + ensemble),2576.8732850045867,8.659171588493116,1678.8968661467622,9.647784447253477,0.8186439717731856,0.8324354310911509,0.18102927545150593,0.12647485682912454,24684.22585148677,537.1797900399015,23.53030303030303,0.22761,2283.895311355591,7.460584878921509,1453.0691261589527,6.677908072211324,1.0,1.0,0.09902421934517802,0.08681294440231768,19729.511337209304,439.3893190512624,27.0,0,0,0,33,1195.7,26.6,34.2,0.48794765840220383,0.07097111324361162 +EBM (default),11.796755761811227,0.06572751565413042,7.163006681498326,0.08347649829212658,0.8635835253158718,0.8477549929945992,0.1830374999527777,0.16150368556735065,102.62018767841327,4.629805865519655,24.12121212121212,0.21297,8.807492733001709,0.05507326126098633,5.903346415390749,0.07107973737946784,1.0,1.0,0.10385145665987439,0.07181719260065285,94.535974439501,3.64412233970632,25.0,1,0,0,32,1181.5,19.7,27.7,0.47451790633608815,0.08863853761833779 +NN_TORCH (tuned),9157.856176080126,0.15434888637427127,4282.323362629224,0.16049520877603024,0.9028056592953255,0.8478607297444255,0.16098971093479852,0.1266285348824375,78848.62406573945,9.627911202663915,24.21212121212121,0.22879,7053.366072177887,0.11448168754577637,3460.094702877122,0.12080316789438152,1.0,1.0,0.100147151360233,0.0767812248241212,57294.450258986595,7.364003194231025,26.0,0,0,0,33,1180.8,29.6,32.8,0.4724517906336088,0.0541395385794855 +XT (tuned),708.1867504842354,0.17320121418346057,501.4565343765227,0.21536245368256726,0.8983919342324714,0.8254754999387586,0.19117416642895727,0.14491121163966522,7205.038232093607,11.521354503327439,24.363636363636363,0.22797,686.9523606300354,0.13843560218811035,418.38755533099174,0.14728453101181402,1.0,1.0,0.14108852963031537,0.08719461637839616,5299.380855419872,9.92312806131416,29.0,0,1,0,32,1177.5,31.6,33.9,0.4690082644628099,0.07720911378928608 +FASTAI (tuned),2576.8732850045867,0.7244977951049805,1678.8968661467622,0.8750374778043916,0.9106015193205547,0.8764958794077937,0.19413067872981046,0.15220862282527267,24684.22585148677,50.26766962756969,26.424242424242426,0.24751,2283.895311355591,0.6439375877380371,1453.0691261589527,0.6368904113769531,1.0,1.0,0.11504721620397895,0.09909086376803719,19729.511337209304,38.24136055200847,29.0,0,0,1,32,1137.4,24.0,28.0,0.4221763085399449,0.06099896037696948 +RF (tuned + ensemble),888.3357556805466,1.4584405927947073,548.982788660307,1.6144500937500732,0.9290486520150778,0.8960318667641056,0.19738033192313834,0.17186838833922394,7986.192963738265,94.79279363134923,26.545454545454547,0.21825,787.3376877307892,1.127267837524414,563.9330272159958,0.9968510626638947,1.0,1.0,0.139739054041219,0.10570563621989555,6864.068149879297,78.90150886133937,30.0,0,1,0,32,1132.8,30.9,28.8,0.4194214876033058,0.06303853697962555 +XGB (default),5.387323235020493,0.2821633526773164,3.3893656086420445,0.3968518697927455,0.9745229998669913,0.9361657110247427,0.17931429995723666,0.16147859650626267,47.27320839563511,19.56191216890218,26.742424242424242,0.22354,4.284018039703369,0.21862435340881348,2.860024247198545,0.18955767154693604,1.0,1.0,0.13522420046121553,0.11358382514493719,45.34698317461044,11.901745335164652,26.0,0,0,0,33,1127.2,33.9,32.0,0.41494490358126723,0.043444759087478385 +GBM (default),6.021721796555952,0.4273146427038944,3.870147482423626,0.35290963126649316,0.9731114574560898,0.9397121832606754,0.1843843692545117,0.15599546383930282,60.70161264382517,26.35196183486977,27.46969696969697,0.2303,5.221571445465088,0.33568358421325684,3.2788508925910955,0.17879690442766463,1.0,1.0,0.14170832154325363,0.10597679216116598,53.27869689108153,15.293676047219583,28.0,0,0,0,33,1112.3,28.8,25.3,0.3984159779614325,0.04080878338304606 +RF (tuned),888.3357556805466,0.14608911311987674,548.982788660307,0.1899222961302186,0.963610556213676,0.9318908517251746,0.2070290354979825,0.18088388732982483,7986.192963738265,9.898822250146974,28.893939393939394,0.22167,787.3376877307892,0.12934398651123047,563.9330272159958,0.1492024419788353,1.0,1.0,0.1400874358486981,0.11247571843237328,6864.068149879297,8.186121069044912,32.0,0,0,0,33,1082.3,26.8,32.2,0.36604683195592286,0.04429294351440158 +NN_TORCH (default),27.938863674799602,0.20885326645591043,17.594693580825485,0.25765115579582876,0.9742281682834123,0.9524320228717569,0.2253376720643945,0.18988826600753597,276.3144916310567,14.424726839721131,31.71212121212121,0.22004,19.96923542022705,0.14620447158813477,9.388451067337588,0.15027597806179566,1.0,1.0,0.13014412363246453,0.12823638255534808,185.13016820561077,8.78781289003576,33.0,0,0,0,33,1020.7,27.2,33.6,0.30199724517906334,0.036752903803687106 +RF (default),1.2340893889918472,0.07513474695610278,0.5622938182798829,0.08939393564694124,0.9962098401734872,0.9734227200703227,0.2534816082440083,0.2480752866143397,10.185916203069231,5.20171567420914,32.40909090909091,0.23701,0.7245135307312012,0.060329437255859375,0.4841926739300568,0.0743608035126163,1.0,1.0,0.2227776088333242,0.1728884254431697,8.226093372880863,4.642861593290882,35.5,0,0,0,33,1000.0,0.0,0.0,0.28615702479338845,0.03860234188006132 +FASTAI (default),10.49087964404713,0.5738813010129061,5.992322657632511,0.6140012973845629,0.9848306008996557,0.948017188154322,0.23594574398074705,0.19925668968075635,93.5145549982301,37.08947361427536,32.45454545454545,0.23988,7.7834436893463135,0.5681214332580566,5.198076387326242,0.522444012045375,1.0,1.0,0.1527465876997327,0.13328152321740847,81.9890544250646,35.934326641384644,35.0,0,0,0,33,1001.5,30.0,34.2,0.28512396694214875,0.03481251704937602 +XT (default),0.765689365791552,0.07577740062366832,0.4465789210364869,0.09028580254631946,0.9701664672021088,0.9623072738578689,0.26320068415327297,0.2608920619312433,6.382092612032232,5.230500351711613,32.60606060606061,0.23857,0.6523852348327637,0.06471061706542969,0.3792952342205737,0.07494776627979297,1.0,1.0,0.1985320515377561,0.1548442777574803,6.583290644538605,4.555164209010363,37.0,0,0,0,33,994.8,33.4,29.9,0.28168044077134985,0.0359958489572328 +LR (tuned + ensemble),170.2873171820785,0.3659532792640455,120.20623845518064,0.3201269296402589,0.9564945706206911,0.9695543000627747,0.32166161721990294,0.3394855129044893,1708.0827888193253,25.164726190305664,35.696969696969695,0.24535,154.81117272377014,0.17154169082641602,97.92847529053688,0.18593118581206958,1.0,1.0,0.2711660718812774,0.2552683896620274,1268.2145563260167,9.250054132082282,40.0,1,0,0,32,910.8,31.4,27.6,0.21143250688705234,0.05761039805708786 +LR (tuned),170.2873171820785,0.1145554528091893,120.20623845518064,0.1042144714095375,0.972064260567699,0.9768461535084743,0.3270099717736157,0.3489597328244356,1708.0827888193253,6.107453287725022,36.27272727272727,0.25429,154.81117272377014,0.0533757209777832,97.92847529053688,0.07892496065012959,1.0,1.0,0.2768258727778956,0.2552683896620274,1268.2145563260167,4.226983992546795,40.5,0,0,1,32,891.9,41.9,39.9,0.19834710743801653,0.036991725643326805 +LR (default),4.627540183789803,0.13403422904737067,2.892082672901587,0.14710367960415885,0.9799295757292101,0.9852238425828255,0.34114552202826137,0.3963674134235127,43.81259012975329,8.95784223571136,37.36363636363637,0.25429,4.974438667297363,0.08619952201843262,2.750714412453488,0.09439525477373414,1.0,1.0,0.2768258727778956,0.31807101339614474,41.29590265237998,5.564972891186484,41.0,0,0,0,33,853.0,33.4,41.8,0.17355371900826447,0.02886590268191161 +KNN (tuned + ensemble),14.287633382912839,0.2593685569185199,6.199230307814771,0.19812645438519322,1.0,0.9855306269698921,0.4624782345305368,0.6172406857495044,57.516079843298705,12.788063330642764,40.333333333333336,0.34601,5.862637996673584,0.10712385177612305,3.133583813905716,0.15733957985747313,1.0,1.0,0.42992491292377444,0.7074394089684692,52.1575390239796,10.510779636451476,43.0,0,0,0,33,729.9,32.4,52.9,0.10606060606060606,0.028878560297108464 +KNN (tuned),14.287633382912839,0.05143869284427527,6.199230307814771,0.03996037829183494,1.0,0.9822666150912611,0.4838576492847778,0.6736420641406983,57.516079843298705,2.4985748964581607,41.53030303030303,0.34412,5.862637996673584,0.032462120056152344,3.133583813905716,0.030069828033447266,1.0,1.0,0.5139348134152102,0.7621498558446018,52.1575390239796,1.9337322466384677,44.0,0,0,1,32,655.9,48.7,41.1,0.07885674931129477,0.03285773876289711 +KNN (default),0.25405634533275256,0.029383905006177498,0.12227935577421162,0.030852453478009863,1.0,1.0,0.5649455267110001,0.9467359600572002,1.0021213058512708,1.6500080471330536,44.303030303030305,0.40785,0.12821459770202637,0.01713418960571289,0.06314403921930106,0.02475904764267444,1.0,1.0,0.5282115941198384,1.0,1.0,1.2284888285212276,45.0,0,0,0,33,359.7,60.5,79.2,0.01584022038567493,0.02261023826175175 diff --git a/data/lite/tabpfn-imputed/time_plot.pdf b/data/lite/tabpfn-imputed/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f2bdeb007d54c61b3417557d2f49bb5ec01325a4 Binary files /dev/null and b/data/lite/tabpfn-imputed/time_plot.pdf differ diff --git a/data/lite/tabpfn-imputed/time_plot.png.zip b/data/lite/tabpfn-imputed/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..6f0e3ccc91f02672563bb824aa7ccd079725c03d --- /dev/null +++ b/data/lite/tabpfn-imputed/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8281f988524a289b887d9a6c5120da78438cab8dacd60abf00a08318b3e5ccfd +size 338023 diff --git a/data/lite/tabpfn-imputed/tuning-impact-elo-horizontal.pdf b/data/lite/tabpfn-imputed/tuning-impact-elo-horizontal.pdf new file mode 100644 index 0000000000000000000000000000000000000000..af534c8a3a677a1a4d6d693302b33823eba1c10c Binary files /dev/null and b/data/lite/tabpfn-imputed/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabpfn-imputed/tuning-impact-elo-horizontal.png.zip b/data/lite/tabpfn-imputed/tuning-impact-elo-horizontal.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..8f8480ab54345da3876478b98a3f85d22684f638 --- /dev/null +++ b/data/lite/tabpfn-imputed/tuning-impact-elo-horizontal.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5b15ce390860be3922a4c9965b53a152d55606c54198204a9d37ec6671413be +size 152011 diff --git a/data/lite/tabpfn-imputed/tuning-impact-elo.pdf b/data/lite/tabpfn-imputed/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0c184bfed7508ecb4ee44af5675c3eb2c55ed9eb Binary files /dev/null and b/data/lite/tabpfn-imputed/tuning-impact-elo.pdf differ diff --git a/data/lite/tabpfn-imputed/tuning-impact-elo.png.zip b/data/lite/tabpfn-imputed/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4a2e750ccdffc68a87ad738e9317d03b03a7a5b1 --- /dev/null +++ b/data/lite/tabpfn-imputed/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9429891f99c30725dc400e81832c4a4ccebc3d49698b55f60565c62202b928b8 +size 147465 diff --git a/data/lite/tabpfn-reg/figures/critical-diagram.pdf b/data/lite/tabpfn-reg/figures/critical-diagram.pdf index 3c3dac348f15e3dfcd2bd8046b88af8a5763ba0d..a56c8cc992f9f2b5147c8acd65db56de2e213f46 100644 Binary files a/data/lite/tabpfn-reg/figures/critical-diagram.pdf and b/data/lite/tabpfn-reg/figures/critical-diagram.pdf differ diff --git a/data/lite/tabpfn-reg/figures/critical-diagram.png.zip b/data/lite/tabpfn-reg/figures/critical-diagram.png.zip index c65d44c30e148888564e1a80ab7144a04860a410..b0d9e0836f9bab75d6778bebb2c532c5d844beb3 100644 --- a/data/lite/tabpfn-reg/figures/critical-diagram.png.zip +++ b/data/lite/tabpfn-reg/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ec83f43ebb5f6213a4b3f92e0a5d5f645a9b348bda4f766515094c51a222a574 -size 310561 +oid sha256:231ca61939689d6c87e995fd992e1ea8225a316bc3802e701a4ff06c55b388bd +size 310699 diff --git a/data/lite/tabpfn-reg/time_plot.pdf b/data/lite/tabpfn-reg/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..395729549d74241274715e6686d0988e4a7cfc6e Binary files /dev/null and b/data/lite/tabpfn-reg/time_plot.pdf differ diff --git a/data/lite/tabpfn-reg/time_plot.png.zip b/data/lite/tabpfn-reg/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..32a79cc4f92d6d0a429cc3caf7b1761a896a1ded --- /dev/null +++ b/data/lite/tabpfn-reg/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08148d66d82223d159ed3f1a34a83aa808abf52750da638f3ab6fa2ba12f8da2 +size 80035 diff --git a/data/lite/tabpfn-reg/tuning-impact-elo-horizontal.pdf b/data/lite/tabpfn-reg/tuning-impact-elo-horizontal.pdf index 3409f93f92550558545a950f5d846c3ab73aaffc..8608821045098752d32d018f89372a680eded23c 100644 Binary files a/data/lite/tabpfn-reg/tuning-impact-elo-horizontal.pdf and b/data/lite/tabpfn-reg/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabpfn-reg/tuning-impact-elo-horizontal.png.zip b/data/lite/tabpfn-reg/tuning-impact-elo-horizontal.png.zip index 4c40ae41478b8e43d4fe239a012e7b8d51ea0421..916ef32f7480be82ba4f435e21e43fa21c3a0725 100644 --- a/data/lite/tabpfn-reg/tuning-impact-elo-horizontal.png.zip +++ b/data/lite/tabpfn-reg/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9859c5a16c8717d1573d1ed6129806a2416b285f78b12a19ecb04779cef94382 -size 135623 +oid sha256:abbde62f120caad595e6b12e606784d2e0d94109f2aedb72bd873b2232f8dbe8 +size 135785 diff --git a/data/lite/tabpfn-reg/tuning-impact-elo.pdf b/data/lite/tabpfn-reg/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..58811520cb28e2b74d2a9fddb473661c917e4e2d Binary files /dev/null and b/data/lite/tabpfn-reg/tuning-impact-elo.pdf differ diff --git a/data/lite/tabpfn-reg/tuning-impact-elo.png.zip b/data/lite/tabpfn-reg/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1536e6ec3411c98bbf2aa16074d9b0cbed547386 --- /dev/null +++ b/data/lite/tabpfn-reg/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b342957d4e046ad834d26acaa0f2d38a77c7287846e3c7148ab3eece5d7fe26f +size 133598 diff --git a/data/lite/tabpfn-tabicl-cls/figures/critical-diagram.pdf b/data/lite/tabpfn-tabicl-cls/figures/critical-diagram.pdf index 78193941da7724a2f31fcf9a4bdc0c1cdd8fda2b..0a460b6197162ba48ca46cc724de9f1e9fd2492c 100644 Binary files a/data/lite/tabpfn-tabicl-cls/figures/critical-diagram.pdf and b/data/lite/tabpfn-tabicl-cls/figures/critical-diagram.pdf differ diff --git a/data/lite/tabpfn-tabicl-cls/figures/critical-diagram.png.zip b/data/lite/tabpfn-tabicl-cls/figures/critical-diagram.png.zip index ee6c1a909c106e11df9c28c59d1ea06ad8d90858..ef97ddc5d0e7092e96060d85a9cdb1aafba9f627 100644 --- a/data/lite/tabpfn-tabicl-cls/figures/critical-diagram.png.zip +++ b/data/lite/tabpfn-tabicl-cls/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5a5e1e5d299642f413424cfe6d5802a988af48d82649e3847e843cf9179eb4ef -size 319000 +oid sha256:0684c936cc8fe278ae317a428c77274baec79dc8cb975b96cbdafa33117988dd +size 319138 diff --git a/data/lite/tabpfn-tabicl-cls/time_plot.pdf b/data/lite/tabpfn-tabicl-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..56b308f0a4c1312bfad7954c7e9df0752c3ae43a Binary files /dev/null and b/data/lite/tabpfn-tabicl-cls/time_plot.pdf differ diff --git a/data/lite/tabpfn-tabicl-cls/time_plot.png.zip b/data/lite/tabpfn-tabicl-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a12b2d3d30e2094105e12f177ddf882ce0713bf5 --- /dev/null +++ b/data/lite/tabpfn-tabicl-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aef006eca6f89d03785fa1c3d2efa7326030156589e07834aa1ed45c17af01b8 +size 338023 diff --git a/data/lite/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.pdf b/data/lite/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.pdf index 56df749f4b660ba4f866442ad45f444cc674ff6e..063c7b7a766080e15d02ce9e5d0d89631f707be1 100644 Binary files a/data/lite/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.pdf and b/data/lite/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.png.zip b/data/lite/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.png.zip index 11f6de74fa20ca51951d5e65c8838b9e75acc2bf..700269105248f09ca1f0291e7ef01ce7390a91d3 100644 --- a/data/lite/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.png.zip +++ b/data/lite/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4e3537446452cbef6ee98d645ac7ca084376854b8328e0703c07f9a66eb7f233 -size 138602 +oid sha256:12b7adee61e7a48e76a7f0ccfb22ce4df78aebb776fcb089cbc5dcbf98513fe2 +size 138764 diff --git a/data/lite/tabpfn-tabicl-cls/tuning-impact-elo.pdf b/data/lite/tabpfn-tabicl-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..4b1f1c5908216a4bbe9f6eb3ef5e6cbe3d893acf Binary files /dev/null and b/data/lite/tabpfn-tabicl-cls/tuning-impact-elo.pdf differ diff --git a/data/lite/tabpfn-tabicl-cls/tuning-impact-elo.png.zip b/data/lite/tabpfn-tabicl-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..43e67568bbec44d5e55c81b3612d3038a82fb684 --- /dev/null +++ b/data/lite/tabpfn-tabicl-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18317c34623e99764742e33a353a178c9b9c73b9b971c3d0f8072a6979e3f358 +size 128232 diff --git a/data/lite/tabpfn-tabicl-imputed-cls/figures/critical-diagram.pdf b/data/lite/tabpfn-tabicl-imputed-cls/figures/critical-diagram.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6463b0571a2481634766c992408ba49557572972 Binary files /dev/null and b/data/lite/tabpfn-tabicl-imputed-cls/figures/critical-diagram.pdf differ diff --git a/data/lite/tabpfn-tabicl-imputed-cls/figures/critical-diagram.png.zip b/data/lite/tabpfn-tabicl-imputed-cls/figures/critical-diagram.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..2532d942d129006f2b97ca0c2137cd0c9d902073 --- /dev/null +++ b/data/lite/tabpfn-tabicl-imputed-cls/figures/critical-diagram.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e6520243c13b51800124700fba0f568e706d0bd9ab7882847b81038413a966b +size 319138 diff --git a/data/lite/tabpfn-tabicl-imputed-cls/leaderboard.tex b/data/lite/tabpfn-tabicl-imputed-cls/leaderboard.tex new file mode 100644 index 0000000000000000000000000000000000000000..470e5847143808188c07aa37c1d5e5b76d097272 --- /dev/null +++ b/data/lite/tabpfn-tabicl-imputed-cls/leaderboard.tex @@ -0,0 +1,53 @@ +\begin{tabular}{llcccccrr} +\toprule +\textbf{Model} & \textbf{Elo ($\uparrow$)} & \textbf{Norm.} & \textbf{Avg.} & \textbf{Harm.} & \textbf{\#wins ($\uparrow$)} & \textbf{Improva-} & \textbf{Train time} & \textbf{Predict time} \\ + & & \textbf{score ($\uparrow$)} & \textbf{rank ($\downarrow$)} & \textbf{mean} & & \textbf{bility ($\downarrow$)} & \textbf{per 1K [s]} & \textbf{per 1K [s]} \\ + & & & & \textbf{rank ($\downarrow$)} & & & & \\ +\midrule +TabPFNv2 (T+E) & \textcolor{gold}{\textbf{1507${}_{-45,+32}$}} & \textcolor{gold}{\textbf{0.700}} & \textcolor{gold}{\textbf{8.5}} & \textcolor{gold}{\textbf{2.8}} & \textcolor{gold}{\textbf{6}} & \textcolor{gold}{\textbf{8.0\%}} & 3361.32 & 51.67 \\ +AutoGluon 1.3 (4h) & \textcolor{silver}{\textbf{1434${}_{-31,+33}$}} & 0.440 & \textcolor{silver}{\textbf{11.0}} & 6.4 & 1 & \textcolor{silver}{\textbf{10.4\%}} & 2085.73 & 2.16 \\ +RealMLP (T+E) & \textcolor{bronze}{\textbf{1408${}_{-30,+28}$}} & 0.402 & \textcolor{bronze}{\textbf{12.1}} & 6.9 & 0 & 12.3\% & 5575.69 & 4.58 \\ +TabPFNv2 (T) & 1402${}_{-33,+32}$ & \textcolor{silver}{\textbf{0.544}} & 12.2 & \textcolor{bronze}{\textbf{4.5}} & 1 & 10.6\% & 3361.32 & 0.53 \\ +TabM (T+E) & 1392${}_{-40,+33}$ & 0.483 & 12.8 & 6.2 & 1 & 12.0\% & 2899.02 & 1.46 \\ +TabICL (D) & 1378${}_{-31,+37}$ & \textcolor{bronze}{\textbf{0.543}} & 13.4 & \textcolor{silver}{\textbf{3.9}} & \textcolor{silver}{\textbf{4}} & \textcolor{bronze}{\textbf{10.5\%}} & 9.07 & 2.01 \\ +LightGBM (T+E) & 1349${}_{-33,+29}$ & 0.319 & 14.8 & 11.6 & 0 & 14.1\% & 691.46 & 1.46 \\ +TabPFNv2 (D) & 1334${}_{-35,+35}$ & 0.499 & 15.5 & \textcolor{bronze}{\textbf{4.5}} & \textcolor{bronze}{\textbf{3}} & 12.1\% & 4.28 & 0.51 \\ +CatBoost (T+E) & 1332${}_{-36,+33}$ & 0.271 & 15.8 & 9.8 & 0 & 13.5\% & 1394.45 & 0.62 \\ +TabM (T) & 1330${}_{-31,+33}$ & 0.387 & 15.7 & 7.7 & 0 & 13.1\% & 2899.02 & 0.17 \\ +ModernNCA (T) & 1326${}_{-31,+31}$ & 0.255 & 16.0 & 6.4 & 2 & 12.7\% & 6020.89 & 0.45 \\ +XGBoost (T+E) & 1314${}_{-33,+29}$ & 0.222 & 16.6 & 11.4 & 0 & 15.2\% & 785.90 & 1.28 \\ +CatBoost (D) & 1308${}_{-32,+30}$ & 0.228 & 16.8 & 9.6 & 1 & 14.8\% & 5.74 & 0.14 \\ +CatBoost (T) & 1299${}_{-28,+42}$ & 0.247 & 17.2 & 13.3 & 0 & 14.4\% & 1394.45 & 0.05 \\ +ModernNCA (T+E) & 1292${}_{-35,+27}$ & 0.307 & 17.7 & 9.4 & 0 & 14.4\% & 6020.89 & 8.48 \\ +XGBoost (T) & 1290${}_{-26,+30}$ & 0.181 & 17.8 & 9.8 & 1 & 15.4\% & 785.90 & 0.21 \\ +LightGBM (T) & 1288${}_{-29,+30}$ & 0.262 & 17.6 & 11.9 & 0 & 15.1\% & 691.46 & 0.26 \\ +TabM (D) & 1281${}_{-25,+30}$ & 0.288 & 18.2 & 8.1 & 1 & 16.2\% & 11.60 & 0.15 \\ +EBM (T+E) & 1277${}_{-29,+37}$ & 0.186 & 18.4 & 8.2 & 0 & 17.2\% & 1142.31 & 0.24 \\ +TorchMLP (T+E) & 1247${}_{-28,+31}$ & 0.199 & 19.9 & 12.7 & 0 & 15.7\% & 2792.14 & 2.18 \\ +RealMLP (T) & 1231${}_{-29,+34}$ & 0.201 & 20.8 & 13.3 & 0 & 16.5\% & 5575.69 & 0.24 \\ +EBM (T) & 1227${}_{-32,+37}$ & 0.120 & 20.9 & 11.1 & 0 & 18.0\% & 1142.31 & 0.03 \\ +EBM (D) & 1207${}_{-35,+29}$ & 0.153 & 22.2 & 9.7 & 1 & 18.7\% & 5.57 & 0.07 \\ +FastaiMLP (T+E) & 1202${}_{-34,+29}$ & 0.230 & 22.3 & 12.4 & 0 & 19.1\% & 1350.04 & 6.57 \\ +ModernNCA (D) & 1188${}_{-28,+29}$ & 0.183 & 22.8 & 9.8 & 1 & 19.4\% & 13.67 & 0.34 \\ +TabDPT (D) & 1179${}_{-36,+29}$ & 0.258 & 23.5 & 7.1 & 1 & 18.4\% & 28.08 & 8.76 \\ +RealMLP (D) & 1179${}_{-31,+24}$ & 0.083 & 23.6 & 11.0 & 1 & 17.5\% & 35.27 & 0.25 \\ +TorchMLP (T) & 1168${}_{-36,+38}$ & 0.086 & 24.0 & 17.9 & 0 & 17.8\% & 2792.14 & 0.13 \\ +ExtraTrees (T+E) & 1155${}_{-36,+29}$ & 0.102 & 24.9 & 16.7 & 0 & 20.4\% & 378.34 & 1.37 \\ +FastaiMLP (T) & 1140${}_{-32,+30}$ & 0.113 & 25.5 & 14.7 & 0 & 20.7\% & 1350.04 & 0.61 \\ +XGBoost (D) & 1130${}_{-32,+28}$ & 0.032 & 26.0 & 21.9 & 0 & 19.1\% & 2.66 & 0.17 \\ +ExtraTrees (T) & 1128${}_{-26,+31}$ & 0.072 & 26.0 & 13.8 & 0 & 20.6\% & 378.34 & 0.15 \\ +RandomForest (T+E) & 1114${}_{-34,+28}$ & 0.057 & 27.0 & 14.8 & 0 & 21.0\% & 527.27 & 1.16 \\ +LightGBM (D) & 1098${}_{-30,+32}$ & 0.030 & 27.6 & 24.0 & 0 & 20.1\% & 3.01 & 0.18 \\ +RandomForest (T) & 1064${}_{-35,+30}$ & 0.023 & 29.2 & 22.2 & 0 & 21.9\% & 527.27 & 0.13 \\ +TorchMLP (D) & 1006${}_{-30,+39}$ & 0.030 & 31.7 & 26.7 & 0 & 24.8\% & 8.93 & 0.17 \\ +RandomForest (D) & 1000${}_{-0,+0}$ & 0.005 & 32.2 & 24.6 & 0 & 27.3\% & 0.44 & 0.06 \\ +FastaiMLP (D) & 1000${}_{-32,+30}$ & 0.019 & 32.2 & 27.7 & 0 & 24.6\% & 4.68 & 0.55 \\ +ExtraTrees (D) & 949${}_{-32,+36}$ & 0.015 & 34.1 & 29.1 & 0 & 29.3\% & 0.36 & 0.07 \\ +Linear (T+E) & 946${}_{-35,+33}$ & 0.055 & 34.2 & 15.0 & 1 & 31.3\% & 89.12 & 0.20 \\ +Linear (T) & 925${}_{-44,+35}$ & 0.035 & 34.9 & 24.7 & 0 & 32.0\% & 89.12 & 0.08 \\ +Linear (D) & 892${}_{-29,+36}$ & 0.025 & 36.0 & 33.0 & 0 & 33.3\% & 2.43 & 0.10 \\ +KNN (T+E) & 746${}_{-31,+45}$ & 0.000 & 39.9 & 33.1 & 0 & 47.7\% & 3.50 & 0.16 \\ +KNN (T) & 676${}_{-48,+42}$ & 0.000 & 41.2 & 28.3 & 0 & 50.2\% & 3.50 & 0.03 \\ +KNN (D) & 384${}_{-116,+61}$ & 0.000 & 44.3 & 44.2 & 0 & 59.5\% & 0.07 & 0.02 \\ +\bottomrule +\end{tabular} \ No newline at end of file diff --git a/data/lite/tabpfn-tabicl-imputed-cls/tabarena_leaderboard.csv b/data/lite/tabpfn-tabicl-imputed-cls/tabarena_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..df059e4121634d1f0f2b668e430505bc14507b44 --- /dev/null +++ b/data/lite/tabpfn-tabicl-imputed-cls/tabarena_leaderboard.csv @@ -0,0 +1,46 @@ +method,time_train_s,time_infer_s,time_train_s_per_1K,time_infer_s_per_1K,normalized-error,normalized-error-task,champ_delta,loss_rescaled,time_train_s_rescaled,time_infer_s_rescaled,rank,median_metric_error,median_time_train_s,median_time_infer_s,median_time_train_s_per_1K,median_time_infer_s_per_1K,median_normalized-error,median_normalized-error-task,median_champ_delta,median_loss_rescaled,median_time_train_s_rescaled,median_time_infer_s_rescaled,median_rank,rank=1_count,rank=2_count,rank=3_count,rank>3_count,elo,elo+,elo-,winrate,mrr +TABPFNV2 (tuned + ensemble),14624.10810300937,166.6936253951146,3837.093570770289,71.22922446033486,0.2999799624009034,0.37900004959687766,0.08031051995518385,0.05455935044977521,73712.17169915051,6326.444952096131,8.461538461538462,0.1749,8447.90291428566,59.9476752281189,3361.317280362242,51.668880171806535,0.21441671773452015,0.3480554663987826,0.03548854933740464,0.027745461982937078,32774.70384527536,4094.471326370238,5.5,6,1,3,16,1506.7,31.8,44.7,0.8304195804195804,0.358094093982103 +AutoGluon 1.3 (4h),5811.540137483524,3.0737958137805643,3715.2599271484114,3.044362316385498,0.5599854226187498,0.5617828950350577,0.10445421398736063,0.0599680756233326,37983.87924037632,165.48830983683717,11.01923076923077,0.17751,5213.819254875183,2.0895845890045166,2085.7270446537464,2.161349128608518,0.5600808158121042,0.5574000428410242,0.06078825021132733,0.06659288302785793,30345.10100676535,118.3021811054546,9.0,1,0,1,24,1433.5,32.8,30.7,0.7722902097902098,0.15518544636191695 +REALMLP (tuned + ensemble),17853.666671056013,5.060677555891184,6069.468633023759,5.352972929535142,0.5983187605703805,0.5697719962134024,0.1225244655490201,0.0697158337547381,114488.59335090281,312.9299756303972,12.115384615384615,0.180765,10652.850141763687,4.7910391092300415,5575.693006637695,4.577857176513236,0.5696681015084324,0.5408431020221902,0.06341551053522204,0.04576778429435464,88499.22644289241,286.2007603461078,11.5,0,1,2,23,1407.7,28.0,30.0,0.7473776223776224,0.1450476689017345 +TABPFNV2 (tuned),14624.10810300937,5.890562671881455,3837.093570770289,2.9216612176257093,0.4562783530638654,0.539616756848536,0.10636474393434797,0.07576264928807884,73712.17169915051,208.74723417223507,12.25,0.19132,8447.90291428566,0.5261704921722412,3361.317280362242,0.52729181819818,0.34109768962419973,0.6143846899224816,0.08017059655741715,0.050216477642017796,32774.70384527536,36.401677545743524,11.0,1,5,2,18,1402.3,31.6,32.9,0.7443181818181818,0.22277582323851844 +TABM (tuned + ensemble),7183.65333878994,2.169150572556716,3570.712955098323,1.861754182648195,0.5166525238887076,0.5534887980632361,0.11956349986158774,0.08309123144106954,42989.483218042995,121.47528176372664,12.846153846153847,0.18310500000000002,6058.230099320412,1.6269241571426392,2899.0220559668596,1.460880880345262,0.47136645123500964,0.5213443194889227,0.07336602828104005,0.040297742432766875,37327.19260107514,109.27107326295308,9.5,1,0,1,24,1392.2,32.7,39.4,0.7307692307692307,0.16156485134544094 +TABICL (default),21.54241045621725,2.6437206268310547,10.219926189931016,1.9528462364644343,0.45732829959408833,0.5557361534084676,0.10499055070178752,0.06310686388674132,140.84761892371782,137.0823191069963,13.365384615384615,0.177865,18.648964643478394,2.1483960151672363,9.073519792448254,2.007502492806867,0.46959481282562443,0.6652418475567421,0.05302697309380272,0.028777901101857722,117.57303754113096,107.76328227971135,13.5,4,2,1,19,1378.5,36.4,30.5,0.7189685314685315,0.2556026489858902 +GBM (tuned + ensemble),1386.6633568268555,2.647538212629465,855.5354588276103,2.6861264004396594,0.6806482110513491,0.7106186510085695,0.141362130997881,0.08782930779014528,10078.423717398238,182.57724180654643,14.788461538461538,0.18084,1317.1929507255554,2.008246660232544,691.4641967411458,1.4553717527055694,0.7005535661954161,0.7652961068739543,0.07965351598262033,0.06332006129013433,9059.644744264673,90.72358893879789,14.75,0,0,1,25,1349.4,28.6,33.0,0.6866258741258742,0.08652272628643233 +TABPFNV2 (default),12.006092300781837,1.2122490681134737,5.3148066442099084,0.6434373636958165,0.5013642769155608,0.6076681115357273,0.12052806276021125,0.08926430649670257,74.57934257049592,50.66248833495304,15.538461538461538,0.194575,9.273962259292603,0.4255213737487793,4.283948847917442,0.5110342254172261,0.3771347187714357,0.8078260737500734,0.07359963710195983,0.07618105155863322,62.366726819804335,28.80821414198816,14.5,3,2,2,19,1333.5,34.9,34.7,0.6695804195804196,0.22455622157365326 +TABM (tuned),7183.65333878994,0.2019361532651461,3570.712955098323,0.2168923191387062,0.6127855914191946,0.6153990479247885,0.130856425990661,0.09537399493661194,42989.483218042995,13.062348029072668,15.692307692307692,0.18589499999999998,6058.230099320412,0.1839456558227539,2899.0220559668596,0.171251855223312,0.5915500043256848,0.6288985041617685,0.09148128983327741,0.05588464476989181,37327.19260107514,9.17800659262771,12.0,0,2,2,22,1329.7,32.2,30.3,0.666083916083916,0.13000506341067208 +CAT (tuned + ensemble),4861.749095384891,1.0365588940106905,3035.7630585976663,0.9271931106606893,0.728920075896456,0.6786642048762535,0.13487992339677388,0.0909217961956513,26561.38119334446,53.29220521181529,15.76923076923077,0.178855,3259.4502482414246,0.9432240724563599,1394.4481643221718,0.6180738220288715,0.7992439551459185,0.7007978686240202,0.07405507598059591,0.051124014208117546,20607.594190900367,47.175332427485074,15.0,0,1,1,24,1331.9,32.8,35.8,0.6643356643356644,0.10243020698237862 +MNCA (tuned),11104.727954295966,0.5286179322462815,5977.187221364461,0.4814995680859316,0.7448490525012423,0.6406839211091188,0.12655502884420408,0.10326715793661159,78433.1838326286,30.191390941699975,16.01923076923077,0.18580999999999998,11212.920736551285,0.4423438310623169,6020.892835479641,0.4537708228509102,0.8177816514691232,0.7061138774961084,0.08668289690877401,0.05913687649157323,67721.7850080898,25.373417518222958,14.25,2,0,0,24,1325.6,30.3,30.3,0.6586538461538461,0.15586362718013336 +XGB (tuned + ensemble),1868.2429302564035,2.459977333362286,1131.3403541498637,3.0908212073715786,0.7784275988369416,0.7326751610286906,0.1521183469738161,0.10487866753851346,11151.77336671448,133.9681639809692,16.576923076923077,0.184395,1441.2213016748428,1.317580223083496,785.8975953484313,1.2809013038285242,0.8316341855574652,0.7341106406501618,0.1144861696628025,0.054690456125919155,11226.485943771408,74.70529109142498,13.5,0,0,2,24,1313.5,28.5,32.1,0.6459790209790209,0.08752950287187947 +CAT (default),173.12264615755814,0.2530022859573364,152.19981148640542,0.1746769705615129,0.7721520925382083,0.7508782741378199,0.14762383159906592,0.09673912909834526,503.5985284620293,11.812145121206303,16.75,0.18153,12.185660362243652,0.2613861560821533,5.736216485552211,0.13945330874004275,0.8708469956136012,0.8764795848440701,0.09472627265148731,0.048694739189140704,101.89620933712393,7.710770185087604,19.25,1,0,0,25,1308.5,29.6,31.7,0.6420454545454546,0.10408056855425275 +CAT (tuned),4861.749095384891,0.10756405500265268,3035.7630585976663,0.11792367197674246,0.7533425274831995,0.7418580085557809,0.14394572275280534,0.09786470048169181,26561.38119334446,7.02054598025311,17.192307692307693,0.179485,3259.4502482414246,0.05912292003631592,1394.4481643221718,0.05275041900206211,0.8504867849422679,0.7781645153943832,0.09050829480547845,0.05413571715045103,20607.594190900367,2.886943526422101,17.0,0,0,0,26,1299.2,41.1,27.8,0.631993006993007,0.07539691390572127 +GBM (tuned),1386.6633568268555,0.6239433655372033,855.5354588276103,0.6651164927916418,0.737683616453633,0.7596688264282848,0.15123309377717353,0.10299158861394081,10078.423717398238,42.45419816933193,17.634615384615383,0.18125,1317.1929507255554,0.32417428493499756,691.4641967411458,0.2643910314838089,0.7826548575969626,0.7981479775650631,0.08157588989660797,0.06910038121186715,9059.644744264673,20.34411810847653,18.5,0,1,0,25,1288.1,29.9,29.0,0.6219405594405595,0.08417253395400366 +MNCA (tuned + ensemble),11104.727954295966,11.626816547833956,5977.187221364461,9.077822725159232,0.6933896363411076,0.6660665157088731,0.1444822450466191,0.11574155496848519,78433.1838326286,625.8156905319904,17.653846153846153,0.196755,11212.920736551285,9.569536089897156,6020.892835479641,8.47783782257851,0.722368050936999,0.6622270828972778,0.09664772425546841,0.06590198674037494,67721.7850080898,462.66839224080115,15.75,0,0,2,24,1292.0,26.5,34.9,0.6215034965034965,0.10668541415880203 +XGB (tuned),1868.2429302564035,0.6344939470291138,1131.3403541498637,0.845858557757576,0.8187393857188691,0.7667685498472205,0.15449551107917392,0.10860550617326425,11151.77336671448,35.07178353015141,17.807692307692307,0.18658,1441.2213016748428,0.2507840394973755,785.8975953484313,0.21279594317419753,0.9017943832788667,0.7453177957281472,0.12200656150282413,0.06269228828209442,11226.485943771408,15.348416723363933,16.5,1,0,0,25,1289.8,29.4,25.3,0.618006993006993,0.10251976073586389 +TABM (default),26.77778566800631,0.20130640726823074,16.983704596070464,0.1991903141757489,0.7120650416184634,0.7086289666607134,0.1622519047697801,0.11661283756574645,212.38711719883958,11.49822311803021,18.153846153846153,0.18572,20.835053324699402,0.1561201810836792,11.598826273977398,0.14759862387674094,0.7417274447397321,0.7810887350534926,0.09137336521284833,0.06275561870896407,109.84095896574044,8.743598466674035,18.0,1,0,1,24,1281.0,29.5,24.9,0.6101398601398601,0.12406889112890061 +EBM (tuned + ensemble),2866.551095375648,0.3536289380146907,1716.6253639310917,0.31694700723366953,0.8144475987903695,0.7364009363908267,0.17180309869898544,0.13289820608093436,16005.580219570395,21.915829440477204,18.423076923076923,0.18934,1826.563945055008,0.30398499965667725,1142.3094021828929,0.2383204663966832,0.9368603191709515,0.9207311313375326,0.07547386841907339,0.05109125672741019,15335.47890204938,12.950523458443941,21.5,0,3,0,23,1276.6,36.7,28.2,0.6040209790209791,0.12124171082532008 +NN_TORCH (tuned + ensemble),8269.36125588417,3.6064914281551657,3548.397093003735,3.1069146452932506,0.8005534385996745,0.7867221786490863,0.15745609955540044,0.10096884964933617,54687.42205511106,206.78432079512532,19.884615384615383,0.186795,7021.069797754288,2.925987720489502,2792.139673756541,2.175346818589723,0.9508009219506811,0.917687560600783,0.0823266490663982,0.0717701052690009,44532.50472025028,168.68500762383525,20.0,0,0,1,25,1247.3,30.8,27.1,0.5708041958041958,0.07895497881555723 +REALMLP (tuned),17853.666671056013,0.2349767226439256,6069.468633023759,0.2967994896112342,0.7992732991174807,0.8048895624469216,0.16530717153664265,0.11169886093388302,114488.59335090281,15.66992951032636,20.76923076923077,0.179705,10652.850141763687,0.22168636322021484,5575.693006637695,0.23576453293502733,0.895201963654964,0.9670251309899789,0.1283441158885112,0.08568734793187416,88499.22644289241,12.792606929662101,20.5,0,1,0,25,1231.0,33.9,28.5,0.5506993006993007,0.07503622635930923 +EBM (tuned),2866.551095375648,0.04391795855302077,1716.6253639310917,0.042622423373582956,0.8796773591220505,0.7989370532749394,0.18031150115586747,0.13991074423102637,16005.580219570395,2.6200464262605005,20.865384615384617,0.19075999999999999,1826.563945055008,0.031216979026794434,1142.3094021828929,0.02993136666804911,1.0,1.0,0.08520888190451481,0.06958046400412875,15335.47890204938,1.3258183958395588,23.75,0,1,1,24,1227.1,36.6,31.5,0.548513986013986,0.08978323529643507 +EBM (default),11.305762657752403,0.07217896901644193,6.708906519026521,0.08642030214738769,0.8468070324259693,0.8110280220039858,0.18686874456979283,0.14250462388903842,76.4056328740172,4.809914969274192,22.23076923076923,0.19205,8.478393912315369,0.060648202896118164,5.571264059656597,0.07129895504487138,1.0,1.0,0.10192934818149862,0.06122058058323697,66.99828845613939,3.601780880785588,24.75,1,0,0,25,1206.9,28.3,34.8,0.5174825174825175,0.10315367705023558 +FASTAI (tuned + ensemble),2663.3156125178702,9.71036501114185,1626.2333617037298,10.35375213848753,0.7698173487890432,0.7888707552608327,0.19143489263209343,0.11424870338763603,20608.72361153851,575.9084247062348,22.326923076923077,0.18179499999999998,2361.516105532646,9.277374625205994,1350.0449662692658,6.569572430045136,0.9823907390988809,1.0,0.10018841132165368,0.07677125815183776,13001.178549428518,549.1440903510916,24.0,0,0,0,26,1202.1,28.7,33.1,0.5152972027972028,0.08032410389081089 +MNCA (default),29.761946293023918,0.45968284056736874,15.939524477151517,0.3666958147944243,0.8169283513779425,0.8227126311705688,0.19359838536157442,0.10940904253610519,187.22763978272167,23.803143758716253,22.826923076923077,0.19357,23.457369804382324,0.36945486068725586,13.667502577653629,0.33894040253174296,0.9808195818002508,0.9797952351969426,0.13240284359564197,0.08931940966654422,154.11579909730304,19.118442709424343,21.5,1,1,0,24,1187.8,28.8,27.9,0.5039335664335665,0.1019829541933546 +TABDPT (default),72.55373939184042,21.63674608560709,32.1753170626944,27.14316288922542,0.7421770023997575,0.7479465935897589,0.18444900328904643,0.12119989176698268,513.8148563269098,1550.6457261531152,23.48076923076923,0.20388,69.82611036300659,21.28597331047058,28.076698775708525,8.760817737139146,1.0,1.0,0.1080149448719816,0.07851722429401914,429.0853998300959,1143.1582415263433,29.5,1,3,0,22,1178.7,28.4,35.5,0.48907342657342656,0.14118801082755247 +REALMLP (default),110.65054762363434,0.2512630499326266,36.51262336985926,0.304026477962315,0.9170988508971886,0.8257233846816412,0.17480849590114966,0.10965875529591303,716.9172586317429,16.106317028225142,23.634615384615383,0.179885,61.38514173030853,0.19377470016479492,35.26610472383891,0.25260632993704635,1.0,1.0,0.10725973083151863,0.09589271240176966,562.1682174661271,13.084740916081461,26.25,1,0,0,25,1178.7,23.8,30.2,0.4855769230769231,0.09086138282523847 +NN_TORCH (tuned),8269.36125588417,0.1732868873156034,3548.397093003735,0.17387289280012472,0.9141063957909149,0.842287949515361,0.1775257283605692,0.11814031211601564,54687.42205511106,10.378521059679809,24.0,0.184685,7021.069797754288,0.11897110939025879,2792.139673756541,0.13285542411599796,1.0,1.0,0.10704543711737907,0.07835117203069997,44532.50472025028,7.793728816499673,25.0,0,0,0,26,1167.5,37.1,35.1,0.4772727272727273,0.055970780933021255 +XT (tuned + ensemble),723.008487197069,1.6600552705618052,480.9309873883464,1.918436680526551,0.8984673764067701,0.8698563213577479,0.20350469181458591,0.1535920856593196,5917.229990900821,105.19348400260608,24.884615384615383,0.188425,702.5076073408127,1.3369704484939575,378.34316304246136,1.3734038445009942,1.0,1.0,0.1315762721206637,0.09230140454586463,3777.3315625688056,84.73782880976256,28.0,0,0,0,26,1154.8,28.9,35.7,0.4571678321678322,0.059958692919405074 +FASTAI (tuned),2663.3156125178702,0.7872952864720271,1626.2333617037298,0.9244876829128756,0.8865326975991655,0.8432447700175844,0.206866728565722,0.1387333836548531,20608.72361153851,52.86743788138416,25.53846153846154,0.18316,2361.516105532646,0.6897614002227783,1350.0449662692658,0.612778202858813,1.0,1.0,0.1372818526486348,0.08670903152513522,13001.178549428518,36.29076458115673,27.5,0,0,1,25,1140.3,29.4,31.8,0.4423076923076923,0.06820649009803559 +XT (tuned),723.008487197069,0.16853698400350717,480.9309873883464,0.21789332007450063,0.9275522820640911,0.8584728603742081,0.20621433350726176,0.1564465527864028,5917.229990900821,11.278027514695932,26.0,0.189205,702.5076073408127,0.14003515243530273,378.34316304246136,0.15364019245636173,1.0,1.0,0.1505192879837034,0.09210625451126611,3777.3315625688056,8.882695855316769,29.5,0,1,0,25,1128.1,31.0,25.7,0.4318181818181818,0.07221148701643333 +XGB (default),5.406896040989802,0.29052331814399135,3.2395402815762044,0.40571388124316665,0.967663807523489,0.9208621475953046,0.1913484639295636,0.1583121152662077,36.135643016538445,18.079059576557942,26.01923076923077,0.18918000000000001,4.358284831047058,0.2170184850692749,2.659926162500852,0.1702856674842303,1.0,1.0,0.13622845464672223,0.08590682453074228,32.63170368298432,10.069746654781287,24.25,0,0,0,26,1129.9,27.4,31.1,0.4313811188811189,0.04570122330728964 +RF (tuned + ensemble),911.769657428448,1.55073488675631,531.6502580092293,1.7081982422422,0.9428934399339288,0.9018568954164017,0.20973262545633878,0.18169828635432628,6577.080710276883,100.32600883678194,26.96153846153846,0.18825,798.9582680463791,1.2464795112609863,527.2734284753022,1.1554577826899797,1.0,1.0,0.15587829515575163,0.09511037806823772,5029.2193365439925,80.30745598154598,30.0,0,1,0,25,1114.2,27.3,33.7,0.40996503496503495,0.06740230643960834 +GBM (default),5.678787359824548,0.359198652780973,3.357556396858338,0.23593366680622332,0.9701181894525962,0.9244030054522838,0.2009750941665907,0.16096135302268902,43.778662055665606,16.224185451530264,27.557692307692307,0.19459,5.089803218841553,0.34710729122161865,3.0120719035705887,0.17665561056592852,1.0,1.0,0.14940090168048453,0.08991864451874558,30.743592100025186,14.007062707115537,28.0,0,0,0,26,1097.5,31.5,29.5,0.3964160839160839,0.04169946841702806 +RF (tuned),911.769657428448,0.15133640399345985,531.6502580092293,0.19063529883897276,0.9772506936041817,0.9414350435106434,0.21876669694055456,0.18885525418574936,6577.080710276883,9.67186030508054,29.25,0.18775,798.9582680463791,0.13221728801727295,527.2734284753022,0.1275233993660041,1.0,1.0,0.16269556435701765,0.11206528012392108,5029.2193365439925,7.908293435451007,31.5,0,0,0,26,1064.0,29.3,35.0,0.35795454545454547,0.04504401000608687 +NN_TORCH (default),23.867143649321335,0.24185629991384652,13.87033788200038,0.29581404152824126,0.9698948249425869,0.9457939759123197,0.24766534415358382,0.1805126846920292,159.99336582070887,16.359564345871572,31.71153846153846,0.19833499999999998,19.20789933204651,0.18236374855041504,8.931817801759234,0.17241159209016133,1.0,1.0,0.1806992032072448,0.1282818632772945,162.04458259531137,9.195674192084631,33.0,0,0,0,26,1005.9,38.8,30.0,0.30201048951048953,0.03748733782316879 +FASTAI (default),10.92040595641503,0.607048539014963,5.789474679574097,0.6364629667404095,0.9807465319111015,0.9340218157343317,0.2458147993858494,0.18433902398032298,78.41470283692,37.58331718018816,32.15384615384615,0.19146000000000002,9.578287720680237,0.6503585577011108,4.679521651364521,0.5473457672031852,1.0,1.0,0.15595107304948763,0.13206479856027808,61.09577848128312,35.985779724786894,35.0,0,0,0,26,999.9,29.7,31.1,0.291958041958042,0.036046519543766274 +RF (default),1.1224354964036207,0.07520443659562331,0.5160590091828214,0.08504536046248315,0.9951894125278877,0.9662672985507942,0.2733008673941071,0.2607556340607851,7.699488729035172,4.8652515281017985,32.21153846153846,0.213715,0.8405723571777344,0.06134629249572754,0.44070860935344924,0.06492877134601137,1.0,1.0,0.22304594727380544,0.1387641342747614,5.263965350091326,4.042169058695611,35.75,0,0,0,26,1000.0,0.0,0.0,0.2906468531468531,0.04063169893049472 +XT (default),0.7937821791722224,0.076584870998676,0.45161836217627827,0.08730195757590078,0.9851460583683391,0.9723807339815378,0.2928638810006597,0.2925631030122461,5.4850410746785805,4.998284751448414,34.11538461538461,0.22224,0.7121238708496094,0.07150018215179443,0.3597485800421042,0.06793435113493712,1.0,1.0,0.21784002712442435,0.1545266336174043,3.777771179368423,4.378376796908203,38.0,0,0,0,26,949.4,35.6,31.6,0.24737762237762237,0.034356084606725006 +LR (tuned + ensemble),175.34701893879816,0.4311328667860765,114.81546568819135,0.35152858974096185,0.9447815704031849,0.9613573808489063,0.31304194938194396,0.27486950659356457,1394.89696800973,28.50704727597215,34.21153846153846,0.219115,157.3445065021515,0.20991504192352295,89.11614089252916,0.1966045265631563,1.0,1.0,0.260779813420035,0.18354757952857997,1033.8951605639145,9.91761243845308,37.5,1,0,0,25,946.4,32.2,34.1,0.24519230769230768,0.06657835051950235 +LR (tuned),175.34701893879816,0.13578540545243484,114.81546568819135,0.11644774040693313,0.9645430999513104,0.9706124256069096,0.3197153495116481,0.2866149774984877,1394.89696800973,6.834352988282761,34.92307692307692,0.21990500000000002,157.3445065021515,0.06698489189147949,89.11614089252916,0.08340922333653941,1.0,1.0,0.2839064101424172,0.1973963943493853,1033.8951605639145,4.726037355894221,38.75,0,0,1,25,925.0,34.9,43.3,0.229020979020979,0.04041420009021431 +LR (default),4.7979896068573,0.14980417031508225,2.784461238483506,0.15310137710602748,0.9745259999639975,0.9812456463551247,0.3329427914389331,0.32581993219914024,36.357934158034695,9.005429323015887,35.98076923076923,0.227395,4.980341196060181,0.0988316535949707,2.4349667711867333,0.10027059217560387,1.0,1.0,0.283906387231373,0.21846589850078024,26.549049340848278,5.855361941404224,39.0,0,0,0,26,891.8,35.4,28.6,0.20498251748251747,0.030292514887996096 +KNN (tuned + ensemble),16.958975911140442,0.2818152537712684,7.194345160625263,0.20399826229470497,1.0,0.9816350265387093,0.47672042775227436,0.5833708185909443,56.76087640724927,13.193730328032911,39.92307692307692,0.297435,6.915013313293457,0.11628735065460205,3.5025132784274384,0.1592449370444793,1.0,1.0,0.4130987117462378,0.6739246116612398,52.78507887384432,10.328630925855668,43.0,0,0,0,26,745.5,44.9,30.5,0.11538461538461539,0.030206633151293592 +KNN (tuned),16.958975911140442,0.05680100734417255,7.194345160625263,0.04235176068048229,1.0,0.9774922422312161,0.5019983015456773,0.6443397962703555,56.76087640724927,2.623837177847182,41.21153846153846,0.301285,6.915013313293457,0.03549385070800781,3.5025132784274384,0.03322185201916636,1.0,1.0,0.5143946348760557,0.7015107115204589,52.78507887384432,1.9257403822827532,44.0,0,0,1,25,676.5,41.9,47.1,0.08610139860139861,0.035393141891706566 +KNN (default),0.30434812949253964,0.03149266426379864,0.14343197725298135,0.03163346219935531,1.0,1.0,0.5949762374279601,0.9484018378342699,1.002692426657382,1.6405188884998085,44.26923076923077,0.34604,0.17687928676605225,0.01871800422668457,0.0745054444441075,0.021103033819676938,1.0,1.0,0.6151111085811185,1.0,1.0,1.0188814221661033,45.0,0,0,0,26,383.7,60.2,115.9,0.016608391608391608,0.02263422903807308 diff --git a/data/lite/tabpfn-tabicl-imputed-cls/time_plot.pdf b/data/lite/tabpfn-tabicl-imputed-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8edd76956cd2a10c995248f5368aab91c1d95a8e Binary files /dev/null and b/data/lite/tabpfn-tabicl-imputed-cls/time_plot.pdf differ diff --git a/data/lite/tabpfn-tabicl-imputed-cls/time_plot.png.zip b/data/lite/tabpfn-tabicl-imputed-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..669e39e8270f7da85e7b36bf8a2e9a19f2841fec --- /dev/null +++ b/data/lite/tabpfn-tabicl-imputed-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21929263a17fea6e4606c60804a7f47da0f2c36a3e6f1a9c025752bd2e1bb7f9 +size 338023 diff --git a/data/lite/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf b/data/lite/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf new file mode 100644 index 0000000000000000000000000000000000000000..87bd678408494e6f4b284f97ad3909dfa1d131ff Binary files /dev/null and b/data/lite/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip b/data/lite/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..82771d031f285d261c303eb89c5de690ae716c90 --- /dev/null +++ b/data/lite/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:962b84540c16a1b1a8dbd376ade4f203bae3973046e47328ebf472d09eb92452 +size 138764 diff --git a/data/lite/tabpfn-tabicl-imputed-cls/tuning-impact-elo.pdf b/data/lite/tabpfn-tabicl-imputed-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..20fbe501eb4aebcdfa4a1f3f5b319ccf3cbdd3ef Binary files /dev/null and b/data/lite/tabpfn-tabicl-imputed-cls/tuning-impact-elo.pdf differ diff --git a/data/lite/tabpfn-tabicl-imputed-cls/tuning-impact-elo.png.zip b/data/lite/tabpfn-tabicl-imputed-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7b19498ac7b96eb251ead9814013e84d88eeba27 --- /dev/null +++ b/data/lite/tabpfn-tabicl-imputed-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eca92b6e3236d39e595bb813e31e294d60194b0ec8e5e5b6cf4819bac57c06f +size 128232 diff --git a/data/lite/tabpfn-tabicl-imputed/figures/critical-diagram.pdf b/data/lite/tabpfn-tabicl-imputed/figures/critical-diagram.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6f16dfd4f62ba3b9af215ae3a7e4fcafaccf71c0 Binary files /dev/null and b/data/lite/tabpfn-tabicl-imputed/figures/critical-diagram.pdf differ diff --git a/data/lite/tabpfn-tabicl-imputed/figures/critical-diagram.png.zip b/data/lite/tabpfn-tabicl-imputed/figures/critical-diagram.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..5fac335c979d6b39203ca83cab7e879a9092a458 --- /dev/null +++ b/data/lite/tabpfn-tabicl-imputed/figures/critical-diagram.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cccbaac14f3330c7adeb010c8f1b06441c8e04792a5f7ac8b215d00d18e724d5 +size 319138 diff --git a/data/lite/tabpfn-tabicl-imputed/leaderboard.tex b/data/lite/tabpfn-tabicl-imputed/leaderboard.tex new file mode 100644 index 0000000000000000000000000000000000000000..470e5847143808188c07aa37c1d5e5b76d097272 --- /dev/null +++ b/data/lite/tabpfn-tabicl-imputed/leaderboard.tex @@ -0,0 +1,53 @@ +\begin{tabular}{llcccccrr} +\toprule +\textbf{Model} & \textbf{Elo ($\uparrow$)} & \textbf{Norm.} & \textbf{Avg.} & \textbf{Harm.} & \textbf{\#wins ($\uparrow$)} & \textbf{Improva-} & \textbf{Train time} & \textbf{Predict time} \\ + & & \textbf{score ($\uparrow$)} & \textbf{rank ($\downarrow$)} & \textbf{mean} & & \textbf{bility ($\downarrow$)} & \textbf{per 1K [s]} & \textbf{per 1K [s]} \\ + & & & & \textbf{rank ($\downarrow$)} & & & & \\ +\midrule +TabPFNv2 (T+E) & \textcolor{gold}{\textbf{1507${}_{-45,+32}$}} & \textcolor{gold}{\textbf{0.700}} & \textcolor{gold}{\textbf{8.5}} & \textcolor{gold}{\textbf{2.8}} & \textcolor{gold}{\textbf{6}} & \textcolor{gold}{\textbf{8.0\%}} & 3361.32 & 51.67 \\ +AutoGluon 1.3 (4h) & \textcolor{silver}{\textbf{1434${}_{-31,+33}$}} & 0.440 & \textcolor{silver}{\textbf{11.0}} & 6.4 & 1 & \textcolor{silver}{\textbf{10.4\%}} & 2085.73 & 2.16 \\ +RealMLP (T+E) & \textcolor{bronze}{\textbf{1408${}_{-30,+28}$}} & 0.402 & \textcolor{bronze}{\textbf{12.1}} & 6.9 & 0 & 12.3\% & 5575.69 & 4.58 \\ +TabPFNv2 (T) & 1402${}_{-33,+32}$ & \textcolor{silver}{\textbf{0.544}} & 12.2 & \textcolor{bronze}{\textbf{4.5}} & 1 & 10.6\% & 3361.32 & 0.53 \\ +TabM (T+E) & 1392${}_{-40,+33}$ & 0.483 & 12.8 & 6.2 & 1 & 12.0\% & 2899.02 & 1.46 \\ +TabICL (D) & 1378${}_{-31,+37}$ & \textcolor{bronze}{\textbf{0.543}} & 13.4 & \textcolor{silver}{\textbf{3.9}} & \textcolor{silver}{\textbf{4}} & \textcolor{bronze}{\textbf{10.5\%}} & 9.07 & 2.01 \\ +LightGBM (T+E) & 1349${}_{-33,+29}$ & 0.319 & 14.8 & 11.6 & 0 & 14.1\% & 691.46 & 1.46 \\ +TabPFNv2 (D) & 1334${}_{-35,+35}$ & 0.499 & 15.5 & \textcolor{bronze}{\textbf{4.5}} & \textcolor{bronze}{\textbf{3}} & 12.1\% & 4.28 & 0.51 \\ +CatBoost (T+E) & 1332${}_{-36,+33}$ & 0.271 & 15.8 & 9.8 & 0 & 13.5\% & 1394.45 & 0.62 \\ +TabM (T) & 1330${}_{-31,+33}$ & 0.387 & 15.7 & 7.7 & 0 & 13.1\% & 2899.02 & 0.17 \\ +ModernNCA (T) & 1326${}_{-31,+31}$ & 0.255 & 16.0 & 6.4 & 2 & 12.7\% & 6020.89 & 0.45 \\ +XGBoost (T+E) & 1314${}_{-33,+29}$ & 0.222 & 16.6 & 11.4 & 0 & 15.2\% & 785.90 & 1.28 \\ +CatBoost (D) & 1308${}_{-32,+30}$ & 0.228 & 16.8 & 9.6 & 1 & 14.8\% & 5.74 & 0.14 \\ +CatBoost (T) & 1299${}_{-28,+42}$ & 0.247 & 17.2 & 13.3 & 0 & 14.4\% & 1394.45 & 0.05 \\ +ModernNCA (T+E) & 1292${}_{-35,+27}$ & 0.307 & 17.7 & 9.4 & 0 & 14.4\% & 6020.89 & 8.48 \\ +XGBoost (T) & 1290${}_{-26,+30}$ & 0.181 & 17.8 & 9.8 & 1 & 15.4\% & 785.90 & 0.21 \\ +LightGBM (T) & 1288${}_{-29,+30}$ & 0.262 & 17.6 & 11.9 & 0 & 15.1\% & 691.46 & 0.26 \\ +TabM (D) & 1281${}_{-25,+30}$ & 0.288 & 18.2 & 8.1 & 1 & 16.2\% & 11.60 & 0.15 \\ +EBM (T+E) & 1277${}_{-29,+37}$ & 0.186 & 18.4 & 8.2 & 0 & 17.2\% & 1142.31 & 0.24 \\ +TorchMLP (T+E) & 1247${}_{-28,+31}$ & 0.199 & 19.9 & 12.7 & 0 & 15.7\% & 2792.14 & 2.18 \\ +RealMLP (T) & 1231${}_{-29,+34}$ & 0.201 & 20.8 & 13.3 & 0 & 16.5\% & 5575.69 & 0.24 \\ +EBM (T) & 1227${}_{-32,+37}$ & 0.120 & 20.9 & 11.1 & 0 & 18.0\% & 1142.31 & 0.03 \\ +EBM (D) & 1207${}_{-35,+29}$ & 0.153 & 22.2 & 9.7 & 1 & 18.7\% & 5.57 & 0.07 \\ +FastaiMLP (T+E) & 1202${}_{-34,+29}$ & 0.230 & 22.3 & 12.4 & 0 & 19.1\% & 1350.04 & 6.57 \\ +ModernNCA (D) & 1188${}_{-28,+29}$ & 0.183 & 22.8 & 9.8 & 1 & 19.4\% & 13.67 & 0.34 \\ +TabDPT (D) & 1179${}_{-36,+29}$ & 0.258 & 23.5 & 7.1 & 1 & 18.4\% & 28.08 & 8.76 \\ +RealMLP (D) & 1179${}_{-31,+24}$ & 0.083 & 23.6 & 11.0 & 1 & 17.5\% & 35.27 & 0.25 \\ +TorchMLP (T) & 1168${}_{-36,+38}$ & 0.086 & 24.0 & 17.9 & 0 & 17.8\% & 2792.14 & 0.13 \\ +ExtraTrees (T+E) & 1155${}_{-36,+29}$ & 0.102 & 24.9 & 16.7 & 0 & 20.4\% & 378.34 & 1.37 \\ +FastaiMLP (T) & 1140${}_{-32,+30}$ & 0.113 & 25.5 & 14.7 & 0 & 20.7\% & 1350.04 & 0.61 \\ +XGBoost (D) & 1130${}_{-32,+28}$ & 0.032 & 26.0 & 21.9 & 0 & 19.1\% & 2.66 & 0.17 \\ +ExtraTrees (T) & 1128${}_{-26,+31}$ & 0.072 & 26.0 & 13.8 & 0 & 20.6\% & 378.34 & 0.15 \\ +RandomForest (T+E) & 1114${}_{-34,+28}$ & 0.057 & 27.0 & 14.8 & 0 & 21.0\% & 527.27 & 1.16 \\ +LightGBM (D) & 1098${}_{-30,+32}$ & 0.030 & 27.6 & 24.0 & 0 & 20.1\% & 3.01 & 0.18 \\ +RandomForest (T) & 1064${}_{-35,+30}$ & 0.023 & 29.2 & 22.2 & 0 & 21.9\% & 527.27 & 0.13 \\ +TorchMLP (D) & 1006${}_{-30,+39}$ & 0.030 & 31.7 & 26.7 & 0 & 24.8\% & 8.93 & 0.17 \\ +RandomForest (D) & 1000${}_{-0,+0}$ & 0.005 & 32.2 & 24.6 & 0 & 27.3\% & 0.44 & 0.06 \\ +FastaiMLP (D) & 1000${}_{-32,+30}$ & 0.019 & 32.2 & 27.7 & 0 & 24.6\% & 4.68 & 0.55 \\ +ExtraTrees (D) & 949${}_{-32,+36}$ & 0.015 & 34.1 & 29.1 & 0 & 29.3\% & 0.36 & 0.07 \\ +Linear (T+E) & 946${}_{-35,+33}$ & 0.055 & 34.2 & 15.0 & 1 & 31.3\% & 89.12 & 0.20 \\ +Linear (T) & 925${}_{-44,+35}$ & 0.035 & 34.9 & 24.7 & 0 & 32.0\% & 89.12 & 0.08 \\ +Linear (D) & 892${}_{-29,+36}$ & 0.025 & 36.0 & 33.0 & 0 & 33.3\% & 2.43 & 0.10 \\ +KNN (T+E) & 746${}_{-31,+45}$ & 0.000 & 39.9 & 33.1 & 0 & 47.7\% & 3.50 & 0.16 \\ +KNN (T) & 676${}_{-48,+42}$ & 0.000 & 41.2 & 28.3 & 0 & 50.2\% & 3.50 & 0.03 \\ +KNN (D) & 384${}_{-116,+61}$ & 0.000 & 44.3 & 44.2 & 0 & 59.5\% & 0.07 & 0.02 \\ +\bottomrule +\end{tabular} \ No newline at end of file diff --git a/data/lite/tabpfn-tabicl-imputed/tabarena_leaderboard.csv b/data/lite/tabpfn-tabicl-imputed/tabarena_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..df059e4121634d1f0f2b668e430505bc14507b44 --- /dev/null +++ b/data/lite/tabpfn-tabicl-imputed/tabarena_leaderboard.csv @@ -0,0 +1,46 @@ +method,time_train_s,time_infer_s,time_train_s_per_1K,time_infer_s_per_1K,normalized-error,normalized-error-task,champ_delta,loss_rescaled,time_train_s_rescaled,time_infer_s_rescaled,rank,median_metric_error,median_time_train_s,median_time_infer_s,median_time_train_s_per_1K,median_time_infer_s_per_1K,median_normalized-error,median_normalized-error-task,median_champ_delta,median_loss_rescaled,median_time_train_s_rescaled,median_time_infer_s_rescaled,median_rank,rank=1_count,rank=2_count,rank=3_count,rank>3_count,elo,elo+,elo-,winrate,mrr +TABPFNV2 (tuned + ensemble),14624.10810300937,166.6936253951146,3837.093570770289,71.22922446033486,0.2999799624009034,0.37900004959687766,0.08031051995518385,0.05455935044977521,73712.17169915051,6326.444952096131,8.461538461538462,0.1749,8447.90291428566,59.9476752281189,3361.317280362242,51.668880171806535,0.21441671773452015,0.3480554663987826,0.03548854933740464,0.027745461982937078,32774.70384527536,4094.471326370238,5.5,6,1,3,16,1506.7,31.8,44.7,0.8304195804195804,0.358094093982103 +AutoGluon 1.3 (4h),5811.540137483524,3.0737958137805643,3715.2599271484114,3.044362316385498,0.5599854226187498,0.5617828950350577,0.10445421398736063,0.0599680756233326,37983.87924037632,165.48830983683717,11.01923076923077,0.17751,5213.819254875183,2.0895845890045166,2085.7270446537464,2.161349128608518,0.5600808158121042,0.5574000428410242,0.06078825021132733,0.06659288302785793,30345.10100676535,118.3021811054546,9.0,1,0,1,24,1433.5,32.8,30.7,0.7722902097902098,0.15518544636191695 +REALMLP (tuned + ensemble),17853.666671056013,5.060677555891184,6069.468633023759,5.352972929535142,0.5983187605703805,0.5697719962134024,0.1225244655490201,0.0697158337547381,114488.59335090281,312.9299756303972,12.115384615384615,0.180765,10652.850141763687,4.7910391092300415,5575.693006637695,4.577857176513236,0.5696681015084324,0.5408431020221902,0.06341551053522204,0.04576778429435464,88499.22644289241,286.2007603461078,11.5,0,1,2,23,1407.7,28.0,30.0,0.7473776223776224,0.1450476689017345 +TABPFNV2 (tuned),14624.10810300937,5.890562671881455,3837.093570770289,2.9216612176257093,0.4562783530638654,0.539616756848536,0.10636474393434797,0.07576264928807884,73712.17169915051,208.74723417223507,12.25,0.19132,8447.90291428566,0.5261704921722412,3361.317280362242,0.52729181819818,0.34109768962419973,0.6143846899224816,0.08017059655741715,0.050216477642017796,32774.70384527536,36.401677545743524,11.0,1,5,2,18,1402.3,31.6,32.9,0.7443181818181818,0.22277582323851844 +TABM (tuned + ensemble),7183.65333878994,2.169150572556716,3570.712955098323,1.861754182648195,0.5166525238887076,0.5534887980632361,0.11956349986158774,0.08309123144106954,42989.483218042995,121.47528176372664,12.846153846153847,0.18310500000000002,6058.230099320412,1.6269241571426392,2899.0220559668596,1.460880880345262,0.47136645123500964,0.5213443194889227,0.07336602828104005,0.040297742432766875,37327.19260107514,109.27107326295308,9.5,1,0,1,24,1392.2,32.7,39.4,0.7307692307692307,0.16156485134544094 +TABICL (default),21.54241045621725,2.6437206268310547,10.219926189931016,1.9528462364644343,0.45732829959408833,0.5557361534084676,0.10499055070178752,0.06310686388674132,140.84761892371782,137.0823191069963,13.365384615384615,0.177865,18.648964643478394,2.1483960151672363,9.073519792448254,2.007502492806867,0.46959481282562443,0.6652418475567421,0.05302697309380272,0.028777901101857722,117.57303754113096,107.76328227971135,13.5,4,2,1,19,1378.5,36.4,30.5,0.7189685314685315,0.2556026489858902 +GBM (tuned + ensemble),1386.6633568268555,2.647538212629465,855.5354588276103,2.6861264004396594,0.6806482110513491,0.7106186510085695,0.141362130997881,0.08782930779014528,10078.423717398238,182.57724180654643,14.788461538461538,0.18084,1317.1929507255554,2.008246660232544,691.4641967411458,1.4553717527055694,0.7005535661954161,0.7652961068739543,0.07965351598262033,0.06332006129013433,9059.644744264673,90.72358893879789,14.75,0,0,1,25,1349.4,28.6,33.0,0.6866258741258742,0.08652272628643233 +TABPFNV2 (default),12.006092300781837,1.2122490681134737,5.3148066442099084,0.6434373636958165,0.5013642769155608,0.6076681115357273,0.12052806276021125,0.08926430649670257,74.57934257049592,50.66248833495304,15.538461538461538,0.194575,9.273962259292603,0.4255213737487793,4.283948847917442,0.5110342254172261,0.3771347187714357,0.8078260737500734,0.07359963710195983,0.07618105155863322,62.366726819804335,28.80821414198816,14.5,3,2,2,19,1333.5,34.9,34.7,0.6695804195804196,0.22455622157365326 +TABM (tuned),7183.65333878994,0.2019361532651461,3570.712955098323,0.2168923191387062,0.6127855914191946,0.6153990479247885,0.130856425990661,0.09537399493661194,42989.483218042995,13.062348029072668,15.692307692307692,0.18589499999999998,6058.230099320412,0.1839456558227539,2899.0220559668596,0.171251855223312,0.5915500043256848,0.6288985041617685,0.09148128983327741,0.05588464476989181,37327.19260107514,9.17800659262771,12.0,0,2,2,22,1329.7,32.2,30.3,0.666083916083916,0.13000506341067208 +CAT (tuned + ensemble),4861.749095384891,1.0365588940106905,3035.7630585976663,0.9271931106606893,0.728920075896456,0.6786642048762535,0.13487992339677388,0.0909217961956513,26561.38119334446,53.29220521181529,15.76923076923077,0.178855,3259.4502482414246,0.9432240724563599,1394.4481643221718,0.6180738220288715,0.7992439551459185,0.7007978686240202,0.07405507598059591,0.051124014208117546,20607.594190900367,47.175332427485074,15.0,0,1,1,24,1331.9,32.8,35.8,0.6643356643356644,0.10243020698237862 +MNCA (tuned),11104.727954295966,0.5286179322462815,5977.187221364461,0.4814995680859316,0.7448490525012423,0.6406839211091188,0.12655502884420408,0.10326715793661159,78433.1838326286,30.191390941699975,16.01923076923077,0.18580999999999998,11212.920736551285,0.4423438310623169,6020.892835479641,0.4537708228509102,0.8177816514691232,0.7061138774961084,0.08668289690877401,0.05913687649157323,67721.7850080898,25.373417518222958,14.25,2,0,0,24,1325.6,30.3,30.3,0.6586538461538461,0.15586362718013336 +XGB (tuned + ensemble),1868.2429302564035,2.459977333362286,1131.3403541498637,3.0908212073715786,0.7784275988369416,0.7326751610286906,0.1521183469738161,0.10487866753851346,11151.77336671448,133.9681639809692,16.576923076923077,0.184395,1441.2213016748428,1.317580223083496,785.8975953484313,1.2809013038285242,0.8316341855574652,0.7341106406501618,0.1144861696628025,0.054690456125919155,11226.485943771408,74.70529109142498,13.5,0,0,2,24,1313.5,28.5,32.1,0.6459790209790209,0.08752950287187947 +CAT (default),173.12264615755814,0.2530022859573364,152.19981148640542,0.1746769705615129,0.7721520925382083,0.7508782741378199,0.14762383159906592,0.09673912909834526,503.5985284620293,11.812145121206303,16.75,0.18153,12.185660362243652,0.2613861560821533,5.736216485552211,0.13945330874004275,0.8708469956136012,0.8764795848440701,0.09472627265148731,0.048694739189140704,101.89620933712393,7.710770185087604,19.25,1,0,0,25,1308.5,29.6,31.7,0.6420454545454546,0.10408056855425275 +CAT (tuned),4861.749095384891,0.10756405500265268,3035.7630585976663,0.11792367197674246,0.7533425274831995,0.7418580085557809,0.14394572275280534,0.09786470048169181,26561.38119334446,7.02054598025311,17.192307692307693,0.179485,3259.4502482414246,0.05912292003631592,1394.4481643221718,0.05275041900206211,0.8504867849422679,0.7781645153943832,0.09050829480547845,0.05413571715045103,20607.594190900367,2.886943526422101,17.0,0,0,0,26,1299.2,41.1,27.8,0.631993006993007,0.07539691390572127 +GBM (tuned),1386.6633568268555,0.6239433655372033,855.5354588276103,0.6651164927916418,0.737683616453633,0.7596688264282848,0.15123309377717353,0.10299158861394081,10078.423717398238,42.45419816933193,17.634615384615383,0.18125,1317.1929507255554,0.32417428493499756,691.4641967411458,0.2643910314838089,0.7826548575969626,0.7981479775650631,0.08157588989660797,0.06910038121186715,9059.644744264673,20.34411810847653,18.5,0,1,0,25,1288.1,29.9,29.0,0.6219405594405595,0.08417253395400366 +MNCA (tuned + ensemble),11104.727954295966,11.626816547833956,5977.187221364461,9.077822725159232,0.6933896363411076,0.6660665157088731,0.1444822450466191,0.11574155496848519,78433.1838326286,625.8156905319904,17.653846153846153,0.196755,11212.920736551285,9.569536089897156,6020.892835479641,8.47783782257851,0.722368050936999,0.6622270828972778,0.09664772425546841,0.06590198674037494,67721.7850080898,462.66839224080115,15.75,0,0,2,24,1292.0,26.5,34.9,0.6215034965034965,0.10668541415880203 +XGB (tuned),1868.2429302564035,0.6344939470291138,1131.3403541498637,0.845858557757576,0.8187393857188691,0.7667685498472205,0.15449551107917392,0.10860550617326425,11151.77336671448,35.07178353015141,17.807692307692307,0.18658,1441.2213016748428,0.2507840394973755,785.8975953484313,0.21279594317419753,0.9017943832788667,0.7453177957281472,0.12200656150282413,0.06269228828209442,11226.485943771408,15.348416723363933,16.5,1,0,0,25,1289.8,29.4,25.3,0.618006993006993,0.10251976073586389 +TABM (default),26.77778566800631,0.20130640726823074,16.983704596070464,0.1991903141757489,0.7120650416184634,0.7086289666607134,0.1622519047697801,0.11661283756574645,212.38711719883958,11.49822311803021,18.153846153846153,0.18572,20.835053324699402,0.1561201810836792,11.598826273977398,0.14759862387674094,0.7417274447397321,0.7810887350534926,0.09137336521284833,0.06275561870896407,109.84095896574044,8.743598466674035,18.0,1,0,1,24,1281.0,29.5,24.9,0.6101398601398601,0.12406889112890061 +EBM (tuned + ensemble),2866.551095375648,0.3536289380146907,1716.6253639310917,0.31694700723366953,0.8144475987903695,0.7364009363908267,0.17180309869898544,0.13289820608093436,16005.580219570395,21.915829440477204,18.423076923076923,0.18934,1826.563945055008,0.30398499965667725,1142.3094021828929,0.2383204663966832,0.9368603191709515,0.9207311313375326,0.07547386841907339,0.05109125672741019,15335.47890204938,12.950523458443941,21.5,0,3,0,23,1276.6,36.7,28.2,0.6040209790209791,0.12124171082532008 +NN_TORCH (tuned + ensemble),8269.36125588417,3.6064914281551657,3548.397093003735,3.1069146452932506,0.8005534385996745,0.7867221786490863,0.15745609955540044,0.10096884964933617,54687.42205511106,206.78432079512532,19.884615384615383,0.186795,7021.069797754288,2.925987720489502,2792.139673756541,2.175346818589723,0.9508009219506811,0.917687560600783,0.0823266490663982,0.0717701052690009,44532.50472025028,168.68500762383525,20.0,0,0,1,25,1247.3,30.8,27.1,0.5708041958041958,0.07895497881555723 +REALMLP (tuned),17853.666671056013,0.2349767226439256,6069.468633023759,0.2967994896112342,0.7992732991174807,0.8048895624469216,0.16530717153664265,0.11169886093388302,114488.59335090281,15.66992951032636,20.76923076923077,0.179705,10652.850141763687,0.22168636322021484,5575.693006637695,0.23576453293502733,0.895201963654964,0.9670251309899789,0.1283441158885112,0.08568734793187416,88499.22644289241,12.792606929662101,20.5,0,1,0,25,1231.0,33.9,28.5,0.5506993006993007,0.07503622635930923 +EBM (tuned),2866.551095375648,0.04391795855302077,1716.6253639310917,0.042622423373582956,0.8796773591220505,0.7989370532749394,0.18031150115586747,0.13991074423102637,16005.580219570395,2.6200464262605005,20.865384615384617,0.19075999999999999,1826.563945055008,0.031216979026794434,1142.3094021828929,0.02993136666804911,1.0,1.0,0.08520888190451481,0.06958046400412875,15335.47890204938,1.3258183958395588,23.75,0,1,1,24,1227.1,36.6,31.5,0.548513986013986,0.08978323529643507 +EBM (default),11.305762657752403,0.07217896901644193,6.708906519026521,0.08642030214738769,0.8468070324259693,0.8110280220039858,0.18686874456979283,0.14250462388903842,76.4056328740172,4.809914969274192,22.23076923076923,0.19205,8.478393912315369,0.060648202896118164,5.571264059656597,0.07129895504487138,1.0,1.0,0.10192934818149862,0.06122058058323697,66.99828845613939,3.601780880785588,24.75,1,0,0,25,1206.9,28.3,34.8,0.5174825174825175,0.10315367705023558 +FASTAI (tuned + ensemble),2663.3156125178702,9.71036501114185,1626.2333617037298,10.35375213848753,0.7698173487890432,0.7888707552608327,0.19143489263209343,0.11424870338763603,20608.72361153851,575.9084247062348,22.326923076923077,0.18179499999999998,2361.516105532646,9.277374625205994,1350.0449662692658,6.569572430045136,0.9823907390988809,1.0,0.10018841132165368,0.07677125815183776,13001.178549428518,549.1440903510916,24.0,0,0,0,26,1202.1,28.7,33.1,0.5152972027972028,0.08032410389081089 +MNCA (default),29.761946293023918,0.45968284056736874,15.939524477151517,0.3666958147944243,0.8169283513779425,0.8227126311705688,0.19359838536157442,0.10940904253610519,187.22763978272167,23.803143758716253,22.826923076923077,0.19357,23.457369804382324,0.36945486068725586,13.667502577653629,0.33894040253174296,0.9808195818002508,0.9797952351969426,0.13240284359564197,0.08931940966654422,154.11579909730304,19.118442709424343,21.5,1,1,0,24,1187.8,28.8,27.9,0.5039335664335665,0.1019829541933546 +TABDPT (default),72.55373939184042,21.63674608560709,32.1753170626944,27.14316288922542,0.7421770023997575,0.7479465935897589,0.18444900328904643,0.12119989176698268,513.8148563269098,1550.6457261531152,23.48076923076923,0.20388,69.82611036300659,21.28597331047058,28.076698775708525,8.760817737139146,1.0,1.0,0.1080149448719816,0.07851722429401914,429.0853998300959,1143.1582415263433,29.5,1,3,0,22,1178.7,28.4,35.5,0.48907342657342656,0.14118801082755247 +REALMLP (default),110.65054762363434,0.2512630499326266,36.51262336985926,0.304026477962315,0.9170988508971886,0.8257233846816412,0.17480849590114966,0.10965875529591303,716.9172586317429,16.106317028225142,23.634615384615383,0.179885,61.38514173030853,0.19377470016479492,35.26610472383891,0.25260632993704635,1.0,1.0,0.10725973083151863,0.09589271240176966,562.1682174661271,13.084740916081461,26.25,1,0,0,25,1178.7,23.8,30.2,0.4855769230769231,0.09086138282523847 +NN_TORCH (tuned),8269.36125588417,0.1732868873156034,3548.397093003735,0.17387289280012472,0.9141063957909149,0.842287949515361,0.1775257283605692,0.11814031211601564,54687.42205511106,10.378521059679809,24.0,0.184685,7021.069797754288,0.11897110939025879,2792.139673756541,0.13285542411599796,1.0,1.0,0.10704543711737907,0.07835117203069997,44532.50472025028,7.793728816499673,25.0,0,0,0,26,1167.5,37.1,35.1,0.4772727272727273,0.055970780933021255 +XT (tuned + ensemble),723.008487197069,1.6600552705618052,480.9309873883464,1.918436680526551,0.8984673764067701,0.8698563213577479,0.20350469181458591,0.1535920856593196,5917.229990900821,105.19348400260608,24.884615384615383,0.188425,702.5076073408127,1.3369704484939575,378.34316304246136,1.3734038445009942,1.0,1.0,0.1315762721206637,0.09230140454586463,3777.3315625688056,84.73782880976256,28.0,0,0,0,26,1154.8,28.9,35.7,0.4571678321678322,0.059958692919405074 +FASTAI (tuned),2663.3156125178702,0.7872952864720271,1626.2333617037298,0.9244876829128756,0.8865326975991655,0.8432447700175844,0.206866728565722,0.1387333836548531,20608.72361153851,52.86743788138416,25.53846153846154,0.18316,2361.516105532646,0.6897614002227783,1350.0449662692658,0.612778202858813,1.0,1.0,0.1372818526486348,0.08670903152513522,13001.178549428518,36.29076458115673,27.5,0,0,1,25,1140.3,29.4,31.8,0.4423076923076923,0.06820649009803559 +XT (tuned),723.008487197069,0.16853698400350717,480.9309873883464,0.21789332007450063,0.9275522820640911,0.8584728603742081,0.20621433350726176,0.1564465527864028,5917.229990900821,11.278027514695932,26.0,0.189205,702.5076073408127,0.14003515243530273,378.34316304246136,0.15364019245636173,1.0,1.0,0.1505192879837034,0.09210625451126611,3777.3315625688056,8.882695855316769,29.5,0,1,0,25,1128.1,31.0,25.7,0.4318181818181818,0.07221148701643333 +XGB (default),5.406896040989802,0.29052331814399135,3.2395402815762044,0.40571388124316665,0.967663807523489,0.9208621475953046,0.1913484639295636,0.1583121152662077,36.135643016538445,18.079059576557942,26.01923076923077,0.18918000000000001,4.358284831047058,0.2170184850692749,2.659926162500852,0.1702856674842303,1.0,1.0,0.13622845464672223,0.08590682453074228,32.63170368298432,10.069746654781287,24.25,0,0,0,26,1129.9,27.4,31.1,0.4313811188811189,0.04570122330728964 +RF (tuned + ensemble),911.769657428448,1.55073488675631,531.6502580092293,1.7081982422422,0.9428934399339288,0.9018568954164017,0.20973262545633878,0.18169828635432628,6577.080710276883,100.32600883678194,26.96153846153846,0.18825,798.9582680463791,1.2464795112609863,527.2734284753022,1.1554577826899797,1.0,1.0,0.15587829515575163,0.09511037806823772,5029.2193365439925,80.30745598154598,30.0,0,1,0,25,1114.2,27.3,33.7,0.40996503496503495,0.06740230643960834 +GBM (default),5.678787359824548,0.359198652780973,3.357556396858338,0.23593366680622332,0.9701181894525962,0.9244030054522838,0.2009750941665907,0.16096135302268902,43.778662055665606,16.224185451530264,27.557692307692307,0.19459,5.089803218841553,0.34710729122161865,3.0120719035705887,0.17665561056592852,1.0,1.0,0.14940090168048453,0.08991864451874558,30.743592100025186,14.007062707115537,28.0,0,0,0,26,1097.5,31.5,29.5,0.3964160839160839,0.04169946841702806 +RF (tuned),911.769657428448,0.15133640399345985,531.6502580092293,0.19063529883897276,0.9772506936041817,0.9414350435106434,0.21876669694055456,0.18885525418574936,6577.080710276883,9.67186030508054,29.25,0.18775,798.9582680463791,0.13221728801727295,527.2734284753022,0.1275233993660041,1.0,1.0,0.16269556435701765,0.11206528012392108,5029.2193365439925,7.908293435451007,31.5,0,0,0,26,1064.0,29.3,35.0,0.35795454545454547,0.04504401000608687 +NN_TORCH (default),23.867143649321335,0.24185629991384652,13.87033788200038,0.29581404152824126,0.9698948249425869,0.9457939759123197,0.24766534415358382,0.1805126846920292,159.99336582070887,16.359564345871572,31.71153846153846,0.19833499999999998,19.20789933204651,0.18236374855041504,8.931817801759234,0.17241159209016133,1.0,1.0,0.1806992032072448,0.1282818632772945,162.04458259531137,9.195674192084631,33.0,0,0,0,26,1005.9,38.8,30.0,0.30201048951048953,0.03748733782316879 +FASTAI (default),10.92040595641503,0.607048539014963,5.789474679574097,0.6364629667404095,0.9807465319111015,0.9340218157343317,0.2458147993858494,0.18433902398032298,78.41470283692,37.58331718018816,32.15384615384615,0.19146000000000002,9.578287720680237,0.6503585577011108,4.679521651364521,0.5473457672031852,1.0,1.0,0.15595107304948763,0.13206479856027808,61.09577848128312,35.985779724786894,35.0,0,0,0,26,999.9,29.7,31.1,0.291958041958042,0.036046519543766274 +RF (default),1.1224354964036207,0.07520443659562331,0.5160590091828214,0.08504536046248315,0.9951894125278877,0.9662672985507942,0.2733008673941071,0.2607556340607851,7.699488729035172,4.8652515281017985,32.21153846153846,0.213715,0.8405723571777344,0.06134629249572754,0.44070860935344924,0.06492877134601137,1.0,1.0,0.22304594727380544,0.1387641342747614,5.263965350091326,4.042169058695611,35.75,0,0,0,26,1000.0,0.0,0.0,0.2906468531468531,0.04063169893049472 +XT (default),0.7937821791722224,0.076584870998676,0.45161836217627827,0.08730195757590078,0.9851460583683391,0.9723807339815378,0.2928638810006597,0.2925631030122461,5.4850410746785805,4.998284751448414,34.11538461538461,0.22224,0.7121238708496094,0.07150018215179443,0.3597485800421042,0.06793435113493712,1.0,1.0,0.21784002712442435,0.1545266336174043,3.777771179368423,4.378376796908203,38.0,0,0,0,26,949.4,35.6,31.6,0.24737762237762237,0.034356084606725006 +LR (tuned + ensemble),175.34701893879816,0.4311328667860765,114.81546568819135,0.35152858974096185,0.9447815704031849,0.9613573808489063,0.31304194938194396,0.27486950659356457,1394.89696800973,28.50704727597215,34.21153846153846,0.219115,157.3445065021515,0.20991504192352295,89.11614089252916,0.1966045265631563,1.0,1.0,0.260779813420035,0.18354757952857997,1033.8951605639145,9.91761243845308,37.5,1,0,0,25,946.4,32.2,34.1,0.24519230769230768,0.06657835051950235 +LR (tuned),175.34701893879816,0.13578540545243484,114.81546568819135,0.11644774040693313,0.9645430999513104,0.9706124256069096,0.3197153495116481,0.2866149774984877,1394.89696800973,6.834352988282761,34.92307692307692,0.21990500000000002,157.3445065021515,0.06698489189147949,89.11614089252916,0.08340922333653941,1.0,1.0,0.2839064101424172,0.1973963943493853,1033.8951605639145,4.726037355894221,38.75,0,0,1,25,925.0,34.9,43.3,0.229020979020979,0.04041420009021431 +LR (default),4.7979896068573,0.14980417031508225,2.784461238483506,0.15310137710602748,0.9745259999639975,0.9812456463551247,0.3329427914389331,0.32581993219914024,36.357934158034695,9.005429323015887,35.98076923076923,0.227395,4.980341196060181,0.0988316535949707,2.4349667711867333,0.10027059217560387,1.0,1.0,0.283906387231373,0.21846589850078024,26.549049340848278,5.855361941404224,39.0,0,0,0,26,891.8,35.4,28.6,0.20498251748251747,0.030292514887996096 +KNN (tuned + ensemble),16.958975911140442,0.2818152537712684,7.194345160625263,0.20399826229470497,1.0,0.9816350265387093,0.47672042775227436,0.5833708185909443,56.76087640724927,13.193730328032911,39.92307692307692,0.297435,6.915013313293457,0.11628735065460205,3.5025132784274384,0.1592449370444793,1.0,1.0,0.4130987117462378,0.6739246116612398,52.78507887384432,10.328630925855668,43.0,0,0,0,26,745.5,44.9,30.5,0.11538461538461539,0.030206633151293592 +KNN (tuned),16.958975911140442,0.05680100734417255,7.194345160625263,0.04235176068048229,1.0,0.9774922422312161,0.5019983015456773,0.6443397962703555,56.76087640724927,2.623837177847182,41.21153846153846,0.301285,6.915013313293457,0.03549385070800781,3.5025132784274384,0.03322185201916636,1.0,1.0,0.5143946348760557,0.7015107115204589,52.78507887384432,1.9257403822827532,44.0,0,0,1,25,676.5,41.9,47.1,0.08610139860139861,0.035393141891706566 +KNN (default),0.30434812949253964,0.03149266426379864,0.14343197725298135,0.03163346219935531,1.0,1.0,0.5949762374279601,0.9484018378342699,1.002692426657382,1.6405188884998085,44.26923076923077,0.34604,0.17687928676605225,0.01871800422668457,0.0745054444441075,0.021103033819676938,1.0,1.0,0.6151111085811185,1.0,1.0,1.0188814221661033,45.0,0,0,0,26,383.7,60.2,115.9,0.016608391608391608,0.02263422903807308 diff --git a/data/lite/tabpfn-tabicl-imputed/time_plot.pdf b/data/lite/tabpfn-tabicl-imputed/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e1af694051173483ebb136f53425cb4bdaf937c8 Binary files /dev/null and b/data/lite/tabpfn-tabicl-imputed/time_plot.pdf differ diff --git a/data/lite/tabpfn-tabicl-imputed/time_plot.png.zip b/data/lite/tabpfn-tabicl-imputed/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..05f1fd54b683cb81bc0df7d11b506a14eb50f472 --- /dev/null +++ b/data/lite/tabpfn-tabicl-imputed/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd64ee326c959829ad740f5099ee65e6916f7e91bbbdcd8db6f4738a43e47400 +size 338023 diff --git a/data/lite/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.pdf b/data/lite/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a72cf66d12edc64fff1b491f92b912c85d193bdb Binary files /dev/null and b/data/lite/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.png.zip b/data/lite/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..b9b9460c3e1ea8cd841b4e8a043ff1b17f5dfb32 --- /dev/null +++ b/data/lite/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c67f8ab8ad03fe6061f7546edea5902c2fc1b090e50a8091e9a296bf6faabe45 +size 138764 diff --git a/data/lite/tabpfn-tabicl-imputed/tuning-impact-elo.pdf b/data/lite/tabpfn-tabicl-imputed/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7cc3b39ad87eff636f43f40eab0c9b89b5dbdb9e Binary files /dev/null and b/data/lite/tabpfn-tabicl-imputed/tuning-impact-elo.pdf differ diff --git a/data/lite/tabpfn-tabicl-imputed/tuning-impact-elo.png.zip b/data/lite/tabpfn-tabicl-imputed/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4d1e18ec41c2ee0c924cc8eb622bee356ac93094 --- /dev/null +++ b/data/lite/tabpfn-tabicl-imputed/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa4a29bc2875417f487670fd48e091ef6c5cc5e71c2b111d24d130b11ed6c975 +size 128232 diff --git a/data/lite/tabpfn-tabicl/figures/critical-diagram.pdf b/data/lite/tabpfn-tabicl/figures/critical-diagram.pdf index 63826253ffaa42082dab94252de70a5352dbae95..7583f28b3b9c9d5c6df1c3e088ce48048e5fecad 100644 Binary files a/data/lite/tabpfn-tabicl/figures/critical-diagram.pdf and b/data/lite/tabpfn-tabicl/figures/critical-diagram.pdf differ diff --git a/data/lite/tabpfn-tabicl/figures/critical-diagram.png.zip b/data/lite/tabpfn-tabicl/figures/critical-diagram.png.zip index ee6c1a909c106e11df9c28c59d1ea06ad8d90858..60293a8c5f3160394d3a4e2d3c5aeb0504428cd8 100644 --- a/data/lite/tabpfn-tabicl/figures/critical-diagram.png.zip +++ b/data/lite/tabpfn-tabicl/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5a5e1e5d299642f413424cfe6d5802a988af48d82649e3847e843cf9179eb4ef -size 319000 +oid sha256:d1d08e376b3f65950e8f78ae5d485ed5f69b13645e3f0c83d52cf9a929c0c2d8 +size 319138 diff --git a/data/lite/tabpfn-tabicl/time_plot.pdf b/data/lite/tabpfn-tabicl/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2cc027139066c6386b7b6a7df2faaf879a7cf430 Binary files /dev/null and b/data/lite/tabpfn-tabicl/time_plot.pdf differ diff --git a/data/lite/tabpfn-tabicl/time_plot.png.zip b/data/lite/tabpfn-tabicl/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..1b7bb5585bb9a9379ac32b2f681d71c48cf8b454 --- /dev/null +++ b/data/lite/tabpfn-tabicl/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a58df813913a9acdde766e5bed311c1e2875ed84297a00c7ab585233e75dd4b +size 338023 diff --git a/data/lite/tabpfn-tabicl/tuning-impact-elo-horizontal.pdf b/data/lite/tabpfn-tabicl/tuning-impact-elo-horizontal.pdf index 87a84cc29477088afd719cbf70343602077428db..7119e73c93f84d1384ad8bbc0fb5e68275ebd3ee 100644 Binary files a/data/lite/tabpfn-tabicl/tuning-impact-elo-horizontal.pdf and b/data/lite/tabpfn-tabicl/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabpfn-tabicl/tuning-impact-elo-horizontal.png.zip b/data/lite/tabpfn-tabicl/tuning-impact-elo-horizontal.png.zip index 11f6de74fa20ca51951d5e65c8838b9e75acc2bf..8f963ebc684959e47f37084ac14d742b55681970 100644 --- a/data/lite/tabpfn-tabicl/tuning-impact-elo-horizontal.png.zip +++ b/data/lite/tabpfn-tabicl/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4e3537446452cbef6ee98d645ac7ca084376854b8328e0703c07f9a66eb7f233 -size 138602 +oid sha256:5e3860eedb444299bbffc4c88f461adafd412f33609c42f04bc96678a2f4375c +size 138764 diff --git a/data/lite/tabpfn-tabicl/tuning-impact-elo.pdf b/data/lite/tabpfn-tabicl/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..901feb8fa4923b9965c6a3605b3fca98f32eec94 Binary files /dev/null and b/data/lite/tabpfn-tabicl/tuning-impact-elo.pdf differ diff --git a/data/lite/tabpfn-tabicl/tuning-impact-elo.png.zip b/data/lite/tabpfn-tabicl/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a225b171bac549d5339e4a13ac4eba723e47b07b --- /dev/null +++ b/data/lite/tabpfn-tabicl/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adf04eda3e60e1f66af06d3e09df33c3ad3a1c0167deedd994c9313f2d577970 +size 128232 diff --git a/data/lite/tabpfn/figures/critical-diagram.pdf b/data/lite/tabpfn/figures/critical-diagram.pdf index c0b7e8a88aef3faaf9d90d670ab0d4c9e40e1dea..ae9218a167738dd8816fed8c6f1f58bfb49cf908 100644 Binary files a/data/lite/tabpfn/figures/critical-diagram.pdf and b/data/lite/tabpfn/figures/critical-diagram.pdf differ diff --git a/data/lite/tabpfn/figures/critical-diagram.png.zip b/data/lite/tabpfn/figures/critical-diagram.png.zip index cbd8c146f2380f529762ed99d807e44ba4a91a75..1c635bd27b9cd3a9e41c6f57dc1cea931c861b73 100644 --- a/data/lite/tabpfn/figures/critical-diagram.png.zip +++ b/data/lite/tabpfn/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a80f3a070c9b90eb2b3b529b1b5770762faafdcf943b90a0c663fa4bce83f4ba -size 309083 +oid sha256:428e87c203ab14ca273f49b5ceb010a07a35aed4a17e8f482b8e9f8b8ed8c5a7 +size 309221 diff --git a/data/lite/tabpfn/time_plot.pdf b/data/lite/tabpfn/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8e99cfc108f90193f2144769b3ff012b6c3c2e59 Binary files /dev/null and b/data/lite/tabpfn/time_plot.pdf differ diff --git a/data/lite/tabpfn/time_plot.png.zip b/data/lite/tabpfn/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..9479b76236d88036c864f541fc38df647bb61a5d --- /dev/null +++ b/data/lite/tabpfn/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b74a758d406765245c6f095f4e65d97a18e0bba466cc25bcb4279a6cace3efbc +size 338023 diff --git a/data/lite/tabpfn/tuning-impact-elo-horizontal.pdf b/data/lite/tabpfn/tuning-impact-elo-horizontal.pdf index ed99681333859006a61dc9555780af8fd60479f7..15d5db791601260b00788a2b524e6cf9bac0ba07 100644 Binary files a/data/lite/tabpfn/tuning-impact-elo-horizontal.pdf and b/data/lite/tabpfn/tuning-impact-elo-horizontal.pdf differ diff --git a/data/lite/tabpfn/tuning-impact-elo-horizontal.png.zip b/data/lite/tabpfn/tuning-impact-elo-horizontal.png.zip index 8714af302898b55aa7fdaa8b5145e0703276180e..b1b28b666ce791ec8a388e353f3d6ba8b6e95af0 100644 --- a/data/lite/tabpfn/tuning-impact-elo-horizontal.png.zip +++ b/data/lite/tabpfn/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d6829223d4546befaa590b3203a69a424e6c39c904e7414643036ad35f9bb7f2 -size 135098 +oid sha256:dbfbf079198ec860d1fd8e7e9945ceb059cbbf6a46f525a17bffdde3080c7686 +size 135260 diff --git a/data/lite/tabpfn/tuning-impact-elo.pdf b/data/lite/tabpfn/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..720c06260e3781bd27af1a1733a297defa133b07 Binary files /dev/null and b/data/lite/tabpfn/tuning-impact-elo.pdf differ diff --git a/data/lite/tabpfn/tuning-impact-elo.png.zip b/data/lite/tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..789efeaf58db473a48f9d05925bebaa2acfdf6cc --- /dev/null +++ b/data/lite/tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a12fbe0dcf18ba48cdb21b68fb42150b3676ca518eabe25ecb743c02de3011ba +size 129434 diff --git a/data/tabicl-cls/figures/critical-diagram.pdf b/data/tabicl-cls/figures/critical-diagram.pdf index 0052000636bb76edbc0ee7a9a51b7cc8d9ab5867..e9f4ed4bd5277e0d358ed580a1700bba5fba6dc3 100644 Binary files a/data/tabicl-cls/figures/critical-diagram.pdf and b/data/tabicl-cls/figures/critical-diagram.pdf differ diff --git a/data/tabicl-cls/figures/critical-diagram.png.zip b/data/tabicl-cls/figures/critical-diagram.png.zip index eeaf1bd4070bd54d33c25ac9661d9ea7fdfb8040..8d1695d2915780701ba802b2cf73d4b92fe93d82 100644 --- a/data/tabicl-cls/figures/critical-diagram.png.zip +++ b/data/tabicl-cls/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a1b9ef7a3f66ec78df1118b64a21ebb3aa4fc34c4cddd8998137f78f590184b3 -size 312248 +oid sha256:d2502c6434894596f06861b6d42fe317ddd22c970bfb8027312afa42ef68862d +size 312386 diff --git a/data/tabicl-cls/time_plot.pdf b/data/tabicl-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..25acbc0be48753dafe35c5d41d88199779837e85 Binary files /dev/null and b/data/tabicl-cls/time_plot.pdf differ diff --git a/data/tabicl-cls/time_plot.png.zip b/data/tabicl-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e179036fa49ffdead5c7d5fe3255c36f633a6352 --- /dev/null +++ b/data/tabicl-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4f80795d7679a6b36d66608bc689367a29d1910801171cb273b6195957cd591 +size 338716 diff --git a/data/tabicl-cls/tuning-impact-elo-horizontal.pdf b/data/tabicl-cls/tuning-impact-elo-horizontal.pdf index c88e63db62070ccf4e12a45f643893c3393e8360..ab1ffd4ab2331049f8ef921e5e1615cc6b75de93 100644 Binary files a/data/tabicl-cls/tuning-impact-elo-horizontal.pdf and b/data/tabicl-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabicl-cls/tuning-impact-elo-horizontal.png.zip b/data/tabicl-cls/tuning-impact-elo-horizontal.png.zip index 9ba326e93cb3d1dedb1b9ee2b47b2ff585feedb3..808524b1e8f4ee3ac513674c9ebc9b17c9e8a1ba 100644 --- a/data/tabicl-cls/tuning-impact-elo-horizontal.png.zip +++ b/data/tabicl-cls/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e858c95af701d1a8d5d6368f1a8c3669f399c2229854ccff0595e69c3ea3da0d -size 132739 +oid sha256:7e8b39532d8a56d174431e8b65ffa77cf6673c31bc91b58d8d37de413690eabb +size 132901 diff --git a/data/tabicl-cls/tuning-impact-elo.pdf b/data/tabicl-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6c94d6433686991088343c8e7e363f7e347501b6 Binary files /dev/null and b/data/tabicl-cls/tuning-impact-elo.pdf differ diff --git a/data/tabicl-cls/tuning-impact-elo.png.zip b/data/tabicl-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..6859885270fb74d7ce9981fe86132143c96f70ff --- /dev/null +++ b/data/tabicl-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb5328dbacd58e0f3092c6640fb142fe340d20c7d7c0d8089746290a22473670 +size 128063 diff --git a/data/tabicl-imputed-cls/figures/critical-diagram.pdf b/data/tabicl-imputed-cls/figures/critical-diagram.pdf index bf843ada801ed3435fd1bddf53b7a79dd4f5e169..07a2f7ac5bb3c37ae5c5dd4ff8d755bef33b4cef 100644 Binary files a/data/tabicl-imputed-cls/figures/critical-diagram.pdf and b/data/tabicl-imputed-cls/figures/critical-diagram.pdf differ diff --git a/data/tabicl-imputed-cls/figures/critical-diagram.png.zip b/data/tabicl-imputed-cls/figures/critical-diagram.png.zip index dbc8af23f2ed54c53adf16bb2f35a7df366e96ed..fef7f5a7f623d678d619b9e15e4586e704921c78 100644 --- a/data/tabicl-imputed-cls/figures/critical-diagram.png.zip +++ b/data/tabicl-imputed-cls/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:735c80d30000ae29ff299d889176259c503a407d4a93ab4fa31fe0124709c252 -size 318459 +oid sha256:a34150c78f1b02cd447b412fed3b21ae952293517190b22587ddc398ef92f4a0 +size 318597 diff --git a/data/tabicl-imputed-cls/time_plot.pdf b/data/tabicl-imputed-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..27c26c73da0ce80878b6f543bd7c22f66f2e7795 Binary files /dev/null and b/data/tabicl-imputed-cls/time_plot.pdf differ diff --git a/data/tabicl-imputed-cls/time_plot.png.zip b/data/tabicl-imputed-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..222bc672fb4f2f1ba0255735bf24ac1fb19255a3 --- /dev/null +++ b/data/tabicl-imputed-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7c009d5eef43bbcc42dcce1ae2656319f927f899f961c9daa1bbe9730237b1 +size 338716 diff --git a/data/tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf b/data/tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf index 2744db91c9ad47cd90aee02f30bc56b5a6f89687..3fdedbbab219bfe65b97da92bca409b85a420531 100644 Binary files a/data/tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf and b/data/tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip b/data/tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip index 239598955be83ce2de1491934bc83495b5eb0560..35b58f8e3d47296ba20d4b47f0000593a52b7190 100644 --- a/data/tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip +++ b/data/tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:69cec3024c196362180fc485a00abd771d989a4cea1aa97071eb9351886ced78 -size 152921 +oid sha256:bc150a848e67cec17b93063f9144a3b0d55382688690fe324caf94a0837d2fdf +size 153083 diff --git a/data/tabicl-imputed-cls/tuning-impact-elo.pdf b/data/tabicl-imputed-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e9634e66fc425804fb0e173e6d79a59be520e9ed Binary files /dev/null and b/data/tabicl-imputed-cls/tuning-impact-elo.pdf differ diff --git a/data/tabicl-imputed-cls/tuning-impact-elo.png.zip b/data/tabicl-imputed-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7e8302fe0f0a7da6a92552a2d113662aa4bcc10c --- /dev/null +++ b/data/tabicl-imputed-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:230e4310243c53b0ff982e9b3490b2ec6e2e91dd15dd5803f00bf215e850a9bd +size 159604 diff --git a/data/tabicl-imputed/figures/critical-diagram.pdf b/data/tabicl-imputed/figures/critical-diagram.pdf index f32dec8a66eb31cc62912db42102455e71e2c287..fcb919f92e516e9082ed44f12ff3537241d3de37 100644 Binary files a/data/tabicl-imputed/figures/critical-diagram.pdf and b/data/tabicl-imputed/figures/critical-diagram.pdf differ diff --git a/data/tabicl-imputed/figures/critical-diagram.png.zip b/data/tabicl-imputed/figures/critical-diagram.png.zip index dbc8af23f2ed54c53adf16bb2f35a7df366e96ed..2c84ec5bc0b25ce2d7aa435d17602be4b35f2248 100644 --- a/data/tabicl-imputed/figures/critical-diagram.png.zip +++ b/data/tabicl-imputed/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:735c80d30000ae29ff299d889176259c503a407d4a93ab4fa31fe0124709c252 -size 318459 +oid sha256:50ed7229753452c38683f24a0f4716cc96e0c5b83a7dd9121ea47aebdb1347f7 +size 318597 diff --git a/data/tabicl-imputed/time_plot.pdf b/data/tabicl-imputed/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7555b3311b3960cb36389790418d76a716f8234b Binary files /dev/null and b/data/tabicl-imputed/time_plot.pdf differ diff --git a/data/tabicl-imputed/time_plot.png.zip b/data/tabicl-imputed/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3fcc862db8a6e55c007ddb123b6aa6c94eb85d31 --- /dev/null +++ b/data/tabicl-imputed/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9f89e695c44673ec69301a489815efae40bdf8dc94b54420d891b4c1f37afbf +size 338716 diff --git a/data/tabicl-imputed/tuning-impact-elo-horizontal.pdf b/data/tabicl-imputed/tuning-impact-elo-horizontal.pdf index 49cf6a8ead3cc58d08a5720949edacbda39de184..c7e5764f313946ff5e646367724798f7e908ffa5 100644 Binary files a/data/tabicl-imputed/tuning-impact-elo-horizontal.pdf and b/data/tabicl-imputed/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabicl-imputed/tuning-impact-elo-horizontal.png.zip b/data/tabicl-imputed/tuning-impact-elo-horizontal.png.zip index 239598955be83ce2de1491934bc83495b5eb0560..3d85b32395babed3717c2a5b8b417b48f954c830 100644 --- a/data/tabicl-imputed/tuning-impact-elo-horizontal.png.zip +++ b/data/tabicl-imputed/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:69cec3024c196362180fc485a00abd771d989a4cea1aa97071eb9351886ced78 -size 152921 +oid sha256:1ebe5b1f0f84c9a1a3642f9dbab5a5ca02216cd63c9ff9383f56efaf1b269ea3 +size 153083 diff --git a/data/tabicl-imputed/tuning-impact-elo.pdf b/data/tabicl-imputed/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d6ac8fb0ec00d8e24ccd229535710e0721f3f646 Binary files /dev/null and b/data/tabicl-imputed/tuning-impact-elo.pdf differ diff --git a/data/tabicl-imputed/tuning-impact-elo.png.zip b/data/tabicl-imputed/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..9e73d1bb7944b47b81c1a63478832cb0c3bc7cb3 --- /dev/null +++ b/data/tabicl-imputed/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8035a9ec5e642609902b7fe8702f9a8d6e031f7e3a6b859e9287618fb0d5e791 +size 159604 diff --git a/data/tabicl/figures/critical-diagram.pdf b/data/tabicl/figures/critical-diagram.pdf index b17532e0edcd586e793714208586b2412d92e694..fde15e430052f3e9e6e240f4c8d16efa5131c82d 100644 Binary files a/data/tabicl/figures/critical-diagram.pdf and b/data/tabicl/figures/critical-diagram.pdf differ diff --git a/data/tabicl/figures/critical-diagram.png.zip b/data/tabicl/figures/critical-diagram.png.zip index eeaf1bd4070bd54d33c25ac9661d9ea7fdfb8040..66bf92d2d6c7603c64d7d9dc631bcf136ab10657 100644 --- a/data/tabicl/figures/critical-diagram.png.zip +++ b/data/tabicl/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a1b9ef7a3f66ec78df1118b64a21ebb3aa4fc34c4cddd8998137f78f590184b3 -size 312248 +oid sha256:483950fed73ebf8c990a96f07a836a726b4b6796a893f3b98b66a13be74267ff +size 312386 diff --git a/data/tabicl/time_plot.pdf b/data/tabicl/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ba8acd8b418c741006a6e4b7662ee280316523e1 Binary files /dev/null and b/data/tabicl/time_plot.pdf differ diff --git a/data/tabicl/time_plot.png.zip b/data/tabicl/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..29c319ab68f4a31661010614087161f93685444d --- /dev/null +++ b/data/tabicl/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d79c123c11a1646193bfc22afd8c728ab6023332ec221c2efdac618d805aa92 +size 338716 diff --git a/data/tabicl/tuning-impact-elo-horizontal.pdf b/data/tabicl/tuning-impact-elo-horizontal.pdf index ea328edb768a938cc1cd91dd4625f5cfeeba78ea..0875ba8aff7852efe1c2c5a96bad765d7a47b57e 100644 Binary files a/data/tabicl/tuning-impact-elo-horizontal.pdf and b/data/tabicl/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabicl/tuning-impact-elo-horizontal.png.zip b/data/tabicl/tuning-impact-elo-horizontal.png.zip index 9ba326e93cb3d1dedb1b9ee2b47b2ff585feedb3..1a86df5c1685e3e4c6b9d7b3c1650d6ae9e4b034 100644 --- a/data/tabicl/tuning-impact-elo-horizontal.png.zip +++ b/data/tabicl/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e858c95af701d1a8d5d6368f1a8c3669f399c2229854ccff0595e69c3ea3da0d -size 132739 +oid sha256:9c668fec6a2521cbec8d483f9e7dbfb13f2d061183274e18c94fbb3c9a9ab8a0 +size 132901 diff --git a/data/tabicl/tuning-impact-elo.pdf b/data/tabicl/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d6e31ca05000fe35ab9e5514cc48e05632e399b6 Binary files /dev/null and b/data/tabicl/tuning-impact-elo.pdf differ diff --git a/data/tabicl/tuning-impact-elo.png.zip b/data/tabicl/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ebad148c0750bc5b99903259e6ad76a9711269a1 --- /dev/null +++ b/data/tabicl/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:055d9e390eaaf1b6df979f20cf4b309a447cd6aa83ec95e5d2ad85cd2dd5112e +size 128063 diff --git a/data/tabpfn-cls/figures/critical-diagram.pdf b/data/tabpfn-cls/figures/critical-diagram.pdf index 949ed98e2d1001003a66751c800bfd5dddce1761..341e7db67d7a3584ed8256943d5b3511c0068a6c 100644 Binary files a/data/tabpfn-cls/figures/critical-diagram.pdf and b/data/tabpfn-cls/figures/critical-diagram.pdf differ diff --git a/data/tabpfn-cls/figures/critical-diagram.png.zip b/data/tabpfn-cls/figures/critical-diagram.png.zip index c621877d3cafbab56387efc63b4f34b1b84d9942..4b558d2440a82cb15924bc3d2432988aa8bd880a 100644 --- a/data/tabpfn-cls/figures/critical-diagram.png.zip +++ b/data/tabpfn-cls/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5777d71a73c53eaab6e2d3e915fddcdb926df5cae2f76e45b5ac8305230a8a2b -size 315193 +oid sha256:951a7c25c48294e13064121a668983545a168251f73be8b780f23ba559dfcce1 +size 315331 diff --git a/data/tabpfn-cls/time_plot.pdf b/data/tabpfn-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..30b82ec8e8b1ccac70bf7bb4a55eb1bfe333d67c Binary files /dev/null and b/data/tabpfn-cls/time_plot.pdf differ diff --git a/data/tabpfn-cls/time_plot.png.zip b/data/tabpfn-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..737678f94b8cdab7c02c4a07ae8c28818692ff1e --- /dev/null +++ b/data/tabpfn-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:144ee945a4343e0d23ed552e73c5775cdb6cd579c7694a3a8819b4af5972a692 +size 338716 diff --git a/data/tabpfn-cls/tuning-impact-elo-horizontal.pdf b/data/tabpfn-cls/tuning-impact-elo-horizontal.pdf index d8f8e5e7bfab42820d989c4d6c23c26f152c88a9..0e9ea13e731cbc1f03d2ddbcb848a36c05be0be5 100644 Binary files a/data/tabpfn-cls/tuning-impact-elo-horizontal.pdf and b/data/tabpfn-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabpfn-cls/tuning-impact-elo-horizontal.png.zip b/data/tabpfn-cls/tuning-impact-elo-horizontal.png.zip index 5eb5025d2eec888d186b09eb6c45746e09b7076a..ab75e3cc05da9d5d030bbcf210555f83288c6d35 100644 --- a/data/tabpfn-cls/tuning-impact-elo-horizontal.png.zip +++ b/data/tabpfn-cls/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e1edd5fcf6f47ea36a683f43b329716a9ae9358b12b1ee57dfd78dd4cac854ee -size 135646 +oid sha256:eec50403b73020705730e3ae9d1c6f69ec7a1e04e5f011ccdeadde3f127d2637 +size 135808 diff --git a/data/tabpfn-cls/tuning-impact-elo.pdf b/data/tabpfn-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b08d43ca01018d75eea61e17d1d6d961f44d1f83 Binary files /dev/null and b/data/tabpfn-cls/tuning-impact-elo.pdf differ diff --git a/data/tabpfn-cls/tuning-impact-elo.png.zip b/data/tabpfn-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..f0143dca8564068bbed481823035eb418fbd00ea --- /dev/null +++ b/data/tabpfn-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b8aa0e3855f12ba506d90c177e47ee763b0314103d0af4be4a139bc0b81def6 +size 134937 diff --git a/data/tabpfn-imputed-cls/figures/critical-diagram.pdf b/data/tabpfn-imputed-cls/figures/critical-diagram.pdf index 4f5ac9425f25cf258b0de0477760c1fa95cdc716..cd7e3f199bcaa947af7cff734e5ce3c663c02f8d 100644 Binary files a/data/tabpfn-imputed-cls/figures/critical-diagram.pdf and b/data/tabpfn-imputed-cls/figures/critical-diagram.pdf differ diff --git a/data/tabpfn-imputed-cls/figures/critical-diagram.png.zip b/data/tabpfn-imputed-cls/figures/critical-diagram.png.zip index 24e08506593fee7a8727aefce7940752470d6e76..6749bb0317fd0299424d704e6cf9266f6bc807ce 100644 --- a/data/tabpfn-imputed-cls/figures/critical-diagram.png.zip +++ b/data/tabpfn-imputed-cls/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:09fd50f8530f0264c02045700850c01bf4b212bcd6257fa3321f09e02e249a62 -size 319568 +oid sha256:10929267b5db855c9883c1fa9b492892b1a96fb5bdd1e3ff68740d573a1a3a6f +size 319706 diff --git a/data/tabpfn-imputed-cls/time_plot.pdf b/data/tabpfn-imputed-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..eb91df942540c3e5dd077670990951401e673a89 Binary files /dev/null and b/data/tabpfn-imputed-cls/time_plot.pdf differ diff --git a/data/tabpfn-imputed-cls/time_plot.png.zip b/data/tabpfn-imputed-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..22239e3dcc4991ed45512f22e0bdce27272e4f06 --- /dev/null +++ b/data/tabpfn-imputed-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57576e9c18637303585efa659f985e0740b4b9a573522ce92883d155fc86cbe4 +size 338716 diff --git a/data/tabpfn-imputed-cls/tuning-impact-elo-horizontal.pdf b/data/tabpfn-imputed-cls/tuning-impact-elo-horizontal.pdf index aa0fc43af3f5a7832789cced59f877ce408ba43a..b2c2f772f1a1c89df5e095719912a4d3c91a4371 100644 Binary files a/data/tabpfn-imputed-cls/tuning-impact-elo-horizontal.pdf and b/data/tabpfn-imputed-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabpfn-imputed-cls/tuning-impact-elo-horizontal.png.zip b/data/tabpfn-imputed-cls/tuning-impact-elo-horizontal.png.zip index 24481787fa054a7bc9a1fd8d8bbf394bb80459fd..c0bfbc2241255a21cae5ab62372b8f39be0cf223 100644 --- a/data/tabpfn-imputed-cls/tuning-impact-elo-horizontal.png.zip +++ b/data/tabpfn-imputed-cls/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4c7ba763238c3b186cb800f076efde71d7f2bd932c98b1a297881496602be9e3 -size 153061 +oid sha256:af0922de7055ca6f9abc573c902b841d76e30dd525dcf67acfed4945c32339e4 +size 153223 diff --git a/data/tabpfn-imputed-cls/tuning-impact-elo.pdf b/data/tabpfn-imputed-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2083fcdbcff16a56f83620f7d49676872db7e882 Binary files /dev/null and b/data/tabpfn-imputed-cls/tuning-impact-elo.pdf differ diff --git a/data/tabpfn-imputed-cls/tuning-impact-elo.png.zip b/data/tabpfn-imputed-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..4300f39c94eeceec2fd5ec53153a03394732393b --- /dev/null +++ b/data/tabpfn-imputed-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f77a2ca20e44a6a4232cafab1fc24604fd76a701034ef73003769f618029d13b +size 153288 diff --git a/data/tabpfn-imputed-reg/figures/critical-diagram.pdf b/data/tabpfn-imputed-reg/figures/critical-diagram.pdf index c2d8f64d091632b223a523520ca37f310a802f56..35b35138c4eef01ba15aca4c4173eca0872ebd36 100644 Binary files a/data/tabpfn-imputed-reg/figures/critical-diagram.pdf and b/data/tabpfn-imputed-reg/figures/critical-diagram.pdf differ diff --git a/data/tabpfn-imputed-reg/figures/critical-diagram.png.zip b/data/tabpfn-imputed-reg/figures/critical-diagram.png.zip index 5dde710223f33796aaf8b252ca24d174bb54c2c5..f2ffd58fed6d5a2e2f91b2bac4152c509797fc07 100644 --- a/data/tabpfn-imputed-reg/figures/critical-diagram.png.zip +++ b/data/tabpfn-imputed-reg/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:719a41d0fa24badcd2e52ffd016be9f940e6c6ec35e8da65cf5d12e237671d0b -size 321964 +oid sha256:2d3f17a5c5982457aad809afac4a9991da40e90e454dec19d36391c9185204cf +size 322102 diff --git a/data/tabpfn-imputed-reg/time_plot.pdf b/data/tabpfn-imputed-reg/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1d775b5a491aec8771ffa03a4737a5bbdfbef4ba Binary files /dev/null and b/data/tabpfn-imputed-reg/time_plot.pdf differ diff --git a/data/tabpfn-imputed-reg/time_plot.png.zip b/data/tabpfn-imputed-reg/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..b9b4c6eaa84756808f647aa946f97fd8050a3b38 --- /dev/null +++ b/data/tabpfn-imputed-reg/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b378d35bee402c5bf5f90d0218b0ac16f6477f4f752cd05dad986aab38a85db +size 80035 diff --git a/data/tabpfn-imputed-reg/tuning-impact-elo-horizontal.pdf b/data/tabpfn-imputed-reg/tuning-impact-elo-horizontal.pdf index 1145b046e5fb944e340ac9b672f9d2726e6f35df..4ff5eb1c03b43d096485a324058ee583376c66a5 100644 Binary files a/data/tabpfn-imputed-reg/tuning-impact-elo-horizontal.pdf and b/data/tabpfn-imputed-reg/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabpfn-imputed-reg/tuning-impact-elo-horizontal.png.zip b/data/tabpfn-imputed-reg/tuning-impact-elo-horizontal.png.zip index fd6e402fb997d49c8c621af728c7533dc827169a..ef89646f1187ef870526f58ea2f3e9c6bf6e0fe2 100644 --- a/data/tabpfn-imputed-reg/tuning-impact-elo-horizontal.png.zip +++ b/data/tabpfn-imputed-reg/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:170a46b3e8d59d2676db2220300c9c410a9b23398a21d08bdb11d476a44059e7 -size 151423 +oid sha256:59b9c85e87f79fa3f58b140634f00f1d09ac5752aa3b58772afb71b210e6c3d0 +size 151585 diff --git a/data/tabpfn-imputed-reg/tuning-impact-elo.pdf b/data/tabpfn-imputed-reg/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6486965586927de7f7a5301546a318a8fce958f4 Binary files /dev/null and b/data/tabpfn-imputed-reg/tuning-impact-elo.pdf differ diff --git a/data/tabpfn-imputed-reg/tuning-impact-elo.png.zip b/data/tabpfn-imputed-reg/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..de97784e6a900e1ab91c2b1476dc221223b5cdc4 --- /dev/null +++ b/data/tabpfn-imputed-reg/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc323b52e6bf9871761cd2cdcd0096012ca8bce0a9244c862d31a7b42032dae3 +size 143292 diff --git a/data/tabpfn-imputed/figures/critical-diagram.pdf b/data/tabpfn-imputed/figures/critical-diagram.pdf index b082a10a3ad46e65f0226f5d6337f53575e8a5a1..88e0864bc4e0d73896dfb80c3b2571f0488c28fa 100644 Binary files a/data/tabpfn-imputed/figures/critical-diagram.pdf and b/data/tabpfn-imputed/figures/critical-diagram.pdf differ diff --git a/data/tabpfn-imputed/figures/critical-diagram.png.zip b/data/tabpfn-imputed/figures/critical-diagram.png.zip index 2730348b3362dad5e4aa2ad742a14121f30a86a1..88eba5531a5884ba9d7a50b4368dfce098792194 100644 --- a/data/tabpfn-imputed/figures/critical-diagram.png.zip +++ b/data/tabpfn-imputed/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2c170c7c92d1867222868e8e681c72e06638718de9296857e0851e221bc0cb6f -size 321507 +oid sha256:b0c3bc45c8fae63546c8ecf350af0857c7f267371f65a51ef461bfe6873dfc22 +size 321645 diff --git a/data/tabpfn-imputed/time_plot.pdf b/data/tabpfn-imputed/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..56750f2b6de36b16516c15ccf309490a092fd151 Binary files /dev/null and b/data/tabpfn-imputed/time_plot.pdf differ diff --git a/data/tabpfn-imputed/time_plot.png.zip b/data/tabpfn-imputed/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..cdb11148d87799da9f3988a3bb6b149d39bced75 --- /dev/null +++ b/data/tabpfn-imputed/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f304f2447dec318a63c222ad868b56674af73070200330037430f0fed495792 +size 338716 diff --git a/data/tabpfn-imputed/tuning-impact-elo-horizontal.pdf b/data/tabpfn-imputed/tuning-impact-elo-horizontal.pdf index 301c3d795424030dd12d16f2a31b8e92988f89b7..e6642763d68de2ba8106415c6c07d5b361268158 100644 Binary files a/data/tabpfn-imputed/tuning-impact-elo-horizontal.pdf and b/data/tabpfn-imputed/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabpfn-imputed/tuning-impact-elo-horizontal.png.zip b/data/tabpfn-imputed/tuning-impact-elo-horizontal.png.zip index f887f59f0106d027450d392018487f91be293dec..0b36e50db1e6dd19e00000cd6c12061f89f7c31e 100644 --- a/data/tabpfn-imputed/tuning-impact-elo-horizontal.png.zip +++ b/data/tabpfn-imputed/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7db4e4ad114130ee65c0ba99801da0bb51caffc0689a155467c1c033c6597cd2 -size 152291 +oid sha256:fcdd19da5a3d7d004426a6a22a32fb7597c868ea73e0553f0e94ef0df51c32ef +size 152453 diff --git a/data/tabpfn-imputed/tuning-impact-elo.pdf b/data/tabpfn-imputed/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3265cd4e221304c3fd887819d1d79ea861dbe0cb Binary files /dev/null and b/data/tabpfn-imputed/tuning-impact-elo.pdf differ diff --git a/data/tabpfn-imputed/tuning-impact-elo.png.zip b/data/tabpfn-imputed/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..178a939d69052a4e59e7c3961b1beeaeb12730c5 --- /dev/null +++ b/data/tabpfn-imputed/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:544a5ba4488874a52fb79877633e8c626245b8d1e4d58e4aa3268ccbf5c97f71 +size 151477 diff --git a/data/tabpfn-reg/figures/critical-diagram.pdf b/data/tabpfn-reg/figures/critical-diagram.pdf index 461623f5a18604f98216ddb0e1ceef1c3e0d8a17..0c5c6a95dd828d806ddb3ed6818f0e0723e01888 100644 Binary files a/data/tabpfn-reg/figures/critical-diagram.pdf and b/data/tabpfn-reg/figures/critical-diagram.pdf differ diff --git a/data/tabpfn-reg/figures/critical-diagram.png.zip b/data/tabpfn-reg/figures/critical-diagram.png.zip index a05d0f1d48213186f97ece9fdf82cfc5228e0f9e..f990cfc5128ec4db484a08c55437fccc7e22ce0d 100644 --- a/data/tabpfn-reg/figures/critical-diagram.png.zip +++ b/data/tabpfn-reg/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:59e7d8b23ce20236b9223953c24d434e9837eecd4166f8b052d21e73427045cd -size 308249 +oid sha256:fa689c2252e478eea980209ede8cbf8082827ea4eaefb9f9236e864729773458 +size 308387 diff --git a/data/tabpfn-reg/time_plot.pdf b/data/tabpfn-reg/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5ad3efd9c7a5ac68647f1490fdba53f5176a9d1d Binary files /dev/null and b/data/tabpfn-reg/time_plot.pdf differ diff --git a/data/tabpfn-reg/time_plot.png.zip b/data/tabpfn-reg/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..01783d35b601fefd25d81af01276762e5402d860 --- /dev/null +++ b/data/tabpfn-reg/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b824602e3bb896af36b5ce4117cf155f6193e9337ee08d9508466145a4d19ce8 +size 80035 diff --git a/data/tabpfn-reg/tuning-impact-elo-horizontal.pdf b/data/tabpfn-reg/tuning-impact-elo-horizontal.pdf index 7e2509b1b563ab555e842f1adcd5b70facd9707c..979856f95319d4093b95c7809f84e6f34536050e 100644 Binary files a/data/tabpfn-reg/tuning-impact-elo-horizontal.pdf and b/data/tabpfn-reg/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabpfn-reg/tuning-impact-elo-horizontal.png.zip b/data/tabpfn-reg/tuning-impact-elo-horizontal.png.zip index 205fe37d17a46a863e2acb6ebb5e834706a4424a..4e211b7d51a3ea03ba3660350a4f669f29867f7e 100644 --- a/data/tabpfn-reg/tuning-impact-elo-horizontal.png.zip +++ b/data/tabpfn-reg/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1907594925200ee1a568c36817d0b22dc6cad8a4f5fa418c697c597e2e57112d -size 135809 +oid sha256:f5e9136d38657ed3160c3d6ea8f5da5ccd5f39bfb1ecfa8e24afeb04d6ee40fa +size 135971 diff --git a/data/tabpfn-reg/tuning-impact-elo.pdf b/data/tabpfn-reg/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..bdc8280fa4e842cd1a49092bdcd08ab1bd0b56a7 Binary files /dev/null and b/data/tabpfn-reg/tuning-impact-elo.pdf differ diff --git a/data/tabpfn-reg/tuning-impact-elo.png.zip b/data/tabpfn-reg/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..a053309ff9bcbb7d766eda69a1b6c83e0cfa65ef --- /dev/null +++ b/data/tabpfn-reg/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a86d2eba712599bd9f9b51c81d669983559b649ea6174c5aa9dfc113bbd3bea +size 133756 diff --git a/data/tabpfn-tabicl-cls/figures/critical-diagram.pdf b/data/tabpfn-tabicl-cls/figures/critical-diagram.pdf index 54539b5f41dc2f5a4e1b4747a53d81add750da31..d59e6b07fe195e2030bda6210dd2ea858c90f1e4 100644 Binary files a/data/tabpfn-tabicl-cls/figures/critical-diagram.pdf and b/data/tabpfn-tabicl-cls/figures/critical-diagram.pdf differ diff --git a/data/tabpfn-tabicl-cls/figures/critical-diagram.png.zip b/data/tabpfn-tabicl-cls/figures/critical-diagram.png.zip index 24e08506593fee7a8727aefce7940752470d6e76..43831c9f48b18d483f273149394b3b248325c603 100644 --- a/data/tabpfn-tabicl-cls/figures/critical-diagram.png.zip +++ b/data/tabpfn-tabicl-cls/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:09fd50f8530f0264c02045700850c01bf4b212bcd6257fa3321f09e02e249a62 -size 319568 +oid sha256:f7ef5039b3117732fa25fb3d00a2d97959c5bf818cb1f5716bbecb2c7430a8e4 +size 319706 diff --git a/data/tabpfn-tabicl-cls/time_plot.pdf b/data/tabpfn-tabicl-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2b9f993031020bd7b3129d1c281523a4a0a8de24 Binary files /dev/null and b/data/tabpfn-tabicl-cls/time_plot.pdf differ diff --git a/data/tabpfn-tabicl-cls/time_plot.png.zip b/data/tabpfn-tabicl-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..3175d82c7fd3d754e00e0835d6ee7bc7a10440cf --- /dev/null +++ b/data/tabpfn-tabicl-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:464064e8388d063acb40d2c01dc7980147f8aca5c73df3b5150d51c5ddd52f16 +size 338716 diff --git a/data/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.pdf b/data/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.pdf index b9e514d68368e1e6c78f4d305ba10f64b756fcf1..bd12d7814b0cfef3f0f181688d1226b4bb49392f 100644 Binary files a/data/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.pdf and b/data/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.png.zip b/data/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.png.zip index 4f6908fba6c41ba3576608b66bc23a0647bbfbc2..ece6b70ce44536ba823ae97f35de3865792ffd28 100644 --- a/data/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.png.zip +++ b/data/tabpfn-tabicl-cls/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e190a64928288c6f71df2881dc9fadd779b153432c7535ca3ddd8d9d6b7fa4dd -size 139645 +oid sha256:74d15c9be7d3ed3d5f34f41d7af0b53440b06e864fdd6c15c35774afc5bfc3a9 +size 139807 diff --git a/data/tabpfn-tabicl-cls/tuning-impact-elo.pdf b/data/tabpfn-tabicl-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1498aad20b4b30a93b49cec3ee7819ba53505b50 Binary files /dev/null and b/data/tabpfn-tabicl-cls/tuning-impact-elo.pdf differ diff --git a/data/tabpfn-tabicl-cls/tuning-impact-elo.png.zip b/data/tabpfn-tabicl-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ec3067b09b557c49d0ed84059edf9634b5be2976 --- /dev/null +++ b/data/tabpfn-tabicl-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c1539b14526812646f06687a42c12c4642fa9f64c61620d655a3b5b2bcb1cf1 +size 135527 diff --git a/data/tabpfn-tabicl-imputed-cls/figures/critical-diagram.pdf b/data/tabpfn-tabicl-imputed-cls/figures/critical-diagram.pdf index 7b6bfc794b51ee972fdede362d44c2ecd32739f2..5176272d78372530e33cf284d89420dd5f6bb1e8 100644 Binary files a/data/tabpfn-tabicl-imputed-cls/figures/critical-diagram.pdf and b/data/tabpfn-tabicl-imputed-cls/figures/critical-diagram.pdf differ diff --git a/data/tabpfn-tabicl-imputed-cls/figures/critical-diagram.png.zip b/data/tabpfn-tabicl-imputed-cls/figures/critical-diagram.png.zip index 24e08506593fee7a8727aefce7940752470d6e76..aff878e2c0f95fca96be99b17a2e86edfe23d8da 100644 --- a/data/tabpfn-tabicl-imputed-cls/figures/critical-diagram.png.zip +++ b/data/tabpfn-tabicl-imputed-cls/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:09fd50f8530f0264c02045700850c01bf4b212bcd6257fa3321f09e02e249a62 -size 319568 +oid sha256:9c116ae76bdbf0c16b757fcd4fb99bf9dff92093afe1ae055f0aa1b4ac575daf +size 319706 diff --git a/data/tabpfn-tabicl-imputed-cls/time_plot.pdf b/data/tabpfn-tabicl-imputed-cls/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f58b0f0d8817fc78ede329eeefd18dcc30e6336e Binary files /dev/null and b/data/tabpfn-tabicl-imputed-cls/time_plot.pdf differ diff --git a/data/tabpfn-tabicl-imputed-cls/time_plot.png.zip b/data/tabpfn-tabicl-imputed-cls/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..21f4a5b0929873f952cce1ae0afdb47f5e1352cf --- /dev/null +++ b/data/tabpfn-tabicl-imputed-cls/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:514f40c5155cc8204482880d4ee9b5438cee97b36fd0a5afa026c5d69c5bfd67 +size 338716 diff --git a/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf b/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf index fbaa6eee7337be014b3a03eb5a7fbb69000bda9a..9fa27fdff83a674b3df5325e26777259501c5d7c 100644 Binary files a/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf and b/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip b/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip index 4f6908fba6c41ba3576608b66bc23a0647bbfbc2..a7c2e318739b38370bb18d13a893475f8d534a88 100644 --- a/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip +++ b/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e190a64928288c6f71df2881dc9fadd779b153432c7535ca3ddd8d9d6b7fa4dd -size 139645 +oid sha256:876e6af8d9335d67bf0bd49d3f74f574a833d4c52982a734e56a70fa451be490 +size 139807 diff --git a/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo.pdf b/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..dddd7facc6bcb171993743c9694fedb7bb4d3cbc Binary files /dev/null and b/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo.pdf differ diff --git a/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo.png.zip b/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..ef4a56fd319a709938c43d53910daf0607c9d407 --- /dev/null +++ b/data/tabpfn-tabicl-imputed-cls/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:017a21afedfc06671d8cfdd8294ea1ab1400301f07c63028b48ba2551cc36a76 +size 135527 diff --git a/data/tabpfn-tabicl-imputed/figures/critical-diagram.pdf b/data/tabpfn-tabicl-imputed/figures/critical-diagram.pdf index ba1d1bdd9236c4ba3a782b6e642fa34df19a80d4..c5ed0f645699f670a3e020a684ca4f96dd90e095 100644 Binary files a/data/tabpfn-tabicl-imputed/figures/critical-diagram.pdf and b/data/tabpfn-tabicl-imputed/figures/critical-diagram.pdf differ diff --git a/data/tabpfn-tabicl-imputed/figures/critical-diagram.png.zip b/data/tabpfn-tabicl-imputed/figures/critical-diagram.png.zip index 24e08506593fee7a8727aefce7940752470d6e76..148b3556646471a5e139316daf72493d2841f3a1 100644 --- a/data/tabpfn-tabicl-imputed/figures/critical-diagram.png.zip +++ b/data/tabpfn-tabicl-imputed/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:09fd50f8530f0264c02045700850c01bf4b212bcd6257fa3321f09e02e249a62 -size 319568 +oid sha256:9158cf54ecab62ab4d9dcdab05193ab842f9c4ab510f16531825f39ca1000921 +size 319706 diff --git a/data/tabpfn-tabicl-imputed/time_plot.pdf b/data/tabpfn-tabicl-imputed/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..4d913a6f4576c4538deffaefb4e0244eafdeee8a Binary files /dev/null and b/data/tabpfn-tabicl-imputed/time_plot.pdf differ diff --git a/data/tabpfn-tabicl-imputed/time_plot.png.zip b/data/tabpfn-tabicl-imputed/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e07b85213c2dac55cb9adc5ee1cff63ae406c28a --- /dev/null +++ b/data/tabpfn-tabicl-imputed/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a08c3d179e5b0f71e7f3d25305df2eda8e506aa08e605bb0e771fda90697e632 +size 338716 diff --git a/data/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.pdf b/data/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.pdf index 13a8d3fa7c4257472e7dc2359f387fd5e33d9ab2..7026c5eb569558b2b82a918d7495f44555d9ccdc 100644 Binary files a/data/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.pdf and b/data/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.png.zip b/data/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.png.zip index 4f6908fba6c41ba3576608b66bc23a0647bbfbc2..2707cb6b55b75930e8e23133c25ea00f7e4a2ef7 100644 --- a/data/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.png.zip +++ b/data/tabpfn-tabicl-imputed/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e190a64928288c6f71df2881dc9fadd779b153432c7535ca3ddd8d9d6b7fa4dd -size 139645 +oid sha256:947759ed4f4091b086a5ab0a8e54faf3e2dbad77c99c775a84678b6fda053d87 +size 139807 diff --git a/data/tabpfn-tabicl-imputed/tuning-impact-elo.pdf b/data/tabpfn-tabicl-imputed/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..aee1492cda38a7a9ac0cee5671fcdba0edf9dae0 Binary files /dev/null and b/data/tabpfn-tabicl-imputed/tuning-impact-elo.pdf differ diff --git a/data/tabpfn-tabicl-imputed/tuning-impact-elo.png.zip b/data/tabpfn-tabicl-imputed/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..e67da532b63be36d150d5c56964dfc48873b8340 --- /dev/null +++ b/data/tabpfn-tabicl-imputed/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11512c2014d1a1e1116cdfde1dbc43f27bd98e3569950c4bd59853ad325bc68b +size 135527 diff --git a/data/tabpfn-tabicl/figures/critical-diagram.pdf b/data/tabpfn-tabicl/figures/critical-diagram.pdf index cf84ff5e66db01f6a8ffae5075a7b2c6a22ead09..2d018f7b8705dba4b6f337086cc2955140d2f732 100644 Binary files a/data/tabpfn-tabicl/figures/critical-diagram.pdf and b/data/tabpfn-tabicl/figures/critical-diagram.pdf differ diff --git a/data/tabpfn-tabicl/figures/critical-diagram.png.zip b/data/tabpfn-tabicl/figures/critical-diagram.png.zip index 24e08506593fee7a8727aefce7940752470d6e76..18b553f494048e2d2276057280c00419c8cd909f 100644 --- a/data/tabpfn-tabicl/figures/critical-diagram.png.zip +++ b/data/tabpfn-tabicl/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:09fd50f8530f0264c02045700850c01bf4b212bcd6257fa3321f09e02e249a62 -size 319568 +oid sha256:3d329afabf21a31f1e2bb1ec30bd8071b5500091178a7adc429a6efb71638990 +size 319706 diff --git a/data/tabpfn-tabicl/time_plot.pdf b/data/tabpfn-tabicl/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8603ccfca9f7e6679284679a9a102e7173183ae7 Binary files /dev/null and b/data/tabpfn-tabicl/time_plot.pdf differ diff --git a/data/tabpfn-tabicl/time_plot.png.zip b/data/tabpfn-tabicl/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..15786c17b8439f17f0a97be662e3f8d76c9f354b --- /dev/null +++ b/data/tabpfn-tabicl/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbc5562ef49800c4d5b34e63fa6a7d5ab48767e47f5efe57111f7314b254d67c +size 338716 diff --git a/data/tabpfn-tabicl/tuning-impact-elo-horizontal.pdf b/data/tabpfn-tabicl/tuning-impact-elo-horizontal.pdf index a739d2a84e525341cf68a8c79feb8cc88cf557e0..cc73960e6d8fbb793af663f1a63fe4b4d01fc8dc 100644 Binary files a/data/tabpfn-tabicl/tuning-impact-elo-horizontal.pdf and b/data/tabpfn-tabicl/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabpfn-tabicl/tuning-impact-elo-horizontal.png.zip b/data/tabpfn-tabicl/tuning-impact-elo-horizontal.png.zip index 4f6908fba6c41ba3576608b66bc23a0647bbfbc2..252eff2cf43c98c21998431c77cf182c298c90da 100644 --- a/data/tabpfn-tabicl/tuning-impact-elo-horizontal.png.zip +++ b/data/tabpfn-tabicl/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e190a64928288c6f71df2881dc9fadd779b153432c7535ca3ddd8d9d6b7fa4dd -size 139645 +oid sha256:56f92b86299ba6275e157ed18442f0f67ba40c51aaae2f72239eff9d94c22556 +size 139807 diff --git a/data/tabpfn-tabicl/tuning-impact-elo.pdf b/data/tabpfn-tabicl/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..880efcf29c2d975110f43c6c2c926ae0059987b6 Binary files /dev/null and b/data/tabpfn-tabicl/tuning-impact-elo.pdf differ diff --git a/data/tabpfn-tabicl/tuning-impact-elo.png.zip b/data/tabpfn-tabicl/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..6dfc52ee162d5fbfeee9f3bce5dfe1b1db48556f --- /dev/null +++ b/data/tabpfn-tabicl/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fd59a79bcdd6ca93a0abbb9d4e2f112b79f28906d3d9bd2bd80c49ddca7d159 +size 135527 diff --git a/data/tabpfn/figures/critical-diagram.pdf b/data/tabpfn/figures/critical-diagram.pdf index e6cbb3851345655347fab9ec9267a7e19e4627ce..1dbdb87296250beceb671e6d867901cbbcc360a6 100644 Binary files a/data/tabpfn/figures/critical-diagram.pdf and b/data/tabpfn/figures/critical-diagram.pdf differ diff --git a/data/tabpfn/figures/critical-diagram.png.zip b/data/tabpfn/figures/critical-diagram.png.zip index 2e996afbc8c6002ff3fefffd6b0d48a63e2f0a23..014c521179274903f0e157fe8e863b587896c60a 100644 --- a/data/tabpfn/figures/critical-diagram.png.zip +++ b/data/tabpfn/figures/critical-diagram.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7adff6e83d6bdc6996b4755fa44a9b4b221ada33465fbef54eea45ac9c3b93e8 -size 316465 +oid sha256:1de163b5f08cece03e14f0c99057c871810f04760a176b35f51213c48a38f9fc +size 316603 diff --git a/data/tabpfn/time_plot.pdf b/data/tabpfn/time_plot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1b5e95cb937c3bc03ad710b7cb347f6ed2dd0680 Binary files /dev/null and b/data/tabpfn/time_plot.pdf differ diff --git a/data/tabpfn/time_plot.png.zip b/data/tabpfn/time_plot.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..7284ae3772734fdcd85eaa780520b4356a77f61c --- /dev/null +++ b/data/tabpfn/time_plot.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8025b337ec722bdefbe1c3df5a0427f152e7581e76f6a2fed4421d7369ebb574 +size 338716 diff --git a/data/tabpfn/tuning-impact-elo-horizontal.pdf b/data/tabpfn/tuning-impact-elo-horizontal.pdf index 1b43f99538ce393e70bc09f8078cc31ba867c1af..f9eb934f3e6873334157de6f918d5c6cdbe7869c 100644 Binary files a/data/tabpfn/tuning-impact-elo-horizontal.pdf and b/data/tabpfn/tuning-impact-elo-horizontal.pdf differ diff --git a/data/tabpfn/tuning-impact-elo-horizontal.png.zip b/data/tabpfn/tuning-impact-elo-horizontal.png.zip index 0082e1650b4a3bd25e03e26f768da4c4f07b5581..82e78c4fa3bab1fd4ffc524482ad36d7ca2fd8b2 100644 --- a/data/tabpfn/tuning-impact-elo-horizontal.png.zip +++ b/data/tabpfn/tuning-impact-elo-horizontal.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:96f37ca0e8626cc2055e6be64880740be0f6b8950004c69fc545431190ec8d16 -size 135563 +oid sha256:868698d7be109325380ca3f68f9423f5d30ee55fe61f4d35e4b4b62bb6d4b7a9 +size 135725 diff --git a/data/tabpfn/tuning-impact-elo.pdf b/data/tabpfn/tuning-impact-elo.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8973dd5556a44199fb1660a83583d650dd7293cb Binary files /dev/null and b/data/tabpfn/tuning-impact-elo.pdf differ diff --git a/data/tabpfn/tuning-impact-elo.png.zip b/data/tabpfn/tuning-impact-elo.png.zip new file mode 100644 index 0000000000000000000000000000000000000000..49f63fa4f6d2e3f03fd2c0ab0c5a4ef054624a19 --- /dev/null +++ b/data/tabpfn/tuning-impact-elo.png.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a357df7ffb4b884ca3d09a187d8c42a212f5e7034201c1e230b2dca49a68c2e +size 133237 diff --git a/data_pdfs_to_pngs.py b/data_pdfs_to_pngs.py index 7ffc71aec50f3f62080fa4bbb56e013010dcc1da..765c748a1b3448b594b08c2fca7ac48d45476477 100644 --- a/data_pdfs_to_pngs.py +++ b/data_pdfs_to_pngs.py @@ -1,4 +1,4 @@ -"""Helper script to go from PDF to PNGs we can use in HTML on the LB.""" +"""Helper script to go from PDF to PNG ZIP Files we can use in HTML on the LB.""" from __future__ import annotations @@ -7,6 +7,7 @@ import os from pathlib import Path from pdf2image import convert_from_path +import zipfile root_dir = "./data" pdf_paths = glob.glob(os.path.join(root_dir, "**", "*.pdf"), recursive=True) @@ -15,9 +16,15 @@ pdf_paths = glob.glob(os.path.join(root_dir, "**", "*.pdf"), recursive=True) for pdf_path in pdf_paths: # Relative path to recreate folder structure path_to_pdf = Path(pdf_path).resolve() - path_to_png = path_to_pdf.with_suffix(".png.zip") + path_to_png = path_to_pdf.with_suffix(".png") + path_to_zip = path_to_pdf.with_suffix(".png.zip") print(f"Converting {pdf_path}...") images = convert_from_path(pdf_path, dpi=800) for _i, image in enumerate(images): image.save(path_to_png, "PNG") + + with zipfile.ZipFile(path_to_zip, 'w') as zipf: + zipf.write(path_to_png, arcname=path_to_png.name) + + path_to_png.unlink(missing_ok=True) \ No newline at end of file diff --git a/main.py b/main.py index e6b1529aa30b2363c0a7f21c22695780ff0470e9..2d1c8ee2669343860c5d2817848729a225cdb4e5 100644 --- a/main.py +++ b/main.py @@ -2,6 +2,7 @@ from __future__ import annotations from dataclasses import dataclass from pathlib import Path +import zipfile import gradio as gr import pandas as pd @@ -153,9 +154,11 @@ class LBContainer: def make_overview_image( overview_image_name: str | None, data_source: str = "data" ) -> None: - # No Image to display - path_to_image = str(Path(__file__).parent / data_source / overview_image_name) - gr.Image(path_to_image, label="Leaderboard Overview", show_label=True, height=550) + path_to_image = Path(__file__).parent / data_source / overview_image_name + path_to_image_zip = path_to_image.with_suffix(".png.zip") + with zipfile.ZipFile(path_to_image_zip, 'r') as zipf: + zipf.extractall(path_to_image.parent) + gr.Image(str(path_to_image), label="Leaderboard Overview", show_label=True, height=550) def make_overview_leaderboard(lbs: [LBContainer]): @@ -304,6 +307,7 @@ def make_leaderboard(df_leaderboard: pd.DataFrame) -> Leaderboard: def _add_imputed_logic(df_leaderboard, imputed_map: dict[str, float] | None) -> None: df_leaderboard = df_leaderboard.copy() if imputed_map is not None: + df_leaderboard["Imputed"] = False for model_name, imputed_percentage in imputed_map.items(): if imputed_percentage == 100: # Filter methods that are fully imputed. @@ -326,7 +330,7 @@ def _get_lbs() -> tuple[LBContainer, ...]: ta = LBContainer( name="🏅 TabArena", file_name="full-imputed/tabarena_leaderboard.csv", - overview_image_name="full-imputed/tuning-impact-elo.png.zip", + overview_image_name="full-imputed/tuning-impact-elo.png", blurb="Leaderboard for all datasets including all (imputed) models.", imputed_map={ "TabPFNv2": 35.29, @@ -335,16 +339,15 @@ def _get_lbs() -> tuple[LBContainer, ...]: ) ta_lite = LBContainer( name="TabArena-Lite", - # FIXME: get full-imputed based LB - file_name="lite/full/tabarena_leaderboard.csv", - overview_image_name="lite/full/tuning-impact-elo.png.zip", + file_name="lite/full-imputed/tabarena_leaderboard.csv", + overview_image_name="lite/full-imputed/tuning-impact-elo.png", blurb="Leaderboard for one split (1st fold, 1st repeat) for all datasets including all (imputed) models.", imputed_map=None, ) ta_clf = LBContainer( name="TabArena-Classification", file_name="full-imputed-cls/tabarena_leaderboard.csv", - overview_image_name="full-imputed-cls/tuning-impact-elo.png.zip", + overview_image_name="full-imputed-cls/tuning-impact-elo.png", blurb="Leaderboard for all 38 classification datasets including all (imputed) models.", imputed_map={ "TabPFNv2": 31.58, @@ -355,7 +358,7 @@ def _get_lbs() -> tuple[LBContainer, ...]: name="TabArena-Regression", file_name="full-imputed-reg/tabarena_leaderboard.csv", # FIXME: get overview image without TabICL - overview_image_name="full-imputed-reg/tuning-impact-elo.png.zip", + overview_image_name="full-imputed-reg/tuning-impact-elo.png", blurb="Leaderboard for all 13 regression datasets including all (imputed) models.", imputed_map={ "TabPFNv2": 46.15, @@ -365,7 +368,7 @@ def _get_lbs() -> tuple[LBContainer, ...]: ta_tabicl = LBContainer( name="⚡ TabArena (TabICL-compatible)", file_name="tabicl-imputed/tabarena_leaderboard.csv", - overview_image_name="tabicl-imputed/tuning-impact-elo-horizontal.png.zip", + overview_image_name="tabicl-imputed/tuning-impact-elo.png", blurb="Leaderboard for all 36 datasets within the constraints of TabICL including all (imputed) models.", imputed_map={ "TabPFNv2": 27.78, @@ -374,7 +377,7 @@ def _get_lbs() -> tuple[LBContainer, ...]: ta_tabpfn = LBContainer( name="⚡ TabArena (TabPFN-compatible)", file_name="tabpfn-imputed/tabarena_leaderboard.csv", - overview_image_name="tabpfn-imputed/tuning-impact-elo-horizontal.png.zip", + overview_image_name="tabpfn-imputed/tuning-impact-elo.png", blurb="Leaderboard for all 33 datasets within the constraints of TabPFN including all (imputed) models.", imputed_map={ "TabICL": 21.21, @@ -383,7 +386,7 @@ def _get_lbs() -> tuple[LBContainer, ...]: ta_tabpfn_tabicl = LBContainer( name="TabArena (TabPFN-TabICL-compatible)", file_name="tabpfn-tabicl/tabarena_leaderboard.csv", - overview_image_name="tabpfn-tabicl/tuning-impact-elo-horizontal.png.zip", + overview_image_name="tabpfn-tabicl/tuning-impact-elo.png", blurb="Leaderboard for all 26 datasets within the constraints of TabPFN and TabICL including all models.", ) diff --git a/pyproject.toml b/pyproject.toml index ace5887eed625ad6847009eae07bc6c4c69837ac..65ec0b9b5e1d67b1a93bb0fae705dadfa9add0e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,6 @@ dependencies = [ "gradio-leaderboard==0.0.13", "gradio[oauth]==5.33.2", "pandas>=2.2.3", - "pdf2image" ] [project.optional-dependencies] dev = ["pdf2image"] diff --git a/requirements.txt b/requirements.txt index 8800ed8753adb52c9a6860de78ee7672279ccaa9..41156f513fbb56dee2425479cadbfb851a013131 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ APScheduler -gradio==5.33.2 +gradio[oauth]==5.33.2 gradio_leaderboard==0.0.13 gradio_client pandas