| { | |
| "dataset_revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7", | |
| "mteb_dataset_name": "MassiveIntentClassification", | |
| "mteb_version": "1.1.1", | |
| "test": { | |
| "af": { | |
| "accuracy": 0.27347007397444517, | |
| "accuracy_stderr": 0.016128000841000932, | |
| "f1": 0.2550355191625284, | |
| "f1_stderr": 0.016088946046718993, | |
| "main_score": 0.27347007397444517 | |
| }, | |
| "am": { | |
| "accuracy": 0.10655682582380632, | |
| "accuracy_stderr": 0.010686588128898462, | |
| "f1": 0.09141696317946997, | |
| "f1_stderr": 0.009005648578088152, | |
| "main_score": 0.10655682582380632 | |
| }, | |
| "ar": { | |
| "accuracy": 0.1734700739744452, | |
| "accuracy_stderr": 0.01123534608450046, | |
| "f1": 0.15345346511499533, | |
| "f1_stderr": 0.009574013909382766, | |
| "main_score": 0.1734700739744452 | |
| }, | |
| "az": { | |
| "accuracy": 0.20390047074646941, | |
| "accuracy_stderr": 0.015597824298743462, | |
| "f1": 0.21129515472610239, | |
| "f1_stderr": 0.012699840590535529, | |
| "main_score": 0.20390047074646941 | |
| }, | |
| "bn": { | |
| "accuracy": 0.14082044384667114, | |
| "accuracy_stderr": 0.010656233784801052, | |
| "f1": 0.12169922201279884, | |
| "f1_stderr": 0.00957451063279757, | |
| "main_score": 0.14082044384667114 | |
| }, | |
| "cy": { | |
| "accuracy": 0.2710827168796234, | |
| "accuracy_stderr": 0.019516725914961192, | |
| "f1": 0.2544922244403006, | |
| "f1_stderr": 0.01623441230927587, | |
| "main_score": 0.2710827168796234 | |
| }, | |
| "da": { | |
| "accuracy": 0.27780766644250166, | |
| "accuracy_stderr": 0.014484267495825457, | |
| "f1": 0.2696237025531764, | |
| "f1_stderr": 0.010821089391535837, | |
| "main_score": 0.27780766644250166 | |
| }, | |
| "de": { | |
| "accuracy": 0.21768661735036984, | |
| "accuracy_stderr": 0.013917721396912315, | |
| "f1": 0.22377462868662262, | |
| "f1_stderr": 0.010993391873705134, | |
| "main_score": 0.21768661735036984 | |
| }, | |
| "el": { | |
| "accuracy": 0.21967047747141896, | |
| "accuracy_stderr": 0.014556875283759737, | |
| "f1": 0.22427583602797058, | |
| "f1_stderr": 0.011618987999626771, | |
| "main_score": 0.21967047747141896 | |
| }, | |
| "en": { | |
| "accuracy": 0.33221250840618693, | |
| "accuracy_stderr": 0.020155785658632, | |
| "f1": 0.32627621011904495, | |
| "f1_stderr": 0.016365282624155555, | |
| "main_score": 0.33221250840618693 | |
| }, | |
| "es": { | |
| "accuracy": 0.2704774714189643, | |
| "accuracy_stderr": 0.01658024006292214, | |
| "f1": 0.25244455827652784, | |
| "f1_stderr": 0.007406811213078085, | |
| "main_score": 0.2704774714189643 | |
| }, | |
| "evaluation_time": 131.41, | |
| "fa": { | |
| "accuracy": 0.18850033624747814, | |
| "accuracy_stderr": 0.01266827713571537, | |
| "f1": 0.16532690247057452, | |
| "f1_stderr": 0.011364461370733442, | |
| "main_score": 0.18850033624747814 | |
| }, | |
| "fi": { | |
| "accuracy": 0.2593476798924008, | |
| "accuracy_stderr": 0.02024368999096525, | |
| "f1": 0.2412697491234186, | |
| "f1_stderr": 0.016920746748146247, | |
| "main_score": 0.2593476798924008 | |
| }, | |
| "fr": { | |
| "accuracy": 0.25591795561533287, | |
| "accuracy_stderr": 0.014491798206505743, | |
| "f1": 0.23976861730458382, | |
| "f1_stderr": 0.009956686190445204, | |
| "main_score": 0.25591795561533287 | |
| }, | |
| "he": { | |
| "accuracy": 0.17683254875588433, | |
| "accuracy_stderr": 0.013275985560545845, | |
| "f1": 0.15217082232778534, | |
| "f1_stderr": 0.012702549956571814, | |
| "main_score": 0.17683254875588433 | |
| }, | |
| "hi": { | |
| "accuracy": 0.14277067921990588, | |
| "accuracy_stderr": 0.01126825452940497, | |
| "f1": 0.1306156794974721, | |
| "f1_stderr": 0.0068170198109071605, | |
| "main_score": 0.14277067921990588 | |
| }, | |
| "hu": { | |
| "accuracy": 0.25817081371889716, | |
| "accuracy_stderr": 0.021028671037123154, | |
| "f1": 0.24794435268772488, | |
| "f1_stderr": 0.015711909808467518, | |
| "main_score": 0.25817081371889716 | |
| }, | |
| "hy": { | |
| "accuracy": 0.12326832548755884, | |
| "accuracy_stderr": 0.017495987360368538, | |
| "f1": 0.10850963544530287, | |
| "f1_stderr": 0.015231543036697468, | |
| "main_score": 0.12326832548755884 | |
| }, | |
| "id": { | |
| "accuracy": 0.28244788164088763, | |
| "accuracy_stderr": 0.015158758319623931, | |
| "f1": 0.27442212153664336, | |
| "f1_stderr": 0.009848059495204738, | |
| "main_score": 0.28244788164088763 | |
| }, | |
| "is": { | |
| "accuracy": 0.26039004707464697, | |
| "accuracy_stderr": 0.011139082707097882, | |
| "f1": 0.2429180485465988, | |
| "f1_stderr": 0.01044577332498688, | |
| "main_score": 0.26039004707464697 | |
| }, | |
| "it": { | |
| "accuracy": 0.26099529253530596, | |
| "accuracy_stderr": 0.01289788523979756, | |
| "f1": 0.2647963496597501, | |
| "f1_stderr": 0.010358552015224782, | |
| "main_score": 0.26099529253530596 | |
| }, | |
| "ja": { | |
| "accuracy": 0.25383322125084057, | |
| "accuracy_stderr": 0.0111210517557597, | |
| "f1": 0.2724527274982159, | |
| "f1_stderr": 0.017380763708285306, | |
| "main_score": 0.25383322125084057 | |
| }, | |
| "jv": { | |
| "accuracy": 0.26970410221923335, | |
| "accuracy_stderr": 0.013419981857916152, | |
| "f1": 0.24710215925904627, | |
| "f1_stderr": 0.013856093072114278, | |
| "main_score": 0.26970410221923335 | |
| }, | |
| "ka": { | |
| "accuracy": 0.14159381304640214, | |
| "accuracy_stderr": 0.012321944709633425, | |
| "f1": 0.12262797179154113, | |
| "f1_stderr": 0.012386173944165885, | |
| "main_score": 0.14159381304640214 | |
| }, | |
| "km": { | |
| "accuracy": 0.12078009414929387, | |
| "accuracy_stderr": 0.010655172734546527, | |
| "f1": 0.10853886985799321, | |
| "f1_stderr": 0.005187083660832209, | |
| "main_score": 0.12078009414929387 | |
| }, | |
| "kn": { | |
| "accuracy": 0.10144586415601883, | |
| "accuracy_stderr": 0.016630522893108093, | |
| "f1": 0.08629316498328535, | |
| "f1_stderr": 0.011136959604318427, | |
| "main_score": 0.10144586415601883 | |
| }, | |
| "ko": { | |
| "accuracy": 0.23799596503026227, | |
| "accuracy_stderr": 0.012487237222953968, | |
| "f1": 0.21483977434283802, | |
| "f1_stderr": 0.011263633263736639, | |
| "main_score": 0.23799596503026227 | |
| }, | |
| "lv": { | |
| "accuracy": 0.27047747141896433, | |
| "accuracy_stderr": 0.019046690836812475, | |
| "f1": 0.2586288514660441, | |
| "f1_stderr": 0.01918841737445303, | |
| "main_score": 0.27047747141896433 | |
| }, | |
| "ml": { | |
| "accuracy": 0.10823806321452589, | |
| "accuracy_stderr": 0.013372719041442079, | |
| "f1": 0.0956831903025781, | |
| "f1_stderr": 0.010876457400211457, | |
| "main_score": 0.10823806321452589 | |
| }, | |
| "mn": { | |
| "accuracy": 0.14435104236718224, | |
| "accuracy_stderr": 0.009787112320379458, | |
| "f1": 0.14861584951252121, | |
| "f1_stderr": 0.011028588805344698, | |
| "main_score": 0.14435104236718224 | |
| }, | |
| "ms": { | |
| "accuracy": 0.2683254875588433, | |
| "accuracy_stderr": 0.014785715808739976, | |
| "f1": 0.2608474943919197, | |
| "f1_stderr": 0.01577798431823313, | |
| "main_score": 0.2683254875588433 | |
| }, | |
| "my": { | |
| "accuracy": 0.1379287155346335, | |
| "accuracy_stderr": 0.015098073418812856, | |
| "f1": 0.11496310101617802, | |
| "f1_stderr": 0.01113023387276534, | |
| "main_score": 0.1379287155346335 | |
| }, | |
| "nb": { | |
| "accuracy": 0.2529253530598521, | |
| "accuracy_stderr": 0.02192725467381842, | |
| "f1": 0.2409811847728251, | |
| "f1_stderr": 0.018335164031557032, | |
| "main_score": 0.2529253530598521 | |
| }, | |
| "nl": { | |
| "accuracy": 0.2624075319435104, | |
| "accuracy_stderr": 0.016494781034623495, | |
| "f1": 0.25259998680849816, | |
| "f1_stderr": 0.011822721077804391, | |
| "main_score": 0.2624075319435104 | |
| }, | |
| "pl": { | |
| "accuracy": 0.2490921318090115, | |
| "accuracy_stderr": 0.01391170861467293, | |
| "f1": 0.23684726718537474, | |
| "f1_stderr": 0.010038508781153537, | |
| "main_score": 0.2490921318090115 | |
| }, | |
| "pt": { | |
| "accuracy": 0.27310020174848687, | |
| "accuracy_stderr": 0.012410948622250844, | |
| "f1": 0.2628728556240871, | |
| "f1_stderr": 0.011369661369276283, | |
| "main_score": 0.27310020174848687 | |
| }, | |
| "ro": { | |
| "accuracy": 0.21930060524546063, | |
| "accuracy_stderr": 0.015842787989000313, | |
| "f1": 0.21993435727293242, | |
| "f1_stderr": 0.013793031984742025, | |
| "main_score": 0.21930060524546063 | |
| }, | |
| "ru": { | |
| "accuracy": 0.25773369199731, | |
| "accuracy_stderr": 0.01568820616393382, | |
| "f1": 0.2385773912094263, | |
| "f1_stderr": 0.01425853200605966, | |
| "main_score": 0.25773369199731 | |
| }, | |
| "sl": { | |
| "accuracy": 0.29256893073301954, | |
| "accuracy_stderr": 0.020871978199526527, | |
| "f1": 0.266824726822139, | |
| "f1_stderr": 0.015432442287329618, | |
| "main_score": 0.29256893073301954 | |
| }, | |
| "sq": { | |
| "accuracy": 0.17673167451244115, | |
| "accuracy_stderr": 0.014965997148414185, | |
| "f1": 0.1874169530134767, | |
| "f1_stderr": 0.012519990193997124, | |
| "main_score": 0.17673167451244115 | |
| }, | |
| "sv": { | |
| "accuracy": 0.26190316072629455, | |
| "accuracy_stderr": 0.01813209654427075, | |
| "f1": 0.26000111409348475, | |
| "f1_stderr": 0.013012923916693933, | |
| "main_score": 0.26190316072629455 | |
| }, | |
| "sw": { | |
| "accuracy": 0.2890383322125084, | |
| "accuracy_stderr": 0.014872024877456985, | |
| "f1": 0.2716695883514315, | |
| "f1_stderr": 0.009966097116440113, | |
| "main_score": 0.2890383322125084 | |
| }, | |
| "ta": { | |
| "accuracy": 0.13517148621385341, | |
| "accuracy_stderr": 0.010585121882109751, | |
| "f1": 0.10124568781522633, | |
| "f1_stderr": 0.009485791249151044, | |
| "main_score": 0.13517148621385341 | |
| }, | |
| "te": { | |
| "accuracy": 0.12414256893073303, | |
| "accuracy_stderr": 0.013528519375366168, | |
| "f1": 0.1072905194223371, | |
| "f1_stderr": 0.011040476001329725, | |
| "main_score": 0.12414256893073303 | |
| }, | |
| "th": { | |
| "accuracy": 0.17330195023537326, | |
| "accuracy_stderr": 0.016822592724954483, | |
| "f1": 0.17262535319973024, | |
| "f1_stderr": 0.018638781100031133, | |
| "main_score": 0.17330195023537326 | |
| }, | |
| "tl": { | |
| "accuracy": 0.22310020174848688, | |
| "accuracy_stderr": 0.01591004963438819, | |
| "f1": 0.2277017596643771, | |
| "f1_stderr": 0.014061398424045453, | |
| "main_score": 0.22310020174848688 | |
| }, | |
| "tr": { | |
| "accuracy": 0.23066577000672495, | |
| "accuracy_stderr": 0.014424977622637898, | |
| "f1": 0.2250576183714979, | |
| "f1_stderr": 0.013217916991158374, | |
| "main_score": 0.23066577000672495 | |
| }, | |
| "ur": { | |
| "accuracy": 0.12501681237390722, | |
| "accuracy_stderr": 0.008628925316661974, | |
| "f1": 0.11803223778323549, | |
| "f1_stderr": 0.007147317301295282, | |
| "main_score": 0.12501681237390722 | |
| }, | |
| "vi": { | |
| "accuracy": 0.2276059179556153, | |
| "accuracy_stderr": 0.012455189381569469, | |
| "f1": 0.21556310396949008, | |
| "f1_stderr": 0.010334699551825161, | |
| "main_score": 0.2276059179556153 | |
| }, | |
| "zh-CN": { | |
| "accuracy": 0.3492602555480834, | |
| "accuracy_stderr": 0.01870216211586065, | |
| "f1": 0.3372797417177343, | |
| "f1_stderr": 0.01139600455982931, | |
| "main_score": 0.3492602555480834 | |
| }, | |
| "zh-TW": { | |
| "accuracy": 0.2836583725622058, | |
| "accuracy_stderr": 0.011393977873276777, | |
| "f1": 0.28510022067424706, | |
| "f1_stderr": 0.00933112485049988, | |
| "main_score": 0.2836583725622058 | |
| } | |
| } | |
| } |