metadata
tags:
- mteb
model-index:
- name: pythia-14m_mean
results:
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (en)
config: en
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 70.73134328358208
- type: ap
value: 32.35996836729783
- type: f1
value: 64.2137087561157
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (de)
config: de
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 62.291220556745174
- type: ap
value: 76.5427302441011
- type: f1
value: 60.37703210343267
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (en-ext)
config: en-ext
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 67.57871064467767
- type: ap
value: 17.03033311712744
- type: f1
value: 54.821750631894986
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (ja)
config: ja
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 62.51605995717344
- type: ap
value: 14.367489440317666
- type: f1
value: 50.48473578289779
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (en)
config: en
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 29.172000000000004
- type: f1
value: 28.264998641170465
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (de)
config: de
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 25.157999999999998
- type: f1
value: 23.033533062569987
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (es)
config: es
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 26.840000000000003
- type: f1
value: 25.693413738086402
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (fr)
config: fr
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 26.491999999999997
- type: f1
value: 25.6252880863665
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (ja)
config: ja
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 24.448000000000004
- type: f1
value: 23.86460242225935
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 26.412000000000003
- type: f1
value: 25.779710231390755
- task:
type: Retrieval
dataset:
type: arguana
name: MTEB ArguAna
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 5.761
- type: map_at_10
value: 10.267
- type: map_at_100
value: 11.065999999999999
- type: map_at_1000
value: 11.16
- type: map_at_3
value: 8.642
- type: map_at_5
value: 9.474
- type: mrr_at_1
value: 6.046
- type: mrr_at_10
value: 10.365
- type: mrr_at_100
value: 11.178
- type: mrr_at_1000
value: 11.272
- type: mrr_at_3
value: 8.713
- type: mrr_at_5
value: 9.587
- type: ndcg_at_1
value: 5.761
- type: ndcg_at_10
value: 13.055
- type: ndcg_at_100
value: 17.526
- type: ndcg_at_1000
value: 20.578
- type: ndcg_at_3
value: 9.616
- type: ndcg_at_5
value: 11.128
- type: precision_at_1
value: 5.761
- type: precision_at_10
value: 2.212
- type: precision_at_100
value: 0.44400000000000006
- type: precision_at_1000
value: 0.06999999999999999
- type: precision_at_3
value: 4.149
- type: precision_at_5
value: 3.229
- type: recall_at_1
value: 5.761
- type: recall_at_10
value: 22.119
- type: recall_at_100
value: 44.381
- type: recall_at_1000
value: 69.70100000000001
- type: recall_at_3
value: 12.447
- type: recall_at_5
value: 16.145
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-s2s
name: MTEB ArxivClusteringS2S
config: default
split: test
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53
metrics:
- type: v_measure
value: 13.902183567893395
- task:
type: Reranking
dataset:
type: mteb/askubuntudupquestions-reranking
name: MTEB AskUbuntuDupQuestions
config: default
split: test
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54
metrics:
- type: map
value: 47.93210378051478
- type: mrr
value: 60.70318339708921
- task:
type: STS
dataset:
type: mteb/biosses-sts
name: MTEB BIOSSES
config: default
split: test
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a
metrics:
- type: cos_sim_pearson
value: 49.57650220181508
- type: cos_sim_spearman
value: 51.842145113866636
- type: euclidean_pearson
value: 41.2188173176347
- type: euclidean_spearman
value: 41.16840792962046
- type: manhattan_pearson
value: 42.73893519020435
- type: manhattan_spearman
value: 44.384746276312534
- task:
type: Classification
dataset:
type: mteb/banking77
name: MTEB Banking77Classification
config: default
split: test
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300
metrics:
- type: accuracy
value: 46.03896103896104
- type: f1
value: 44.54083818845286
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-p2p
name: MTEB BiorxivClusteringP2P
config: default
split: test
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40
metrics:
- type: v_measure
value: 23.113393015706908
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-s2s
name: MTEB BiorxivClusteringS2S
config: default
split: test
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908
metrics:
- type: v_measure
value: 12.624675113307488
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackAndroidRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 10.105
- type: map_at_10
value: 13.364
- type: map_at_100
value: 13.987
- type: map_at_1000
value: 14.08
- type: map_at_3
value: 12.447
- type: map_at_5
value: 12.992999999999999
- type: mrr_at_1
value: 12.876000000000001
- type: mrr_at_10
value: 16.252
- type: mrr_at_100
value: 16.926
- type: mrr_at_1000
value: 17.004
- type: mrr_at_3
value: 15.235999999999999
- type: mrr_at_5
value: 15.744
- type: ndcg_at_1
value: 12.876000000000001
- type: ndcg_at_10
value: 15.634999999999998
- type: ndcg_at_100
value: 19.173000000000002
- type: ndcg_at_1000
value: 22.168
- type: ndcg_at_3
value: 14.116999999999999
- type: ndcg_at_5
value: 14.767
- type: precision_at_1
value: 12.876000000000001
- type: precision_at_10
value: 2.761
- type: precision_at_100
value: 0.5579999999999999
- type: precision_at_1000
value: 0.101
- type: precision_at_3
value: 6.676
- type: precision_at_5
value: 4.635
- type: recall_at_1
value: 10.105
- type: recall_at_10
value: 19.767000000000003
- type: recall_at_100
value: 36.448
- type: recall_at_1000
value: 58.623000000000005
- type: recall_at_3
value: 15.087
- type: recall_at_5
value: 17.076
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackEnglishRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 7.249999999999999
- type: map_at_10
value: 9.41
- type: map_at_100
value: 9.903
- type: map_at_1000
value: 9.993
- type: map_at_3
value: 8.693
- type: map_at_5
value: 9.052
- type: mrr_at_1
value: 9.299
- type: mrr_at_10
value: 11.907
- type: mrr_at_100
value: 12.424
- type: mrr_at_1000
value: 12.503
- type: mrr_at_3
value: 10.945
- type: mrr_at_5
value: 11.413
- type: ndcg_at_1
value: 9.299
- type: ndcg_at_10
value: 11.278
- type: ndcg_at_100
value: 13.904
- type: ndcg_at_1000
value: 16.642000000000003
- type: ndcg_at_3
value: 9.956
- type: ndcg_at_5
value: 10.488
- type: precision_at_1
value: 9.299
- type: precision_at_10
value: 2.166
- type: precision_at_100
value: 0.45399999999999996
- type: precision_at_1000
value: 0.089
- type: precision_at_3
value: 4.798
- type: precision_at_5
value: 3.427
- type: recall_at_1
value: 7.249999999999999
- type: recall_at_10
value: 14.285
- type: recall_at_100
value: 26.588
- type: recall_at_1000
value: 46.488
- type: recall_at_3
value: 10.309
- type: recall_at_5
value: 11.756
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGamingRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 11.57
- type: map_at_10
value: 15.497
- type: map_at_100
value: 16.036
- type: map_at_1000
value: 16.122
- type: map_at_3
value: 14.309
- type: map_at_5
value: 14.895
- type: mrr_at_1
value: 13.354
- type: mrr_at_10
value: 17.408
- type: mrr_at_100
value: 17.936
- type: mrr_at_1000
value: 18.015
- type: mrr_at_3
value: 16.123
- type: mrr_at_5
value: 16.735
- type: ndcg_at_1
value: 13.354
- type: ndcg_at_10
value: 18.071
- type: ndcg_at_100
value: 21.017
- type: ndcg_at_1000
value: 23.669999999999998
- type: ndcg_at_3
value: 15.644
- type: ndcg_at_5
value: 16.618
- type: precision_at_1
value: 13.354
- type: precision_at_10
value: 2.94
- type: precision_at_100
value: 0.481
- type: precision_at_1000
value: 0.076
- type: precision_at_3
value: 7.001
- type: precision_at_5
value: 4.765
- type: recall_at_1
value: 11.57
- type: recall_at_10
value: 24.147
- type: recall_at_100
value: 38.045
- type: recall_at_1000
value: 58.648
- type: recall_at_3
value: 17.419999999999998
- type: recall_at_5
value: 19.875999999999998
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGisRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 4.463
- type: map_at_10
value: 6.091
- type: map_at_100
value: 6.548
- type: map_at_1000
value: 6.622
- type: map_at_3
value: 5.461
- type: map_at_5
value: 5.768
- type: mrr_at_1
value: 4.746
- type: mrr_at_10
value: 6.431000000000001
- type: mrr_at_100
value: 6.941
- type: mrr_at_1000
value: 7.016
- type: mrr_at_3
value: 5.763
- type: mrr_at_5
value: 6.101999999999999
- type: ndcg_at_1
value: 4.746
- type: ndcg_at_10
value: 7.19
- type: ndcg_at_100
value: 9.604
- type: ndcg_at_1000
value: 12.086
- type: ndcg_at_3
value: 5.88
- type: ndcg_at_5
value: 6.429
- type: precision_at_1
value: 4.746
- type: precision_at_10
value: 1.141
- type: precision_at_100
value: 0.249
- type: precision_at_1000
value: 0.049
- type: precision_at_3
value: 2.448
- type: precision_at_5
value: 1.7850000000000001
- type: recall_at_1
value: 4.463
- type: recall_at_10
value: 10.33
- type: recall_at_100
value: 21.578
- type: recall_at_1000
value: 41.404
- type: recall_at_3
value: 6.816999999999999
- type: recall_at_5
value: 8.06
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackMathematicaRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 1.521
- type: map_at_10
value: 2.439
- type: map_at_100
value: 2.785
- type: map_at_1000
value: 2.858
- type: map_at_3
value: 2.091
- type: map_at_5
value: 2.2560000000000002
- type: mrr_at_1
value: 2.114
- type: mrr_at_10
value: 3.216
- type: mrr_at_100
value: 3.6319999999999997
- type: mrr_at_1000
value: 3.712
- type: mrr_at_3
value: 2.778
- type: mrr_at_5
value: 2.971
- type: ndcg_at_1
value: 2.114
- type: ndcg_at_10
value: 3.1910000000000003
- type: ndcg_at_100
value: 5.165
- type: ndcg_at_1000
value: 7.607
- type: ndcg_at_3
value: 2.456
- type: ndcg_at_5
value: 2.7439999999999998
- type: precision_at_1
value: 2.114
- type: precision_at_10
value: 0.634
- type: precision_at_100
value: 0.189
- type: precision_at_1000
value: 0.049
- type: precision_at_3
value: 1.202
- type: precision_at_5
value: 0.8959999999999999
- type: recall_at_1
value: 1.521
- type: recall_at_10
value: 4.8
- type: recall_at_100
value: 13.877
- type: recall_at_1000
value: 32.1
- type: recall_at_3
value: 2.806
- type: recall_at_5
value: 3.5520000000000005
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackPhysicsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 7.449999999999999
- type: map_at_10
value: 10.065
- type: map_at_100
value: 10.507
- type: map_at_1000
value: 10.599
- type: map_at_3
value: 9.017
- type: map_at_5
value: 9.603
- type: mrr_at_1
value: 9.336
- type: mrr_at_10
value: 12.589
- type: mrr_at_100
value: 13.086
- type: mrr_at_1000
value: 13.161000000000001
- type: mrr_at_3
value: 11.373
- type: mrr_at_5
value: 12.084999999999999
- type: ndcg_at_1
value: 9.336
- type: ndcg_at_10
value: 12.299
- type: ndcg_at_100
value: 14.780999999999999
- type: ndcg_at_1000
value: 17.632
- type: ndcg_at_3
value: 10.302
- type: ndcg_at_5
value: 11.247
- type: precision_at_1
value: 9.336
- type: precision_at_10
value: 2.271
- type: precision_at_100
value: 0.42300000000000004
- type: precision_at_1000
value: 0.08099999999999999
- type: precision_at_3
value: 4.909
- type: precision_at_5
value: 3.5999999999999996
- type: recall_at_1
value: 7.449999999999999
- type: recall_at_10
value: 16.891000000000002
- type: recall_at_100
value: 28.050000000000004
- type: recall_at_1000
value: 49.267
- type: recall_at_3
value: 11.187999999999999
- type: recall_at_5
value: 13.587
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackProgrammersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 4.734
- type: map_at_10
value: 7.045999999999999
- type: map_at_100
value: 7.564
- type: map_at_1000
value: 7.6499999999999995
- type: map_at_3
value: 6.21
- type: map_at_5
value: 6.617000000000001
- type: mrr_at_1
value: 5.936
- type: mrr_at_10
value: 8.624
- type: mrr_at_100
value: 9.193
- type: mrr_at_1000
value: 9.28
- type: mrr_at_3
value: 7.725
- type: mrr_at_5
value: 8.147
- type: ndcg_at_1
value: 5.936
- type: ndcg_at_10
value: 8.81
- type: ndcg_at_100
value: 11.694
- type: ndcg_at_1000
value: 14.526
- type: ndcg_at_3
value: 7.140000000000001
- type: ndcg_at_5
value: 7.8020000000000005
- type: precision_at_1
value: 5.936
- type: precision_at_10
value: 1.701
- type: precision_at_100
value: 0.366
- type: precision_at_1000
value: 0.07200000000000001
- type: precision_at_3
value: 3.463
- type: precision_at_5
value: 2.557
- type: recall_at_1
value: 4.734
- type: recall_at_10
value: 12.733
- type: recall_at_100
value: 25.982
- type: recall_at_1000
value: 47.233999999999995
- type: recall_at_3
value: 8.018
- type: recall_at_5
value: 9.762
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackStatsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 4.293
- type: map_at_10
value: 6.146999999999999
- type: map_at_100
value: 6.487
- type: map_at_1000
value: 6.544999999999999
- type: map_at_3
value: 5.6930000000000005
- type: map_at_5
value: 5.869
- type: mrr_at_1
value: 5.061
- type: mrr_at_10
value: 7.1690000000000005
- type: mrr_at_100
value: 7.542
- type: mrr_at_1000
value: 7.5969999999999995
- type: mrr_at_3
value: 6.646000000000001
- type: mrr_at_5
value: 6.8229999999999995
- type: ndcg_at_1
value: 5.061
- type: ndcg_at_10
value: 7.396
- type: ndcg_at_100
value: 9.41
- type: ndcg_at_1000
value: 11.386000000000001
- type: ndcg_at_3
value: 6.454
- type: ndcg_at_5
value: 6.718
- type: precision_at_1
value: 5.061
- type: precision_at_10
value: 1.319
- type: precision_at_100
value: 0.262
- type: precision_at_1000
value: 0.047
- type: precision_at_3
value: 3.0669999999999997
- type: precision_at_5
value: 1.994
- type: recall_at_1
value: 4.293
- type: recall_at_10
value: 10.221
- type: recall_at_100
value: 19.744999999999997
- type: recall_at_1000
value: 35.399
- type: recall_at_3
value: 7.507999999999999
- type: recall_at_5
value: 8.275
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackTexRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 3.519
- type: map_at_10
value: 4.768
- type: map_at_100
value: 5.034000000000001
- type: map_at_1000
value: 5.087
- type: map_at_3
value: 4.308
- type: map_at_5
value: 4.565
- type: mrr_at_1
value: 4.474
- type: mrr_at_10
value: 6.045
- type: mrr_at_100
value: 6.361999999999999
- type: mrr_at_1000
value: 6.417000000000001
- type: mrr_at_3
value: 5.483
- type: mrr_at_5
value: 5.81
- type: ndcg_at_1
value: 4.474
- type: ndcg_at_10
value: 5.799
- type: ndcg_at_100
value: 7.344
- type: ndcg_at_1000
value: 9.141
- type: ndcg_at_3
value: 4.893
- type: ndcg_at_5
value: 5.309
- type: precision_at_1
value: 4.474
- type: precision_at_10
value: 1.06
- type: precision_at_100
value: 0.217
- type: precision_at_1000
value: 0.045
- type: precision_at_3
value: 2.306
- type: precision_at_5
value: 1.7000000000000002
- type: recall_at_1
value: 3.519
- type: recall_at_10
value: 7.75
- type: recall_at_100
value: 15.049999999999999
- type: recall_at_1000
value: 28.779
- type: recall_at_3
value: 5.18
- type: recall_at_5
value: 6.245
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackUnixRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 6.098
- type: map_at_10
value: 7.918
- type: map_at_100
value: 8.229000000000001
- type: map_at_1000
value: 8.293000000000001
- type: map_at_3
value: 7.138999999999999
- type: map_at_5
value: 7.646
- type: mrr_at_1
value: 7.090000000000001
- type: mrr_at_10
value: 9.293
- type: mrr_at_100
value: 9.669
- type: mrr_at_1000
value: 9.734
- type: mrr_at_3
value: 8.364
- type: mrr_at_5
value: 8.956999999999999
- type: ndcg_at_1
value: 7.090000000000001
- type: ndcg_at_10
value: 9.411999999999999
- type: ndcg_at_100
value: 11.318999999999999
- type: ndcg_at_1000
value: 13.478000000000002
- type: ndcg_at_3
value: 7.837
- type: ndcg_at_5
value: 8.73
- type: precision_at_1
value: 7.090000000000001
- type: precision_at_10
value: 1.558
- type: precision_at_100
value: 0.28400000000000003
- type: precision_at_1000
value: 0.053
- type: precision_at_3
value: 3.42
- type: precision_at_5
value: 2.5749999999999997
- type: recall_at_1
value: 6.098
- type: recall_at_10
value: 12.764000000000001
- type: recall_at_100
value: 21.747
- type: recall_at_1000
value: 38.279999999999994
- type: recall_at_3
value: 8.476
- type: recall_at_5
value: 10.707
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWebmastersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 8.607
- type: map_at_10
value: 10.835
- type: map_at_100
value: 11.285
- type: map_at_1000
value: 11.383000000000001
- type: map_at_3
value: 10.111
- type: map_at_5
value: 10.334999999999999
- type: mrr_at_1
value: 10.671999999999999
- type: mrr_at_10
value: 13.269
- type: mrr_at_100
value: 13.729
- type: mrr_at_1000
value: 13.813
- type: mrr_at_3
value: 12.385
- type: mrr_at_5
value: 12.701
- type: ndcg_at_1
value: 10.671999999999999
- type: ndcg_at_10
value: 12.728
- type: ndcg_at_100
value: 15.312999999999999
- type: ndcg_at_1000
value: 18.160999999999998
- type: ndcg_at_3
value: 11.355
- type: ndcg_at_5
value: 11.605
- type: precision_at_1
value: 10.671999999999999
- type: precision_at_10
value: 2.154
- type: precision_at_100
value: 0.455
- type: precision_at_1000
value: 0.098
- type: precision_at_3
value: 4.941
- type: precision_at_5
value: 3.2809999999999997
- type: recall_at_1
value: 8.607
- type: recall_at_10
value: 16.398
- type: recall_at_100
value: 28.92
- type: recall_at_1000
value: 49.761
- type: recall_at_3
value: 11.844000000000001
- type: recall_at_5
value: 12.792
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWordpressRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 3.826
- type: map_at_10
value: 5.6419999999999995
- type: map_at_100
value: 5.943
- type: map_at_1000
value: 6.005
- type: map_at_3
value: 5.1049999999999995
- type: map_at_5
value: 5.437
- type: mrr_at_1
value: 4.436
- type: mrr_at_10
value: 6.413
- type: mrr_at_100
value: 6.752
- type: mrr_at_1000
value: 6.819999999999999
- type: mrr_at_3
value: 5.884
- type: mrr_at_5
value: 6.18
- type: ndcg_at_1
value: 4.436
- type: ndcg_at_10
value: 6.7989999999999995
- type: ndcg_at_100
value: 8.619
- type: ndcg_at_1000
value: 10.842
- type: ndcg_at_3
value: 5.739
- type: ndcg_at_5
value: 6.292000000000001
- type: precision_at_1
value: 4.436
- type: precision_at_10
value: 1.109
- type: precision_at_100
value: 0.214
- type: precision_at_1000
value: 0.043
- type: precision_at_3
value: 2.588
- type: precision_at_5
value: 1.848
- type: recall_at_1
value: 3.826
- type: recall_at_10
value: 9.655
- type: recall_at_100
value: 18.611
- type: recall_at_1000
value: 36.733
- type: recall_at_3
value: 6.784
- type: recall_at_5
value: 8.17
- task:
type: Classification
dataset:
type: mteb/emotion
name: MTEB EmotionClassification
config: default
split: test
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37
metrics:
- type: accuracy
value: 23.279999999999998
- type: f1
value: 19.87865985032945
- task:
type: Retrieval
dataset:
type: fiqa
name: MTEB FiQA2018
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 1.166
- type: map_at_10
value: 2.283
- type: map_at_100
value: 2.564
- type: map_at_1000
value: 2.6519999999999997
- type: map_at_3
value: 1.867
- type: map_at_5
value: 2.0500000000000003
- type: mrr_at_1
value: 2.932
- type: mrr_at_10
value: 4.852
- type: mrr_at_100
value: 5.306
- type: mrr_at_1000
value: 5.4
- type: mrr_at_3
value: 4.141
- type: mrr_at_5
value: 4.457
- type: ndcg_at_1
value: 2.932
- type: ndcg_at_10
value: 3.5709999999999997
- type: ndcg_at_100
value: 5.489
- type: ndcg_at_1000
value: 8.309999999999999
- type: ndcg_at_3
value: 2.773
- type: ndcg_at_5
value: 2.979
- type: precision_at_1
value: 2.932
- type: precision_at_10
value: 1.049
- type: precision_at_100
value: 0.306
- type: precision_at_1000
value: 0.077
- type: precision_at_3
value: 1.8519999999999999
- type: precision_at_5
value: 1.389
- type: recall_at_1
value: 1.166
- type: recall_at_10
value: 5.178
- type: recall_at_100
value: 13.056999999999999
- type: recall_at_1000
value: 31.708
- type: recall_at_3
value: 2.714
- type: recall_at_5
value: 3.4909999999999997
- task:
type: Classification
dataset:
type: mteb/imdb
name: MTEB ImdbClassification
config: default
split: test
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7
metrics:
- type: accuracy
value: 56.96359999999999
- type: ap
value: 54.16760114570921
- type: f1
value: 56.193845361069116
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (en)
config: en
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 64.39808481532147
- type: f1
value: 63.468270818712625
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (de)
config: de
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 53.961679346294744
- type: f1
value: 51.6707117653683
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (es)
config: es
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 57.018012008005336
- type: f1
value: 54.23413458037234
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (fr)
config: fr
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 48.84434700908236
- type: f1
value: 46.48494180527987
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (hi)
config: hi
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 39.7669415561133
- type: f1
value: 35.50974325529877
- task:
type: Classification
dataset:
type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (th)
config: th
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
value: 42.589511754068724
- type: f1
value: 40.47244422785889
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (en)
config: en
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 34.01276789785682
- type: f1
value: 21.256775922291286
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (de)
config: de
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 33.285432516201745
- type: f1
value: 19.841703666811565
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (es)
config: es
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 32.121414276184126
- type: f1
value: 19.34706868150749
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (fr)
config: fr
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 26.088318196053866
- type: f1
value: 17.22608011891254
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (hi)
config: hi
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 15.320903549659375
- type: f1
value: 9.62002916015258
- task:
type: Classification
dataset:
type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (th)
config: th
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
value: 16.426763110307412
- type: f1
value: 11.023799171137183
- task:
type: Clustering
dataset:
type: mteb/medrxiv-clustering-p2p
name: MTEB MedrxivClusteringP2P
config: default
split: test
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73
metrics:
- type: v_measure
value: 22.08508717069763
- task:
type: Clustering
dataset:
type: mteb/medrxiv-clustering-s2s
name: MTEB MedrxivClusteringS2S
config: default
split: test
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663
metrics:
- type: v_measure
value: 16.58582885790446
- task:
type: Retrieval
dataset:
type: nfcorpus
name: MTEB NFCorpus
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 1.2
- type: map_at_10
value: 1.6400000000000001
- type: map_at_100
value: 1.9789999999999999
- type: map_at_1000
value: 2.554
- type: map_at_3
value: 1.4449999999999998
- type: map_at_5
value: 1.533
- type: mrr_at_1
value: 6.811
- type: mrr_at_10
value: 11.068999999999999
- type: mrr_at_100
value: 12.454
- type: mrr_at_1000
value: 12.590000000000002
- type: mrr_at_3
value: 9.751999999999999
- type: mrr_at_5
value: 10.31
- type: ndcg_at_1
value: 6.3469999999999995
- type: ndcg_at_10
value: 4.941
- type: ndcg_at_100
value: 6.524000000000001
- type: ndcg_at_1000
value: 15.918
- type: ndcg_at_3
value: 5.959
- type: ndcg_at_5
value: 5.395
- type: precision_at_1
value: 6.811
- type: precision_at_10
value: 3.375
- type: precision_at_100
value: 2.0709999999999997
- type: precision_at_1000
value: 1.313
- type: precision_at_3
value: 5.47
- type: precision_at_5
value: 4.396
- type: recall_at_1
value: 1.2
- type: recall_at_10
value: 2.5909999999999997
- type: recall_at_100
value: 9.443999999999999
- type: recall_at_1000
value: 41.542
- type: recall_at_3
value: 1.702
- type: recall_at_5
value: 1.9879999999999998
- task:
type: Retrieval
dataset:
type: quora
name: MTEB QuoraRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 45.708
- type: map_at_10
value: 55.131
- type: map_at_100
value: 55.935
- type: map_at_1000
value: 55.993
- type: map_at_3
value: 52.749
- type: map_at_5
value: 54.166000000000004
- type: mrr_at_1
value: 52.44
- type: mrr_at_10
value: 59.99
- type: mrr_at_100
value: 60.492999999999995
- type: mrr_at_1000
value: 60.522
- type: mrr_at_3
value: 58.285
- type: mrr_at_5
value: 59.305
- type: ndcg_at_1
value: 52.43
- type: ndcg_at_10
value: 59.873
- type: ndcg_at_100
value: 63.086
- type: ndcg_at_1000
value: 64.291
- type: ndcg_at_3
value: 56.291000000000004
- type: ndcg_at_5
value: 58.071
- type: precision_at_1
value: 52.43
- type: precision_at_10
value: 8.973
- type: precision_at_100
value: 1.161
- type: precision_at_1000
value: 0.134
- type: precision_at_3
value: 24.177
- type: precision_at_5
value: 16.073999999999998
- type: recall_at_1
value: 45.708
- type: recall_at_10
value: 69.195
- type: recall_at_100
value: 82.812
- type: recall_at_1000
value: 91.136
- type: recall_at_3
value: 58.938
- type: recall_at_5
value: 63.787000000000006
- task:
type: Clustering
dataset:
type: mteb/reddit-clustering
name: MTEB RedditClustering
config: default
split: test
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb
metrics:
- type: v_measure
value: 13.142048230676806
- task:
type: Clustering
dataset:
type: mteb/reddit-clustering-p2p
name: MTEB RedditClusteringP2P
config: default
split: test
revision: 282350215ef01743dc01b456c7f5241fa8937f16
metrics:
- type: v_measure
value: 26.06687178917052
- task:
type: Retrieval
dataset:
type: scidocs
name: MTEB SCIDOCS
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 0.46499999999999997
- type: map_at_10
value: 0.906
- type: map_at_100
value: 1.127
- type: map_at_1000
value: 1.203
- type: map_at_3
value: 0.72
- type: map_at_5
value: 0.814
- type: mrr_at_1
value: 2.3
- type: mrr_at_10
value: 3.733
- type: mrr_at_100
value: 4.295999999999999
- type: mrr_at_1000
value: 4.412
- type: mrr_at_3
value: 3.183
- type: mrr_at_5
value: 3.458
- type: ndcg_at_1
value: 2.3
- type: ndcg_at_10
value: 1.797
- type: ndcg_at_100
value: 3.376
- type: ndcg_at_1000
value: 6.143
- type: ndcg_at_3
value: 1.763
- type: ndcg_at_5
value: 1.5070000000000001
- type: precision_at_1
value: 2.3
- type: precision_at_10
value: 0.91
- type: precision_at_100
value: 0.32399999999999995
- type: precision_at_1000
value: 0.101
- type: precision_at_3
value: 1.633
- type: precision_at_5
value: 1.3
- type: recall_at_1
value: 0.46499999999999997
- type: recall_at_10
value: 1.8499999999999999
- type: recall_at_100
value: 6.625
- type: recall_at_1000
value: 20.587
- type: recall_at_3
value: 0.9900000000000001
- type: recall_at_5
value: 1.315
- task:
type: STS
dataset:
type: mteb/sickr-sts
name: MTEB SICK-R
config: default
split: test
revision: a6ea5a8cab320b040a23452cc28066d9beae2cee
metrics:
- type: cos_sim_pearson
value: 60.78961481918511
- type: cos_sim_spearman
value: 54.92014630234372
- type: euclidean_pearson
value: 54.91456364340953
- type: euclidean_spearman
value: 50.95537043206628
- type: manhattan_pearson
value: 55.0450005071106
- type: manhattan_spearman
value: 51.227579527791654
- task:
type: STS
dataset:
type: mteb/sts12-sts
name: MTEB STS12
config: default
split: test
revision: a0d554a64d88156834ff5ae9920b964011b16384
metrics:
- type: cos_sim_pearson
value: 43.73124494569395
- type: cos_sim_spearman
value: 43.07629933550637
- type: euclidean_pearson
value: 37.2529484210563
- type: euclidean_spearman
value: 36.68421330216546
- type: manhattan_pearson
value: 37.41673219009712
- type: manhattan_spearman
value: 36.92073705702668
- task:
type: STS
dataset:
type: mteb/sts13-sts
name: MTEB STS13
config: default
split: test
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca
metrics:
- type: cos_sim_pearson
value: 57.17534157059787
- type: cos_sim_spearman
value: 56.86679858348438
- type: euclidean_pearson
value: 54.51552371857776
- type: euclidean_spearman
value: 53.80989851917749
- type: manhattan_pearson
value: 54.44486043632584
- type: manhattan_spearman
value: 53.83487353949481
- task:
type: STS
dataset:
type: mteb/sts14-sts
name: MTEB STS14
config: default
split: test
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375
metrics:
- type: cos_sim_pearson
value: 52.319034960820375
- type: cos_sim_spearman
value: 50.89512224974754
- type: euclidean_pearson
value: 49.19308209408045
- type: euclidean_spearman
value: 47.45736923614355
- type: manhattan_pearson
value: 48.82127080055118
- type: manhattan_spearman
value: 47.20185686489298
- task:
type: STS
dataset:
type: mteb/sts15-sts
name: MTEB STS15
config: default
split: test
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3
metrics:
- type: cos_sim_pearson
value: 61.57602956458427
- type: cos_sim_spearman
value: 62.894640061838956
- type: euclidean_pearson
value: 53.86893407586029
- type: euclidean_spearman
value: 54.68528520514299
- type: manhattan_pearson
value: 53.689614981956815
- type: manhattan_spearman
value: 54.51172839699876
- task:
type: STS
dataset:
type: mteb/sts16-sts
name: MTEB STS16
config: default
split: test
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513
metrics:
- type: cos_sim_pearson
value: 56.2305694109318
- type: cos_sim_spearman
value: 57.885939000786045
- type: euclidean_pearson
value: 50.486043353701994
- type: euclidean_spearman
value: 50.4463227974027
- type: manhattan_pearson
value: 50.73317560427465
- type: manhattan_spearman
value: 50.81397877006027
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (ko-ko)
config: ko-ko
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 55.52162058025664
- type: cos_sim_spearman
value: 59.02220327783535
- type: euclidean_pearson
value: 55.66332330866701
- type: euclidean_spearman
value: 56.829076266662206
- type: manhattan_pearson
value: 55.39181385186973
- type: manhattan_spearman
value: 56.607432176121144
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (ar-ar)
config: ar-ar
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 46.312186899914906
- type: cos_sim_spearman
value: 48.07172073934163
- type: euclidean_pearson
value: 46.957276350776695
- type: euclidean_spearman
value: 43.98800593212707
- type: manhattan_pearson
value: 46.910805787619914
- type: manhattan_spearman
value: 43.96662723946553
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-ar)
config: en-ar
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 16.222172523403835
- type: cos_sim_spearman
value: 17.230258645779042
- type: euclidean_pearson
value: -6.781460243147299
- type: euclidean_spearman
value: -6.884123336780775
- type: manhattan_pearson
value: -4.369061881907372
- type: manhattan_spearman
value: -4.235845433380353
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-de)
config: en-de
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 7.462476431657987
- type: cos_sim_spearman
value: 5.875270645234161
- type: euclidean_pearson
value: -10.79494346180473
- type: euclidean_spearman
value: -11.704529023304776
- type: manhattan_pearson
value: -11.465867974964997
- type: manhattan_spearman
value: -12.428424608287173
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-en)
config: en-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 61.46601840758559
- type: cos_sim_spearman
value: 65.69667638887147
- type: euclidean_pearson
value: 49.531065525619866
- type: euclidean_spearman
value: 53.880480167479725
- type: manhattan_pearson
value: 50.25462221374689
- type: manhattan_spearman
value: 54.22205494276401
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (en-tr)
config: en-tr
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: -12.769479370624031
- type: cos_sim_spearman
value: -12.161427312728382
- type: euclidean_pearson
value: -27.950593491756536
- type: euclidean_spearman
value: -24.925281959398585
- type: manhattan_pearson
value: -25.98778888167475
- type: manhattan_spearman
value: -22.861942388867234
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (es-en)
config: es-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 2.1575763564561727
- type: cos_sim_spearman
value: 1.182204089411577
- type: euclidean_pearson
value: -10.389249806317189
- type: euclidean_spearman
value: -16.078659904264605
- type: manhattan_pearson
value: -9.674301846448607
- type: manhattan_spearman
value: -16.976576817518577
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (es-es)
config: es-es
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 66.16718583059163
- type: cos_sim_spearman
value: 69.95156267898052
- type: euclidean_pearson
value: 64.93174777029739
- type: euclidean_spearman
value: 66.21292533974568
- type: manhattan_pearson
value: 65.2578109632889
- type: manhattan_spearman
value: 66.21830865759128
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (fr-en)
config: fr-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 0.1540829683540524
- type: cos_sim_spearman
value: -2.4072834011003987
- type: euclidean_pearson
value: -18.951775877513473
- type: euclidean_spearman
value: -18.393605606817527
- type: manhattan_pearson
value: -19.609633839454542
- type: manhattan_spearman
value: -19.276064769117912
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (it-en)
config: it-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: -4.22497246932717
- type: cos_sim_spearman
value: -5.747420352346977
- type: euclidean_pearson
value: -16.86351349130112
- type: euclidean_spearman
value: -16.555536618547382
- type: manhattan_pearson
value: -17.45445643482646
- type: manhattan_spearman
value: -17.97322953856309
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
name: MTEB STS17 (nl-en)
config: nl-en
split: test
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
metrics:
- type: cos_sim_pearson
value: 8.559184021676034
- type: cos_sim_spearman
value: 5.600273352595882
- type: euclidean_pearson
value: -10.76482859283058
- type: euclidean_spearman
value: -9.575202768285926
- type: manhattan_pearson
value: -9.48508597350615
- type: manhattan_spearman
value: -9.33387861352172
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (en)
config: en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 30.260087169228978
- type: cos_sim_spearman
value: 43.264174903196015
- type: euclidean_pearson
value: 35.07785877281954
- type: euclidean_spearman
value: 43.41294719372452
- type: manhattan_pearson
value: 36.74996284702431
- type: manhattan_spearman
value: 43.53522851890142
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de)
config: de
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 5.58694979115026
- type: cos_sim_spearman
value: 32.80692337371332
- type: euclidean_pearson
value: 10.53180875461474
- type: euclidean_spearman
value: 31.105269938654033
- type: manhattan_pearson
value: 10.559778015974826
- type: manhattan_spearman
value: 31.452204563072044
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (es)
config: es
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 10.593783873928478
- type: cos_sim_spearman
value: 50.397542574042006
- type: euclidean_pearson
value: 28.122179063209714
- type: euclidean_spearman
value: 50.72847867996529
- type: manhattan_pearson
value: 28.730690148465005
- type: manhattan_spearman
value: 51.019761292483366
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (pl)
config: pl
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: -1.3049499265017876
- type: cos_sim_spearman
value: 16.347130048706084
- type: euclidean_pearson
value: 0.5710147274110128
- type: euclidean_spearman
value: 16.589843077857605
- type: manhattan_pearson
value: 1.1226404198336415
- type: manhattan_spearman
value: 16.410620108636557
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (tr)
config: tr
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: -10.96861909019159
- type: cos_sim_spearman
value: 24.536979219880724
- type: euclidean_pearson
value: -1.3040190807315306
- type: euclidean_spearman
value: 25.061584673761928
- type: manhattan_pearson
value: -0.06525719745037804
- type: manhattan_spearman
value: 25.979295538386893
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (ar)
config: ar
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 1.0599417065503314
- type: cos_sim_spearman
value: 52.055853787103345
- type: euclidean_pearson
value: 23.666828441081776
- type: euclidean_spearman
value: 52.38656753170069
- type: manhattan_pearson
value: 23.398080463967215
- type: manhattan_spearman
value: 52.23849717509109
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (ru)
config: ru
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: -2.847646040977239
- type: cos_sim_spearman
value: 40.5826838357407
- type: euclidean_pearson
value: 9.242304983683113
- type: euclidean_spearman
value: 40.35906851022345
- type: manhattan_pearson
value: 9.645663412799504
- type: manhattan_spearman
value: 40.78106154950966
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 17.761397832130992
- type: cos_sim_spearman
value: 59.98756452345925
- type: euclidean_pearson
value: 37.03125109036693
- type: euclidean_spearman
value: 59.58469212715707
- type: manhattan_pearson
value: 36.828102137170724
- type: manhattan_spearman
value: 59.07036501478588
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (fr)
config: fr
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 22.281212883400205
- type: cos_sim_spearman
value: 48.27687537627578
- type: euclidean_pearson
value: 30.531395629285324
- type: euclidean_spearman
value: 50.349143748970384
- type: manhattan_pearson
value: 30.48762081986554
- type: manhattan_spearman
value: 50.66037165529169
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de-en)
config: de-en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 15.76679673990358
- type: cos_sim_spearman
value: 19.123349126370442
- type: euclidean_pearson
value: 19.21389203087116
- type: euclidean_spearman
value: 23.63276413160338
- type: manhattan_pearson
value: 18.789263824907053
- type: manhattan_spearman
value: 19.962703178974692
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (es-en)
config: es-en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 11.024970397289941
- type: cos_sim_spearman
value: 13.530951900755017
- type: euclidean_pearson
value: 13.473514585343645
- type: euclidean_spearman
value: 16.754702023734914
- type: manhattan_pearson
value: 13.72847275970385
- type: manhattan_spearman
value: 16.673001637012348
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (it)
config: it
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 33.32761589409043
- type: cos_sim_spearman
value: 54.14305778960692
- type: euclidean_pearson
value: 45.30173241170555
- type: euclidean_spearman
value: 54.77422257007743
- type: manhattan_pearson
value: 45.41890064000217
- type: manhattan_spearman
value: 54.533788920795544
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (pl-en)
config: pl-en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 20.045210048995486
- type: cos_sim_spearman
value: 17.597101329633823
- type: euclidean_pearson
value: 32.531726142346145
- type: euclidean_spearman
value: 27.244772040848105
- type: manhattan_pearson
value: 32.74618458514601
- type: manhattan_spearman
value: 25.81220754539242
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh-en)
config: zh-en
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: -13.832846350193021
- type: cos_sim_spearman
value: -8.406778050457863
- type: euclidean_pearson
value: -6.557254855697437
- type: euclidean_spearman
value: -3.5112770921588563
- type: manhattan_pearson
value: -6.493730738275641
- type: manhattan_spearman
value: -2.5922348401468365
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (es-it)
config: es-it
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 26.357929743436664
- type: cos_sim_spearman
value: 37.3417709718339
- type: euclidean_pearson
value: 30.930792572341293
- type: euclidean_spearman
value: 36.061866364725795
- type: manhattan_pearson
value: 31.56982745863155
- type: manhattan_spearman
value: 37.18529502311113
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de-fr)
config: de-fr
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 9.310102041071547
- type: cos_sim_spearman
value: 10.907002693108673
- type: euclidean_pearson
value: 7.361793742296021
- type: euclidean_spearman
value: 9.53967881391466
- type: manhattan_pearson
value: 8.017048631719996
- type: manhattan_spearman
value: 13.537860190039725
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (de-pl)
config: de-pl
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: -5.534456407419709
- type: cos_sim_spearman
value: 17.552638994787724
- type: euclidean_pearson
value: -10.136558594355556
- type: euclidean_spearman
value: 11.055083156366303
- type: manhattan_pearson
value: -11.799223055640773
- type: manhattan_spearman
value: 1.416528760982869
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (fr-pl)
config: fr-pl
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 48.64639760720344
- type: cos_sim_spearman
value: 39.440531887330785
- type: euclidean_pearson
value: 37.75527464173489
- type: euclidean_spearman
value: 39.440531887330785
- type: manhattan_pearson
value: 32.324715276369474
- type: manhattan_spearman
value: 28.17180849095055
- task:
type: STS
dataset:
type: mteb/stsbenchmark-sts
name: MTEB STSBenchmark
config: default
split: test
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831
metrics:
- type: cos_sim_pearson
value: 44.667456983937
- type: cos_sim_spearman
value: 46.04327333618551
- type: euclidean_pearson
value: 44.583522824155104
- type: euclidean_spearman
value: 44.77184813864239
- type: manhattan_pearson
value: 44.54496373721756
- type: manhattan_spearman
value: 44.830873857115996
- task:
type: Reranking
dataset:
type: mteb/scidocs-reranking
name: MTEB SciDocsRR
config: default
split: test
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab
metrics:
- type: map
value: 49.756063724243
- type: mrr
value: 75.29077585450135
- task:
type: Retrieval
dataset:
type: scifact
name: MTEB SciFact
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 14.194
- type: map_at_10
value: 18.756999999999998
- type: map_at_100
value: 19.743
- type: map_at_1000
value: 19.865
- type: map_at_3
value: 16.986
- type: map_at_5
value: 18.024
- type: mrr_at_1
value: 15
- type: mrr_at_10
value: 19.961000000000002
- type: mrr_at_100
value: 20.875
- type: mrr_at_1000
value: 20.982
- type: mrr_at_3
value: 18.056
- type: mrr_at_5
value: 19.406000000000002
- type: ndcg_at_1
value: 15
- type: ndcg_at_10
value: 21.775
- type: ndcg_at_100
value: 26.8
- type: ndcg_at_1000
value: 30.468
- type: ndcg_at_3
value: 18.199
- type: ndcg_at_5
value: 20.111
- type: precision_at_1
value: 15
- type: precision_at_10
value: 3.4000000000000004
- type: precision_at_100
value: 0.607
- type: precision_at_1000
value: 0.094
- type: precision_at_3
value: 7.444000000000001
- type: precision_at_5
value: 5.6000000000000005
- type: recall_at_1
value: 14.194
- type: recall_at_10
value: 30
- type: recall_at_100
value: 53.911
- type: recall_at_1000
value: 83.289
- type: recall_at_3
value: 20.556
- type: recall_at_5
value: 24.972
- task:
type: PairClassification
dataset:
type: mteb/sprintduplicatequestions-pairclassification
name: MTEB SprintDuplicateQuestions
config: default
split: test
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46
metrics:
- type: cos_sim_accuracy
value: 99.35544554455446
- type: cos_sim_ap
value: 62.596006705300724
- type: cos_sim_f1
value: 60.80283353010627
- type: cos_sim_precision
value: 74.20749279538906
- type: cos_sim_recall
value: 51.5
- type: dot_accuracy
value: 99.13564356435643
- type: dot_ap
value: 43.87589686325114
- type: dot_f1
value: 46.99663623258049
- type: dot_precision
value: 45.235892691951896
- type: dot_recall
value: 48.9
- type: euclidean_accuracy
value: 99.2
- type: euclidean_ap
value: 43.44660755386079
- type: euclidean_f1
value: 45.9016393442623
- type: euclidean_precision
value: 52.79583875162549
- type: euclidean_recall
value: 40.6
- type: manhattan_accuracy
value: 99.2
- type: manhattan_ap
value: 43.11790011749347
- type: manhattan_f1
value: 45.11023176936122
- type: manhattan_precision
value: 51.88556566970091
- type: manhattan_recall
value: 39.900000000000006
- type: max_accuracy
value: 99.35544554455446
- type: max_ap
value: 62.596006705300724
- type: max_f1
value: 60.80283353010627
- task:
type: Clustering
dataset:
type: mteb/stackexchange-clustering
name: MTEB StackExchangeClustering
config: default
split: test
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259
metrics:
- type: v_measure
value: 25.71674282500873
- task:
type: Clustering
dataset:
type: mteb/stackexchange-clustering-p2p
name: MTEB StackExchangeClusteringP2P
config: default
split: test
revision: 815ca46b2622cec33ccafc3735d572c266efdb44
metrics:
- type: v_measure
value: 25.465780711520985
- task:
type: Reranking
dataset:
type: mteb/stackoverflowdupquestions-reranking
name: MTEB StackOverflowDupQuestions
config: default
split: test
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69
metrics:
- type: map
value: 35.35656209427094
- type: mrr
value: 35.10693860877685
- task:
type: Retrieval
dataset:
type: trec-covid
name: MTEB TRECCOVID
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 0.074
- type: map_at_10
value: 0.47400000000000003
- type: map_at_100
value: 1.825
- type: map_at_1000
value: 4.056
- type: map_at_3
value: 0.199
- type: map_at_5
value: 0.301
- type: mrr_at_1
value: 34
- type: mrr_at_10
value: 46.06
- type: mrr_at_100
value: 47.506
- type: mrr_at_1000
value: 47.522999999999996
- type: mrr_at_3
value: 44
- type: mrr_at_5
value: 44.4
- type: ndcg_at_1
value: 32
- type: ndcg_at_10
value: 28.633999999999997
- type: ndcg_at_100
value: 18.547
- type: ndcg_at_1000
value: 16.142
- type: ndcg_at_3
value: 32.48
- type: ndcg_at_5
value: 31.163999999999998
- type: precision_at_1
value: 34
- type: precision_at_10
value: 30.4
- type: precision_at_100
value: 18.54
- type: precision_at_1000
value: 7.942
- type: precision_at_3
value: 35.333
- type: precision_at_5
value: 34
- type: recall_at_1
value: 0.074
- type: recall_at_10
value: 0.641
- type: recall_at_100
value: 3.675
- type: recall_at_1000
value: 15.706000000000001
- type: recall_at_3
value: 0.231
- type: recall_at_5
value: 0.367
- task:
type: Classification
dataset:
type: mteb/toxic_conversations_50k
name: MTEB ToxicConversationsClassification
config: default
split: test
revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c
metrics:
- type: accuracy
value: 54.625600000000006
- type: ap
value: 9.425323874806459
- type: f1
value: 42.38724794017267
- task:
type: Classification
dataset:
type: mteb/tweet_sentiment_extraction
name: MTEB TweetSentimentExtractionClassification
config: default
split: test
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a
metrics:
- type: accuracy
value: 42.8494623655914
- type: f1
value: 42.66062148844617
- task:
type: Clustering
dataset:
type: mteb/twentynewsgroups-clustering
name: MTEB TwentyNewsgroupsClustering
config: default
split: test
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449
metrics:
- type: v_measure
value: 12.464890895237952
- task:
type: PairClassification
dataset:
type: mteb/twittersemeval2015-pairclassification
name: MTEB TwitterSemEval2015
config: default
split: test
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1
metrics:
- type: cos_sim_accuracy
value: 79.97854205161829
- type: cos_sim_ap
value: 47.45175747605773
- type: cos_sim_f1
value: 46.55775962660444
- type: cos_sim_precision
value: 41.73640167364017
- type: cos_sim_recall
value: 52.638522427440634
- type: dot_accuracy
value: 77.76718126005842
- type: dot_ap
value: 35.97737653101504
- type: dot_f1
value: 41.1975475754439
- type: dot_precision
value: 29.50165355228646
- type: dot_recall
value: 68.25857519788919
- type: euclidean_accuracy
value: 79.34076414138403
- type: euclidean_ap
value: 45.309577778755134
- type: euclidean_f1
value: 45.09938313913639
- type: euclidean_precision
value: 39.76631748589847
- type: euclidean_recall
value: 52.0844327176781
- type: manhattan_accuracy
value: 79.31692197651546
- type: manhattan_ap
value: 45.2433373222626
- type: manhattan_f1
value: 45.04624986069319
- type: manhattan_precision
value: 38.99286127725256
- type: manhattan_recall
value: 53.324538258575195
- type: max_accuracy
value: 79.97854205161829
- type: max_ap
value: 47.45175747605773
- type: max_f1
value: 46.55775962660444
- task:
type: PairClassification
dataset:
type: mteb/twitterurlcorpus-pairclassification
name: MTEB TwitterURLCorpus
config: default
split: test
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf
metrics:
- type: cos_sim_accuracy
value: 81.76737687740133
- type: cos_sim_ap
value: 64.59241956109807
- type: cos_sim_f1
value: 57.83203629255339
- type: cos_sim_precision
value: 55.50442477876106
- type: cos_sim_recall
value: 60.363412380659064
- type: dot_accuracy
value: 78.96922420149805
- type: dot_ap
value: 56.11775087282065
- type: dot_f1
value: 52.92134831460675
- type: dot_precision
value: 51.524212368728115
- type: dot_recall
value: 54.39636587619341
- type: euclidean_accuracy
value: 80.8611790274382
- type: euclidean_ap
value: 61.28070098354092
- type: euclidean_f1
value: 54.58334971882497
- type: euclidean_precision
value: 55.783297162607504
- type: euclidean_recall
value: 53.43393902063443
- type: manhattan_accuracy
value: 80.72534637326814
- type: manhattan_ap
value: 61.18048430787254
- type: manhattan_f1
value: 54.50978912822061
- type: manhattan_precision
value: 53.435396790178245
- type: manhattan_recall
value: 55.6282722513089
- type: max_accuracy
value: 81.76737687740133
- type: max_ap
value: 64.59241956109807
- type: max_f1
value: 57.83203629255339