|
|
--- |
|
|
model-index: |
|
|
- name: Quark-Emb-1.5b |
|
|
results: |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB AFQMC |
|
|
revision: None |
|
|
split: validation |
|
|
type: C-MTEB/AFQMC |
|
|
metrics: |
|
|
- type: cosine_pearson |
|
|
value: 47.14285927987258 |
|
|
- type: cosine_spearman |
|
|
value: 48.161200368263025 |
|
|
- type: manhattan_pearson |
|
|
value: 46.852921578928694 |
|
|
- type: manhattan_spearman |
|
|
value: 48.0768829644805 |
|
|
- type: euclidean_pearson |
|
|
value: 46.934710408297846 |
|
|
- type: euclidean_spearman |
|
|
value: 48.161200368263025 |
|
|
- type: main_score |
|
|
value: 48.161200368263025 |
|
|
task: |
|
|
type: STS |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB ATEC |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/ATEC |
|
|
metrics: |
|
|
- type: cosine_pearson |
|
|
value: 53.31694395347832 |
|
|
- type: cosine_spearman |
|
|
value: 50.82142054857025 |
|
|
- type: manhattan_pearson |
|
|
value: 55.63018022546727 |
|
|
- type: manhattan_spearman |
|
|
value: 50.808925663235286 |
|
|
- type: euclidean_pearson |
|
|
value: 55.630897902214585 |
|
|
- type: euclidean_spearman |
|
|
value: 50.82142054857025 |
|
|
- type: main_score |
|
|
value: 50.82142054857025 |
|
|
task: |
|
|
type: STS |
|
|
- dataset: |
|
|
config: zh |
|
|
name: MTEB AmazonReviewsClassification (zh) |
|
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
|
split: test |
|
|
type: mteb/amazon_reviews_multi |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 51.93800000000001 |
|
|
- type: accuracy_stderr |
|
|
value: 1.6225030046197138 |
|
|
- type: f1 |
|
|
value: 49.36480272612989 |
|
|
- type: f1_stderr |
|
|
value: 2.402473535325102 |
|
|
- type: main_score |
|
|
value: 51.93800000000001 |
|
|
task: |
|
|
type: Classification |
|
|
- dataset: |
|
|
config: zh |
|
|
name: MTEB AmazonReviewsClassification (zh) |
|
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
|
split: validation |
|
|
type: mteb/amazon_reviews_multi |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 50.757999999999996 |
|
|
- type: accuracy_stderr |
|
|
value: 1.1949041802588176 |
|
|
- type: f1 |
|
|
value: 48.18542841607346 |
|
|
- type: f1_stderr |
|
|
value: 2.025507464835368 |
|
|
- type: main_score |
|
|
value: 50.757999999999996 |
|
|
task: |
|
|
type: Classification |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB BQ |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/BQ |
|
|
metrics: |
|
|
- type: cosine_pearson |
|
|
value: 66.94471481392273 |
|
|
- type: cosine_spearman |
|
|
value: 67.86811107045457 |
|
|
- type: manhattan_pearson |
|
|
value: 65.56778188873142 |
|
|
- type: manhattan_spearman |
|
|
value: 67.83060870618156 |
|
|
- type: euclidean_pearson |
|
|
value: 65.63668085779311 |
|
|
- type: euclidean_spearman |
|
|
value: 67.86811107045457 |
|
|
- type: main_score |
|
|
value: 67.86811107045457 |
|
|
task: |
|
|
type: STS |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB CLSClusteringP2P |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/CLSClusteringP2P |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 58.53706905558472 |
|
|
- type: v_measure_std |
|
|
value: 1.3628784531981595 |
|
|
- type: main_score |
|
|
value: 58.53706905558472 |
|
|
task: |
|
|
type: Clustering |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB CLSClusteringS2S |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/CLSClusteringS2S |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 54.70969139354621 |
|
|
- type: v_measure_std |
|
|
value: 1.938384688132648 |
|
|
- type: main_score |
|
|
value: 54.70969139354621 |
|
|
task: |
|
|
type: Clustering |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB CMedQAv1 |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/CMedQAv1-reranking |
|
|
metrics: |
|
|
- type: map |
|
|
value: 87.79521046311835 |
|
|
- type: mrr |
|
|
value: 90.01547619047618 |
|
|
- type: main_score |
|
|
value: 87.79521046311835 |
|
|
task: |
|
|
type: Reranking |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB CMedQAv2 |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/CMedQAv2-reranking |
|
|
metrics: |
|
|
- type: map |
|
|
value: 87.89916670870878 |
|
|
- type: mrr |
|
|
value: 89.92595238095238 |
|
|
- type: main_score |
|
|
value: 87.89916670870878 |
|
|
task: |
|
|
type: Reranking |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB CmedqaRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/CmedqaRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 25.444 |
|
|
- type: map_at_10 |
|
|
value: 37.763999999999996 |
|
|
- type: map_at_100 |
|
|
value: 39.641999999999996 |
|
|
- type: map_at_1000 |
|
|
value: 39.756 |
|
|
- type: map_at_3 |
|
|
value: 33.742 |
|
|
- type: map_at_5 |
|
|
value: 35.906 |
|
|
- type: mrr_at_1 |
|
|
value: 38.71 |
|
|
- type: mrr_at_10 |
|
|
value: 46.744 |
|
|
- type: mrr_at_100 |
|
|
value: 47.745 |
|
|
- type: mrr_at_1000 |
|
|
value: 47.791 |
|
|
- type: mrr_at_3 |
|
|
value: 44.324000000000005 |
|
|
- type: mrr_at_5 |
|
|
value: 45.696 |
|
|
- type: ndcg_at_1 |
|
|
value: 38.71 |
|
|
- type: ndcg_at_10 |
|
|
value: 44.285000000000004 |
|
|
- type: ndcg_at_100 |
|
|
value: 51.69200000000001 |
|
|
- type: ndcg_at_1000 |
|
|
value: 53.669999999999995 |
|
|
- type: ndcg_at_3 |
|
|
value: 39.273 |
|
|
- type: ndcg_at_5 |
|
|
value: 41.254000000000005 |
|
|
- type: precision_at_1 |
|
|
value: 38.71 |
|
|
- type: precision_at_10 |
|
|
value: 9.825000000000001 |
|
|
- type: precision_at_100 |
|
|
value: 1.583 |
|
|
- type: precision_at_1000 |
|
|
value: 0.183 |
|
|
- type: precision_at_3 |
|
|
value: 22.197 |
|
|
- type: precision_at_5 |
|
|
value: 16.019 |
|
|
- type: recall_at_1 |
|
|
value: 25.444 |
|
|
- type: recall_at_10 |
|
|
value: 54.535999999999994 |
|
|
- type: recall_at_100 |
|
|
value: 85.307 |
|
|
- type: recall_at_1000 |
|
|
value: 98.473 |
|
|
- type: recall_at_3 |
|
|
value: 39.274 |
|
|
- type: recall_at_5 |
|
|
value: 45.580999999999996 |
|
|
- type: main_score |
|
|
value: 44.285000000000004 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB Cmnli |
|
|
revision: None |
|
|
split: validation |
|
|
type: C-MTEB/CMNLI |
|
|
metrics: |
|
|
- type: cos_sim_accuracy |
|
|
value: 89.58508719182201 |
|
|
- type: cos_sim_accuracy_threshold |
|
|
value: 97.09511288861569 |
|
|
- type: cos_sim_ap |
|
|
value: 95.12338246323735 |
|
|
- type: cos_sim_f1 |
|
|
value: 90.19211324570271 |
|
|
- type: cos_sim_f1_threshold |
|
|
value: 97.02014138938755 |
|
|
- type: cos_sim_precision |
|
|
value: 86.80795847750865 |
|
|
- type: cos_sim_recall |
|
|
value: 93.85083002104278 |
|
|
- type: dot_accuracy |
|
|
value: 89.58508719182201 |
|
|
- type: dot_accuracy_threshold |
|
|
value: 97.0951128886157 |
|
|
- type: dot_ap |
|
|
value: 95.13959275940286 |
|
|
- type: dot_f1 |
|
|
value: 90.19211324570271 |
|
|
- type: dot_f1_threshold |
|
|
value: 97.02014138938755 |
|
|
- type: dot_precision |
|
|
value: 86.80795847750865 |
|
|
- type: dot_recall |
|
|
value: 93.85083002104278 |
|
|
- type: euclidean_accuracy |
|
|
value: 89.58508719182201 |
|
|
- type: euclidean_accuracy_threshold |
|
|
value: 24.103473235790947 |
|
|
- type: euclidean_ap |
|
|
value: 95.12338246323735 |
|
|
- type: euclidean_f1 |
|
|
value: 90.19211324570271 |
|
|
- type: euclidean_f1_threshold |
|
|
value: 24.412531977088996 |
|
|
- type: euclidean_precision |
|
|
value: 86.80795847750865 |
|
|
- type: euclidean_recall |
|
|
value: 93.85083002104278 |
|
|
- type: manhattan_accuracy |
|
|
value: 89.57306073361396 |
|
|
- type: manhattan_accuracy_threshold |
|
|
value: 729.1211254739587 |
|
|
- type: manhattan_ap |
|
|
value: 95.12388319543341 |
|
|
- type: manhattan_f1 |
|
|
value: 90.13956654941563 |
|
|
- type: manhattan_f1_threshold |
|
|
value: 733.155723492131 |
|
|
- type: manhattan_precision |
|
|
value: 87.56613756613757 |
|
|
- type: manhattan_recall |
|
|
value: 92.8688332943652 |
|
|
- type: max_accuracy |
|
|
value: 89.58508719182201 |
|
|
- type: max_ap |
|
|
value: 95.13959275940286 |
|
|
- type: max_f1 |
|
|
value: 90.19211324570271 |
|
|
task: |
|
|
type: PairClassification |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB CovidRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/CovidRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 75.29 |
|
|
- type: map_at_10 |
|
|
value: 82.392 |
|
|
- type: map_at_100 |
|
|
value: 82.581 |
|
|
- type: map_at_1000 |
|
|
value: 82.585 |
|
|
- type: map_at_3 |
|
|
value: 80.88300000000001 |
|
|
- type: map_at_5 |
|
|
value: 81.71199999999999 |
|
|
- type: mrr_at_1 |
|
|
value: 75.553 |
|
|
- type: mrr_at_10 |
|
|
value: 82.422 |
|
|
- type: mrr_at_100 |
|
|
value: 82.6 |
|
|
- type: mrr_at_1000 |
|
|
value: 82.604 |
|
|
- type: mrr_at_3 |
|
|
value: 80.927 |
|
|
- type: mrr_at_5 |
|
|
value: 81.765 |
|
|
- type: ndcg_at_1 |
|
|
value: 75.44800000000001 |
|
|
- type: ndcg_at_10 |
|
|
value: 85.655 |
|
|
- type: ndcg_at_100 |
|
|
value: 86.435 |
|
|
- type: ndcg_at_1000 |
|
|
value: 86.541 |
|
|
- type: ndcg_at_3 |
|
|
value: 82.60300000000001 |
|
|
- type: ndcg_at_5 |
|
|
value: 84.062 |
|
|
- type: precision_at_1 |
|
|
value: 75.44800000000001 |
|
|
- type: precision_at_10 |
|
|
value: 9.663 |
|
|
- type: precision_at_100 |
|
|
value: 1.002 |
|
|
- type: precision_at_1000 |
|
|
value: 0.101 |
|
|
- type: precision_at_3 |
|
|
value: 29.329 |
|
|
- type: precision_at_5 |
|
|
value: 18.314 |
|
|
- type: recall_at_1 |
|
|
value: 75.29 |
|
|
- type: recall_at_10 |
|
|
value: 95.838 |
|
|
- type: recall_at_100 |
|
|
value: 99.157 |
|
|
- type: recall_at_1000 |
|
|
value: 100.0 |
|
|
- type: recall_at_3 |
|
|
value: 87.566 |
|
|
- type: recall_at_5 |
|
|
value: 90.991 |
|
|
- type: main_score |
|
|
value: 85.655 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB DuRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/DuRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 27.584999999999997 |
|
|
- type: map_at_10 |
|
|
value: 85.112 |
|
|
- type: map_at_100 |
|
|
value: 87.632 |
|
|
- type: map_at_1000 |
|
|
value: 87.654 |
|
|
- type: map_at_3 |
|
|
value: 59.504999999999995 |
|
|
- type: map_at_5 |
|
|
value: 75.029 |
|
|
- type: mrr_at_1 |
|
|
value: 93.30000000000001 |
|
|
- type: mrr_at_10 |
|
|
value: 95.44200000000001 |
|
|
- type: mrr_at_100 |
|
|
value: 95.498 |
|
|
- type: mrr_at_1000 |
|
|
value: 95.5 |
|
|
- type: mrr_at_3 |
|
|
value: 95.258 |
|
|
- type: mrr_at_5 |
|
|
value: 95.36099999999999 |
|
|
- type: ndcg_at_1 |
|
|
value: 93.30000000000001 |
|
|
- type: ndcg_at_10 |
|
|
value: 91.086 |
|
|
- type: ndcg_at_100 |
|
|
value: 93.089 |
|
|
- type: ndcg_at_1000 |
|
|
value: 93.297 |
|
|
- type: ndcg_at_3 |
|
|
value: 90.432 |
|
|
- type: ndcg_at_5 |
|
|
value: 89.361 |
|
|
- type: precision_at_1 |
|
|
value: 93.30000000000001 |
|
|
- type: precision_at_10 |
|
|
value: 43.21 |
|
|
- type: precision_at_100 |
|
|
value: 4.857 |
|
|
- type: precision_at_1000 |
|
|
value: 0.49 |
|
|
- type: precision_at_3 |
|
|
value: 81.0 |
|
|
- type: precision_at_5 |
|
|
value: 68.28999999999999 |
|
|
- type: recall_at_1 |
|
|
value: 27.584999999999997 |
|
|
- type: recall_at_10 |
|
|
value: 91.73599999999999 |
|
|
- type: recall_at_100 |
|
|
value: 98.648 |
|
|
- type: recall_at_1000 |
|
|
value: 99.751 |
|
|
- type: recall_at_3 |
|
|
value: 61.378 |
|
|
- type: recall_at_5 |
|
|
value: 78.672 |
|
|
- type: main_score |
|
|
value: 91.086 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB EcomRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/EcomRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 55.1 |
|
|
- type: map_at_10 |
|
|
value: 65.268 |
|
|
- type: map_at_100 |
|
|
value: 65.756 |
|
|
- type: map_at_1000 |
|
|
value: 65.765 |
|
|
- type: map_at_3 |
|
|
value: 63.132999999999996 |
|
|
- type: map_at_5 |
|
|
value: 64.25800000000001 |
|
|
- type: mrr_at_1 |
|
|
value: 55.1 |
|
|
- type: mrr_at_10 |
|
|
value: 65.268 |
|
|
- type: mrr_at_100 |
|
|
value: 65.756 |
|
|
- type: mrr_at_1000 |
|
|
value: 65.765 |
|
|
- type: mrr_at_3 |
|
|
value: 63.132999999999996 |
|
|
- type: mrr_at_5 |
|
|
value: 64.25800000000001 |
|
|
- type: ndcg_at_1 |
|
|
value: 55.1 |
|
|
- type: ndcg_at_10 |
|
|
value: 70.15599999999999 |
|
|
- type: ndcg_at_100 |
|
|
value: 72.368 |
|
|
- type: ndcg_at_1000 |
|
|
value: 72.635 |
|
|
- type: ndcg_at_3 |
|
|
value: 65.697 |
|
|
- type: ndcg_at_5 |
|
|
value: 67.741 |
|
|
- type: precision_at_1 |
|
|
value: 55.1 |
|
|
- type: precision_at_10 |
|
|
value: 8.55 |
|
|
- type: precision_at_100 |
|
|
value: 0.955 |
|
|
- type: precision_at_1000 |
|
|
value: 0.098 |
|
|
- type: precision_at_3 |
|
|
value: 24.367 |
|
|
- type: precision_at_5 |
|
|
value: 15.620000000000001 |
|
|
- type: recall_at_1 |
|
|
value: 55.1 |
|
|
- type: recall_at_10 |
|
|
value: 85.5 |
|
|
- type: recall_at_100 |
|
|
value: 95.5 |
|
|
- type: recall_at_1000 |
|
|
value: 97.6 |
|
|
- type: recall_at_3 |
|
|
value: 73.1 |
|
|
- type: recall_at_5 |
|
|
value: 78.10000000000001 |
|
|
- type: main_score |
|
|
value: 70.15599999999999 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB IFlyTek |
|
|
revision: None |
|
|
split: validation |
|
|
type: C-MTEB/IFlyTek-classification |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 52.743362831858406 |
|
|
- type: accuracy_stderr |
|
|
value: 0.4449967616714387 |
|
|
- type: f1 |
|
|
value: 40.13427504900375 |
|
|
- type: f1_stderr |
|
|
value: 0.17565290177989018 |
|
|
- type: main_score |
|
|
value: 52.743362831858406 |
|
|
task: |
|
|
type: Classification |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB JDReview |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/JDReview-classification |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 90.13133208255161 |
|
|
- type: accuracy_stderr |
|
|
value: 0.9647249630155678 |
|
|
- type: ap |
|
|
value: 62.848199712439765 |
|
|
- type: ap_stderr |
|
|
value: 1.986859492917626 |
|
|
- type: f1 |
|
|
value: 85.48543445690254 |
|
|
- type: f1_stderr |
|
|
value: 1.0490059319804828 |
|
|
- type: main_score |
|
|
value: 90.13133208255161 |
|
|
task: |
|
|
type: Classification |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB LCQMC |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/LCQMC |
|
|
metrics: |
|
|
- type: cosine_pearson |
|
|
value: 77.75677384428634 |
|
|
- type: cosine_spearman |
|
|
value: 78.86284859566986 |
|
|
- type: manhattan_pearson |
|
|
value: 79.8032754323316 |
|
|
- type: manhattan_spearman |
|
|
value: 78.85558562163624 |
|
|
- type: euclidean_pearson |
|
|
value: 79.82552324704292 |
|
|
- type: euclidean_spearman |
|
|
value: 78.86284859566986 |
|
|
- type: main_score |
|
|
value: 78.86284859566986 |
|
|
task: |
|
|
type: STS |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB MMarcoReranking |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/Mmarco-reranking |
|
|
metrics: |
|
|
- type: map |
|
|
value: 30.737025407798523 |
|
|
- type: mrr |
|
|
value: 29.26111111111111 |
|
|
- type: main_score |
|
|
value: 30.737025407798523 |
|
|
task: |
|
|
type: Reranking |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB MMarcoRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/MMarcoRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 70.244 |
|
|
- type: map_at_10 |
|
|
value: 78.975 |
|
|
- type: map_at_100 |
|
|
value: 79.253 |
|
|
- type: map_at_1000 |
|
|
value: 79.26100000000001 |
|
|
- type: map_at_3 |
|
|
value: 77.363 |
|
|
- type: map_at_5 |
|
|
value: 78.364 |
|
|
- type: mrr_at_1 |
|
|
value: 72.521 |
|
|
- type: mrr_at_10 |
|
|
value: 79.514 |
|
|
- type: mrr_at_100 |
|
|
value: 79.75 |
|
|
- type: mrr_at_1000 |
|
|
value: 79.757 |
|
|
- type: mrr_at_3 |
|
|
value: 78.095 |
|
|
- type: mrr_at_5 |
|
|
value: 78.987 |
|
|
- type: ndcg_at_1 |
|
|
value: 72.521 |
|
|
- type: ndcg_at_10 |
|
|
value: 82.395 |
|
|
- type: ndcg_at_100 |
|
|
value: 83.554 |
|
|
- type: ndcg_at_1000 |
|
|
value: 83.774 |
|
|
- type: ndcg_at_3 |
|
|
value: 79.341 |
|
|
- type: ndcg_at_5 |
|
|
value: 81.036 |
|
|
- type: precision_at_1 |
|
|
value: 72.521 |
|
|
- type: precision_at_10 |
|
|
value: 9.812 |
|
|
- type: precision_at_100 |
|
|
value: 1.038 |
|
|
- type: precision_at_1000 |
|
|
value: 0.106 |
|
|
- type: precision_at_3 |
|
|
value: 29.694 |
|
|
- type: precision_at_5 |
|
|
value: 18.712999999999997 |
|
|
- type: recall_at_1 |
|
|
value: 70.244 |
|
|
- type: recall_at_10 |
|
|
value: 92.35 |
|
|
- type: recall_at_100 |
|
|
value: 97.419 |
|
|
- type: recall_at_1000 |
|
|
value: 99.16199999999999 |
|
|
- type: recall_at_3 |
|
|
value: 84.303 |
|
|
- type: recall_at_5 |
|
|
value: 88.325 |
|
|
- type: main_score |
|
|
value: 82.395 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: zh-CN |
|
|
name: MTEB MassiveIntentClassification (zh-CN) |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
split: test |
|
|
type: mteb/amazon_massive_intent |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 76.3752521856086 |
|
|
- type: accuracy_stderr |
|
|
value: 1.3911220977886072 |
|
|
- type: f1 |
|
|
value: 73.38330839246518 |
|
|
- type: f1_stderr |
|
|
value: 0.9864886479418102 |
|
|
- type: main_score |
|
|
value: 76.3752521856086 |
|
|
task: |
|
|
type: Classification |
|
|
- dataset: |
|
|
config: zh-CN |
|
|
name: MTEB MassiveScenarioClassification (zh-CN) |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
split: test |
|
|
type: mteb/amazon_massive_scenario |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 81.8022864828514 |
|
|
- type: accuracy_stderr |
|
|
value: 1.4060452754762354 |
|
|
- type: f1 |
|
|
value: 80.85164585310973 |
|
|
- type: f1_stderr |
|
|
value: 1.2664399398388577 |
|
|
- type: main_score |
|
|
value: 81.8022864828514 |
|
|
task: |
|
|
type: Classification |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB MedicalRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/MedicalRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 57.199999999999996 |
|
|
- type: map_at_10 |
|
|
value: 63.346999999999994 |
|
|
- type: map_at_100 |
|
|
value: 63.852 |
|
|
- type: map_at_1000 |
|
|
value: 63.88700000000001 |
|
|
- type: map_at_3 |
|
|
value: 61.967000000000006 |
|
|
- type: map_at_5 |
|
|
value: 62.66199999999999 |
|
|
- type: mrr_at_1 |
|
|
value: 57.3 |
|
|
- type: mrr_at_10 |
|
|
value: 63.397000000000006 |
|
|
- type: mrr_at_100 |
|
|
value: 63.902 |
|
|
- type: mrr_at_1000 |
|
|
value: 63.937 |
|
|
- type: mrr_at_3 |
|
|
value: 62.017 |
|
|
- type: mrr_at_5 |
|
|
value: 62.712 |
|
|
- type: ndcg_at_1 |
|
|
value: 57.199999999999996 |
|
|
- type: ndcg_at_10 |
|
|
value: 66.38300000000001 |
|
|
- type: ndcg_at_100 |
|
|
value: 69.267 |
|
|
- type: ndcg_at_1000 |
|
|
value: 70.233 |
|
|
- type: ndcg_at_3 |
|
|
value: 63.44499999999999 |
|
|
- type: ndcg_at_5 |
|
|
value: 64.71000000000001 |
|
|
- type: precision_at_1 |
|
|
value: 57.199999999999996 |
|
|
- type: precision_at_10 |
|
|
value: 7.6 |
|
|
- type: precision_at_100 |
|
|
value: 0.905 |
|
|
- type: precision_at_1000 |
|
|
value: 0.098 |
|
|
- type: precision_at_3 |
|
|
value: 22.567 |
|
|
- type: precision_at_5 |
|
|
value: 14.16 |
|
|
- type: recall_at_1 |
|
|
value: 57.199999999999996 |
|
|
- type: recall_at_10 |
|
|
value: 76.0 |
|
|
- type: recall_at_100 |
|
|
value: 90.5 |
|
|
- type: recall_at_1000 |
|
|
value: 98.2 |
|
|
- type: recall_at_3 |
|
|
value: 67.7 |
|
|
- type: recall_at_5 |
|
|
value: 70.8 |
|
|
- type: main_score |
|
|
value: 66.38300000000001 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB MultilingualSentiment |
|
|
revision: None |
|
|
split: validation |
|
|
type: C-MTEB/MultilingualSentiment-classification |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 80.12333333333335 |
|
|
- type: accuracy_stderr |
|
|
value: 0.31377628265303376 |
|
|
- type: f1 |
|
|
value: 80.26166732998303 |
|
|
- type: f1_stderr |
|
|
value: 0.2836457609943486 |
|
|
- type: main_score |
|
|
value: 80.12333333333335 |
|
|
task: |
|
|
type: Classification |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB Ocnli |
|
|
revision: None |
|
|
split: validation |
|
|
type: C-MTEB/OCNLI |
|
|
metrics: |
|
|
- type: cos_sim_accuracy |
|
|
value: 87.54737412019492 |
|
|
- type: cos_sim_accuracy_threshold |
|
|
value: 96.99121475650863 |
|
|
- type: cos_sim_ap |
|
|
value: 91.71816430648396 |
|
|
- type: cos_sim_f1 |
|
|
value: 88.27655310621243 |
|
|
- type: cos_sim_f1_threshold |
|
|
value: 96.8697507135398 |
|
|
- type: cos_sim_precision |
|
|
value: 83.98474737845567 |
|
|
- type: cos_sim_recall |
|
|
value: 93.03062302006336 |
|
|
- type: dot_accuracy |
|
|
value: 87.54737412019492 |
|
|
- type: dot_accuracy_threshold |
|
|
value: 96.99121475650863 |
|
|
- type: dot_ap |
|
|
value: 91.71816430648396 |
|
|
- type: dot_f1 |
|
|
value: 88.27655310621243 |
|
|
- type: dot_f1_threshold |
|
|
value: 96.86975071353979 |
|
|
- type: dot_precision |
|
|
value: 83.98474737845567 |
|
|
- type: dot_recall |
|
|
value: 93.03062302006336 |
|
|
- type: euclidean_accuracy |
|
|
value: 87.54737412019492 |
|
|
- type: euclidean_accuracy_threshold |
|
|
value: 24.530733065589622 |
|
|
- type: euclidean_ap |
|
|
value: 91.71816430648396 |
|
|
- type: euclidean_f1 |
|
|
value: 88.27655310621243 |
|
|
- type: euclidean_f1_threshold |
|
|
value: 25.020988098238107 |
|
|
- type: euclidean_precision |
|
|
value: 83.98474737845567 |
|
|
- type: euclidean_recall |
|
|
value: 93.03062302006336 |
|
|
- type: manhattan_accuracy |
|
|
value: 87.27666486193829 |
|
|
- type: manhattan_accuracy_threshold |
|
|
value: 752.4905438529156 |
|
|
- type: manhattan_ap |
|
|
value: 91.70647280240597 |
|
|
- type: manhattan_f1 |
|
|
value: 88.08920425747591 |
|
|
- type: manhattan_f1_threshold |
|
|
value: 752.4905438529156 |
|
|
- type: manhattan_precision |
|
|
value: 84.69785575048732 |
|
|
- type: manhattan_recall |
|
|
value: 91.76346356916578 |
|
|
- type: max_accuracy |
|
|
value: 87.54737412019492 |
|
|
- type: max_ap |
|
|
value: 91.71816430648396 |
|
|
- type: max_f1 |
|
|
value: 88.27655310621243 |
|
|
task: |
|
|
type: PairClassification |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB OnlineShopping |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/OnlineShopping-classification |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 94.46999999999998 |
|
|
- type: accuracy_stderr |
|
|
value: 0.2865309756378883 |
|
|
- type: ap |
|
|
value: 93.00417328431348 |
|
|
- type: ap_stderr |
|
|
value: 0.5383352662551945 |
|
|
- type: f1 |
|
|
value: 94.4618263222835 |
|
|
- type: f1_stderr |
|
|
value: 0.2840342094212124 |
|
|
- type: main_score |
|
|
value: 94.46999999999998 |
|
|
task: |
|
|
type: Classification |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB PAWSX |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/PAWSX |
|
|
metrics: |
|
|
- type: cosine_pearson |
|
|
value: 46.85211982536296 |
|
|
- type: cosine_spearman |
|
|
value: 49.917839688145996 |
|
|
- type: manhattan_pearson |
|
|
value: 49.66820248148123 |
|
|
- type: manhattan_spearman |
|
|
value: 49.94013555794742 |
|
|
- type: euclidean_pearson |
|
|
value: 49.63608491973345 |
|
|
- type: euclidean_spearman |
|
|
value: 49.917839688145996 |
|
|
- type: main_score |
|
|
value: 49.917839688145996 |
|
|
task: |
|
|
type: STS |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB QBQTC |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/QBQTC |
|
|
metrics: |
|
|
- type: cosine_pearson |
|
|
value: 55.18355221701257 |
|
|
- type: cosine_spearman |
|
|
value: 54.67390932826382 |
|
|
- type: manhattan_pearson |
|
|
value: 53.32847494683504 |
|
|
- type: manhattan_spearman |
|
|
value: 54.61660160532041 |
|
|
- type: euclidean_pearson |
|
|
value: 53.405599174765364 |
|
|
- type: euclidean_spearman |
|
|
value: 54.67390932826382 |
|
|
- type: main_score |
|
|
value: 54.67390932826382 |
|
|
task: |
|
|
type: STS |
|
|
- dataset: |
|
|
config: zh |
|
|
name: MTEB STS22 (zh) |
|
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
|
split: test |
|
|
type: mteb/sts22-crosslingual-sts |
|
|
metrics: |
|
|
- type: cosine_pearson |
|
|
value: 67.89319522460808 |
|
|
- type: cosine_spearman |
|
|
value: 68.98524514928238 |
|
|
- type: manhattan_pearson |
|
|
value: 67.65257700660463 |
|
|
- type: manhattan_spearman |
|
|
value: 69.17199742136434 |
|
|
- type: euclidean_pearson |
|
|
value: 67.52535570217756 |
|
|
- type: euclidean_spearman |
|
|
value: 68.98524514928238 |
|
|
- type: main_score |
|
|
value: 68.98524514928238 |
|
|
task: |
|
|
type: STS |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB STSB |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/STSB |
|
|
metrics: |
|
|
- type: cosine_pearson |
|
|
value: 75.4871803618505 |
|
|
- type: cosine_spearman |
|
|
value: 76.17471665593993 |
|
|
- type: manhattan_pearson |
|
|
value: 75.73597640243183 |
|
|
- type: manhattan_spearman |
|
|
value: 76.20048941210949 |
|
|
- type: euclidean_pearson |
|
|
value: 75.66172628182565 |
|
|
- type: euclidean_spearman |
|
|
value: 76.17471665593993 |
|
|
- type: main_score |
|
|
value: 76.17471665593993 |
|
|
task: |
|
|
type: STS |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB T2Reranking |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/T2Reranking |
|
|
metrics: |
|
|
- type: map |
|
|
value: 67.45036855302303 |
|
|
- type: mrr |
|
|
value: 78.15107441080697 |
|
|
- type: main_score |
|
|
value: 67.45036855302303 |
|
|
task: |
|
|
type: Reranking |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB T2Retrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/T2Retrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 28.094 |
|
|
- type: map_at_10 |
|
|
value: 79.367 |
|
|
- type: map_at_100 |
|
|
value: 82.89800000000001 |
|
|
- type: map_at_1000 |
|
|
value: 82.953 |
|
|
- type: map_at_3 |
|
|
value: 55.782 |
|
|
- type: map_at_5 |
|
|
value: 68.667 |
|
|
- type: mrr_at_1 |
|
|
value: 91.237 |
|
|
- type: mrr_at_10 |
|
|
value: 93.399 |
|
|
- type: mrr_at_100 |
|
|
value: 93.479 |
|
|
- type: mrr_at_1000 |
|
|
value: 93.482 |
|
|
- type: mrr_at_3 |
|
|
value: 93.029 |
|
|
- type: mrr_at_5 |
|
|
value: 93.273 |
|
|
- type: ndcg_at_1 |
|
|
value: 91.237 |
|
|
- type: ndcg_at_10 |
|
|
value: 86.368 |
|
|
- type: ndcg_at_100 |
|
|
value: 89.637 |
|
|
- type: ndcg_at_1000 |
|
|
value: 90.16300000000001 |
|
|
- type: ndcg_at_3 |
|
|
value: 87.691 |
|
|
- type: ndcg_at_5 |
|
|
value: 86.462 |
|
|
- type: precision_at_1 |
|
|
value: 91.237 |
|
|
- type: precision_at_10 |
|
|
value: 42.841 |
|
|
- type: precision_at_100 |
|
|
value: 5.047 |
|
|
- type: precision_at_1000 |
|
|
value: 0.517 |
|
|
- type: precision_at_3 |
|
|
value: 76.708 |
|
|
- type: precision_at_5 |
|
|
value: 64.428 |
|
|
- type: recall_at_1 |
|
|
value: 28.094 |
|
|
- type: recall_at_10 |
|
|
value: 85.181 |
|
|
- type: recall_at_100 |
|
|
value: 95.953 |
|
|
- type: recall_at_1000 |
|
|
value: 98.63 |
|
|
- type: recall_at_3 |
|
|
value: 57.267999999999994 |
|
|
- type: recall_at_5 |
|
|
value: 71.75399999999999 |
|
|
- type: main_score |
|
|
value: 86.368 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB TNews |
|
|
revision: None |
|
|
split: validation |
|
|
type: C-MTEB/TNews-classification |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 55.482 |
|
|
- type: accuracy_stderr |
|
|
value: 0.3268577672321692 |
|
|
- type: f1 |
|
|
value: 53.57211848235611 |
|
|
- type: f1_stderr |
|
|
value: 0.3511138517262321 |
|
|
- type: main_score |
|
|
value: 55.482 |
|
|
task: |
|
|
type: Classification |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB ThuNewsClusteringP2P |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/ThuNewsClusteringP2P |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 79.44895384385426 |
|
|
- type: v_measure_std |
|
|
value: 2.315777338929376 |
|
|
- type: main_score |
|
|
value: 79.44895384385426 |
|
|
task: |
|
|
type: Clustering |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB ThuNewsClusteringS2S |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/ThuNewsClusteringS2S |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 76.95904984506356 |
|
|
- type: v_measure_std |
|
|
value: 2.244801218820472 |
|
|
- type: main_score |
|
|
value: 76.95904984506356 |
|
|
task: |
|
|
type: Clustering |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB VideoRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/VideoRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 65.60000000000001 |
|
|
- type: map_at_10 |
|
|
value: 75.24499999999999 |
|
|
- type: map_at_100 |
|
|
value: 75.51 |
|
|
- type: map_at_1000 |
|
|
value: 75.519 |
|
|
- type: map_at_3 |
|
|
value: 73.68299999999999 |
|
|
- type: map_at_5 |
|
|
value: 74.638 |
|
|
- type: mrr_at_1 |
|
|
value: 65.60000000000001 |
|
|
- type: mrr_at_10 |
|
|
value: 75.24499999999999 |
|
|
- type: mrr_at_100 |
|
|
value: 75.51 |
|
|
- type: mrr_at_1000 |
|
|
value: 75.519 |
|
|
- type: mrr_at_3 |
|
|
value: 73.68299999999999 |
|
|
- type: mrr_at_5 |
|
|
value: 74.638 |
|
|
- type: ndcg_at_1 |
|
|
value: 65.60000000000001 |
|
|
- type: ndcg_at_10 |
|
|
value: 79.338 |
|
|
- type: ndcg_at_100 |
|
|
value: 80.585 |
|
|
- type: ndcg_at_1000 |
|
|
value: 80.772 |
|
|
- type: ndcg_at_3 |
|
|
value: 76.189 |
|
|
- type: ndcg_at_5 |
|
|
value: 77.915 |
|
|
- type: precision_at_1 |
|
|
value: 65.60000000000001 |
|
|
- type: precision_at_10 |
|
|
value: 9.19 |
|
|
- type: precision_at_100 |
|
|
value: 0.976 |
|
|
- type: precision_at_1000 |
|
|
value: 0.099 |
|
|
- type: precision_at_3 |
|
|
value: 27.800000000000004 |
|
|
- type: precision_at_5 |
|
|
value: 17.52 |
|
|
- type: recall_at_1 |
|
|
value: 65.60000000000001 |
|
|
- type: recall_at_10 |
|
|
value: 91.9 |
|
|
- type: recall_at_100 |
|
|
value: 97.6 |
|
|
- type: recall_at_1000 |
|
|
value: 99.0 |
|
|
- type: recall_at_3 |
|
|
value: 83.39999999999999 |
|
|
- type: recall_at_5 |
|
|
value: 87.6 |
|
|
- type: main_score |
|
|
value: 79.338 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB Waimai |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/waimai-classification |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 89.9 |
|
|
- type: accuracy_stderr |
|
|
value: 0.7861297602813425 |
|
|
- type: ap |
|
|
value: 76.33068327298966 |
|
|
- type: ap_stderr |
|
|
value: 1.6404446239337744 |
|
|
- type: f1 |
|
|
value: 88.66175970131309 |
|
|
- type: f1_stderr |
|
|
value: 0.7269675835542363 |
|
|
- type: main_score |
|
|
value: 89.9 |
|
|
task: |
|
|
type: Classification |
|
|
tags: |
|
|
- mteb |
|
|
--- |
|
|
# quark-llm-embedding-1.5B |
|
|
|
|
|
- Chinese Text Embedding Model developed by Alibaba Quark-LLM Team. Details will be published later. |
