Merge branch 'main' into model_size_parameters

app.py

@@ -215,6 +215,17 @@ TASK_LIST_RETRIEVAL_FR = [
     "XPQARetrieval (fr)",
 ]
 
+TASK_LIST_RETRIEVAL_LAW = [
+    "AILACasedocs",
+    "AILAStatutes",
+    "GerDaLIRSmall",
+    "LeCaRDv2",
+    "LegalBenchConsumerContractsQA",
+    "LegalBenchCorporateLobbying",
+    "LegalQuAD",
+    "LegalSummarization",
+]
+
 TASK_LIST_RETRIEVAL_PL = [
     "ArguAna-PL",
     "DBPedia-PL",
@@ -324,6 +335,7 @@ def make_clickable_model(model_name, link=None):
 # Models without metadata, thus we cannot fetch their results naturally
 EXTERNAL_MODELS = [
     "Baichuan-text-embedding",
+    "Cohere-embed-english-v3.0",
     "Cohere-embed-multilingual-v3.0",
     "Cohere-embed-multilingual-light-v3.0",
     "DanskBERT",
@@ -342,6 +354,7 @@ EXTERNAL_MODELS = [
     "bert-base-swedish-cased",
     "bert-base-uncased",
     "bge-base-zh-v1.5",
+    "bge-large-en-v1.5",
     "bge-large-zh-v1.5",
     "bge-large-zh-noinstruct",
     "bge-small-zh-v1.5",
@@ -364,6 +377,8 @@ EXTERNAL_MODELS = [
     "gelectra-base",
     "gelectra-large",
     "glove.6B.300d",
+    "google-gecko.text-embedding-preview-0409",
+    "google-gecko-256.text-embedding-preview-0409",
     "gottbert-base",
     "gtr-t5-base",
     "gtr-t5-large",
@@ -434,6 +449,7 @@ EXTERNAL_MODELS = [
 ]
 
 EXTERNAL_MODEL_TO_LINK = {
+    "Cohere-embed-english-v3.0": "https://huggingface.co/Cohere/Cohere-embed-english-v3.0",
     "Cohere-embed-multilingual-v3.0": "https://huggingface.co/Cohere/Cohere-embed-multilingual-v3.0",
     "Cohere-embed-multilingual-light-v3.0": "https://huggingface.co/Cohere/Cohere-embed-multilingual-light-v3.0",
     "allenai-specter": "https://huggingface.co/sentence-transformers/allenai-specter",
@@ -450,6 +466,7 @@ EXTERNAL_MODEL_TO_LINK = {
     "bert-base-swedish-cased": "https://huggingface.co/KB/bert-base-swedish-cased",
     "bert-base-uncased": "https://huggingface.co/bert-base-uncased",
     "bge-base-zh-v1.5": "https://huggingface.co/BAAI/bge-base-zh-v1.5",
+    "bge-large-en-v1.5": "https://huggingface.co/BAAI/bge-large-en-v1.5",
     "bge-large-zh-v1.5": "https://huggingface.co/BAAI/bge-large-zh-v1.5",
     "bge-large-zh-noinstruct": "https://huggingface.co/BAAI/bge-large-zh-noinstruct",
     "bge-small-zh-v1.5": "https://huggingface.co/BAAI/bge-small-zh-v1.5",
@@ -480,6 +497,8 @@ EXTERNAL_MODEL_TO_LINK = {
     "gelectra-base": "https://huggingface.co/deepset/gelectra-base",
     "gelectra-large": "https://huggingface.co/deepset/gelectra-large",
     "glove.6B.300d": "https://huggingface.co/sentence-transformers/average_word_embeddings_glove.6B.300d",
+    "google-gecko.text-embedding-preview-0409": "https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings#latest_models",
+    "google-gecko-256.text-embedding-preview-0409": "https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings#latest_models",
     "gottbert-base": "https://huggingface.co/uklfr/gottbert-base",
     "gtr-t5-base": "https://huggingface.co/sentence-transformers/gtr-t5-base",
     "gtr-t5-large": "https://huggingface.co/sentence-transformers/gtr-t5-large",
@@ -553,6 +572,7 @@ EXTERNAL_MODEL_TO_LINK = {
 }
 
 EXTERNAL_MODEL_TO_DIM = {
+    "Cohere-embed-english-v3.0": 1024,
     "Cohere-embed-multilingual-v3.0": 1024,
     "Cohere-embed-multilingual-light-v3.0": 384,
     "all-MiniLM-L12-v2": 384,
@@ -568,6 +588,7 @@ EXTERNAL_MODEL_TO_DIM = {
     "bert-base-swedish-cased": 768,
     "bert-base-uncased": 768,
     "bge-base-zh-v1.5": 768,
+    "bge-large-en-v1.5": 1024,
     "bge-large-zh-v1.5": 1024,
     "bge-large-zh-noinstruct": 1024,
     "bge-small-zh-v1.5": 512,
@@ -601,6 +622,8 @@ EXTERNAL_MODEL_TO_DIM = {
     "gelectra-base": 768,
     "gelectra-large": 1024,
     "glove.6B.300d": 300,
+    "google-gecko.text-embedding-preview-0409": 768,
+    "google-gecko-256.text-embedding-preview-0409": 256,
     "gottbert-base": 768,    
     "gtr-t5-base": 768,
     "gtr-t5-large": 768,
@@ -671,6 +694,7 @@ EXTERNAL_MODEL_TO_DIM = {
 }
 
 EXTERNAL_MODEL_TO_SEQLEN = {
+    "Cohere-embed-english-v3.0": 512,
     "Cohere-embed-multilingual-v3.0": 512,
     "Cohere-embed-multilingual-light-v3.0": 512,    
     "all-MiniLM-L12-v2": 512,
@@ -686,6 +710,7 @@ EXTERNAL_MODEL_TO_SEQLEN = {
     "bert-base-swedish-cased": 512,
     "bert-base-uncased": 512,
     "bge-base-zh-v1.5": 512,
+    "bge-large-en-v1.5": 512,
     "bge-large-zh-v1.5": 512,
     "bge-large-zh-noinstruct": 512,
     "bge-small-zh-v1.5": 512,
@@ -715,6 +740,8 @@ EXTERNAL_MODEL_TO_SEQLEN = {
     "gbert-large": 512,
     "gelectra-base": 512,
     "gelectra-large": 512,
+    "google-gecko.text-embedding-preview-0409": 2048,
+    "google-gecko-256.text-embedding-preview-0409": 2048,
     "gottbert-base": 512,
     "glove.6B.300d": "N/A",
     "gtr-t5-base": 512,
@@ -904,6 +931,8 @@ PROPRIETARY_MODELS = {
     "voyage-code-2",
     "voyage-lite-01-instruct",
     "voyage-lite-02-instruct",
+    "google-gecko.text-embedding-preview-0409",
+    "google-gecko-256.text-embedding-preview-0409",
 }
 PROPRIETARY_MODELS = {
     make_clickable_model(model, link=EXTERNAL_MODEL_TO_LINK.get(model, "https://huggingface.co/spaces/mteb/leaderboard"))
@@ -1151,7 +1180,7 @@ def add_task(examples):
         examples["mteb_task"] = "PairClassification"
     elif examples["mteb_dataset_name"] in norm(TASK_LIST_RERANKING + TASK_LIST_RERANKING_FR + TASK_LIST_RERANKING_ZH):
         examples["mteb_task"] = "Reranking"
-    elif examples["mteb_dataset_name"] in norm(TASK_LIST_RETRIEVAL_NORM + TASK_LIST_RETRIEVAL_FR + TASK_LIST_RETRIEVAL_PL + TASK_LIST_RETRIEVAL_ZH):
+    elif examples["mteb_dataset_name"] in norm(TASK_LIST_RETRIEVAL_NORM + TASK_LIST_RETRIEVAL_FR + TASK_LIST_RETRIEVAL_PL + TASK_LIST_RETRIEVAL_ZH + TASK_LIST_RETRIEVAL_LAW):
         examples["mteb_task"] = "Retrieval"
     elif examples["mteb_dataset_name"] in norm(TASK_LIST_STS + TASK_LIST_STS_FR + TASK_LIST_STS_PL + TASK_LIST_STS_ZH):
         examples["mteb_task"] = "STS"
@@ -1569,6 +1598,7 @@ DATA_CLASSIFICATION_SV = get_mteb_data(["Classification"], [], TASK_LIST_CLASSIF
 DATA_CLASSIFICATION_OTHER = get_mteb_data(["Classification"], [], TASK_LIST_CLASSIFICATION_OTHER)[["Rank", "Model", "Model Size (Million Parameters)", "Average"] + TASK_LIST_CLASSIFICATION_OTHER]
 DATA_CLUSTERING_DE = get_mteb_data(["Clustering"], [], TASK_LIST_CLUSTERING_DE)[["Rank", "Model", "Model Size (Million Parameters)", "Average"] + TASK_LIST_CLUSTERING_DE]
 DATA_STS_OTHER = get_mteb_data(["STS"], [], TASK_LIST_STS_OTHER)[["Rank", "Model", "Model Size (Million Parameters)", "Average"] + TASK_LIST_STS_OTHER]
+DATA_RETRIEVAL_LAW = get_mteb_data(["Retrieval"], [], TASK_LIST_RETRIEVAL_LAW)[["Rank", "Model", "Model Size (Million Parameters)", "Average"] + TASK_LIST_RETRIEVAL_LAW]
 
 # Exact, add all non-nan integer values for every dataset
 NUM_SCORES = 0
@@ -1602,6 +1632,7 @@ for d in [
     DATA_RETRIEVAL_FR,
     DATA_RETRIEVAL_PL,
     DATA_RETRIEVAL_ZH,
+    DATA_RETRIEVAL_LAW,
     DATA_STS_EN,
     DATA_STS_FR,
     DATA_STS_PL,
@@ -1893,6 +1924,14 @@ data = {
                 "data": DATA_RETRIEVAL_FR,
                 "refresh": partial(get_mteb_data, tasks=["Retrieval"], datasets=TASK_LIST_RETRIEVAL_FR)
             },
+            {
+                "language": "Law",
+                "language_long": "English, German, Chinese",
+                "description": "**Retrieval Law Leaderboard** 🔎⚖️",
+                "credits": "[Voyage AI](https://www.voyageai.com/)",
+                "data": DATA_RETRIEVAL_LAW,
+                "refresh": partial(get_mteb_data, tasks=["Retrieval"], datasets=TASK_LIST_RETRIEVAL_LAW)
+            },
             {
                 "language": "Polish",
                 "description": "**Retrieval Polish Leaderboard** 🔎🇵🇱",