Add Global MMLU benchmark

README.md

@@ -11,6 +11,7 @@ datasets:
 - openlanguagedata/flores_plus
 - google/fleurs
 - mozilla-foundation/common_voice_1_0
+- CohereForAI/Global-MMLU
 models:
 - meta-llama/Llama-3.3-70B-Instruct
 - mistralai/Mistral-Small-24B-Instruct-2501

evals/backend.py

@@ -22,7 +22,7 @@ def mean(lst):
     return sum(lst) / len(lst) if lst else None
 
 
-task_metrics = ["translation_from_bleu", "translation_to_bleu", "classification_accuracy"]
+task_metrics = ["translation_from_bleu", "translation_to_bleu", "classification_accuracy", "mmlu_accuracy"]
 
 
 def make_model_table(df, models):

evals/main.py

@@ -12,6 +12,8 @@ from tasks import tasks
 # ===== config =====
 
 n_sentences = 10
+n_languages = 3
+n_models = 3
 
 # ===== run evaluation and aggregate results =====
 
@@ -22,9 +24,8 @@ async def evaluate():
         task(model, lang.bcp_47, i)
         for task in tasks
         for i in range(n_sentences)
-        for lang in languages.iloc[:100].itertuples()
-        for model in models["id"]
-        if lang.in_benchmark
+        for lang in languages.iloc[:n_languages].itertuples()
+        for model in models["id"].iloc[:n_models]
     ]
     return await tqdm_asyncio.gather(*results, miniters=1)
 

evals/models.py

@@ -116,7 +116,7 @@ async def transcribe(path, model="elevenlabs/scribe_v1"):
             raise ValueError(f"Model {model} not supported")
 
 
-models = pd.DataFrame(models, columns=["id"]).iloc[:3]
+models = pd.DataFrame(models, columns=["id"])
 
 
 @cache

evals/tasks.py

@@ -8,6 +8,7 @@ from datasets_.flores import flores_sentences
 from joblib.memory import Memory
 from languages import languages, script_name
 from models import complete, transcribe
+from datasets import load_dataset
 
 cache = Memory(location=".cache", verbose=0).cache
 bleu = evaluate.load("bleu")
@@ -185,6 +186,45 @@ async def mlm_and_evaluate(model, language_bcp_47, nr):
         }
     ]
 
+@cache
+def _load_dataset(dataset, subset):
+    return load_dataset(dataset, subset)
+
+@cache
+async def mmlu_and_evaluate(model, language_bcp_47, nr):
+    data = _load_dataset("CohereForAI/Global-MMLU", language_bcp_47)
+    item = data["test"][nr]
+    def format_item(item):
+        return f"""{item['question']}
+        
+        A: {item['option_a']}
+        B: {item['option_b']}
+        C: {item['option_c']}
+        D: {item['option_d']}
+        
+        A|B|C|D?"""
+    messages = []
+    for example in data["dev"].select(range(5)):
+        messages += [{"role": "user", "content": format_item(example)}, {"role": "assistant", "content": example["answer"]}]
+    messages += [{"role": "user", "content": format_item(item)}]
+    reply = await complete(
+        model=model,
+        messages=messages,
+        temperature=0,
+        max_tokens=1,
+    )
+    print(reply.choices[0].message.content.strip())
+    acc = int(reply.choices[0].message.content.strip() == item["answer"])
+    return [
+        {
+            "model": model,
+            "bcp_47": language_bcp_47,
+            "task": "mmlu",
+            "metric": "accuracy",
+            "score": acc,
+            "sentence_nr": nr,
+        }
+    ]
 
 @cache
 async def transcribe_and_evaluate(model, language_bcp_47, nr):
@@ -217,11 +257,11 @@ async def transcribe_and_evaluate(model, language_bcp_47, nr):
         }
     ]
 
-
 tasks = [
     partial(translate_and_evaluate, mode="from"),
     partial(translate_and_evaluate, mode="to"),
     classify_and_evaluate,
     # mlm_and_evaluate,
+    mmlu_and_evaluate,
     # transcribe_and_evaluate,
 ]