Model output submission.

app.py

@@ -19,14 +19,13 @@ from src.display.utils import (
     EVAL_COLS,
     EVAL_TYPES,
     AutoEvalColumn,
-    ModelType,
+    ModelTraining,
     fields,
-    WeightType,
-    Precision
+    MalteseTraining
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+from src.submission.submit import add_new_eval, read_configuration
 
 
 def restart_space():
@@ -71,8 +70,15 @@ def init_leaderboard(dataframe):
         search_columns=[AutoEvalColumn.model.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            ColumnFilter(AutoEvalColumn.model_training.name, type="checkboxgroup", label="Model types"),
             ColumnFilter(AutoEvalColumn.maltese_training.name, type="checkboxgroup", label="Maltese training"),
+            ColumnFilter(
+                AutoEvalColumn.language_count.name,
+                type="slider",
+                min=1,
+                max=1000,
+                label="Number of languages during training",
+            ),
             ColumnFilter(
                 AutoEvalColumn.params.name,
                 type="slider",
@@ -80,6 +86,8 @@ def init_leaderboard(dataframe):
                 max=150,
                 label="Select the number of parameters (B)",
             ),
+            ColumnFilter(AutoEvalColumn.prompt_version.name, type="checkboxgroup", label="Prompt Version"),
+            ColumnFilter(AutoEvalColumn.n_shot.name, type="slider", min=0, max=100, label="Number of Shots"),
             ColumnFilter(
                 AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
             ),
@@ -145,45 +153,71 @@ with demo:
                 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
 
             with gr.Row():
+                files = gr.File(
+                    label="Files (Configuration File & Prediction Outputs)",
+                    file_count="directory",
+                    type="filepath",
+                )
+
+            with gr.Row(equal_height=True):
                 with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.NK],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
+                    model_name = gr.Textbox(
+                        label="Model name",
+                        info="Read automatically from the results file.",
+                        interactive=False,
+                    )
+                    version = gr.Textbox(
+                        label="Prompt Version",
+                        info="Read automatically from the results file.",
+                        interactive=False,
+                    )
+                    n_shots = gr.Number(
+                        label="Number of Shots",
+                        info="Read automatically from the results file.",
+                        interactive=False,
                     )
 
                 with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
+                    model_training = gr.Dropdown(
+                        choices=[t.to_str(": ") for t in ModelTraining if t != ModelTraining.NK],
+                        label="Model Training",
+                        info="How to model is trained.",
                         multiselect=False,
-                        value="float16",
+                        value=None,
                         interactive=True,
                     )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
+                    maltese_training = gr.Dropdown(
+                        choices=[t.to_str(": ") for t in MalteseTraining if t != ModelTraining.NK],
+                        label="Maltese Training",
+                        info="The last stage of training in which Maltese was included.",
                         multiselect=False,
-                        value="Original",
+                        value=None,
+                        interactive=True,
+                    )
+                    language_count = gr.Number(
+                        label="Number of languages",
+                        info="Include languages for all training stages. Set to 0 if unknown.",
+                        minimum=0,
                         interactive=True,
                     )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
 
             submit_button = gr.Button("Submit Eval")
             submission_result = gr.Markdown()
+
+            configuration = gr.State()
+            file_paths = gr.State()
+            files.change(read_configuration,
+                         files,
+                         [configuration, file_paths, model_name, version, n_shots, submission_result])
+
             submit_button.click(
                 add_new_eval,
                 [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
+                    model_training,
+                    maltese_training,
+                    language_count,
+                    configuration,
+                    file_paths
                 ],
                 submission_result,
             )
@@ -201,4 +235,4 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch()

requirements.txt

@@ -9,6 +9,8 @@ huggingface-hub>=0.18.0
 matplotlib
 numpy
 pandas
+protobuf
+pydantic==2.10.6
 python-dateutil
 tqdm
 transformers

src/about.py

@@ -26,22 +26,22 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("sentiment", "f1,none", "Sentiment Analysis (F1)", TaskType.NLU)
-    task1 = Task("sib200", "f1,none", "SIB200 (F1)", TaskType.NLU)
-    task2 = Task("taxi1500", "f1,none", "Taxi1500 (F1)", TaskType.NLU)
-    task3 = Task("maltese_news_categories", "loglikelihood,none", "Maltese News Categories (F1)", TaskType.NLU)
-    task4 = Task("multi_eurlex", "loglikelihood,none", "MultiEURLEX (F1)", TaskType.NLU)
-    task5 = Task("belebele", "acc,none", "Belebele (Accuracy)", TaskType.NLU)
-    task6 = Task("opus100_en-mt", "bleu,none", "OPUS-100 EN→MT (BLEU)", TaskType.NLG, False)
-    task7 = Task("opus100_en-mt", "chrf,none", "OPUS-100 EN→MT (ChrF)", TaskType.NLG)
-    task8 = Task("flores200_en-mt", "bleu,none", "Flores-200 EN→MT (BLEU)", TaskType.NLG, False)
-    task9 = Task("flores200_en-mt", "chrf,none", "Flores-200 EN→MT (ChrF)", TaskType.NLG)
-    task10 = Task("webnlg", "chrf,none", "WebNLG (ChrF)", TaskType.NLG)
-    task11 = Task("webnlg", "rouge,none", "WebNLG (Rouge-L)", TaskType.NLG, False)
-    task12 = Task("eurlex_sum", "chrf,none", "EUR-Lex-Sum (ChrF)", TaskType.NLG, False)
-    task13 = Task("eurlex_sum", "rouge,none", "EUR-Lex-Sum (Rouge-L)", TaskType.NLG)
-    task14 = Task("maltese_news_headlines", "chrf,none", "Maltese News Headlines (ChrF)", TaskType.NLG, False)
-    task15 = Task("maltese_news_headlines", "rouge,none", "Maltese News Headlines (Rouge-L)", TaskType.NLG)
+    task0 = Task("sentiment_mlt", "f1", "Sentiment Analysis (F1)", TaskType.NLU)
+    task1 = Task("sib200_mlt", "f1", "SIB200 (F1)", TaskType.NLU)
+    task2 = Task("taxi1500_mlt", "f1", "Taxi1500 (F1)", TaskType.NLU)
+    task3 = Task("maltese_news_categories", "loglikelihood", "Maltese News Categories (F1)", TaskType.NLU)
+    task4 = Task("multieurlex_mlt", "loglikelihood", "MultiEURLEX (F1)", TaskType.NLU)
+    task5 = Task("belebele_mlt", "acc", "Belebele (Accuracy)", TaskType.NLU)
+    task6 = Task("opus100_eng-mlt", "bleu", "OPUS-100 EN→MT (BLEU)", TaskType.NLG, False)
+    task7 = Task("opus100_eng-mlt", "chrf", "OPUS-100 EN→MT (ChrF)", TaskType.NLG)
+    task8 = Task("flores200_eng-mlt", "bleu", "Flores-200 EN→MT (BLEU)", TaskType.NLG, False)
+    task9 = Task("flores200_eng-mlt", "chrf", "Flores-200 EN→MT (ChrF)", TaskType.NLG)
+    task10 = Task("webnlg_mlt", "chrf", "WebNLG (ChrF)", TaskType.NLG)
+    task11 = Task("webnlg_mlt", "rouge", "WebNLG (Rouge-L)", TaskType.NLG, False)
+    task12 = Task("eurlexsum_mlt", "chrf", "EUR-Lex-Sum (ChrF)", TaskType.NLG, False)
+    task13 = Task("eurlexsum_mlt", "rouge", "EUR-Lex-Sum (Rouge-L)", TaskType.NLG)
+    task14 = Task("maltese_news_headlines", "chrf", "Maltese News Headlines (ChrF)", TaskType.NLG, False)
+    task15 = Task("maltese_news_headlines", "rouge", "Maltese News Headlines (Rouge-L)", TaskType.NLG)
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
@@ -66,33 +66,9 @@ To reproduce our results, here is the commands you can run:
 """
 
 EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+To include new results on this benchmark, follow the instructions on our [GitHub Repository](https://github.com/MLRS/MELABench/tree/main/prompting).
+You can then upload the output files which should include the configuration/results file and all the prediction files.
+In addition, we ask for additional metadata about model training. 
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"

src/display/utils.py

@@ -26,6 +26,8 @@ auto_eval_column_dict = []
 # Init
 auto_eval_column_dict.append(["model_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["n_shot", ColumnContent, ColumnContent("N-Shot", "number", False)])
+auto_eval_column_dict.append(["prompt_version", ColumnContent, ColumnContent("Version", "str", False)])
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average (All) ⬆️", "number", True)])
 for task_type in TaskType:
@@ -33,9 +35,9 @@ for task_type in TaskType:
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", task.value.is_primary_metric)])
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["model_training", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["maltese_training", ColumnContent, ColumnContent("Maltese Training", "str", False)])
-auto_eval_column_dict.append(["num_languages", ColumnContent, ColumnContent("#Languages", "number", False)])
+auto_eval_column_dict.append(["language_count", ColumnContent, ColumnContent("#Languages", "number", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
 auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
@@ -53,9 +55,10 @@ AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
     precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
+    n_shot = ColumnContent("n_shot", "int", True)
+    prompt_version = ColumnContent("prompt_version", "str", True)
+    seed = ColumnContent("seed", "int", True)
     status = ColumnContent("status", "str", True)
 
 ## All the model information that we might need
@@ -66,7 +69,7 @@ class ModelDetails:
     symbol: str = "" # emoji
 
 
-class ModelType(Enum):
+class ModelTraining(Enum):
     PT = ModelDetails(name="pre-trained", symbol="PT")
     FT = ModelDetails(name="fine-tuned", symbol="FT")
     IT = ModelDetails(name="instruction-tuned", symbol="IT")
@@ -78,13 +81,13 @@ class ModelType(Enum):
     @staticmethod
     def from_str(type):
         type = type or ""
-        if type == "PT":
-            return ModelType.PT
-        if type == "FT":
-            return ModelType.FT
-        if type == "IT":
-            return ModelType.IT
-        return ModelType.NK
+        if "PT" in type:
+            return ModelTraining.PT
+        if "FT" in type:
+            return ModelTraining.FT
+        if "IT" in type:
+            return ModelTraining.IT
+        return ModelTraining.NK
 
 
 class MalteseTraining(Enum):
@@ -100,13 +103,13 @@ class MalteseTraining(Enum):
     @staticmethod
     def from_str(type):
         type = type or ""
-        if type == "NO":
+        if "NO" in type:
             return MalteseTraining.NO
-        if type == "PT":
+        if "PT" in type:
             return MalteseTraining.PT
-        if type == "FT":
+        if "FT" in type:
             return MalteseTraining.FT
-        if type == "IT":
+        if "IT" in type:
             return MalteseTraining.IT
         return MalteseTraining.NK
 

src/envs.py

@@ -4,17 +4,21 @@ from huggingface_hub import HfApi
 
 # Info to change for your repository
 # ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
 
-OWNER = "MLRS" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+PROMPT_VERSIONS = [version.strip() for version in
+                   os.environ.get("PROMPT_VERSIONS", "1.0_english,1.0_maltese").split(",")]
+
+OWNER = "MLRS"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/MELABench"
 QUEUE_REPO = f"{OWNER}/MELABench_requests"
+PREDICTIONS_REPO = f"{OWNER}/MELABench_predictions"
 RESULTS_REPO = f"{OWNER}/MELABench_results"
 
 # If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
+CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
 EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")

src/leaderboard/read_evals.py

@@ -3,12 +3,12 @@ import json
 import os
 from collections import defaultdict
 from dataclasses import dataclass
+from pathlib import Path
 
-import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, MalteseTraining
+from src.display.utils import AutoEvalColumn, ModelTraining, Tasks, Precision, WeightType, MalteseTraining
 from src.envs import TOKEN, API
 from src.submission.check_validity import is_model_on_hub, get_model_size
 
@@ -24,9 +24,12 @@ class EvalResult:
     revision: str # commit hash, "" if main
     results: dict
     precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.NK # Pretrained, fine tuned, ...
+    n_shot: int = 0
+    prompt_version: str = "1.0_english"
+    seed: int = 0
+    model_training: ModelTraining = ModelTraining.NK # Pretrained, fine tuned, ...
     maltese_training: MalteseTraining = MalteseTraining.NK # none, pre-training, ...
-    num_languages: int = None
+    language_count: int = None
     weight_type: WeightType = WeightType.Original # Original or Adapter
     architecture: str = "Unknown" 
     license: str = "?"
@@ -36,46 +39,39 @@ class EvalResult:
     still_on_hub: bool = False
 
     @classmethod
-    def init_from_json_file(self, json_filepath):
+    def init_from_json_files(self, seed_directory):
         """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
+        with open(list(seed_directory.values())[0][0]) as fp:
             data = json.load(fp)
 
         config = data.get("config")
-        metadata = data.get("metadata")
-
         precision = Precision.from_str(config.get("model_dtype"))
 
-        model_type = ModelType.from_str(metadata.get("model_type"))
+        n_shot = config.get("n_shot")
+
+        prompt_version = config.get("prompt_version")
+
+        seed = config.get("seed")
 
-        maltese_training = MalteseTraining.from_str(metadata.get("maltese_training"))
+        model_training = ModelTraining.from_str(config.get("model_training"))
 
-        num_languages = metadata.get("num_languages")
+        maltese_training = MalteseTraining.from_str(config.get("maltese_training"))
+
+        language_count = config.get("language_count")
 
         model_size = config.get("model_num_parameters")
 
         # Get model and org
-        org_and_model = config.get("model_name", None)
+        org_and_model = config.get("model", None)
         org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
         full_model = "/".join(org_and_model)
 
         revision = config.get("model_sha", config.get("model_revision", "main"))
 
-        model_args = {
-            **dict({tuple(arg.split("=")) for arg in config.get("model_args", "").split(",") if len(arg) > 0}),
-            "revision": revision,
-            "trust_remote_code": True,
-            "cache_dir": None
-        }
+        model_args = config.get("model_args")
+        model_args["revision"] = revision
+        model_args["trust_remote_code"] = True
+        model_args["cache_dir"] = None
         base_model = None
         if "pretrained" in model_args:
             base_model = model_args.pop("pretrained")
@@ -100,17 +96,30 @@ class EvalResult:
                 pass
 
         # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
+        results = defaultdict(dict)
+        for seed, file_paths in seed_directory.items():
+            for file_path in file_paths:
+                with open(file_path) as file:
+                    data = json.load(file)["results"]
+
+                for task in Tasks:
+                    task = task.value
+                    if task.benchmark not in data or task.metric not in data[task.benchmark]:
+                        continue
+                    score = data[task.benchmark][task.metric]
+                    if task.metric in ("accuracy", "f1", "loglikelihood", "rouge"):
+                        score *= 100
+                    results[task.benchmark + "_" + task.metric][seed] = score
+
+        results = {task: np.mean(list(seed_results.values())) for task, seed_results in results.items()}
 
-            mean_acc = np.mean(accs)
-            results[task.benchmark] = mean_acc
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+        result_key = f"{'_'.join(org_and_model)}_{revision}_{precision.value.name}_{n_shot}_{prompt_version}_{seed}"
 
         return self(
             eval_name=result_key,
@@ -118,15 +127,18 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            model_type=model_type,
+            model_training=model_training,
             maltese_training=maltese_training,
-            num_languages=num_languages or "?",
-            precision=precision,  
+            language_count=language_count or "?",
+            precision=precision,
             revision=revision,
+            n_shot=n_shot,
+            prompt_version=prompt_version,
+            seed=seed,
             still_on_hub=still_on_hub,
             architecture=architecture,
             likes=likes or "?",
-            num_params=round(model_size / 1e9, 3),
+            num_params=model_size and round(model_size / 1e9, 3),
             license=license,
         )
 
@@ -137,7 +149,7 @@ class EvalResult:
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.model_training = ModelTraining.from_str(request.get("model_training", ""))
             self.weight_type = WeightType[request.get("weight_type", "Original")]
             self.license = request.get("license", "?")
             self.likes = request.get("likes", 0)
@@ -152,10 +164,12 @@ class EvalResult:
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
+            AutoEvalColumn.n_shot.name: self.n_shot,
+            AutoEvalColumn.prompt_version.name: self.prompt_version,
+            AutoEvalColumn.model_training.name: self.model_training.value.name,
             AutoEvalColumn.maltese_training.name: self.maltese_training.value.name,
-            AutoEvalColumn.model_symbol.name: self.model_type.value.symbol + "/" + self.maltese_training.value.symbol,
-            AutoEvalColumn.num_languages.name: self.num_languages,
+            AutoEvalColumn.model_symbol.name: self.model_training.value.symbol + "/" + self.maltese_training.value.symbol,
+            AutoEvalColumn.language_count.name: self.language_count,
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
@@ -169,7 +183,7 @@ class EvalResult:
 
         results_by_task_type = defaultdict(list)
         for task in Tasks:
-            result = self.results[task.value.benchmark]
+            result = self.results.get(task.value.benchmark + "_" + task.value.metric)
             data_dict[task.value.col_name] = result
             if task.value.is_primary_metric:
                 results_by_task_type[task.value.task_type].append(result)
@@ -205,28 +219,19 @@ def get_request_file_for_model(requests_path, model_name, precision):
     return request_file
 
 
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+def get_raw_eval_results(results_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
+    model_result_filepaths = defaultdict(lambda: defaultdict(list))
 
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
+    for directory_path in Path(results_path).rglob("*-shot/*/*/"):
+        for file_path in directory_path.rglob("*-seed/results_*.json"):
+            seed = file_path.parent.name.removesuffix("-seed")
+            model_result_filepaths[directory_path.relative_to(results_path)][seed].append(file_path)
 
     eval_results = {}
-    for model_result_filepath in model_result_filepaths:
+    for model_result_filepath in model_result_filepaths.values():
         # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result = EvalResult.init_from_json_files(model_result_filepath)
 
         # Store results of same eval together
         eval_name = eval_result.eval_name

src/populate.py

@@ -1,5 +1,5 @@
 import json
-import os
+from pathlib import Path
 
 import pandas as pd
 
@@ -10,44 +10,28 @@ from src.leaderboard.read_evals import get_raw_eval_results
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
+    raw_data = get_raw_eval_results(results_path)
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[cols].round(decimals=2)
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
     return df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
     all_evals = []
 
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
+    for file_path in Path(save_path).rglob("requests_*.json"):
+        with open(file_path) as fp:
+            data = json.load(fp)["leaderboard"]
+
+        data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+        data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+        all_evals.append(data)
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
     running_list = [e for e in all_evals if e["status"] == "RUNNING"]

src/submission/check_validity.py

@@ -1,6 +1,5 @@
 import json
 import os
-from collections import defaultdict
 from typing import Any
 
 import huggingface_hub
@@ -75,11 +74,19 @@ def get_model_arch(model_info: ModelInfo):
     """Gets the model architecture from the configuration"""
     return model_info.config.get("architectures", "Unknown")
 
+def get_model_properties(configuration: dict) -> tuple[str, str, str, int, str, int]:
+    model_name = configuration["model_name_sanitized"]
+    revision = configuration["config"]["model_revision"]
+    precision = configuration["config"]["model_dtype"].split(".")[-1]
+    seed = configuration["config"]["random_seed"]
+    n_shot = list(configuration["n-shot"].values())[0]
+    prompt_version = list(configuration["versions"].values())[0]
+    return model_name, revision, precision, seed, prompt_version, n_shot
+
 def already_submitted_models(requested_models_dir: str) -> set[str]:
     """Gather a list of already submitted models to avoid duplicates"""
     depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
+    run_names = []
 
     for root, _, files in os.walk(requested_models_dir):
         current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
@@ -89,12 +96,8 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
 
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
+                    properties = get_model_properties(info)
+                    run_names.append("_".join([str(property) for property in properties]))
 
-    return set(file_names), users_to_submission_dates
+    return set(run_names)

src/submission/submit.py

@@ -1,117 +1,157 @@
 import json
 import os
+import re
 from datetime import datetime, timezone
+from pathlib import Path
 
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
+from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO, PROMPT_VERSIONS, PREDICTIONS_REPO
+from src.submission.check_validity import already_submitted_models, is_model_on_hub, get_model_properties
 
 REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
 
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
+def read_configuration(file_paths):
+    configuration_file_paths = list(filter(lambda file_path: file_path.name.endswith(".json"), file_paths or []))
+    if len(configuration_file_paths) != 1:
+        return None, None, None, None, None, styled_error(f"Expected exactly one configuration file but found {len(configuration_file_paths)}!")
 
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    configuration_file_path = file_paths.pop(file_paths.index(configuration_file_paths[0]))
 
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
+    try:
+        with open(configuration_file_path.name, "r", encoding="utf-8") as f:
+            data = json.load(f)
+    except Exception:
+        return None, None, None, None, None, styled_error("Failed to read configuration file!")
 
-    model_args = {}
+    try:
+        model_name = data["model_name"]
+        model_args = {
+            **dict({tuple(arg.split("=")) for arg in data["config"].get("model_args", "").split(",") if len(arg) > 0}),
+            "revision": data["config"]["model_revision"],
+            "trust_remote_code": True,
+            "cache_dir": None
+        }
+        base_model = model_args.pop("pretrained")
+        model_on_hub, error, _ = is_model_on_hub(model_name=base_model, model_args=model_args, token=TOKEN, test_tokenizer=True)
+        if not model_on_hub:
+            return None, None, model_name, None, None, styled_error(f"Model {model_name} {error}")
+
+        limit = data["config"]["limit"]
+        if limit is not None:
+            return None, None, model_name, None, None, styled_error(f"Only full results are accepted but found a specified limit of {limit}!")
+
+        prediction_files = {}
+        versions = {}
+        n_shots = {}
+        for task_name, _ in data["configs"].items():
+            sample_files = list(filter(lambda file_path: re.search(rf"samples_{task_name}_.*\.jsonl", file_path.name), file_paths))
+            if len(sample_files) == 0:
+                return None, None, model_name, None, None, styled_error(f"No prediction file found for configured task {task_name}!")
+
+            prediction_files[task_name] = str(file_paths.pop(file_paths.index(sample_files[0])))
+
+            versions[task_name] = data["versions"][task_name]
+            n_shots[task_name] = data["n-shot"][task_name]
+        if len(prediction_files) == 0:
+            return None, None, model_name, None, None, styled_error("No tasks found in configuration!")
+
+        versions = set(versions.values())
+        if len(versions) != 1:
+            return None, None, model_name, None, None, styled_error(f"All tasks should have the same version but found {versions}!")
+        version = list(versions)[0]
+        if version not in PROMPT_VERSIONS:
+            return None, None, model_name, None, None, styled_error(f"Unknown version {version}, should be one of {PROMPT_VERSIONS}!")
+
+        n_shots = set(n_shots.values())
+        if len(n_shots) != 1:
+            return None, None, model_name, version, None, styled_error(f"All tasks should have the same number of shots but found {n_shots}!")
+        n_shot = list(n_shots)[0]
+    except KeyError:
+        return None, None, model_name, None, None, styled_error("Wrong configuration file format!")
+
+    if len(file_paths) > 0:
+        ignored_files = [Path(file_path).name for file_path in file_paths]
+        return data, prediction_files, model_name, version, n_shot, styled_warning(f"The following files will be ignored: {ignored_files}")
+    return data, prediction_files, model_name, version, n_shot, styled_message("Files parsed successfully, verify that read metadata is correct before submitting")
 
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-    model_args["revision"] = revision
 
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, model_args=model_args, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
+def add_new_eval(
+    model_training: str,
+    maltese_training: str,
+    language_count: int,
+    configuration: dict,
+    prediction_files: dict[str, str],
+):
+    global REQUESTED_MODELS
+    if not REQUESTED_MODELS:
+        REQUESTED_MODELS = already_submitted_models(EVAL_REQUESTS_PATH)
 
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, model_args=model_args, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%f")
 
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
+    if configuration is None or configuration == {} or prediction_files is None or prediction_files == {}:
+        return styled_error("No files selected for upload, please upload an output folder (or wait for the files to finish uploading).")
 
-    model_size = get_model_size(model_info=model_info, precision=precision)
+    if model_training is None or model_training == "":
+        return styled_error("Please select the model's overall training.")
 
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
+    if maltese_training is None or maltese_training == "":
+        return styled_error("Please select the model's Maltese training.")
 
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
+    if language_count is None or language_count < 1:
+        language_count = None
+
+    model_name, revision, precision, seed, prompt_version, n_shot = get_model_properties(configuration)
+    model_id = configuration["model_name"]
 
     # Seems good, creating the eval
     print("Adding new eval")
 
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
+    # Check for duplicate submission
+    if f"{model_name}_{revision}_{precision}_{seed}_{prompt_version}_{n_shot}" in REQUESTED_MODELS:
+        return styled_warning("This model has been already submitted.")
+
+    request = {
+        "model": model_id,
+        "model_args": dict({tuple(arg.split("=")) for arg in configuration["config"].get("model_args", "").split(",") if len(arg) > 0}),
         "revision": revision,
         "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
+        "seed": seed,
+        "n_shot": n_shot,
+        "prompt_version": prompt_version,
+        "tasks": list(configuration["configs"].keys()),
+        "model_training": model_training,
+        "maltese_training": maltese_training,
+        "language_count": language_count,
         "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
+        "status": "PENDING",
     }
 
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    for task_name, file_path in prediction_files.items():
+        print(f"Uploading {model_id} {task_name} prediction file")
+        API.upload_file(
+            path_or_fileobj=file_path,
+            path_in_repo=f"{n_shot}-shot_{prompt_version}/{model_name}_{revision}_{precision}/{seed}-seed/samples_{task_name}_{current_time}.jsonl",
+            repo_id=PREDICTIONS_REPO,
+            repo_type="dataset",
+            commit_message=f"Add {configuration['model_name']} {task_name} {n_shot}-shot outputs",
+        )
+
+    print(f"Creating {model_id} configruation file")
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{model_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+    out_path = f"{OUT_DIR}/requests_{model_name}_{revision}_{precision}_{n_shot}shot_{prompt_version}_{seed}seed_{current_time}.json"
 
     with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
+        f.write(json.dumps({"leaderboard": request, "configuration": configuration}, ensure_ascii=False, indent=2))
 
-    print("Uploading eval file")
+    print(f"Uploading {model_id} configuration file")
     API.upload_file(
         path_or_fileobj=out_path,
         path_in_repo=out_path.split("eval-queue/")[1],
         repo_id=QUEUE_REPO,
         repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
+        commit_message=f"Add {configuration['model_name']} {n_shot}-shot to eval queue",
     )
 
     # Remove the local file