sync wi FIDLE

src/about.py

@@ -12,8 +12,12 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    task0 = Task("community|ifeval-fr|0", "norm_acc", "IFEval-Fr")
+    task1 = Task("community|pr-fouras|0", "pr-fouras-qem", "Pr-Fouras")
+    task2 = Task("community|kangourou-to|0", "norm_acc", "Kangourou-TO")
+    task3 = Task("community|gpqa-fr|0", "norm_acc", "GPQA-Fr")
+    task4 = Task("community|bac-fr|0", "bac-fr-qem", "Bac-Fr")
+    task5 = Task("community|sornette|0", "norm_acc", "Sornette")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
@@ -21,11 +25,29 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">
+ <img src="https://www.deepmama.com/images/fideval.png" alt="FIDLE Evaluator" width="100%">
+ </h1> 
+"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+-------------------------
+# FIDLE LLM-FR Leaderboard 🏆
+
+This is a leaderboard exclusively **in French**. We do not intend to become a reference for LLM evaluations. This is for informational and educational purposes only. Please cross-reference with other, more official leaderboards.
+
+**Note: The assessments have been adapted to the Reasoning Language Model**: all *tasks* are in generative mode, with no limit on token generation.
+* **IFEval-Fr** : French Translation of [IFEval](https://huggingface.co/datasets/google/IFEval)
+* **Pr-Fouras** : "Père Fouras"'s Riddles (ex : [fan site](https://www.fan-fortboyard.fr/pages/fanzone/enigmes-du-pere-fouras/))
+* **Sornette** : Classification of texts (GORAFI, wikipedia, le saviez-vous, ...) into 4 categories - `burlesque et fantaisiste`, `ludique et didactique`, `insidieux et mensonger`, `moral et accablant`
+* **Kangourou-TO** : MATH Quizzes [Kangourou](www.mathkang.org). *Text Only* : Only questions without figures.
+
+**Model Types**:
+* 🪨 - Base, Pretrained, Foundation Model
+* 💬 - Chat Model (Instruct, RLHF, DPO, ...)
+* 💅🏻 - Fine-tuned Model
+* 🤔 - Reasoning Model
 """
 
 # Which evaluations are you running? how can people reproduce what you have?

src/display/utils.py

@@ -23,6 +23,7 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
+auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent("R", "number", True, never_hidden=True)])
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
@@ -30,15 +31,15 @@ auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+#auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+#auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+#auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", True)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+#auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+#auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -62,10 +63,10 @@ class ModelDetails:
 
 
 class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    PT = ModelDetails(name="pretrained", symbol="🪨")
+    FT = ModelDetails(name="fine-tuned", symbol="💅🏻")
+    IFT = ModelDetails(name="instruction-tuned", symbol="💬")
+    RL = ModelDetails(name="RL-tuned", symbol="🤔")
     Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
@@ -73,13 +74,13 @@ class ModelType(Enum):
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
+        if "fine-tuned" in type or "💅🏻" in type:
             return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
+        if "pretrained" in type or "🪨" in type:
             return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
+        if "RL-tuned" in type or "🤔" in type:
             return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
+        if "instruction-tuned" in type or "💬" in type:
             return ModelType.IFT
         return ModelType.Unknown
 

src/envs.py

@@ -6,12 +6,12 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "FIDLE-CNRS" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+REPO_ID = f"{OWNER}/LLM-FR_leaderboard"
+QUEUE_REPO = f"{OWNER}/LLM-FR_requests"
+RESULTS_REPO = f"{OWNER}/LLM-FR_results"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")

src/leaderboard/read_evals.py

@@ -31,6 +31,7 @@ class EvalResult:
     num_params: int = 0
     date: str = "" # submission date of request file
     still_on_hub: bool = False
+    rank: int = 0
 
     @classmethod
     def init_from_json_file(self, json_filepath):
@@ -38,7 +39,7 @@ class EvalResult:
         with open(json_filepath) as fp:
             data = json.load(fp)
 
-        config = data.get("config")
+        config = data.get("config_general")
 
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))
@@ -109,21 +110,24 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        # weighted average calculation
+        task_weights = [0.25, 0.25, 0.25, 0.09, 0.09, 0.07]
+        average = sum(np.array([v for v in self.results.values() if v is not None]) * np.array(task_weights)) / sum(task_weights)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
+            #AutoEvalColumn.model_type.name: self.model_type.value.name,
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
+            #AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            #AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
+            #AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            #AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            AutoEvalColumn.rank.name: self.rank,
         }
 
         for task in Tasks:
@@ -133,7 +137,7 @@ class EvalResult:
 
 
 def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+    """Selects the correct request file for a given model."""
     request_files = os.path.join(
         requests_path,
         f"{model_name}_eval_request_*.json",

src/populate.py

@@ -14,11 +14,17 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False).reset_index(drop=True)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
+
+    # add rank column
+    df[AutoEvalColumn.rank.name] = df.index + 1
+    df.loc[0, AutoEvalColumn.rank.name] = '1 🥇'
+    df.loc[1, AutoEvalColumn.rank.name] = '2 🥈'
+    df.loc[2, AutoEvalColumn.rank.name] = '3 🥉'
     return df
 
 

src/submission/check_validity.py

@@ -31,7 +31,7 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
 
     return True, ""
 
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=True, test_tokenizer=False) -> tuple[bool, str]:
     """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
     try:
         config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)

src/submission/submit.py

@@ -81,7 +81,7 @@ def add_new_eval(
         "revision": revision,
         "precision": precision,
         "weight_type": weight_type,
-        "status": "PENDING",
+        "status": "FINISHED",
         "submitted_time": current_time,
         "model_type": model_type,
         "likes": model_info.likes,