Hugging Face Hub check fix for PEFT models.

src/leaderboard/read_evals.py

@@ -70,8 +70,17 @@ class EvalResult:
 
         revision = config.get("model_sha", config.get("model_revision", "main"))
 
+        model_args = {
+            **dict({tuple(arg.split("=")) for arg in config.get("model_args", "").split(",") if len(arg) > 0}),
+            "revision": revision,
+            "trust_remote_code": True,
+            "cache_dir": None
+        }
+        base_model = None
+        if "pretrained" in model_args:
+            base_model = model_args.pop("pretrained")
         still_on_hub, _, model_config = is_model_on_hub(
-            full_model, revision, trust_remote_code=True, test_tokenizer=False
+            base_model or full_model, model_args, test_tokenizer=False, token=TOKEN,
         )
         architecture = "?"
         if model_config is not None:
@@ -82,7 +91,7 @@ class EvalResult:
         likes = 0
         if still_on_hub:
             try:
-                model_info = API.model_info(repo_id=full_model, revision=revision)
+                model_info = API.model_info(repo_id=full_model, revision=revision, token=TOKEN)
                 if not model_size:
                     model_size = get_model_size(model_info=model_info, precision=precision)
                 license = model_info.cardData.get("license")

src/submission/check_validity.py

@@ -31,13 +31,14 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
 
     return True, ""
 
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str, Any]:
+def is_model_on_hub(model_name: str, model_args: dict = None, token: str = None, test_tokenizer=False) -> tuple[bool, str, Any]:
     """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+    model_args = model_args or {}
     try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+        config = AutoConfig.from_pretrained(model_name, token=token, **model_args)
         if test_tokenizer:
             try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+                tk = AutoTokenizer.from_pretrained(model_name, token=token, **model_args)
             except ValueError as e:
                 return (
                     False,

src/submission/submit.py

@@ -39,18 +39,21 @@ def add_new_eval(
     if model_type is None or model_type == "":
         return styled_error("Please select a model type.")
 
+    model_args = {}
+
     # Does the model actually exist?
     if revision == "":
         revision = "main"
+    model_args["revision"] = revision
 
     # Is the model on the hub?
     if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, model_args=model_args, token=TOKEN, test_tokenizer=True)
         if not base_model_on_hub:
             return styled_error(f'Base model "{base_model}" {error}')
 
     if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+        model_on_hub, error, _ = is_model_on_hub(model_name=model, model_args=model_args, token=TOKEN, test_tokenizer=True)
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')