Update run_eval.py

run_eval.py

@@ -34,10 +34,21 @@ login(token)
 DATASET_REPO = os.environ["HF_DATASET_REPO"]
 api = HfApi()
 
-# ───── Evaluate each adapter ─────
-all_rows = []
 METRICS_TO_KEEP = {"acc", "accuracy", "acc_stderr", "f1", "exact_match"}
+all_rows = []
+
+# ───── Safe tokenizer loading ─────
+def load_tokenizer(model_id: str):
+    try:
+        return AutoTokenizer.from_pretrained(model_id, use_fast=True)
+    except Exception as e1:
+        print(f"Fast tokenizer failed for {model_id}: {e1}")
+        try:
+            return AutoTokenizer.from_pretrained(model_id, use_fast=False)
+        except Exception as e2:
+            raise RuntimeError(f"Failed to load tokenizer for {model_id}: {e2}") from e2
 
+# ───── Evaluate each adapter ─────
 for cfg in CONFIGS:
     base_model_id = cfg["base_model"]
     adapter_repo = cfg["adapter_repo"]
@@ -45,34 +56,49 @@ for cfg in CONFIGS:
     tasks = cfg["tasks"]
 
     print(f"\nLoading base model: {base_model_id}")
-    tokenizer = AutoTokenizer.from_pretrained(base_model_id, use_fast=False)
+    tokenizer = load_tokenizer(base_model_id)
+
+    if "llama" in base_model_id.lower():
+        try:
+            tokenizer.legacy = False 
+        except:
+            pass
 
-    # Try causal first, fallback to encoder
     try:
         base_model = AutoModelForCausalLM.from_pretrained(
             base_model_id,
             trust_remote_code=True,
             use_safetensors=True
-        )        
+        )
         is_encoder = False
-    except:
+        print("Loaded as Causal LM")
+    except Exception as e:
+        print(f"⚠️ Failed to load causal LM: {e}")
         base_model = AutoModelForSequenceClassification.from_pretrained(
             base_model_id,
             trust_remote_code=True,
             use_safetensors=True
         )
         is_encoder = True
+        print("Loaded as Sequence Classification model")
+
     try:
         info = model_info(adapter_repo)
         files = [f.rfilename for f in info.siblings]
         if "adapter_config.json" not in files:
             print(f"{adapter_repo} is not a valid PEFT adapter (missing adapter_config.json)")
-            continue  # skip
+            continue
     except Exception as e:
-        print(f"Failed to inspect {adapter_repo}: {e}")
+        print(f"Failed to inspect adapter {adapter_repo}: {e}")
         continue
-    peft_model = PeftModel.from_pretrained(base_model, adapter_repo)
-    merged_model = peft_model.merge_and_unload()
+
+    try:
+        peft_model = PeftModel.from_pretrained(base_model, adapter_repo)
+        merged_model = peft_model.merge_and_unload()
+    except Exception as e:
+        print(f"Failed to apply adapter {adapter_repo}: {e}")
+        continue
+
     device = "cuda" if torch.cuda.is_available() else "cpu"
     merged_model.to(device)
     merged_model.eval()
@@ -81,12 +107,22 @@ for cfg in CONFIGS:
         merged_model.save_pretrained(td)
         tokenizer.save_pretrained(td)
 
+        # Verify tokenizer object
+        if not hasattr(tokenizer, "vocab_size"):
+            print("Invalid tokenizer loaded. Skipping.")
+            continue
+
         hf_lm = HFLM(
             pretrained=td,
             batch_size=8 if not is_encoder else 16,
             device=device,
         )
-        res = evaluator.simple_evaluate(model=hf_lm, tasks=tasks)
+
+        try:
+            res = evaluator.simple_evaluate(model=hf_lm, tasks=tasks)
+        except Exception as e:
+            print(f"Evaluation failed for {adapter_repo}: {e}")
+            continue
 
     meta = {
         "model_id": adapter_repo,
@@ -117,15 +153,12 @@ with tempfile.TemporaryDirectory() as tmp:
     )
     df_existing = pd.read_parquet(current_path)
     df_combined = pd.concat([df_existing, df_new], ignore_index=True)
-
     df_combined = df_combined.sort_values("run_date")
-
-
     df_combined["value"] = pd.to_numeric(df_combined["value"], errors="coerce")
 
     print("Existing rows:", len(df_existing))
     print("New rows:", len(df_new))
-    print("Combined rows (pre-dedup):", len(df_existing) + len(df_new))
+    print("Combined (pre-dedup):", len(df_existing) + len(df_new))
     print("Final rows (after dedup):", len(df_combined))
 
     out = Path("peft_bench.parquet")