debug

src/leaderboard/read_evals.py

@@ -188,7 +188,7 @@ class EvalResult_MIB_CAUSALGRAPH:
     eval_name: str        
     method_name: str      
     results: Dict
-    
+
     def init_from_json_file(self, json_filepath):
         """Inits results from the method result file"""
         with open(json_filepath) as fp:
@@ -229,30 +229,60 @@ class EvalResult_MIB_CAUSALGRAPH:
             results=results
         )
 
-    def to_dict(self):
-        """Converts the Eval Result to a dict for dataframe display"""
-        data_dict = {
-            "eval_name": self.eval_name,
-            "Method": self.method_name,
-        }
+    # def to_dict(self):
+    #     """Converts the Eval Result to a dict for dataframe display"""
+    #     data_dict = {
+    #         "eval_name": self.eval_name,
+    #         "Method": self.method_name,
+    #     }
         
-        # Process each model's results
-        for model_id, model_results in self.results.items():
-            for task, task_scores in model_results.items():
-                # Calculate layer-averaged scores for each intervention
-                intervention_scores = defaultdict(list)
+    #     # Process each model's results
+    #     for model_id, model_results in self.results.items():
+    #         for task, task_scores in model_results.items():
+    #             # Calculate layer-averaged scores for each intervention
+    #             intervention_scores = defaultdict(list)
                 
-                for layer_data in task_scores:
-                    for score_data in layer_data['scores']:
-                        intervention = score_data['intervention']
-                        intervention_scores[intervention].append(score_data['score'])
+    #             for layer_data in task_scores:
+    #                 for score_data in layer_data['scores']:
+    #                     intervention = score_data['intervention']
+    #                     intervention_scores[intervention].append(score_data['score'])
                 
-                # Average across layers for each intervention
-                for intervention, scores in intervention_scores.items():
-                    col_name = f"{model_id}_{task}_{intervention}".lower()
-                    data_dict[col_name] = round(np.mean(scores), 3)
+    #             # Average across layers for each intervention
+    #             for intervention, scores in intervention_scores.items():
+    #                 col_name = f"{model_id}_{task}_{intervention}".lower()
+    #                 data_dict[col_name] = round(np.mean(scores), 3)
         
-        return data_dict
+    #     return data_dict
+    def to_dict(self):
+    """Converts the Eval Result to a dict for dataframe display"""
+    data_dict = {
+        "eval_name": self.eval_name,
+        "Method": self.method_name,
+    }
+        
+    # Process each model's results
+    for model_id, task_scores in self.results.items():
+        model_name = model_id.lower()  # Lowercase for consistency
+        
+        # Each task_scores contains layer data
+        for layer_data in task_scores.get("MCQA", []):
+            layer = layer_data.get("layer")
+            layer_scores = layer_data.get("layer_scores", [])
+            
+            # Process each intervention and counterfactual
+            for intervention_data in layer_scores:
+                intervention = intervention_data["intervention"][0]
+                counterfactual_scores = intervention_data["counterfactual_scores"]
+                
+                for cf_score in counterfactual_scores:
+                    counterfactual = cf_score["counterfactual"][0]
+                    score = cf_score["score"]
+                    
+                    # Column name matches what we defined in utils.py
+                    col_name = f"{model_name}_layer{layer}_{intervention}_{counterfactual}".lower()
+                    data_dict[col_name] = score
+
+    return data_dict
 
 
 # def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str) -> List[EvalResult_MIB_CAUSALGRAPH]: