caulsal-track debug

src/leaderboard/read_evals.py

@@ -174,103 +174,100 @@ def get_raw_eval_results_mib_subgraph(results_path: str, requests_path: str) ->
 
 
 
-# def process_single_json(json_file: Dict[str, Any], method_counter: int) -> pd.DataFrame:
-#     """
-#     Process a single JSON file and convert it to a DataFrame.
+def process_single_json(json_file: Dict[str, Any], method_counter: int) -> pd.DataFrame:
+    """
+    Process a single JSON file and convert it to a DataFrame.
     
-#     Args:
-#         json_file: Dictionary containing the analysis results
-#         method_counter: Counter for handling duplicate method names
+    Args:
+        json_file: Dictionary containing the analysis results
+        method_counter: Counter for handling duplicate method names
         
-#     Returns:
-#         pd.DataFrame: DataFrame for single method with MODEL_TASK_INTERVENTION as columns
-#     """
-#     method_name = json_file['method_name']
-#     unique_method_name = f"{method_name}_{method_counter}"
-#     method_scores = []
-    
-#     for result in json_file['results']:
-#         model = result['model_id']
+    Returns:
+        pd.DataFrame: DataFrame for single method with MODEL_TASK_INTERVENTION as columns
+    """
+    method_name = json_file['method_name']
+    unique_method_name = f"{method_name}_{method_counter}"
+    method_scores = []
+    
+    for result in json_file['results']:
+        model = result['model_id']
         
-#         for task, scores in result['task_scores'].items():
-#             # Process each layer's data
-#             intervention_scores = defaultdict(list)
+        for task, scores in result['task_scores'].items():
+            # Process each layer's data
+            intervention_scores = defaultdict(list)
             
-#             for layer_data in scores:
-#                 for intervention_data in layer_data['layer_scores']:
-#                     # Calculate average score for counterfactuals
-#                     avg_cf_score = np.mean([
-#                         cf['score'] 
-#                         for cf in intervention_data['counterfactual_scores']
-#                     ])
+            for layer_data in scores:
+                for intervention_data in layer_data['layer_scores']:
+                    # Calculate average score for counterfactuals
+                    avg_cf_score = np.mean([
+                        cf['score'] 
+                        for cf in intervention_data['counterfactual_scores']
+                    ])
                     
-#                     if np.isnan(avg_cf_score):
-#                         avg_cf_score = 0.0
+                    if np.isnan(avg_cf_score):
+                        avg_cf_score = 0.0
                         
-#                     # Group scores by intervention
-#                     intervention_key = '_'.join(intervention_data['intervention'])
-#                     intervention_scores[intervention_key].append(avg_cf_score)
+                    # Group scores by intervention
+                    intervention_key = '_'.join(intervention_data['intervention'])
+                    intervention_scores[intervention_key].append(avg_cf_score)
             
-#             # Average across layers for each intervention
-#             for intervention, layer_scores in intervention_scores.items():
-#                 column = f"{model}_{task}_{intervention}"
-#                 avg_score = np.mean(layer_scores) if layer_scores else 0.0
-#                 method_scores.append((column, f"{avg_score:.3f}"))
-    
-#     # Sort by column names for consistency
-#     method_scores.sort(key=lambda x: x[0])
-#     data = {
-#         unique_method_name: {
-#             col: score for col, score in method_scores
-#         }
-#     }
-    
-#     return pd.DataFrame.from_dict(data, orient='index')
-
-# def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str) -> List[EvalResult_MIB_CAUSALGRAPH]:
-#     model_result_filepaths = []
-    
-#     # print(f"Scanning directory: {results_path}")
-#     for root, dirnames, files in os.walk(results_path):
-#         # print(f"Current directory: {root}")
-#         # print(f"Found files: {files}")
-#         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-#             continue
+            # Average across layers for each intervention
+            for intervention, layer_scores in intervention_scores.items():
+                column = f"{model}_{task}_{intervention}"
+                avg_score = np.mean(layer_scores) if layer_scores else 0.0
+                method_scores.append((column, f"{avg_score:.3f}"))
+    
+    # Sort by column names for consistency
+    method_scores.sort(key=lambda x: x[0])
+    data = {
+        unique_method_name: {
+            col: score for col, score in method_scores
+        }
+    }
+    
+    return pd.DataFrame.from_dict(data, orient='index')
+
+def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str) -> List[EvalResult_MIB_CAUSALGRAPH]:
+    model_result_filepaths = []
+    
+    # print(f"Scanning directory: {results_path}")
+    for root, dirnames, files in os.walk(results_path):
+        # print(f"Current directory: {root}")
+        # print(f"Found files: {files}")
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
         
-#         try:
-#             files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-#         except dateutil.parser._parser.ParserError:
-#             files = [files[-1]]
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
 
-#         for file in files:
-#             model_result_filepaths.append(os.path.join(root, file))
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
 
-#     # print(f"Found json files: {model_result_filepaths}")
+    # print(f"Found json files: {model_result_filepaths}")
 
-#     method_counters = defaultdict(int)
-#     dataframes = []
+    method_counters = defaultdict(int)
+    dataframes = []
     
-#     for json_file in model_result_filepaths:
-#         try:
-#             with open(filepath, 'r') as f:
-#                 json_data = json.load(f)
-#                 method_name = json_data['method_name']
-#                 method_counters[method_name] += 1
+    for json_file in model_result_filepaths:
+        try:
+            with open(filepath, 'r') as f:
+                json_data = json.load(f)
+                method_name = json_data['method_name']
+                method_counters[method_name] += 1
             
-#             # Process single JSON file
-#             df = process_single_json(json_data, method_counters[method_name])
-#             dataframes.append(df)
-#         except Exception as e:
-#             print(f"Error processing {json_file}: {e}")
-#             continue
-    
-#     # # Concatenate all DataFrames
-#     # if dataframes:
-#     #     final_df = pd.concat(dataframes, axis=0)
-#     #     return final_df
-#     # else:
-#     #     return pd.DataFrame()
-#     return dataframes
+            # Process single JSON file
+            df = process_single_json(json_data, method_counters[method_name])
+            dataframes.append(df)
+        except Exception as e:
+            print(f"Error processing {json_file}: {e}")
+            continue
+    
+    return dataframes
+
+
+
 
 from dataclasses import dataclass
 import json
@@ -285,7 +282,6 @@ from collections import defaultdict
 @dataclass
 class EvalResult_MIB_CAUSALGRAPH:
     """Represents one full evaluation for a method across all models in MIB for causal graph track."""
-    eval_name: str        # method name as identifier
     method_name: str      # name of the interpretation method
     results: Dict         # nested dict of results for each model and task
 
@@ -298,7 +294,7 @@ class EvalResult_MIB_CAUSALGRAPH:
         
         # Initialize results dictionary
         results = {}
-        for task in ["ioi", "mcqa", "arithmetic_addition", "arithmetic_subtraction", "arc_easy", "arc_challenge"]:
+        for task in ["MCQA"]:
             results[task] = {}
             
         # Process each model's results
@@ -330,7 +326,6 @@ class EvalResult_MIB_CAUSALGRAPH:
                 }
 
         return EvalResult_MIB_CAUSALGRAPH(
-            eval_name=method_name,
             method_name=method_name,
             results=results
         )
@@ -338,7 +333,6 @@ class EvalResult_MIB_CAUSALGRAPH:
     def to_dict(self, metric_type="average"):
         """Converts the Eval Result to a dict for dataframe display"""
         data_dict = {
-            "eval_name": self.eval_name,
             "Method": self.method_name,
         }
     
@@ -359,7 +353,7 @@ class EvalResult_MIB_CAUSALGRAPH:
                 data_dict[col_name] = f"{avg_score:.3f}"
                 all_scores.append(avg_score)
     
-        data_dict["Average"] = f"{np.mean(all_scores):.3f}" if all_scores else '-'
+        data_dict["Average"] = f"{np.mean(all_scores):.3f}"
         return data_dict
 
 
@@ -480,8 +474,8 @@ def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str)
     # Create the detailed DataFrame
     detailed_df = pd.DataFrame(data_dicts)
     detailed_df.set_index("Method", inplace=True)
-    if "eval_name" in detailed_df.columns:
-        detailed_df.drop("eval_name", axis=1, inplace=True)
+    # if "eval_name" in detailed_df.columns:
+    #     detailed_df.drop("eval_name", axis=1, inplace=True)
 
     # Create aggregated DataFrame
     aggregated_df = aggregate_methods(detailed_df)