debug

src/leaderboard/read_evals.py

@@ -259,6 +259,36 @@ class EvalResult_MIB_CAUSALGRAPH:
         
         return data_dict
 
+
+# def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str) -> List[EvalResult_MIB_CAUSALGRAPH]:
+#     """Extract evaluation results for MIB causalgraph"""
+#     model_result_filepaths = []
+    
+#     for root, dirnames, files in os.walk(results_path):
+#         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+#             continue
+        
+#         try:
+#             files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+#         except dateutil.parser._parser.ParserError:
+#             files = [files[-1]]
+
+#         for file in files:
+#             model_result_filepaths.append(os.path.join(root, file))
+
+#     eval_results = []
+#     for filepath in model_result_filepaths:
+#         try:
+#             eval_result = EvalResult_MIB_CAUSALGRAPH("", "", {})
+#             result = eval_result.init_from_json_file(filepath)
+#             result.to_dict()  # Verify conversion works
+#             eval_results.append(result)
+#         except Exception as e:
+#             print(f"Error processing {filepath}: {e}")
+#             continue
+
+#     return eval_results
+
 def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str) -> List[EvalResult_MIB_CAUSALGRAPH]:
     model_result_filepaths = []
     

src/populate.py

@@ -1,7 +1,7 @@
 import json
 import os
-
 import pandas as pd
+from typing import List, Dict, Tuple
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, AutoEvalColumnMultimodal, EvalQueueColumn
@@ -109,6 +109,26 @@ def create_intervention_averaged_df(df: pd.DataFrame) -> pd.DataFrame:
     
     return averaged_df
 
+# def get_leaderboard_df_mib_causalgraph(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+#     """Creates a dataframe from all the MIB causal graph experiment results"""
+#     print(f"results_path is {results_path}, requests_path is {requests_path}")
+#     raw_data = get_raw_eval_results_mib_causalgraph(results_path, requests_path)
+#     print(f"raw_data is {raw_data}")
+    
+#     # Convert each result to dict format for detailed df
+#     all_data_json = [v.to_dict() for v in raw_data]
+#     detailed_df = pd.DataFrame.from_records(all_data_json)
+#     print(f"detailed_df is: {detailed_df}")
+
+#     # Create and print other views for debugging/reference
+#     aggregated_df = aggregate_methods(detailed_df)
+#     print(f"aggregated_df is: {aggregated_df}")
+    
+#     intervention_averaged_df = create_intervention_averaged_df(aggregated_df)
+#     print(f"intervention_averaged_df is: {intervention_averaged_df}")
+    
+#     # Only return detailed_df for display
+#     return detailed_df
 
 def get_leaderboard_df_mib_causalgraph(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
     """Creates three dataframes from all the MIB causal graph experiment results"""