debug

src/about.py

@@ -44,15 +44,17 @@ class TasksMib_Subgraph(Enum):
     task1 = TaskMIB_Subgraph("mcqa", ["meta_llama", "qwen", "gpt2"], "mcqa", ["edge_counts", "faithfulness"])
 
 
-@dataclass 
-class TaskMIB_Causalgraph:
-    benchmark: str      # MCQA
-    models: list[str]   # List of all models
-    layers: list[str]   # 0-31
-    col_name: str       # display name in leaderboard
-    interventions: list[str]  # output_token, output_location
-    counterfactuals: list[str]  # symbol_counterfactual, etc.
-    metrics: list[str]  # score
+
+
+# @dataclass 
+# class TaskMIB_Causalgraph:
+#     benchmark: str      # MCQA
+#     models: list[str]   # List of all models
+#     layers: list[str]   # 0-31
+#     col_name: str       # display name in leaderboard
+#     interventions: list[str]  # output_token, output_location
+#     counterfactuals: list[str]  # symbol_counterfactual, etc.
+#     metrics: list[str]  # score
 
 # class TasksMib_Causalgraph(Enum):
 #     task0 = TaskMIB_Causalgraph(
@@ -77,11 +79,38 @@ class TaskMIB_Causalgraph:
 #          "answerPosition_randomLetter_counterfactual"],
 #         ["score"]
 #     )
+
+# class TasksMib_Causalgraph(Enum):
+#     task0 = TaskMIB_Causalgraph(
+#         "MCQA", 
+#         ["qwen2forcausallm", "gemma2forcausallm", "llamaforcausallm"],  # Use lowercase consistently
+#         [str(i) for i in range(32)],
+#         "mcqa",
+#         ["output_token", "output_location"],
+#         ["randomLetter_counterfactual", "answerPosition_counterfactual", 
+#          "answerPosition_randomLetter_counterfactual"],
+#         ["score"]
+#     )
+
+@dataclass 
+class TaskMIB_Causalgraph:
+    benchmark: str      
+    models: list[str]   
+    layers: dict[str, list[str]]  # Different layers for each model
+    col_name: str      
+    interventions: list[str]  
+    counterfactuals: list[str]  
+    metrics: list[str]  
+
 class TasksMib_Causalgraph(Enum):
     task0 = TaskMIB_Causalgraph(
         "MCQA", 
-        ["qwen2forcausallm", "gemma2forcausallm", "llamaforcausallm"],  # Use lowercase consistently
-        [str(i) for i in range(32)],
+        ["Qwen2ForCausalLM", "Gemma2ForCausalLM", "LlamaForCausalLM"],
+        {
+            "Qwen2ForCausalLM": [str(i) for i in range(24)],    # 0-23
+            "Gemma2ForCausalLM": [str(i) for i in range(26)],   # 0-25
+            "LlamaForCausalLM": [str(i) for i in range(32)]     # 0-31
+        },
         "mcqa",
         ["output_token", "output_location"],
         ["randomLetter_counterfactual", "answerPosition_counterfactual", 

src/display/utils.py

@@ -163,24 +163,45 @@ BENCHMARK_COLS_MIB_CAUSALGRAPH = []
 
 
 
+# auto_eval_column_dict_mib_causalgraph = []
+
+# # Method name column
+# auto_eval_column_dict_mib_causalgraph.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
+
+# # Add eval_name column
+# auto_eval_column_dict_mib_causalgraph.append(["eval_name", ColumnContent, ColumnContent("eval_name", "str", True)])
+
+# # For each model-task-intervention-counterfactual combination
+# for task in TasksMib_Causalgraph:
+#     for model in task.value.models:  # Use exact model names with correct casing
+#         model_name = model  # Don't convert to lowercase
+#         for layer in task.value.layers:
+#             for intervention in task.value.interventions:
+#                 for counterfactual in task.value.counterfactuals:
+#                     # Match exact format from the actual data
+#                     col_name = f"{model_name}_layer{layer}_{intervention}_{counterfactual}"
+#                     # Use the exact column name as both the field name and display name
+#                     auto_eval_column_dict_mib_causalgraph.append([
+#                         col_name, 
+#                         ColumnContent, 
+#                         ColumnContent(col_name, "number", True)
+#                     ])
+
+
+
 auto_eval_column_dict_mib_causalgraph = []
 
 # Method name column
 auto_eval_column_dict_mib_causalgraph.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
-
-# Add eval_name column
 auto_eval_column_dict_mib_causalgraph.append(["eval_name", ColumnContent, ColumnContent("eval_name", "str", True)])
 
 # For each model-task-intervention-counterfactual combination
 for task in TasksMib_Causalgraph:
-    for model in task.value.models:  # Use exact model names with correct casing
-        model_name = model  # Don't convert to lowercase
-        for layer in task.value.layers:
+    for model in task.value.models:
+        for layer in task.value.layers[model]:  # Use model-specific layers
             for intervention in task.value.interventions:
                 for counterfactual in task.value.counterfactuals:
-                    # Match exact format from the actual data
-                    col_name = f"{model_name}_layer{layer}_{intervention}_{counterfactual}"
-                    # Use the exact column name as both the field name and display name
+                    col_name = f"{model}_layer{layer}_{intervention}_{counterfactual}"
                     auto_eval_column_dict_mib_causalgraph.append([
                         col_name, 
                         ColumnContent,