Addressing the need to update the requests dataset to COMPLETED, FAILED, etc. based on model *and task*.

process_runs.py

@@ -76,32 +76,32 @@ def update_requests(requests, all_attempts, failed_attempts):
     # run_dir="/runs/${experiment_name}/${backend_model}/${now}", where
     # ${backend_model} is ${organization}/${model_name}
     for line in all_attempts:
+        line = line.strip()
         print(f"Checking {line}")
         split_run_dir = line.strip().strip("/").split("/")
         print(f"Processing run directory {split_run_dir}")
         task = split_run_dir[1]
         print(f"Task is {task}")
+        # The naming of the optimum benchmark configs uses an underscore.
+        # The naming of the HF Api list models function uses a hyphen.
+        # We therefore need to adapt this task string name depending on
+        # which part of our pipeline we're talking to.
+        hyphenated_task_name = "-".join(task.split("_"))
         model = "/".join([split_run_dir[2], split_run_dir[3]])
-        print(f"Model is is {model}")
+        print(f"Model is {model}")
         traceback_error = check_for_traceback(line)
         if traceback_error != "":
             print("Found a traceback error!")
             print(traceback_error)
-            requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == task), ['status']] = FAILED
-            requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == task), ['error_message']] = traceback_error
+            requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == hyphenated_task_name), ['status']] = FAILED
+            requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == hyphenated_task_name), ['error_message']] = traceback_error
         elif line in failed_attempts:
             print(f"Job failed, but not sure why -- didn't find a traceback in {line}.")
-            print(f"Setting {model}, {task}, status {PENDING} to {FAILED}.")
-            print("DEBUG: PENDING status dataframe:")
-            print(requests_df[(requests_df["status"] == PENDING)])
-            print("DEBUG: model dataframe:")
-            print(requests_df[(requests_df["model"] == model)])
-            print("DEBUG: task dataframe:")
-            print(requests_df[(requests_df["task"] == task)])
-            print(requests_df[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == task)])
-            requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == task), ['status']] = FAILED
+            print(f"Setting {model}, {hyphenated_task_name}, status {PENDING} to {FAILED}.")
+            print(requests_df[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == hyphenated_task_name)])
+            requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == hyphenated_task_name), ['status']] = FAILED
         else:
-            requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == task), ['status']] = COMPLETED
+            requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == hyphenated_task_name), ['status']] = COMPLETED
     updated_dset = Dataset.from_pandas(requests_df)
     return updated_dset
 
@@ -116,7 +116,6 @@ if __name__ == '__main__':
     )
     # Update requests dataset based on whether things have failed or not.
     print(f"Examining the run directory for each model & task to determine if it {FAILED} or {COMPLETED}.")
-    print(f"Setting the corresponding line in {REQUESTS_DSET} to {FAILED} or {COMPLETED} based on what's in the directory.")
     requests = load_dataset(f"{REQUESTS_DSET}", split="test", token=TOKEN)
     all_attempts = open(f"{args.attempts}", "r+").readlines()
     failed_attempts = open(f"{args.failed_attempts}", "r+").readlines()