Update app.py

testing the gemini audio understanding

app.py

@@ -4,6 +4,7 @@ import pytz
 import time
 import wikipediaapi
 import yaml
+from google import genai
 from tools.final_answer import FinalAnswerTool
 
 import os
@@ -158,14 +159,25 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
     answers_payload = []
     print(f"Running agent on {len(questions_data)} questions...")
     for item in questions_data:
+        if !item.get("file_name"):
+            continue
+            
         task_id = item.get("task_id")
-        question_text = item.get("question") + " think hard to answer."
+        question_text = item.get("question") + " think hard to answer."            
         if not task_id or question_text is None:
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
+            # check if question_text contains an mp3 filename
+            if item.get("file_name") and ".mp3" in question_text:
+                client = genai.Client(api_key=os.environ.get("GEMINI_KEY"))
+                mp3_file = client.files.upload(file=f"/files/{item.get('task_id')}")
+                audio_description = client.models.generate_content(
+                    model="gemini-2.0-flash", contents=["Describe this audio clip", mp3_file]
+                )
+                question_text = f"{question_text} {audio_description.text}"
             # check if the file_name is not empty
-            if item.get("file_name"):
+            elif item.get("file_name"):
                 question_text = f"{question_text} Here is the file: https://agents-course-unit4-scoring.hf.space/files/{item.get('task_id')}"
             submitted_answer = agent(question_text)
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
@@ -180,55 +192,55 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
         print("Agent did not produce any answers to submit.")
         return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
 
-    # return "Questions parsed.", pd.DataFrame(results_log)
+    return "Questions parsed.", pd.DataFrame(results_log)
     
     # 4. Prepare Submission 
-    submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
-    status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
-    print(status_update)
+    # submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
+    # status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
+    # print(status_update)
 
     # 5. Submit
-    print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
-    try:
-        response = requests.post(submit_url, json=submission_data, timeout=60)
-        response.raise_for_status()
-        result_data = response.json()
-        final_status = (
-            f"Submission Successful!\n"
-            f"User: {result_data.get('username')}\n"
-            f"Overall Score: {result_data.get('score', 'N/A')}% "
-            f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
-            f"Message: {result_data.get('message', 'No message received.')}"
-        )
-        print("Submission successful.")
-        results_df = pd.DataFrame(results_log)
-        return final_status, results_df
-    except requests.exceptions.HTTPError as e:
-        error_detail = f"Server responded with status {e.response.status_code}."
-        try:
-            error_json = e.response.json()
-            error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
-        except requests.exceptions.JSONDecodeError:
-            error_detail += f" Response: {e.response.text[:500]}"
-        status_message = f"Submission Failed: {error_detail}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-    except requests.exceptions.Timeout:
-        status_message = "Submission Failed: The request timed out."
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-    except requests.exceptions.RequestException as e:
-        status_message = f"Submission Failed: Network error - {e}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-    except Exception as e:
-        status_message = f"An unexpected error occurred during submission: {e}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
+    # print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
+    # try:
+    #     response = requests.post(submit_url, json=submission_data, timeout=60)
+    #     response.raise_for_status()
+    #     result_data = response.json()
+    #     final_status = (
+    #         f"Submission Successful!\n"
+    #         f"User: {result_data.get('username')}\n"
+    #         f"Overall Score: {result_data.get('score', 'N/A')}% "
+    #         f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
+    #         f"Message: {result_data.get('message', 'No message received.')}"
+    #     )
+    #     print("Submission successful.")
+    #     results_df = pd.DataFrame(results_log)
+    #     return final_status, results_df
+    # except requests.exceptions.HTTPError as e:
+    #     error_detail = f"Server responded with status {e.response.status_code}."
+    #     try:
+    #         error_json = e.response.json()
+    #         error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
+    #     except requests.exceptions.JSONDecodeError:
+    #         error_detail += f" Response: {e.response.text[:500]}"
+    #     status_message = f"Submission Failed: {error_detail}"
+    #     print(status_message)
+    #     results_df = pd.DataFrame(results_log)
+    #     return status_message, results_df
+    # except requests.exceptions.Timeout:
+    #     status_message = "Submission Failed: The request timed out."
+    #     print(status_message)
+    #     results_df = pd.DataFrame(results_log)
+    #     return status_message, results_df
+    # except requests.exceptions.RequestException as e:
+    #     status_message = f"Submission Failed: Network error - {e}"
+    #     print(status_message)
+    #     results_df = pd.DataFrame(results_log)
+    #     return status_message, results_df
+    # except Exception as e:
+    #     status_message = f"An unexpected error occurred during submission: {e}"
+    #     print(status_message)
+    #     results_df = pd.DataFrame(results_log)
+    #     return status_message, results_df
 
 
 # --- Build Gradio Interface using Blocks ---