Updated call method in BasicAgent and system prompt

app.py

@@ -2,8 +2,9 @@ import os
 import gradio as gr
 import requests
 import pandas as pd
-from typing import List
+from typing import List, Dict, Any
 from dotenv import load_dotenv
+import json
 
 # LlamaIndex Imports
 from llama_index.core.llms import LLM
@@ -70,15 +71,15 @@ class BasicAgent:
     
     def _build_agent(self) -> ReActAgent:
         """Build and return the agent."""
-        # Load system prompt from file
+        # Load system prompt from file and append output format requirements
         try:
             with open("system_prompt.txt", "r", encoding="utf-8") as f:
                 system_prompt = f.read()
+            # Append output format to system prompt
+            system_prompt = f"{system_prompt}\n\nIMPORTANT OUTPUT FORMAT:\n{OUTPUT_FORMAT}"
         except Exception as e:
             print(f"Error loading system prompt: {e}")
-            system_prompt = """You are an intelligent agent designed to answer a wide variety of questions.
-You can use tools when necessary to look up information, perform calculations, or process special text formats.
-Always provide concise, accurate answers based on the question asked."""
+            system_prompt = f"You are an intelligent agent designed to answer a wide variety of questions.\n\nIMPORTANT OUTPUT FORMAT:\n{OUTPUT_FORMAT}"
         
         return ReActAgent.from_tools(
             tools=self.tools,
@@ -93,12 +94,31 @@ Always provide concise, accurate answers based on the question asked."""
         try:
             # Process the question
             response = self.agent.query(question)
-            answer = str(response)
-            print(f"Agent generated answer (first 50 chars): {answer[:50]}...")
-            return answer
+            answer_text = str(response)
+            
+            # Extract the FINAL ANSWER part if it exists
+            if "FINAL ANSWER:" in answer_text:
+                reasoning_trace = answer_text.split("FINAL ANSWER:")[0].strip()
+                model_answer = answer_text.split("FINAL ANSWER:")[1].strip()
+                
+                # Include the reasoning trace in the response but formatted for JSON
+                result = {
+                    "model_answer": model_answer,
+                    "reasoning_trace": reasoning_trace
+                }
+                
+                # Return just the answer part for direct evaluation
+                print(f"Agent generated answer: {model_answer[:50]}..." if len(model_answer) > 50 else f"Agent generated answer: {model_answer}")
+                return json.dumps(result)
+            else:
+                # If no FINAL ANSWER pattern, return the whole response
+                print(f"No 'FINAL ANSWER' found in response. Returning full response.")
+                return json.dumps({"model_answer": answer_text, "reasoning_trace": ""})
+                
         except Exception as e:
             print(f"Error generating answer: {e}")
-            return f"I encountered an error while answering your question: {str(e)}"
+            error_msg = f"I encountered an error while answering your question: {str(e)}"
+            return json.dumps({"model_answer": error_msg, "reasoning_trace": ""})
 
 def run_and_submit_all(profile: gr.OAuthProfile | None):
     """
@@ -153,6 +173,9 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
     # 3. Run your Agent
     results_log = []
     answers_payload = []
+    # Also create JSONL file for submission
+    jsonl_output = []
+    
     print(f"Running agent on {len(questions_data)} questions...")
     for item in questions_data:
         task_id = item.get("task_id")
@@ -161,17 +184,58 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
-            submitted_answer = agent(question_text)
+            # Get agent response which is now a JSON string
+            agent_response_json = agent(question_text)
+            agent_response = json.loads(agent_response_json)
+            
+            model_answer = agent_response.get("model_answer", "")
+            reasoning_trace = agent_response.get("reasoning_trace", "")
+            
+            # Format for submission payload
+            submitted_answer = model_answer
+            
+            # Add to answers payload
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
-            results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
+            
+            # Add to results log for display
+            results_log.append({
+                "Task ID": task_id, 
+                "Question": question_text, 
+                "Submitted Answer": submitted_answer,
+                "Reasoning": reasoning_trace[:100] + "..." if len(reasoning_trace) > 100 else reasoning_trace
+            })
+            
+            # Add to JSONL output
+            jsonl_output.append({
+                "task_id": task_id,
+                "model_answer": model_answer,
+                "reasoning_trace": reasoning_trace
+            })
+            
         except Exception as e:
              print(f"Error running agent on task {task_id}: {e}")
-             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
+             error_msg = f"AGENT ERROR: {e}"
+             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": error_msg})
+             answers_payload.append({"task_id": task_id, "submitted_answer": error_msg})
+             jsonl_output.append({
+                 "task_id": task_id,
+                 "model_answer": error_msg,
+                 "reasoning_trace": ""
+             })
 
     if not answers_payload:
         print("Agent did not produce any answers to submit.")
         return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
 
+    # Save JSONL output to file
+    try:
+        with open("submissions.jsonl", "w") as f:
+            for item in jsonl_output:
+                f.write(json.dumps(item) + "\n")
+        print("Saved submissions to submissions.jsonl")
+    except Exception as e:
+        print(f"Error saving submissions.jsonl: {e}")
+
     # 4. Prepare Submission 
     submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
     status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."

system_prompt.txt

@@ -26,6 +26,8 @@ When using tools:
 - For recent events or specialized topics, use web search
 - If a question is unclear, ask for clarification
 
-Always provide concise, accurate answers that directly address the question. Format your response according to any specific instructions in the question. If you're uncertain, state your confidence level and what additional information would help.
+Always provide concise, accurate answers that directly address the question. Format your response according to any specific instructions in the question.
+
+IMPORTANT: After providing your thoughts and reasoning about the question, ALWAYS end your response with "FINAL ANSWER: [your answer]" where your answer should be as concise as possible. If asked for a number, don't use commas or units. If asked for text, avoid articles and abbreviations unless specifically requested.
 
 Remember: The quality of your answers will be evaluated based on accuracy, relevance, and adherence to specified formats.