Added a user-specific answer cache

evaluator.py

@@ -57,14 +57,11 @@ class Evaluator():
             questions = self.get_questions()
             return questions[random.randint(0, len(questions)-1)]
 
-    def _read_answer_file(self, file_prefix) -> List[str]:
-        
-        logger.info(f"File prefix: {file_prefix}")
-        
-        with open("answers.json", "r") as f:
+    def _read_answer_file(self, username) -> List[str]:
+        """Read the question answer pairs from a user-specific answer file."""
+        with open(f"answers_{username}.json", "r") as f:
             pairs = [QuestionAnswerPair(**pair) for pair in json.load(f)]
             formatted_data = [pair.get_answer() for pair in pairs]
-        # Return count and the formatted data
         return formatted_data
 
     def submit_answers(self, username: str) -> str:
@@ -73,6 +70,7 @@ class Evaluator():
             answers_payload = self._read_answer_file({username})
         except FileNotFoundError:
             return "Click 'Get One Answer' or 'Get All Answers' to run before trying to submit."
+        
         agent_code = f"https://huggingface.co/spaces/{self.settings.space_id}/tree/main"
         submission_data = {
             "username": self.settings.username,

runner.py

@@ -1,5 +1,5 @@
 from settings import Settings
-from models import QuestionAnswerPair
+from models import Question, QuestionAnswerPair
 from agent import BasicAgent
 import pandas as pd
 import logging
@@ -14,9 +14,10 @@ class Runner():
     def __init__(self, settings: Settings):
         self.settings = settings
 
-    def _save_pairs(self, pairs: list[QuestionAnswerPair]):
+    def _save_pairs(self, pairs: list[QuestionAnswerPair], username: str):
+        """Write the question answer pairs to a user-specific file."""
         answers = [pair.model_dump() for pair in pairs if pair is not None]
-        with open("answers.json", "w") as f:
+        with open(f"answers_{username}.json", "w") as f:
             json.dump(answers, f, indent=4)
 
     def _enrich_question_text(self, item):
@@ -33,7 +34,7 @@ class Runner():
             question_text = f"{question_text} file_name: {file_name} (use tools to fetch the file)"
         return question_text
 
-    async def _run_agent_async(self, item):
+    async def _run_agent_async(self, item: Question):
         """Runs the agent asynchronously."""
         task_id = item.task_id
         question_text = self._enrich_question_text(item)
@@ -45,12 +46,12 @@ class Runner():
         return QuestionAnswerPair(task_id=task_id,
                                   question=item.question, answer=str(answer))
 
-    def _assign_questions(self, questions):
+    def _assign_questions(self, questions: list[Question]):
         """Runs the asynchronous loop and returns task outputs."""
         tasks = [self._run_agent_async(item) for item in questions]
         return asyncio.gather(*tasks)
 
-    def run_agent(self, questions) -> pd.DataFrame:
+    def run_agent(self, questions: list[Question], username: str) -> pd.DataFrame:
         """Run the agent(s) async, save answers and return a dataframe"""
         # Assign questions to agents and wait
         try:
@@ -67,7 +68,7 @@ class Runner():
         pairs = run_tasks_in_thread()
 
         # save json to disk and return a dataframe
-        self._save_pairs(pairs)
+        self._save_pairs(pairs, username)
         results_log = [pair.model_dump() for pair in pairs if pair is not None]
         if not results_log:
             logger.warning("Agent did not produce any answers to submit.")