Added grader

tests/candidate.py

@@ -13,13 +13,13 @@ from api.llm import LLMManager
 from config import config
 from resources.data import fixed_messages, topic_lists
 from resources.prompts import prompts
-from tests.tessting_prompts import candidate_prompt
+from tests.testing_prompts import candidate_prompt
 
 load_dotenv()
 
 
 def complete_interview(interview_type, exp_name, requirements="", difficulty="", topic="", model="gpt-3.5-turbo"):
-    client = OpenAI()
+    client = OpenAI(url="https://api.openai.com/v1")
     llm = LLMManager(config, prompts)
     llm_name = config.llm.name
 

tests/grader.py

@@ -0,0 +1,43 @@
+import json
+
+from openai import OpenAI
+
+from tests.testing_prompts import grader_prompt
+
+
+def grade(json_file_path, model="gpt-4-turbo"):
+    client = OpenAI(url="https://api.openai.com/v1")
+
+    with open(json_file_path) as file:
+        interview_data = json.load(file)
+
+    messages = [
+        {"role": "system", "content": grader_prompt},
+        {"role": "user", "content": f"Interview data: {interview_data}"},
+        {"role": "user", "content": "Please evaluate the interview."},
+    ]
+
+    response = client.chat.completions.create(model=model, messages=messages, temperature=1, response_format={"type": "json_object"})
+    feedback = json.loads(response.choices[0].message.content)
+
+    feedback["file_name"] = json_file_path
+    feedback["agent_llm"] = interview_data["interviewer_llm"]
+    feedback["candidate_llm"] = interview_data["candidate_llm"]
+    feedback["type"] = interview_data["inputs"]["interview_type"]
+    feedback["difficulty"] = interview_data["inputs"]["difficulty"]
+    feedback["topic"] = interview_data["inputs"]["topic"]
+    feedback["average_response_time_seconds"] = interview_data["average_response_time_seconds"]
+    feedback["number_of_messages"] = len(interview_data["transcript"])
+
+    scores = [
+        feedback[x]
+        for x in feedback
+        if x.startswith("interviewer_") or x.startswith("feedback_") or x.startswith("problem_") and feedback[x] is not None
+    ]
+    feedback["overall_score"] = sum(scores) / len(scores)
+
+    # save results to json file in the same folder as the interview data
+    with open(json_file_path.replace(".json", "_feedback.json"), "w") as file:
+        json.dump(feedback, file, indent=4)
+
+    return feedback

tests/test_e2e.py

@@ -1,6 +1,12 @@
 from tests.candidate import complete_interview
+from tests.grader import grade
 
 
 def test_complete_interview():
-    file_path, interview_data = complete_interview("coding", "test", model="gpt-3.5-turbo")
-    assert True
+    for _ in range(3):
+        file_path, _ = complete_interview("coding", "test", model="gpt-3.5-turbo")
+        feedback = grade(file_path, model="gpt-4-turbo")
+        assert feedback["overall_score"] > 0.5
+        if feedback["overall_score"] > 0.8:
+            return
+    assert False