New description

app.py

@@ -43,8 +43,10 @@ def get_evaluation_questions():
     Both lists must be of equal length.
     """
     questions_str = os.environ.get("TEST_QUESTION_1")
+    docs_str = os.environ.get("TEST_DOCUMENTS_1")
     expected_str = os.environ.get("TEST_EXPECTED_1")
-    if not questions_str or not expected_str:
+
+    if not questions_str or not expected_str or not docs_str:
         return []
     try:
         questions_list = json.loads(questions_str)
@@ -56,10 +58,18 @@ def get_evaluation_questions():
     except Exception as e:
         print(f"Error parsing expected answers: {str(e)}")
         return []
-    if len(questions_list) != len(expected_list):
+    try:
+        docs_list = json.loads(docs_str)
+    except Exception as e:
+        print(f"Error parsing documents: {str(e)}")
+        return []
+    
+    # Ensure all lists are of the same length.
+    if len(questions_list) != len(expected_list) or len(questions_list) != len(docs_list):
         print("Mismatch in length: questions list and expected answers list must have the same length.")
         return []
-    return [{"question": q, "expected": e} for q, e in zip(questions_list, expected_list)]
+    
+    return [{"question": q, "expected": e, "docs": d} for q, e, d in zip(questions_list, expected_list, docs_list)]
 
 # Load evaluation questions at startup.
 EVALUATION_QUESTIONS = get_evaluation_questions()
@@ -129,6 +139,7 @@ def submit_prompt(email, name, system_prompt):
     # Process each evaluation question.
     for item in EVALUATION_QUESTIONS:
         question = item["question"]
+        docs = item["docs"]
         expected = item["expected"]
         try:
             response = client.chat.completions.create(
@@ -224,122 +235,109 @@ def build_interface():
     Constructs the Gradio interface with a submission button and single-submission mechanism.
     """
     with gr.Blocks() as demo:
-        gr.Markdown("# Applicant Task: Writing a System Prompt")
-        gr.Markdown("## Document and Clause Level Classification")
+        gr.Markdown("# Applicant Task: Target Company & Law Firm Identification")
+        gr.Markdown("## Identifying Parties, Law Firms, and Target Company Presence")
+        
         # General description
         gr.Markdown("""
-        Applicants must create a system prompt for a language model that classifies user requests about finding information into two specific categories:  
-        1. **Document Level**: Determines whether the query refers to a single document or multiple documents.  
-        2. **Clause Level**: Identifies whether the query is focused on:  
-            - A single clause,  
-            - Multiple clauses, or  
-            - General information not constrained to any specific clause.
+        This task involves processing a user query to determine the relevance to the intended task, followed by analyzing textual data to extract information about law firms representing parties (Buyer, Seller, and Third Parties) and verifying the presence of a target company. 
 
-        Imagine the user is a legal professional who is trying to find information related to their work. We need this classification to guide our downstream approach on how to retrieve this information.
+        The system is designed to sequentially leverage three LLM functions:
         
-        That's why the model must return a valid JSON object with the following structure:  
+        **Step 1:** LLM1 determines if the user's query mentions any target company.  
+        - If no target company is found, LLM1 responds with a message wrapped in `<user_message></user_message>` XML tags to inform the user that the query is irrelevant to this task.  
+        - If the query contains a target company, LLM1 moves forward with a formatted acknowledgment of the identified target company.
+
+        **Step 2:** LLM2 examines four separate paragraphs provided as input. For each paragraph, it extracts specific information about:
+        - The Buyer's representative law firm.
+        - The Seller's representative law firm.
+        - Any third-party law firm present.
+        - Whether the target company is mentioned in the paragraph.
+        Each paragraph's results are formatted and concatenated for the next step.
+
+        **Step 3:** LLM3 compiles the information from all analyzed paragraphs and outputs a structured JSON object with the following keys:
         
-        ```
+        ```json
         {
-          "document_level": "single/multiple",
-          "clause_level": "single/multiple/general"
+        "buyer_firm": "string",
+        "seller_firm": "string",
+        "third_party": "string",
+        "contains_target_firm": boolean
         }
         ```
         
-        The goal is to ensure that the model's output adheres to the precscibed JSON structure and accurately classifies 7 test queries into the two respective categories. 
+        The goal is to identify the representative law firms of involved parties and determine if the target company is mentioned, ensuring the results are structured and accurate.
         
-        This task is designed to evaluate your ability to craft prompts by adhering to the required structure, without relying on constrained decoding or "JSON mode," while simultaneously providing accurate responses.
+        **Key Considerations:**
+        - The output must adhere to the prescribed JSON format for the final step.
+        - Ensure the system can accurately extract and classify relevant information from the input paragraphs.
         """)
 
         # Example Inputs and Outputs in an Accordion
-        with gr.Accordion("Example Inputs and Expected Outputs", open=False):
+        with gr.Accordion("Example Workflow", open=False):
             gr.Markdown("""
-            1. **User Message Example 1:**  
-            - *"Please provide the contract for the lease agreement."*  
-            - **Expected Output:**  
-            ```
-            {"document_level": "single", "clause_level": "general"}
-            ```
-    
-            2. **User Message Example 2:**  
-            - *"I need all clauses related to termination in the employment contract."*  
-            - **Expected Output:**  
-            ```
-            {"document_level": "single", "clause_level": "multiple"}
-            ```
-    
-            3. **User Message Example 3:**  
-            - *"Can you send me the financial reports and the partnership agreement?"*  
-            - **Expected Output:**  
-            ```
-            {"document_level": "multiple", "clause_level": "general"}
-            ```
-    
-            4. **User Message Example 4:**  
-            - *"What are the key clauses in the NDA?"*  
-            - **Expected Output:**  
-            ```
-            {"document_level": "single", "clause_level": "multiple"}
-            ```
-    
-            5. **User Message Example 5:**  
-            - *"Tell me about the company’s financials."*  
-            - **Expected Output:**  
-            ```
-            {"document_level": "single", "clause_level": "general"}
-            ```
-    
-            6. **User Message Example 6:**  
-            - *"Provide all contracts and their confidentiality clauses."*  
-            - **Expected Output:**  
-            ```
-            {"document_level": "multiple", "clause_level": "multiple"}
-            ```
-    
-            7. **User Message Example 7:**  
-            - *"Extract the arbitration clause from this service agreement."*  
-            - **Expected Output:**  
-            ```
-            {"document_level": "single", "clause_level": "single"}
+            **Example Query and System Output:**
+
+            **User Query:**  
+            *"Is Kirkland present?"*
+
+            Step 1 (LLM1):  
+            - If no target company is identified:  
+            Output: `<user_message>Query is not relevant to the intended task.</user_message>`  
+
+            - If a target company is identified:  
+            Output: *"The query mentions the target company Kirkland."*
+
+            Step 2 (LLM2 for Paragraphs):  
+            **Input Paragraph Example:**  
+            *"Representation agreements between Buyers and Kirkland & Ellis are included."*  
+
+            **Output:**  
+            *"Buyer Firm: Kirkland & Ellis, Seller Firm: None, Third Party Firm: None, Contains Target Firm: True."*  
+
+            Step 3 (LLM3 Final Output):  
+            Compiled JSON:
+            ```json
+            {
+            "buyer_firm": "Kirkland & Ellis",
+            "seller_firm": null,
+            "third_party": null,
+            "contains_target_firm": true
+            }
             ```
             """)
 
-        # Challenge instructions in another Accordion
-        with gr.Accordion("Task Instructions", open=False):
+        # Challenge instructions and testing guidance
+        with gr.Accordion("Task Instructions and Testing", open=False):
             gr.Markdown("""
-            - Design a system prompt that ensures gpt4o-mini generates outputs like those above when given similar user messages.
-            
-              The system prompt should:
-              1. Specify formatting requirements (e.g., *"Output must be a valid JSON object"*). 
-                  - Note that we are not using constrained decoding or any sort of JSON mode; if not correctly prompted, the LLM will output plain text.
-                  - All LLM responses will be passed to json.loads(response), responses that fail the json parsing are deemed incorrect (beware of tripple backtricks etc.)
-              2. Emphasize strict adherence to classification definitions:
-                  - *Single Document:* Refers to one document.
-                  - *Multiple Documents:* Refers to more than one document.
-                  - *Single Clause:* Refers to one specific clause.
-                  - *Multiple Clauses:* Refers to more than one specific clause.
-                  - *General Information:* Refers to general content not tied to specific clauses.
+            - Design prompts that ensure proper interaction between the three LLM systems, with each step contributing to the final output.
+            - Ensure strict adherence to JSON formatting requirements (e.g., no extra characters that may cause JSON parsing errors).
+            - Test extensively to verify accurate law firm and target company identification.
             
-            **You can only submit once, so test your system prompt thoroughly before submission!**
+            **Output Requirements:**
+            - Ensure final LLM3 JSON output has the following keys:
+            - `"buyer_firm"`
+            - `"seller_firm"`
+            - `"third_party"`
+            - `"contains_target_firm"`
+            - Values must be accurately extracted or classified based on LLM2's parsed data.
 
-            You will be scored according to the following criteria with respect to the outputs of 7 test user messages:
-            
-                - Response is valid JSON
-                - The response contains the keys: "document_level" and "clause_level"
-                - The values for each of the keys are correct
+            **Hints for Crafting System Prompts:**
+            - Explicitly specify formatting requirements at each step.
+            - Clarify the task definitions and expected classifications in each system prompt for LLM1, LLM2, and LLM3.
+            - Test using diverse sample data for robustness.
+
+            You can only submit once, so validate your system prompts thoroughly using mock queries and example data before final submission.
             
             Good Luck!
-            
-              """)
+            """)
+
+        gr.Markdown("""
+        Enter your name and email below, as listed in your CV, and submit your designed prompts.
         
-        gr.Markdown(
-            """Please enter the same name and email as listed in your CV and submit your system prompt below.
-            
-            You can only submit once, try to test and build out your system prompt using gpt4o-mini with temp=1 before submitting your solution.
-            
-            We look forward to your submission!
-            """
-        )
+        Remember: Focus on clarity, accuracy, and structured responses to achieve a high score!
+        """)
+
 
         email_input = gr.Textbox(label="Email", placeholder="[email protected]")
         name_input = gr.Textbox(label="First Name, Last Name", placeholder="John, Smith")
@@ -356,10 +354,12 @@ def build_interface():
             inputs=[email_input, name_input, system_prompt_input],
             outputs=output_text,
         )
+        
+        
     
     return demo
 
 if __name__ == "__main__":
     interface = build_interface()
     # Launch the app on 0.0.0.0 so it is accessible externally (e.g., in a container).
-    interface.launch(server_name="0.0.0.0", server_port=7860)
+    interface.launch(server_name="0.0.0.0", server_port=7860)