implement round specific evaluation

_static/html/evaluation_instruction.html

@@ -8,13 +8,10 @@
       <li><strong>Reason:</strong> Write down the reason for your choice in the free text box labeled "The reason of your choice".</li>
       <li><strong>Evaluation Criteria:</strong> After making your decision, please evaluate the system in the following aspects:
         <ul>
-          <li><strong>Trust:</strong> How much do you trust the financial advisor? Answer from 1 to 100.</li>
-          <li><strong>Satisfaction:</strong> How satisfied are you with the financial advisor? Answer from 1 to 100.</li>
-          <li><strong>Knowledgeable:</strong> How knowledgeable do you feel after interacting with the financial advisor? Answer from 1 to 100.</li>
-          <li><strong>Helpful:</strong> How helpful do you find the financial advisor? Answer from 1 to 100.</li>
+          <li><strong>Likelihood of Purchase:</strong> Rate the likelihood of purchase, with 1 being very low and 7 being very high.</li>
+          <li><strong>Confidence in Your Decision:</strong> Rate the confidence of your decision, with 1 being very low and 7 being very high.</li>
         </ul>
       </li>
-      <li>After completing all the evaluation criteria, <strong>click the "Send: Evaluation" button</strong> to submit your responses.</li>
+      <li>After completing all the evaluation criteria, <strong>click the "Send: Evaluation" button</strong> to submit your responses. <strong>Once you submit, you cannot resubmit, so please review your answers carefully before submitting.</strong></li>
     </ol>
-    <p style="font-size: 16px; color: #34495e;">Thank you for your participation in this evaluation process!</p>
   </div>

app.py

@@ -258,7 +258,9 @@ def add_user_profile_to_system_instruction(
             # print(f"Preference Summary:{summ}")
         system_instruction += f"\nUser Profile collected in the previous conversations: {user_preference_elicitation_data['summary_history']}\n"
     else:
-        system_instruction += f"\nUser Profile collected in the previous conversations: {user_preference_elicitation_data['history']}\n"
+        system_instruction += (
+            f"\nUser Profile collected in the previous conversations: {user_preference_elicitation_data['history']}\n"
+        )
     return system_instruction
 
 
@@ -300,38 +302,13 @@ def create_demo():
                     lines=5,
                 )
             with gr.Row():
-                trust = gr.Slider(
-                    label="Trust",
-                    minimum=1,
-                    maximum=100,
-                    value=50,
-                    info="How much do you trust the financial advisor? Answer from 1 to 100. A score of 100 means you have complete trust in the financial advisor, while a score of 1 means you have no trust at all.",
-                    step=1,
-                )
-                satisfaction = gr.Slider(
-                    label="Satisfaction",
-                    minimum=1,
-                    maximum=100,
-                    value=50,
-                    info="How satisfied are you with the financial advisor? Answer from 1 to 100. A score of 100 means you are completely satisfied, while a score of 1 means you are not satisfied at all.",
-                    step=1,
+                likelihood = gr.Radio(
+                    [1, 2, 3, 4, 5, 6, 7], label="Likelihood of Purchase (1 = Very Low, 7 = Very High)", show_label=True
                 )
-            with gr.Row():
-                knowledgeable = gr.Slider(
-                    label="Knowledgeable",
-                    minimum=1,
-                    maximum=100,
-                    value=50,
-                    info="How knowledgeable do you feel after interacting with the financial advisor? Answer from 1 to 100. A score of 100 means you feel very knowledgeable, while a score of 1 means you feel not knowledgeable at all.",
-                    step=1,
-                )
-                helpful = gr.Slider(
-                    label="Helpful",
-                    minimum=1,
-                    maximum=100,
-                    value=50,
-                    info="How helpful do you find the financial advisor? Answer from 1 to 100. A score of 100 means you find the financial advisor very helpful, while a score of 1 means you find the financial advisor not helpful at all.",
-                    step=1,
+                confidence = gr.Radio(
+                    [1, 2, 3, 4, 5, 6, 7],
+                    label="Confidence in Your Decision (1 = Very Low, 7 = Very High)",
+                    show_label=True,
                 )
             evaluation_send_button = gr.Button(value="Send: Evaluation")
         return {
@@ -344,10 +321,8 @@ def create_demo():
             "msg": msg,
             "dropdown": dropdown,
             "reason": reason,
-            "trust": trust,
-            "satisfaction": satisfaction,
-            "knowledgeable": knowledgeable,
-            "helpful": helpful,
+            "likelihood": likelihood,
+            "confidence": confidence,
             "evaluation_send_button": evaluation_send_button,
             "general_instruction": general_instruction,
             "round_instruction": round_instruction,
@@ -419,10 +394,8 @@ def create_demo():
             msg,
             dropdown,
             reason,
-            trust,
-            satisfaction,
-            knowledgeable,
-            helpful,
+            likelihood,
+            confidence,
             evaluation_send_button,
         ) = (
             tabs["comp"],
@@ -434,10 +407,8 @@ def create_demo():
             tabs["msg"],
             tabs["dropdown"],
             tabs["reason"],
-            tabs["trust"],
-            tabs["satisfaction"],
-            tabs["knowledgeable"],
-            tabs["helpful"],
+            tabs["likelihood"],
+            tabs["confidence"],
             tabs["evaluation_send_button"],
         )
         system_instruction = ""
@@ -510,16 +481,14 @@ def create_demo():
             [tab_session, chatbot],
         )
         evaluation_send_button.click(
-            lambda user_id, comp, tab_session, dropdown, reason, trust, satisfaction, knowledgeable, helpful, evaluation_send_button: respond_evaluation(
+            lambda user_id, comp, tab_session, dropdown, reason, likelihood, confidence, evaluation_send_button: respond_evaluation(
                 user_id,
                 tab_session,
                 {
                     "selection": dropdown,
                     "reason": reason,
-                    "trust": trust,
-                    "satisfaction": satisfaction,
-                    "knowledgeable": knowledgeable,
-                    "helpful": helpful,
+                    "likelihood": likelihood,
+                    "confidence": confidence,
                 },
                 comp,
                 evaluation_send_button,
@@ -530,13 +499,11 @@ def create_demo():
                 tab_session,
                 dropdown,
                 reason,
-                trust,
-                satisfaction,
-                knowledgeable,
-                helpful,
+                likelihood,
+                confidence,
                 evaluation_send_button,
             ],
-            [tab_session, dropdown, reason, trust, satisfaction, knowledgeable, helpful, evaluation_send_button],
+            [tab_session, dropdown, reason, likelihood, confidence, evaluation_send_button],
         )
 
     def click_control_preference_stage(tabs, user_id, user_preference_elicitation_session):
@@ -925,10 +892,8 @@ def create_demo():
             tab_data,
             evals["selection"],
             evals["reason"],
-            evals["trust"],
-            evals["satisfaction"],
-            evals["knowledgeable"],
-            evals["helpful"],
+            evals["likelihood"],
+            evals["confidence"],
             evaluation_send_button,
         )
 

user_feedback/round_evaluation_a7d67f45-e195-4752-9702-d9cafd065fd8.json

@@ -0,0 +1 @@
+{"user_id": "demo_1_1", "uuid": "a7d67f45-e195-4752-9702-d9cafd065fd8", "timestamp": "2024-10-17 18:23:18", "type": "round_evaluation", "value": {"selection": null, "reason": "", "likelihood": 7, "confidence": null, "company": "Amazon.com, Inc."}}