update app.py with o1-mini

app.py

@@ -7,42 +7,44 @@ from dotenv import load_dotenv
 
 load_dotenv()
 
-# Combined template for fact checking
-user_template = """You are a thorough and precise fact-checking assistant. Your role is to:
-1. Analyze statements for factual accuracy
-2. Identify potential inaccuracies or misconceptions
-3. Provide evidence-based corrections when needed
-4. Note any missing context or nuances
-5. Maintain an objective and unbiased perspective
-
-Frame your responses in these sections:
-- Claim Analysis
-- Evidence Assessment
-- Factual Correction (if needed)
-- Additional Context
-
-User statement to verify: {input}
-
-Please consider:
-- The accuracy of specific claims
-- The reliability of any cited sources
-- Any missing context
-- Common misconceptions about this topic
-
-Analyze the statement step by step and provide a clear assessment.
+# Template for LLM system vibe checking
+user_template = """You are an assistant helping to perform quick evaluations of LLM-powered systems. Your role is to:
+1. Help identify potential critical failure points in the system
+2. Assess basic functionality and obvious issues
+3. Look for significant problems that would be immediately noticeable
+4. Provide a cursory but meaningful evaluation
+5. Focus on crucial functions where failure would be severe
+
+Frame your assessment in these sections:
+- Basic Functionality Check
+- Critical Issues Assessment
+- Obvious Failure Points
+- Quick Recommendations
+
+System or component to evaluate: {input}
+
+Key areas to examine:
+- Core functionality problems
+- Obvious response issues
+- Critical safety concerns
+- Basic performance problems
+- User-facing issues
+
+Provide an informal but insightful evaluation focusing on major concerns.
 """
 
 @cl.on_chat_start
 async def start_chat():
-    # Welcome message with fact checking introduction
+    # Welcome message with LLM system vibe check introduction
     await cl.Message(
-        content="👋 Welcome to the Fact Check Assistant! I'm here to help analyze statements "
-        "for factual accuracy. Feel free to share any statement you'd like to verify, such as:\n\n"
-        "1. Historical claims or dates\n"
-        "2. Scientific statements\n"
-        "3. Statistical claims\n"
-        "4. Quotes or attributions\n"
-        "5. Common beliefs or assertions\n"
+        content="👋 Welcome to the LLM System Vibe Check Assistant! I'll help you perform quick evaluations "
+        "of LLM-powered systems. Share any component or behavior you want to evaluate, such as:\n\n"
+        "1. Response quality or consistency\n"
+        "2. Safety mechanism effectiveness\n"
+        "3. Basic functionality issues\n"
+        "4. User interaction problems\n"
+        "5. Critical system behaviors\n\n"
+        "Remember: This is meant to be a cursory check for obvious issues, not a comprehensive evaluation."
     ).send()
 
     settings = {
@@ -84,12 +86,12 @@ async def main(message: cl.Message):
     prompt.completion = msg.content
     msg.prompt = prompt
 
-    # Add indicator emoji based on fact-check result
-    if any(word in msg.content.lower() for word in ['correct', 'accurate', 'true', 'verified']):
-        await msg.stream_token(" ✅")
-    elif any(word in msg.content.lower() for word in ['incorrect', 'false', 'inaccurate', 'misleading']):
-        await msg.stream_token(" ❌")
+    # Add indicator emoji based on evaluation result
+    if any(word in msg.content.lower() for word in ['critical', 'severe', 'serious', 'failing']):
+        await msg.stream_token(" 🚨")  # Critical issues found
+    elif any(word in msg.content.lower() for word in ['minor', 'small', 'minimal']):
+        await msg.stream_token(" ⚠️")  # Minor issues found
     else:
-        await msg.stream_token(" ℹ️")
+        await msg.stream_token(" ✅")  # No obvious issues
 
     await msg.send()