Update app.py

app.py

@@ -358,7 +358,7 @@ Select one action and its input. Output JSON only."""
             logger.warning(f"PUI_GRADIO [{request_id}]: Missing input for {action_type}. Falling back.")
             action_type = "quick_respond" # Fallback to quick_respond logic above
             final_system_prompt_str += " Respond directly. (Note: A web action was attempted but failed due to missing input)."
-            final_user_prompt_content_str = f"History:\n{history_str_for_prompt}\nGuidelines:\n{initial_insights_ctx_str}\nQuery: \"{user_input}\"\nResponse:"
+            final_user_prompt_content_str = f"Current Conversation History (User/AI turns):\n{history_str_for_prompt}\n\nGuiding Principles (Learned Rules):\n{initial_insights_ctx_str}\n\nUser's Current Query: \"{user_input}\"\n\nYour concise and helpful response:"
         else:
             yield "status", f"<i>[Web: '{query_or_url[:60]}'...]</i>"
             web_results_data = []
@@ -387,12 +387,12 @@ Select one action and its input. Output JSON only."""
             
             yield "status", "<i>[Synthesizing web report...]</i>"
             final_system_prompt_str += " Generate a report/answer from web content, history, & guidelines. Cite URLs as [Source X]."
-            final_user_prompt_content_str = f"History:\n{history_str_for_prompt}\nGuidelines:\n{initial_insights_ctx_str}\n\nWeb Content Found:\n{scraped_content_for_prompt}\n\nUser's Query: \"{user_input}\"\n\nYour report/response (cite sources like [Source 1], [Source 2]):"
+            final_user_prompt_content_str = f"Current Conversation History:\n{history_str_for_prompt}\n\nGuiding Principles:\n{initial_insights_ctx_str}\n\nWeb Content Found:\n{scraped_content_for_prompt}\n\nUser's Query: \"{user_input}\"\n\nYour report/response (cite sources like [Source 1], [Source 2]):"
     
     else: # Fallback if action_type is somehow unknown
         logger.warning(f"PUI_GRADIO [{request_id}]: Unknown action_type '{action_type}'. Defaulting.")
         final_system_prompt_str += " Respond directly. (Internal state error)."
-        final_user_prompt_content_str = f"History:\n{history_str_for_prompt}\nGuidelines:\n{initial_insights_ctx_str}\nQuery: \"{user_input}\"\nResponse:"
+        final_user_prompt_content_str = f"Current Conversation History (User/AI turns):\n{history_str_for_prompt}\n\nGuiding Principles (Learned Rules):\n{initial_insights_ctx_str}\n\nUser's Current Query: \"{user_input}\"\n\nYour concise and helpful response:"
 
 
     # --- Final LLM Call for Response Generation ---
@@ -461,12 +461,50 @@ def deferred_learning_and_memory_task(user_input: str, bot_response: str, provid
         relevant_existing_rules = retrieve_insights_simple_keywords(reflection_context_query, k_insights=10) # Max 10 rules for context
         existing_rules_ctx_str = "\n".join([f"- \"{rule}\"" for rule in relevant_existing_rules]) if relevant_existing_rules else "No specific existing rules found as highly relevant for direct comparison."
         
-        # Use the long system prompt for insight generation from ai-learn
-        insight_sys_prompt = """You are an expert AI knowledge base curator... (Your full long prompt here from ai-learn's deferred_learning_and_memory)... Output ONLY the JSON list."""
+        # Insight generation prompt (ensure this is the full, detailed prompt from ai-learn)
+        insight_sys_prompt = """You are an expert AI knowledge base curator. Your primary function is to meticulously analyze an interaction and update the AI's guiding principles (insights/rules) to improve its future performance and self-understanding.
+You MUST output a JSON list of operation objects. This list can and SHOULD contain MULTIPLE distinct operations if various learnings occurred.
+Each operation object in the JSON list must have:
+1. "action": A string, either "add" (for entirely new rules) or "update" (to replace an existing rule with a better one).
+2. "insight": A string, the full, refined insight text including its [TYPE|SCORE] prefix (e.g., "[CORE_RULE|1.0] My name is Lumina, an AI assistant.").
+3. "old_insight_to_replace" (ONLY for "update" action): A string, the *exact, full text* of an existing insight that the new "insight" should replace.
+**Your Reflection Process (Consider each step and generate operations accordingly):**
+**STEP 1: Core Identity & Purpose Review (Result: Primarily 'update' operations)**
+   - Examine all `CORE_RULE`s related to my identity (name, fundamental purpose, core unchanging capabilities, origin) from the "Potentially Relevant Existing Rules".
+   - **CONSOLIDATE & MERGE:** If multiple `CORE_RULE`s state similar aspects (e.g., multiple name declarations like 'Lumina' and 'LearnerAI', or slightly different purpose statements), you MUST merge them into ONE definitive, comprehensive `CORE_RULE`.
+   - The new "insight" will be this single, merged rule. Propose separate "update" operations to replace *each* redundant or less accurate core identity rule with this new canonical one.
+   - Prioritize user-assigned names or the most specific, recently confirmed information. If the interaction summary clarifies a name or core function, ensure this is reflected.
+**STEP 2: New Distinct Learnings (Result: Primarily 'add' operations)**
+   - Did I learn any completely new, distinct facts (e.g., "The user's project is codenamed 'Bluefire'")?
+   - Did I demonstrate or get told about a new skill/capability not previously documented (e.g., "I can now generate mermaid diagrams based on descriptions")?
+   - Did the user express a strong, general preference that should guide future interactions (e.g., "User prefers responses to start with a direct answer, then explanation")?
+   - For these, propose 'add' operations. Assign `CORE_RULE` for truly fundamental new facts/capabilities, otherwise `RESPONSE_PRINCIPLE` or `BEHAVIORAL_ADJUSTMENT`. Ensure these are genuinely NEW and not just rephrasing of existing non-core rules.
+**STEP 3: Refinements to Existing Behaviors/Principles (Result: 'update' operations for non-core rules)**
+   - Did I learn to modify or improve an existing behavior, response style, or operational guideline (that is NOT part of core identity)?
+   - For example, if an existing `RESPONSE_PRINCIPLE` was "Be formal," and the interaction showed the user prefers informality, update that principle.
+   - Propose 'update' operations for the relevant `RESPONSE_PRINCIPLE` or `BEHAVIORAL_ADJUSTMENT`. Only update if the change is significant.
+**General Guidelines:**
+- If no new insights, updates, or consolidations are warranted from the interaction, output an empty JSON list: `[]`.
+- Ensure the "insight" field (for both add/update) always contains the properly formatted insight string: `[TYPE|SCORE] Text`. TYPE can be `CORE_RULE`, `RESPONSE_PRINCIPLE`, `BEHAVIORAL_ADJUSTMENT`. Scores should reflect confidence/importance.
+- Be precise with "old_insight_to_replace" – it must *exactly* match an existing rule string from the "Potentially Relevant Existing Rules" context.
+- Aim for a comprehensive set of operations that reflects ALL key learnings from the interaction.
+- Output ONLY the JSON list. No other text, explanations, or markdown.
+**Example of a comprehensive JSON output with MULTIPLE operations:**
+[
+  {"action": "update", "old_insight_to_replace": "[CORE_RULE|1.0] My designated name is 'LearnerAI'.", "insight": "[CORE_RULE|1.0] I am Lumina, an AI assistant designed to chat, provide information, and remember context like the secret word 'rocksyrup'."},
+  {"action": "update", "old_insight_to_replace": "[CORE_RULE|1.0] I'm Lumina, the AI designed to chat with you.", "insight": "[CORE_RULE|1.0] I am Lumina, an AI assistant designed to chat, provide information, and remember context like the secret word 'rocksyrup'."},
+  {"action": "add", "insight": "[CORE_RULE|0.9] I am capable of searching the internet for current weather information if asked."},
+  {"action": "add", "insight": "[RESPONSE_PRINCIPLE|0.8] When user provides positive feedback, acknowledge it warmly."},
+  {"action": "update", "old_insight_to_replace": "[RESPONSE_PRINCIPLE|0.7] Avoid mentioning old conversations.", "insight": "[RESPONSE_PRINCIPLE|0.85] Avoid mentioning old conversations unless the user explicitly refers to them or it's highly relevant to the current query."}
+]"""
         insight_user_prompt = f"""Interaction Summary:\n{summary_for_reflection}\n
 Potentially Relevant Existing Rules (Review these carefully for consolidation or refinement):\n{existing_rules_ctx_str}\n
 Guiding principles that were considered during THIS interaction (these might offer clues for new rules or refinements):\n{json.dumps([p['original'] for p in parsed_insights_for_reflection if 'original' in p]) if parsed_insights_for_reflection else "None"}\n
-Task: Based on your reflection process... (Your full long task description here from ai-learn's deferred_learning_and_memory)... Output JSON only."""
+Task: Based on your three-step reflection process (Core Identity, New Learnings, Refinements):
+1.  **Consolidate CORE_RULEs:** Merge similar identity/purpose rules from "Potentially Relevant Existing Rules" into single, definitive statements using "update" operations. Replace multiple old versions with the new canonical one.
+2.  **Add New Learnings:** Identify and "add" any distinct new facts, skills, or important user preferences learned from the "Interaction Summary".
+3.  **Update Existing Principles:** "Update" any non-core principles from "Potentially Relevant Existing Rules" if the "Interaction Summary" provided a clear refinement.
+Combine all findings into a single JSON list of operations. If there are multiple distinct changes based on the interaction and existing rules, ensure your list reflects all of them. Output JSON only."""
         
         insight_gen_messages = [{"role": "system", "content": insight_sys_prompt}, {"role": "user", "content": insight_user_prompt}]
 
@@ -553,13 +591,13 @@ def handle_gradio_chat_submit(user_message_text: str,
                               ui_api_key_text: str | None,
                               custom_system_prompt_text: str):
     
+    global current_chat_session_history # <--- ***** ADD THIS LINE *****
+
     # Initialize UI update variables
     cleared_input_text = "" # To clear the user input box
     updated_gradio_history = list(gradio_chat_history_list) # Copy current display history
     current_status_text = "Initializing..."
     # Default values for output components to yield immediately
-    # These should match the types of the output components in demo.launch
-    # Use dummy values that match the component types if needed
     default_detected_outputs_md = gr.Markdown(value="*Processing...*")
     default_formatted_output_text = gr.Textbox(value="*Waiting for AI response...*")
     default_download_button = gr.DownloadButton(interactive=False, value=None, visible=False)
@@ -586,7 +624,12 @@ def handle_gradio_chat_submit(user_message_text: str,
     if len(internal_processing_history) > (MAX_HISTORY_TURNS * 2 + 1): # +1 for potential system prompt
         # Simple truncation from the beginning, preserving last N turns
         # More sophisticated: keep system prompt if present, then truncate older user/assistant pairs
-        internal_processing_history = internal_processing_history[-(MAX_HISTORY_TURNS * 2):]
+        # This simple truncation might lose a system prompt if it was the very first item
+        # A better approach if system prompts are always first:
+        if internal_processing_history[0]["role"] == "system" and len(internal_processing_history) > (MAX_HISTORY_TURNS * 2 +1) :
+             internal_processing_history = [internal_processing_history[0]] + internal_processing_history[-(MAX_HISTORY_TURNS * 2):]
+        else:
+             internal_processing_history = internal_processing_history[-(MAX_HISTORY_TURNS * 2):]
 
 
     final_bot_response_text_accumulated = ""
@@ -671,8 +714,13 @@ def handle_gradio_chat_submit(user_message_text: str,
         current_chat_session_history.append({"role": "assistant", "content": final_bot_response_text_accumulated})
         
         # Trim global history if it exceeds max turns
-        if len(current_chat_session_history) > (MAX_HISTORY_TURNS * 2):
-            current_chat_session_history = current_chat_session_history[-(MAX_HISTORY_TURNS * 2):]
+        if len(current_chat_session_history) > (MAX_HISTORY_TURNS * 2): # Check total length
+            # If a system prompt is the first message, preserve it during trimming
+            if current_chat_session_history[0]["role"] == "system" and len(current_chat_session_history) > (MAX_HISTORY_TURNS * 2 + 1):
+                current_chat_session_history = [current_chat_session_history[0]] + current_chat_session_history[-(MAX_HISTORY_TURNS * 2):]
+            else: # No system prompt or length is fine with system prompt
+                 current_chat_session_history = current_chat_session_history[-(MAX_HISTORY_TURNS * 2):]
+
 
         # Start deferred learning task in a background thread
         logger.info(f"Starting deferred learning task for user: '{user_message_text[:40]}...'")
@@ -705,17 +753,12 @@ def ui_upload_rules_action(uploaded_file_obj, progress=gr.Progress()):
     if not uploaded_file_obj: return "No file provided for rules upload."
     
     try:
-        content = uploaded_file_obj.decode('utf-8') # Gradio File component gives bytes
-    except AttributeError: # If it's already a string (e.g. from temp file path)
-        try:
-            with open(uploaded_file_obj.name, 'r', encoding='utf-8') as f: # .name if it's a temp file object
-                content = f.read()
-        except Exception as e_read:
-            logger.error(f"Error reading uploaded rules file: {e_read}")
-            return f"Error reading file: {e_read}"
-    except Exception as e_decode:
-        logger.error(f"Error decoding uploaded rules file: {e_decode}")
-        return f"Error decoding file content: {e_decode}"
+        # Gradio's File component gives a temp file object, access its name
+        with open(uploaded_file_obj.name, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except Exception as e_read:
+        logger.error(f"Error reading uploaded rules file: {e_read}")
+        return f"Error reading file: {e_read}"
 
     if not content.strip(): return "Uploaded rules file is empty."
 
@@ -726,6 +769,7 @@ def ui_upload_rules_action(uploaded_file_obj, progress=gr.Progress()):
         potential_rules = [r.strip() for r in content.splitlines() if r.strip()]
     
     total_to_process = len(potential_rules)
+    if total_to_process == 0: return "No rules found in file to process."
     progress(0, desc="Starting rules upload...")
 
     for idx, rule_text in enumerate(potential_rules):
@@ -738,16 +782,66 @@ def ui_upload_rules_action(uploaded_file_obj, progress=gr.Progress()):
         # A better save_rule_to_file could return: "added", "duplicate", "invalid_format", "error"
         
         # Re-check for existing before trying to save for more accurate "skipped_count"
-        existing_rules_snapshot = load_rules_from_file() # Could be slow if called repeatedly
-        if rule_text in existing_rules_snapshot:
-            skipped_count +=1
-        elif save_rule_to_file(rule_text): # save_rule_to_file will log its own errors/skips
-            added_count += 1
-        else: # Failed to save for other reasons (format error logged by save_rule_to_file, or file write error)
-            error_count += 1
+        # This is inefficient for very large rule sets, but ok for moderate numbers.
+        # Consider optimizing if this becomes a bottleneck.
+        # existing_rules_snapshot = load_rules_from_file() 
+        # if rule_text in existing_rules_snapshot:
+        #     skipped_count +=1
+        # el
+        if save_rule_to_file(rule_text): # save_rule_to_file logs its own errors/skips for duplicates/format
+            # save_rule_to_file returns True if saved, False if duplicate, invalid, or error
+            # This logic needs refinement if save_rule_to_file doesn't distinguish duplicate vs error
+            # Assuming save_rule_to_file now only returns True on actual addition.
+            # The memory_logic.py version of save_rule_to_file returns False for duplicates already.
+            added_count +=1
+        else:
+            # Check if it was a duplicate to categorize skipped_count, otherwise it's an error_count
+            # This requires knowing if the rule format was valid first. save_rule_to_file does this.
+            # If save_rule_to_file returned False, it means it was either a duplicate, invalid format, or save error.
+            # To simplify UI feedback, we'll just count as 'not added'.
+            # More detailed feedback would require save_rule_to_file to return distinct status codes.
+            # For now, if it wasn't added, and we can't easily tell if it was a duplicate vs format error here,
+            # let's assume it's an error for this counter, as save_rule_to_file already logs duplicates.
+            is_valid_format = bool(re.match(r"\[(CORE_RULE|RESPONSE_PRINCIPLE|BEHAVIORAL_ADJUSTMENT|GENERAL_LEARNING)\|([\d\.]+?)\](.*)", rule_text, re.I|re.DOTALL))
+            if not is_valid_format: # It was a format error
+                 error_count += 1
+            else: # It was a duplicate (already logged by save_rule_to_file) or a file write error
+                 # if rule_text in load_rules_from_file(): # this check is redundant if save_rule_to_file does it
+                 #    skipped_count +=1
+                 # else: # True error
+                 #    error_count += 1
+                 # To avoid re-loading rules file constantly, rely on save_rule_to_file's logging.
+                 # If it returns False and it wasn't added, it's complex to distinguish here without more info.
+                 # Let's simplify: if not added, increment a general "not_added_or_error" count.
+                 # The current save_rule_to_file will log "already exists". We need to avoid double counting.
+                 # Solution: save_rule_to_file should be the sole decider.
+                 # Let's adjust how we count based on a hypothetical enhanced save_rule_to_file:
+                 # result = save_rule_to_file_detailed(rule_text)
+                 # if result == "added": added_count += 1
+                 # elif result == "duplicate": skipped_count += 1
+                 # elif result == "invalid_format": error_count += 1
+                 # else: error_count +=1 # file write error
+                 # For now, with current save_rule_to_file:
+                 # The logic in save_rule_to_file already prevents adding duplicates and invalid formats.
+                 # So, if save_rule_to_file returns False, it means it was a duplicate, invalid, or actual save error.
+                 # We need better differentiation for accurate counts.
+                 # This simple count will be approximate for "skipped" vs "error".
+                 # The provided save_rule_to_file handles duplicates and format errors internally and returns False.
+                 # We can't easily tell which from just "False" here.
+                 # So we'll assume False means it wasn't added for *some* reason (duplicate, format, error).
+                 pass # The `save_rule_to_file` will log the reason if it's a duplicate or format error.
+                      # If it's a true file write error, that's also logged.
+                      # For UI, we can only show how many were *successfully* added.
         progress((idx + 1) / total_to_process, desc=f"Processed {idx+1}/{total_to_process} rules...")
 
-    msg = f"Rules Upload: Processed {total_to_process}. Added: {added_count}, Skipped (duplicates): {skipped_count}, Errors/Not Added: {error_count}."
+    # After loop, get final counts for summary.
+    # This means we must load rules again to find out how many are new vs existing for skipped_count.
+    # This is inefficient. A better `save_rule_to_file` that returns status is needed for good UI feedback.
+    # For now, the UI feedback will be simpler:
+    final_rules_list = load_rules_from_file()
+    actual_added_this_run = added_count # This count is based on `save_rule_to_file` returning True only for new, valid rules.
+
+    msg = f"Rules Upload: Attempted {total_to_process}. Successfully added: {actual_added_this_run}. Others skipped (duplicates/invalid format/errors - check logs)."
     logger.info(msg)
     return msg
 
@@ -761,17 +855,11 @@ def ui_upload_memories_action(uploaded_file_obj, progress=gr.Progress()):
     if not uploaded_file_obj: return "No file provided for memories upload."
 
     try:
-        content = uploaded_file_obj.decode('utf-8')
-    except AttributeError:
-        try:
-            with open(uploaded_file_obj.name, 'r', encoding='utf-8') as f:
-                content = f.read()
-        except Exception as e_read:
-            logger.error(f"Error reading uploaded memories file: {e_read}")
-            return f"Error reading file: {e_read}"
-    except Exception as e_decode:
-        logger.error(f"Error decoding uploaded memories file: {e_decode}")
-        return f"Error decoding file content: {e_decode}"
+        with open(uploaded_file_obj.name, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except Exception as e_read:
+        logger.error(f"Error reading uploaded memories file: {e_read}")
+        return f"Error reading file: {e_read}"
 
     if not content.strip(): return "Uploaded memories file is empty."
 
@@ -800,17 +888,33 @@ def ui_upload_memories_action(uploaded_file_obj, progress=gr.Progress()):
         return "No valid memory objects found in the uploaded file."
 
     total_to_process = len(memory_objects_to_process)
+    if total_to_process == 0: return "No memory objects to process after parsing."
     progress(0, desc="Starting memories upload...")
 
+    # Duplicate check for memories is harder without unique IDs.
+    # `save_memory_to_file` currently just appends.
+    # For simplicity, this upload will append all validly formatted memories.
+    # A more robust system would check for duplicates based on content signature or timestamp.
+    existing_memories_timestamps_content = set() # For basic duplicate check in this upload session
+    # Pre-load existing to avoid adding true duplicates if possible
+    # for mem in load_memories_from_file():
+    #    existing_memories_timestamps_content.add((mem.get("timestamp"), mem.get("user_input"), mem.get("bot_response")))
+
+
     for idx, mem_data in enumerate(memory_objects_to_process):
-        if not isinstance(mem_data, dict) or not all(k in mem_data for k in ["user_input", "bot_response", "metrics"]): # Timestamp optional for upload
+        if not isinstance(mem_data, dict) or not all(k in mem_data for k in ["user_input", "bot_response", "metrics"]): # Timestamp is good to have but save_memory_to_file adds it if missing
             format_error_count += 1
             continue
         
-        # For file-based, duplicate check on save might be too slow.
-        # memory_logic.save_memory_to_file just appends.
+        # Simple duplicate check for this session (not against all stored if large)
+        # current_sig = (mem_data.get("timestamp"), mem_data.get("user_input"), mem_data.get("bot_response"))
+        # if current_sig in existing_memories_timestamps_content:
+        #     # This is a duplicate from the file itself or already in store
+        #     continue # Skip adding
+
         if save_memory_to_file(mem_data["user_input"], mem_data["bot_response"], mem_data["metrics"]):
             added_count += 1
+            # existing_memories_timestamps_content.add(current_sig) # Add to session set
         else:
             save_error_count += 1 # Error during file write
         progress((idx + 1) / total_to_process, desc=f"Processed {idx+1}/{total_to_process} memories...")
@@ -974,7 +1078,8 @@ if __name__ == "__main__":
     # No explicit data loading into globals needed here if handlers load on demand.
 
     app_port = int(os.getenv("GRADIO_PORT", 7860))
-    app_server_name = os.getenv("GRADIO_SERVER_NAME", "0.0.0.0")
+    # Default to 127.0.0.1 for local access unless 0.0.0.0 is explicitly set for broader network access
+    app_server_name = os.getenv("GRADIO_SERVER_NAME", "127.0.0.1") 
     app_debug_mode = os.getenv("GRADIO_DEBUG", "False").lower() == "true"
     app_share_mode = os.getenv("GRADIO_SHARE", "False").lower() == "true"