Update app.py

app.py

@@ -18,7 +18,7 @@ from memory_logic import (
     initialize_memory_system,
     add_memory_entry, retrieve_memories_semantic, get_all_memories_cached, clear_all_memory_data_backend,
     add_rule_entry, retrieve_rules_semantic, remove_rule_entry, get_all_rules_cached, clear_all_rules_data_backend,
-    save_faiss_indices_to_disk, STORAGE_BACKEND as MEMORY_STORAGE_BACKEND # Import for UI
+    save_faiss_indices_to_disk, STORAGE_BACKEND as MEMORY_STORAGE_BACKEND
 )
 from websearch_logic import scrape_url, search_and_scrape_duckduckgo, search_and_scrape_google
 
@@ -156,7 +156,7 @@ def process_user_interaction_gradio(user_input: str, provider_name: str, model_d
             yield "status", "<i>[Synthesizing web report...]</i>"
             final_system_prompt_str += " Generate report/answer from web content, history, & guidelines. Cite URLs as [Source X]."
             final_user_prompt_content_str = f"History:\n{history_str_for_prompt}\nGuidelines:\n{initial_insights_ctx_str}\nWeb Content:\n{scraped_content}\nQuery: \"{user_input}\"\nReport/Response (cite sources [Source X]):"
-    else: # Fallback
+    else: 
         final_system_prompt_str += " Respond directly (unknown action path)."
         final_user_prompt_content_str = f"History:\n{history_str_for_prompt}\nGuidelines:\n{initial_insights_ctx_str}\nQuery: \"{user_input}\"\nResponse:"
     final_llm_messages = [{"role": "system", "content": final_system_prompt_str}, {"role": "user", "content": final_user_prompt_content_str}]
@@ -182,8 +182,49 @@ def deferred_learning_and_memory_task(user_input: str, bot_response: str, provid
         add_memory_entry(user_input, metrics, bot_response)
         summary = f"User:\"{user_input}\"\nAI:\"{bot_response}\"\nMetrics(takeaway):{metrics.get('takeaway','N/A')},Success:{metrics.get('response_success_score','N/A')}"
         existing_rules_ctx = "\n".join([f"- \"{r}\"" for r in retrieve_rules_semantic(f"{summary}\n{user_input}", k=10)]) or "No existing rules context."
-        insight_sys_prompt = """You are an expert AI knowledge base curator... (Your full long prompt from ai-learn)... Output ONLY JSON list.""" # Ensure this is the FULL prompt
-        insight_user_prompt = f"""Interaction Summary:\n{summary}\nRelevant Existing Rules:\n{existing_rules_ctx}\nConsidered Principles:\n{json.dumps([p['original'] for p in insights_reflected if 'original' in p]) if insights_reflected else "None"}\nTask: Generate JSON list of add/update operations... (Full task description from ai-learn)"""
+        insight_sys_prompt = """You are an expert AI knowledge base curator. Your primary function is to meticulously analyze an interaction and update the AI's guiding principles (insights/rules) to improve its future performance and self-understanding.
+You MUST output a JSON list of operation objects. This list can and SHOULD contain MULTIPLE distinct operations if various learnings occurred.
+Each operation object in the JSON list must have:
+1. "action": A string, either "add" (for entirely new rules) or "update" (to replace an existing rule with a better one).
+2. "insight": A string, the full, refined insight text including its [TYPE|SCORE] prefix (e.g., "[CORE_RULE|1.0] My name is Lumina, an AI assistant.").
+3. "old_insight_to_replace" (ONLY for "update" action): A string, the *exact, full text* of an existing insight that the new "insight" should replace.
+**Your Reflection Process (Consider each step and generate operations accordingly):**
+**STEP 1: Core Identity & Purpose Review (Result: Primarily 'update' operations)**
+   - Examine all `CORE_RULE`s related to my identity (name, fundamental purpose, core unchanging capabilities, origin) from the "Potentially Relevant Existing Rules".
+   - **CONSOLIDATE & MERGE:** If multiple `CORE_RULE`s state similar aspects (e.g., multiple name declarations like 'Lumina' and 'LearnerAI', or slightly different purpose statements), you MUST merge them into ONE definitive, comprehensive `CORE_RULE`.
+   - The new "insight" will be this single, merged rule. Propose separate "update" operations to replace *each* redundant or less accurate core identity rule with this new canonical one.
+   - Prioritize user-assigned names or the most specific, recently confirmed information. If the interaction summary clarifies a name or core function, ensure this is reflected.
+**STEP 2: New Distinct Learnings (Result: Primarily 'add' operations)**
+   - Did I learn any completely new, distinct facts (e.g., "The user's project is codenamed 'Bluefire'")?
+   - Did I demonstrate or get told about a new skill/capability not previously documented (e.g., "I can now generate mermaid diagrams based on descriptions")?
+   - Did the user express a strong, general preference that should guide future interactions (e.g., "User prefers responses to start with a direct answer, then explanation")?
+   - For these, propose 'add' operations. Assign `CORE_RULE` for truly fundamental new facts/capabilities, otherwise `RESPONSE_PRINCIPLE` or `BEHAVIORAL_ADJUSTMENT`. Ensure these are genuinely NEW and not just rephrasing of existing non-core rules.
+**STEP 3: Refinements to Existing Behaviors/Principles (Result: 'update' operations for non-core rules)**
+   - Did I learn to modify or improve an existing behavior, response style, or operational guideline (that is NOT part of core identity)?
+   - For example, if an existing `RESPONSE_PRINCIPLE` was "Be formal," and the interaction showed the user prefers informality, update that principle.
+   - Propose 'update' operations for the relevant `RESPONSE_PRINCIPLE` or `BEHAVIORAL_ADJUSTMENT`. Only update if the change is significant.
+**General Guidelines:**
+- If no new insights, updates, or consolidations are warranted from the interaction, output an empty JSON list: `[]`.
+- Ensure the "insight" field (for both add/update) always contains the properly formatted insight string: `[TYPE|SCORE] Text`. TYPE can be `CORE_RULE`, `RESPONSE_PRINCIPLE`, `BEHAVIORAL_ADJUSTMENT`. Scores should reflect confidence/importance.
+- Be precise with "old_insight_to_replace" – it must *exactly* match an existing rule string from the "Potentially Relevant Existing Rules" context.
+- Aim for a comprehensive set of operations that reflects ALL key learnings from the interaction.
+- Output ONLY the JSON list. No other text, explanations, or markdown.
+**Example of a comprehensive JSON output with MULTIPLE operations:**
+[
+  {"action": "update", "old_insight_to_replace": "[CORE_RULE|1.0] My designated name is 'LearnerAI'.", "insight": "[CORE_RULE|1.0] I am Lumina, an AI assistant designed to chat, provide information, and remember context like the secret word 'rocksyrup'."},
+  {"action": "update", "old_insight_to_replace": "[CORE_RULE|1.0] I'm Lumina, the AI designed to chat with you.", "insight": "[CORE_RULE|1.0] I am Lumina, an AI assistant designed to chat, provide information, and remember context like the secret word 'rocksyrup'."},
+  {"action": "add", "insight": "[CORE_RULE|0.9] I am capable of searching the internet for current weather information if asked."},
+  {"action": "add", "insight": "[RESPONSE_PRINCIPLE|0.8] When user provides positive feedback, acknowledge it warmly."},
+  {"action": "update", "old_insight_to_replace": "[RESPONSE_PRINCIPLE|0.7] Avoid mentioning old conversations.", "insight": "[RESPONSE_PRINCIPLE|0.85] Avoid mentioning old conversations unless the user explicitly refers to them or it's highly relevant to the current query."}
+]"""
+        insight_user_prompt = f"""Interaction Summary:\n{summary}\n
+Potentially Relevant Existing Rules (Review these carefully for consolidation or refinement):\n{existing_rules_ctx}\n
+Guiding principles that were considered during THIS interaction (these might offer clues for new rules or refinements):\n{json.dumps([p['original'] for p in insights_reflected if 'original' in p]) if insights_reflected else "None"}\n
+Task: Based on your three-step reflection process (Core Identity, New Learnings, Refinements):
+1.  **Consolidate CORE_RULEs:** Merge similar identity/purpose rules from "Potentially Relevant Existing Rules" into single, definitive statements using "update" operations. Replace multiple old versions with the new canonical one.
+2.  **Add New Learnings:** Identify and "add" any distinct new facts, skills, or important user preferences learned from the "Interaction Summary".
+3.  **Update Existing Principles:** "Update" any non-core principles from "Potentially Relevant Existing Rules" if the "Interaction Summary" provided a clear refinement.
+Combine all findings into a single JSON list of operations. If there are multiple distinct changes based on the interaction and existing rules, ensure your list reflects all of them. Output JSON only."""
         insight_msgs = [{"role":"system", "content":insight_sys_prompt}, {"role":"user", "content":insight_user_prompt}]
         insight_prov, insight_model_disp = provider, model_disp_name
         insight_env_model = os.getenv("INSIGHT_MODEL_OVERRIDE")
@@ -264,56 +305,66 @@ def handle_gradio_chat_submit(user_msg_txt: str, gr_hist_list: list, sel_prov_na
         hist_len_check = MAX_HISTORY_TURNS * 2
         if current_chat_session_history and current_chat_session_history[0]["role"] == "system": hist_len_check +=1
         if len(current_chat_session_history) > hist_len_check:
-            current_chat_session_history = ([current_chat_session_history[0]] if current_chat_session_history[0]["role"] == "system" else []) + current_chat_session_history[- (MAX_HISTORY_TURNS * 2):]
+            current_chat_session_history = ([current_chat_session_history[0]] if current_chat_session_history[0]["role"] == "system" else []) + current_chat_session_history[-(MAX_HISTORY_TURNS * 2):]
         threading.Thread(target=deferred_learning_and_memory_task, args=(user_msg_txt, final_bot_resp_acc, sel_prov_name, sel_model_disp_name, insights_used_parsed, ui_api_key.strip() if ui_api_key else None), daemon=True).start()
         status_txt = "Response complete. Background learning initiated."
     else: status_txt = "Processing finished; no response or error."
     yield (cleared_input, updated_gr_hist, status_txt, def_detect_out_md, def_fmt_out_txt, def_dl_btn)
 
 def ui_view_rules_action_fn(): return "\n\n---\n\n".join(get_all_rules_cached()) or "No rules found."
-def ui_upload_rules_action_fn(file_obj, progress=gr.Progress()):
-    if not file_obj: return "No file."
-    try: content = open(file_obj.name, 'r', encoding='utf-8').read()
-    except Exception as e: return f"Error reading file: {e}"
-    if not content.strip(): return "File empty."
+def ui_upload_rules_action_fn(uploaded_file_obj, progress=gr.Progress()):
+    if not uploaded_file_obj: return "No file provided for rules upload."
+    try:
+        with open(uploaded_file_obj.name, 'r', encoding='utf-8') as f: content = f.read()
+    except Exception as e_read: return f"Error reading file: {e_read}"
+    if not content.strip(): return "Uploaded rules file is empty."
+    added_count, skipped_count, error_count = 0,0,0
     potential_rules = content.split("\n\n---\n\n")
     if len(potential_rules) == 1 and "\n" in content: potential_rules = [r.strip() for r in content.splitlines() if r.strip()]
-    if not potential_rules: return "No rules found in file."
-    added, skipped, errors = 0,0,0; total = len(potential_rules)
+    total_to_process = len(potential_rules)
+    if total_to_process == 0: return "No rules found in file to process."
+    progress(0, desc="Starting rules upload...")
     for idx, rule_text in enumerate(potential_rules):
-        if not rule_text.strip(): continue
-        success, status = add_rule_entry(rule_text.strip())
-        if success: added +=1
-        elif status == "duplicate": skipped +=1
-        else: errors +=1
-        progress((idx+1)/total, desc=f"Processing {idx+1}/{total} rules...")
-    return f"Rules Upload: Total {total}. Added: {added}, Skipped (duplicates): {skipped}, Errors/Invalid: {errors}."
+        rule_text = rule_text.strip()
+        if not rule_text: continue
+        success, status_msg = add_rule_entry(rule_text)
+        if success: added_count += 1
+        elif status_msg == "duplicate": skipped_count += 1
+        else: error_count += 1
+        progress((idx + 1) / total_to_process, desc=f"Processed {idx+1}/{total_to_process} rules...")
+    msg = f"Rules Upload: Processed {total_to_process}. Added: {added_count}, Skipped (duplicates): {skipped_count}, Errors/Invalid: {error_count}."
+    logger.info(msg); return msg
 
 def ui_view_memories_action_fn(): return get_all_memories_cached() or []
-def ui_upload_memories_action_fn(file_obj, progress=gr.Progress()):
-    if not file_obj: return "No file."
-    try: content = open(file_obj.name, 'r', encoding='utf-8').read()
-    except Exception as e: return f"Error reading file: {e}"
-    if not content.strip(): return "File empty."
-    mem_objs, fmt_errors, added, save_errors = [], 0,0,0
+def ui_upload_memories_action_fn(uploaded_file_obj, progress=gr.Progress()):
+    if not uploaded_file_obj: return "No file provided for memories upload."
+    try:
+        with open(uploaded_file_obj.name, 'r', encoding='utf-8') as f: content = f.read()
+    except Exception as e_read: return f"Error reading file: {e_read}"
+    if not content.strip(): return "Uploaded memories file is empty."
+    added_count, format_error_count, save_error_count = 0,0,0
+    memory_objects_to_process = []
     try:
-        parsed = json.loads(content)
-        mem_objs = parsed if isinstance(parsed, list) else [parsed]
+        parsed_json = json.loads(content)
+        memory_objects_to_process = parsed_json if isinstance(parsed_json, list) else [parsed_json]
     except json.JSONDecodeError:
         for line in content.splitlines():
             if line.strip():
-                try: mem_objs.append(json.loads(line))
-                except: fmt_errors+=1
-    if not mem_objs and fmt_errors == 0: return "No valid memories in file."
-    total = len(mem_objs)
-    for idx, mem_data in enumerate(mem_objs):
+                try: memory_objects_to_process.append(json.loads(line))
+                except: format_error_count += 1
+    if not memory_objects_to_process and format_error_count == 0: return "No valid memory objects found."
+    total_to_process = len(memory_objects_to_process)
+    if total_to_process == 0: return "No memory objects to process."
+    progress(0, desc="Starting memories upload...")
+    for idx, mem_data in enumerate(memory_objects_to_process):
         if isinstance(mem_data, dict) and all(k in mem_data for k in ["user_input", "bot_response", "metrics"]):
             success, _ = add_memory_entry(mem_data["user_input"], mem_data["metrics"], mem_data["bot_response"])
-            if success: added +=1
-            else: save_errors +=1
-        else: fmt_errors +=1
-        progress((idx+1)/total, desc=f"Processing {idx+1}/{total} memories...")
-    return f"Memories Upload: Total {total}. Added: {added}, Format Errors: {fmt_errors}, Save Errors: {save_errors}."
+            if success: added_count += 1
+            else: save_error_count += 1
+        else: format_error_count += 1
+        progress((idx + 1) / total_to_process, desc=f"Processed {idx+1}/{total_to_process} memories...")
+    msg = f"Memories Upload: Processed {total_to_process}. Added: {added_count}, Format Errors: {format_error_count}, Save Errors: {save_error_count}."
+    logger.info(msg); return msg
 
 custom_theme = gr.themes.Base(primary_hue="teal", secondary_hue="purple", neutral_hue="zinc", text_size="sm", spacing_size="sm", radius_size="sm", font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"])
 custom_css = """ body { font-family: 'Inter', sans-serif; } .gradio-container { max-width: 96% !important; margin: auto !important; padding-top: 1rem !important; } footer { display: none !important; } .gr-button { white-space: nowrap; } .gr-input, .gr-textarea textarea, .gr-dropdown input { border-radius: 8px !important; } .gr-chatbot .message { border-radius: 10px !important; box-shadow: 0 2px 5px rgba(0,0,0,0.08) !important; } .prose { h1 { font-size: 1.8rem; margin-bottom: 0.6em; margin-top: 0.8em; } h2 { font-size: 1.4rem; margin-bottom: 0.5em; margin-top: 0.7em; } h3 { font-size: 1.15rem; margin-bottom: 0.4em; margin-top: 0.6em; } p { margin-bottom: 0.8em; line-height: 1.65; } ul, ol { margin-left: 1.5em; margin-bottom: 0.8em; } code { background-color: #f1f5f9; padding: 0.2em 0.45em; border-radius: 4px; font-size: 0.9em; } pre > code { display: block; padding: 0.8em; overflow-x: auto; background-color: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px;}} .compact-group .gr-input-label, .compact-group .gr-dropdown-label { font-size: 0.8rem !important; padding-bottom: 2px !important;}"""
@@ -322,6 +373,7 @@ with gr.Blocks(theme=custom_theme, css=custom_css, title="AI Research Mega Agent
     gr.Markdown("# 🚀 AI Research Mega Agent (Advanced Memory & Dynamic Models)", elem_classes="prose")
     avail_provs, def_prov = get_available_providers(), get_available_providers()[0] if get_available_providers() else None
     def_models, def_model = get_model_display_names_for_provider(def_prov) if def_prov else [], get_default_model_display_name_for_provider(def_prov) if def_prov else None
+    save_faiss_ram_btn = None
     with gr.Row():
         with gr.Column(scale=1, min_width=320):
             gr.Markdown("## ⚙️ Configuration", elem_classes="prose")
@@ -334,7 +386,7 @@ with gr.Blocks(theme=custom_theme, css=custom_css, title="AI Research Mega Agent
                 with gr.Group(elem_classes="compact-group"):
                     gr.Markdown("### System Prompt", elem_classes="prose")
                     sys_prompt_tb = gr.Textbox(label="Custom System Prompt Base", lines=6, value=DEFAULT_SYSTEM_PROMPT, interactive=True)
-            with gr.Accordion("Knowledge Management (Backend: " + MEMORY_STORAGE_BACKEND + ")", open=False): # Show backend type
+            with gr.Accordion("Knowledge Management (Backend: " + MEMORY_STORAGE_BACKEND + ")", open=False):
                 gr.Markdown("### Rules (Insights)", elem_classes="prose"); view_rules_btn = gr.Button("View Rules"); upload_rules_fobj = gr.File(label="Upload Rules (.txt/.jsonl)", file_types=[".txt", ".jsonl"]); rules_stat_tb = gr.Textbox(label="Status", interactive=False, lines=2); clear_rules_btn = gr.Button("⚠️ Clear All Rules", variant="stop")
                 if MEMORY_STORAGE_BACKEND == "RAM": save_faiss_ram_btn = gr.Button("Save FAISS Indices (RAM Backend)")
                 gr.Markdown("### Memories", elem_classes="prose"); view_mems_btn = gr.Button("View Memories"); upload_mems_fobj = gr.File(label="Upload Memories (.jsonl)", file_types=[".jsonl"]); mems_stat_tb = gr.Textbox(label="Status", interactive=False, lines=2); clear_mems_btn = gr.Button("⚠️ Clear All Memories", variant="stop")
@@ -344,7 +396,7 @@ with gr.Blocks(theme=custom_theme, css=custom_css, title="AI Research Mega Agent
             agent_stat_tb = gr.Textbox(label="Agent Status", interactive=False, lines=1, value="Initializing...")
             with gr.Tabs():
                 with gr.TabItem("📝 Report/Output"): gr.Markdown("AI's full response/report.", elem_classes="prose"); fmt_report_tb = gr.Textbox(label="Output", lines=20, interactive=True, show_copy_button=True, value="*Responses appear here...*"); dl_report_btn = gr.DownloadButton(label="Download Report", interactive=False, visible=False)
-                with gr.TabItem("🔍 Details / Data"): gr.Markdown("Intermediate details, loaded data.", elem_classes="prose"); detect_out_md = gr.Markdown("*Insights used or details show here...*"); gr.HTML("<hr style='margin:1em 0;'>"); gr.Markdown("### Rules Viewer", elem_classes="prose"); rules_disp_ta = gr.TextArea(label="Rules Snapshot", lines=10, interactive=False); gr.HTML("<hr style='margin:1em 0;'>"); gr.Markdown("### Memories Viewer", elem_classes="prose"); mems_disp_json = gr.JSON(label="Memories Snapshot")
+                with gr.TabItem("🔍 Details / Data"): gr.Markdown("View intermediate details, loaded data.", elem_classes="prose"); detect_out_md = gr.Markdown("*Insights used or details show here...*"); gr.HTML("<hr style='margin:1em 0;'>"); gr.Markdown("### Rules Viewer", elem_classes="prose"); rules_disp_ta = gr.TextArea(label="Rules Snapshot", lines=10, interactive=False); gr.HTML("<hr style='margin:1em 0;'>"); gr.Markdown("### Memories Viewer", elem_classes="prose"); mems_disp_json = gr.JSON(label="Memories Snapshot")
     def dyn_upd_model_dd(sel_prov_dyn:str): models_dyn, def_model_dyn = get_model_display_names_for_provider(sel_prov_dyn), get_default_model_display_name_for_provider(sel_prov_dyn); return gr.Dropdown(choices=models_dyn, value=def_model_dyn, interactive=True)
     prov_sel_dd.change(fn=dyn_upd_model_dd, inputs=prov_sel_dd, outputs=model_sel_dd)
     chat_ins = [user_msg_tb, main_chat_disp, prov_sel_dd, model_sel_dd, api_key_tb, sys_prompt_tb]
@@ -353,7 +405,9 @@ with gr.Blocks(theme=custom_theme, css=custom_css, title="AI Research Mega Agent
     view_rules_btn.click(fn=ui_view_rules_action_fn, outputs=rules_disp_ta)
     upload_rules_fobj.upload(fn=ui_upload_rules_action_fn, inputs=[upload_rules_fobj], outputs=[rules_stat_tb], show_progress="full").then(fn=ui_view_rules_action_fn, outputs=rules_disp_ta)
     clear_rules_btn.click(fn=lambda: "All rules cleared." if clear_all_rules_data_backend() else "Error clearing rules.", outputs=rules_stat_tb).then(fn=ui_view_rules_action_fn, outputs=rules_disp_ta)
-    if MEMORY_STORAGE_BACKEND == "RAM": save_faiss_ram_btn.click(fn=save_faiss_indices_to_disk, outputs=None, success_message="Attempted to save FAISS indices to disk.")
+    if MEMORY_STORAGE_BACKEND == "RAM" and save_faiss_ram_btn is not None:
+        def save_faiss_action_with_feedback_fn(): save_faiss_indices_to_disk(); gr.Info("Attempted to save FAISS indices to disk.")
+        save_faiss_ram_btn.click(fn=save_faiss_action_with_feedback_fn, inputs=None, outputs=None)
     view_mems_btn.click(fn=ui_view_memories_action_fn, outputs=mems_disp_json)
     upload_mems_fobj.upload(fn=ui_upload_memories_action_fn, inputs=[upload_mems_fobj], outputs=[mems_stat_tb], show_progress="full").then(fn=ui_view_memories_action_fn, outputs=mems_disp_json)
     clear_mems_btn.click(fn=lambda: "All memories cleared." if clear_all_memory_data_backend() else "Error clearing memories.", outputs=mems_stat_tb).then(fn=ui_view_memories_action_fn, outputs=mems_disp_json)