Update app.py

app.py

@@ -177,6 +177,56 @@ def process_user_interaction_gradio(user_input: str, provider_name: str, model_d
     logger.info(f"PUI_GRADIO [{request_id}]: Finished. Total: {time.time() - process_start_time:.2f}s. Resp len: {len(final_bot_text)}")
     yield "final_response_and_insights", {"response": final_bot_text, "insights_used": parsed_initial_insights_list}
 
+def repair_json_string_newslines(json_like_string: str) -> str:
+    """
+    Attempts to fix unescaped literal newlines within string literals in a JSON-like string.
+    This is a heuristic and focuses on the most common LLM error for this task.
+    It converts literal \n, \r, \r\n characters found *inside* what it
+    determines to be string literals into the two-character sequence "\\n".
+    """
+    output = []
+    i = 0
+    n = len(json_like_string)
+    in_string_literal = False
+    
+    while i < n:
+        char = json_like_string[i]
+        
+        if char == '"':
+            # Check if this quote is escaped (preceded by an odd number of backslashes)
+            num_backslashes = 0
+            k = i - 1
+            while k >= 0 and json_like_string[k] == '\\':
+                num_backslashes += 1
+                k -= 1
+            
+            if num_backslashes % 2 == 0: # This quote is not escaped, it's a delimiter
+                in_string_literal = not in_string_literal
+            output.append(char)
+            i += 1
+            continue
+
+        if in_string_literal:
+            if char == '\n': # Literal newline
+                output.append('\\\\n') # Append literal backslash then 'n'
+            elif char == '\r':
+                if i + 1 < n and json_like_string[i+1] == '\n': # CRLF
+                    output.append('\\\\n')
+                    i += 1 # Consume the \n as well, it's part of the CRLF pair
+                else: # Standalone CR
+                    output.append('\\\\n')
+            # We are not attempting to fix other escape issues here,
+            # as newlines are the primary cause of "Unterminated string".
+            # If we see a backslash, we assume it's either a correct escape
+            # or part of content that doesn't break basic parsing like a raw newline does.
+            else:
+                output.append(char)
+        else: # Not in string literal
+            output.append(char)
+        i += 1
+            
+    return "".join(output)
+
 def deferred_learning_and_memory_task(user_input: str, bot_response: str, provider: str, model_disp_name: str, insights_reflected: list[dict], api_key_override: str = None):
     start_time, task_id = time.time(), os.urandom(4).hex()
     logger.info(f"DEFERRED [{task_id}]: START User='{user_input[:40]}...', Bot='{bot_response[:40]}...'")
@@ -186,49 +236,48 @@ def deferred_learning_and_memory_task(user_input: str, bot_response: str, provid
         add_memory_entry(user_input, metrics, bot_response)
         summary = f"User:\"{user_input}\"\nAI:\"{bot_response}\"\nMetrics(takeaway):{metrics.get('takeaway','N/A')},Success:{metrics.get('response_success_score','N/A')}"
         existing_rules_ctx = "\n".join([f"- \"{r}\"" for r in retrieve_rules_semantic(f"{summary}\n{user_input}", k=10)]) or "No existing rules context."
+        
         insight_sys_prompt = """You are an expert AI knowledge base curator. Your primary function is to meticulously analyze an interaction and update the AI's guiding principles (insights/rules) to improve its future performance and self-understanding.
 **CRITICAL OUTPUT REQUIREMENT: You MUST output a single, valid JSON list of operation objects.**
 This list can and SHOULD contain MULTIPLE distinct operations if various learnings occurred.
 If no operations are warranted, output an empty JSON list: `[]`.
 ABSOLUTELY NO other text, explanations, or markdown should precede or follow this JSON list.
+Your output will be directly parsed by Python's `json.loads()` function.
+
 Each operation object in the JSON list must have these keys and string values:
 1.  `"action"`: A string, either `"add"` (for entirely new rules) or `"update"` (to replace an existing rule with a better one).
 2.  `"insight"`: A string, the full, refined insight text including its `[TYPE|SCORE]` prefix (e.g., `"[CORE_RULE|1.0] My name is Lumina, an AI assistant."`).
 3.  `"old_insight_to_replace"`: (ONLY for `"update"` action) A string, the *exact, full text* of an existing insight that the new `"insight"` should replace. If action is `"add"`, this key should be omitted or its value should be `null` or an empty string.
-**CRITICAL JSON STRING FORMATTING RULES (for values of "insight" and "old_insight_to_replace"):**
+
+**ULTRA-CRITICAL JSON STRING FORMATTING RULES (ESPECIALLY FOR THE "insight" FIELD):**
 -   All string values MUST be enclosed in double quotes (`"`).
 -   Any literal double quote (`"`) character *within* the string content MUST be escaped as `\\"`.
 -   Any literal backslash (`\\`) character *within* the string content MUST be escaped as `\\\\`.
--   Any newline characters *within* the string content MUST be escaped as `\\n`. Avoid literal newlines in JSON string values; use `\\n` instead.
-    THIS IS ESPECIALLY IMPORTANT FOR THE 'insight' FIELD, as the insight text itself can be multi-line. Each internal newline in the original text MUST be represented as `\\n` in the JSON string value.
-    *Example of correctly escaped insight string in JSON:*
-    `"insight": "[RESPONSE_PRINCIPLE|0.8] User prefers concise answers, stating: \\"Just the facts!\\". Avoid verbose explanations unless asked.\\nFollow up with a question if appropriate."`
+-   **NEWLINES ARE THE #1 CAUSE OF ERRORS. READ CAREFULLY:**
+    -   If the *content* of your `insight` text (or any other string value) spans multiple lines OR contains newline characters for formatting, EACH such newline character *inside the string's content* MUST be represented as the two-character sequence: a backslash followed by an 'n' (i.e., `\\n`).
+    -   **DO NOT use a literal newline character (pressing Enter) inside a JSON string value.** This will break the JSON parser and result in an "Unterminated string" error.
+    -   **Example of BAD JSON (literal newline):**
+        `"insight": "This is line one.
+        This is line two."`
+        (This is INVALID because of the actual newline between "one." and "This". The parser stops reading the string at "one.")
+    -   **Example of GOOD JSON (escaped newline using `\\n`):**
+        `"insight": "This is line one.\\nThis is line two."`
+        (This is VALID. The `\\n` sequence correctly tells the parser there's a newline *within* the string's content.)
+-   The entire JSON response (the list `[...]`) can be pretty-printed with newlines *between* JSON elements (like after a comma or a brace) for readability, but *within* any quoted string value, the newline rule above is absolute.
+
 **Your Reflection Process (Consider each step and generate operations accordingly):**
-**STEP 1: Core Identity & Purpose Review (Result: Primarily 'update' operations)**
-   - Examine all `CORE_RULE`s related to my identity (name, fundamental purpose, core unchanging capabilities, origin) from the "Potentially Relevant Existing Rules".
-   - **CONSOLIDATE & MERGE:** If multiple `CORE_RULE`s state similar aspects (e.g., multiple name declarations like 'Lumina' and 'LearnerAI', or slightly different purpose statements), you MUST merge them into ONE definitive, comprehensive `CORE_RULE`.
-   - The new "insight" will be this single, merged rule. Propose separate "update" operations to replace *each* redundant or less accurate core identity rule with this new canonical one.
-   - Prioritize user-assigned names or the most specific, recently confirmed information. If the interaction summary clarifies a name or core function, ensure this is reflected.
-**STEP 2: New Distinct Learnings (Result: Primarily 'add' operations)**
-   - Did I learn any completely new, distinct facts (e.g., "The user's project is codenamed 'Bluefire'")?
-   - Did I demonstrate or get told about a new skill/capability not previously documented (e.g., "I can now generate mermaid diagrams based on descriptions")?
-   - Did the user express a strong, general preference that should guide future interactions (e.g., "User prefers responses to start with a direct answer, then explanation")?
-   - For these, propose 'add' operations. Assign `CORE_RULE` for truly fundamental new facts/capabilities, otherwise `RESPONSE_PRINCIPLE` or `BEHAVIORAL_ADJUSTMENT`. Ensure these are genuinely NEW and not just rephrasing of existing non-core rules.
-**STEP 3: Refinements to Existing Behaviors/Principles (Result: 'update' operations for non-core rules)**
-   - Did I learn to modify or improve an existing behavior, response style, or operational guideline (that is NOT part of core identity)?
-   - For example, if an existing `RESPONSE_PRINCIPLE` was "Be formal," and the interaction showed the user prefers informality, update that principle.
-   - Propose 'update' operations for the relevant `RESPONSE_PRINCIPLE` or `BEHAVIORAL_ADJUSTMENT`. Only update if the change is significant.
+... (rest of reflection process - STEP 1, STEP 2, STEP 3 - remains the same) ...
+
 **General Guidelines for Insight Content and Actions:**
-- Ensure the "insight" field (for both add/update) always contains the properly formatted insight string: `[TYPE|SCORE] Text`. `TYPE` can be `CORE_RULE`, `RESPONSE_PRINCIPLE`, `BEHAVIORAL_ADJUSTMENT`. Scores should reflect confidence/importance (0.0-1.0).
-- Be precise with "old_insight_to_replace" – it must *exactly* match an existing rule string from the "Potentially Relevant Existing Rules" context.
-- Aim for a comprehensive set of operations that reflects ALL key learnings from the interaction.
-**Example of a comprehensive JSON output with MULTIPLE operations (This is how your output should look):**
+... (General Guidelines remain the same) ...
+
+**Example of a comprehensive JSON output (Pay close attention to `\\n` for newlines within insight text if the insight content spans multiple lines):**
 [
   {"action": "update", "old_insight_to_replace": "[CORE_RULE|1.0] My designated name is 'LearnerAI'.", "insight": "[CORE_RULE|1.0] I am Lumina, an AI assistant designed to chat, provide information, and remember context like the secret word 'rocksyrup'."},
-  {"action": "update", "old_insight_to_replace": "[CORE_RULE|1.0] I'm Lumina, the AI designed to chat with you.", "insight": "[CORE_RULE|1.0] I am Lumina, an AI assistant designed to chat, provide information, and remember context like the secret word 'rocksyrup'."},
-  {"action": "add", "insight": "[CORE_RULE|0.9] I am capable of searching the internet for current weather information if asked."},
-  {"action": "add", "insight": "[RESPONSE_PRINCIPLE|0.8] When user provides positive feedback, acknowledge it warmly."},
-  {"action": "update", "old_insight_to_replace": "[RESPONSE_PRINCIPLE|0.7] Avoid mentioning old conversations.", "insight": "[RESPONSE_PRINCIPLE|0.85] Avoid mentioning old conversations unless the user explicitly refers to them or it's highly relevant to the current query."}
+  {"action": "update", "old_insight_to_replace": "[CORE_RULE|1.0] I'm Lumina, the AI designed to chat with you.", "insight": "[CORE_RULE|1.0] I am Lumina, an AI assistant designed to chat, provide information, and remember context like the secret word 'rocksyrup'.\\nMy purpose is to assist the user with research and tasks."},
+  {"action": "add", "insight": "[CORE_RULE|0.9] I am capable of searching the internet for current weather information if asked.\\nThis capability was confirmed on [date] based on user query regarding weather."},
+  {"action": "add", "insight": "[RESPONSE_PRINCIPLE|0.8] When user provides positive feedback like 'good job', acknowledge it warmly with phrases like 'Thank you!' or 'Glad I could help!'."},
+  {"action": "update", "old_insight_to_replace": "[RESPONSE_PRINCIPLE|0.7] Avoid mentioning old conversations.", "insight": "[RESPONSE_PRINCIPLE|0.85] Avoid mentioning old conversations unless the user explicitly refers to them or it's highly relevant to the current query.\\nThis rule was updated due to user preference for more contextual continuity when appropriate."}
 ]
 """
         insight_user_prompt = f"""Interaction Summary:\n{summary}\n
@@ -239,7 +288,7 @@ Task: Based on your three-step reflection process (Core Identity, New Learnings,
 2.  **Add New Learnings:** Identify and "add" any distinct new facts, skills, or important user preferences learned from the "Interaction Summary".
 3.  **Update Existing Principles:** "Update" any non-core principles from "Potentially Relevant Existing Rules" if the "Interaction Summary" provided a clear refinement.
 Combine all findings into a single JSON list of operations. If there are multiple distinct changes based on the interaction and existing rules, ensure your list reflects all of them. Output JSON only, adhering to all specified formatting rules.
-**ULTRA-IMPORTANT FINAL REMINDER: YOUR ENTIRE RESPONSE MUST BE A SINGLE, VALID JSON LIST. DOUBLE-CHECK ALL STRING VALUES FOR CORRECTLY ESCAPED NEWLINES (\\n) AND QUOTES (\\\") BEFORE OUTPUTTING. INVALID JSON WILL BE REJECTED.**
+**ULTRA-IMPORTANT FINAL REMINDER: YOUR ENTIRE RESPONSE MUST BE A SINGLE, VALID JSON LIST. DOUBLE-CHECK ALL STRING VALUES, ESPECIALLY THE 'insight' TEXT, FOR CORRECTLY ESCAPED NEWLINES (using the two characters `\\` and `n`, i.e., `\\n`) AND QUOTES (using `\\"`). AN UNESCAPED LITERAL NEWLINE CHARACTER (ASCII 10) WITHIN AN 'insight' STRING VALUE WILL BREAK THE JSON AND CAUSE THE UPDATE TO FAIL. BE METICULOUS. VALIDATE YOUR JSON STRUCTURE AND STRING CONTENTS BEFORE FINALIZING YOUR OUTPUT.**
 """
         insight_msgs = [{"role":"system", "content":insight_sys_prompt}, {"role":"user", "content":insight_user_prompt}]
         insight_prov, insight_model_disp = provider, model_disp_name
@@ -249,41 +298,72 @@ Combine all findings into a single JSON list of operations. If there are multipl
             i_d_n = next((dn for dn, mid in MODELS_BY_PROVIDER.get(i_p.lower(), {}).get("models", {}).items() if mid == i_id), None)
             if i_d_n: insight_prov, insight_model_disp = i_p, i_d_n
         logger.info(f"DEFERRED [{task_id}]: Generating insights with {insight_prov}/{insight_model_disp}")
-        raw_ops_json = "".join(list(call_model_stream(provider=insight_prov, model_display_name=insight_model_disp, messages=insight_msgs, api_key_override=api_key_override, temperature=0.05, max_tokens=2000))).strip()
+        
+        raw_ops_json_full = "".join(list(call_model_stream(provider=insight_prov, model_display_name=insight_model_disp, messages=insight_msgs, api_key_override=api_key_override, temperature=0.0, max_tokens=2000))).strip()
         ops, processed_count = [], 0
-        json_match_ops = re.search(r"```json\s*(\[.*?\])\s*```", raw_ops_json, re.DOTALL|re.I) or re.search(r"(\[.*?\])", raw_ops_json, re.DOTALL)
         
+        json_match_ops = re.search(r"```json\s*(\[.*?\])\s*```", raw_ops_json_full, re.DOTALL|re.I) or \
+                         re.search(r"(\[.*?\])", raw_ops_json_full, re.DOTALL)
+
+        ops_json_str = None
         if json_match_ops:
+            ops_json_str = json_match_ops.group(1)
             try: 
-                ops_json_str = json_match_ops.group(1)
                 ops = json.loads(ops_json_str)
-            except Exception as e: 
-                logger.error(f"DEFERRED [{task_id}]: JSON ops parse error: {e}. Raw content parsed: {ops_json_str[:500]}")
-        else:
-             logger.info(f"DEFERRED [{task_id}]: No JSON list structure found in LLM output. Raw output (first 500 chars): {raw_ops_json[:500]}")
+            except json.JSONDecodeError as e: 
+                error_char_pos = e.pos
+                context_window = 40
+                start_context = max(0, error_char_pos - context_window)
+                end_context = min(len(ops_json_str), error_char_pos + context_window)
+                problem_context = ops_json_str[start_context:end_context].replace("\n", "\\n") # Show newlines escaped in log
+                
+                logger.warning(
+                    f"DEFERRED [{task_id}]: Initial JSON ops parse error: {e}. "
+                    f"Error at char {error_char_pos}. Context around error: '...{problem_context}...'. "
+                    f"Attempting to repair newlines in the JSON string."
+                )
+                # logger.debug(f"DEFERRED [{task_id}]: Problematic JSON string before repair:\n>>>>>>>>>>\n{ops_json_str}\n<<<<<<<<<<")
 
+                repaired_json_str = repair_json_string_newslines(ops_json_str)
+                # logger.debug(f"DEFERRED [{task_id}]: JSON string after repair attempt:\n>>>>>>>>>>\n{repaired_json_str}\n<<<<<<<<<<")
+                
+                try:
+                    ops = json.loads(repaired_json_str)
+                    logger.info(f"DEFERRED [{task_id}]: JSON successfully parsed after repair attempt.")
+                except json.JSONDecodeError as e2:
+                    logger.error(
+                        f"DEFERRED [{task_id}]: JSON ops parse error EVEN AFTER REPAIR: {e2}. "
+                        f"The repair attempt was not sufficient. Skipping insight operations for this turn."
+                    )
+                    # Log the string that failed after repair for further debugging
+                    # logger.error(f"DEFERRED [{task_id}]: Repaired JSON string that still failed:\n>>>>>>>>>>\n{repaired_json_str}\n<<<<<<<<<<")
+                    ops = [] 
+        else:
+             logger.info(f"DEFERRED [{task_id}]: No JSON list structure (e.g., starting with '[') found in LLM output. Full raw output:\n>>>>>>>>>>\n{raw_ops_json_full}\n<<<<<<<<<<")
+        
         if isinstance(ops, list) and ops:
-            logger.info(f"DEFERRED [{task_id}]: LLM provided {len(ops)} insight ops.")
+            logger.info(f"DEFERRED [{task_id}]: LLM provided {len(ops)} insight ops to process.")
             for op_idx, op in enumerate(ops):
                 if not isinstance(op, dict): 
                     logger.warning(f"DEFERRED [{task_id}]: Op {op_idx}: Skipped non-dict item in ops list: {op}")
                     continue
                 
                 action = op.get("action","").lower()
-                insight_text = op.get("insight","") # Keep as is initially for logging
-                old_insight = op.get("old_insight_to_replace","") # Keep as is initially
+                insight_text_raw = op.get("insight") 
+                old_insight_raw = op.get("old_insight_to_replace")
 
-                if not isinstance(insight_text, str):
-                    logger.warning(f"DEFERRED [{task_id}]: Op {op_idx}: Skipped op due to non-string insight_text: {op}")
+                if not isinstance(insight_text_raw, str):
+                    logger.warning(f"DEFERRED [{task_id}]: Op {op_idx}: Skipped op due to non-string insight_text (type: {type(insight_text_raw)}): {op}")
                     continue
-                insight_text = insight_text.strip() # Now strip
-
-                if not isinstance(old_insight, str) and old_insight is not None : # Allow None for old_insight
-                    logger.warning(f"DEFERRED [{task_id}]: Op {op_idx}: Skipped op due to non-string old_insight_to_replace: {op}")
-                    continue
-                if old_insight is not None:
-                    old_insight = old_insight.strip() # Strip if not None
-
+                insight_text = insight_text_raw.strip()
+
+                old_insight = None
+                if old_insight_raw is not None:
+                    if not isinstance(old_insight_raw, str):
+                        logger.warning(f"DEFERRED [{task_id}]: Op {op_idx}: Skipped op due to non-string/non-null old_insight_to_replace (type: {type(old_insight_raw)}): {op}")
+                        continue
+                    old_insight = old_insight_raw.strip()
+                
                 if not insight_text or not re.match(r"\[(CORE_RULE|RESPONSE_PRINCIPLE|BEHAVIORAL_ADJUSTMENT|GENERAL_LEARNING)\|([\d\.]+?)\]", insight_text, re.I|re.DOTALL): 
                     logger.warning(f"DEFERRED [{task_id}]: Op {op_idx}: Skipped op due to invalid/empty insight_text format: '{insight_text[:100]}...' from op: {op}")
                     continue
@@ -293,21 +373,25 @@ Combine all findings into a single JSON list of operations. If there are multipl
                     if success: processed_count +=1
                     else: logger.warning(f"DEFERRED [{task_id}]: Op {op_idx} (add): Failed to add rule '{insight_text[:50]}...'. Status: {status_msg}")
                 elif action == "update":
-                    if old_insight and old_insight != insight_text: 
-                        remove_success = remove_rule_entry(old_insight)
-                        if not remove_success:
-                             logger.warning(f"DEFERRED [{task_id}]: Op {op_idx} (update): Failed to remove old rule '{old_insight[:50]}...' before adding new.")
-                    success, status_msg = add_rule_entry(insight_text)
+                    if old_insight: 
+                        if old_insight != insight_text: 
+                            remove_success = remove_rule_entry(old_insight)
+                            if not remove_success:
+                                 logger.warning(f"DEFERRED [{task_id}]: Op {op_idx} (update): Failed to remove old rule '{old_insight[:50]}...' before adding new. This might lead to duplicates if the new rule is different.")
+                        else: 
+                             logger.info(f"DEFERRED [{task_id}]: Op {op_idx} (update): Old insight is identical to new insight. Skipping removal and effectively treating as 'add if not present'.")
+                    
+                    success, status_msg = add_rule_entry(insight_text) 
                     if success: processed_count +=1
-                    else: logger.warning(f"DEFERRED [{task_id}]: Op {op_idx} (update): Failed to add updated rule '{insight_text[:50]}...'. Status: {status_msg}")
+                    else: logger.warning(f"DEFERRED [{task_id}]: Op {op_idx} (update): Failed to add/update rule '{insight_text[:50]}...'. Status: {status_msg}")
                 else:
                     logger.warning(f"DEFERRED [{task_id}]: Op {op_idx}: Skipped op due to unknown action '{action}': {op}")
 
             logger.info(f"DEFERRED [{task_id}]: Processed {processed_count} insight ops out of {len(ops)} received.")
-        elif not json_match_ops : # Already logged if no JSON structure found
-            pass
-        else: # json_match_ops was true, but ops list is empty or not a list (e.g. parsing error led to ops not being a list)
-            logger.info(f"DEFERRED [{task_id}]: No valid list of insight ops from LLM after parsing. Raw match (first 500 chars): {json_match_ops.group(1)[:500] if json_match_ops else 'N/A'}")
+        elif not json_match_ops : 
+            pass 
+        else: 
+            logger.info(f"DEFERRED [{task_id}]: No valid list of insight ops from LLM after parsing was attempted. Raw match (ops_json_str - first 500 chars):\n>>>>>>>>>>\n{ops_json_str[:500] if ops_json_str else 'N/A'}\n<<<<<<<<<<")
             
     except Exception as e: logger.error(f"DEFERRED [{task_id}]: CRITICAL ERROR in deferred task: {e}", exc_info=True)
     logger.info(f"DEFERRED [{task_id}]: END. Total: {time.time() - start_time:.2f}s")
@@ -431,10 +515,33 @@ def ui_upload_rules_action_fn(uploaded_file_obj, progress=gr.Progress()):
     except Exception as e_read: return f"Error reading file: {e_read}"
     if not content.strip(): return "Uploaded rules file is empty."
     added_count, skipped_count, error_count = 0,0,0
-    potential_rules = content.split("\n\n---\n\n")
-    if len(potential_rules) == 1 and "\n" in content: potential_rules = [r.strip() for r in content.splitlines() if r.strip()]
+    # For .txt, split by '---'
+    if uploaded_file_obj.name.lower().endswith(".txt"):
+        potential_rules = content.split("\n\n---\n\n")
+        if len(potential_rules) == 1 and "\n" in content: # Fallback for simple newline separation in .txt
+             potential_rules = [r.strip() for r in content.splitlines() if r.strip()]
+    elif uploaded_file_obj.name.lower().endswith(".jsonl"):
+        potential_rules = []
+        for line in content.splitlines():
+            if line.strip():
+                try:
+                    # Expect each line to be a JSON string containing the rule text
+                    rule_text_in_json_string = json.loads(line)
+                    if isinstance(rule_text_in_json_string, str):
+                        potential_rules.append(rule_text_in_json_string)
+                    else:
+                        logger.warning(f"Rule Upload: Skipped non-string rule from JSONL: {rule_text_in_json_string}")
+                        error_count +=1
+                except json.JSONDecodeError:
+                    logger.warning(f"Rule Upload: Failed to parse JSONL line for rule: {line}")
+                    error_count +=1
+    else:
+        return "Unsupported file type for rules. Please use .txt or .jsonl."
+
+
     total_to_process = len(potential_rules)
-    if total_to_process == 0: return "No rules found in file to process."
+    if total_to_process == 0 and error_count == 0: return "No rules found in file to process."
+    
     progress(0, desc="Starting rules upload...")
     for idx, rule_text in enumerate(potential_rules):
         rule_text = rule_text.strip()
@@ -442,9 +549,10 @@ def ui_upload_rules_action_fn(uploaded_file_obj, progress=gr.Progress()):
         success, status_msg = add_rule_entry(rule_text)
         if success: added_count += 1
         elif status_msg == "duplicate": skipped_count += 1
-        else: error_count += 1
-        progress((idx + 1) / total_to_process, desc=f"Processed {idx+1}/{total_to_process} rules...")
-    msg = f"Rules Upload: Processed {total_to_process}. Added: {added_count}, Skipped (duplicates): {skipped_count}, Errors/Invalid: {error_count}."
+        else: error_count += 1 # Increment error count for add_rule_entry failures too
+        progress((idx + 1) / total_to_process if total_to_process > 0 else 1, desc=f"Processed {idx+1}/{total_to_process} rules...")
+    
+    msg = f"Rules Upload: Total lines/segments processed: {total_to_process}. Added: {added_count}, Skipped (duplicates): {skipped_count}, Errors/Invalid: {error_count}."
     logger.info(msg); return msg
 
 def ui_refresh_memories_display_fn(): return get_all_memories_cached() or [] 
@@ -480,22 +588,33 @@ def ui_upload_memories_action_fn(uploaded_file_obj, progress=gr.Progress()):
     if not content.strip(): return "Uploaded memories file is empty."
     added_count, format_error_count, save_error_count = 0,0,0
     memory_objects_to_process = []
-    try:
-        # Try parsing as a single JSON list first
-        parsed_json = json.loads(content)
-        if isinstance(parsed_json, list):
-            memory_objects_to_process = parsed_json
-        else: # If it's a single object, wrap it in a list
-            memory_objects_to_process = [parsed_json]
-    except json.JSONDecodeError: # If not a single JSON list, try JSONL
-        for line in content.splitlines():
+    
+    file_ext = os.path.splitext(uploaded_file_obj.name.lower())[1]
+
+    if file_ext == ".json":
+        try:
+            parsed_json = json.loads(content)
+            if isinstance(parsed_json, list):
+                memory_objects_to_process = parsed_json
+            elif isinstance(parsed_json, dict): # Single object
+                memory_objects_to_process = [parsed_json]
+            else:
+                format_error_count = 1 # Not a list or object
+        except json.JSONDecodeError:
+            format_error_count = 1 # Invalid JSON
+    elif file_ext == ".jsonl":
+        for line_num, line in enumerate(content.splitlines()):
             if line.strip():
-                try: memory_objects_to_process.append(json.loads(line))
-                except json.JSONDecodeError: format_error_count += 1
+                try: 
+                    memory_objects_to_process.append(json.loads(line))
+                except json.JSONDecodeError: 
+                    logger.warning(f"Memories Upload: JSONL line {line_num+1} parse error: {line[:100]}")
+                    format_error_count += 1
+    else:
+        return "Unsupported file type for memories. Please use .json or .jsonl."
                                 
-    if not memory_objects_to_process and format_error_count > 0 and not content.strip().startswith("["):
-         # If many format errors and not starting like a list, it was probably meant to be JSONL but all lines failed
-         return f"Memories Upload: File does not seem to be a valid JSON array or JSONL. Format errors on all lines attempted for JSONL."
+    if not memory_objects_to_process and format_error_count > 0 :
+         return f"Memories Upload: File parsing failed. Found {format_error_count} format errors."
     elif not memory_objects_to_process:
          return "No valid memory objects found in the uploaded file."
 
@@ -508,10 +627,12 @@ def ui_upload_memories_action_fn(uploaded_file_obj, progress=gr.Progress()):
             success, _ = add_memory_entry(mem_data["user_input"], mem_data["metrics"], mem_data["bot_response"])
             if success: added_count += 1
             else: save_error_count += 1
-        else: format_error_count += 1 # Count errors if structure is wrong even after successful line parse
+        else: 
+            logger.warning(f"Memories Upload: Skipped invalid memory object structure: {str(mem_data)[:100]}")
+            format_error_count += 1 
         progress((idx + 1) / total_to_process, desc=f"Processed {idx+1}/{total_to_process} memories...")
     
-    msg = f"Memories Upload: Processed {total_to_process} objects. Added: {added_count}, Format Errors (incl. structure issues): {format_error_count}, Save Errors: {save_error_count}."
+    msg = f"Memories Upload: Processed {total_to_process} objects. Added: {added_count}, Format/Structure Errors: {format_error_count}, Save Errors: {save_error_count}."
     logger.info(msg); return msg
 
 
@@ -614,7 +735,7 @@ with gr.Blocks(
                             rules_disp_ta = gr.TextArea(
                                 label="Current Rules (Read-only, Edit via Upload/Save)", lines=10, 
                                 placeholder="Rules will appear here. Use 'Save Edited Text' or 'Upload File' to modify.", 
-                                interactive=True # Keep interactive for copy-pasting, but primary edit via save button
+                                interactive=True 
                             )
                             gr.Markdown("To edit rules, modify the text above and click 'Save Edited Text', or upload a new file.")
                             save_edited_rules_btn = gr.Button("💾 Save Edited Text", variant="primary")
@@ -662,8 +783,8 @@ with gr.Blocks(
     user_msg_submit_event = user_msg_tb.submit(**chat_event_args)
 
     for event in [send_btn_click_event, user_msg_submit_event]:
-        event.then(fn=ui_refresh_rules_display_fn, inputs=None, outputs=rules_disp_ta)
-        event.then(fn=ui_refresh_memories_display_fn, inputs=None, outputs=mems_disp_json)
+        event.then(fn=ui_refresh_rules_display_fn, inputs=None, outputs=rules_disp_ta, show_progress=False)
+        event.then(fn=ui_refresh_memories_display_fn, inputs=None, outputs=mems_disp_json, show_progress=False)
 
     # Rules Management events
     dl_rules_btn.click(fn=ui_download_rules_action_fn, inputs=None, outputs=dl_rules_btn)
@@ -672,74 +793,49 @@ with gr.Blocks(
         if not edited_rules_text.strip():
             return "No rules text to save."
         
-        # Clear existing rules before adding from textarea to avoid duplicates if user edits and saves repeatedly
-        # Alternatively, implement a more sophisticated diff/update, but clearing and re-adding is simpler for now.
-        # For now, let's assume add_rule_entry handles duplicates by skipping.
-        # clear_all_rules_data_backend() # Option: clear all first
-        # logger.info("Cleared all rules before saving edited text.")
-
         potential_rules = edited_rules_text.split("\n\n---\n\n")
-        if len(potential_rules) == 1 and "\n" in edited_rules_text: # Fallback for simple newline separation
+        if len(potential_rules) == 1 and "\n" in edited_rules_text: 
             potential_rules = [r.strip() for r in edited_rules_text.splitlines() if r.strip()]
         
         if not potential_rules:
             return "No rules found to process from editor."
 
         added, skipped, errors = 0, 0, 0
-        total = len(potential_rules)
-        progress(0, desc=f"Saving {total} rules from editor...")
-        
-        # To prevent duplicates from the text area itself if it contained them,
-        # process unique rules from the input text.
         unique_rules_to_process = sorted(list(set(filter(None, [r.strip() for r in potential_rules]))))
         
-        # We need to identify rules to remove if they are no longer in the text area
-        # and rules to add if they are new. This is complex.
-        # Simpler: clear all existing rules, then add all from text area.
-        # This is destructive if the user only wanted to add/modify a few.
-        # For now, let's stick to `add_rule_entry` which skips duplicates.
-        # The user should be aware that "Save Edited Text" adds new rules from the text area
-        # and does not remove rules that are no longer present in the text area.
-        # To remove, they should use "Clear All Rules" or upload a new definitive list.
-
-        # For a "Save Edited Text" to truly reflect the text area as the source of truth:
-        # 1. Get current rules from text area.
-        # 2. Get rules currently in the system.
-        # 3. Rules to add = (rules in text area) - (rules in system)
-        # 4. Rules to remove = (rules in system) - (rules in text area)
-        # This is too complex for this iteration. The current add_rule_entry handles duplicates.
-        # Let's make `save_edited_rules_action_fn` clear and re-add for simplicity if that's desired.
-        # OR, it just adds new ones and updates existing ones if `add_rule_entry` can be made to update.
-        # Current `add_rule_entry` adds if not exact duplicate.
-        # Sticking to simpler "add if not present":
+        total_unique = len(unique_rules_to_process)
+        if total_unique == 0: return "No unique, non-empty rules found in editor text."
+
+        progress(0, desc=f"Saving {total_unique} unique rules from editor...")
         
         for idx, rule_text in enumerate(unique_rules_to_process):
-            success, status_msg = add_rule_entry(rule_text) # add_rule_entry handles duplicates
+            success, status_msg = add_rule_entry(rule_text) 
             if success: added += 1
             elif status_msg == "duplicate": skipped += 1
             else: errors += 1
-            progress((idx + 1) / len(unique_rules_to_process), desc=f"Processed {idx+1}/{len(unique_rules_to_process)} rules...")
+            progress((idx + 1) / total_unique, desc=f"Processed {idx+1}/{total_unique} rules...")
         
-        return f"Editor Save: Added: {added}, Skipped (duplicates): {skipped}, Errors/Invalid: {errors} from {len(unique_rules_to_process)} unique rules in text."
+        return f"Editor Save: Added: {added}, Skipped (duplicates): {skipped}, Errors/Invalid: {errors} from {total_unique} unique rules in text."
 
     save_edited_rules_btn.click(
         fn=save_edited_rules_action_fn,
         inputs=[rules_disp_ta],
         outputs=[rules_stat_tb],
         show_progress="full"
-    ).then(fn=ui_refresh_rules_display_fn, outputs=rules_disp_ta)
+    ).then(fn=ui_refresh_rules_display_fn, outputs=rules_disp_ta, show_progress=False)
 
     upload_rules_fobj.upload(
         fn=ui_upload_rules_action_fn,
         inputs=[upload_rules_fobj],
         outputs=[rules_stat_tb],
         show_progress="full"
-    ).then(fn=ui_refresh_rules_display_fn, outputs=rules_disp_ta)
+    ).then(fn=ui_refresh_rules_display_fn, outputs=rules_disp_ta, show_progress=False)
 
     clear_rules_btn.click(
         fn=lambda: ("All rules cleared." if clear_all_rules_data_backend() else "Error clearing rules."),
-        outputs=rules_stat_tb
-    ).then(fn=ui_refresh_rules_display_fn, outputs=rules_disp_ta)
+        outputs=rules_stat_tb,
+        show_progress=False
+    ).then(fn=ui_refresh_rules_display_fn, outputs=rules_disp_ta, show_progress=False)
 
     # Memories Management events
     dl_mems_btn.click(fn=ui_download_memories_action_fn, inputs=None, outputs=dl_mems_btn)
@@ -749,12 +845,13 @@ with gr.Blocks(
         inputs=[upload_mems_fobj],
         outputs=[mems_stat_tb],
         show_progress="full"
-    ).then(fn=ui_refresh_memories_display_fn, outputs=mems_disp_json)
+    ).then(fn=ui_refresh_memories_display_fn, outputs=mems_disp_json, show_progress=False)
 
     clear_mems_btn.click(
         fn=lambda: ("All memories cleared." if clear_all_memory_data_backend() else "Error clearing memories."),
-        outputs=mems_stat_tb
-    ).then(fn=ui_refresh_memories_display_fn, outputs=mems_disp_json)
+        outputs=mems_stat_tb,
+        show_progress=False
+    ).then(fn=ui_refresh_memories_display_fn, outputs=mems_disp_json, show_progress=False)
 
     if MEMORY_STORAGE_BACKEND == "RAM" and 'save_faiss_sidebar_btn' in locals():
         def save_faiss_action_with_feedback_sidebar_fn():
@@ -768,31 +865,28 @@ with gr.Blocks(
         backend_status = "AI Systems Initialized. Ready."
         rules_on_load = ui_refresh_rules_display_fn()
         mems_on_load = ui_refresh_memories_display_fn()
-        # Initial population of display areas
-        return backend_status, rules_on_load, mems_on_load, gr.Markdown(visible=False), gr.Textbox(value="*Waiting...*", interactive=True), gr.DownloadButton(interactive=False, value=None, visible=False)
-
-
-    # Ensure all outputs for app_load_fn match the number of output components
-    # chat_outs are: [user_msg_tb, main_chat_disp, agent_stat_tb, detect_out_md, fmt_report_tb, dl_report_btn]
-    # demo.load outputs are: [agent_stat_tb, rules_disp_ta, mems_disp_json]
-    # Let's adjust app_load_fn to return values for the components it's supposed to update.
+        return (
+            backend_status, 
+            rules_on_load, 
+            mems_on_load,
+            gr.Markdown(visible=False), 
+            gr.Textbox(value="*Waiting...*", interactive=True), 
+            gr.DownloadButton(interactive=False, value=None, visible=False)
+        )
     
     initial_load_outputs = [
         agent_stat_tb, 
         rules_disp_ta, 
         mems_disp_json,
-        # Outputs for chat area that might need reset/initial state, matching chat_outs if possible
-        # user_msg_tb (cleared), main_chat_disp (cleared), agent_stat_tb (updated by app_load)
-        # detect_out_md, fmt_report_tb, dl_report_btn
-        detect_out_md, # from chat_outs
-        fmt_report_tb, # from chat_outs
-        dl_report_btn  # from chat_outs
+        detect_out_md, 
+        fmt_report_tb, 
+        dl_report_btn  
     ]
     demo.load(fn=app_load_fn, inputs=None, outputs=initial_load_outputs)
 
 
 if __name__ == "__main__":
-    logger.info(f"Starting Gradio AI Research Mega Agent (v6.0 - JSON Fix & UI Upload Simplification, Memory: {MEMORY_STORAGE_BACKEND})...")
+    logger.info(f"Starting Gradio AI Research Mega Agent (v6.1 - JSON Repair & UI Polish, Memory: {MEMORY_STORAGE_BACKEND})...")
     app_port = int(os.getenv("GRADIO_PORT", 7860))
     app_server = os.getenv("GRADIO_SERVER_NAME", "127.0.0.1")
     app_debug = os.getenv("GRADIO_DEBUG", "False").lower() == "true"