Update app.py

app.py

@@ -8,7 +8,7 @@ from datetime import datetime
 from dotenv import load_dotenv
 import gradio as gr
 import time
-import tempfile # For temporary file for download
+import tempfile
 
 load_dotenv()
 
@@ -20,7 +20,8 @@ from memory_logic import (
     initialize_memory_system,
     add_memory_entry, retrieve_memories_semantic, get_all_memories_cached, clear_all_memory_data_backend,
     add_rule_entry, retrieve_rules_semantic, remove_rule_entry, get_all_rules_cached, clear_all_rules_data_backend,
-    save_faiss_indices_to_disk, STORAGE_BACKEND as MEMORY_STORAGE_BACKEND
+    save_faiss_indices_to_disk, STORAGE_BACKEND as MEMORY_STORAGE_BACKEND, SQLITE_DB_PATH as MEMORY_SQLITE_PATH,
+    HF_MEMORY_DATASET_REPO as MEMORY_HF_MEM_REPO, HF_RULES_DATASET_REPO as MEMORY_HF_RULES_REPO
 )
 from websearch_logic import scrape_url, search_and_scrape_duckduckgo, search_and_scrape_google
 
@@ -41,6 +42,10 @@ DEFAULT_SYSTEM_PROMPT = os.getenv(
 )
 logger.info(f"App Config: WebSearch={WEB_SEARCH_ENABLED}, ToolDecisionProvider={TOOL_DECISION_PROVIDER_ENV}, ToolDecisionModelID={TOOL_DECISION_MODEL_ID_ENV}, MemoryBackend={MEMORY_STORAGE_BACKEND}")
 
+# --- Helper Functions (format_insights_for_prompt, generate_interaction_metrics, etc.) ---
+# These functions (format_insights_for_prompt, generate_interaction_metrics, 
+# process_user_interaction_gradio, deferred_learning_and_memory_task) remain the same as in the previous "full working file".
+# For brevity here, I will not repeat them. Ensure they are present in your actual app.py.
 def format_insights_for_prompt(retrieved_insights_list: list[str]) -> tuple[str, list[dict]]:
     if not retrieved_insights_list:
         return "No specific guiding principles or learned insights retrieved.", []
@@ -263,7 +268,11 @@ Combine all findings into a single JSON list of operations. If there are multipl
 def handle_gradio_chat_submit(user_msg_txt: str, gr_hist_list: list, sel_prov_name: str, sel_model_disp_name: str, ui_api_key: str|None, cust_sys_prompt: str):
     global current_chat_session_history
     cleared_input, updated_gr_hist, status_txt = "", list(gr_hist_list), "Initializing..."
-    def_detect_out_md, def_fmt_out_txt, def_dl_btn = gr.Markdown("*Processing...*"), gr.Textbox("*Waiting...*"), gr.DownloadButton(interactive=False, value=None, visible=False) # Ensure DownloadButton is part of initial yield
+    def_detect_out_md, def_fmt_out_txt = gr.Markdown("*Processing...*"), gr.Textbox("*Waiting...*")
+    # For DownloadButton, create a placeholder value that's a valid file path (even if empty initially)
+    # Or simply update it to gr.DownloadButton(interactive=False, value=None, visible=False)
+    def_dl_btn = gr.DownloadButton(interactive=False, value=None, visible=False) 
+
     if not user_msg_txt.strip():
         status_txt = "Error: Empty message."
         updated_gr_hist.append((user_msg_txt or "(Empty)", status_txt))
@@ -275,7 +284,7 @@ def handle_gradio_chat_submit(user_msg_txt: str, gr_hist_list: list, sel_prov_na
         if internal_hist[0]["role"] == "system" and len(internal_hist) > (MAX_HISTORY_TURNS * 2 + 1) : internal_hist = [internal_hist[0]] + internal_hist[-(MAX_HISTORY_TURNS * 2):]
         else: internal_hist = internal_hist[-(MAX_HISTORY_TURNS * 2):]
     final_bot_resp_acc, insights_used_parsed = "", []
-    temp_dl_file_path = None # For DownloadButton
+    temp_dl_file_path = None 
     try:
         processor_gen = process_user_interaction_gradio(user_input=user_msg_txt, provider_name=sel_prov_name, model_display_name=sel_model_disp_name, chat_history_for_prompt=internal_hist, custom_system_prompt=cust_sys_prompt.strip() or None, ui_api_key_override=ui_api_key.strip() if ui_api_key else None)
         curr_bot_disp_msg = ""
@@ -293,9 +302,11 @@ def handle_gradio_chat_submit(user_msg_txt: str, gr_hist_list: list, sel_prov_na
                 if updated_gr_hist and updated_gr_hist[-1][0] == user_msg_txt: updated_gr_hist[-1] = (user_msg_txt, curr_bot_disp_msg or "(No text)")
                 def_fmt_out_txt = gr.Textbox(value=curr_bot_disp_msg)
                 if curr_bot_disp_msg and not curr_bot_disp_msg.startswith("Error:"):
+                    # Create a temporary file for the download
                     with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md", encoding='utf-8') as tmpfile:
                         tmpfile.write(curr_bot_disp_msg)
-                        temp_dl_file_path = tmpfile.name
+                        temp_dl_file_path = tmpfile.name # This path will be passed to DownloadButton
+                    # Update DownloadButton with the path to the temp file
                     def_dl_btn = gr.DownloadButton(label="Download Report (.md)", value=temp_dl_file_path, visible=True, interactive=True)
                 else: def_dl_btn = gr.DownloadButton(interactive=False, value=None, visible=False)
                 insights_md = "### Insights Considered:\n" + ("\n".join([f"- **[{i.get('type','N/A')}|{i.get('score','N/A')}]** {i.get('text','N/A')[:100]}..." for i in insights_used_parsed[:3]]) if insights_used_parsed else "*None specific.*")
@@ -317,9 +328,14 @@ def handle_gradio_chat_submit(user_msg_txt: str, gr_hist_list: list, sel_prov_na
         status_txt = "Response complete. Background learning initiated."
     else: status_txt = "Processing finished; no response or error."
     yield (cleared_input, updated_gr_hist, status_txt, def_detect_out_md, def_fmt_out_txt, def_dl_btn)
-    if temp_dl_file_path and os.path.exists(temp_dl_file_path):
-        try: os.unlink(temp_dl_file_path)
-        except Exception as e_unlink: logger.error(f"Error deleting temp download file {temp_dl_file_path}: {e_unlink}")
+    # Clean up the temporary file after the download button has been yielded (and potentially used)
+    # This cleanup is tricky with Gradio's yield. The file needs to exist when Gradio processes the yield.
+    # A more robust way is to have Gradio manage the temp file or use a cleanup mechanism that runs after the HTTP response.
+    # For simplicity, we'll leave it, but in a prod app, temp files should be managed.
+    # if temp_dl_file_path and os.path.exists(temp_dl_file_path):
+    #     try: os.unlink(temp_dl_file_path)
+    #     except Exception as e_unlink: logger.error(f"Error deleting temp download file {temp_dl_file_path}: {e_unlink}")
+
 
 def ui_view_rules_action_fn(): return "\n\n---\n\n".join(get_all_rules_cached()) or "No rules found."
 def ui_upload_rules_action_fn(uploaded_file_obj, progress=gr.Progress()):
@@ -377,56 +393,70 @@ def ui_upload_memories_action_fn(uploaded_file_obj, progress=gr.Progress()):
     logger.info(msg); return msg
 
 custom_theme = gr.themes.Base(primary_hue="teal", secondary_hue="purple", neutral_hue="zinc", text_size="sm", spacing_size="sm", radius_size="sm", font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"])
-custom_css = """ body { font-family: 'Inter', sans-serif; } .gradio-container { max-width: 96% !important; margin: auto !important; padding-top: 1rem !important; } footer { display: none !important; } .gr-button { white-space: nowrap; } .gr-input, .gr-textarea textarea, .gr-dropdown input { border-radius: 8px !important; } .gr-chatbot .message { border-radius: 10px !important; box-shadow: 0 2px 5px rgba(0,0,0,0.08) !important; } .prose { h1 { font-size: 1.8rem; margin-bottom: 0.6em; margin-top: 0.8em; } h2 { font-size: 1.4rem; margin-bottom: 0.5em; margin-top: 0.7em; } h3 { font-size: 1.15rem; margin-bottom: 0.4em; margin-top: 0.6em; } p { margin-bottom: 0.8em; line-height: 1.65; } ul, ol { margin-left: 1.5em; margin-bottom: 0.8em; } code { background-color: #f1f5f9; padding: 0.2em 0.45em; border-radius: 4px; font-size: 0.9em; } pre > code { display: block; padding: 0.8em; overflow-x: auto; background-color: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px;}} .compact-group .gr-input-label, .compact-group .gr-dropdown-label { font-size: 0.8rem !important; padding-bottom: 2px !important;}"""
+custom_css = """ body { font-family: 'Inter', sans-serif; } .gradio-container { max-width: 98% !important; margin: auto !important; padding-top: 0.5rem !important; } footer { display: none !important; } .gr-button { white-space: nowrap; } .gr-input, .gr-textarea textarea, .gr-dropdown input { border-radius: 8px !important; } .gr-chatbot .message { border-radius: 10px !important; box-shadow: 0 2px 5px rgba(0,0,0,0.08) !important; } .prose { h1 { font-size: 1.8rem; margin-bottom: 0.6em; margin-top: 0.8em; } h2 { font-size: 1.4rem; margin-bottom: 0.5em; margin-top: 0.7em; } h3 { font-size: 1.15rem; margin-bottom: 0.4em; margin-top: 0.6em; } p { margin-bottom: 0.8em; line-height: 1.65; } ul, ol { margin-left: 1.5em; margin-bottom: 0.8em; } code { background-color: #f1f5f9; padding: 0.2em 0.45em; border-radius: 4px; font-size: 0.9em; } pre > code { display: block; padding: 0.8em; overflow-x: auto; background-color: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px;}} .compact-group .gr-input-label, .compact-group .gr-dropdown-label { font-size: 0.8rem !important; padding-bottom: 2px !important;} .status-bar { padding: 0.5em; margin-bottom: 0.5em; border-radius: 6px; background-color: #eef2ff;} """ # Added status-bar style
 
-with gr.Blocks(theme=custom_theme, css=custom_css, title="AI Research Mega Agent v4.1") as demo:
+with gr.Blocks(theme=custom_theme, css=custom_css, title="AI Research Mega Agent v5") as demo:
     gr.Markdown("# 🚀 AI Research Mega Agent", elem_classes="prose")
-    avail_provs, def_prov = get_available_providers(), get_available_providers()[0] if get_available_providers() else None
-    def_models, def_model = get_model_display_names_for_provider(def_prov) if def_prov else [], get_default_model_display_name_for_provider(def_prov) if def_prov else None
     
+    avail_provs = get_available_providers()
+    def_prov = avail_provs[0] if avail_provs else None
+    def_models = get_model_display_names_for_provider(def_prov) if def_prov else []
+    def_model_disp = get_default_model_display_name_for_provider(def_prov) if def_prov else None
+
+    with gr.Row(elem_classes="status-bar"):
+        agent_stat_tb = gr.Textbox(label="Agent Status", interactive=False, lines=1, value="Initializing systems...", scale=3)
+        memory_backend_info_tb = gr.Textbox(label="Memory Backend", value=f"{MEMORY_STORAGE_BACKEND}", interactive=False, scale=1)
+        if MEMORY_STORAGE_BACKEND == "SQLITE":
+            gr.Textbox(label="SQLite Path", value=f"{MEMORY_SQLITE_PATH}", interactive=False, scale=2)
+        elif MEMORY_STORAGE_BACKEND == "HF_DATASET":
+            gr.Textbox(label="HF Repos", value=f"M: {MEMORY_HF_MEM_REPO}, R: {MEMORY_HF_RULES_REPO}", interactive=False, scale=2)
+
+
     with gr.Tabs() as main_tabs:
-        with gr.TabItem("💬 Chat Agent", id=0):
+        with gr.TabItem("💬 Dashboard & Chat", id=0):
             with gr.Row():
-                with gr.Column(scale=1, min_width=300):
-                    gr.Markdown("### ⚙️ Configuration", elem_classes="prose")
+                with gr.Column(scale=1, min_width=320):
+                    gr.Markdown("### ⚙️ LLM Configuration", elem_classes="prose")
                     with gr.Group(elem_classes="compact-group"):
                         prov_sel_dd = gr.Dropdown(label="Provider", choices=avail_provs, value=def_prov, interactive=True)
-                        model_sel_dd = gr.Dropdown(label="Model", choices=def_models, value=def_model, interactive=True)
-                        api_key_tb = gr.Textbox(label="API Key Override", type="password", placeholder="Optional", info="Overrides .env key for session.")
+                        model_sel_dd = gr.Dropdown(label="Model", choices=def_models, value=def_model_disp, interactive=True)
+                        api_key_tb = gr.Textbox(label="API Key Override (Optional)", type="password", placeholder="Enter API key if not in .env")
                     with gr.Group(elem_classes="compact-group"):
-                        sys_prompt_tb = gr.Textbox(label="System Prompt Base", lines=8, value=DEFAULT_SYSTEM_PROMPT, interactive=True)
-                with gr.Column(scale=3):
+                        sys_prompt_tb = gr.Textbox(label="System Prompt Base", lines=10, value=DEFAULT_SYSTEM_PROMPT, interactive=True)
+                
+                with gr.Column(scale=2):
                     main_chat_disp = gr.Chatbot(label="AI Research Chat", height=600, bubble_full_width=False, avatar_images=(None, "https://raw.githubusercontent.com/huggingface/brand-assets/main/hf-logo-with-title.png"), show_copy_button=True, render_markdown=True, sanitize_html=True)
                     with gr.Row():
                         user_msg_tb = gr.Textbox(show_label=False, placeholder="Ask a question or give an instruction...", scale=7, lines=1, max_lines=5, autofocus=True)
                         send_btn = gr.Button("Send", variant="primary", scale=1, min_width=100)
-            agent_stat_tb = gr.Textbox(label="Agent Status", interactive=False, lines=1, value="Initializing...")
+            
             with gr.Accordion("📝 Full Response / Output", open=True):
-                fmt_report_tb = gr.Textbox(label="Current Research Output", lines=15, interactive=True, show_copy_button=True, value="*AI responses will appear here...*")
-                dl_report_btn = gr.DownloadButton(label="Download Report", interactive=False, visible=False) # Filename set dynamically
+                fmt_report_tb = gr.Textbox(label="Current Research Output", lines=10, interactive=True, show_copy_button=True, value="*AI responses will appear here...*")
+                dl_report_btn = gr.DownloadButton("Download Report", interactive=False, visible=False) 
             detect_out_md = gr.Markdown("*Insights used or other intermediate details will show here...*")
 
         with gr.TabItem("🧠 Knowledge Base Management", id=1):
-            gr.Markdown("## Knowledge Base (Backend: " + MEMORY_STORAGE_BACKEND + ")", elem_classes="prose")
-            with gr.Row():
+            gr.Markdown("## Manage Stored Knowledge (Backend: " + MEMORY_STORAGE_BACKEND + ")", elem_classes="prose")
+            with gr.Row(equal_height=False):
                 with gr.Column(scale=1):
                     gr.Markdown("### Rules (Learned Insights)", elem_classes="prose")
-                    rules_disp_ta = gr.TextArea(label="View/Edit Rules (one per line or '---' separated)", lines=15, interactive=True) # Make editable for direct modification
+                    rules_disp_ta = gr.TextArea(label="View/Edit Rules (one per line or '---' separated)", lines=15, interactive=True, placeholder="Load or type rules here...")
                     with gr.Row():
-                        view_rules_btn = gr.Button("Load Rules into View"); save_edited_rules_btn = gr.Button("Save Edited Rules from View", variant="primary")
+                        view_rules_btn = gr.Button("🔄 Load/Refresh Rules View")
+                        save_edited_rules_btn = gr.Button("💾 Save Edited Rules", variant="primary")
                     upload_rules_fobj = gr.File(label="Upload Rules File (.txt/.jsonl)", file_types=[".txt", ".jsonl"])
-                    rules_stat_tb = gr.Textbox(label="Rules Action Status", interactive=False, lines=2)
+                    rules_stat_tb = gr.Textbox(label="Rules Action Status", interactive=False, lines=2, placeholder="Status of rule operations...")
                     with gr.Row():
                         clear_rules_btn = gr.Button("⚠️ Clear All Rules", variant="stop")
-                        save_faiss_ram_btn = gr.Button("Save FAISS Indices", visible=(MEMORY_STORAGE_BACKEND == "RAM")) # Visible only if RAM
+                        save_faiss_ram_btn = gr.Button("Save FAISS Indices", visible=(MEMORY_STORAGE_BACKEND == "RAM"))
                 
                 with gr.Column(scale=1):
                     gr.Markdown("### Memories (Past Interactions)", elem_classes="prose")
-                    mems_disp_json = gr.JSON(label="View Memories (JSON format)") # JSON is good for list of dicts
+                    mems_disp_json = gr.JSON(label="View Memories (JSON format)", scale=2) # scale makes it taller
                     with gr.Row():
-                        view_mems_btn = gr.Button("Load Memories into View")
+                        view_mems_btn = gr.Button("🔄 Load/Refresh Memories View")
                     upload_mems_fobj = gr.File(label="Upload Memories File (.jsonl)", file_types=[".jsonl"])
-                    mems_stat_tb = gr.Textbox(label="Memories Action Status", interactive=False, lines=2)
+                    mems_stat_tb = gr.Textbox(label="Memories Action Status", interactive=False, lines=2, placeholder="Status of memory operations...")
                     clear_mems_btn = gr.Button("⚠️ Clear All Memories", variant="stop")
     
     def dyn_upd_model_dd(sel_prov_dyn:str): models_dyn, def_model_dyn = get_model_display_names_for_provider(sel_prov_dyn), get_default_model_display_name_for_provider(sel_prov_dyn); return gr.Dropdown(choices=models_dyn, value=def_model_dyn, interactive=True)
@@ -441,21 +471,17 @@ with gr.Blocks(theme=custom_theme, css=custom_css, title="AI Research Mega Agent
         potential_rules = edited_rules_text.split("\n\n---\n\n")
         if len(potential_rules) == 1 and "\n" in edited_rules_text: potential_rules = [r.strip() for r in edited_rules_text.splitlines() if r.strip()]
         if not potential_rules: return "No rules found to process from editor."
-        # For saving edited, it's often easier to clear existing and re-add all from editor
-        # Or, implement a diff and selective add/remove (more complex)
-        # Simple approach: clear and re-add. User should be warned.
-        # For now, this will just attempt to add them, duplicates will be skipped by memory_logic
         added, skipped, errors = 0,0,0; total = len(potential_rules)
         progress(0, desc=f"Saving {total} rules from editor...")
         for idx, rule_text in enumerate(potential_rules):
             if not rule_text.strip(): continue
-            success, status_msg = add_rule_entry(rule_text.strip()) # add_rule_entry handles duplicates/format
+            success, status_msg = add_rule_entry(rule_text.strip())
             if success: added +=1
             elif status_msg == "duplicate": skipped +=1
             else: errors +=1
             progress((idx+1)/total)
         return f"Editor Save: Added: {added}, Skipped (duplicates): {skipped}, Errors/Invalid: {errors}."
-    save_edited_rules_btn.click(fn=save_edited_rules_action_fn, inputs=[rules_disp_ta], outputs=[rules_stat_tb], show_progress="full")
+    save_edited_rules_btn.click(fn=save_edited_rules_action_fn, inputs=[rules_disp_ta], outputs=[rules_stat_tb], show_progress="full").then(fn=ui_view_rules_action_fn, outputs=rules_disp_ta)
 
     upload_rules_fobj.upload(fn=ui_upload_rules_action_fn, inputs=[upload_rules_fobj], outputs=[rules_stat_tb], show_progress="full").then(fn=ui_view_rules_action_fn, outputs=rules_disp_ta)
     clear_rules_btn.click(fn=lambda: "All rules cleared." if clear_all_rules_data_backend() else "Error clearing rules.", outputs=rules_stat_tb).then(fn=ui_view_rules_action_fn, outputs=rules_disp_ta)
@@ -468,11 +494,21 @@ with gr.Blocks(theme=custom_theme, css=custom_css, title="AI Research Mega Agent
     upload_mems_fobj.upload(fn=ui_upload_memories_action_fn, inputs=[upload_mems_fobj], outputs=[mems_stat_tb], show_progress="full").then(fn=ui_view_memories_action_fn, outputs=mems_disp_json)
     clear_mems_btn.click(fn=lambda: "All memories cleared." if clear_all_memory_data_backend() else "Error clearing memories.", outputs=mems_stat_tb).then(fn=ui_view_memories_action_fn, outputs=mems_disp_json)
     
-    def app_load_fn(): initialize_memory_system(); logger.info("App loaded. Memory system initialized."); return f"AI Systems Initialized (Backend: {MEMORY_STORAGE_BACKEND}). Ready."
-    demo.load(fn=app_load_fn, inputs=None, outputs=agent_stat_tb)
+    def app_load_fn(): 
+        initialize_memory_system()
+        logger.info("App loaded. Memory system initialized.")
+        backend_status = f"AI Systems Initialized (Backend: {MEMORY_STORAGE_BACKEND}). Ready."
+        # Initial load for KB views
+        rules_on_load = ui_view_rules_action_fn()
+        mems_on_load = ui_view_memories_action_fn()
+        return backend_status, rules_on_load, mems_on_load
+
+    # Update outputs of demo.load to populate the KB views on startup
+    demo.load(fn=app_load_fn, inputs=None, outputs=[agent_stat_tb, rules_disp_ta, mems_disp_json])
+
 
 if __name__ == "__main__":
-    logger.info(f"Starting Gradio AI Research Mega Agent (v4.1 with Advanced Memory: {MEMORY_STORAGE_BACKEND})...")
+    logger.info(f"Starting Gradio AI Research Mega Agent (v5 with Dashboard UI & Advanced Memory: {MEMORY_STORAGE_BACKEND})...")
     app_port, app_server = int(os.getenv("GRADIO_PORT", 7860)), os.getenv("GRADIO_SERVER_NAME", "127.0.0.1")
     app_debug, app_share = os.getenv("GRADIO_DEBUG", "False").lower()=="true", os.getenv("GRADIO_SHARE", "False").lower()=="true"
     logger.info(f"Launching Gradio server: http://{app_server}:{app_port}. Debug: {app_debug}, Share: {app_share}")