dynamic core selection according to agentic api's output.

data_processing.py

@@ -40,32 +40,50 @@ def parse_suggestions_from_report(report_text):
 
 
 def llm_generate_analysis_plan_with_history(llm_model, natural_language_query, chat_history):
-    """
-    Generates a complete analysis plan from a user query, considering chat history
-    and dynamic field suggestions from an external API.
-    """
-    search_fields, search_name = [], ""
-    try:
-        # Call the external API to get dynamic field suggestions
-        search_fields, search_name = get_search_list_params(natural_language_query)
-        print(f"Successfully retrieved {len(search_fields)} dynamic fields.")
-    except Exception as e:
-        print(f"Warning: Could not retrieve dynamic search fields. Proceeding without them. Error: {e}")
-
-    # Generate the prompt, including the (potentially empty) search_fields
-    prompt = get_analysis_plan_prompt(natural_language_query, chat_history, search_fields)
-
-    try:
-        response = llm_model.generate_content(prompt)
-        cleaned_text = re.sub(r'```json\s*|\s*```', '', response.text, flags=re.MULTILINE | re.DOTALL).strip()
-        plan = json.loads(cleaned_text)
-        # Return the plan and the retrieved fields for UI display
-        return plan, search_fields
-    except Exception as e:
-        raw_response_text = response.text if 'response' in locals() else 'N/A'
-        print(f"Error in llm_generate_analysis_plan_with_history: {e}\nRaw Response:\n{raw_response_text}")
-        # Return None for the plan but still return search_fields for debugging in the UI
-        return None, search_fields
+        """
+        Generates a complete analysis plan from a user query, considering chat history
+        and dynamic field suggestions from an external API.
+        """
+        search_fields, search_name, field_mappings = [], "", {}
+        try:
+            # Call the external API to get dynamic fields, core name, and mappings
+            search_fields, search_name, field_mappings = get_search_list_params(natural_language_query)
+            print(f"API returned core: '{search_name}' with {len(search_fields)} fields and {len(field_mappings)} mappings.")
+        except Exception as e:
+            print(f"Warning: Could not retrieve dynamic search fields. Proceeding without them. Error: {e}")
+
+        # Determine the core name, default to 'news' if not provided by the API
+        core_name = search_name if search_name else 'news'
+
+        # Apply the field mappings to the suggestions before sending them to the LLM
+        mapped_search_fields = []
+        if search_fields and field_mappings:
+            for field in search_fields:
+                original_name = field.get('field_name')
+                # Create a new dict to avoid modifying the original
+                mapped_field = field.copy()
+                if original_name in field_mappings:
+                    mapped_field['field_name'] = field_mappings[original_name]
+                    print(f"Mapped field '{original_name}' to '{mapped_field['field_name']}'")
+                mapped_search_fields.append(mapped_field)
+        else:
+            mapped_search_fields = search_fields
+
+
+        # Generate the prompt, passing the mapped fields and the dynamic core name
+        prompt = get_analysis_plan_prompt(natural_language_query, chat_history, mapped_search_fields, core_name)
+
+        try:
+            response = llm_model.generate_content(prompt)
+            cleaned_text = re.sub(r'```json\s*|\s*```', '', response.text, flags=re.MULTILINE | re.DOTALL).strip()
+            plan = json.loads(cleaned_text)
+            # Return the plan, the mapped fields for UI display, and the core name
+            return plan, mapped_search_fields, core_name
+        except Exception as e:
+            raw_response_text = response.text if 'response' in locals() else 'N/A'
+            print(f"Error in llm_generate_analysis_plan_with_history: {e}\nRaw Response:\n{raw_response_text}")
+            # Return None for the plan but still return other data for debugging
+            return None, mapped_search_fields, core_name
 
 
 
@@ -79,17 +97,15 @@ def execute_quantitative_query(solr_client, plan):
             "rows": 0,
             "json.facet": json.dumps(plan['quantitative_request']['json.facet'])
         }
-
-        # Build the full Solr URL manually (for logging)
-        base_url = "http://69.167.186.48:8983/solr/news/select"
+        # Build the full Solr URL manually (for logging) from the client's current URL
+        base_url = f"{solr_client.url}/select"
         query_string = urllib.parse.urlencode(params)
         full_url = f"{base_url}?{query_string}"
-
-        print(f"[DEBUG] Solr QUANTITIVE query URL: {full_url}")
+        print(f"[DEBUG] Solr QUANTITATIVE query URL: {full_url}")
         results = solr_client.search(**params)
         return results.raw_response.get("facets", {}), full_url
     except Exception as e:
-        print(f"Error in quantitative query: {e}")
+        print(f"Error in quantitative query on core specified in client ({solr_client.url}): {e}")
         return None, None
 
 def execute_qualitative_query(solr_client, plan):
@@ -104,15 +120,15 @@ def execute_qualitative_query(solr_client, plan):
             "fl": "*,score",
             **qual_request
         }
-        base_url = "http://69.167.186.48:8983/solr/news/select"
+        # Build the full Solr URL manually (for logging) from the client's current URL
+        base_url = f"{solr_client.url}/select"
         query_string = urllib.parse.urlencode(params)
         full_url = f"{base_url}?{query_string}"
-
         print(f"[DEBUG] Solr QUALITATIVE query URL: {full_url}")
         results = solr_client.search(**params)
         return results.grouped, full_url
     except Exception as e:
-        print(f"Error in qualitative query: {e}")
+        print(f"Error in qualitative query on core specified in client ({solr_client.url}): {e}")
         return None, None
 
 def llm_synthesize_enriched_report_stream(llm_model, query, quantitative_data, qualitative_data, plan):

extract_results.py

@@ -1,28 +1,67 @@
 import requests
 import json
 import yaml
+import re
+
+def _parse_mappings(mapping_str: str) -> dict:
+    """Parses the field mapping string into a dictionary."""
+    mappings = {}
+    if not mapping_str:
+        return mappings
+    # Example line: "Mapping 'company_territory' to 'owner_company_territory' under 'compound'."
+    pattern = re.compile(r"Mapping '([^']*)' to '([^']*)'")
+    for line in mapping_str.split('\n'):
+        match = pattern.search(line)
+        if match:
+            original_field, mapped_field = match.groups()
+            mappings[original_field] = mapped_field
+    return mappings
 
 def get_search_list_params(query, k=20):
     """
-    Returns tuple: (search_fields, search_name)
+    Connects to the external API, parses the stream, and returns the core name,
+    search fields, and field mappings.
+
+    Returns tuple: (search_fields, search_name, field_mappings)
     """
     url = "https://aitest.ebalina.com/stream"
     
-    response = requests.post(url, 
-                           headers={'Content-Type': 'application/json'},
-                           json={"query": query, "k": k},
-                           stream=True)
+    response = requests.post(
+        url,
+        headers={'Content-Type': 'application/json'},
+        json={"query": query, "k": k},
+        stream=True
+    )
     
+    search_fields = []
+    search_name = ""
+    field_mappings_str = ""
+
     for line in response.iter_lines():
         if line and line.startswith(b'data: '):
             try:
-                data = json.loads(line[6:])
-                if data.get('log_title') == 'Search List Result':
-                    yaml_data = yaml.safe_load(data['content'])
-                    # As requested, ignoring 'search_name' and only returning fields
-                    return yaml_data.get('search_fields', []), yaml_data.get('search_name', '')
-            except:
+                line_str = line.decode('utf-8')[6:]
+                if not line_str or line_str.isspace():
+                    continue
+                
+                data = json.loads(line_str)
+                log_title = data.get('log_title')
+
+                if log_title == 'Search List Result':
+                    content = data.get('content', '')
+                    if content:
+                        yaml_data = yaml.safe_load(content)
+                        print("DEBUG:", yaml_data)
+                        # This is the dynamic core name
+                        search_name = yaml_data.get('search_name', '')
+                        search_fields = yaml_data.get('search_fields', [])
+
+                elif log_title == 'Field Mapping Outputs':
+                    field_mappings_str = data.get('content', '')
+
+            except (json.JSONDecodeError, yaml.YAMLError, AttributeError):
                 continue
     
-    # Return empty list if no valid data is found
-    return [], ""
+    field_mappings = _parse_mappings(field_mappings_str)
+    
+    return search_fields, search_name, field_mappings

llm_prompts.py

@@ -11,15 +11,17 @@ import datetime
 import json
 from solr_metadata import format_metadata_for_prompt
 
-def get_analysis_plan_prompt(natural_language_query, chat_history, search_fields=None):
+def get_analysis_plan_prompt(natural_language_query, chat_history, search_fields=None, core_name="news"):
     """
     Generates the prompt for creating a Solr analysis plan from a user query.
     Args:
         natural_language_query (str): The user's query.
         chat_history (list): A list of previous user and bot messages.
         search_fields (list, optional): A list of dictionaries with 'field_name' and 'field_value'.
+        core_name (str): The name of the Solr core to use for field metadata.
     """
-    formatted_field_info = format_metadata_for_prompt()
+    # Dynamically get field info for the specified core
+    formatted_field_info = format_metadata_for_prompt(core_name)
     formatted_history = ""
     for user_msg, bot_msg in chat_history:
         if user_msg:
@@ -27,6 +29,7 @@ def get_analysis_plan_prompt(natural_language_query, chat_history, search_fields
 
     dynamic_fields_prompt_section = ""
     if search_fields:
+        # The search_fields are now pre-mapped, so we can use them directly
         formatted_fields = "\n".join([f"  - {field['field_name']}: {field['field_value']}" for field in search_fields])
         dynamic_fields_prompt_section = f"""
 --- 
@@ -84,7 +87,7 @@ This is the most critical part of your task. A bad choice leads to a useless, bo
 *   **IMPLICIT COUNT:** If the user asks a "what," "who," "how many," or "most common" question without specifying a value metric, the measure is `count`.
 
 ---
-### FIELD DEFINITIONS (Your Source of Truth)
+### FIELD DEFINITIONS (Your Source of Truth for Core: {core_name})
 
 {formatted_field_info}
 {dynamic_fields_prompt_section}

ui.py

@@ -21,6 +21,7 @@ from data_processing import (
     parse_suggestions_from_report
 )
 
+
 def create_ui(llm_model, solr_client):
     """
     Builds the Gradio UI and wires up all the event handlers.
@@ -36,28 +37,39 @@ def create_ui(llm_model, solr_client):
             with gr.Column(scale=4):
                 gr.Markdown("# PharmaCircle AI Data Analyst")
             with gr.Column(scale=1):
-                clear_button = gr.Button("🔄 Start New Analysis", variant="primary")
+                clear_button = gr.Button(
+                    "🔄 Start New Analysis", variant="primary")
 
         gr.Markdown("Ask a question to begin your analysis. I will generate an analysis plan, retrieve quantitative and qualitative data, create a visualization, and write an enriched report.")
 
         with gr.Row():
             with gr.Column(scale=1):
-                chatbot = gr.Chatbot(label="Analysis Chat Log", height=700, show_copy_button=True)
-                msg_textbox = gr.Textbox(placeholder="Ask a question, e.g., 'Show me the top 5 companies by total deal value in 2023'", label="Your Question", interactive=True)
+                chatbot = gr.Chatbot(
+                    label="Analysis Chat Log", height=700, show_copy_button=True)
+                msg_textbox = gr.Textbox(
+                    placeholder="Ask a question, e.g., 'Show me the top 5 companies by total deal value in 2023'", label="Your Question", interactive=True)
 
             with gr.Column(scale=2):
                 with gr.Accordion("Dynamic Field Suggestions", open=False):
-                    suggestions_display = gr.Markdown("Suggestions from the external API will appear here...", visible=True)
+                    suggestions_display = gr.Markdown(
+                        "Suggestions from the external API will appear here...", visible=True)
                 with gr.Accordion("Generated Analysis Plan", open=False):
-                    plan_display = gr.Markdown("Plan will appear here...", visible=True)
+                    plan_display = gr.Markdown(
+                        "Plan will appear here...", visible=True)
                 with gr.Accordion("Retrieved Quantitative Data", open=False):
-                    quantitative_url_display = gr.Markdown("Quantitative URL will appear here...", visible=False)
-                    quantitative_data_display = gr.Markdown("Aggregate data will appear here...", visible=False)
+                    quantitative_url_display = gr.Markdown(
+                        "Quantitative URL will appear here...", visible=False)
+                    quantitative_data_display = gr.Markdown(
+                        "Aggregate data will appear here...", visible=False)
                 with gr.Accordion("Retrieved Qualitative Data (Examples)", open=False):
-                    qualitative_url_display = gr.Markdown("Qualitative URL will appear here...", visible=False)
-                    qualitative_data_display = gr.Markdown("Example data will appear here...", visible=False)
-                plot_display = gr.Image(label="Visualization", type="filepath", visible=False)
-                report_display = gr.Markdown("Report will be streamed here...", visible=False)
+                    qualitative_url_display = gr.Markdown(
+                        "Qualitative URL will appear here...", visible=False)
+                    qualitative_data_display = gr.Markdown(
+                        "Example data will appear here...", visible=False)
+                plot_display = gr.Image(
+                    label="Visualization", type="filepath", visible=False)
+                report_display = gr.Markdown(
+                    "Report will be streamed here...", visible=False)
 
         def process_analysis_flow(user_input, history, state):
             """
@@ -80,12 +92,12 @@ def create_ui(llm_model, solr_client):
             history.append((user_input, f"Analyzing: '{query_context}'\n\n*Generating analysis plan...*"))
             yield (history, state, None, None, None, None, None, None, None, None)
 
-            # Generate plan and get search field suggestions
-            analysis_plan, search_fields = llm_generate_analysis_plan_with_history(llm_model, query_context, history)
+            # Generate plan and get search field suggestions. This now returns the core name.
+            analysis_plan, mapped_search_fields, core_name = llm_generate_analysis_plan_with_history(llm_model, query_context, history)
 
             # Update and display search field suggestions in its own accordion
-            if search_fields:
-                suggestions_md = "**External API Suggestions:**\n" + "\n".join([f"- `{field['field_name']}`: `{field['field_value']}`" for field in search_fields])
+            if mapped_search_fields:
+                suggestions_md = "**API Suggestions (with mappings applied):**\n" + "\n".join([f"- `{field['field_name']}`: `{field['field_value']}`" for field in mapped_search_fields])
                 suggestions_display_update = gr.update(value=suggestions_md, visible=True)
             else:
                 suggestions_display_update = gr.update(value="No suggestions were returned from the external API.", visible=True)
@@ -95,30 +107,45 @@ def create_ui(llm_model, solr_client):
                 yield (history, state, None, None, None, None, None, None, None, suggestions_display_update)
                 return
 
-            history.append((None, "✅ Analysis plan generated!"))
+            history.append((None, f"✅ Analysis plan generated for core: **`{core_name}`**"))
             plan_summary = f"""
-*   **Analysis Dimension:** `{analysis_plan.get('analysis_dimension')}`
-*   **Analysis Measure:** `{analysis_plan.get('analysis_measure')}`
-*   **Query Filter:** `{analysis_plan.get('query_filter')}`
-"""
+        *   **Analysis Dimension:** `{analysis_plan.get('analysis_dimension')}`
+        *   **Analysis Measure:** `{analysis_plan.get('analysis_measure')}`
+        *   **Query Filter:** `{analysis_plan.get('query_filter')}`
+        """
             history.append((None, plan_summary))
-            formatted_plan = f"**Full Analysis Plan:**\n```json\n{json.dumps(analysis_plan, indent=2)}\n```"
+            formatted_plan = f"**Full Analysis Plan (Core: `{core_name}`):**\n```json\n{json.dumps(analysis_plan, indent=2)}\n```"
             yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), None, None, None, None, suggestions_display_update)
 
             history.append((None, "*Executing queries for aggregates and examples...*"))
             yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), None, None, None, None, suggestions_display_update)
 
+            # --- DYNAMIC CORE SWITCH (Thread-safe) ---
+            original_solr_url = solr_client.url
+            # Correctly construct the new URL by replacing the last component (the core name)
+            base_url = original_solr_url.rsplit('/', 1)[0]
+            new_url = f"{base_url}/{core_name}"
+            solr_client.url = new_url
+            print(f"[INFO] Switched Solr client to core: {core_name} at URL: {solr_client.url}")
+            # ---
+
             # Execute queries in parallel
             aggregate_data, quantitative_url = None, None
             example_data, qualitative_url = None, None
-            with concurrent.futures.ThreadPoolExecutor() as executor:
-                future_agg = executor.submit(execute_quantitative_query, solr_client, analysis_plan)
-                future_ex = executor.submit(execute_qualitative_query, solr_client, analysis_plan)
-                aggregate_data, quantitative_url = future_agg.result()
-                example_data, qualitative_url = future_ex.result()
+            try:
+                with concurrent.futures.ThreadPoolExecutor() as executor:
+                    future_agg = executor.submit(execute_quantitative_query, solr_client, analysis_plan)
+                    future_ex = executor.submit(execute_qualitative_query, solr_client, analysis_plan)
+                    aggregate_data, quantitative_url = future_agg.result()
+                    example_data, qualitative_url = future_ex.result()
+            finally:
+                # --- IMPORTANT: Reset client to default URL ---
+                solr_client.url = original_solr_url
+                print(f"[INFO] Reset Solr client to default URL: {original_solr_url}")
+                # ---
 
             if not aggregate_data or aggregate_data.get('count', 0) == 0:
-                history.append((None, "No data was found for your query. Please try a different question."))
+                history.append((None, f"No data was found for your query in the '{core_name}' core. Please try a different question."))
                 yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), None, None, None, None, suggestions_display_update)
                 return
 
@@ -178,7 +205,8 @@ def create_ui(llm_model, solr_client):
         msg_textbox.submit(
             fn=process_analysis_flow,
             inputs=[msg_textbox, chatbot, state],
-            outputs=[chatbot, state, plot_display, report_display, plan_display, quantitative_url_display, quantitative_data_display, qualitative_url_display, qualitative_data_display, suggestions_display],
+            outputs=[chatbot, state, plot_display, report_display, plan_display, quantitative_url_display,
+                     quantitative_data_display, qualitative_url_display, qualitative_data_display, suggestions_display],
         ).then(
             lambda: gr.update(value=""),
             None,
@@ -189,8 +217,9 @@ def create_ui(llm_model, solr_client):
         clear_button.click(
             fn=reset_all,
             inputs=None,
-            outputs=[chatbot, state, msg_textbox, plot_display, report_display, plan_display, quantitative_url_display, quantitative_data_display, qualitative_url_display, qualitative_data_display, suggestions_display],
+            outputs=[chatbot, state, msg_textbox, plot_display, report_display, plan_display, quantitative_url_display,
+                     quantitative_data_display, qualitative_url_display, qualitative_data_display, suggestions_display],
             queue=False
         )
 
-    return demo
+    return demo