Update app2.py

app2.py

@@ -27,6 +27,8 @@ import numpy as np
 import tarfile
 import gzip
 import math
+import random # Added for dummy chatbot response variation
+import pandas as pd # Added for potential data analysis
 
 # Setup enhanced logging with more detailed formatting
 logging.basicConfig(
@@ -1132,6 +1134,281 @@ def generate_qr_codes(data: Union[str, Dict, List], combined: bool = True) -> Li
 # process_inputs function already handles calling them and getting
 # the combined list of results.
 
+# --- Chatbot Logic ---
+def respond_to_chat(message: str, chat_history: List[Tuple[str, str]], chatbot_data: Optional[List[Dict]]) -> Tuple[List[Tuple[str, str]], List[Dict]]:
+    """Responds to user chat messages based on the loaded JSON data."""
+    if chatbot_data is None or not chatbot_data:
+        chat_history.append((message, "Please process some data first using the other tabs before chatting."))
+        return chat_history, chatbot_data
+
+    # Add user message to history
+    chat_history.append((message, "")) # Add empty string for bot response initially
+
+    response = ""
+    lower_message = message.lower().strip()
+
+    try:
+        # Attempt to flatten the data structure for easier querying if it's nested
+        # This is a simplified approach; a real implementation might need a more
+        # sophisticated data traversal/query engine.
+        flat_data = []
+        for item in chatbot_data:
+            # Recursively flatten dictionaries and lists within the item
+            def flatten_item(d, parent_key='', sep='_'):
+                items = []
+                for k, v in d.items():
+                    new_key = parent_key + sep + k if parent_key else k
+                    if isinstance(v, dict):
+                        items.extend(flatten_item(v, new_key, sep=sep).items())
+                    elif isinstance(v, list):
+                        # Flatten lists by processing each element
+                        for i, elem in enumerate(v):
+                            if isinstance(elem, (dict, list)):
+                                items.extend(flatten_item({f'{new_key}_{i}': elem}, sep=sep).items())
+                            else:
+                                items.append((f'{new_key}_{i}', elem))
+                    else:
+                        items.append((new_key, v))
+                return dict(items)
+
+            if isinstance(item, dict):
+                flat_data.append(flatten_item(item))
+            elif isinstance(item, list):
+                 # If a top-level item is a list, try flattening its elements
+                 for sub_item in item:
+                     if isinstance(sub_item, dict):
+                         flat_data.append(flatten_item(sub_item))
+                     else:
+                          # Handle non-dict items in the top-level list if necessary
+                          flat_data.append({'value': sub_item}) # Wrap in dict
+
+
+        # Use pandas DataFrame for easier querying and analysis if flat_data is suitable
+        df = None
+        if flat_data:
+            try:
+                # Attempt to create a DataFrame. This might fail if structures are highly inconsistent.
+                df = pd.DataFrame(flat_data)
+                logger.debug(f"Created DataFrame with shape: {df.shape}")
+                logger.debug(f"DataFrame columns: {list(df.columns)}")
+            except Exception as e:
+                logger.warning(f"Could not create pandas DataFrame from processed data: {e}. Falling back to manual processing.")
+                df = None # Ensure df is None if creation failed
+
+
+        # --- Complex Queries and Analysis ---
+
+        if df is not None:
+            # Example: How many unique values in a column?
+            match = re.search(r'how many unique values in (\w+)', lower_message)
+            if match:
+                column_name = match.group(1)
+                if column_name in df.columns:
+                    unique_count = df[column_name].nunique()
+                    response = f"There are {unique_count} unique values in the '{column_name}' column."
+                else:
+                    response = f"I couldn't find a column named '{column_name}' in the data. Available columns are: {', '.join(df.columns)}"
+
+            # Example: What is the average/sum/min/max of a numeric column?
+            match = re.search(r'what is the (average|sum|min|max) of (\w+)', lower_message)
+            if match:
+                operation, column_name = match.groups()
+                if column_name in df.columns:
+                    try:
+                        # Attempt to convert column to numeric, coercing errors
+                        numeric_col = pd.to_numeric(df[column_name], errors='coerce')
+                        # Drop NaNs that resulted from coercion
+                        numeric_col = numeric_col.dropna()
+
+                        if not numeric_col.empty:
+                            if operation == 'average':
+                                result = numeric_col.mean()
+                                response = f"The average of '{column_name}' is {result:.2f}."
+                            elif operation == 'sum':
+                                result = numeric_col.sum()
+                                response = f"The sum of '{column_name}' is {result:.2f}."
+                            elif operation == 'min':
+                                result = numeric_col.min()
+                                response = f"The minimum of '{column_name}' is {result}."
+                            elif operation == 'max':
+                                result = numeric_col.max()
+                                response = f"The maximum of '{column_name}' is {result}."
+                            else:
+                                response = "I can calculate average, sum, min, or max."
+                        else:
+                            response = f"The column '{column_name}' does not contain numeric values that I can analyze."
+                    except Exception as e:
+                        response = f"An error occurred while calculating the {operation} of '{column_name}': {e}"
+                        logger.error(f"Error calculating {operation} for column '{column_name}': {e}")
+                else:
+                    response = f"I couldn't find a column named '{column_name}'. Available columns are: {', '.join(df.columns)}"
+
+            # Example: Filter data based on a simple condition (e.g., price > 100)
+            match = re.search(r'show me items where (\w+)\s*([<>=!]+)\s*(\w+)', lower_message)
+            if match:
+                column_name, operator, value_str = match.groups()
+                try:
+                    # Attempt to infer value type (numeric, string)
+                    try:
+                        value = float(value_str)
+                        is_numeric_comparison = True
+                    except ValueError:
+                        value = value_str.strip("'\"") # Remove quotes for string comparison
+                        is_numeric_comparison = False
+
+                    if column_name in df.columns:
+                        if is_numeric_comparison:
+                            # Ensure the column is numeric for comparison
+                            numeric_col = pd.to_numeric(df[column_name], errors='coerce')
+                            filtered_df = df.loc[pd.notna(numeric_col)] # Filter out rows where conversion failed
+
+                            if operator == '>':
+                                filtered_results = filtered_df[numeric_col > value]
+                            elif operator == '<':
+                                filtered_results = filtered_df[numeric_col < value]
+                            elif operator == '>=':
+                                filtered_results = filtered_df[numeric_col >= value]
+                            elif operator == '<=':
+                                filtered_results = filtered_df[numeric_col <= value]
+                            elif operator == '==':
+                                filtered_results = filtered_df[numeric_col == value]
+                            elif operator == '!=':
+                                filtered_results = filtered_df[numeric_col != value]
+                            else:
+                                filtered_results = pd.DataFrame() # Empty if operator is not recognized
+                                response = f"Unsupported numeric operator: {operator}. Try >, <, >=, <=, ==, !=."
+
+                            if not filtered_results.empty:
+                                preview = filtered_results.to_json(orient='records', indent=2)[:500] + "..." if len(filtered_results.to_json()) > 500 else filtered_results.to_json(orient='records', indent=2)
+                                response = f"Here are the items where '{column_name}' {operator} {value_str}:\n```json\n{preview}\n```"
+                            elif 'response' not in locals(): # Only set if not already set by unsupported operator
+                                response = f"No items found where '{column_name}' {operator} {value_str}."
+
+                        elif operator == '==': # Simple string equality
+                            filtered_results = df[df[column_name] == value]
+                            if not filtered_results.empty:
+                                preview = filtered_results.to_json(orient='records', indent=2)[:500] + "..." if len(filtered_results.to_json()) > 500 else filtered_results.to_json(orient='records', indent=2)
+                                response = f"Here are the items where '{column_name}' is '{value}':\n```json\n{preview}\n```"
+                            else:
+                                response = f"No items found where '{column_name}' is '{value}'."
+                        elif operator == '!=': # Simple string inequality
+                            filtered_results = df[df[column_name] != value]
+                            if not filtered_results.empty:
+                                preview = filtered_results.to_json(orient='records', indent=2)[:500] + "..." if len(filtered_results.to_json()) > 500 else filtered_results.to_json(orient='records', indent=2)
+                                response = f"Here are the items where '{column_name}' is not '{value}':\n```json\n{preview}\n```"
+                            else:
+                                response = f"All items have '{column_name}' as '{value}' or the column doesn't exist."
+                        else:
+                             response = f"Unsupported operator for string comparison: {operator}. Try == or !=."
+
+
+                    else:
+                        response = f"I couldn't find a column named '{column_name}'. Available columns are: {', '.join(df.columns)}"
+
+                except Exception as e:
+                     response = f"An error occurred while filtering data: {e}"
+                     logger.error(f"Error filtering data based on condition: {e}")
+
+
+        # --- General Queries ---
+
+        if not response: # If no specific analysis/query matched yet
+            if "how many items" in lower_message or "number of items" in lower_message:
+                if isinstance(chatbot_data, list):
+                    response = f"There are {len(chatbot_data)} top-level items in the processed data."
+                elif isinstance(chatbot_data, dict):
+                    response = "The processed data is a single dictionary, not a list of items."
+                else:
+                    response = "The processed data is not a standard list or dictionary structure."
+
+            elif "what is the structure" in lower_message or "tell me about the data" in lower_message:
+                if isinstance(chatbot_data, list) and chatbot_data:
+                    sample_item = chatbot_data[0]
+                    response = f"The data is a list containing {len(chatbot_data)} items. The first item has the following top-level keys: {list(sample_item.keys())}. I can try to tell you more about specific keys if you like."
+                elif isinstance(chatbot_data, dict):
+                     response = f"The data is a dictionary with the following top-level keys: {list(chatbot_data.keys())}."
+                else:
+                    response = "The processed data is not a standard list or dictionary structure that I can easily describe."
+
+            elif "show me" in lower_message or "get me" in lower_message or "extract" in lower_message:
+                 # Basic extraction if DataFrame wasn't created or query wasn't complex
+                 parts = lower_message.split("show me")
+                 if len(parts) > 1:
+                     key_request = parts[1].strip().split(" ")[0]
+                     extracted_values = []
+                     if isinstance(chatbot_data, list):
+                         for item in chatbot_data:
+                             if isinstance(item, dict) and key_request in item:
+                                 extracted_values.append(item[key_request])
+                     elif isinstance(chatbot_data, dict) and key_request in chatbot_data:
+                         extracted_values.append(chatbot_data[key_request])
+
+                     if extracted_values:
+                         preview = json.dumps(extracted_values, indent=2)[:500] + "..." if len(json.dumps(extracted_values)) > 500 else json.dumps(extracted_values, indent=2)
+                         response = f"Here are the values for '{key_request}':\n```json\n{preview}\n```"
+                     else:
+                         response = f"I couldn't find a key named '{key_request}' in the top level of the data items."
+                 else:
+                     response = "What specifically would you like me to show or extract?"
+
+            # --- Speculation about Modifications ---
+            elif "how can i modify" in lower_message or "how to change" in lower_message or "can i add" in lower_message or "can i remove" in lower_message:
+                 response = "I cannot directly modify the data here, but I can tell you how you *could* modify it. What kind of change are you considering (e.g., adding an item, changing a value, removing a field)?"
+
+            elif "add a field" in lower_message or "add a column" in lower_message:
+                 response = "To add a field (or column if the data is tabular), you would typically iterate through each item (or row) in the data and add the new key-value pair. For example, adding a 'status' field with a default value."
+
+            elif "change a value" in lower_message or "update a field" in lower_message:
+                 response = "To change a value, you would need to identify the specific item(s) and the field you want to update. You could use a condition (like filtering) to find the right items and then assign a new value to the field."
+
+            elif "remove a field" in lower_message or "delete a column" in lower_message:
+                 response = "To remove a field, you would iterate through each item and delete the specified key. Be careful, as this is irreversible."
+
+            elif "restructure" in lower_message or "change the format" in lower_message:
+                 response = "Restructuring data involves transforming it into a different shape. This could mean flattening nested objects, grouping items, or pivoting data. This often requires writing custom code to map the old structure to the new one."
+
+            elif "what if i" in lower_message or "if i changed" in lower_message:
+                 response = "Tell me what specific change you're contemplating, and I can speculate on the potential impact or how you might approach it programmatically."
+
+
+            # --- General Conversation / Fallback ---
+            elif "hello" in lower_message or "hi" in lower_message:
+                response = random.choice(["Hello! How can I help you understand the processed data?", "Hi there! What's on your mind about this data?", "Hey! Ask me anything about the data you've loaded."])
+
+            elif "thank you" in lower_message or "thanks" in lower_message:
+                response = random.choice(["You're welcome!", "Glad I could help.", "No problem! Let me know if you have more questions about the data."])
+
+            elif "clear chat" in lower_message:
+                 chat_history = [] # Clear history
+                 response = "Chat history cleared."
+
+            elif not response: # If still no specific response matched
+                 response = random.choice([
+                    "I can analyze the data you've processed. What would you like to know?",
+                    "Ask me about the number of items, the structure, or values of specific fields.",
+                    "I can perform basic analysis like counting unique values or calculating sums/averages if the data is suitable.",
+                    "Tell me what you want to extract or filter from the data.",
+                    "I'm still learning, but I can try to answer questions about the data structure and content."
+                 ])
+
+    except Exception as e:
+        logger.error(f"Chatbot runtime error: {e}")
+        response = f"An internal error occurred while processing your request: {e}"
+        response += "\nPlease try rephrasing your question or clear the chat history."
+
+
+    # Update the last message in history with the bot's response
+    if chat_history and chat_history[-1][1] == "":
+        chat_history[-1] = (chat_history[-1][0], response)
+    else:
+        # This case should ideally not happen if the initial append was done correctly
+        chat_history.append(("", response)) # Append bot response if something went wrong with initial append
+
+
+    return chat_history, chatbot_data # Return updated history and data state
+
+
+# --- Gradio Interface Definition ---
 def create_modern_interface():
     """Create a modern and visually appealing Gradio interface"""
 
@@ -1268,6 +1545,7 @@ def create_modern_interface():
             )
 
         qr_code_paths = gr.State([])
+        chatbot_data = gr.State(None) # State to hold the processed JSON data for the chatbot
         gr.Markdown("""
         # 🌐 Advanced Data Processing & QR Code Generator
         Transform your data into beautifully designed, sequenced QR codes with our cutting-edge processor.
@@ -1322,6 +1600,15 @@ def create_modern_interface():
             viewport_output = gr.HTML(label="QR Code Sequence Viewport")
             enabled_qr_codes = gr.State([]) # To store the enabled/disabled state
 
+        with gr.Tab("🤖 Chat with Data") as chat_tab:
+             chat_history = gr.State([]) # State to hold chat history
+             chatbot = gr.Chatbot(label="Data Chatbot")
+             with gr.Row():
+                  chat_input = gr.Textbox(label="Your Message", placeholder="Ask me about the processed data...")
+                  send_msg_btn = gr.Button("Send")
+             clear_chat_btn = gr.Button("Clear Chat History")
+
+
         # Load example data
         def load_example():
             example = {
@@ -1353,7 +1640,7 @@ def create_modern_interface():
             return json.dumps(example, indent=2)
 
         def clear_input():
-            return "", None, "" # Clear url, files, text
+            return "", None, "", None # Clear url, files, text, and chatbot data state
 
         def update_viewport(paths, enabled_states):
             if not paths:
@@ -1478,10 +1765,12 @@ def create_modern_interface():
                 logger.error(f"Overall processing error in process_inputs: {e}")
                 processing_status_messages.append(f"❌ An unexpected error occurred during processing: {str(e)}")
 
+            # Return processed data for the chatbot state
             return (
                 final_json_output,
                 [str(path) for path in qr_paths], # Gradio Gallery expects list of paths (strings)
-                "\n".join(processing_status_messages) # Join status messages
+                "\n".join(processing_status_messages), # Join status messages
+                final_json_output # Pass the processed data to the chatbot_data state
             )
 
         def on_qr_generation(qr_paths_list):
@@ -1497,12 +1786,12 @@ def create_modern_interface():
 
         # Link events
         example_btn.click(load_example, inputs=[], outputs=text_input)
-        clear_btn.click(clear_input, inputs=[], outputs=[url_input, file_input, text_input]) # Clear all inputs
+        clear_btn.click(clear_input, inputs=[], outputs=[url_input, file_input, text_input, chatbot_data]) # Clear all inputs including chatbot data state
 
         process_btn.click(
             process_inputs,
             inputs=[url_input, file_input, text_input, combine_data, crawl_depth_slider], # Pass crawl_depth_slider value
-            outputs=[output_json, output_gallery, output_text]
+            outputs=[output_json, output_gallery, output_text, chatbot_data] # Output to chatbot_data state
         ).then( # Chain a .then() to update the QR paths state and trigger viewport update
             on_qr_generation,
             inputs=[output_gallery], # Get the list of paths from the gallery output
@@ -1512,6 +1801,34 @@ def create_modern_interface():
         # The viewport tab's select event will trigger update_viewport to render the grid
         viewport_tab.select(update_viewport, inputs=[qr_code_paths, enabled_qr_codes], outputs=[viewport_output])
 
+        # Chatbot interactions
+        send_msg_btn.click(
+            respond_to_chat,
+            inputs=[chat_input, chat_history, chatbot_data], # Pass chatbot_data state
+            outputs=[chatbot, chatbot_data] # Update chatbot display and chatbot_data state
+        ).then(
+            lambda: "", # Clear the input box after sending
+            inputs=None,
+            outputs=chat_input
+        )
+
+        chat_input.submit( # Allow sending message by pressing Enter
+            respond_to_chat,
+            inputs=[chat_input, chat_history, chatbot_data], # Pass chatbot_data state
+            outputs=[chatbot, chatbot_data] # Update chatbot display and chatbot_data state
+        ).then(
+            lambda: "", # Clear the input box after sending
+            inputs=None,
+            outputs=chat_input
+        )
+
+        clear_chat_btn.click(
+            lambda: [], # Clear chat history
+            inputs=None,
+            outputs=chatbot
+        )
+
+
         # Add helpful documentation
         gr.Markdown("""
         ### 🚀 Features
@@ -1524,6 +1841,7 @@ def create_modern_interface():
         - **Sequential QR Codes**: Maintains data integrity across multiple codes by chunking the combined/individual processed data.
         - **QR Code Viewport**: Visualize generated QR codes in a sequenced square grid with options to enable/disable individual codes for selective scanning/sharing.
         - **Modern Design**: Clean, responsive interface with visual feedback.
+        - **Data Chatbot**: Interact conversationally with the processed JSON data to ask questions about its structure, content, or request specific information.
         ### 💡 Tips
         1.  **URLs**: Enter multiple URLs separated by commas or newlines. The processor will attempt to fetch and structure the content based on its type, following links up to the specified **Crawl Depth**.
         2.  **Files**: Upload any type of file. The processor will attempt to handle supported text-based files, archives (.zip, .tar, .gz), and specific document/structured formats.
@@ -1532,6 +1850,7 @@ def create_modern_interface():
         5.  **QR Codes**: Choose whether to "Combine all data into sequence" or generate separate sequences for each input item.
         6.  **Processing**: Monitor the "Processing Status" box for real-time updates and notes about errors or processing steps.
         7.  **Output**: The "Processed Data" JSON box shows the structured data extracted from your inputs. The "Generated QR Codes" gallery shows the QR code images.
+        8.  **Chatbot**: After processing data, go to the "Chat with Data" tab to ask questions about the JSON output.
         ### ⚙️ QR Code Viewport Instructions
         1.  Navigate to the **QR Code Viewport** tab after generating QR codes.
         2.  The generated QR codes will be displayed in a grid based on their total count.