Optimize v2-search with enhanced URL processing and search capabilities

- Enhanced URL validation with domain checking and smart truncation
- Improved content extraction targeting main/article elements
- Increased dynamic URL limit to 3 with better error handling
- Fixed research template integration after field simplification
- Added backward compatibility wrapper for fetch_url_content
- Extended timeout to 15s for better reliability

app.py

@@ -13,15 +13,19 @@ from support_docs import create_support_docs, export_conversation_to_markdown
 
 # Simple URL content fetching using requests and BeautifulSoup
 def get_grounding_context_simple(urls):
-    """Fetch grounding context using simple HTTP requests"""
+    """Fetch grounding context using enhanced HTTP requests"""
     if not urls:
         return ""
     
     context_parts = []
     for i, url in enumerate(urls, 1):
         if url and url.strip():
-            content = fetch_url_content(url.strip())
-            context_parts.append(f"Context from URL {i} ({url}):\n{content}")
+            # Use enhanced URL extraction for any URLs within the URL text
+            extracted_urls = extract_urls_from_text(url.strip())
+            target_url = extracted_urls[0] if extracted_urls else url.strip()
+            
+            content = enhanced_fetch_url_content(target_url)
+            context_parts.append(f"Context from URL {i} ({target_url}):\n{content}")
     
     if context_parts:
         return "\n\n" + "\n\n".join(context_parts) + "\n\n"
@@ -42,9 +46,82 @@ load_dotenv()
 import re
 
 def extract_urls_from_text(text):
-    """Extract URLs from text using regex"""
+    """Extract URLs from text using regex with enhanced validation"""
     url_pattern = r'https?://[^\s<>"{}|\\^`\[\]"]+'
-    return re.findall(url_pattern, text)
+    urls = re.findall(url_pattern, text)
+    
+    # Basic URL validation and cleanup
+    validated_urls = []
+    for url in urls:
+        # Remove trailing punctuation that might be captured
+        url = url.rstrip('.,!?;:')
+        # Basic domain validation
+        if '.' in url and len(url) > 10:
+            validated_urls.append(url)
+    
+    return validated_urls
+
+def validate_url_domain(url):
+    """Basic URL domain validation"""
+    try:
+        from urllib.parse import urlparse
+        parsed = urlparse(url)
+        # Check for valid domain structure
+        if parsed.netloc and '.' in parsed.netloc:
+            return True
+    except:
+        pass
+    return False
+
+def enhanced_fetch_url_content(url, enable_search_validation=False):
+    """Enhanced URL content fetching with optional search validation"""
+    if not validate_url_domain(url):
+        return f"Invalid URL format: {url}"
+    
+    try:
+        # Enhanced headers for better compatibility
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
+            'Accept-Encoding': 'gzip, deflate',
+            'Connection': 'keep-alive'
+        }
+        
+        response = requests.get(url, timeout=15, headers=headers)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.content, 'html.parser')
+        
+        # Enhanced content cleaning
+        for element in soup(["script", "style", "nav", "header", "footer", "aside", "form", "button"]):
+            element.decompose()
+        
+        # Extract main content preferentially
+        main_content = soup.find('main') or soup.find('article') or soup.find('div', class_=lambda x: x and 'content' in x.lower()) or soup
+        text = main_content.get_text()
+        
+        # Enhanced text cleaning
+        lines = (line.strip() for line in text.splitlines())
+        chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+        text = ' '.join(chunk for chunk in chunks if chunk and len(chunk) > 2)
+        
+        # Smart truncation - try to end at sentence boundaries
+        if len(text) > 4000:
+            truncated = text[:4000]
+            last_period = truncated.rfind('.')
+            if last_period > 3000:  # If we can find a reasonable sentence break
+                text = truncated[:last_period + 1]
+            else:
+                text = truncated + "..."
+        
+        return text if text.strip() else "No readable content found at this URL"
+        
+    except requests.exceptions.Timeout:
+        return f"Timeout error fetching {url} (15s limit exceeded)"
+    except requests.exceptions.RequestException as e:
+        return f"Error fetching {url}: {str(e)}"
+    except Exception as e:
+        return f"Error processing content from {url}: {str(e)}"
 
 # Template for generated space app (based on mvp_simple.py)
 SPACE_TEMPLATE = '''import gradio as gr
@@ -358,31 +435,8 @@ MODELS = [
 ]
 
 def fetch_url_content(url):
-    """Fetch and extract text content from a URL"""
-    try:
-        response = requests.get(url, timeout=10)
-        response.raise_for_status()
-        soup = BeautifulSoup(response.content, 'html.parser')
-        
-        # Remove script and style elements
-        for script in soup(["script", "style"]):
-            script.decompose()
-        
-        # Get text content
-        text = soup.get_text()
-        
-        # Clean up whitespace
-        lines = (line.strip() for line in text.splitlines())
-        chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
-        text = ' '.join(chunk for chunk in chunks if chunk)
-        
-        # Truncate to ~4000 characters
-        if len(text) > 4000:
-            text = text[:4000] + "..."
-        
-        return text
-    except Exception as e:
-        return f"Error fetching {url}: {str(e)}"
+    """Fetch and extract text content from a URL - maintained for backward compatibility"""
+    return enhanced_fetch_url_content(url)
 
 def get_grounding_context(urls):
     """Fetch context from grounding URLs"""
@@ -697,35 +751,16 @@ def on_preview_combined(name, description, system_prompt, enable_research_assist
         )
     
     try:
-        # Combine system prompt components if research assistant is enabled
-        if enable_research_assistant:
-            if not role_purpose or not role_purpose.strip():
-                return (
-                    {},
-                    gr.update(value="**Error:** Please provide a Role and Purpose for the research assistant", visible=True),
-                    gr.update(visible=False),
-                    gr.update(value="Configuration will appear here after preview generation.")
-                )
-            system_prompt_parts = []
-            if role_purpose and role_purpose.strip():
-                system_prompt_parts.append(role_purpose.strip())
-            if intended_audience and intended_audience.strip():
-                system_prompt_parts.append(intended_audience.strip())
-            if key_tasks and key_tasks.strip():
-                system_prompt_parts.append(key_tasks.strip())
-            if additional_context and additional_context.strip():
-                system_prompt_parts.append(additional_context.strip())
-            
-            final_system_prompt = " ".join(system_prompt_parts)
-        else:
-            if not system_prompt or not system_prompt.strip():
-                return (
-                    {},
-                    gr.update(value="**Error:** Please provide a System Prompt for the assistant", visible=True),
-                    gr.update(visible=False),
-                    gr.update(value="Configuration will appear here after preview generation.")
-                )
-            final_system_prompt = system_prompt.strip()
+        # Use the system prompt directly (research assistant toggle already updates it)
+        if not system_prompt or not system_prompt.strip():
+            return (
+                {},
+                gr.update(value="**Error:** Please provide a System Prompt for the assistant", visible=True),
+                gr.update(visible=False),
+                gr.update(value="Configuration will appear here after preview generation.")
+            )
+        
+        final_system_prompt = system_prompt.strip()
         
         # Create configuration for preview
         config_data = {
@@ -881,8 +916,8 @@ def preview_chat_response(message, history, config_data, url1="", url2="", url3=
             urls_in_message = extract_urls_from_text(message)
             if urls_in_message:
                 dynamic_context_parts = []
-                for url in urls_in_message[:2]:  # Limit to 2 URLs in preview
-                    content = fetch_url_content(url)
+                for url in urls_in_message[:3]:  # Increased limit to 3 URLs with enhanced processing
+                    content = enhanced_fetch_url_content(url)
                     dynamic_context_parts.append(f"\n\nDynamic context from {url}:\n{content}")
                 if dynamic_context_parts:
                     dynamic_context = "\n".join(dynamic_context_parts)
@@ -988,27 +1023,11 @@ def on_generate(name, description, system_prompt, enable_research_assistant, mod
         if enable_vector_rag and rag_tool_state:
             rag_data = rag_tool_state.get_serialized_data()
         
-        # Combine system prompt components if research assistant is enabled
-        if enable_research_assistant:
-            # Use the research assistant fields if enabled
-            if not role_purpose or not role_purpose.strip():
-                return gr.update(value="Error: Please provide a Role and Purpose for the research assistant", visible=True), gr.update(visible=False)
-            system_prompt_parts = []
-            if role_purpose and role_purpose.strip():
-                system_prompt_parts.append(role_purpose.strip())
-            if intended_audience and intended_audience.strip():
-                system_prompt_parts.append(intended_audience.strip())
-            if key_tasks and key_tasks.strip():
-                system_prompt_parts.append(key_tasks.strip())
-            if additional_context and additional_context.strip():
-                system_prompt_parts.append(additional_context.strip())
-            
-            final_system_prompt = " ".join(system_prompt_parts)
-        else:
-            # Use the direct system prompt field
-            if not system_prompt or not system_prompt.strip():
-                return gr.update(value="Error: Please provide a System Prompt for the assistant", visible=True), gr.update(visible=False)
-            final_system_prompt = system_prompt.strip()
+        # Use the system prompt directly (research assistant toggle already updates it)
+        if not system_prompt or not system_prompt.strip():
+            return gr.update(value="Error: Please provide a System Prompt for the assistant", visible=True), gr.update(visible=False)
+        
+        final_system_prompt = system_prompt.strip()
         
         filename = generate_zip(name, description, final_system_prompt, model, api_key_var, temperature, max_tokens, examples_text, access_code, enable_dynamic_urls, url1, url2, url3, url4, enable_vector_rag, rag_data)
         

claude.local.md

@@ -0,0 +1,314 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+Chat UI Helper is a Gradio-based tool for generating and configuring chat interfaces for HuggingFace Spaces. It creates deployable packages with custom assistants, web scraping capabilities, and optional vector RAG functionality.
+
+## Core Architecture
+
+### Main Application Flow (`app.py`)
+The application follows a three-tab Gradio interface pattern:
+1. **Configuration Tab**: Space setup, assistant configuration, tool settings (lines 1267-1589)
+2. **Sandbox Preview Tab**: Interactive testing with real OpenRouter API integration (lines 1591-1699) 
+3. **Support Docs Tab**: Comprehensive guidance and templates via `support_docs.py`
+
+### Template Generation System
+- `SPACE_TEMPLATE` (lines 50-347): Complete HuggingFace Space template with export functionality and legacy tuple format compatibility
+- `generate_zip()` function (lines 562-652): Orchestrates package creation with all dependencies
+- Key template variables: `{system_prompt}`, `{model}`, `{enable_vector_rag}`, `{api_key_var}`, `{grounding_urls}`, `{enable_dynamic_urls}`
+
+### Preview Sandbox Architecture (Enhanced)
+- Real OpenRouter API integration in preview mode (`preview_chat_response()` line 855)
+- URL context testing with dynamic add/remove functionality
+- Configuration-aware responses using exact model and parameters from user configuration
+- Fallback messaging when `OPENROUTER_API_KEY` environment variable not set
+- Legacy tuple format compatibility for Gradio 4.44.1 ChatInterface
+- **Comprehensive Debugging**: Enhanced error handling with detailed API response validation (lines 928-955)
+  - Empty response detection and logging
+  - API structure validation (choices, message, content)
+  - Request payload debugging for troubleshooting
+  - Timeout handling (30 seconds) for API requests
+
+### Document Processing Pipeline (RAG)
+- **RAGTool** (`rag_tool.py`): Main orchestrator with 10MB file size validation (lines 19-79)
+- **DocumentProcessor** (`document_processor.py`): PDF/DOCX/TXT/MD parsing with semantic chunking (800 chars, 100 overlap)
+- **VectorStore** (`vector_store.py`): FAISS-based similarity search and base64 serialization
+
+### Web Scraping Architecture
+Simple HTTP + BeautifulSoup approach (replacing previous Crawl4AI):
+- `fetch_url_content()` (lines 390-415): Basic requests with timeout and user-agent headers
+- Content cleaning: Removes scripts, styles, navigation elements
+- Content limits: ~4000 character truncation for context management
+- URL content caching: `get_cached_grounding_context()` (line 1019) prevents redundant fetches
+- `extract_urls_from_text()` (line 44): Regex-based URL extraction for dynamic fetching
+
+## Development-Only Utilities
+
+### MCP Servers
+- **Gradio Docs**: Available at https://gradio-docs-mcp.hf.space/gradio_api/mcp/sse
+- Use `gradio_docs.py` utility for development assistance
+- **CRITICAL**: Do NOT import in main application - this is for development tooling only
+
+Usage for development:
+```bash
+python -c "from gradio_docs import gradio_docs; print(gradio_docs.search_docs('ChatInterface'))"
+```
+
+## Development Commands
+
+### Environment Setup
+**Important**: This application requires Python ≥3.10 for Gradio 5.x compatibility.
+
+```bash
+# Recommended: Use Python 3.11+ environment
+python3.11 -m venv venv311
+source venv311/bin/activate  # or venv311\Scripts\activate on Windows
+pip install -r requirements.txt
+```
+
+### Running the Application
+```bash
+# With virtual environment activated
+python app.py
+```
+
+### Testing Commands
+```bash
+# Test vector database functionality (requires all RAG dependencies)
+python test_vector_db.py
+
+# Test OpenRouter API key validation
+python test_api_key.py
+
+# Test minimal Gradio functionality (for debugging)
+python test_minimal.py
+
+# Test preview functionality components (new)
+python test_preview.py
+
+# Test individual RAG components
+python -c "from test_vector_db import test_document_processing; test_document_processing()"
+python -c "from test_vector_db import test_vector_store; test_vector_store()"
+python -c "from test_vector_db import test_rag_tool; test_rag_tool()"
+```
+
+### Pre-Test Setup for RAG Components
+```bash
+# Create test document for vector database testing
+echo "This is a test document for RAG functionality testing." > test_document.txt
+
+# Verify all dependencies are installed
+python -c "import sentence_transformers, faiss, fitz; print('RAG dependencies available')"
+```
+
+### Key Dependencies and Versions
+
+#### Required Dependencies
+- **Gradio ≥4.44.1**: Main UI framework (5.37.0 recommended for Python ≥3.10)
+- **requests ≥2.32.3**: HTTP requests for web content fetching
+- **beautifulsoup4 ≥4.12.3**: HTML parsing for web scraping
+- **python-dotenv ≥1.0.0**: Environment variable management
+
+#### Optional RAG Dependencies
+- **sentence-transformers ≥2.2.2**: Text embeddings
+- **faiss-cpu ==1.7.4**: Vector similarity search
+- **PyMuPDF ≥1.23.0**: PDF text extraction
+- **python-docx ≥0.8.11**: DOCX document processing
+- **numpy ==1.26.4**: Numerical operations
+
+## Configuration Patterns
+
+### Conditional Dependency Loading
+```python
+try:
+    from rag_tool import RAGTool
+    HAS_RAG = True
+except ImportError:
+    HAS_RAG = False
+    RAGTool = None
+```
+This pattern allows graceful degradation when optional vector dependencies are unavailable.
+
+### Template Variable Substitution
+Generated spaces use these key substitutions:
+- `{system_prompt}`: Combined assistant configuration
+- `{grounding_urls}`: Static URL list for context
+- `{enable_dynamic_urls}`: Runtime URL fetching capability
+- `{enable_vector_rag}`: Document search integration
+- `{rag_data_json}`: Serialized embeddings and chunks
+- `{api_key_var}`: Customizable API key environment variable name
+
+### Access Control Pattern
+- Environment variable `SPACE_ACCESS_CODE` for student access control
+- Global state management for session-based access in generated spaces
+- Security-first approach storing credentials as HuggingFace Spaces secrets
+
+### RAG Integration Workflow
+1. Documents uploaded through Gradio File component with conditional visibility (`HAS_RAG` flag)
+2. Processed via DocumentProcessor (PDF/DOCX/TXT/MD support) in `process_documents()` function
+3. Chunked and embedded using sentence-transformers (800 chars, 100 overlap)
+4. FAISS index created and serialized to base64 for deployment portability
+5. Embedded in generated template via `{rag_data_json}` template variable
+
+## Implementation Notes
+
+### Research Template System (Simplified)
+- **Simple Toggle**: `toggle_research_assistant()` function (line 1225) now provides simple on/off functionality
+- **Direct System Prompt**: Enables predefined academic research prompt with DOI verification and LibKey integration
+- **Auto-Enable Dynamic URLs**: Research template automatically enables dynamic URL fetching for academic sources
+- **Template Content**: Academic inquiry focus with DOI-verified sources, fact-checking, and proper citation requirements
+- **Note**: Previous complex field system (Role and Purpose, Intended Audience, Key Tasks, Additional Context) has been removed for simplified architecture
+
+### State Management Across Tabs
+- Extensive use of `gr.State()` for maintaining session data
+- Cross-tab functionality through shared state variables (`sandbox_state`, `preview_config_state`)
+- URL content caching to prevent redundant web requests (`url_content_cache` global variable)
+- Preview debugging with comprehensive error handling and API response validation
+
+### Gradio Compatibility and Message Format Handling
+- **Target Version**: Gradio 5.37.0 (requires Python ≥3.10)
+- **Legacy Support**: Gradio 4.44.1 compatibility with JSON schema workarounds
+- **Message Format**: Preview uses legacy tuple format `[user_msg, bot_msg]` for ChatInterface compatibility
+- **Generated Spaces**: Use modern dictionary format `{"role": "user", "content": "..."}` for OpenRouter API
+
+### Security Considerations
+- Never embed API keys or access codes in generated templates
+- Environment variable pattern for all sensitive configuration (`{api_key_var}` template variable)
+- Input validation for uploaded files and URL processing
+- Content length limits for web scraping operations
+
+## Testing Infrastructure
+
+### Current Test Structure
+- `test_vector_db.py`: Comprehensive RAG component testing (196 lines)
+- `test_api_key.py`: OpenRouter API validation (85 lines)  
+- `test_minimal.py`: Basic Gradio functionality debugging (20 lines)
+- `test_preview.py`: Preview functionality component testing (URL extraction, fetching, chat response)
+
+### Test Dependencies
+RAG testing requires: `sentence-transformers`, `faiss-cpu`, `PyMuPDF`, `python-docx`
+
+Core testing requires: `gradio`, `requests`, `beautifulsoup4`, `python-dotenv`
+
+### Testing Status
+- **Functional**: Three main test files covering core functionality
+- **Missing**: Automated test scripts referenced in TEST_PROCEDURE.md (`quick_test.sh`, `full_test.sh`) are documented but not implemented
+- **Usage**: Run individual Python test modules directly
+
+## File Structure Notes
+
+### Generated Space Structure
+All generated HuggingFace Spaces follow consistent structure:
+1. Configuration section with environment variable loading
+2. Web scraping functions (simple HTTP requests with BeautifulSoup)
+3. RAG context retrieval (if enabled)
+4. OpenRouter API integration with conversation history
+5. Gradio ChatInterface with access control
+
+### Development Files Not For Production
+- `gradio_docs.py`: MCP server integration (development only)
+- `test_*.py`: Testing utilities
+- `TEST_PROCEDURE.md`: Comprehensive testing methodology
+- `file_upload_proposal.md`: Technical architecture proposals
+
+## Known Issues and Compatibility
+
+### Gradio 4.44.1 JSON Schema Bug
+- **Issue**: TypeError in `json_schema_to_python_type` prevents app startup in some environments
+- **Symptom**: "argument of type 'bool' is not iterable" error during API schema generation
+- **Workaround**: Individual component functions work correctly (as verified by `test_preview.py`)
+- **Solution**: Upgrade to Gradio 5.x for full compatibility, or wait for Gradio 4.x patch
+
+### Message Format Compatibility
+- **Preview Mode**: Uses legacy tuple format `[user_msg, bot_msg]` for Gradio 4.44.1 ChatInterface
+- **Generated Spaces**: Use modern dictionary format for OpenRouter API compatibility
+- **Cross-Version Support**: Template generation handles both formats appropriately
+
+### Python Version Requirements
+- **Minimum**: Python 3.9 (for Gradio 4.44.1)
+- **Recommended**: Python 3.11+ (for Gradio 5.x and optimal performance)
+
+## Common Claude Code Anti-Patterns to Avoid
+
+### Message Format Reversion
+**❌ Don't revert to:** New dictionary format in preview functions
+```python
+# WRONG - breaks Gradio 4.44.1 ChatInterface
+history.append({"role": "user", "content": message})
+history.append({"role": "assistant", "content": response})
+```
+**✅ Keep:** Legacy tuple format for preview compatibility
+```python
+# CORRECT - works with current Gradio ChatInterface
+history.append([message, response])
+```
+
+### Template Variable Substitution
+**❌ Don't change:** Template string escaping patterns in `SPACE_TEMPLATE`
+- Keep double backslashes: `\\n\\n` (becomes `\n\n` after Python string processing)
+- Keep double braces: `{{variable}}` (becomes `{variable}` after format())
+- **Reason**: Template undergoes two levels of processing (Python format + HuggingFace deployment)
+
+### Research Template Function Signature
+**✅ Current Implementation:** Simplified function signature for research template
+```python
+# CURRENT - simplified toggle with direct system prompt management
+def toggle_research_assistant(enable_research):
+    if enable_research:
+        return (gr.update(value=combined_prompt), gr.update(value=True))
+    else:
+        return (gr.update(value=""), gr.update(value=False))
+```
+**❌ Don't revert to:** Complex field management patterns that are no longer needed
+- The research template no longer uses separate fields for role, audience, tasks, context
+- Current implementation directly manages system prompt and dynamic URL setting only
+
+### Import Organization Anti-Patterns
+**❌ Don't move:** `extract_urls_from_text()` back into template string
+- Function must remain in main app code (line 44) for preview functionality
+- Template version is for generated spaces only
+
+### URL Management Simplification
+**❌ Don't remove:** Dynamic URL add/remove functionality
+- Keep `add_urls()`, `remove_urls()`, `add_chat_urls()`, `remove_chat_urls()` functions
+- Maintain URL count state management with `gr.State()`
+- **Reason**: Users expect scalable URL input interface
+
+### Preview Functionality Degradation
+**❌ Don't revert to:** Simple mock responses in preview
+```python
+# WRONG - provides no real testing value
+def preview_chat_response(message, history, config_data):
+    return "", history + [[message, "Mock response"]]
+```
+**✅ Keep:** Real API integration with comprehensive debugging
+- Actual OpenRouter API calls when `OPENROUTER_API_KEY` is set
+- URL context fetching and processing
+- Configuration-aware responses using exact user settings
+- Comprehensive debugging for empty responses and API errors (lines 928-955)
+
+### Research Template Simplification
+**✅ Current Implementation:** Simplified research template system
+- Simple toggle functionality without complex field management
+- Direct system prompt injection for academic research use cases
+- Auto-enables dynamic URL fetching for academic sources
+- **Reason**: Simplified architecture reduces maintenance complexity while preserving core functionality
+
+### Conditional Dependency Loading
+**❌ Don't remove:** `HAS_RAG` flag and conditional imports
+```python
+# WRONG - breaks installations without vector dependencies
+from rag_tool import RAGTool
+```
+**✅ Keep:** Graceful degradation pattern
+```python
+# CORRECT - allows app to work without optional dependencies
+try:
+    from rag_tool import RAGTool
+    HAS_RAG = True
+except ImportError:
+    HAS_RAG = False
+    RAGTool = None
+```