Add URL caching system to prevent re-crawling in conversations

- Implement global url_content_cache dictionary
- Add get_cached_grounding_context function with cache key generation
- Add cache status display and clear functionality
- Optimize performance by avoiding duplicate URL fetches
- Cache persists across conversation turns

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>

app.py

@@ -26,6 +26,8 @@ SPACE_DESCRIPTION = "{description}"
 SYSTEM_PROMPT = """{system_prompt}"""
 MODEL = "{model}"
 GROUNDING_URLS = {grounding_urls}
+ACCESS_CODE = "{access_code}"
+ENABLE_DYNAMIC_URLS = {enable_dynamic_urls}
 
 # Get API key from environment - customizable variable name
 API_KEY = os.environ.get("{api_key_var}")
@@ -63,11 +65,21 @@ def fetch_url_content(url):
     except Exception as e:
         return f"Error fetching {{url}}: {{str(e)}}"
 
+# Global cache for URL content to avoid re-crawling in generated spaces
+_url_content_cache = {{}}
+
 def get_grounding_context():
-    """Fetch context from grounding URLs"""
+    """Fetch context from grounding URLs with caching"""
     if not GROUNDING_URLS:
         return ""
     
+    # Create cache key from URLs
+    cache_key = tuple(sorted([url for url in GROUNDING_URLS if url and url.strip()]))
+    
+    # Check cache first
+    if cache_key in _url_content_cache:
+        return _url_content_cache[cache_key]
+    
     context_parts = []
     for i, url in enumerate(GROUNDING_URLS, 1):
         if url.strip():
@@ -75,8 +87,20 @@ def get_grounding_context():
             context_parts.append(f"Context from URL {{i}} ({{url}}):\\n{{content}}")
     
     if context_parts:
-        return "\\n\\n" + "\\n\\n".join(context_parts) + "\\n\\n"
-    return ""
+        result = "\\n\\n" + "\\n\\n".join(context_parts) + "\\n\\n"
+    else:
+        result = ""
+    
+    # Cache the result
+    _url_content_cache[cache_key] = result
+    return result
+
+import re
+
+def extract_urls_from_text(text):
+    """Extract URLs from text using regex"""
+    url_pattern = r'https?://[^\\s<>"{{}}|\\^`\\[\\]"]+'
+    return re.findall(url_pattern, text)
 
 def generate_response(message, history):
     """Generate response using OpenRouter API"""
@@ -87,6 +111,18 @@ def generate_response(message, history):
     # Get grounding context
     grounding_context = get_grounding_context()
     
+    # If dynamic URLs are enabled, check message for URLs to fetch
+    if ENABLE_DYNAMIC_URLS:
+        urls_in_message = extract_urls_from_text(message)
+        if urls_in_message:
+            # Fetch content from URLs mentioned in the message
+            dynamic_context_parts = []
+            for url in urls_in_message[:3]:  # Limit to 3 URLs per message
+                content = fetch_url_content(url)
+                dynamic_context_parts.append(f"\\n\\nDynamic context from {{url}}:\\n{{content}}")
+            if dynamic_context_parts:
+                grounding_context += "\\n".join(dynamic_context_parts)
+    
     # Build enhanced system prompt with grounding context
     enhanced_system_prompt = SYSTEM_PROMPT + grounding_context
     
@@ -132,13 +168,64 @@ def generate_response(message, history):
     except Exception as e:
         return f"Error: {{str(e)}}"
 
-# Create simple Gradio interface using ChatInterface
-demo = gr.ChatInterface(
-    fn=generate_response,
-    title=SPACE_NAME,
-    description=SPACE_DESCRIPTION,
-    examples={examples}
-)
+# Access code verification
+access_granted = gr.State(False)
+
+def verify_access_code(code):
+    \"\"\"Verify the access code\"\"\"
+    if not ACCESS_CODE:
+        return gr.update(visible=False), gr.update(visible=True), True
+    
+    if code == ACCESS_CODE:
+        return gr.update(visible=False), gr.update(visible=True), True
+    else:
+        return gr.update(visible=True, value="❌ Incorrect access code. Please try again."), gr.update(visible=False), False
+
+def protected_generate_response(message, history, access_state):
+    \"\"\"Protected response function that checks access\"\"\"
+    if not access_state:
+        return "Please enter the access code to continue."
+    return generate_response(message, history)
+
+# Create interface with access code protection
+with gr.Blocks(title=SPACE_NAME) as demo:
+    gr.Markdown(f"# {{SPACE_NAME}}")
+    gr.Markdown(SPACE_DESCRIPTION)
+    
+    # Access code section (shown only if ACCESS_CODE is set)
+    with gr.Column(visible=bool(ACCESS_CODE)) as access_section:
+        gr.Markdown("### 🔐 Access Required")
+        gr.Markdown("Please enter the access code provided by your instructor:")
+        
+        access_input = gr.Textbox(
+            label="Access Code",
+            placeholder="Enter access code...",
+            type="password"
+        )
+        access_btn = gr.Button("Submit", variant="primary")
+        access_error = gr.Markdown(visible=False)
+    
+    # Main chat interface (hidden until access granted)
+    with gr.Column(visible=not bool(ACCESS_CODE)) as chat_section:
+        chat_interface = gr.ChatInterface(
+            fn=lambda msg, hist: protected_generate_response(msg, hist, access_granted.value),
+            title="",  # Title already shown above
+            description="",  # Description already shown above
+            examples={examples}
+        )
+    
+    # Connect access verification
+    if ACCESS_CODE:
+        access_btn.click(
+            verify_access_code,
+            inputs=[access_input],
+            outputs=[access_error, chat_section, access_granted]
+        )
+        access_input.submit(
+            verify_access_code,
+            inputs=[access_input],
+            outputs=[access_error, chat_section, access_granted]
+        )
 
 if __name__ == "__main__":
     demo.launch()
@@ -237,13 +324,25 @@ pinned: false
 5. Value: Your OpenRouter API key
 6. Click "Add"
 
-### Step 4: Get Your API Key
+{f'''### Step 4: Configure Access Control (Optional)
+Your Space is configured with access code protection. Students will need to enter the access code to use the chatbot.
+
+**Access Code**: `{config['access_code']}`
+
+To disable access protection:
+1. Edit `app.py` in your Space
+2. Change `ACCESS_CODE = "{config['access_code']}"` to `ACCESS_CODE = ""`
+3. The Space will rebuild automatically
+
+''' if config['access_code'] else ''}
+
+### Step {4 if not config['access_code'] else 5}: Get Your API Key
 1. Go to https://openrouter.ai/keys
 2. Sign up/login if needed
 3. Click "Create Key"
 4. Copy the key (starts with `sk-or-`)
 
-### Step 5: Test Your Space
+### Step {5 if not config['access_code'] else 6}: Test Your Space
 - Go back to "App" tab
 - Your Space should be running!
 - Try the example prompts or ask a question
@@ -253,7 +352,9 @@ pinned: false
 - **Model**: {config['model']}
 - **Temperature**: {config['temperature']}
 - **Max Tokens**: {config['max_tokens']}
-- **API Key Variable**: {config['api_key_var']}
+- **API Key Variable**: {config['api_key_var']}{f"""
+- **Access Code**: {config['access_code']} (Students need this to access the chatbot)""" if config['access_code'] else ""}{f"""
+- **Dynamic URL Fetching**: Enabled (Assistant can fetch URLs mentioned in conversations)""" if config.get('enable_dynamic_urls') else ""}
 
 ## Customization
 
@@ -284,7 +385,7 @@ def create_requirements():
     """Generate requirements.txt"""
     return "gradio==4.44.1\nrequests==2.32.3\ncrawl4ai==0.4.245"
 
-def generate_zip(name, description, system_prompt, model, api_key_var, temperature, max_tokens, examples_text, url1="", url2="", url3="", url4=""):
+def generate_zip(name, description, system_prompt, model, api_key_var, temperature, max_tokens, examples_text, access_code="", enable_dynamic_urls=False, url1="", url2="", url3="", url4=""):
     """Generate deployable zip file"""
     
     # Process examples
@@ -314,7 +415,9 @@ def generate_zip(name, description, system_prompt, model, api_key_var, temperatu
         'temperature': temperature,
         'max_tokens': int(max_tokens),
         'examples': examples_json,
-        'grounding_urls': json.dumps(grounding_urls)
+        'grounding_urls': json.dumps(grounding_urls),
+        'access_code': access_code or "",
+        'enable_dynamic_urls': enable_dynamic_urls
     }
     
     # Generate files
@@ -341,7 +444,7 @@ def generate_zip(name, description, system_prompt, model, api_key_var, temperatu
     return filename
 
 # Define callback functions outside the interface
-def on_generate(name, description, system_prompt, model, api_key_var, temperature, max_tokens, examples_text, url1, url2, url3, url4):
+def on_generate(name, description, system_prompt, model, api_key_var, temperature, max_tokens, examples_text, access_code, enable_dynamic_urls, url1, url2, url3, url4):
     if not name or not name.strip():
         return gr.update(value="Error: Please provide a Space Title", visible=True), gr.update(visible=False)
     
@@ -349,7 +452,7 @@ def on_generate(name, description, system_prompt, model, api_key_var, temperatur
         return gr.update(value="Error: Please provide a System Prompt", visible=True), gr.update(visible=False)
     
     try:
-        filename = generate_zip(name, description, system_prompt, model, api_key_var, temperature, max_tokens, examples_text, url1, url2, url3, url4)
+        filename = generate_zip(name, description, system_prompt, model, api_key_var, temperature, max_tokens, examples_text, access_code, enable_dynamic_urls, url1, url2, url3, url4)
         
         success_msg = f"""**Deployment package ready!**
 
@@ -373,6 +476,40 @@ def on_generate(name, description, system_prompt, model, api_key_var, temperatur
     except Exception as e:
         return gr.update(value=f"Error: {str(e)}", visible=True), gr.update(visible=False)
 
+# Global cache for URL content to avoid re-crawling
+url_content_cache = {}
+
+def get_cached_grounding_context(urls):
+    """Get grounding context with caching to avoid re-crawling same URLs"""
+    if not urls:
+        return ""
+    
+    # Filter valid URLs
+    valid_urls = [url for url in urls if url and url.strip()]
+    if not valid_urls:
+        return ""
+    
+    # Create cache key from sorted URLs
+    cache_key = tuple(sorted(valid_urls))
+    
+    # Check if we already have this content cached
+    if cache_key in url_content_cache:
+        return url_content_cache[cache_key]
+    
+    # If not cached, fetch using Crawl4AI
+    grounding_context = get_grounding_context_crawl4ai(valid_urls)
+    
+    # Cache the result
+    url_content_cache[cache_key] = grounding_context
+    
+    return grounding_context
+
+def respond_with_cache_update(message, chat_history, url1="", url2="", url3="", url4=""):
+    """Wrapper that updates cache status after responding"""
+    msg, history = respond(message, chat_history, url1, url2, url3, url4)
+    cache_status = get_cache_status()
+    return msg, history, cache_status
+
 def respond(message, chat_history, url1="", url2="", url3="", url4=""):
     # Make actual API request to OpenRouter
     import os
@@ -386,9 +523,9 @@ def respond(message, chat_history, url1="", url2="", url3="", url4=""):
         chat_history.append([message, response])
         return "", chat_history
     
-    # Get grounding context from URLs using Crawl4AI
+    # Get grounding context from URLs using cached approach
     grounding_urls = [url1, url2, url3, url4]
-    grounding_context = get_grounding_context_crawl4ai(grounding_urls)
+    grounding_context = get_cached_grounding_context(grounding_urls)
     
     # Build enhanced system prompt with grounding context
     base_system_prompt = """You are an expert assistant specializing in Gradio configurations for HuggingFace Spaces. You have deep knowledge of:
@@ -409,9 +546,13 @@ Provide specific, technical guidance focused on Gradio implementation details an
         "content": enhanced_system_prompt
     }]
     
-    # Add conversation history - Gradio 4.x uses list/tuple format
+    # Add conversation history - Support both new messages format and legacy tuple format
     for chat in chat_history:
-        if isinstance(chat, (list, tuple)) and len(chat) >= 2:
+        if isinstance(chat, dict):
+            # New format: {"role": "user", "content": "..."}
+            messages.append(chat)
+        elif isinstance(chat, (list, tuple)) and len(chat) >= 2:
+            # Legacy format: ("user msg", "bot msg")
             user_msg, assistant_msg = chat[0], chat[1]
             if user_msg:
                 messages.append({"role": "user", "content": user_msg})
@@ -430,7 +571,7 @@ Provide specific, technical guidance focused on Gradio implementation details an
                 "Content-Type": "application/json"
             },
             json={
-                "model": "google/gemma-3-27b-it",
+                "model": "google/gemini-2.0-flash-001",
                 "messages": messages,
                 "temperature": 0.7,
                 "max_tokens": 500
@@ -445,12 +586,69 @@ Provide specific, technical guidance focused on Gradio implementation details an
     except Exception as e:
         assistant_response = f"Error: {str(e)}"
     
-    chat_history.append([message, assistant_response])
+    chat_history.append({"role": "user", "content": message})
+    chat_history.append({"role": "assistant", "content": assistant_response})
     return "", chat_history
 
 def clear_chat():
     return "", []
 
+def clear_url_cache():
+    """Clear the URL content cache"""
+    global url_content_cache
+    url_content_cache.clear()
+    return "✅ URL cache cleared. Next request will re-fetch content."
+
+def get_cache_status():
+    """Get current cache status"""
+    if not url_content_cache:
+        return "🔄 No URLs cached"
+    return f"💾 {len(url_content_cache)} URL set(s) cached"
+
+def add_urls(count):
+    """Show additional URL fields"""
+    if count == 2:
+        return (gr.update(visible=True), gr.update(visible=False), 
+                gr.update(value="+ Add URLs"), gr.update(visible=True), 3)
+    elif count == 3:
+        return (gr.update(visible=True), gr.update(visible=True), 
+                gr.update(value="Max URLs", interactive=False), gr.update(visible=True), 4)
+    else:
+        return (gr.update(), gr.update(), gr.update(), gr.update(), count)
+
+def remove_urls(count):
+    """Hide URL fields"""
+    if count == 4:
+        return (gr.update(visible=True), gr.update(visible=False, value=""), 
+                gr.update(value="+ Add URLs", interactive=True), gr.update(visible=True), 3)
+    elif count == 3:
+        return (gr.update(visible=False, value=""), gr.update(visible=False, value=""), 
+                gr.update(value="+ Add URLs", interactive=True), gr.update(visible=False), 2)
+    else:
+        return (gr.update(), gr.update(), gr.update(), gr.update(), count)
+
+def add_chat_urls(count):
+    """Show additional chat URL fields"""
+    if count == 2:
+        return (gr.update(visible=True), gr.update(visible=False), 
+                gr.update(value="+ Add URLs"), gr.update(visible=True), 3)
+    elif count == 3:
+        return (gr.update(visible=True), gr.update(visible=True), 
+                gr.update(value="Max URLs", interactive=False), gr.update(visible=True), 4)
+    else:
+        return (gr.update(), gr.update(), gr.update(), gr.update(), count)
+
+def remove_chat_urls(count):
+    """Hide chat URL fields"""
+    if count == 4:
+        return (gr.update(visible=True), gr.update(visible=False, value=""), 
+                gr.update(value="+ Add URLs", interactive=True), gr.update(visible=True), 3)
+    elif count == 3:
+        return (gr.update(visible=False, value=""), gr.update(visible=False, value=""), 
+                gr.update(value="+ Add URLs", interactive=True), gr.update(visible=False), 2)
+    else:
+        return (gr.update(), gr.update(), gr.update(), gr.update(), count)
+
 # Create Gradio interface with proper tab structure
 with gr.Blocks(title="Chat U/I Helper") as demo:
     with gr.Tabs():
@@ -485,11 +683,24 @@ with gr.Blocks(title="Chat U/I Helper") as demo:
                     info="Name for the secret in HuggingFace Space settings"
                 )
                 
+                access_code = gr.Textbox(
+                    label="Access Code (Optional)",
+                    placeholder="Leave empty for public access, or enter code for student access",
+                    info="If set, students must enter this code to access the chatbot",
+                    type="password"
+                )
+                
                 system_prompt = gr.Textbox(
                     label="System Prompt",
                     placeholder="You are a research assistant...",
                     lines=4,
-                    value="You are a knowledgeable academic research assistant. Provide thoughtful, evidence-based guidance for scholarly work, literature reviews, and academic writing. Support students and researchers with clear explanations and critical thinking."
+                    value="You are a research assistant that provides link-grounded information through Crawl4AI web fetching. Every claim must be supported by retrieved sources: fetch content from URLs to verify accuracy, and cite with precise links—never rely on memory for facts. Ground your responses in the provided URL contexts and any additional URLs you're instructed to fetch."
+                )
+                
+                enable_dynamic_urls = gr.Checkbox(
+                    label="Enable Dynamic URL Fetching",
+                    value=False,
+                    info="Allow the assistant to fetch additional URLs mentioned in conversations (uses Crawl4AI)"
                 )
                 
                 examples_text = gr.Textbox(
@@ -499,32 +710,42 @@ with gr.Blocks(title="Chat U/I Helper") as demo:
                     info="These will appear as clickable examples in the chat interface"
                 )
                 
-                gr.Markdown("### URL Grounding (Optional)")
-                gr.Markdown("Add up to 4 URLs to provide context. Content will be fetched and added to the system prompt.")
-                
-                url1 = gr.Textbox(
-                    label="URL 1",
-                    placeholder="https://example.com/page1",
-                    info="First URL for context grounding"
-                )
-                
-                url2 = gr.Textbox(
-                    label="URL 2", 
-                    placeholder="https://example.com/page2",
-                    info="Second URL for context grounding"
-                )
-                
-                url3 = gr.Textbox(
-                    label="URL 3",
-                    placeholder="https://example.com/page3", 
-                    info="Third URL for context grounding"
-                )
-                
-                url4 = gr.Textbox(
-                    label="URL 4",
-                    placeholder="https://example.com/page4",
-                    info="Fourth URL for context grounding"
-                )
+                with gr.Accordion("URL Grounding (Optional)", open=False):
+                    gr.Markdown("Add URLs to provide context. Content will be fetched and added to the system prompt.")
+                    
+                    # Initial URL fields
+                    url1 = gr.Textbox(
+                        label="URL 1",
+                        placeholder="https://example.com/page1",
+                        info="First URL for context grounding"
+                    )
+                    
+                    url2 = gr.Textbox(
+                        label="URL 2", 
+                        placeholder="https://example.com/page2",
+                        info="Second URL for context grounding"
+                    )
+                    
+                    # Additional URL fields (initially hidden)
+                    url3 = gr.Textbox(
+                        label="URL 3",
+                        placeholder="https://example.com/page3", 
+                        info="Third URL for context grounding",
+                        visible=False
+                    )
+                    
+                    url4 = gr.Textbox(
+                        label="URL 4",
+                        placeholder="https://example.com/page4",
+                        info="Fourth URL for context grounding",
+                        visible=False
+                    )
+                    
+                    # URL management buttons
+                    with gr.Row():
+                        add_url_btn = gr.Button("+ Add URLs", size="sm")
+                        remove_url_btn = gr.Button("- Remove URLs", size="sm", visible=False)
+                    url_count = gr.State(2)  # Track number of visible URLs
                 
                 with gr.Row():
                     temperature = gr.Slider(
@@ -549,10 +770,23 @@ with gr.Blocks(title="Chat U/I Helper") as demo:
                 status = gr.Markdown(visible=False)
                 download_file = gr.File(label="Download your zip package", visible=False)
             
+            # Connect the URL management buttons
+            add_url_btn.click(
+                add_urls,
+                inputs=[url_count],
+                outputs=[url3, url4, add_url_btn, remove_url_btn, url_count]
+            )
+            
+            remove_url_btn.click(
+                remove_urls,
+                inputs=[url_count],
+                outputs=[url3, url4, add_url_btn, remove_url_btn, url_count]
+            )
+            
             # Connect the generate button
             generate_btn.click(
                 on_generate,
-                inputs=[name, description, system_prompt, model, api_key_var, temperature, max_tokens, examples_text, url1, url2, url3, url4],
+                inputs=[name, description, system_prompt, model, api_key_var, temperature, max_tokens, examples_text, access_code, enable_dynamic_urls, url1, url2, url3, url4],
                 outputs=[status, download_file]
             )
         
@@ -564,8 +798,9 @@ with gr.Blocks(title="Chat U/I Helper") as demo:
             with gr.Column():
                 chatbot = gr.Chatbot(
                     value=[],
-                    label="Chat Support Assistant",
-                    height=400
+                    label="Chat Support Assistant", 
+                    height=400,
+                    type="messages"
                 )
                 msg = gr.Textbox(
                     label="Ask about configuring chat UIs for courses, research, or custom HuggingFace Spaces",
@@ -582,19 +817,37 @@ with gr.Blocks(title="Chat U/I Helper") as demo:
                     )
                     chat_url2 = gr.Textbox(
                         label="URL 2", 
-                        value="https://huggingface.co/docs/hub/en/spaces-config-reference",
-                        info="Spaces Configuration Reference"
+                        value="",
+                        placeholder="https://example.com/page2",
+                        info="Additional context URL"
                     )
+                    
+                    # Additional URL fields for chat (initially hidden)
                     chat_url3 = gr.Textbox(
                         label="URL 3",
-                        value="https://huggingface.co/docs/hub/en/spaces-settings",
-                        info="Spaces Settings Documentation"
+                        placeholder="https://example.com/page3",
+                        info="Additional context URL",
+                        visible=False
                     )
+                    
                     chat_url4 = gr.Textbox(
                         label="URL 4",
-                        value="https://huggingface.co/docs/hub/en/spaces-sdks-gradio",
-                        info="Gradio SDK for Spaces"
+                        placeholder="https://example.com/page4",
+                        info="Additional context URL",
+                        visible=False
                     )
+                    
+                    # Chat URL management buttons
+                    with gr.Row():
+                        add_chat_url_btn = gr.Button("+ Add URLs", size="sm")
+                        remove_chat_url_btn = gr.Button("- Remove URLs", size="sm", visible=False)
+                    chat_url_count = gr.State(2)  # Track number of visible chat URLs
+                    
+                    # Cache controls
+                    with gr.Row():
+                        cache_status = gr.Markdown("🔄 No URLs cached")
+                        clear_cache_btn = gr.Button("Clear URL Cache", size="sm")
+                
                 with gr.Row():
                     submit = gr.Button("Send", variant="primary")
                     clear = gr.Button("Clear")
@@ -611,9 +864,25 @@ with gr.Blocks(title="Chat U/I Helper") as demo:
                     inputs=msg
                 )
             
+            # Connect the chat URL management buttons
+            add_chat_url_btn.click(
+                add_chat_urls,
+                inputs=[chat_url_count],
+                outputs=[chat_url3, chat_url4, add_chat_url_btn, remove_chat_url_btn, chat_url_count]
+            )
+            
+            remove_chat_url_btn.click(
+                remove_chat_urls,
+                inputs=[chat_url_count],
+                outputs=[chat_url3, chat_url4, add_chat_url_btn, remove_chat_url_btn, chat_url_count]
+            )
+            
+            # Connect cache controls
+            clear_cache_btn.click(clear_url_cache, outputs=[cache_status])
+            
             # Connect the chat functionality
-            submit.click(respond, [msg, chatbot, chat_url1, chat_url2, chat_url3, chat_url4], [msg, chatbot])
-            msg.submit(respond, [msg, chatbot, chat_url1, chat_url2, chat_url3, chat_url4], [msg, chatbot])
+            submit.click(respond_with_cache_update, [msg, chatbot, chat_url1, chat_url2, chat_url3, chat_url4], [msg, chatbot, cache_status])
+            msg.submit(respond_with_cache_update, [msg, chatbot, chat_url1, chat_url2, chat_url3, chat_url4], [msg, chatbot, cache_status])
             clear.click(clear_chat, outputs=[msg, chatbot])
 
 if __name__ == "__main__":