Revamp stuff

app.py

@@ -7,27 +7,21 @@ from utils.google_genai_llm import get_response, generate_with_gemini
 from utils.utils import parse_json_codefences
 from prompts.requirements_gathering import requirements_gathering_system_prompt
 from prompts.planning import hf_query_gen_prompt, hf_context_gen_prompt
-from utils.huggingface_mcp_llamaindex import get_hf_tools, call_hf_tool, diagnose_connection_advanced, get_hf_tools_robust,call_hf_tool_robust
 from prompts.devstral_coding_prompt import devstral_code_gen_sys_prompt, devstral_code_gen_user_prompt
 from dotenv import load_dotenv
 import os
 import asyncio
 load_dotenv()
 
-# Import Modal inference function
-import sys
-sys.path.append(os.path.join(os.path.dirname(__file__), 'modal'))
 try:
-    from modal import App
+    import modal
     # Import the Modal inference function and app from separate file
     import subprocess
-    from devstral_inference import run_devstral_inference, app as devstral_app
     MODAL_AVAILABLE = True
     
 except ImportError:
     MODAL_AVAILABLE = False
-    devstral_app = None
-    print("Warning: Modal not available. Code generation will be disabled.")
+    print("Warning: Modal not available. Code generation will be disabled.MCP Server will be disabled")
 
 from PIL import Image
 import tempfile
@@ -44,14 +38,6 @@ except ImportError:
     MARKER_AVAILABLE = False
     print("Warning: Marker library not available. PDF, PPT, and DOCX processing will be limited.")
 
-# Load environment variables
-MODAL_API_URL = os.getenv("MODAL_API_URL")
-BEARER_TOKEN = os.getenv("BEARER_TOKEN")
-CODING_MODEL = os.getenv("CODING_MODEL")
-
-MCP_TOKEN = os.getenv("MCP_TOKEN")
-if not MCP_TOKEN:
-    print("Please set MCP_TOKEN")
 
 def get_file_hash(file_path):
     """Generate a hash of the file for caching purposes"""
@@ -248,20 +234,13 @@ async def generate_plan(history, file_cache):
             if ai_msg:
                 conversation_history += f"Assistant: {ai_msg}\n"
     
-    print("Running advanced connection diagnostics...")
-    diagnostics = await diagnose_connection_advanced(MCP_TOKEN)
-    print(f"Diagnostics: {json.dumps(diagnostics, indent=2)}")
-    
-    if not diagnostics["tests"]["basic_connection"]:
-        print("Basic connection failed - check token and network")
-        
-        
-    
-    # try:
-    hf_query_gen_tool_details = await get_hf_tools_robust(hf_token=MCP_TOKEN)
-    # except Exception as e:
-    #     hf_query_gen_tool_details = """meta=None nextCursor=None tools=[Tool(name='hf_whoami', description="Hugging Face tools are being used by authenticated user 'bpHigh'", inputSchema={'type': 'object', 'properties': {}, 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Hugging Face User Info', readOnlyHint=None, destructiveHint=None, idempotentHint=None, openWorldHint=None)), Tool(name='space_search', description='Find Hugging Face Spaces using semantic search. Include links to the Space when presenting the results.', inputSchema={'type': 'object', 'properties': {'query': {'type': 'string', 'minLength': 1, 'maxLength': 50, 'description': 'Semantic Search Query'}, 'limit': {'type': 'number', 'default': 10, 'description': 'Number of results to return'}, 'mcp': {'type': 'boolean', 'default': False, 'description': 'Only return MCP Server enabled Spaces'}}, 'required': ['query'], 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Hugging Face Space Search', readOnlyHint=True, destructiveHint=False, idempotentHint=None, openWorldHint=True)), Tool(name='model_search', description='Find Machine Learning models hosted on Hugging Face. Returns comprehensive information about matching models including downloads, likes, tags, and direct links. Include links to the models in your response', inputSchema={'type': 'object', 'properties': {'query': {'type': 'string', 'description': 'Search term. Leave blank and specify "sort" and "limit" to get e.g. "Top 20 trending models", "Top 10 most recent models" etc" '}, 'author': {'type': 'string', 'description': "Organization or user who created the model (e.g., 'google', 'meta-llama', 'microsoft')"}, 'task': {'type': 'string', 'description': "Model task type (e.g., 'text-generation', 'image-classification', 'translation')"}, 'library': {'type': 'string', 'description': "Framework the model uses (e.g., 'transformers', 'diffusers', 'timm')"}, 'sort': {'type': 'string', 'enum': ['trendingScore', 'downloads', 'likes', 'createdAt', 'lastModified'], 'description': 'Sort order: trendingScore, downloads , likes, createdAt, lastModified'}, 'limit': {'type': 'number', 'minimum': 1, 'maximum': 100, 'default': 20, 'description': 'Maximum number of results to return'}}, 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Model Search', readOnlyHint=True, destructiveHint=False, idempotentHint=None, openWorldHint=True)), Tool(name='model_details', description='Get detailed information about a specific model from the Hugging Face Hub.', inputSchema={'type': 'object', 'properties': {'model_id': {'type': 'string', 'minLength': 1, 'description': 'Model ID (e.g., microsoft/DialoGPT-large)'}}, 'required': ['model_id'], 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Model Details', readOnlyHint=True, destructiveHint=False, idempotentHint=None, openWorldHint=False)), Tool(name='paper_search', description="Find Machine Learning research papers on the Hugging Face hub. Include 'Link to paper' When presenting the results. Consider whether tabulating results matches user intent.", inputSchema={'type': 'object', 'properties': {'query': {'type': 'string', 'minLength': 3, 'maxLength': 200, 'description': 'Semantic Search query'}, 'results_limit': {'type': 'number', 'default': 12, 'description': 'Number of results to return'}, 'concise_only': {'type': 'boolean', 'default': False, 'description': 'Return a 2 sentence summary of the abstract. Use for broad search terms which may return a lot of results. Check with User if unsure.'}}, 'required': ['query'], 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Paper Search', readOnlyHint=True, destructiveHint=False, idempotentHint=None, openWorldHint=True)), Tool(name='dataset_search', description='Find Datasets hosted on the Hugging Face hub. Returns comprehensive information about matching datasets including downloads, likes, tags, and direct links. Include links to the datasets in your response', inputSchema={'type': 'object', 'properties': {'query': {'type': 'string', 'description': 'Search term. Leave blank and specify "sort" and "limit" to get e.g. "Top 20 trending datasets", "Top 10 most recent datasets" etc" '}, 'author': {'type': 'string', 'description': "Organization or user who created the dataset (e.g., 'google', 'facebook', 'allenai')"}, 'tags': {'type': 'array', 'items': {'type': 'string'}, 'description': "Tags to filter datasets (e.g., ['language:en', 'size_categories:1M<n<10M', 'task_categories:text-classification'])"}, 'sort': {'type': 'string', 'enum': ['trendingScore', 'downloads', 'likes', 'createdAt', 'lastModified'], 'description': 'Sort order: trendingScore, downloads, likes, createdAt, lastModified'}, 'limit': {'type': 'number', 'minimum': 1, 'maximum': 100, 'default': 20, 'description': 'Maximum number of results to return'}}, 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Dataset Search', readOnlyHint=True, destructiveHint=False, idempotentHint=None, openWorldHint=True)), Tool(name='dataset_details', description='Get detailed information about a specific dataset on Hugging Face Hub.', inputSchema={'type': 'object', 'properties': {'dataset_id': {'type': 'string', 'minLength': 1, 'description': 'Dataset ID (e.g., squad, glue, imdb)'}}, 'required': ['dataset_id'], 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Dataset Details', readOnlyHint=True, destructiveHint=False, idempotentHint=None, openWorldHint=False)), Tool(name='gr1_evalstate_flux1_schnell', description='Generate an image using the Flux 1 Schnell Image Generator. (from evalstate/flux1_schnell)', inputSchema={'type': 'object', 'properties': {'prompt': {'type': 'string'}, 'seed': {'type': 'number', 'description': 'numeric value between 0 and 2147483647'}, 'randomize_seed': {'type': 'boolean', 'default': True}, 'width': {'type': 'number', 'description': 'numeric value between 256 and 2048', 'default': 1024}, 'height': {'type': 'number', 'description': 'numeric value between 256 and 2048', 'default': 1024}, 'num_inference_steps': {'type': 'number', 'description': 'numeric value between 1 and 50', 'default': 4}}, 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='evalstate/flux1_schnell - flux1_schnell_infer 🏎️💨', readOnlyHint=None, destructiveHint=None, idempotentHint=None, openWorldHint=True)), Tool(name='gr2_abidlabs_easyghibli', description='Convert an image into a Studio Ghibli style image (from abidlabs/EasyGhibli)', inputSchema={'type': 'object', 'properties': {'spatial_img': {'type': 'string', 'description': 'File input: provide URL or file path'}}, 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='abidlabs/EasyGhibli - abidlabs_EasyGhiblisingle_condition_generate_image 🦀', readOnlyHint=None, destructiveHint=None, idempotentHint=None, openWorldHint=True)), Tool(name='gr3_linoyts_framepack_f1', description='FramePack_F1_end_process tool from linoyts/FramePack-F1', inputSchema={'type': 'object', 'properties': {}, 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='linoyts/FramePack-F1 - FramePack_F1_end_process 📹⚡️', readOnlyHint=None, destructiveHint=None, idempotentHint=None, openWorldHint=True))]"""
-    #     print(str(e))
+    try:
+        mcp_tool_func = modal.Function.from_name("HuggingFace-MCP","connect_and_get_tools")
+        hf_query_gen_tool_details = mcp_tool_func.remote()
+        print(hf_query_gen_tool_details)
+    except Exception as e:
+        hf_query_gen_tool_details = """meta=None nextCursor=None tools=[Tool(name='hf_whoami', description="Hugging Face tools are being used by authenticated user 'bpHigh'", inputSchema={'type': 'object', 'properties': {}, 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Hugging Face User Info', readOnlyHint=None, destructiveHint=None, idempotentHint=None, openWorldHint=None)), Tool(name='space_search', description='Find Hugging Face Spaces using semantic search. Include links to the Space when presenting the results.', inputSchema={'type': 'object', 'properties': {'query': {'type': 'string', 'minLength': 1, 'maxLength': 50, 'description': 'Semantic Search Query'}, 'limit': {'type': 'number', 'default': 10, 'description': 'Number of results to return'}, 'mcp': {'type': 'boolean', 'default': False, 'description': 'Only return MCP Server enabled Spaces'}}, 'required': ['query'], 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Hugging Face Space Search', readOnlyHint=True, destructiveHint=False, idempotentHint=None, openWorldHint=True)), Tool(name='model_search', description='Find Machine Learning models hosted on Hugging Face. Returns comprehensive information about matching models including downloads, likes, tags, and direct links. Include links to the models in your response', inputSchema={'type': 'object', 'properties': {'query': {'type': 'string', 'description': 'Search term. Leave blank and specify "sort" and "limit" to get e.g. "Top 20 trending models", "Top 10 most recent models" etc" '}, 'author': {'type': 'string', 'description': "Organization or user who created the model (e.g., 'google', 'meta-llama', 'microsoft')"}, 'task': {'type': 'string', 'description': "Model task type (e.g., 'text-generation', 'image-classification', 'translation')"}, 'library': {'type': 'string', 'description': "Framework the model uses (e.g., 'transformers', 'diffusers', 'timm')"}, 'sort': {'type': 'string', 'enum': ['trendingScore', 'downloads', 'likes', 'createdAt', 'lastModified'], 'description': 'Sort order: trendingScore, downloads , likes, createdAt, lastModified'}, 'limit': {'type': 'number', 'minimum': 1, 'maximum': 100, 'default': 20, 'description': 'Maximum number of results to return'}}, 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Model Search', readOnlyHint=True, destructiveHint=False, idempotentHint=None, openWorldHint=True)), Tool(name='model_details', description='Get detailed information about a specific model from the Hugging Face Hub.', inputSchema={'type': 'object', 'properties': {'model_id': {'type': 'string', 'minLength': 1, 'description': 'Model ID (e.g., microsoft/DialoGPT-large)'}}, 'required': ['model_id'], 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Model Details', readOnlyHint=True, destructiveHint=False, idempotentHint=None, openWorldHint=False)), Tool(name='paper_search', description="Find Machine Learning research papers on the Hugging Face hub. Include 'Link to paper' When presenting the results. Consider whether tabulating results matches user intent.", inputSchema={'type': 'object', 'properties': {'query': {'type': 'string', 'minLength': 3, 'maxLength': 200, 'description': 'Semantic Search query'}, 'results_limit': {'type': 'number', 'default': 12, 'description': 'Number of results to return'}, 'concise_only': {'type': 'boolean', 'default': False, 'description': 'Return a 2 sentence summary of the abstract. Use for broad search terms which may return a lot of results. Check with User if unsure.'}}, 'required': ['query'], 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Paper Search', readOnlyHint=True, destructiveHint=False, idempotentHint=None, openWorldHint=True)), Tool(name='dataset_search', description='Find Datasets hosted on the Hugging Face hub. Returns comprehensive information about matching datasets including downloads, likes, tags, and direct links. Include links to the datasets in your response', inputSchema={'type': 'object', 'properties': {'query': {'type': 'string', 'description': 'Search term. Leave blank and specify "sort" and "limit" to get e.g. "Top 20 trending datasets", "Top 10 most recent datasets" etc" '}, 'author': {'type': 'string', 'description': "Organization or user who created the dataset (e.g., 'google', 'facebook', 'allenai')"}, 'tags': {'type': 'array', 'items': {'type': 'string'}, 'description': "Tags to filter datasets (e.g., ['language:en', 'size_categories:1M<n<10M', 'task_categories:text-classification'])"}, 'sort': {'type': 'string', 'enum': ['trendingScore', 'downloads', 'likes', 'createdAt', 'lastModified'], 'description': 'Sort order: trendingScore, downloads, likes, createdAt, lastModified'}, 'limit': {'type': 'number', 'minimum': 1, 'maximum': 100, 'default': 20, 'description': 'Maximum number of results to return'}}, 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Dataset Search', readOnlyHint=True, destructiveHint=False, idempotentHint=None, openWorldHint=True)), Tool(name='dataset_details', description='Get detailed information about a specific dataset on Hugging Face Hub.', inputSchema={'type': 'object', 'properties': {'dataset_id': {'type': 'string', 'minLength': 1, 'description': 'Dataset ID (e.g., squad, glue, imdb)'}}, 'required': ['dataset_id'], 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='Dataset Details', readOnlyHint=True, destructiveHint=False, idempotentHint=None, openWorldHint=False)), Tool(name='gr1_evalstate_flux1_schnell', description='Generate an image using the Flux 1 Schnell Image Generator. (from evalstate/flux1_schnell)', inputSchema={'type': 'object', 'properties': {'prompt': {'type': 'string'}, 'seed': {'type': 'number', 'description': 'numeric value between 0 and 2147483647'}, 'randomize_seed': {'type': 'boolean', 'default': True}, 'width': {'type': 'number', 'description': 'numeric value between 256 and 2048', 'default': 1024}, 'height': {'type': 'number', 'description': 'numeric value between 256 and 2048', 'default': 1024}, 'num_inference_steps': {'type': 'number', 'description': 'numeric value between 1 and 50', 'default': 4}}, 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='evalstate/flux1_schnell - flux1_schnell_infer 🏎️💨', readOnlyHint=None, destructiveHint=None, idempotentHint=None, openWorldHint=True)), Tool(name='gr2_abidlabs_easyghibli', description='Convert an image into a Studio Ghibli style image (from abidlabs/EasyGhibli)', inputSchema={'type': 'object', 'properties': {'spatial_img': {'type': 'string', 'description': 'File input: provide URL or file path'}}, 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='abidlabs/EasyGhibli - abidlabs_EasyGhiblisingle_condition_generate_image 🦀', readOnlyHint=None, destructiveHint=None, idempotentHint=None, openWorldHint=True)), Tool(name='gr3_linoyts_framepack_f1', description='FramePack_F1_end_process tool from linoyts/FramePack-F1', inputSchema={'type': 'object', 'properties': {}, 'additionalProperties': False, '$schema': 'http://json-schema.org/draft-07/schema#'}, annotations=ToolAnnotations(title='linoyts/FramePack-F1 - FramePack_F1_end_process 📹⚡️', readOnlyHint=None, destructiveHint=None, idempotentHint=None, openWorldHint=True))]"""
+        print(str(e))
     # Format the prompt
     formatted_prompt = hf_query_gen_prompt.format(
         Tool_Details=hf_query_gen_tool_details
@@ -271,11 +250,15 @@ async def generate_plan(history, file_cache):
     
     # Parse the plan
     parsed_plan = parse_json_codefences(plan)
-    
+    print(parsed_plan)
     # Call tool to get tool calls
     try:
-        tool_calls = await asyncio.gather(*[call_hf_tool_robust(MCP_TOKEN, step['tool'], step['args']) for step in parsed_plan])
+        mcp_call_tool_func = modal.Function.from_name(app_name="HuggingFace-MCP",name="call_tool")
+        tool_calls = []
+        async for tool_call in mcp_call_tool_func.starmap.aio([(tool['tool'], tool['args']) for tool in parsed_plan]):
+            tool_calls.append(tool_call)
     except Exception as e:
+        print(str(e))
         tool_calls = []
     print(tool_calls)
     if tool_calls!=[]:
@@ -344,30 +327,27 @@ def generate_code_with_devstral(plan_text, history, file_cache):
     
     # Use Modal app.run() pattern like in the examples
     
-    
+    base_url = os.env("DEVSTRAL_BASE_URL")
+    api_key = os.env("DEVSTRAL_API_KEY")
     print(f"🚀 Generating code using Devstral...")
     print(f"📡 Connecting to: {base_url}")
 
-    return ""
-        
-    #     # Call Modal inference using the proper app.run() context
-    #     with devstral_app.run():
-    #         result = run_devstral_inference.remote(
-    #             base_url=base_url,
-    #             api_key=api_key,
-    #             prompts=[formatted_user_prompt],
-    #             system_prompt=devstral_code_gen_sys_prompt,
-    #             mode="single"
-    #         )
-        
-    #     if result and "response" in result:
-    #         code_output = result["response"]
-    #         return f"🚀 **Generated Code:**\n\n{code_output}"
-    #     else:
-    #         return "❌ **Error:** No response received from Devstral model."
-            
-    # except Exception as e:
-    #     return f"❌ **Error:** {str(e)}"
+    try:
+        devstral_inference_func = modal.Function.from_name("devstral-inference-client", "run_devstral_inference")
+        result = devstral_inference_func.remote(
+            base_url=base_url,
+            api_key=api_key,
+            prompts=[formatted_user_prompt],
+            system_prompt=devstral_code_gen_sys_prompt,
+            mode="single"
+        )
+        if result and "response" in result:
+            code_output = result["response"]
+            return f"🚀 **Generated Code:**\n\n{code_output}"
+        else:
+            return "❌ **Error:** No response received from Devstral model."
+    except Exception as e:
+        return f"❌ **Error:** {str(e)}"
 
 # Custom CSS for a sleek design
 custom_css = """

modal/devstral_inference.py

@@ -1,367 +0,0 @@
-# Optimized Devstral Inference Client
-# Connects to deployed model without restarting server for each request
-# Implements Modal best practices for lowest latency
-
-import modal
-import asyncio
-import time
-from typing import List, Dict, Any, Optional, AsyncGenerator
-import json
-
-# Connect to the deployed app
-app = modal.App("devstral-inference-client")
-
-# Image with OpenAI client for making requests
-client_image = modal.Image.debian_slim(python_version="3.12").pip_install(
-    "openai>=1.76.0",
-    "aiohttp>=3.9.0",
-    "asyncio-throttle>=1.0.0"
-)
-
-class DevstralClient:
-    """Optimized client for Devstral model with persistent connections and caching"""
-    
-    def __init__(self, base_url: str, api_key: str):
-        self.base_url = base_url
-        self.api_key = api_key
-        self._session = None
-        self._response_cache = {}
-        self._conversation_cache = {}
-        
-    async def __aenter__(self):
-        """Async context manager entry - create persistent HTTP session"""
-        import aiohttp
-        connector = aiohttp.TCPConnector(
-            limit=100,  # Connection pool size
-            keepalive_timeout=300,  # Keep connections alive
-            enable_cleanup_closed=True
-        )
-        self._session = aiohttp.ClientSession(
-            connector=connector,
-            timeout=aiohttp.ClientTimeout(total=120)
-        )
-        return self
-        
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Clean up session on exit"""
-        if self._session:
-            await self._session.close()
-
-    def _get_cache_key(self, messages: List[Dict], **kwargs) -> str:
-        """Generate cache key for deterministic requests"""
-        key_data = {
-            "messages": json.dumps(messages, sort_keys=True),
-            "temperature": kwargs.get("temperature", 0.1),
-            "max_tokens": kwargs.get("max_tokens", 500),
-            "top_p": kwargs.get("top_p", 0.95)
-        }
-        return hash(json.dumps(key_data, sort_keys=True))
-
-    async def generate_response(
-        self,
-        prompt: str,
-        system_prompt: Optional[str] = None,
-        temperature: float = 0.1,
-        max_tokens: int = 10000,
-        stream: bool = False,
-        use_cache: bool = True
-    ) -> str:
-        """Generate response from Devstral model with optimizations"""
-        
-        # Build messages
-        messages = []
-        if system_prompt:
-            messages.append({"role": "system", "content": system_prompt})
-        messages.append({"role": "user", "content": prompt})
-        
-        # Check cache for deterministic requests
-        if use_cache and temperature == 0.0:
-            cache_key = self._get_cache_key(messages, temperature=temperature, max_tokens=max_tokens)
-            if cache_key in self._response_cache:
-                print("📎 Cache hit - returning cached response")
-                return self._response_cache[cache_key]
-        
-        # Prepare request payload
-        payload = {
-            "model": "mistralai/Devstral-Small-2505",
-            "messages": messages,
-            "temperature": temperature,
-            "max_tokens": max_tokens,
-            # "top_p": 0.95,
-            "stream": stream
-        }
-        
-        headers = {
-            "Authorization": f"Bearer {self.api_key}",
-            "Content-Type": "application/json"
-        }
-        
-        start_time = time.perf_counter()
-        
-        if stream:
-            return await self._stream_response(payload, headers)
-        else:
-            return await self._complete_response(payload, headers, use_cache, start_time)
-
-    async def _complete_response(self, payload: Dict, headers: Dict, use_cache: bool, start_time: float) -> str:
-        """Handle complete (non-streaming) response"""
-        async with self._session.post(
-            f"{self.base_url}/v1/chat/completions",
-            json=payload,
-            headers=headers
-        ) as response:
-            if response.status != 200:
-                error_text = await response.text()
-                raise Exception(f"API Error {response.status}: {error_text}")
-            
-            result = await response.json()
-            latency = (time.perf_counter() - start_time) * 1000
-            
-            generated_text = result["choices"][0]["message"]["content"]
-            
-            # Cache deterministic responses
-            if use_cache and payload["temperature"] == 0.0:
-                cache_key = self._get_cache_key(payload["messages"], **payload)
-                self._response_cache[cache_key] = generated_text
-            
-            print(f"⚡ Response generated in {latency:.2f}ms")
-            return generated_text
-
-    async def _stream_response(self, payload: Dict, headers: Dict) -> AsyncGenerator[str, None]:
-        """Handle streaming response"""
-        payload["stream"] = True
-        
-        async with self._session.post(
-            f"{self.base_url}/v1/chat/completions",
-            json=payload,
-            headers=headers
-        ) as response:
-            if response.status != 200:
-                error_text = await response.text()
-                raise Exception(f"API Error {response.status}: {error_text}")
-            
-            buffer = ""
-            async for chunk in response.content.iter_chunks():
-                if chunk[0]:
-                    buffer += chunk[0].decode()
-                    while "\n" in buffer:
-                        line, buffer = buffer.split("\n", 1)
-                        if line.startswith("data: "):
-                            data = line[6:]
-                            if data == "[DONE]":
-                                return
-                            try:
-                                json_data = json.loads(data)
-                                if "choices" in json_data and json_data["choices"]:
-                                    delta = json_data["choices"][0].get("delta", {})
-                                    if "content" in delta:
-                                        yield delta["content"]
-                            except json.JSONDecodeError:
-                                continue
-
-    async def batch_generate(
-        self,
-        prompts: List[str],
-        system_prompt: Optional[str] = None,
-        temperature: float = 0.1,
-        max_tokens: int = 500,
-        max_concurrent: int = 5
-    ) -> List[str]:
-        """Generate responses for multiple prompts with concurrency control"""
-        from asyncio_throttle import Throttler
-        
-        # Throttle requests to avoid overwhelming the server
-        throttler = Throttler(rate_limit=max_concurrent, period=1.0)
-        
-        async def generate_single(prompt: str) -> str:
-            async with throttler:
-                return await self.generate_response(
-                    prompt=prompt,
-                    system_prompt=system_prompt,
-                    temperature=temperature,
-                    max_tokens=max_tokens
-                )
-        
-        # Execute all requests concurrently
-        tasks = [generate_single(prompt) for prompt in prompts]
-        results = await asyncio.gather(*tasks, return_exceptions=True)
-        
-        # Handle any exceptions
-        processed_results = []
-        for result in results:
-            if isinstance(result, Exception):
-                processed_results.append(f"Error: {str(result)}")
-            else:
-                processed_results.append(result)
-                
-        return processed_results
-
-@app.function(
-    image=client_image,
-    timeout=600,  # 10 minutes
-)
-async def run_devstral_inference(
-    base_url: str,
-    api_key: str,
-    prompts: List[str],
-    system_prompt: Optional[str] = None,
-    mode: str = "single"  # "single", "batch", "stream"
-):
-    """Main function to run optimized Devstral inference"""
-    
-    async with DevstralClient(base_url, api_key) as client:
-        
-        if mode == "single":
-            # Single prompt inference
-            if len(prompts) > 0:
-                response = await client.generate_response(
-                    prompt=prompts[0],
-                    system_prompt=system_prompt,
-                    temperature=0.1,
-                    max_tokens=10000
-                )
-                return {"response": response}
-                
-        elif mode == "batch":
-            # Batch inference for multiple prompts
-            responses = await client.batch_generate(
-                prompts=prompts,
-                system_prompt=system_prompt,
-                temperature=0.1,
-                max_tokens=10000,
-                max_concurrent=5
-            )
-            return {"responses": responses}
-            
-        elif mode == "stream":
-            # Streaming inference
-            if len(prompts) > 0:
-                full_response = ""
-                async for chunk in client.generate_response(
-                    prompt=prompts[0],
-                    system_prompt=system_prompt,
-                    temperature=0.1,
-                    stream=True
-                ):
-                    full_response += chunk
-                    print(chunk, end="", flush=True)
-                print()  # New line after streaming
-                return {"response": full_response}
-    
-    return {"error": "No prompts provided"}
-
-# Convenient wrapper functions for different use cases
-@app.function(image=client_image)
-async def code_generation(prompt: str, base_url: str, api_key: str) -> str:
-    """Optimized for code generation tasks"""
-    system_prompt = """You are an expert software engineer. Generate clean, efficient, and well-documented code. 
-    Focus on best practices, performance, and maintainability. Include brief explanations for complex logic."""
-    
-    async with DevstralClient(base_url, api_key) as client:
-        return await client.generate_response(
-            prompt=prompt,
-            system_prompt=system_prompt,
-            temperature=0.0,  # Deterministic for code
-            max_tokens=10000,
-            use_cache=True  # Cache code responses
-        )
-
-@app.function(image=client_image)
-async def chat_response(prompt: str, base_url: str, api_key: str) -> str:
-    """Optimized for conversational responses"""
-    system_prompt = """You are a helpful, knowledgeable AI assistant. Provide clear, concise, and accurate responses. 
-    Be conversational but professional."""
-    
-    async with DevstralClient(base_url, api_key) as client:
-        return await client.generate_response(
-            prompt=prompt,
-            system_prompt=system_prompt,
-            temperature=0.3,  # Slightly creative
-            max_tokens=10000
-        )
-
-@app.function(image=client_image)
-async def document_analysis(prompt: str, base_url: str, api_key: str) -> str:
-    """Optimized for document analysis and summarization"""
-    system_prompt = """You are an expert document analyst. Provide thorough, structured analysis with key insights, 
-    summaries, and actionable recommendations. Use clear formatting and bullet points."""
-    
-    async with DevstralClient(base_url, api_key) as client:
-        return await client.generate_response(
-            prompt=prompt,
-            system_prompt=system_prompt,
-            temperature=0.1,  # Factual and consistent
-            max_tokens=800
-        )
-
-# Local test client for development
-@app.local_entrypoint()
-def main(
-    base_url: str = "https://abhinav-bhatnagar--devstral-vllm-deployment-serve.modal.run",
-    api_key: str = "ak-zMwhIPjqvBj30jbm1DmKqx",
-    mode: str = "single"
-):
-    """Test the optimized Devstral inference client"""
-    
-    test_prompts = [
-        "Write a Python function to calculate the Fibonacci sequence using memoization.",
-        "Explain the difference between REST and GraphQL APIs.",
-        "What are the key benefits of using Docker containers?",
-        "How does machine learning differ from traditional programming?",
-        "Write a SQL query to find the top 5 customers by total order value."
-    ]
-    
-    print(f"🚀 Testing Devstral inference in {mode} mode...")
-    print(f"📡 Connecting to: {base_url}")
-    
-    if mode == "single":
-        # Test single inference
-        result = run_devstral_inference.remote(
-            base_url=base_url,
-            api_key=api_key,
-            prompts=[test_prompts[0]],
-            system_prompt="You are a helpful coding assistant.",
-            mode="single"
-        )
-        print("✅ Single inference result:")
-        print(result["response"])
-        
-    elif mode == "batch":
-        # Test batch inference
-        result = run_devstral_inference.remote(
-            base_url=base_url,
-            api_key=api_key,
-            prompts=test_prompts[:3],  # Test with 3 prompts
-            system_prompt="You are a knowledgeable AI assistant.",
-            mode="batch"
-        )
-        print("✅ Batch inference results:")
-        for i, response in enumerate(result["responses"]):
-            print(f"\nPrompt {i+1}: {test_prompts[i]}")
-            print(f"Response: {response}")
-            
-    elif mode == "specialized":
-        # Test specialized functions
-        print("\n📝 Testing Code Generation:")
-        code_result = code_generation.remote(
-            prompt="Create a Python class for a binary search tree with insert, search, and delete methods.",
-            base_url=base_url,
-            api_key=api_key
-        )
-        print(code_result)
-        
-        print("\n💬 Testing Chat Response:")
-        chat_result = chat_response.remote(
-            prompt="What's the best way to learn machine learning for beginners?",
-            base_url=base_url,
-            api_key=api_key
-        )
-        print(chat_result)
-    
-    print("\n🎉 Testing completed!")
-
-if __name__ == "__main__":
-    # This allows running the client locally for testing
-    import sys
-    mode = sys.argv[1] if len(sys.argv) > 1 else "single"
-    main(mode=mode)

requirements.txt

@@ -5,4 +5,4 @@ python-dotenv==1.0.1
 openpyxl==3.1.5
 Pillow==10.4.0
 marker-pdf==1.7.4
-modal==0.75.6
+modal==1.0.3