Update app.py

app.py

@@ -252,7 +252,10 @@ def respond(message, history, model, temperature, num_calls, use_web_search):
 logging.basicConfig(level=logging.DEBUG)
 
 def get_response_from_cloudflare(query, num_calls=3, temperature=0.2):
-    headers = {"Authorization": f"Bearer {API_TOKEN}"}
+    headers = {
+        "Authorization": f"Bearer {API_TOKEN}",
+        "Content-Type": "application/json"
+    }
     model = "@cf/meta/llama-3.1-8b-instruct"
 
     logging.debug(f"API_BASE_URL: {API_BASE_URL}")
@@ -265,27 +268,33 @@ def get_response_from_cloudflare(query, num_calls=3, temperature=0.2):
         {"role": "user", "content": prompt}
     ]
 
+    payload = {
+        "messages": inputs,
+        "stream": True
+    }
+
     full_response = ""
     for i in range(num_calls):
         try:
-            response = requests.post(f"{API_BASE_URL}{model}", headers=headers, json={"messages": inputs})
-            
-            logging.debug(f"Cloudflare API Response Status: {response.status_code}")
-            logging.debug(f"Cloudflare API Response Headers: {response.headers}")
-            logging.debug(f"Cloudflare API Response Content: {response.text[:500]}...")  # First 500 characters
-            
-            if response.status_code == 200:
-                json_response = response.json()
-                if 'result' in json_response and 'response' in json_response['result']:
-                    chunk = json_response['result']['response']
-                    full_response += chunk
-                    yield full_response
+            with requests.post(f"{API_BASE_URL}{model}", headers=headers, json=payload, stream=True) as response:
+                logging.debug(f"Cloudflare API Response Status: {response.status_code}")
+                logging.debug(f"Cloudflare API Response Headers: {response.headers}")
+                
+                if response.status_code == 200:
+                    for line in response.iter_lines():
+                        if line:
+                            try:
+                                json_response = json.loads(line.decode('utf-8').split('data: ')[1])
+                                if 'response' in json_response:
+                                    chunk = json_response['response']
+                                    full_response += chunk
+                                    yield full_response
+                            except (json.JSONDecodeError, IndexError) as e:
+                                logging.error(f"Error parsing streaming response: {str(e)}")
+                                continue
                 else:
-                    logging.error(f"Unexpected JSON structure: {json_response}")
-                    yield "I apologize, but I received an unexpected response format. Please try again later."
-            else:
-                logging.error(f"HTTP Error: {response.status_code}, Response: {response.text}")
-                yield f"I apologize, but I encountered an HTTP error: {response.status_code}. Please try again later."
+                    logging.error(f"HTTP Error: {response.status_code}, Response: {response.text}")
+                    yield f"I apologize, but I encountered an HTTP error: {response.status_code}. Please try again later."
         except Exception as e:
             logging.error(f"Error in generating response from Cloudflare: {str(e)}")
             yield f"I apologize, but an error occurred: {str(e)}. Please try again later."
@@ -306,7 +315,7 @@ After writing the document, please provide a list of sources used in your respon
     if model == "@cf/meta/llama-3.1-8b-instruct":
         # Use Cloudflare API
         for response in get_response_from_cloudflare(prompt, num_calls, temperature):
-            yield response, ""  # Yield response without sources
+            yield response, ""  # Yield streaming response without sources
     else:
         # Use Hugging Face API
         client = InferenceClient(model, token=huggingface_token)