Update app.py

app.py

@@ -1,4 +1,5 @@
 from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import EventSourceResponse
 from llama_cpp import Llama
 import time
 import os
@@ -31,24 +32,23 @@ async def chat(request: Request):
 
         output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True)
 
-        response_text_chunk = ''
-
         async def generate_chunks():
-            nonlocal response_text_chunk
+            nonlocal output_stream
+            response_text_chunk = ''
             try:
                 while True:
                     chunk = next(output_stream)
                     if chunk.get('choices') and chunk['choices'][0].get('text'):
                         response_text_chunk += chunk['choices'][0]['text']
                         typewrite(response_text_chunk, delay=0.00)
-                        yield {"response": response_text_chunk}
+                        yield f"data: {response_text_chunk}\n\n"
             except StopIteration:
                 pass
 
         if ask == 'clear':
             os.system("cls")
 
-        return {"chunks": generate_chunks()}
+        return EventSourceResponse(generate_chunks())
 
     except Exception as e:
         print(f"Exception: {e}")