Update app.py

app.py

@@ -1,55 +1,34 @@
-from fastapi import FastAPI, HTTPException, Request
-from llama_cpp import Llama
+from fastapi import FastAPI, Form, HTMLResponse
+from fastapi.responses import StreamingResponse
 import time
-import os
 
 app = FastAPI()
 
-llm = Llama(model_path="llama-2-7b-chat.Q3_K_S.gguf", n_ctx=2048, n_batch=512, use_mlock=True, n_threads=8)
+def generate_responses(prompt):
+    from llama_cpp import Llama
 
-def typewrite(text, delay=0.01):
-    for char in text:
-        print(char, end='', flush=True)
-        time.sleep(delay)
-    print(end='', flush=True)  # Print newline to move to the next line
+    llm = Llama(model_path="llama-2-7b-chat.Q3_K_S.gguf", n_ctx=2048, n_batch=512, use_mlock=True, n_threads=8)
 
-@app.post("/chat")
-async def chat(request: Request):
-    try:
-        data = await request.json()
-        user_input = data.get("user_input")
-
-        if not user_input:
-            raise HTTPException(status_code=400, detail="Missing 'user_input' field in the request JSON.")
-
-        os.system("cls")
-        print("Chatbot by Aritra Roy & DVLH")
-
-        ask = user_input
-
-        prompt = f"Llama-2-Chat [INST] <<SYS>>You're an assistant named Tusti. You are Developed by Aritra Roy. Don't share any false information.<</SYS>> {ask} [/INST]"
-
-        output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True)
-
-        response_text_chunk = ''
+    output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True)
 
+    try:
         while True:
             try:
                 chunk = next(output_stream)
                 if chunk.get('choices') and chunk['choices'][0].get('text'):
-                    response_text_chunk += chunk['choices'][0]['text']
-                    typewrite(response_text_chunk, delay=0.00)
+                    response_text_chunk = chunk['choices'][0]['text']
+                    yield response_text_chunk
             except StopIteration:
                 break
+    except StopIteration:
+        pass
 
-        if ask == 'clear':
-            os.system("cls")
-
-        return {"response": response_text_chunk}
-
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
+@app.post("/chat")
+async def chat(input_text: str = Form(...)):
+    prompt = f"Llama-2-Chat [INST] <<SYS>>You're a assistant named Tusti.You are Developed by Aritra Roy.Don't share any false information.<</SYS>> {input_text} [/INST]"
+    return StreamingResponse(generate_responses(prompt), media_type="text/plain")
 
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=7860)
+
+    uvicorn.run(app, host="127.0.0.1", port=8000)