Update app.py

app.py

@@ -1,2 +1,50 @@
+from fastapi import FastAPI, HTTPException
+from llama_cpp import Llama
+import time
 import os
-os.system('autotrain Ilm --train --ARLLAMa --model TinyPixel/ Llama-2-7B-bf16-sharded --timdettmers/openassistant-guanaco -use_peft-use_int4 --learning_rate 2e-4 --train_batch_size 2 --num_train_epochs 3 --trainer sft --model_max_length 2048')
+
+app = FastAPI()
+
+llm = Llama(model_path="llama-2-7b-chat.Q3_K_S.gguf", n_ctx=2048, n_batch=512, use_mlock=True, n_threads=8)
+
+def typewrite(text, delay=0.01):
+    for char in text:
+        print(char, end='', flush=True)
+        time.sleep(delay)
+    print(end='', flush=True)  # Print newline to move to the next line
+
+@app.post("/chat")
+async def chat(user_input: str):
+    try:
+        os.system("cls")
+        print("Chatbot by Aritra Roy & DVLH")
+
+        ask = user_input
+
+        prompt = f"Llama-2-Chat [INST] <<SYS>>You're an assistant named Tusti. You are Developed by Aritra Roy. Don't share any false information.<</SYS>> {ask} [/INST]"
+
+        # Send the request with stream=True
+        output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True)
+
+        response_text_chunk = ''
+
+        while True:
+            try:
+                chunk = next(output_stream)
+                if chunk.get('choices') and chunk['choices'][0].get('text'):
+                    response_text_chunk += chunk['choices'][0]['text']
+                    typewrite(response_text_chunk, delay=0.00)  # live response
+            except StopIteration:
+                break
+
+        if ask == 'clear':
+            os.system("cls")
+
+        return {"response": response_text_chunk}
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=7860)