app.py

from fastapi import FastAPI, HTTPException
from llama_cpp import Llama
import time
import os

app = FastAPI()

llm = Llama(model_path="llama-2-7b-chat.Q3_K_S.gguf", n_ctx=2048, n_batch=512, use_mlock=True, n_threads=8)

def typewrite(text, delay=0.01):
    for char in text:
        print(char, end='', flush=True)
        time.sleep(delay)
    print(end='', flush=True)  # Print newline to move to the next line

@app.post("/chat")
async def chat(user_input: str):
    try:
        os.system("cls")
        print("Chatbot by Aritra Roy & DVLH")

        ask = user_input

        prompt = f"Llama-2-Chat [INST] <<SYS>>You're an assistant named Tusti. You are Developed by Aritra Roy. Don't share any false information.<</SYS>> {ask} [/INST]"

        # Send the request with stream=True
        output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True)

        response_text_chunk = ''

        while True:
            try:
                chunk = next(output_stream)
                if chunk.get('choices') and chunk['choices'][0].get('text'):
                    response_text_chunk += chunk['choices'][0]['text']
                    typewrite(response_text_chunk, delay=0.00)  # live response
            except StopIteration:
                break

        if ask == 'clear':
            os.system("cls")

        return {"response": response_text_chunk}

    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=7860)