app.py

from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import StreamingResponse
from llama_cpp import Llama
#
app = FastAPI()

llm = Llama(model_path="llama-2-7b-chat.Q3_K_S.gguf", n_ctx=2048, n_batch=512, use_mlock=True, n_threads=8)

def generate_output_stream(prompt):
    output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True)
    while True:
        try:
            chunk = next(output_stream)
            if chunk.get('choices') and chunk['choices'][0].get('text'):
                text = chunk['choices'][0]['text']
                yield f"{text}\n"
        except StopIteration:
            break

def clear_screen():
    os.system("cls")

@app.post("/chat")
async def chat(request: Request):
    try:
        data = await request.json()
        user_input = data.get("user_input")

        if not user_input:
            raise HTTPException(status_code=400, detail="Missing 'user_input' field in the request JSON.")

        ask = user_input

        prompt = f"Llama-2-Chat [INST] <<SYS>>You're an assistant named Tusti. You are Developed by Aritra Roy. Don't share any false information.<</SYS>> {ask} [/INST]"

        if ask == 'clear':
            clear_screen()

        return StreamingResponse(generate_output_stream(prompt), media_type="text/plain")

    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=7860)