Spaces:
Runtime error
Runtime error
File size: 1,771 Bytes
342aa4b 856e571 342aa4b 856e571 342aa4b 856e571 342aa4b 856e571 342aa4b 856e571 342aa4b 856e571 342aa4b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
from fastapi import FastAPI
from fastapi.responses import StreamingResponse
from starlette.responses import HTMLResponse
from llama_cpp import Llama
import time
import os
app = FastAPI()
llm = Llama(model_path="llama-2-7b-chat.Q3_K_S.gguf", n_ctx=2048, n_batch=512, use_mlock=True, n_threads=8)
time.sleep(8)
os.system("cls")
print("Chatbot by Aritra Roy & DVLH")
import warnings
warnings.filterwarnings("ignore")
def generate_responses(ask):
prompt = f"Llama-2-Chat [INST] <<SYS>>You're an assistant named Tusti. You are developed by Aritra Roy. Don't share any false information.<</SYS>> {ask} [/INST]"
output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True)
try:
while True:
try:
chunk = next(output_stream)
if chunk.get('choices') and chunk['choices'][0].get('text'):
response_text_chunk = chunk['choices'][0]['text']
yield response_text_chunk
except StopIteration:
break
except StopIteration:
pass
@app.get("/", response_class=HTMLResponse)
def home():
return """
<html>
<head>
<title>Chatbot Streaming</title>
</head>
<body>
<h1>Chatbot Streaming</h1>
<form action="/ask/" method="post">
<label for="ask">You:</label>
<input type="text" id="ask" name="ask" required>
<button type="submit">Ask</button>
</form>
</body>
</html>
"""
@app.post("/ask/")
def ask_endpoint(ask: str):
return StreamingResponse(generate_responses(ask))
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=7856) |