Spaces:
Runtime error
Runtime error
| from fastapi import FastAPI | |
| from fastapi.responses import StreamingResponse | |
| from starlette.responses import HTMLResponse | |
| from llama_cpp import Llama | |
| import time | |
| import os | |
| app = FastAPI() | |
| llm = Llama(model_path="llama-2-7b-chat.Q3_K_S.gguf", n_ctx=2048, n_batch=512, use_mlock=True, n_threads=8) | |
| time.sleep(8) | |
| os.system("cls") | |
| print("Chatbot by Aritra Roy & DVLH") | |
| import warnings | |
| warnings.filterwarnings("ignore") | |
| def generate_responses(ask): | |
| prompt = f"Llama-2-Chat [INST] <<SYS>>You're an assistant named Tusti. You are developed by Aritra Roy. Don't share any false information.<</SYS>> {ask} [/INST]" | |
| output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True) | |
| try: | |
| while True: | |
| try: | |
| chunk = next(output_stream) | |
| if chunk.get('choices') and chunk['choices'][0].get('text'): | |
| response_text_chunk = chunk['choices'][0]['text'] | |
| yield response_text_chunk | |
| except StopIteration: | |
| break | |
| except StopIteration: | |
| pass | |
| def home(): | |
| return """ | |
| <html> | |
| <head> | |
| <title>Chatbot Streaming</title> | |
| </head> | |
| <body> | |
| <h1>Chatbot Streaming</h1> | |
| <form action="/ask/" method="post"> | |
| <label for="ask">You:</label> | |
| <input type="text" id="ask" name="ask" required> | |
| <button type="submit">Ask</button> | |
| </form> | |
| </body> | |
| </html> | |
| """ | |
| def ask_endpoint(ask: str): | |
| return StreamingResponse(generate_responses(ask)) | |
| if __name__ == "__main__": | |
| import uvicorn | |
| uvicorn.run(app, host="0.0.0.0", port=7856) |