Spaces:
Runtime error
Runtime error
| from fastapi import FastAPI, WebSocket | |
| from fastapi.responses import HTMLResponse | |
| from llama_cpp import Llama | |
| import os | |
| app = FastAPI() | |
| llm = Llama(model_path="llama-2-7b-chat.Q3_K_S.gguf", n_ctx=2048, n_batch=512, use_mlock=True, n_threads=8) | |
| html = """ | |
| <!DOCTYPE html> | |
| <html> | |
| <head> | |
| <title>Chatbot by Aritra Roy & DVLH</title> | |
| </head> | |
| <body> | |
| <h1>Chatbot by Aritra Roy & DVLH</h1> | |
| <div> | |
| <ul id="chat"></ul> | |
| <input type="text" id="user_input" onkeydown="sendMessage(event)"> | |
| </div> | |
| <script> | |
| var chat = document.getElementById('chat'); | |
| var socket = new WebSocket('ws://localhost:7860/ws/chat'); | |
| socket.onmessage = function(event) { | |
| var li = document.createElement('li'); | |
| li.appendChild(document.createTextNode(event.data)); | |
| chat.appendChild(li); | |
| }; | |
| function sendMessage(event) { | |
| if (event.key === 'Enter') { | |
| var user_input = document.getElementById('user_input').value; | |
| var li = document.createElement('li'); | |
| li.appendChild(document.createTextNode('> ' + user_input)); | |
| chat.appendChild(li); | |
| socket.send(user_input); | |
| document.getElementById('user_input').value = ''; | |
| } | |
| } | |
| </script> | |
| </body> | |
| </html> | |
| """ | |
| async def chat(websocket: WebSocket): | |
| await websocket.accept() | |
| while True: | |
| try: | |
| user_input = await websocket.receive_text() | |
| os.system("cls") | |
| ask = user_input | |
| prompt = f"Llama-2-Chat [INST] <<SYS>>You're an assistant named Tusti. You are Developed by Aritra Roy. Don't share any false information.<</SYS>> {ask} [/INST]" | |
| output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True) | |
| while True: | |
| try: | |
| chunk = next(output_stream) | |
| if chunk.get('choices') and chunk['choices'][0].get('text'): | |
| await websocket.send_text(chunk['choices'][0]['text']) | |
| except StopIteration: | |
| break | |
| if ask == 'clear': | |
| os.system("cls") | |
| except Exception as e: | |
| await websocket.send_text(str(e)) | |
| if __name__ == "__main__": | |
| import uvicorn | |
| uvicorn.run(app, host="0.0.0.0", port=7860) |