Spaces:

Manofem
/

Mode_transformers__nn

Runtime error

App Files Files Community

Manofem commited on Dec 6, 2023

Commit

69a8774

1 Parent(s): d4d23c8

Update app.py

Browse files

Files changed (1) hide show

app.py +58 -30

app.py CHANGED Viewed

@@ -1,49 +1,77 @@
-from fastapi import FastAPI, HTTPException
 from llama_cpp import Llama
-import time
 import os
 app = FastAPI()
 llm = Llama(model_path="llama-2-7b-chat.Q3_K_S.gguf", n_ctx=2048, n_batch=512, use_mlock=True, n_threads=8)
-def typewrite(text, delay=0.01):
-    for char in text:
-        print(char, end='', flush=True)
-        time.sleep(delay)
-    print(end='', flush=True)  # Print newline to move to the next line
-@app.post("/chat")
-async def chat(user_input: str):
-    try:
-        os.system("cls")
-        print("Chatbot by Aritra Roy & DVLH")
-        ask = user_input
-        prompt = f"Llama-2-Chat [INST] <<SYS>>You're an assistant named Tusti. You are Developed by Aritra Roy. Don't share any false information.<</SYS>> {ask} [/INST]"
-        # Send the request with stream=True
-        output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True)
-        response_text_chunk = ''
-        while True:
-            try:
-                chunk = next(output_stream)
-                if chunk.get('choices') and chunk['choices'][0].get('text'):
-                    response_text_chunk += chunk['choices'][0]['text']
-                    typewrite(response_text_chunk, delay=0.00)  # live response
-            except StopIteration:
-                break
-        if ask == 'clear':
-            os.system("cls")
-        return {"response": response_text_chunk}
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
 if __name__ == "__main__":
     import uvicorn

+from fastapi import FastAPI, WebSocket
+from fastapi.responses import HTMLResponse
 from llama_cpp import Llama
 import os
 app = FastAPI()
 llm = Llama(model_path="llama-2-7b-chat.Q3_K_S.gguf", n_ctx=2048, n_batch=512, use_mlock=True, n_threads=8)
+html = """
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>Chatbot by Aritra Roy & DVLH</title>
+    </head>
+    <body>
+        <h1>Chatbot by Aritra Roy & DVLH</h1>
+        <div>
+            <ul id="chat"></ul>
+            <input type="text" id="user_input" onkeydown="sendMessage(event)">
+        </div>
+        <script>
+            var chat = document.getElementById('chat');
+            var socket = new WebSocket('ws://localhost:7860/ws/chat');
+            socket.onmessage = function(event) {
+                var li = document.createElement('li');
+                li.appendChild(document.createTextNode(event.data));
+                chat.appendChild(li);
+            };
+            function sendMessage(event) {
+                if (event.key === 'Enter') {
+                    var user_input = document.getElementById('user_input').value;
+                    var li = document.createElement('li');
+                    li.appendChild(document.createTextNode('> ' + user_input));
+                    chat.appendChild(li);
+                    socket.send(user_input);
+                    document.getElementById('user_input').value = '';
+                }
+            }
+        </script>
+    </body>
+</html>
+"""
+@app.websocket("/ws/chat")
+async def chat(websocket: WebSocket):
+    await websocket.accept()
+    while True:
+        try:
+            user_input = await websocket.receive_text()
+            os.system("cls")
+            ask = user_input
+            prompt = f"Llama-2-Chat [INST] <<SYS>>You're an assistant named Tusti. You are Developed by Aritra Roy. Don't share any false information.<</SYS>> {ask} [/INST]"
+            output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True)
+            while True:
+                try:
+                    chunk = next(output_stream)
+                    if chunk.get('choices') and chunk['choices'][0].get('text'):
+                        await websocket.send_text(chunk['choices'][0]['text'])
+                except StopIteration:
+                    break
+            if ask == 'clear':
+                os.system("cls")
+        except Exception as e:
+            await websocket.send_text(str(e))
 if __name__ == "__main__":
     import uvicorn