Hugging Face's logo

Spaces:
Manofem
/
Mode_transformers__nn
Runtime error

App Files Community
1
Manofem commited on
Commit
966212c
·
1 Parent(s): 69a8774

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +65 -63
app.py CHANGED
@@ -1,78 +1,80 @@
1
- from fastapi import FastAPI, WebSocket
2
- from fastapi.responses import HTMLResponse
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  from llama_cpp import Llama
 
4
  import os
5
 
6
  app = FastAPI()
7
 
8
  llm = Llama(model_path="llama-2-7b-chat.Q3_K_S.gguf", n_ctx=2048, n_batch=512, use_mlock=True, n_threads=8)
9
 
10
- html = """
11
- <!DOCTYPE html>
12
- <html>
13
- <head>
14
- <title>Chatbot by Aritra Roy & DVLH</title>
15
- </head>
16
- <body>
17
- <h1>Chatbot by Aritra Roy & DVLH</h1>
18
- <div>
19
- <ul id="chat"></ul>
20
- <input type="text" id="user_input" onkeydown="sendMessage(event)">
21
- </div>
22
- <script>
23
- var chat = document.getElementById('chat');
24
-
25
- var socket = new WebSocket('ws://localhost:7860/ws/chat');
26
-
27
- socket.onmessage = function(event) {
28
- var li = document.createElement('li');
29
- li.appendChild(document.createTextNode(event.data));
30
- chat.appendChild(li);
31
- };
32
-
33
- function sendMessage(event) {
34
- if (event.key === 'Enter') {
35
- var user_input = document.getElementById('user_input').value;
36
- var li = document.createElement('li');
37
- li.appendChild(document.createTextNode('> ' + user_input));
38
- chat.appendChild(li);
39
- socket.send(user_input);
40
- document.getElementById('user_input').value = '';
41
- }
42
- }
43
- </script>
44
- </body>
45
- </html>
46
- """
47
-
48
- @app.websocket("/ws/chat")
49
- async def chat(websocket: WebSocket):
50
- await websocket.accept()
51
-
52
- while True:
53
- try:
54
- user_input = await websocket.receive_text()
55
- os.system("cls")
56
 
57
- ask = user_input
58
- prompt = f"Llama-2-Chat [INST] <<SYS>>You're an assistant named Tusti. You are Developed by Aritra Roy. Don't share any false information.<</SYS>> {ask} [/INST]"
 
 
 
59
 
60
- output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True)
61
 
62
- while True:
63
- try:
64
- chunk = next(output_stream)
65
- if chunk.get('choices') and chunk['choices'][0].get('text'):
66
- await websocket.send_text(chunk['choices'][0]['text'])
67
- except StopIteration:
68
- break
 
 
 
 
 
 
 
 
 
 
 
69
 
70
- if ask == 'clear':
71
- os.system("cls")
72
 
73
- except Exception as e:
74
- await websocket.send_text(str(e))
75
 
76
  if __name__ == "__main__":
77
  import uvicorn
78
- uvicorn.run(app, host="0.0.0.0", port=7860)
 
1
+ Hugging Face's logo
2
+ Search models, datasets, users...
3
+
4
+ Spaces:
5
+
6
+ Manofem
7
+ /
8
+ Mode_transformers__nn
9
+
10
+ like
11
+ 0
12
+
13
+ App
14
+ Files
15
+ Community
16
+ Settings
17
+ Mode_transformers__nn
18
+ /
19
+ app.py
20
+ Manofem's picture
21
+ Manofem
22
+ Update app.py
23
+ e318921
24
+ raw
25
+ history
26
+ blame
27
+ edit
28
+ delete
29
+ No virus
30
+ 1.58 kB
31
+ from fastapi import FastAPI, HTTPException
32
  from llama_cpp import Llama
33
+ import time
34
  import os
35
 
36
  app = FastAPI()
37
 
38
  llm = Llama(model_path="llama-2-7b-chat.Q3_K_S.gguf", n_ctx=2048, n_batch=512, use_mlock=True, n_threads=8)
39
 
40
+ def typewrite(text, delay=0.01):
41
+ for char in text:
42
+ print(char, end='', flush=True)
43
+ time.sleep(delay)
44
+ print(end='', flush=True) # Print newline to move to the next line
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
 
46
+ @app.post("/chat")
47
+ async def chat(user_input: str):
48
+ try:
49
+ os.system("cls")
50
+ print("Chatbot by Aritra Roy & DVLH")
51
 
52
+ ask = user_input
53
 
54
+ prompt = f"Llama-2-Chat [INST] <<SYS>>You're an assistant named Tusti. You are Developed by Aritra Roy. Don't share any false information.<</SYS>> {ask} [/INST]"
55
+
56
+ # Send the request with stream=True
57
+ output_stream = llm(prompt, max_tokens=1024, echo=False, temperature=0.2, top_p=0.1, stream=True)
58
+
59
+ response_text_chunk = ''
60
+
61
+ while True:
62
+ try:
63
+ chunk = next(output_stream)
64
+ if chunk.get('choices') and chunk['choices'][0].get('text'):
65
+ response_text_chunk += chunk['choices'][0]['text']
66
+ typewrite(response_text_chunk, delay=0.00) # live response
67
+ except StopIteration:
68
+ break
69
+
70
+ if ask == 'clear':
71
+ os.system("cls")
72
 
73
+ return {"response": response_text_chunk}
 
74
 
75
+ except Exception as e:
76
+ raise HTTPException(status_code=500, detail=str(e))
77
 
78
  if __name__ == "__main__":
79
  import uvicorn
80
+ uvicorn.run(app, host="0.0.0.0", port=7860)