fixed streaming issue
Browse files
main.py
CHANGED
@@ -1,30 +1,3 @@
|
|
1 |
-
from fastapi import FastAPI, Request
|
2 |
-
from fastapi.responses import StreamingResponse, JSONResponse
|
3 |
-
from pydantic import BaseModel
|
4 |
-
import requests
|
5 |
-
import time
|
6 |
-
import json
|
7 |
-
from typing import List, Optional
|
8 |
-
from models import AVAILABLE_MODELS
|
9 |
-
|
10 |
-
app = FastAPI()
|
11 |
-
|
12 |
-
def unix_id():
|
13 |
-
return str(int(time.time() * 1000))
|
14 |
-
|
15 |
-
class Message(BaseModel):
|
16 |
-
role: str
|
17 |
-
content: str
|
18 |
-
|
19 |
-
class ChatRequest(BaseModel):
|
20 |
-
messages: List[Message]
|
21 |
-
model: str
|
22 |
-
stream: Optional[bool] = False
|
23 |
-
|
24 |
-
@app.get("/v1/models")
|
25 |
-
async def list_models():
|
26 |
-
return {"object": "list", "data": AVAILABLE_MODELS}
|
27 |
-
|
28 |
@app.post("/v1/chat/completions")
|
29 |
async def chat_completion(request: ChatRequest):
|
30 |
headers = {
|
@@ -42,6 +15,7 @@ async def chat_completion(request: ChatRequest):
|
|
42 |
|
43 |
if request.stream:
|
44 |
def event_stream():
|
|
|
45 |
with requests.post(
|
46 |
"https://www.chatwithmono.xyz/api/chat",
|
47 |
headers=headers,
|
@@ -51,9 +25,13 @@ async def chat_completion(request: ChatRequest):
|
|
51 |
) as response:
|
52 |
for line in response.iter_lines(decode_unicode=True):
|
53 |
if line.startswith("0:"):
|
54 |
-
|
55 |
-
|
56 |
-
|
|
|
|
|
|
|
|
|
57 |
yield "data: [DONE]\n\n"
|
58 |
return StreamingResponse(event_stream(), media_type="text/event-stream")
|
59 |
|
@@ -72,7 +50,7 @@ async def chat_completion(request: ChatRequest):
|
|
72 |
if chunk.startswith("0:"):
|
73 |
try:
|
74 |
piece = json.loads(chunk[2:])
|
75 |
-
assistant_response += piece
|
76 |
except:
|
77 |
continue
|
78 |
elif chunk.startswith(("e:", "d:")):
|
@@ -91,7 +69,7 @@ async def chat_completion(request: ChatRequest):
|
|
91 |
"index": 0,
|
92 |
"message": {
|
93 |
"role": "assistant",
|
94 |
-
"content": assistant_response
|
95 |
},
|
96 |
"finish_reason": "stop"
|
97 |
}],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
@app.post("/v1/chat/completions")
|
2 |
async def chat_completion(request: ChatRequest):
|
3 |
headers = {
|
|
|
15 |
|
16 |
if request.stream:
|
17 |
def event_stream():
|
18 |
+
sent_done = False
|
19 |
with requests.post(
|
20 |
"https://www.chatwithmono.xyz/api/chat",
|
21 |
headers=headers,
|
|
|
25 |
) as response:
|
26 |
for line in response.iter_lines(decode_unicode=True):
|
27 |
if line.startswith("0:"):
|
28 |
+
try:
|
29 |
+
content_piece = json.loads(line[2:])
|
30 |
+
yield f'data: {json.dumps({"choices": [{"delta": {"content": content_piece}, "finish_reason": None}]})}\n\n'
|
31 |
+
except:
|
32 |
+
continue
|
33 |
+
elif line.startswith(("e:", "d:")) and not sent_done:
|
34 |
+
sent_done = True
|
35 |
yield "data: [DONE]\n\n"
|
36 |
return StreamingResponse(event_stream(), media_type="text/event-stream")
|
37 |
|
|
|
50 |
if chunk.startswith("0:"):
|
51 |
try:
|
52 |
piece = json.loads(chunk[2:])
|
53 |
+
assistant_response += piece # this is just a string fragment
|
54 |
except:
|
55 |
continue
|
56 |
elif chunk.startswith(("e:", "d:")):
|
|
|
69 |
"index": 0,
|
70 |
"message": {
|
71 |
"role": "assistant",
|
72 |
+
"content": assistant_response # correctly concatenated string
|
73 |
},
|
74 |
"finish_reason": "stop"
|
75 |
}],
|