AIMaster7 commited on
Commit
a6877b8
·
verified ·
1 Parent(s): 4bdce4b

fixed streaming issue

Browse files
Files changed (1) hide show
  1. main.py +10 -32
main.py CHANGED
@@ -1,30 +1,3 @@
1
- from fastapi import FastAPI, Request
2
- from fastapi.responses import StreamingResponse, JSONResponse
3
- from pydantic import BaseModel
4
- import requests
5
- import time
6
- import json
7
- from typing import List, Optional
8
- from models import AVAILABLE_MODELS
9
-
10
- app = FastAPI()
11
-
12
- def unix_id():
13
- return str(int(time.time() * 1000))
14
-
15
- class Message(BaseModel):
16
- role: str
17
- content: str
18
-
19
- class ChatRequest(BaseModel):
20
- messages: List[Message]
21
- model: str
22
- stream: Optional[bool] = False
23
-
24
- @app.get("/v1/models")
25
- async def list_models():
26
- return {"object": "list", "data": AVAILABLE_MODELS}
27
-
28
  @app.post("/v1/chat/completions")
29
  async def chat_completion(request: ChatRequest):
30
  headers = {
@@ -42,6 +15,7 @@ async def chat_completion(request: ChatRequest):
42
 
43
  if request.stream:
44
  def event_stream():
 
45
  with requests.post(
46
  "https://www.chatwithmono.xyz/api/chat",
47
  headers=headers,
@@ -51,9 +25,13 @@ async def chat_completion(request: ChatRequest):
51
  ) as response:
52
  for line in response.iter_lines(decode_unicode=True):
53
  if line.startswith("0:"):
54
- piece = line[2:]
55
- yield f"data: {json.dumps({'choices': [{'delta': {'content': piece}, 'finish_reason': None}]})}\n\n"
56
- elif line.startswith(("e:", "d:")):
 
 
 
 
57
  yield "data: [DONE]\n\n"
58
  return StreamingResponse(event_stream(), media_type="text/event-stream")
59
 
@@ -72,7 +50,7 @@ async def chat_completion(request: ChatRequest):
72
  if chunk.startswith("0:"):
73
  try:
74
  piece = json.loads(chunk[2:])
75
- assistant_response += piece
76
  except:
77
  continue
78
  elif chunk.startswith(("e:", "d:")):
@@ -91,7 +69,7 @@ async def chat_completion(request: ChatRequest):
91
  "index": 0,
92
  "message": {
93
  "role": "assistant",
94
- "content": assistant_response
95
  },
96
  "finish_reason": "stop"
97
  }],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  @app.post("/v1/chat/completions")
2
  async def chat_completion(request: ChatRequest):
3
  headers = {
 
15
 
16
  if request.stream:
17
  def event_stream():
18
+ sent_done = False
19
  with requests.post(
20
  "https://www.chatwithmono.xyz/api/chat",
21
  headers=headers,
 
25
  ) as response:
26
  for line in response.iter_lines(decode_unicode=True):
27
  if line.startswith("0:"):
28
+ try:
29
+ content_piece = json.loads(line[2:])
30
+ yield f'data: {json.dumps({"choices": [{"delta": {"content": content_piece}, "finish_reason": None}]})}\n\n'
31
+ except:
32
+ continue
33
+ elif line.startswith(("e:", "d:")) and not sent_done:
34
+ sent_done = True
35
  yield "data: [DONE]\n\n"
36
  return StreamingResponse(event_stream(), media_type="text/event-stream")
37
 
 
50
  if chunk.startswith("0:"):
51
  try:
52
  piece = json.loads(chunk[2:])
53
+ assistant_response += piece # this is just a string fragment
54
  except:
55
  continue
56
  elif chunk.startswith(("e:", "d:")):
 
69
  "index": 0,
70
  "message": {
71
  "role": "assistant",
72
+ "content": assistant_response # correctly concatenated string
73
  },
74
  "finish_reason": "stop"
75
  }],