nikeshn commited on
Commit
e26f1cc
·
verified ·
1 Parent(s): e711654

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +9 -8
app.py CHANGED
@@ -885,18 +885,19 @@ async def chat(req: ChatRequest, request: Request):
885
  async def stream_response() -> AsyncGenerator[str, None]:
886
  full_text = ""
887
  try:
888
- # Plain streaming — no tool calls
889
- async with client.chat.completions.stream(
890
  model=MODEL,
891
  messages=messages,
892
  max_tokens=600,
893
  temperature=0.3,
894
- ) as stream:
895
- async for chunk in stream:
896
- token = chunk.choices[0].delta.content if chunk.choices else None
897
- if token:
898
- full_text += token
899
- yield f'data: {json.dumps({"token": token, "done": False})}\n\n'
 
900
 
901
  # Extract route from reply using brace-counting
902
  reply, route, prefill = extract_route(full_text)
 
885
  async def stream_response() -> AsyncGenerator[str, None]:
886
  full_text = ""
887
  try:
888
+ # Plain streaming — compatible with openai==1.30.0
889
+ response = await client.chat.completions.create(
890
  model=MODEL,
891
  messages=messages,
892
  max_tokens=600,
893
  temperature=0.3,
894
+ stream=True,
895
+ )
896
+ async for chunk in response:
897
+ token = chunk.choices[0].delta.content if chunk.choices else None
898
+ if token:
899
+ full_text += token
900
+ yield f'data: {json.dumps({"token": token, "done": False})}\n\n'
901
 
902
  # Extract route from reply using brace-counting
903
  reply, route, prefill = extract_route(full_text)