Spaces:
Running
Running
Upload app.py
Browse files
app.py
CHANGED
|
@@ -885,18 +885,19 @@ async def chat(req: ChatRequest, request: Request):
|
|
| 885 |
async def stream_response() -> AsyncGenerator[str, None]:
|
| 886 |
full_text = ""
|
| 887 |
try:
|
| 888 |
-
# Plain streaming —
|
| 889 |
-
|
| 890 |
model=MODEL,
|
| 891 |
messages=messages,
|
| 892 |
max_tokens=600,
|
| 893 |
temperature=0.3,
|
| 894 |
-
|
| 895 |
-
|
| 896 |
-
|
| 897 |
-
|
| 898 |
-
|
| 899 |
-
|
|
|
|
| 900 |
|
| 901 |
# Extract route from reply using brace-counting
|
| 902 |
reply, route, prefill = extract_route(full_text)
|
|
|
|
| 885 |
async def stream_response() -> AsyncGenerator[str, None]:
|
| 886 |
full_text = ""
|
| 887 |
try:
|
| 888 |
+
# Plain streaming — compatible with openai==1.30.0
|
| 889 |
+
response = await client.chat.completions.create(
|
| 890 |
model=MODEL,
|
| 891 |
messages=messages,
|
| 892 |
max_tokens=600,
|
| 893 |
temperature=0.3,
|
| 894 |
+
stream=True,
|
| 895 |
+
)
|
| 896 |
+
async for chunk in response:
|
| 897 |
+
token = chunk.choices[0].delta.content if chunk.choices else None
|
| 898 |
+
if token:
|
| 899 |
+
full_text += token
|
| 900 |
+
yield f'data: {json.dumps({"token": token, "done": False})}\n\n'
|
| 901 |
|
| 902 |
# Extract route from reply using brace-counting
|
| 903 |
reply, route, prefill = extract_route(full_text)
|