response = requests.post(API_URL, headers=headers, json=payload, stream=True)
for line in response.iter_lines():
if not line.startswith(b"data:"):
continue
if line.strip() == b"data: [DONE]":
return
yield json.loads(line.decode("utf-8").lstrip("data:").rstrip("/n"))
chunks = query({
"messages": [
{
"role": "user",
"content": "What is the capital of France?"
}
],
"model": "dphn/Dolphin-Mistral-24B-Venice-Edition",
"stream": True,
})
for chunk in chunks:
print(chunk["choices"][0]["delta"]["content"], end="")
