G_AI / app /server /chat.py
superxu520's picture
fix-type-errors
50b60b4
Raw History Blame Contribute Delete
43.3 kB
import base64
import hashlib
import io
import uuid
from pathlib import Path
from typing import Any, AsyncGenerator, Sequence
import orjson
from fastapi import APIRouter, Depends, Request
from fastapi.responses import StreamingResponse
from gemini_webapi import ModelOutput
from gemini_webapi.client import ChatSession
from gemini_webapi.constants import Model
from gemini_webapi.types.image import GeneratedImage, Image
from loguru import logger
from ..models import (
AnthropicMessageRequest,
ChatCompletionRequest,
ConversationInStore,
Message,
ModelListResponse,
ResponseCreateRequest,
ResponseImageGenerationCall,
ResponseOutputContent,
ResponseToolCall,
ResponseToolChoice,
ResponseUsage,
Tool,
ToolChoiceFunction,
)
from ..services import GeminiClientPool, GeminiClientWrapper, LMDBConversationStore
from ..services.chat_session_service import (
execute_core_chat_session as shared_execute_core_chat_session,
)
from ..services.chat_session_service import (
find_reusable_session as shared_find_reusable_session,
)
from ..services.chat_session_service import send_with_split as shared_send_with_split
from ..utils import g_config
from ..utils.helper import (
detect_image_extension,
estimate_tokens,
extract_image_dimensions,
)
from .anthropic_handler import AnthropicHandlerDependencies, handle_anthropic_messages
from .chat_support import (
StructuredOutputRequirement as SharedStructuredOutputRequirement,
)
from .chat_support import (
build_image_generation_instruction,
build_structured_requirement,
calculate_usage,
create_chat_completion_standard_payload,
create_responses_standard_payload,
get_available_models,
get_default_model_name,
instructions_to_messages,
prepare_messages_for_model,
process_llm_output,
response_items_to_messages,
)
from .middleware import get_image_store_dir, get_image_token, get_temp_dir, verify_api_key
from .openai_chat_handler import OpenAIChatHandlerDependencies, handle_chat_completion
from .responses_handler import ResponsesHandlerDependencies, handle_responses_api
from .streaming_filter import StreamingOutputFilter
MAX_CHARS_PER_REQUEST = int(g_config.gemini.max_chars_per_request * 0.9)
METADATA_TTL_MINUTES = 15
router = APIRouter()
StructuredOutputRequirement = SharedStructuredOutputRequirement
# --- Helper Functions ---
async def _image_to_base64(
image: Image, temp_dir: Path
) -> tuple[str, int | None, int | None, str, str]:
"""Persist an image provided by gemini_webapi and return base64 plus dimensions, filename, and hash."""
if isinstance(image, GeneratedImage):
try:
saved_path = await image.save(path=str(temp_dir), full_size=True)
except Exception as e:
logger.warning(
f"Failed to download full-size GeneratedImage, retrying with default size: {e}"
)
saved_path = await image.save(path=str(temp_dir), full_size=False)
else:
saved_path = await image.save(path=str(temp_dir))
if not saved_path:
raise ValueError("Failed to save generated image")
original_path = Path(saved_path)
data = original_path.read_bytes()
suffix = original_path.suffix
if not suffix:
detected_ext = detect_image_extension(data)
suffix = detected_ext or (".png" if isinstance(image, GeneratedImage) else ".jpg")
random_name = f"img_{uuid.uuid4().hex}{suffix}"
new_path = temp_dir / random_name
original_path.rename(new_path)
width, height = extract_image_dimensions(data)
filename = random_name
file_hash = hashlib.sha256(data).hexdigest()
return base64.b64encode(data).decode("ascii"), width, height, filename, file_hash
def _persist_conversation(
db: LMDBConversationStore,
model_name: str,
client_id: str,
metadata: list[str | None],
messages: list[Message],
storage_output: str | None,
tool_calls: list[Any] | None,
thoughts: str | None = None,
) -> str | None:
"""Unified logic to save conversation history to LMDB."""
try:
current_assistant_message = Message(
role="assistant",
content=storage_output or None,
tool_calls=tool_calls or None,
reasoning_content=thoughts or None,
)
full_history = [*messages, current_assistant_message]
cleaned_history = db.sanitize_assistant_messages(full_history)
conv = ConversationInStore(
model=model_name,
client_id=client_id,
metadata=metadata,
messages=cleaned_history,
)
key = db.store(conv)
logger.debug(f"Conversation saved to LMDB with key: {key[:12]}")
return key
except Exception as e:
logger.warning(f"Failed to save {len(messages) + 1} messages to LMDB: {e}")
return None
async def _find_reusable_session(
db: LMDBConversationStore,
pool: GeminiClientPool,
model: Model,
messages: list[Message],
) -> tuple[ChatSession | None, GeminiClientWrapper | None, list[Message]]:
"""Delegate reusable session lookup to the shared session service."""
return await shared_find_reusable_session(
db,
pool,
model,
messages,
metadata_ttl_minutes=METADATA_TTL_MINUTES,
)
async def _send_with_split(
session: ChatSession,
text: str,
files: Sequence[Path | str | io.BytesIO] | None = None,
stream: bool = False,
) -> AsyncGenerator[ModelOutput, None] | ModelOutput:
"""Delegate Gemini send/split logic to the shared session service."""
return await shared_send_with_split(
session,
text,
files=files,
stream=stream,
max_chars_per_request=MAX_CHARS_PER_REQUEST,
)
# --- Response Builders & Streaming ---
def _create_real_streaming_response(
generator: AsyncGenerator[ModelOutput, None],
completion_id: str,
created_time: int,
model_name: str,
messages: list[Message],
db: LMDBConversationStore,
model: Model,
client_wrapper: GeminiClientWrapper,
session: ChatSession,
base_url: str,
structured_requirement: StructuredOutputRequirement | None = None,
) -> StreamingResponse:
"""
Create a real-time streaming response.
Reconciles manual delta accumulation with the model's final authoritative state.
"""
async def generate_stream():
full_thoughts, full_text = "", ""
has_started = False
all_outputs: list[ModelOutput] = []
suppressor = StreamingOutputFilter()
try:
async for chunk in generator:
all_outputs.append(chunk)
if not has_started:
data = {
"id": completion_id,
"object": "chat.completion.chunk",
"created": created_time,
"model": model_name,
"choices": [
{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}
],
}
yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n"
has_started = True
if hasattr(chunk, "thoughts_delta") and chunk.thoughts_delta:
t_delta = str(chunk.thoughts_delta)
full_thoughts += t_delta
data = {
"id": completion_id,
"object": "chat.completion.chunk",
"created": created_time,
"model": model_name,
"choices": [
{
"index": 0,
"delta": {"reasoning_content": t_delta},
"finish_reason": None,
}
],
}
yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n"
elif chunk.thoughts and len(chunk.thoughts) > len(full_thoughts):
t_delta = chunk.thoughts[len(full_thoughts) :]
data = {
"id": completion_id,
"object": "chat.completion.chunk",
"created": created_time,
"model": model_name,
"choices": [
{
"index": 0,
"delta": {"reasoning_content": t_delta},
"finish_reason": None,
}
],
}
yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n"
full_thoughts = chunk.thoughts
# Robust text delta calculation
new_text = chunk.text or ""
# Strip transient triple-backtick fence which is often added by the web-scraping backend
# during streaming snapshots. We will restore the real ending after the loop.
display_text = StreamingOutputFilter.TRAILING_FENCE_RE.sub("", new_text)
if len(display_text) > len(full_text):
text_delta = display_text[len(full_text) :]
filter_events = suppressor.process(text_delta)
for event in filter_events:
delta = {}
if event["type"] == "text":
delta = {"content": event["content"]}
elif event["type"] == "tool_start":
delta = {
"tool_calls": [
{
"index": event["index"],
"id": event["id"],
"type": "function",
"function": {"name": event["name"], "arguments": ""},
}
]
}
elif event["type"] == "tool_delta":
delta = {
"tool_calls": [
{
"index": event["index"],
"function": {"arguments": event["content"]},
}
]
}
if delta:
data = {
"id": completion_id,
"object": "chat.completion.chunk",
"created": created_time,
"model": model_name,
"choices": [
{
"index": 0,
"delta": delta,
"finish_reason": None,
}
],
}
yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n"
full_text = display_text
except Exception as e:
logger.exception(f"Error during OpenAI streaming: {e}")
yield f"data: {orjson.dumps({'error': {'message': 'Streaming error occurred.', 'type': 'server_error', 'param': None, 'code': None}}).decode('utf-8')}\n\n"
return
if all_outputs:
final_chunk = all_outputs[-1]
final_text = final_chunk.text or ""
if len(final_text) > len(full_text):
# Yield any real trailing content (like the actual closing fence)
final_delta = final_text[len(full_text) :]
for event in suppressor.process(final_delta):
if event["type"] == "text":
yield f"data: {orjson.dumps({'id': completion_id, 'object': 'chat.completion.chunk', 'created': created_time, 'model': model_name, 'choices': [{'index': 0, 'delta': {'content': event['content']}, 'finish_reason': None}]}).decode('utf-8')}\n\n"
full_text = final_text
if final_chunk.thoughts:
full_thoughts = final_chunk.thoughts
for event in suppressor.flush():
delta = {}
if event["type"] == "text":
delta = {"content": event["content"]}
elif event["type"] == "tool_start":
delta = {
"tool_calls": [
{
"index": event["index"],
"id": event["id"],
"type": "function",
"function": {"name": event["name"], "arguments": ""},
}
]
}
elif event["type"] == "tool_delta":
delta = {
"tool_calls": [
{"index": event["index"], "function": {"arguments": event["content"]}}
]
}
if delta:
data = {
"id": completion_id,
"object": "chat.completion.chunk",
"created": created_time,
"model": model_name,
"choices": [{"index": 0, "delta": delta, "finish_reason": None}],
}
yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n"
_thoughts, assistant_text, storage_output, tool_calls = process_llm_output(
full_thoughts, full_text, structured_requirement
)
images = []
seen_urls = set()
for out in all_outputs:
if out.images:
for img in out.images:
# Use the image URL as a stable identifier across chunks
if img.url not in seen_urls:
images.append(img)
seen_urls.add(img.url)
image_markdown = ""
seen_hashes = set()
for image in images:
try:
image_store = get_image_store_dir()
_, _, _, filename, file_hash = await _image_to_base64(image, image_store)
if file_hash in seen_hashes:
# Duplicate content, delete the file and skip
(image_store / filename).unlink(missing_ok=True)
continue
seen_hashes.add(file_hash)
img_url = (
f"![{filename}]({base_url}images/{filename}?token={get_image_token(filename)})"
)
image_markdown += f"\n\n{img_url}"
except Exception as exc:
logger.warning(f"Failed to process image in OpenAI stream: {exc}")
if image_markdown:
assistant_text += image_markdown
storage_output += image_markdown
yield f"data: {orjson.dumps({'id': completion_id, 'object': 'chat.completion.chunk', 'created': created_time, 'model': model_name, 'choices': [{'index': 0, 'delta': {'content': image_markdown}, 'finish_reason': None}]}).decode('utf-8')}\n\n"
p_tok, c_tok, t_tok, r_tok = calculate_usage(
messages, assistant_text, tool_calls, full_thoughts
)
usage = {
"prompt_tokens": p_tok,
"completion_tokens": c_tok,
"total_tokens": t_tok,
"completion_tokens_details": {"reasoning_tokens": r_tok},
}
final_delta = {}
if tool_calls:
# Add index field to each tool_call for OpenAI compatibility
final_delta["tool_calls"] = [
{**call.model_dump(mode="json"), "index": i} for i, call in enumerate(tool_calls)
]
data = {
"id": completion_id,
"object": "chat.completion.chunk",
"created": created_time,
"model": model_name,
"choices": [
{
"index": 0,
"delta": final_delta,
"finish_reason": "tool_calls" if tool_calls else "stop",
}
],
"usage": usage,
}
_persist_conversation(
db,
model.model_name,
client_wrapper.id,
session.metadata,
messages, # This should be the prepared messages
storage_output,
tool_calls,
full_thoughts,
)
yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n"
yield "data: [DONE]\n\n"
return StreamingResponse(generate_stream(), media_type="text/event-stream")
def _create_responses_real_streaming_response(
generator: AsyncGenerator[ModelOutput, None],
response_id: str,
created_time: int,
model_name: str,
messages: list[Message],
db: LMDBConversationStore,
model: Model,
client_wrapper: GeminiClientWrapper,
session: ChatSession,
request: ResponseCreateRequest,
image_store: Path,
base_url: str,
structured_requirement: StructuredOutputRequirement | None = None,
) -> StreamingResponse:
"""
Create a real-time streaming response for the Responses API.
Ensures final accumulated text and thoughts are synchronized.
"""
base_event = {
"id": response_id,
"object": "response",
"created_at": created_time,
"model": model_name,
}
async def generate_stream():
yield f"data: {orjson.dumps({**base_event, 'type': 'response.created', 'response': {'id': response_id, 'object': 'response', 'created_at': created_time, 'model': model_name, 'status': 'in_progress', 'metadata': request.metadata, 'input': None, 'tools': request.tools, 'tool_choice': request.tool_choice}}).decode('utf-8')}\n\n"
message_id = f"msg_{uuid.uuid4().hex}"
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.added', 'output_index': 0, 'item': {'id': message_id, 'type': 'message', 'role': 'assistant', 'content': []}}).decode('utf-8')}\n\n"
full_thoughts, full_text = "", ""
last_chunk_was_thought = False
all_outputs: list[ModelOutput] = []
suppressor = StreamingOutputFilter()
try:
async for chunk in generator:
all_outputs.append(chunk)
# Robust thought delta calculation
new_thoughts = chunk.thoughts or ""
if len(new_thoughts) > len(full_thoughts):
t_delta = new_thoughts[len(full_thoughts) :]
if not last_chunk_was_thought and not full_thoughts:
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': '<think>'}).decode('utf-8')}\n\n"
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': t_delta}).decode('utf-8')}\n\n"
full_thoughts = new_thoughts
last_chunk_was_thought = True
# Robust text delta calculation
new_text = chunk.text or ""
# Strip transient triple-backtick fence which is often added by the web-scraping backend
# during streaming snapshots. We will restore the real ending after the loop.
display_text = StreamingOutputFilter.TRAILING_FENCE_RE.sub("", new_text)
if len(display_text) > len(full_text):
text_delta = display_text[len(full_text) :]
if last_chunk_was_thought:
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': '</think>\n'}).decode('utf-8')}\n\n"
last_chunk_was_thought = False
filter_events = suppressor.process(text_delta)
for event in filter_events:
if event["type"] == "text":
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': event['content']}).decode('utf-8')}\n\n"
# Note: This API currently handles tool calls as discrete items at the end.
# We ignore internal tool call events in the text stream for now.
full_text = display_text
except Exception as e:
logger.exception(f"Error during Responses API streaming: {e}")
yield f"data: {orjson.dumps({**base_event, 'type': 'error', 'error': {'message': 'Streaming error.'}}).decode('utf-8')}\n\n"
return
if all_outputs:
final_chunk = all_outputs[-1]
final_text = final_chunk.text or ""
if len(final_text) > len(full_text):
# Yield any real trailing content (like the actual closing fence)
final_delta = final_text[len(full_text) :]
for event in suppressor.process(final_delta):
if event["type"] == "text":
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': event['content']}).decode('utf-8')}\n\n"
full_text = final_text
if final_chunk.thoughts:
full_thoughts = final_chunk.thoughts
if last_chunk_was_thought:
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': '</think>\n'}).decode('utf-8')}\n\n"
for event in suppressor.flush():
if event["type"] == "text":
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': event['content']}).decode('utf-8')}\n\n"
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.done', 'output_index': 0}).decode('utf-8')}\n\n"
_thoughts, assistant_text, storage_output, detected_tool_calls = process_llm_output(
full_thoughts, full_text, structured_requirement
)
images = []
seen_urls = set()
for out in all_outputs:
if out.images:
for img in out.images:
if img.url not in seen_urls:
images.append(img)
seen_urls.add(img.url)
# Check if image generation was forced via tool_choice (same logic as non-streaming)
image_generation_forced = (
request.tool_choice is not None
and isinstance(request.tool_choice, ResponseToolChoice)
and request.tool_choice.type == "image_generation"
)
if image_generation_forced and not images and not assistant_text:
logger.warning(
"Image generation was forced via tool_choice but no images or text were returned in stream."
)
yield f"data: {orjson.dumps({**base_event, 'type': 'error', 'error': {'message': 'No images returned from forced image generation request.'}}).decode('utf-8')}\n\n"
return
response_contents, image_call_items = [], []
seen_hashes = set()
for image in images:
try:
image_base64, width, height, filename, file_hash = await _image_to_base64(
image, image_store
)
if file_hash in seen_hashes:
(image_store / filename).unlink(missing_ok=True)
continue
seen_hashes.add(file_hash)
img_format = "png" if isinstance(image, GeneratedImage) else "jpeg"
image_url = (
f"![{filename}]({base_url}images/{filename}?token={get_image_token(filename)})"
)
image_call_items.append(
ResponseImageGenerationCall(
id=filename.rsplit(".", 1)[0],
result=image_base64,
output_format=img_format,
size=f"{width}x{height}" if width and height else None,
)
)
response_contents.append(ResponseOutputContent(type="output_text", text=image_url))
except Exception as exc:
logger.warning(f"Failed to process image in stream: {exc}")
if assistant_text:
response_contents.append(ResponseOutputContent(type="output_text", text=assistant_text))
if not response_contents:
response_contents.append(ResponseOutputContent(type="output_text", text=""))
# Aggregate images for storage
image_markdown = ""
for img_call in image_call_items:
fname = f"{img_call.id}.{img_call.output_format}"
img_url = f"![{fname}]({base_url}images/{fname}?token={get_image_token(fname)})"
image_markdown += f"\n\n{img_url}"
if image_markdown:
storage_output += image_markdown
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.done', 'output_index': 0, 'item': {'id': message_id, 'type': 'message', 'role': 'assistant', 'content': [c.model_dump(mode='json') for c in response_contents]}}).decode('utf-8')}\n\n"
current_idx = 1
for call in detected_tool_calls:
tc_item = ResponseToolCall(id=call.id, status="completed", function=call.function)
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.added', 'output_index': current_idx, 'item': tc_item.model_dump(mode='json')}).decode('utf-8')}\n\n"
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.done', 'output_index': current_idx, 'item': tc_item.model_dump(mode='json')}).decode('utf-8')}\n\n"
current_idx += 1
for img_call in image_call_items:
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.added', 'output_index': current_idx, 'item': img_call.model_dump(mode='json')}).decode('utf-8')}\n\n"
yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.done', 'output_index': current_idx, 'item': img_call.model_dump(mode='json')}).decode('utf-8')}\n\n"
current_idx += 1
p_tok, c_tok, t_tok, r_tok = calculate_usage(
messages, assistant_text, detected_tool_calls, full_thoughts
)
usage = ResponseUsage(
input_tokens=p_tok,
output_tokens=c_tok,
total_tokens=t_tok,
output_tokens_details={"reasoning_tokens": r_tok},
)
payload = create_responses_standard_payload(
response_id,
created_time,
model_name,
detected_tool_calls,
image_call_items,
response_contents,
usage,
request,
None,
full_thoughts,
)
_persist_conversation(
db,
model.model_name,
client_wrapper.id,
session.metadata,
messages,
storage_output,
detected_tool_calls,
full_thoughts,
)
yield f"data: {orjson.dumps({**base_event, 'type': 'response.completed', 'response': payload.model_dump(mode='json')}).decode('utf-8')}\n\n"
yield "data: [DONE]\n\n"
return StreamingResponse(generate_stream(), media_type="text/event-stream")
# --- Core Session Logic ---
async def _execute_core_chat_session(
request_model: str,
messages: list[Message],
tmp_dir: Path,
tools: list[Tool] | None = None,
tool_choice: str | ToolChoiceFunction | None = None,
stream: bool = False,
extra_instructions: list[str] | None = None,
response_format: dict[str, Any] | None = None,
):
"""
Delegate shared Gemini session execution to the session service and
preserve the tuple contract expected by existing handlers.
"""
result = await shared_execute_core_chat_session(
request_model=request_model,
messages=messages,
tmp_dir=tmp_dir,
tools=tools,
tool_choice=tool_choice,
stream=stream,
extra_instructions=extra_instructions,
response_format=response_format,
metadata_ttl_minutes=METADATA_TTL_MINUTES,
)
return (
result.response_or_stream,
result.completion_id,
result.created_time,
result.session,
result.client,
result.db,
result.model,
result.prepared_messages,
result.structured_requirement,
)
async def _create_anthropic_streaming_response(
generator: AsyncGenerator[ModelOutput, None],
completion_id: str,
created_time: int,
model_name: str,
messages: list[Message],
db: LMDBConversationStore | None = None,
model: Model | None = None,
client_wrapper: GeminiClientWrapper | None = None,
session: ChatSession | None = None,
):
"""
Stream Gemini output as Anthropic SSE events.
Now supports structured tool call streaming.
"""
async def generate_stream():
# 1. Message Start
msg_start = {
"type": "message_start",
"message": {
"id": completion_id,
"type": "message",
"role": "assistant",
"content": [],
"model": model_name,
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": 0, "output_tokens": 0},
},
}
yield f"event: message_start\ndata: {orjson.dumps(msg_start).decode('utf-8')}\n\n"
full_text = ""
suppressor = StreamingOutputFilter()
content_block_index = 0
in_text_block = False
last_chunk: ModelOutput | None = None
# 3. Stream Deltas
try:
async for chunk in generator:
last_chunk = chunk
if chunk.text:
display_text = StreamingOutputFilter.TRAILING_FENCE_RE.sub("", chunk.text)
if len(display_text) > len(full_text):
delta_text = display_text[len(full_text) :]
full_text = display_text
filter_events = suppressor.process(delta_text)
for event in filter_events:
if event["type"] == "text":
if not in_text_block:
# Start a new text block if not in one
yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'text', 'text': ''}}).decode('utf-8')}\n\n"
in_text_block = True
delta = {
"type": "content_block_delta",
"index": content_block_index,
"delta": {"type": "text_delta", "text": event["content"]},
}
yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n"
elif event["type"] == "tool_start":
if in_text_block:
yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n"
content_block_index += 1
in_text_block = False
# Anthropic tool_use start
yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'tool_use', 'id': event['id'], 'name': event['name'], 'input': {}}}).decode('utf-8')}\n\n"
elif event["type"] == "tool_delta":
# Anthropic tool_use delta (input_json_delta)
delta = {
"type": "content_block_delta",
"index": content_block_index,
"delta": {
"type": "input_json_delta",
"partial_json": event["content"],
},
}
yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n"
elif event["type"] == "tool_end":
yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n"
content_block_index += 1
# We stay in "not in_text_block" state until next text event
# Handle final delta after loop ends
final_text = ""
final_thoughts = ""
if last_chunk:
final_text = last_chunk.text or ""
final_thoughts = last_chunk.thoughts or ""
if len(final_text) > len(full_text):
final_delta = final_text[len(full_text) :]
for event in suppressor.process(final_delta):
if event["type"] == "text":
if not in_text_block:
yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'text', 'text': ''}}).decode('utf-8')}\n\n"
in_text_block = True
delta = {
"type": "content_block_delta",
"index": content_block_index,
"delta": {"type": "text_delta", "text": event["content"]},
}
yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n"
full_text = final_text
# Final verification of tool calls (Built-in tools like image_retrieval often appear at the end)
_thoughts_out, _assistant_visible, assistant_storage, tool_calls = process_llm_output(
final_thoughts, full_text, None
)
# If we detected tool_calls that weren't captured by suppressor yet, emit them now
# (Note: suppressor currently handles manual formatting, but built-in tools are caught here)
# To avoid duplicates if suppressor already found them, we could check event IDs.
# But normally built-in tools won't trigger the regex-based suppressor.
if tool_calls:
for tcall in tool_calls:
# Emit as a new content block if it's a tool_use
if in_text_block:
yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n"
content_block_index += 1
in_text_block = False
yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'tool_use', 'id': tcall.id, 'name': tcall.function.name, 'input': {}}}).decode('utf-8')}\n\n"
yield f"event: content_block_delta\ndata: {orjson.dumps({'type': 'content_block_delta', 'index': content_block_index, 'delta': {'type': 'input_json_delta', 'partial_json': tcall.function.arguments}}).decode('utf-8')}\n\n"
yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n"
content_block_index += 1
# Handle trailing buffer
for event in suppressor.flush():
if event["type"] == "text":
if not in_text_block:
yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'text', 'text': ''}}).decode('utf-8')}\n\n"
in_text_block = True
delta = {
"type": "content_block_delta",
"index": content_block_index,
"delta": {"type": "text_delta", "text": event["content"]},
}
yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n"
elif event["type"] == "tool_start":
if in_text_block:
yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n"
content_block_index += 1
in_text_block = False
yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'tool_use', 'id': event['id'], 'name': event['name'], 'input': {}}}).decode('utf-8')}\n\n"
elif event["type"] == "tool_delta":
delta = {
"type": "content_block_delta",
"index": content_block_index,
"delta": {"type": "input_json_delta", "partial_json": event["content"]},
}
yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n"
elif event["type"] == "tool_end":
yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n"
content_block_index += 1
except Exception as e:
logger.exception(f"Error during Anthropic streaming: {e}")
yield f"event: error\ndata: {orjson.dumps({'type': 'error', 'error': {'type': 'api_error', 'message': str(e)}}).decode('utf-8')}\n\n"
return
if in_text_block:
yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n"
# 5. Message Delta (Stop Reason & Usage)
c_tok = estimate_tokens(full_text)
msg_delta = {
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
"usage": {"output_tokens": c_tok},
}
yield f"event: message_delta\ndata: {orjson.dumps(msg_delta).decode('utf-8')}\n\n"
# 6. Message Stop
yield f"event: message_stop\ndata: {orjson.dumps({'type': 'message_stop'}).decode('utf-8')}\n\n"
# 7. Persist conversation if session context was provided
if (
db is not None
and model is not None
and client_wrapper is not None
and session is not None
):
_persist_conversation(
db,
model.model_name,
client_wrapper.id,
session.metadata,
messages,
assistant_storage,
tool_calls,
final_thoughts if final_thoughts else None,
)
return StreamingResponse(generate_stream(), media_type="text/event-stream")
# --- Main Router Endpoints ---
@router.get("/v1/models", response_model=ModelListResponse)
async def list_models(api_key: str = Depends(verify_api_key)):
models = get_available_models()
return ModelListResponse(data=models)
@router.post("/v1/chat/completions")
async def create_chat_completion(
request: ChatCompletionRequest,
raw_request: Request,
api_key: str = Depends(verify_api_key),
tmp_dir: Path = Depends(get_temp_dir),
image_store: Path = Depends(get_image_store_dir),
):
return await handle_chat_completion(
request=request,
raw_request=raw_request,
tmp_dir=tmp_dir,
image_store=image_store,
deps=OpenAIChatHandlerDependencies(
execute_core_chat_session=_execute_core_chat_session,
create_real_streaming_response=_create_real_streaming_response,
create_chat_completion_standard_payload=create_chat_completion_standard_payload,
process_llm_output=process_llm_output,
calculate_usage=calculate_usage,
persist_conversation=_persist_conversation,
get_image_token=get_image_token,
),
)
@router.post("/v1/messages")
async def create_anthropic_message(
request: AnthropicMessageRequest,
raw_request: Request,
api_key: str = Depends(verify_api_key),
tmp_dir: Path = Depends(get_temp_dir),
):
return await handle_anthropic_messages(
request=request,
raw_request=raw_request,
tmp_dir=tmp_dir,
deps=AnthropicHandlerDependencies(
execute_core_chat_session=_execute_core_chat_session,
create_anthropic_streaming_response=_create_anthropic_streaming_response,
process_llm_output=process_llm_output,
calculate_usage=calculate_usage,
persist_conversation=_persist_conversation,
),
)
@router.post("/v1/responses")
async def create_response(
request: ResponseCreateRequest,
raw_request: Request,
api_key: str = Depends(verify_api_key),
tmp_dir: Path = Depends(get_temp_dir),
image_store: Path = Depends(get_image_store_dir),
):
return await handle_responses_api(
request=request,
raw_request=raw_request,
tmp_dir=tmp_dir,
image_store=image_store,
deps=ResponsesHandlerDependencies(
response_items_to_messages=response_items_to_messages,
instructions_to_messages=instructions_to_messages,
build_structured_requirement=build_structured_requirement,
build_image_generation_instruction=build_image_generation_instruction,
prepare_messages_for_model=prepare_messages_for_model,
get_default_model_name=get_default_model_name,
create_responses_real_streaming_response=_create_responses_real_streaming_response,
create_responses_standard_payload=create_responses_standard_payload,
process_llm_output=process_llm_output,
calculate_usage=calculate_usage,
persist_conversation=_persist_conversation,
get_image_token=get_image_token,
),
)