import base64 import hashlib import io import uuid from pathlib import Path from typing import Any, AsyncGenerator, Sequence import orjson from fastapi import APIRouter, Depends, Request from fastapi.responses import StreamingResponse from gemini_webapi import ModelOutput from gemini_webapi.client import ChatSession from gemini_webapi.constants import Model from gemini_webapi.types.image import GeneratedImage, Image from loguru import logger from ..models import ( AnthropicMessageRequest, ChatCompletionRequest, ConversationInStore, Message, ModelListResponse, ResponseCreateRequest, ResponseImageGenerationCall, ResponseOutputContent, ResponseToolCall, ResponseToolChoice, ResponseUsage, Tool, ToolChoiceFunction, ) from ..services import GeminiClientPool, GeminiClientWrapper, LMDBConversationStore from ..services.chat_session_service import ( execute_core_chat_session as shared_execute_core_chat_session, ) from ..services.chat_session_service import ( find_reusable_session as shared_find_reusable_session, ) from ..services.chat_session_service import send_with_split as shared_send_with_split from ..utils import g_config from ..utils.helper import ( detect_image_extension, estimate_tokens, extract_image_dimensions, ) from .anthropic_handler import AnthropicHandlerDependencies, handle_anthropic_messages from .chat_support import ( StructuredOutputRequirement as SharedStructuredOutputRequirement, ) from .chat_support import ( build_image_generation_instruction, build_structured_requirement, calculate_usage, create_chat_completion_standard_payload, create_responses_standard_payload, get_available_models, get_default_model_name, instructions_to_messages, prepare_messages_for_model, process_llm_output, response_items_to_messages, ) from .middleware import get_image_store_dir, get_image_token, get_temp_dir, verify_api_key from .openai_chat_handler import OpenAIChatHandlerDependencies, handle_chat_completion from .responses_handler import ResponsesHandlerDependencies, handle_responses_api from .streaming_filter import StreamingOutputFilter MAX_CHARS_PER_REQUEST = int(g_config.gemini.max_chars_per_request * 0.9) METADATA_TTL_MINUTES = 15 router = APIRouter() StructuredOutputRequirement = SharedStructuredOutputRequirement # --- Helper Functions --- async def _image_to_base64( image: Image, temp_dir: Path ) -> tuple[str, int | None, int | None, str, str]: """Persist an image provided by gemini_webapi and return base64 plus dimensions, filename, and hash.""" if isinstance(image, GeneratedImage): try: saved_path = await image.save(path=str(temp_dir), full_size=True) except Exception as e: logger.warning( f"Failed to download full-size GeneratedImage, retrying with default size: {e}" ) saved_path = await image.save(path=str(temp_dir), full_size=False) else: saved_path = await image.save(path=str(temp_dir)) if not saved_path: raise ValueError("Failed to save generated image") original_path = Path(saved_path) data = original_path.read_bytes() suffix = original_path.suffix if not suffix: detected_ext = detect_image_extension(data) suffix = detected_ext or (".png" if isinstance(image, GeneratedImage) else ".jpg") random_name = f"img_{uuid.uuid4().hex}{suffix}" new_path = temp_dir / random_name original_path.rename(new_path) width, height = extract_image_dimensions(data) filename = random_name file_hash = hashlib.sha256(data).hexdigest() return base64.b64encode(data).decode("ascii"), width, height, filename, file_hash def _persist_conversation( db: LMDBConversationStore, model_name: str, client_id: str, metadata: list[str | None], messages: list[Message], storage_output: str | None, tool_calls: list[Any] | None, thoughts: str | None = None, ) -> str | None: """Unified logic to save conversation history to LMDB.""" try: current_assistant_message = Message( role="assistant", content=storage_output or None, tool_calls=tool_calls or None, reasoning_content=thoughts or None, ) full_history = [*messages, current_assistant_message] cleaned_history = db.sanitize_assistant_messages(full_history) conv = ConversationInStore( model=model_name, client_id=client_id, metadata=metadata, messages=cleaned_history, ) key = db.store(conv) logger.debug(f"Conversation saved to LMDB with key: {key[:12]}") return key except Exception as e: logger.warning(f"Failed to save {len(messages) + 1} messages to LMDB: {e}") return None async def _find_reusable_session( db: LMDBConversationStore, pool: GeminiClientPool, model: Model, messages: list[Message], ) -> tuple[ChatSession | None, GeminiClientWrapper | None, list[Message]]: """Delegate reusable session lookup to the shared session service.""" return await shared_find_reusable_session( db, pool, model, messages, metadata_ttl_minutes=METADATA_TTL_MINUTES, ) async def _send_with_split( session: ChatSession, text: str, files: Sequence[Path | str | io.BytesIO] | None = None, stream: bool = False, ) -> AsyncGenerator[ModelOutput, None] | ModelOutput: """Delegate Gemini send/split logic to the shared session service.""" return await shared_send_with_split( session, text, files=files, stream=stream, max_chars_per_request=MAX_CHARS_PER_REQUEST, ) # --- Response Builders & Streaming --- def _create_real_streaming_response( generator: AsyncGenerator[ModelOutput, None], completion_id: str, created_time: int, model_name: str, messages: list[Message], db: LMDBConversationStore, model: Model, client_wrapper: GeminiClientWrapper, session: ChatSession, base_url: str, structured_requirement: StructuredOutputRequirement | None = None, ) -> StreamingResponse: """ Create a real-time streaming response. Reconciles manual delta accumulation with the model's final authoritative state. """ async def generate_stream(): full_thoughts, full_text = "", "" has_started = False all_outputs: list[ModelOutput] = [] suppressor = StreamingOutputFilter() try: async for chunk in generator: all_outputs.append(chunk) if not has_started: data = { "id": completion_id, "object": "chat.completion.chunk", "created": created_time, "model": model_name, "choices": [ {"index": 0, "delta": {"role": "assistant"}, "finish_reason": None} ], } yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n" has_started = True if hasattr(chunk, "thoughts_delta") and chunk.thoughts_delta: t_delta = str(chunk.thoughts_delta) full_thoughts += t_delta data = { "id": completion_id, "object": "chat.completion.chunk", "created": created_time, "model": model_name, "choices": [ { "index": 0, "delta": {"reasoning_content": t_delta}, "finish_reason": None, } ], } yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n" elif chunk.thoughts and len(chunk.thoughts) > len(full_thoughts): t_delta = chunk.thoughts[len(full_thoughts) :] data = { "id": completion_id, "object": "chat.completion.chunk", "created": created_time, "model": model_name, "choices": [ { "index": 0, "delta": {"reasoning_content": t_delta}, "finish_reason": None, } ], } yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n" full_thoughts = chunk.thoughts # Robust text delta calculation new_text = chunk.text or "" # Strip transient triple-backtick fence which is often added by the web-scraping backend # during streaming snapshots. We will restore the real ending after the loop. display_text = StreamingOutputFilter.TRAILING_FENCE_RE.sub("", new_text) if len(display_text) > len(full_text): text_delta = display_text[len(full_text) :] filter_events = suppressor.process(text_delta) for event in filter_events: delta = {} if event["type"] == "text": delta = {"content": event["content"]} elif event["type"] == "tool_start": delta = { "tool_calls": [ { "index": event["index"], "id": event["id"], "type": "function", "function": {"name": event["name"], "arguments": ""}, } ] } elif event["type"] == "tool_delta": delta = { "tool_calls": [ { "index": event["index"], "function": {"arguments": event["content"]}, } ] } if delta: data = { "id": completion_id, "object": "chat.completion.chunk", "created": created_time, "model": model_name, "choices": [ { "index": 0, "delta": delta, "finish_reason": None, } ], } yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n" full_text = display_text except Exception as e: logger.exception(f"Error during OpenAI streaming: {e}") yield f"data: {orjson.dumps({'error': {'message': 'Streaming error occurred.', 'type': 'server_error', 'param': None, 'code': None}}).decode('utf-8')}\n\n" return if all_outputs: final_chunk = all_outputs[-1] final_text = final_chunk.text or "" if len(final_text) > len(full_text): # Yield any real trailing content (like the actual closing fence) final_delta = final_text[len(full_text) :] for event in suppressor.process(final_delta): if event["type"] == "text": yield f"data: {orjson.dumps({'id': completion_id, 'object': 'chat.completion.chunk', 'created': created_time, 'model': model_name, 'choices': [{'index': 0, 'delta': {'content': event['content']}, 'finish_reason': None}]}).decode('utf-8')}\n\n" full_text = final_text if final_chunk.thoughts: full_thoughts = final_chunk.thoughts for event in suppressor.flush(): delta = {} if event["type"] == "text": delta = {"content": event["content"]} elif event["type"] == "tool_start": delta = { "tool_calls": [ { "index": event["index"], "id": event["id"], "type": "function", "function": {"name": event["name"], "arguments": ""}, } ] } elif event["type"] == "tool_delta": delta = { "tool_calls": [ {"index": event["index"], "function": {"arguments": event["content"]}} ] } if delta: data = { "id": completion_id, "object": "chat.completion.chunk", "created": created_time, "model": model_name, "choices": [{"index": 0, "delta": delta, "finish_reason": None}], } yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n" _thoughts, assistant_text, storage_output, tool_calls = process_llm_output( full_thoughts, full_text, structured_requirement ) images = [] seen_urls = set() for out in all_outputs: if out.images: for img in out.images: # Use the image URL as a stable identifier across chunks if img.url not in seen_urls: images.append(img) seen_urls.add(img.url) image_markdown = "" seen_hashes = set() for image in images: try: image_store = get_image_store_dir() _, _, _, filename, file_hash = await _image_to_base64(image, image_store) if file_hash in seen_hashes: # Duplicate content, delete the file and skip (image_store / filename).unlink(missing_ok=True) continue seen_hashes.add(file_hash) img_url = ( f"![{filename}]({base_url}images/{filename}?token={get_image_token(filename)})" ) image_markdown += f"\n\n{img_url}" except Exception as exc: logger.warning(f"Failed to process image in OpenAI stream: {exc}") if image_markdown: assistant_text += image_markdown storage_output += image_markdown yield f"data: {orjson.dumps({'id': completion_id, 'object': 'chat.completion.chunk', 'created': created_time, 'model': model_name, 'choices': [{'index': 0, 'delta': {'content': image_markdown}, 'finish_reason': None}]}).decode('utf-8')}\n\n" p_tok, c_tok, t_tok, r_tok = calculate_usage( messages, assistant_text, tool_calls, full_thoughts ) usage = { "prompt_tokens": p_tok, "completion_tokens": c_tok, "total_tokens": t_tok, "completion_tokens_details": {"reasoning_tokens": r_tok}, } final_delta = {} if tool_calls: # Add index field to each tool_call for OpenAI compatibility final_delta["tool_calls"] = [ {**call.model_dump(mode="json"), "index": i} for i, call in enumerate(tool_calls) ] data = { "id": completion_id, "object": "chat.completion.chunk", "created": created_time, "model": model_name, "choices": [ { "index": 0, "delta": final_delta, "finish_reason": "tool_calls" if tool_calls else "stop", } ], "usage": usage, } _persist_conversation( db, model.model_name, client_wrapper.id, session.metadata, messages, # This should be the prepared messages storage_output, tool_calls, full_thoughts, ) yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n" yield "data: [DONE]\n\n" return StreamingResponse(generate_stream(), media_type="text/event-stream") def _create_responses_real_streaming_response( generator: AsyncGenerator[ModelOutput, None], response_id: str, created_time: int, model_name: str, messages: list[Message], db: LMDBConversationStore, model: Model, client_wrapper: GeminiClientWrapper, session: ChatSession, request: ResponseCreateRequest, image_store: Path, base_url: str, structured_requirement: StructuredOutputRequirement | None = None, ) -> StreamingResponse: """ Create a real-time streaming response for the Responses API. Ensures final accumulated text and thoughts are synchronized. """ base_event = { "id": response_id, "object": "response", "created_at": created_time, "model": model_name, } async def generate_stream(): yield f"data: {orjson.dumps({**base_event, 'type': 'response.created', 'response': {'id': response_id, 'object': 'response', 'created_at': created_time, 'model': model_name, 'status': 'in_progress', 'metadata': request.metadata, 'input': None, 'tools': request.tools, 'tool_choice': request.tool_choice}}).decode('utf-8')}\n\n" message_id = f"msg_{uuid.uuid4().hex}" yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.added', 'output_index': 0, 'item': {'id': message_id, 'type': 'message', 'role': 'assistant', 'content': []}}).decode('utf-8')}\n\n" full_thoughts, full_text = "", "" last_chunk_was_thought = False all_outputs: list[ModelOutput] = [] suppressor = StreamingOutputFilter() try: async for chunk in generator: all_outputs.append(chunk) # Robust thought delta calculation new_thoughts = chunk.thoughts or "" if len(new_thoughts) > len(full_thoughts): t_delta = new_thoughts[len(full_thoughts) :] if not last_chunk_was_thought and not full_thoughts: yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': ''}).decode('utf-8')}\n\n" yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': t_delta}).decode('utf-8')}\n\n" full_thoughts = new_thoughts last_chunk_was_thought = True # Robust text delta calculation new_text = chunk.text or "" # Strip transient triple-backtick fence which is often added by the web-scraping backend # during streaming snapshots. We will restore the real ending after the loop. display_text = StreamingOutputFilter.TRAILING_FENCE_RE.sub("", new_text) if len(display_text) > len(full_text): text_delta = display_text[len(full_text) :] if last_chunk_was_thought: yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': '\n'}).decode('utf-8')}\n\n" last_chunk_was_thought = False filter_events = suppressor.process(text_delta) for event in filter_events: if event["type"] == "text": yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': event['content']}).decode('utf-8')}\n\n" # Note: This API currently handles tool calls as discrete items at the end. # We ignore internal tool call events in the text stream for now. full_text = display_text except Exception as e: logger.exception(f"Error during Responses API streaming: {e}") yield f"data: {orjson.dumps({**base_event, 'type': 'error', 'error': {'message': 'Streaming error.'}}).decode('utf-8')}\n\n" return if all_outputs: final_chunk = all_outputs[-1] final_text = final_chunk.text or "" if len(final_text) > len(full_text): # Yield any real trailing content (like the actual closing fence) final_delta = final_text[len(full_text) :] for event in suppressor.process(final_delta): if event["type"] == "text": yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': event['content']}).decode('utf-8')}\n\n" full_text = final_text if final_chunk.thoughts: full_thoughts = final_chunk.thoughts if last_chunk_was_thought: yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': '\n'}).decode('utf-8')}\n\n" for event in suppressor.flush(): if event["type"] == "text": yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': event['content']}).decode('utf-8')}\n\n" yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.done', 'output_index': 0}).decode('utf-8')}\n\n" _thoughts, assistant_text, storage_output, detected_tool_calls = process_llm_output( full_thoughts, full_text, structured_requirement ) images = [] seen_urls = set() for out in all_outputs: if out.images: for img in out.images: if img.url not in seen_urls: images.append(img) seen_urls.add(img.url) # Check if image generation was forced via tool_choice (same logic as non-streaming) image_generation_forced = ( request.tool_choice is not None and isinstance(request.tool_choice, ResponseToolChoice) and request.tool_choice.type == "image_generation" ) if image_generation_forced and not images and not assistant_text: logger.warning( "Image generation was forced via tool_choice but no images or text were returned in stream." ) yield f"data: {orjson.dumps({**base_event, 'type': 'error', 'error': {'message': 'No images returned from forced image generation request.'}}).decode('utf-8')}\n\n" return response_contents, image_call_items = [], [] seen_hashes = set() for image in images: try: image_base64, width, height, filename, file_hash = await _image_to_base64( image, image_store ) if file_hash in seen_hashes: (image_store / filename).unlink(missing_ok=True) continue seen_hashes.add(file_hash) img_format = "png" if isinstance(image, GeneratedImage) else "jpeg" image_url = ( f"![{filename}]({base_url}images/{filename}?token={get_image_token(filename)})" ) image_call_items.append( ResponseImageGenerationCall( id=filename.rsplit(".", 1)[0], result=image_base64, output_format=img_format, size=f"{width}x{height}" if width and height else None, ) ) response_contents.append(ResponseOutputContent(type="output_text", text=image_url)) except Exception as exc: logger.warning(f"Failed to process image in stream: {exc}") if assistant_text: response_contents.append(ResponseOutputContent(type="output_text", text=assistant_text)) if not response_contents: response_contents.append(ResponseOutputContent(type="output_text", text="")) # Aggregate images for storage image_markdown = "" for img_call in image_call_items: fname = f"{img_call.id}.{img_call.output_format}" img_url = f"![{fname}]({base_url}images/{fname}?token={get_image_token(fname)})" image_markdown += f"\n\n{img_url}" if image_markdown: storage_output += image_markdown yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.done', 'output_index': 0, 'item': {'id': message_id, 'type': 'message', 'role': 'assistant', 'content': [c.model_dump(mode='json') for c in response_contents]}}).decode('utf-8')}\n\n" current_idx = 1 for call in detected_tool_calls: tc_item = ResponseToolCall(id=call.id, status="completed", function=call.function) yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.added', 'output_index': current_idx, 'item': tc_item.model_dump(mode='json')}).decode('utf-8')}\n\n" yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.done', 'output_index': current_idx, 'item': tc_item.model_dump(mode='json')}).decode('utf-8')}\n\n" current_idx += 1 for img_call in image_call_items: yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.added', 'output_index': current_idx, 'item': img_call.model_dump(mode='json')}).decode('utf-8')}\n\n" yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.done', 'output_index': current_idx, 'item': img_call.model_dump(mode='json')}).decode('utf-8')}\n\n" current_idx += 1 p_tok, c_tok, t_tok, r_tok = calculate_usage( messages, assistant_text, detected_tool_calls, full_thoughts ) usage = ResponseUsage( input_tokens=p_tok, output_tokens=c_tok, total_tokens=t_tok, output_tokens_details={"reasoning_tokens": r_tok}, ) payload = create_responses_standard_payload( response_id, created_time, model_name, detected_tool_calls, image_call_items, response_contents, usage, request, None, full_thoughts, ) _persist_conversation( db, model.model_name, client_wrapper.id, session.metadata, messages, storage_output, detected_tool_calls, full_thoughts, ) yield f"data: {orjson.dumps({**base_event, 'type': 'response.completed', 'response': payload.model_dump(mode='json')}).decode('utf-8')}\n\n" yield "data: [DONE]\n\n" return StreamingResponse(generate_stream(), media_type="text/event-stream") # --- Core Session Logic --- async def _execute_core_chat_session( request_model: str, messages: list[Message], tmp_dir: Path, tools: list[Tool] | None = None, tool_choice: str | ToolChoiceFunction | None = None, stream: bool = False, extra_instructions: list[str] | None = None, response_format: dict[str, Any] | None = None, ): """ Delegate shared Gemini session execution to the session service and preserve the tuple contract expected by existing handlers. """ result = await shared_execute_core_chat_session( request_model=request_model, messages=messages, tmp_dir=tmp_dir, tools=tools, tool_choice=tool_choice, stream=stream, extra_instructions=extra_instructions, response_format=response_format, metadata_ttl_minutes=METADATA_TTL_MINUTES, ) return ( result.response_or_stream, result.completion_id, result.created_time, result.session, result.client, result.db, result.model, result.prepared_messages, result.structured_requirement, ) async def _create_anthropic_streaming_response( generator: AsyncGenerator[ModelOutput, None], completion_id: str, created_time: int, model_name: str, messages: list[Message], db: LMDBConversationStore | None = None, model: Model | None = None, client_wrapper: GeminiClientWrapper | None = None, session: ChatSession | None = None, ): """ Stream Gemini output as Anthropic SSE events. Now supports structured tool call streaming. """ async def generate_stream(): # 1. Message Start msg_start = { "type": "message_start", "message": { "id": completion_id, "type": "message", "role": "assistant", "content": [], "model": model_name, "stop_reason": None, "stop_sequence": None, "usage": {"input_tokens": 0, "output_tokens": 0}, }, } yield f"event: message_start\ndata: {orjson.dumps(msg_start).decode('utf-8')}\n\n" full_text = "" suppressor = StreamingOutputFilter() content_block_index = 0 in_text_block = False last_chunk: ModelOutput | None = None # 3. Stream Deltas try: async for chunk in generator: last_chunk = chunk if chunk.text: display_text = StreamingOutputFilter.TRAILING_FENCE_RE.sub("", chunk.text) if len(display_text) > len(full_text): delta_text = display_text[len(full_text) :] full_text = display_text filter_events = suppressor.process(delta_text) for event in filter_events: if event["type"] == "text": if not in_text_block: # Start a new text block if not in one yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'text', 'text': ''}}).decode('utf-8')}\n\n" in_text_block = True delta = { "type": "content_block_delta", "index": content_block_index, "delta": {"type": "text_delta", "text": event["content"]}, } yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n" elif event["type"] == "tool_start": if in_text_block: yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" content_block_index += 1 in_text_block = False # Anthropic tool_use start yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'tool_use', 'id': event['id'], 'name': event['name'], 'input': {}}}).decode('utf-8')}\n\n" elif event["type"] == "tool_delta": # Anthropic tool_use delta (input_json_delta) delta = { "type": "content_block_delta", "index": content_block_index, "delta": { "type": "input_json_delta", "partial_json": event["content"], }, } yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n" elif event["type"] == "tool_end": yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" content_block_index += 1 # We stay in "not in_text_block" state until next text event # Handle final delta after loop ends final_text = "" final_thoughts = "" if last_chunk: final_text = last_chunk.text or "" final_thoughts = last_chunk.thoughts or "" if len(final_text) > len(full_text): final_delta = final_text[len(full_text) :] for event in suppressor.process(final_delta): if event["type"] == "text": if not in_text_block: yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'text', 'text': ''}}).decode('utf-8')}\n\n" in_text_block = True delta = { "type": "content_block_delta", "index": content_block_index, "delta": {"type": "text_delta", "text": event["content"]}, } yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n" full_text = final_text # Final verification of tool calls (Built-in tools like image_retrieval often appear at the end) _thoughts_out, _assistant_visible, assistant_storage, tool_calls = process_llm_output( final_thoughts, full_text, None ) # If we detected tool_calls that weren't captured by suppressor yet, emit them now # (Note: suppressor currently handles manual formatting, but built-in tools are caught here) # To avoid duplicates if suppressor already found them, we could check event IDs. # But normally built-in tools won't trigger the regex-based suppressor. if tool_calls: for tcall in tool_calls: # Emit as a new content block if it's a tool_use if in_text_block: yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" content_block_index += 1 in_text_block = False yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'tool_use', 'id': tcall.id, 'name': tcall.function.name, 'input': {}}}).decode('utf-8')}\n\n" yield f"event: content_block_delta\ndata: {orjson.dumps({'type': 'content_block_delta', 'index': content_block_index, 'delta': {'type': 'input_json_delta', 'partial_json': tcall.function.arguments}}).decode('utf-8')}\n\n" yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" content_block_index += 1 # Handle trailing buffer for event in suppressor.flush(): if event["type"] == "text": if not in_text_block: yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'text', 'text': ''}}).decode('utf-8')}\n\n" in_text_block = True delta = { "type": "content_block_delta", "index": content_block_index, "delta": {"type": "text_delta", "text": event["content"]}, } yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n" elif event["type"] == "tool_start": if in_text_block: yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" content_block_index += 1 in_text_block = False yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'tool_use', 'id': event['id'], 'name': event['name'], 'input': {}}}).decode('utf-8')}\n\n" elif event["type"] == "tool_delta": delta = { "type": "content_block_delta", "index": content_block_index, "delta": {"type": "input_json_delta", "partial_json": event["content"]}, } yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n" elif event["type"] == "tool_end": yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" content_block_index += 1 except Exception as e: logger.exception(f"Error during Anthropic streaming: {e}") yield f"event: error\ndata: {orjson.dumps({'type': 'error', 'error': {'type': 'api_error', 'message': str(e)}}).decode('utf-8')}\n\n" return if in_text_block: yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" # 5. Message Delta (Stop Reason & Usage) c_tok = estimate_tokens(full_text) msg_delta = { "type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": c_tok}, } yield f"event: message_delta\ndata: {orjson.dumps(msg_delta).decode('utf-8')}\n\n" # 6. Message Stop yield f"event: message_stop\ndata: {orjson.dumps({'type': 'message_stop'}).decode('utf-8')}\n\n" # 7. Persist conversation if session context was provided if ( db is not None and model is not None and client_wrapper is not None and session is not None ): _persist_conversation( db, model.model_name, client_wrapper.id, session.metadata, messages, assistant_storage, tool_calls, final_thoughts if final_thoughts else None, ) return StreamingResponse(generate_stream(), media_type="text/event-stream") # --- Main Router Endpoints --- @router.get("/v1/models", response_model=ModelListResponse) async def list_models(api_key: str = Depends(verify_api_key)): models = get_available_models() return ModelListResponse(data=models) @router.post("/v1/chat/completions") async def create_chat_completion( request: ChatCompletionRequest, raw_request: Request, api_key: str = Depends(verify_api_key), tmp_dir: Path = Depends(get_temp_dir), image_store: Path = Depends(get_image_store_dir), ): return await handle_chat_completion( request=request, raw_request=raw_request, tmp_dir=tmp_dir, image_store=image_store, deps=OpenAIChatHandlerDependencies( execute_core_chat_session=_execute_core_chat_session, create_real_streaming_response=_create_real_streaming_response, create_chat_completion_standard_payload=create_chat_completion_standard_payload, process_llm_output=process_llm_output, calculate_usage=calculate_usage, persist_conversation=_persist_conversation, get_image_token=get_image_token, ), ) @router.post("/v1/messages") async def create_anthropic_message( request: AnthropicMessageRequest, raw_request: Request, api_key: str = Depends(verify_api_key), tmp_dir: Path = Depends(get_temp_dir), ): return await handle_anthropic_messages( request=request, raw_request=raw_request, tmp_dir=tmp_dir, deps=AnthropicHandlerDependencies( execute_core_chat_session=_execute_core_chat_session, create_anthropic_streaming_response=_create_anthropic_streaming_response, process_llm_output=process_llm_output, calculate_usage=calculate_usage, persist_conversation=_persist_conversation, ), ) @router.post("/v1/responses") async def create_response( request: ResponseCreateRequest, raw_request: Request, api_key: str = Depends(verify_api_key), tmp_dir: Path = Depends(get_temp_dir), image_store: Path = Depends(get_image_store_dir), ): return await handle_responses_api( request=request, raw_request=raw_request, tmp_dir=tmp_dir, image_store=image_store, deps=ResponsesHandlerDependencies( response_items_to_messages=response_items_to_messages, instructions_to_messages=instructions_to_messages, build_structured_requirement=build_structured_requirement, build_image_generation_instruction=build_image_generation_instruction, prepare_messages_for_model=prepare_messages_for_model, get_default_model_name=get_default_model_name, create_responses_real_streaming_response=_create_responses_real_streaming_response, create_responses_standard_payload=create_responses_standard_payload, process_llm_output=process_llm_output, calculate_usage=calculate_usage, persist_conversation=_persist_conversation, get_image_token=get_image_token, ), )