Spaces:
Runtime error
Runtime error
Download app/server/chat.py from 520superxu520/G_AI: direct link, hf CLI and curl.
- Browser
- Download file 43.3 kB
-
https://huggingface.co/spaces/520superxu520/G_AI/resolve/main/app/server/chat.py
- Command line
-
hf download hf://spaces/520superxu520/G_AI/app/server/chat.py
-
curl -L -o chat.py https://huggingface.co/spaces/520superxu520/G_AI/resolve/main/app/server/chat.py
43.3 kB
| import base64 | |
| import hashlib | |
| import io | |
| import uuid | |
| from pathlib import Path | |
| from typing import Any, AsyncGenerator, Sequence | |
| import orjson | |
| from fastapi import APIRouter, Depends, Request | |
| from fastapi.responses import StreamingResponse | |
| from gemini_webapi import ModelOutput | |
| from gemini_webapi.client import ChatSession | |
| from gemini_webapi.constants import Model | |
| from gemini_webapi.types.image import GeneratedImage, Image | |
| from loguru import logger | |
| from ..models import ( | |
| AnthropicMessageRequest, | |
| ChatCompletionRequest, | |
| ConversationInStore, | |
| Message, | |
| ModelListResponse, | |
| ResponseCreateRequest, | |
| ResponseImageGenerationCall, | |
| ResponseOutputContent, | |
| ResponseToolCall, | |
| ResponseToolChoice, | |
| ResponseUsage, | |
| Tool, | |
| ToolChoiceFunction, | |
| ) | |
| from ..services import GeminiClientPool, GeminiClientWrapper, LMDBConversationStore | |
| from ..services.chat_session_service import ( | |
| execute_core_chat_session as shared_execute_core_chat_session, | |
| ) | |
| from ..services.chat_session_service import ( | |
| find_reusable_session as shared_find_reusable_session, | |
| ) | |
| from ..services.chat_session_service import send_with_split as shared_send_with_split | |
| from ..utils import g_config | |
| from ..utils.helper import ( | |
| detect_image_extension, | |
| estimate_tokens, | |
| extract_image_dimensions, | |
| ) | |
| from .anthropic_handler import AnthropicHandlerDependencies, handle_anthropic_messages | |
| from .chat_support import ( | |
| StructuredOutputRequirement as SharedStructuredOutputRequirement, | |
| ) | |
| from .chat_support import ( | |
| build_image_generation_instruction, | |
| build_structured_requirement, | |
| calculate_usage, | |
| create_chat_completion_standard_payload, | |
| create_responses_standard_payload, | |
| get_available_models, | |
| get_default_model_name, | |
| instructions_to_messages, | |
| prepare_messages_for_model, | |
| process_llm_output, | |
| response_items_to_messages, | |
| ) | |
| from .middleware import get_image_store_dir, get_image_token, get_temp_dir, verify_api_key | |
| from .openai_chat_handler import OpenAIChatHandlerDependencies, handle_chat_completion | |
| from .responses_handler import ResponsesHandlerDependencies, handle_responses_api | |
| from .streaming_filter import StreamingOutputFilter | |
| MAX_CHARS_PER_REQUEST = int(g_config.gemini.max_chars_per_request * 0.9) | |
| METADATA_TTL_MINUTES = 15 | |
| router = APIRouter() | |
| StructuredOutputRequirement = SharedStructuredOutputRequirement | |
| # --- Helper Functions --- | |
| async def _image_to_base64( | |
| image: Image, temp_dir: Path | |
| ) -> tuple[str, int | None, int | None, str, str]: | |
| """Persist an image provided by gemini_webapi and return base64 plus dimensions, filename, and hash.""" | |
| if isinstance(image, GeneratedImage): | |
| try: | |
| saved_path = await image.save(path=str(temp_dir), full_size=True) | |
| except Exception as e: | |
| logger.warning( | |
| f"Failed to download full-size GeneratedImage, retrying with default size: {e}" | |
| ) | |
| saved_path = await image.save(path=str(temp_dir), full_size=False) | |
| else: | |
| saved_path = await image.save(path=str(temp_dir)) | |
| if not saved_path: | |
| raise ValueError("Failed to save generated image") | |
| original_path = Path(saved_path) | |
| data = original_path.read_bytes() | |
| suffix = original_path.suffix | |
| if not suffix: | |
| detected_ext = detect_image_extension(data) | |
| suffix = detected_ext or (".png" if isinstance(image, GeneratedImage) else ".jpg") | |
| random_name = f"img_{uuid.uuid4().hex}{suffix}" | |
| new_path = temp_dir / random_name | |
| original_path.rename(new_path) | |
| width, height = extract_image_dimensions(data) | |
| filename = random_name | |
| file_hash = hashlib.sha256(data).hexdigest() | |
| return base64.b64encode(data).decode("ascii"), width, height, filename, file_hash | |
| def _persist_conversation( | |
| db: LMDBConversationStore, | |
| model_name: str, | |
| client_id: str, | |
| metadata: list[str | None], | |
| messages: list[Message], | |
| storage_output: str | None, | |
| tool_calls: list[Any] | None, | |
| thoughts: str | None = None, | |
| ) -> str | None: | |
| """Unified logic to save conversation history to LMDB.""" | |
| try: | |
| current_assistant_message = Message( | |
| role="assistant", | |
| content=storage_output or None, | |
| tool_calls=tool_calls or None, | |
| reasoning_content=thoughts or None, | |
| ) | |
| full_history = [*messages, current_assistant_message] | |
| cleaned_history = db.sanitize_assistant_messages(full_history) | |
| conv = ConversationInStore( | |
| model=model_name, | |
| client_id=client_id, | |
| metadata=metadata, | |
| messages=cleaned_history, | |
| ) | |
| key = db.store(conv) | |
| logger.debug(f"Conversation saved to LMDB with key: {key[:12]}") | |
| return key | |
| except Exception as e: | |
| logger.warning(f"Failed to save {len(messages) + 1} messages to LMDB: {e}") | |
| return None | |
| async def _find_reusable_session( | |
| db: LMDBConversationStore, | |
| pool: GeminiClientPool, | |
| model: Model, | |
| messages: list[Message], | |
| ) -> tuple[ChatSession | None, GeminiClientWrapper | None, list[Message]]: | |
| """Delegate reusable session lookup to the shared session service.""" | |
| return await shared_find_reusable_session( | |
| db, | |
| pool, | |
| model, | |
| messages, | |
| metadata_ttl_minutes=METADATA_TTL_MINUTES, | |
| ) | |
| async def _send_with_split( | |
| session: ChatSession, | |
| text: str, | |
| files: Sequence[Path | str | io.BytesIO] | None = None, | |
| stream: bool = False, | |
| ) -> AsyncGenerator[ModelOutput, None] | ModelOutput: | |
| """Delegate Gemini send/split logic to the shared session service.""" | |
| return await shared_send_with_split( | |
| session, | |
| text, | |
| files=files, | |
| stream=stream, | |
| max_chars_per_request=MAX_CHARS_PER_REQUEST, | |
| ) | |
| # --- Response Builders & Streaming --- | |
| def _create_real_streaming_response( | |
| generator: AsyncGenerator[ModelOutput, None], | |
| completion_id: str, | |
| created_time: int, | |
| model_name: str, | |
| messages: list[Message], | |
| db: LMDBConversationStore, | |
| model: Model, | |
| client_wrapper: GeminiClientWrapper, | |
| session: ChatSession, | |
| base_url: str, | |
| structured_requirement: StructuredOutputRequirement | None = None, | |
| ) -> StreamingResponse: | |
| """ | |
| Create a real-time streaming response. | |
| Reconciles manual delta accumulation with the model's final authoritative state. | |
| """ | |
| async def generate_stream(): | |
| full_thoughts, full_text = "", "" | |
| has_started = False | |
| all_outputs: list[ModelOutput] = [] | |
| suppressor = StreamingOutputFilter() | |
| try: | |
| async for chunk in generator: | |
| all_outputs.append(chunk) | |
| if not has_started: | |
| data = { | |
| "id": completion_id, | |
| "object": "chat.completion.chunk", | |
| "created": created_time, | |
| "model": model_name, | |
| "choices": [ | |
| {"index": 0, "delta": {"role": "assistant"}, "finish_reason": None} | |
| ], | |
| } | |
| yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n" | |
| has_started = True | |
| if hasattr(chunk, "thoughts_delta") and chunk.thoughts_delta: | |
| t_delta = str(chunk.thoughts_delta) | |
| full_thoughts += t_delta | |
| data = { | |
| "id": completion_id, | |
| "object": "chat.completion.chunk", | |
| "created": created_time, | |
| "model": model_name, | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "delta": {"reasoning_content": t_delta}, | |
| "finish_reason": None, | |
| } | |
| ], | |
| } | |
| yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n" | |
| elif chunk.thoughts and len(chunk.thoughts) > len(full_thoughts): | |
| t_delta = chunk.thoughts[len(full_thoughts) :] | |
| data = { | |
| "id": completion_id, | |
| "object": "chat.completion.chunk", | |
| "created": created_time, | |
| "model": model_name, | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "delta": {"reasoning_content": t_delta}, | |
| "finish_reason": None, | |
| } | |
| ], | |
| } | |
| yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n" | |
| full_thoughts = chunk.thoughts | |
| # Robust text delta calculation | |
| new_text = chunk.text or "" | |
| # Strip transient triple-backtick fence which is often added by the web-scraping backend | |
| # during streaming snapshots. We will restore the real ending after the loop. | |
| display_text = StreamingOutputFilter.TRAILING_FENCE_RE.sub("", new_text) | |
| if len(display_text) > len(full_text): | |
| text_delta = display_text[len(full_text) :] | |
| filter_events = suppressor.process(text_delta) | |
| for event in filter_events: | |
| delta = {} | |
| if event["type"] == "text": | |
| delta = {"content": event["content"]} | |
| elif event["type"] == "tool_start": | |
| delta = { | |
| "tool_calls": [ | |
| { | |
| "index": event["index"], | |
| "id": event["id"], | |
| "type": "function", | |
| "function": {"name": event["name"], "arguments": ""}, | |
| } | |
| ] | |
| } | |
| elif event["type"] == "tool_delta": | |
| delta = { | |
| "tool_calls": [ | |
| { | |
| "index": event["index"], | |
| "function": {"arguments": event["content"]}, | |
| } | |
| ] | |
| } | |
| if delta: | |
| data = { | |
| "id": completion_id, | |
| "object": "chat.completion.chunk", | |
| "created": created_time, | |
| "model": model_name, | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "delta": delta, | |
| "finish_reason": None, | |
| } | |
| ], | |
| } | |
| yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n" | |
| full_text = display_text | |
| except Exception as e: | |
| logger.exception(f"Error during OpenAI streaming: {e}") | |
| yield f"data: {orjson.dumps({'error': {'message': 'Streaming error occurred.', 'type': 'server_error', 'param': None, 'code': None}}).decode('utf-8')}\n\n" | |
| return | |
| if all_outputs: | |
| final_chunk = all_outputs[-1] | |
| final_text = final_chunk.text or "" | |
| if len(final_text) > len(full_text): | |
| # Yield any real trailing content (like the actual closing fence) | |
| final_delta = final_text[len(full_text) :] | |
| for event in suppressor.process(final_delta): | |
| if event["type"] == "text": | |
| yield f"data: {orjson.dumps({'id': completion_id, 'object': 'chat.completion.chunk', 'created': created_time, 'model': model_name, 'choices': [{'index': 0, 'delta': {'content': event['content']}, 'finish_reason': None}]}).decode('utf-8')}\n\n" | |
| full_text = final_text | |
| if final_chunk.thoughts: | |
| full_thoughts = final_chunk.thoughts | |
| for event in suppressor.flush(): | |
| delta = {} | |
| if event["type"] == "text": | |
| delta = {"content": event["content"]} | |
| elif event["type"] == "tool_start": | |
| delta = { | |
| "tool_calls": [ | |
| { | |
| "index": event["index"], | |
| "id": event["id"], | |
| "type": "function", | |
| "function": {"name": event["name"], "arguments": ""}, | |
| } | |
| ] | |
| } | |
| elif event["type"] == "tool_delta": | |
| delta = { | |
| "tool_calls": [ | |
| {"index": event["index"], "function": {"arguments": event["content"]}} | |
| ] | |
| } | |
| if delta: | |
| data = { | |
| "id": completion_id, | |
| "object": "chat.completion.chunk", | |
| "created": created_time, | |
| "model": model_name, | |
| "choices": [{"index": 0, "delta": delta, "finish_reason": None}], | |
| } | |
| yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n" | |
| _thoughts, assistant_text, storage_output, tool_calls = process_llm_output( | |
| full_thoughts, full_text, structured_requirement | |
| ) | |
| images = [] | |
| seen_urls = set() | |
| for out in all_outputs: | |
| if out.images: | |
| for img in out.images: | |
| # Use the image URL as a stable identifier across chunks | |
| if img.url not in seen_urls: | |
| images.append(img) | |
| seen_urls.add(img.url) | |
| image_markdown = "" | |
| seen_hashes = set() | |
| for image in images: | |
| try: | |
| image_store = get_image_store_dir() | |
| _, _, _, filename, file_hash = await _image_to_base64(image, image_store) | |
| if file_hash in seen_hashes: | |
| # Duplicate content, delete the file and skip | |
| (image_store / filename).unlink(missing_ok=True) | |
| continue | |
| seen_hashes.add(file_hash) | |
| img_url = ( | |
| f"})" | |
| ) | |
| image_markdown += f"\n\n{img_url}" | |
| except Exception as exc: | |
| logger.warning(f"Failed to process image in OpenAI stream: {exc}") | |
| if image_markdown: | |
| assistant_text += image_markdown | |
| storage_output += image_markdown | |
| yield f"data: {orjson.dumps({'id': completion_id, 'object': 'chat.completion.chunk', 'created': created_time, 'model': model_name, 'choices': [{'index': 0, 'delta': {'content': image_markdown}, 'finish_reason': None}]}).decode('utf-8')}\n\n" | |
| p_tok, c_tok, t_tok, r_tok = calculate_usage( | |
| messages, assistant_text, tool_calls, full_thoughts | |
| ) | |
| usage = { | |
| "prompt_tokens": p_tok, | |
| "completion_tokens": c_tok, | |
| "total_tokens": t_tok, | |
| "completion_tokens_details": {"reasoning_tokens": r_tok}, | |
| } | |
| final_delta = {} | |
| if tool_calls: | |
| # Add index field to each tool_call for OpenAI compatibility | |
| final_delta["tool_calls"] = [ | |
| {**call.model_dump(mode="json"), "index": i} for i, call in enumerate(tool_calls) | |
| ] | |
| data = { | |
| "id": completion_id, | |
| "object": "chat.completion.chunk", | |
| "created": created_time, | |
| "model": model_name, | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "delta": final_delta, | |
| "finish_reason": "tool_calls" if tool_calls else "stop", | |
| } | |
| ], | |
| "usage": usage, | |
| } | |
| _persist_conversation( | |
| db, | |
| model.model_name, | |
| client_wrapper.id, | |
| session.metadata, | |
| messages, # This should be the prepared messages | |
| storage_output, | |
| tool_calls, | |
| full_thoughts, | |
| ) | |
| yield f"data: {orjson.dumps(data).decode('utf-8')}\n\n" | |
| yield "data: [DONE]\n\n" | |
| return StreamingResponse(generate_stream(), media_type="text/event-stream") | |
| def _create_responses_real_streaming_response( | |
| generator: AsyncGenerator[ModelOutput, None], | |
| response_id: str, | |
| created_time: int, | |
| model_name: str, | |
| messages: list[Message], | |
| db: LMDBConversationStore, | |
| model: Model, | |
| client_wrapper: GeminiClientWrapper, | |
| session: ChatSession, | |
| request: ResponseCreateRequest, | |
| image_store: Path, | |
| base_url: str, | |
| structured_requirement: StructuredOutputRequirement | None = None, | |
| ) -> StreamingResponse: | |
| """ | |
| Create a real-time streaming response for the Responses API. | |
| Ensures final accumulated text and thoughts are synchronized. | |
| """ | |
| base_event = { | |
| "id": response_id, | |
| "object": "response", | |
| "created_at": created_time, | |
| "model": model_name, | |
| } | |
| async def generate_stream(): | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.created', 'response': {'id': response_id, 'object': 'response', 'created_at': created_time, 'model': model_name, 'status': 'in_progress', 'metadata': request.metadata, 'input': None, 'tools': request.tools, 'tool_choice': request.tool_choice}}).decode('utf-8')}\n\n" | |
| message_id = f"msg_{uuid.uuid4().hex}" | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.added', 'output_index': 0, 'item': {'id': message_id, 'type': 'message', 'role': 'assistant', 'content': []}}).decode('utf-8')}\n\n" | |
| full_thoughts, full_text = "", "" | |
| last_chunk_was_thought = False | |
| all_outputs: list[ModelOutput] = [] | |
| suppressor = StreamingOutputFilter() | |
| try: | |
| async for chunk in generator: | |
| all_outputs.append(chunk) | |
| # Robust thought delta calculation | |
| new_thoughts = chunk.thoughts or "" | |
| if len(new_thoughts) > len(full_thoughts): | |
| t_delta = new_thoughts[len(full_thoughts) :] | |
| if not last_chunk_was_thought and not full_thoughts: | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': '<think>'}).decode('utf-8')}\n\n" | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': t_delta}).decode('utf-8')}\n\n" | |
| full_thoughts = new_thoughts | |
| last_chunk_was_thought = True | |
| # Robust text delta calculation | |
| new_text = chunk.text or "" | |
| # Strip transient triple-backtick fence which is often added by the web-scraping backend | |
| # during streaming snapshots. We will restore the real ending after the loop. | |
| display_text = StreamingOutputFilter.TRAILING_FENCE_RE.sub("", new_text) | |
| if len(display_text) > len(full_text): | |
| text_delta = display_text[len(full_text) :] | |
| if last_chunk_was_thought: | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': '</think>\n'}).decode('utf-8')}\n\n" | |
| last_chunk_was_thought = False | |
| filter_events = suppressor.process(text_delta) | |
| for event in filter_events: | |
| if event["type"] == "text": | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': event['content']}).decode('utf-8')}\n\n" | |
| # Note: This API currently handles tool calls as discrete items at the end. | |
| # We ignore internal tool call events in the text stream for now. | |
| full_text = display_text | |
| except Exception as e: | |
| logger.exception(f"Error during Responses API streaming: {e}") | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'error', 'error': {'message': 'Streaming error.'}}).decode('utf-8')}\n\n" | |
| return | |
| if all_outputs: | |
| final_chunk = all_outputs[-1] | |
| final_text = final_chunk.text or "" | |
| if len(final_text) > len(full_text): | |
| # Yield any real trailing content (like the actual closing fence) | |
| final_delta = final_text[len(full_text) :] | |
| for event in suppressor.process(final_delta): | |
| if event["type"] == "text": | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': event['content']}).decode('utf-8')}\n\n" | |
| full_text = final_text | |
| if final_chunk.thoughts: | |
| full_thoughts = final_chunk.thoughts | |
| if last_chunk_was_thought: | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': '</think>\n'}).decode('utf-8')}\n\n" | |
| for event in suppressor.flush(): | |
| if event["type"] == "text": | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.delta', 'output_index': 0, 'delta': event['content']}).decode('utf-8')}\n\n" | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_text.done', 'output_index': 0}).decode('utf-8')}\n\n" | |
| _thoughts, assistant_text, storage_output, detected_tool_calls = process_llm_output( | |
| full_thoughts, full_text, structured_requirement | |
| ) | |
| images = [] | |
| seen_urls = set() | |
| for out in all_outputs: | |
| if out.images: | |
| for img in out.images: | |
| if img.url not in seen_urls: | |
| images.append(img) | |
| seen_urls.add(img.url) | |
| # Check if image generation was forced via tool_choice (same logic as non-streaming) | |
| image_generation_forced = ( | |
| request.tool_choice is not None | |
| and isinstance(request.tool_choice, ResponseToolChoice) | |
| and request.tool_choice.type == "image_generation" | |
| ) | |
| if image_generation_forced and not images and not assistant_text: | |
| logger.warning( | |
| "Image generation was forced via tool_choice but no images or text were returned in stream." | |
| ) | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'error', 'error': {'message': 'No images returned from forced image generation request.'}}).decode('utf-8')}\n\n" | |
| return | |
| response_contents, image_call_items = [], [] | |
| seen_hashes = set() | |
| for image in images: | |
| try: | |
| image_base64, width, height, filename, file_hash = await _image_to_base64( | |
| image, image_store | |
| ) | |
| if file_hash in seen_hashes: | |
| (image_store / filename).unlink(missing_ok=True) | |
| continue | |
| seen_hashes.add(file_hash) | |
| img_format = "png" if isinstance(image, GeneratedImage) else "jpeg" | |
| image_url = ( | |
| f"})" | |
| ) | |
| image_call_items.append( | |
| ResponseImageGenerationCall( | |
| id=filename.rsplit(".", 1)[0], | |
| result=image_base64, | |
| output_format=img_format, | |
| size=f"{width}x{height}" if width and height else None, | |
| ) | |
| ) | |
| response_contents.append(ResponseOutputContent(type="output_text", text=image_url)) | |
| except Exception as exc: | |
| logger.warning(f"Failed to process image in stream: {exc}") | |
| if assistant_text: | |
| response_contents.append(ResponseOutputContent(type="output_text", text=assistant_text)) | |
| if not response_contents: | |
| response_contents.append(ResponseOutputContent(type="output_text", text="")) | |
| # Aggregate images for storage | |
| image_markdown = "" | |
| for img_call in image_call_items: | |
| fname = f"{img_call.id}.{img_call.output_format}" | |
| img_url = f"})" | |
| image_markdown += f"\n\n{img_url}" | |
| if image_markdown: | |
| storage_output += image_markdown | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.done', 'output_index': 0, 'item': {'id': message_id, 'type': 'message', 'role': 'assistant', 'content': [c.model_dump(mode='json') for c in response_contents]}}).decode('utf-8')}\n\n" | |
| current_idx = 1 | |
| for call in detected_tool_calls: | |
| tc_item = ResponseToolCall(id=call.id, status="completed", function=call.function) | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.added', 'output_index': current_idx, 'item': tc_item.model_dump(mode='json')}).decode('utf-8')}\n\n" | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.done', 'output_index': current_idx, 'item': tc_item.model_dump(mode='json')}).decode('utf-8')}\n\n" | |
| current_idx += 1 | |
| for img_call in image_call_items: | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.added', 'output_index': current_idx, 'item': img_call.model_dump(mode='json')}).decode('utf-8')}\n\n" | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.output_item.done', 'output_index': current_idx, 'item': img_call.model_dump(mode='json')}).decode('utf-8')}\n\n" | |
| current_idx += 1 | |
| p_tok, c_tok, t_tok, r_tok = calculate_usage( | |
| messages, assistant_text, detected_tool_calls, full_thoughts | |
| ) | |
| usage = ResponseUsage( | |
| input_tokens=p_tok, | |
| output_tokens=c_tok, | |
| total_tokens=t_tok, | |
| output_tokens_details={"reasoning_tokens": r_tok}, | |
| ) | |
| payload = create_responses_standard_payload( | |
| response_id, | |
| created_time, | |
| model_name, | |
| detected_tool_calls, | |
| image_call_items, | |
| response_contents, | |
| usage, | |
| request, | |
| None, | |
| full_thoughts, | |
| ) | |
| _persist_conversation( | |
| db, | |
| model.model_name, | |
| client_wrapper.id, | |
| session.metadata, | |
| messages, | |
| storage_output, | |
| detected_tool_calls, | |
| full_thoughts, | |
| ) | |
| yield f"data: {orjson.dumps({**base_event, 'type': 'response.completed', 'response': payload.model_dump(mode='json')}).decode('utf-8')}\n\n" | |
| yield "data: [DONE]\n\n" | |
| return StreamingResponse(generate_stream(), media_type="text/event-stream") | |
| # --- Core Session Logic --- | |
| async def _execute_core_chat_session( | |
| request_model: str, | |
| messages: list[Message], | |
| tmp_dir: Path, | |
| tools: list[Tool] | None = None, | |
| tool_choice: str | ToolChoiceFunction | None = None, | |
| stream: bool = False, | |
| extra_instructions: list[str] | None = None, | |
| response_format: dict[str, Any] | None = None, | |
| ): | |
| """ | |
| Delegate shared Gemini session execution to the session service and | |
| preserve the tuple contract expected by existing handlers. | |
| """ | |
| result = await shared_execute_core_chat_session( | |
| request_model=request_model, | |
| messages=messages, | |
| tmp_dir=tmp_dir, | |
| tools=tools, | |
| tool_choice=tool_choice, | |
| stream=stream, | |
| extra_instructions=extra_instructions, | |
| response_format=response_format, | |
| metadata_ttl_minutes=METADATA_TTL_MINUTES, | |
| ) | |
| return ( | |
| result.response_or_stream, | |
| result.completion_id, | |
| result.created_time, | |
| result.session, | |
| result.client, | |
| result.db, | |
| result.model, | |
| result.prepared_messages, | |
| result.structured_requirement, | |
| ) | |
| async def _create_anthropic_streaming_response( | |
| generator: AsyncGenerator[ModelOutput, None], | |
| completion_id: str, | |
| created_time: int, | |
| model_name: str, | |
| messages: list[Message], | |
| db: LMDBConversationStore | None = None, | |
| model: Model | None = None, | |
| client_wrapper: GeminiClientWrapper | None = None, | |
| session: ChatSession | None = None, | |
| ): | |
| """ | |
| Stream Gemini output as Anthropic SSE events. | |
| Now supports structured tool call streaming. | |
| """ | |
| async def generate_stream(): | |
| # 1. Message Start | |
| msg_start = { | |
| "type": "message_start", | |
| "message": { | |
| "id": completion_id, | |
| "type": "message", | |
| "role": "assistant", | |
| "content": [], | |
| "model": model_name, | |
| "stop_reason": None, | |
| "stop_sequence": None, | |
| "usage": {"input_tokens": 0, "output_tokens": 0}, | |
| }, | |
| } | |
| yield f"event: message_start\ndata: {orjson.dumps(msg_start).decode('utf-8')}\n\n" | |
| full_text = "" | |
| suppressor = StreamingOutputFilter() | |
| content_block_index = 0 | |
| in_text_block = False | |
| last_chunk: ModelOutput | None = None | |
| # 3. Stream Deltas | |
| try: | |
| async for chunk in generator: | |
| last_chunk = chunk | |
| if chunk.text: | |
| display_text = StreamingOutputFilter.TRAILING_FENCE_RE.sub("", chunk.text) | |
| if len(display_text) > len(full_text): | |
| delta_text = display_text[len(full_text) :] | |
| full_text = display_text | |
| filter_events = suppressor.process(delta_text) | |
| for event in filter_events: | |
| if event["type"] == "text": | |
| if not in_text_block: | |
| # Start a new text block if not in one | |
| yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'text', 'text': ''}}).decode('utf-8')}\n\n" | |
| in_text_block = True | |
| delta = { | |
| "type": "content_block_delta", | |
| "index": content_block_index, | |
| "delta": {"type": "text_delta", "text": event["content"]}, | |
| } | |
| yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n" | |
| elif event["type"] == "tool_start": | |
| if in_text_block: | |
| yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" | |
| content_block_index += 1 | |
| in_text_block = False | |
| # Anthropic tool_use start | |
| yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'tool_use', 'id': event['id'], 'name': event['name'], 'input': {}}}).decode('utf-8')}\n\n" | |
| elif event["type"] == "tool_delta": | |
| # Anthropic tool_use delta (input_json_delta) | |
| delta = { | |
| "type": "content_block_delta", | |
| "index": content_block_index, | |
| "delta": { | |
| "type": "input_json_delta", | |
| "partial_json": event["content"], | |
| }, | |
| } | |
| yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n" | |
| elif event["type"] == "tool_end": | |
| yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" | |
| content_block_index += 1 | |
| # We stay in "not in_text_block" state until next text event | |
| # Handle final delta after loop ends | |
| final_text = "" | |
| final_thoughts = "" | |
| if last_chunk: | |
| final_text = last_chunk.text or "" | |
| final_thoughts = last_chunk.thoughts or "" | |
| if len(final_text) > len(full_text): | |
| final_delta = final_text[len(full_text) :] | |
| for event in suppressor.process(final_delta): | |
| if event["type"] == "text": | |
| if not in_text_block: | |
| yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'text', 'text': ''}}).decode('utf-8')}\n\n" | |
| in_text_block = True | |
| delta = { | |
| "type": "content_block_delta", | |
| "index": content_block_index, | |
| "delta": {"type": "text_delta", "text": event["content"]}, | |
| } | |
| yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n" | |
| full_text = final_text | |
| # Final verification of tool calls (Built-in tools like image_retrieval often appear at the end) | |
| _thoughts_out, _assistant_visible, assistant_storage, tool_calls = process_llm_output( | |
| final_thoughts, full_text, None | |
| ) | |
| # If we detected tool_calls that weren't captured by suppressor yet, emit them now | |
| # (Note: suppressor currently handles manual formatting, but built-in tools are caught here) | |
| # To avoid duplicates if suppressor already found them, we could check event IDs. | |
| # But normally built-in tools won't trigger the regex-based suppressor. | |
| if tool_calls: | |
| for tcall in tool_calls: | |
| # Emit as a new content block if it's a tool_use | |
| if in_text_block: | |
| yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" | |
| content_block_index += 1 | |
| in_text_block = False | |
| yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'tool_use', 'id': tcall.id, 'name': tcall.function.name, 'input': {}}}).decode('utf-8')}\n\n" | |
| yield f"event: content_block_delta\ndata: {orjson.dumps({'type': 'content_block_delta', 'index': content_block_index, 'delta': {'type': 'input_json_delta', 'partial_json': tcall.function.arguments}}).decode('utf-8')}\n\n" | |
| yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" | |
| content_block_index += 1 | |
| # Handle trailing buffer | |
| for event in suppressor.flush(): | |
| if event["type"] == "text": | |
| if not in_text_block: | |
| yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'text', 'text': ''}}).decode('utf-8')}\n\n" | |
| in_text_block = True | |
| delta = { | |
| "type": "content_block_delta", | |
| "index": content_block_index, | |
| "delta": {"type": "text_delta", "text": event["content"]}, | |
| } | |
| yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n" | |
| elif event["type"] == "tool_start": | |
| if in_text_block: | |
| yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" | |
| content_block_index += 1 | |
| in_text_block = False | |
| yield f"event: content_block_start\ndata: {orjson.dumps({'type': 'content_block_start', 'index': content_block_index, 'content_block': {'type': 'tool_use', 'id': event['id'], 'name': event['name'], 'input': {}}}).decode('utf-8')}\n\n" | |
| elif event["type"] == "tool_delta": | |
| delta = { | |
| "type": "content_block_delta", | |
| "index": content_block_index, | |
| "delta": {"type": "input_json_delta", "partial_json": event["content"]}, | |
| } | |
| yield f"event: content_block_delta\ndata: {orjson.dumps(delta).decode('utf-8')}\n\n" | |
| elif event["type"] == "tool_end": | |
| yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" | |
| content_block_index += 1 | |
| except Exception as e: | |
| logger.exception(f"Error during Anthropic streaming: {e}") | |
| yield f"event: error\ndata: {orjson.dumps({'type': 'error', 'error': {'type': 'api_error', 'message': str(e)}}).decode('utf-8')}\n\n" | |
| return | |
| if in_text_block: | |
| yield f"event: content_block_stop\ndata: {orjson.dumps({'type': 'content_block_stop', 'index': content_block_index}).decode('utf-8')}\n\n" | |
| # 5. Message Delta (Stop Reason & Usage) | |
| c_tok = estimate_tokens(full_text) | |
| msg_delta = { | |
| "type": "message_delta", | |
| "delta": {"stop_reason": "end_turn", "stop_sequence": None}, | |
| "usage": {"output_tokens": c_tok}, | |
| } | |
| yield f"event: message_delta\ndata: {orjson.dumps(msg_delta).decode('utf-8')}\n\n" | |
| # 6. Message Stop | |
| yield f"event: message_stop\ndata: {orjson.dumps({'type': 'message_stop'}).decode('utf-8')}\n\n" | |
| # 7. Persist conversation if session context was provided | |
| if ( | |
| db is not None | |
| and model is not None | |
| and client_wrapper is not None | |
| and session is not None | |
| ): | |
| _persist_conversation( | |
| db, | |
| model.model_name, | |
| client_wrapper.id, | |
| session.metadata, | |
| messages, | |
| assistant_storage, | |
| tool_calls, | |
| final_thoughts if final_thoughts else None, | |
| ) | |
| return StreamingResponse(generate_stream(), media_type="text/event-stream") | |
| # --- Main Router Endpoints --- | |
| async def list_models(api_key: str = Depends(verify_api_key)): | |
| models = get_available_models() | |
| return ModelListResponse(data=models) | |
| async def create_chat_completion( | |
| request: ChatCompletionRequest, | |
| raw_request: Request, | |
| api_key: str = Depends(verify_api_key), | |
| tmp_dir: Path = Depends(get_temp_dir), | |
| image_store: Path = Depends(get_image_store_dir), | |
| ): | |
| return await handle_chat_completion( | |
| request=request, | |
| raw_request=raw_request, | |
| tmp_dir=tmp_dir, | |
| image_store=image_store, | |
| deps=OpenAIChatHandlerDependencies( | |
| execute_core_chat_session=_execute_core_chat_session, | |
| create_real_streaming_response=_create_real_streaming_response, | |
| create_chat_completion_standard_payload=create_chat_completion_standard_payload, | |
| process_llm_output=process_llm_output, | |
| calculate_usage=calculate_usage, | |
| persist_conversation=_persist_conversation, | |
| get_image_token=get_image_token, | |
| ), | |
| ) | |
| async def create_anthropic_message( | |
| request: AnthropicMessageRequest, | |
| raw_request: Request, | |
| api_key: str = Depends(verify_api_key), | |
| tmp_dir: Path = Depends(get_temp_dir), | |
| ): | |
| return await handle_anthropic_messages( | |
| request=request, | |
| raw_request=raw_request, | |
| tmp_dir=tmp_dir, | |
| deps=AnthropicHandlerDependencies( | |
| execute_core_chat_session=_execute_core_chat_session, | |
| create_anthropic_streaming_response=_create_anthropic_streaming_response, | |
| process_llm_output=process_llm_output, | |
| calculate_usage=calculate_usage, | |
| persist_conversation=_persist_conversation, | |
| ), | |
| ) | |
| async def create_response( | |
| request: ResponseCreateRequest, | |
| raw_request: Request, | |
| api_key: str = Depends(verify_api_key), | |
| tmp_dir: Path = Depends(get_temp_dir), | |
| image_store: Path = Depends(get_image_store_dir), | |
| ): | |
| return await handle_responses_api( | |
| request=request, | |
| raw_request=raw_request, | |
| tmp_dir=tmp_dir, | |
| image_store=image_store, | |
| deps=ResponsesHandlerDependencies( | |
| response_items_to_messages=response_items_to_messages, | |
| instructions_to_messages=instructions_to_messages, | |
| build_structured_requirement=build_structured_requirement, | |
| build_image_generation_instruction=build_image_generation_instruction, | |
| prepare_messages_for_model=prepare_messages_for_model, | |
| get_default_model_name=get_default_model_name, | |
| create_responses_real_streaming_response=_create_responses_real_streaming_response, | |
| create_responses_standard_payload=create_responses_standard_payload, | |
| process_llm_output=process_llm_output, | |
| calculate_usage=calculate_usage, | |
| persist_conversation=_persist_conversation, | |
| get_image_token=get_image_token, | |
| ), | |
| ) | |