from __future__ import annotations import uuid from dataclasses import dataclass from pathlib import Path from typing import Any, Callable import orjson from fastapi import HTTPException, Request from gemini_webapi import ModelOutput from gemini_webapi.types.image import GeneratedImage, Image from loguru import logger from ..models import ( ContentItem, Message, ResponseCreateRequest, ResponseImageGenerationCall, ResponseImageTool, ResponseOutputContent, ResponseToolChoice, ResponseUsage, Tool, ToolChoiceFunction, ) from ..services.chat_session_service import ( execute_core_chat_session as shared_execute_core_chat_session, ) from ..services.client import GeminiClientWrapper from ..utils.helper import detect_image_extension, extract_image_dimensions, safe_json_loads StreamFactory = Callable[..., Any] PersistConversationFn = Callable[..., str | None] ProcessOutputFn = Callable[..., tuple[str | None, str, str, list[Any]]] CalculateUsageFn = Callable[..., tuple[int, int, int, int]] PayloadFactory = Callable[..., Any] ResponseItemsToMessagesFn = Callable[..., tuple[list[Any], Any]] InstructionsToMessagesFn = Callable[..., list[Any]] BuildStructuredRequirementFn = Callable[..., Any] BuildImageInstructionFn = Callable[..., str | None] PrepareMessagesFn = Callable[..., list[Any]] GetDefaultModelNameFn = Callable[[], str] @dataclass(slots=True) class ResponsesHandlerDependencies: """Injected runtime hooks extracted from the chat router.""" response_items_to_messages: ResponseItemsToMessagesFn instructions_to_messages: InstructionsToMessagesFn build_structured_requirement: BuildStructuredRequirementFn build_image_generation_instruction: BuildImageInstructionFn prepare_messages_for_model: PrepareMessagesFn get_default_model_name: GetDefaultModelNameFn create_responses_real_streaming_response: StreamFactory create_responses_standard_payload: PayloadFactory process_llm_output: ProcessOutputFn calculate_usage: CalculateUsageFn persist_conversation: PersistConversationFn get_image_token: Callable[[str], str] def _extract_image_generation_prompt(tool_calls: list[Any] | None) -> str | None: """Extract a user-facing prompt from an image_generation tool call payload.""" if not tool_calls: return None for tool_call in tool_calls: function = getattr(tool_call, "function", None) if not function or function.name != "image_generation": continue raw_arguments = (function.arguments or "").strip() if not raw_arguments: return None try: parsed_arguments = safe_json_loads(raw_arguments) except Exception: try: parsed_arguments = orjson.loads(raw_arguments) except orjson.JSONDecodeError: parsed_arguments = raw_arguments if isinstance(parsed_arguments, dict): for key in ("prompt", "query", "input", "text"): value = parsed_arguments.get(key) if isinstance(value, str) and value.strip(): return value.strip() return orjson.dumps(parsed_arguments).decode("utf-8") if isinstance(parsed_arguments, str) and parsed_arguments.strip(): return parsed_arguments.strip() return raw_arguments return None async def _image_to_base64( image: Image, temp_dir: Path ) -> tuple[str, int | None, int | None, str, str]: """Persist an image provided by gemini_webapi and return base64 plus dimensions, filename, and hash.""" import base64 import hashlib if isinstance(image, GeneratedImage): try: saved_path = await image.save(path=str(temp_dir), full_size=True) except Exception as e: logger.warning( f"Failed to download full-size GeneratedImage, retrying with default size: {e}" ) saved_path = await image.save(path=str(temp_dir), full_size=False) else: saved_path = await image.save(path=str(temp_dir)) if not saved_path: raise ValueError("Failed to save generated image") original_path = Path(saved_path) data = original_path.read_bytes() suffix = original_path.suffix if not suffix: detected_ext = detect_image_extension(data) suffix = detected_ext or (".png" if isinstance(image, GeneratedImage) else ".jpg") random_name = f"img_{uuid.uuid4().hex}{suffix}" new_path = temp_dir / random_name original_path.rename(new_path) width, height = extract_image_dimensions(data) filename = random_name file_hash = hashlib.sha256(data).hexdigest() return base64.b64encode(data).decode("ascii"), width, height, filename, file_hash def _should_use_direct_image_generation_path( image_tools: list[ResponseImageTool], standard_tools: list[Tool], request: ResponseCreateRequest, ) -> bool: """ Use a provider-aware direct image generation path when the request is a pure image-generation call. """ if request.stream: return False if standard_tools: return False if not image_tools: return False if request.tool_choice is None: return True return ( isinstance(request.tool_choice, ResponseToolChoice) and request.tool_choice.type == "image_generation" ) def _build_direct_image_generation_messages( conversation_messages: list[Message], extra_instructions: list[str] | None, ) -> list[Message]: """ Build a provider-native message list for direct image generation. """ prepared_messages = [msg.model_copy(deep=True) for msg in conversation_messages] instructions = [instr for instr in (extra_instructions or []) if instr] instructions.append( "For this request, use Gemini's native image generation capability directly. " "Do not emit tool calls, function-call wrappers, JSON, or any intermediate tool " "planning. Return at least one generated image. Avoid text-only responses unless " "a very short caption is strictly necessary." ) combined_instructions = "\n\n".join(instructions) if ( prepared_messages and prepared_messages[0].role == "system" and isinstance(prepared_messages[0].content, str) ): existing = prepared_messages[0].content or "" if combined_instructions not in existing: separator = "\n\n" if existing else "" prepared_messages[0].content = f"{existing}{separator}{combined_instructions}" else: prepared_messages.insert(0, Message(role="system", content=combined_instructions)) direct_generation_hint = ( "Generate the requested image directly now. " "Return the image itself and do not output tool calls or JSON." ) for msg in reversed(prepared_messages): if msg.role != "user" or msg.content is None: continue if isinstance(msg.content, str): if direct_generation_hint not in msg.content: msg.content = f"{msg.content}\n\n{direct_generation_hint}" break if isinstance(msg.content, list): for part in reversed(msg.content): if isinstance(part, dict): if part.get("type") != "text": continue current_text = part.get("text") or "" if direct_generation_hint not in current_text: part["text"] = f"{current_text}\n\n{direct_generation_hint}" break if getattr(part, "type", None) != "text": continue current_text = getattr(part, "text", "") or "" if direct_generation_hint not in current_text: part.text = f"{current_text}\n\n{direct_generation_hint}" # type: ignore[assignment] break else: msg.content.append(ContentItem(type="text", text=direct_generation_hint)) break return prepared_messages async def handle_responses_api( request: ResponseCreateRequest, raw_request: Request, tmp_dir: Path, image_store: Path, deps: ResponsesHandlerDependencies, ) -> Any: """ Handle the OpenAI-compatible Responses API endpoint. """ base_url = str(raw_request.base_url) base_messages, normalized_input = deps.response_items_to_messages(request.input) structured_requirement = deps.build_structured_requirement(request.response_format) extra_instructions = [structured_requirement.instruction] if structured_requirement else [] standard_tools: list[Tool] = [] image_tools: list[ResponseImageTool] = [] if request.tools: for tool in request.tools: if isinstance(tool, Tool): standard_tools.append(tool) elif isinstance(tool, ResponseImageTool): image_tools.append(tool) elif isinstance(tool, dict): if tool.get("type") == "function": standard_tools.append(Tool.model_validate(tool)) elif tool.get("type") == "image_generation": image_tools.append(ResponseImageTool.model_validate(tool)) image_instruction = deps.build_image_generation_instruction( image_tools, request.tool_choice if isinstance(request.tool_choice, ResponseToolChoice) else None, ) if image_instruction: extra_instructions.append(image_instruction) preface_messages = deps.instructions_to_messages(request.instructions) conversation_messages = ( [*preface_messages, *base_messages] if preface_messages else base_messages ) model_tool_choice = ( request.tool_choice if isinstance(request.tool_choice, (str, ToolChoiceFunction)) else None ) request.model = deps.get_default_model_name() direct_image_generation = _should_use_direct_image_generation_path( image_tools, standard_tools, request, ) if direct_image_generation: execution_messages = _build_direct_image_generation_messages( conversation_messages, extra_instructions or None, ) execution_tools = None execution_tool_choice = None execution_extra_instructions = None else: execution_messages = conversation_messages execution_tools = standard_tools or None execution_tool_choice = model_tool_choice execution_extra_instructions = extra_instructions or None result = await shared_execute_core_chat_session( request_model=request.model, messages=execution_messages, tmp_dir=tmp_dir, tools=execution_tools, tool_choice=execution_tool_choice, stream=bool(request.stream), extra_instructions=execution_extra_instructions, response_format=request.response_format, ) response_or_stream = result.response_or_stream response_id = f"resp_{uuid.uuid4().hex}" created_time = result.created_time session = result.session client = result.client db = result.db model = result.model messages = result.prepared_messages structured_requirement = result.structured_requirement if request.stream: return deps.create_responses_real_streaming_response( response_or_stream, response_id, created_time, model.model_name, messages, db, model, client, session, request, image_store, base_url, structured_requirement, ) if not isinstance(response_or_stream, ModelOutput): raise HTTPException(status_code=502, detail="Malformed non-stream response.") try: thoughts = getattr(response_or_stream, "thoughts", None) raw_clean = GeminiClientWrapper.extract_output( response_or_stream, include_thoughts=False, ) except Exception as exc: logger.exception("Gemini parsing failed") raise HTTPException(status_code=502, detail="Malformed response.") from exc thoughts, assistant_text, storage_output, tool_calls = deps.process_llm_output( thoughts, raw_clean, structured_requirement, ) images = response_or_stream.images or [] if direct_image_generation and tool_calls and not images: logger.warning( "Direct image generation path still produced tool calls without image payload. " "This likely indicates an upstream Gemini Web image generation failure." ) image_generation_prompt = _extract_image_generation_prompt(tool_calls) if image_generation_prompt and not assistant_text: assistant_text = image_generation_prompt logger.debug( "Response: text_len=%s, images=%s, tool_calls=%s", len(assistant_text), len(images), len(tool_calls), ) if not assistant_text and not images and not tool_calls: logger.warning("Gemini returned an empty response (no text, images, or tool calls)") if hasattr(response_or_stream, "candidates") and response_or_stream.candidates: logger.debug(f"Candidates: {response_or_stream.candidates}") image_generation_forced = ( request.tool_choice is not None and isinstance(request.tool_choice, ResponseToolChoice) and request.tool_choice.type == "image_generation" ) if image_generation_forced and not images: logger.warning( "Image generation was requested but Gemini Web did not return an image payload." ) raise HTTPException( status_code=502, detail="Image generation request completed without returning an image.", ) contents: list[ResponseOutputContent] = [] image_calls: list[ResponseImageGenerationCall] = [] seen_hashes: set[str] = set() for image in images: try: image_base64, width, height, filename, file_hash = await _image_to_base64( image, image_store, ) if file_hash in seen_hashes: (image_store / filename).unlink(missing_ok=True) continue seen_hashes.add(file_hash) image_url = ( f"![{filename}]({base_url}images/{filename}?token={deps.get_image_token(filename)})" ) contents.append(ResponseOutputContent(type="output_text", text=image_url)) image_calls.append( ResponseImageGenerationCall( id=filename.rsplit(".", 1)[0], result=image_base64, output_format="png" if isinstance(image, GeneratedImage) else "jpeg", size=f"{width}x{height}" if width and height else None, ) ) except Exception as exc: logger.warning(f"Image error: {exc}") if assistant_text: contents.append(ResponseOutputContent(type="output_text", text=assistant_text)) if not contents: contents.append(ResponseOutputContent(type="output_text", text="")) image_markdown = "" for image_call in image_calls: filename = f"{image_call.id}.{image_call.output_format}" image_url = ( f"![{filename}]({base_url}images/{filename}?token={deps.get_image_token(filename)})" ) image_markdown += f"\n\n{image_url}" if image_markdown: storage_output += image_markdown input_tokens, output_tokens, total_tokens, reasoning_tokens = deps.calculate_usage( messages, assistant_text, tool_calls, thoughts, ) usage = ResponseUsage( input_tokens=input_tokens, output_tokens=output_tokens, total_tokens=total_tokens, output_tokens_details={"reasoning_tokens": reasoning_tokens}, ) payload = deps.create_responses_standard_payload( response_id, created_time, model.model_name, None if direct_image_generation else tool_calls, image_calls, contents, usage, request, normalized_input, thoughts, ) deps.persist_conversation( db, model.model_name, client.id, session.metadata, messages, storage_output, None if direct_image_generation else tool_calls, thoughts, ) return payload