File size: 16,671 Bytes
5eeb2b8
 
 
 
 
50b60b4
5eeb2b8
74ba44d
5eeb2b8
 
828be99
5eeb2b8
 
 
2d5de6c
9a4d479
5eeb2b8
 
 
 
 
 
 
 
 
2d5de6c
 
 
5eeb2b8
828be99
5eeb2b8
 
 
828be99
 
5eeb2b8
 
 
50b60b4
 
 
5eeb2b8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
74ba44d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
828be99
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9a4d479
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
922bb0f
9a4d479
 
 
 
922bb0f
 
 
 
 
 
9a4d479
 
 
 
 
 
 
 
 
 
 
 
 
 
922bb0f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2d5de6c
 
 
 
 
 
 
 
922bb0f
 
8e8fd9d
922bb0f
8e8fd9d
922bb0f
 
 
 
 
9a4d479
 
 
5eeb2b8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9a4d479
 
 
 
5eeb2b8
 
9a4d479
2d5de6c
9a4d479
 
 
2d5de6c
 
 
9a4d479
2d5de6c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5eeb2b8
2d5de6c
5eeb2b8
2d5de6c
 
 
 
 
 
828be99
5eeb2b8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
828be99
5eeb2b8
 
 
 
 
 
 
 
828be99
 
5eeb2b8
 
 
 
 
9a4d479
 
 
 
74ba44d
9a4d479
 
 
74ba44d
5eeb2b8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9a4d479
5eeb2b8
9a4d479
5eeb2b8
 
 
9a4d479
5eeb2b8
 
 
 
 
 
 
 
828be99
5eeb2b8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
828be99
5eeb2b8
 
 
828be99
5eeb2b8
 
 
 
 
828be99
5eeb2b8
 
 
 
 
 
9a4d479
5eeb2b8
 
 
 
 
828be99
5eeb2b8
 
 
 
 
 
 
 
 
9a4d479
828be99
5eeb2b8
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
from __future__ import annotations

import uuid
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable

import orjson
from fastapi import HTTPException, Request
from gemini_webapi import ModelOutput
from gemini_webapi.types.image import GeneratedImage, Image
from loguru import logger

from ..models import (
    ContentItem,
    Message,
    ResponseCreateRequest,
    ResponseImageGenerationCall,
    ResponseImageTool,
    ResponseOutputContent,
    ResponseToolChoice,
    ResponseUsage,
    Tool,
    ToolChoiceFunction,
)
from ..services.chat_session_service import (
    execute_core_chat_session as shared_execute_core_chat_session,
)
from ..services.client import GeminiClientWrapper
from ..utils.helper import detect_image_extension, extract_image_dimensions, safe_json_loads

StreamFactory = Callable[..., Any]
PersistConversationFn = Callable[..., str | None]
ProcessOutputFn = Callable[..., tuple[str | None, str, str, list[Any]]]
CalculateUsageFn = Callable[..., tuple[int, int, int, int]]
PayloadFactory = Callable[..., Any]
ResponseItemsToMessagesFn = Callable[..., tuple[list[Any], Any]]
InstructionsToMessagesFn = Callable[..., list[Any]]
BuildStructuredRequirementFn = Callable[..., Any]
BuildImageInstructionFn = Callable[..., str | None]
PrepareMessagesFn = Callable[..., list[Any]]
GetDefaultModelNameFn = Callable[[], str]


@dataclass(slots=True)
class ResponsesHandlerDependencies:
    """Injected runtime hooks extracted from the chat router."""

    response_items_to_messages: ResponseItemsToMessagesFn
    instructions_to_messages: InstructionsToMessagesFn
    build_structured_requirement: BuildStructuredRequirementFn
    build_image_generation_instruction: BuildImageInstructionFn
    prepare_messages_for_model: PrepareMessagesFn
    get_default_model_name: GetDefaultModelNameFn
    create_responses_real_streaming_response: StreamFactory
    create_responses_standard_payload: PayloadFactory
    process_llm_output: ProcessOutputFn
    calculate_usage: CalculateUsageFn
    persist_conversation: PersistConversationFn
    get_image_token: Callable[[str], str]


def _extract_image_generation_prompt(tool_calls: list[Any] | None) -> str | None:
    """Extract a user-facing prompt from an image_generation tool call payload."""
    if not tool_calls:
        return None

    for tool_call in tool_calls:
        function = getattr(tool_call, "function", None)
        if not function or function.name != "image_generation":
            continue

        raw_arguments = (function.arguments or "").strip()
        if not raw_arguments:
            return None

        try:
            parsed_arguments = safe_json_loads(raw_arguments)
        except Exception:
            try:
                parsed_arguments = orjson.loads(raw_arguments)
            except orjson.JSONDecodeError:
                parsed_arguments = raw_arguments

        if isinstance(parsed_arguments, dict):
            for key in ("prompt", "query", "input", "text"):
                value = parsed_arguments.get(key)
                if isinstance(value, str) and value.strip():
                    return value.strip()
            return orjson.dumps(parsed_arguments).decode("utf-8")

        if isinstance(parsed_arguments, str) and parsed_arguments.strip():
            return parsed_arguments.strip()

        return raw_arguments

    return None


async def _image_to_base64(
    image: Image, temp_dir: Path
) -> tuple[str, int | None, int | None, str, str]:
    """Persist an image provided by gemini_webapi and return base64 plus dimensions, filename, and hash."""
    import base64
    import hashlib

    if isinstance(image, GeneratedImage):
        try:
            saved_path = await image.save(path=str(temp_dir), full_size=True)
        except Exception as e:
            logger.warning(
                f"Failed to download full-size GeneratedImage, retrying with default size: {e}"
            )
            saved_path = await image.save(path=str(temp_dir), full_size=False)
    else:
        saved_path = await image.save(path=str(temp_dir))

    if not saved_path:
        raise ValueError("Failed to save generated image")

    original_path = Path(saved_path)
    data = original_path.read_bytes()
    suffix = original_path.suffix

    if not suffix:
        detected_ext = detect_image_extension(data)
        suffix = detected_ext or (".png" if isinstance(image, GeneratedImage) else ".jpg")

    random_name = f"img_{uuid.uuid4().hex}{suffix}"
    new_path = temp_dir / random_name
    original_path.rename(new_path)

    width, height = extract_image_dimensions(data)
    filename = random_name
    file_hash = hashlib.sha256(data).hexdigest()
    return base64.b64encode(data).decode("ascii"), width, height, filename, file_hash


def _should_use_direct_image_generation_path(
    image_tools: list[ResponseImageTool],
    standard_tools: list[Tool],
    request: ResponseCreateRequest,
) -> bool:
    """
    Use a provider-aware direct image generation path when the request is a pure
    image-generation call.
    """
    if request.stream:
        return False

    if standard_tools:
        return False

    if not image_tools:
        return False

    if request.tool_choice is None:
        return True

    return (
        isinstance(request.tool_choice, ResponseToolChoice)
        and request.tool_choice.type == "image_generation"
    )


def _build_direct_image_generation_messages(
    conversation_messages: list[Message],
    extra_instructions: list[str] | None,
) -> list[Message]:
    """
    Build a provider-native message list for direct image generation.
    """
    prepared_messages = [msg.model_copy(deep=True) for msg in conversation_messages]

    instructions = [instr for instr in (extra_instructions or []) if instr]
    instructions.append(
        "For this request, use Gemini's native image generation capability directly. "
        "Do not emit tool calls, function-call wrappers, JSON, or any intermediate tool "
        "planning. Return at least one generated image. Avoid text-only responses unless "
        "a very short caption is strictly necessary."
    )

    combined_instructions = "\n\n".join(instructions)
    if (
        prepared_messages
        and prepared_messages[0].role == "system"
        and isinstance(prepared_messages[0].content, str)
    ):
        existing = prepared_messages[0].content or ""
        if combined_instructions not in existing:
            separator = "\n\n" if existing else ""
            prepared_messages[0].content = f"{existing}{separator}{combined_instructions}"
    else:
        prepared_messages.insert(0, Message(role="system", content=combined_instructions))

    direct_generation_hint = (
        "Generate the requested image directly now. "
        "Return the image itself and do not output tool calls or JSON."
    )

    for msg in reversed(prepared_messages):
        if msg.role != "user" or msg.content is None:
            continue

        if isinstance(msg.content, str):
            if direct_generation_hint not in msg.content:
                msg.content = f"{msg.content}\n\n{direct_generation_hint}"
            break

        if isinstance(msg.content, list):
            for part in reversed(msg.content):
                if isinstance(part, dict):
                    if part.get("type") != "text":
                        continue
                    current_text = part.get("text") or ""
                    if direct_generation_hint not in current_text:
                        part["text"] = f"{current_text}\n\n{direct_generation_hint}"
                    break

                if getattr(part, "type", None) != "text":
                    continue
                current_text = getattr(part, "text", "") or ""
                if direct_generation_hint not in current_text:
                    part.text = f"{current_text}\n\n{direct_generation_hint}"  # type: ignore[assignment]
                break
            else:
                msg.content.append(ContentItem(type="text", text=direct_generation_hint))
            break

    return prepared_messages


async def handle_responses_api(
    request: ResponseCreateRequest,
    raw_request: Request,
    tmp_dir: Path,
    image_store: Path,
    deps: ResponsesHandlerDependencies,
) -> Any:
    """
    Handle the OpenAI-compatible Responses API endpoint.
    """
    base_url = str(raw_request.base_url)

    base_messages, normalized_input = deps.response_items_to_messages(request.input)
    structured_requirement = deps.build_structured_requirement(request.response_format)
    extra_instructions = [structured_requirement.instruction] if structured_requirement else []

    standard_tools: list[Tool] = []
    image_tools: list[ResponseImageTool] = []
    if request.tools:
        for tool in request.tools:
            if isinstance(tool, Tool):
                standard_tools.append(tool)
            elif isinstance(tool, ResponseImageTool):
                image_tools.append(tool)
            elif isinstance(tool, dict):
                if tool.get("type") == "function":
                    standard_tools.append(Tool.model_validate(tool))
                elif tool.get("type") == "image_generation":
                    image_tools.append(ResponseImageTool.model_validate(tool))

    image_instruction = deps.build_image_generation_instruction(
        image_tools,
        request.tool_choice if isinstance(request.tool_choice, ResponseToolChoice) else None,
    )
    if image_instruction:
        extra_instructions.append(image_instruction)

    preface_messages = deps.instructions_to_messages(request.instructions)
    conversation_messages = (
        [*preface_messages, *base_messages] if preface_messages else base_messages
    )
    model_tool_choice = (
        request.tool_choice if isinstance(request.tool_choice, (str, ToolChoiceFunction)) else None
    )

    request.model = deps.get_default_model_name()

    direct_image_generation = _should_use_direct_image_generation_path(
        image_tools,
        standard_tools,
        request,
    )

    if direct_image_generation:
        execution_messages = _build_direct_image_generation_messages(
            conversation_messages,
            extra_instructions or None,
        )
        execution_tools = None
        execution_tool_choice = None
        execution_extra_instructions = None
    else:
        execution_messages = conversation_messages
        execution_tools = standard_tools or None
        execution_tool_choice = model_tool_choice
        execution_extra_instructions = extra_instructions or None

    result = await shared_execute_core_chat_session(
        request_model=request.model,
        messages=execution_messages,
        tmp_dir=tmp_dir,
        tools=execution_tools,
        tool_choice=execution_tool_choice,
        stream=bool(request.stream),
        extra_instructions=execution_extra_instructions,
        response_format=request.response_format,
    )

    response_or_stream = result.response_or_stream
    response_id = f"resp_{uuid.uuid4().hex}"
    created_time = result.created_time
    session = result.session
    client = result.client
    db = result.db
    model = result.model
    messages = result.prepared_messages
    structured_requirement = result.structured_requirement

    if request.stream:
        return deps.create_responses_real_streaming_response(
            response_or_stream,
            response_id,
            created_time,
            model.model_name,
            messages,
            db,
            model,
            client,
            session,
            request,
            image_store,
            base_url,
            structured_requirement,
        )

    if not isinstance(response_or_stream, ModelOutput):
        raise HTTPException(status_code=502, detail="Malformed non-stream response.")

    try:
        thoughts = getattr(response_or_stream, "thoughts", None)
        raw_clean = GeminiClientWrapper.extract_output(
            response_or_stream,
            include_thoughts=False,
        )
    except Exception as exc:
        logger.exception("Gemini parsing failed")
        raise HTTPException(status_code=502, detail="Malformed response.") from exc

    thoughts, assistant_text, storage_output, tool_calls = deps.process_llm_output(
        thoughts,
        raw_clean,
        structured_requirement,
    )
    images = response_or_stream.images or []

    if direct_image_generation and tool_calls and not images:
        logger.warning(
            "Direct image generation path still produced tool calls without image payload. "
            "This likely indicates an upstream Gemini Web image generation failure."
        )
        image_generation_prompt = _extract_image_generation_prompt(tool_calls)
        if image_generation_prompt and not assistant_text:
            assistant_text = image_generation_prompt

    logger.debug(
        "Response: text_len=%s, images=%s, tool_calls=%s",
        len(assistant_text),
        len(images),
        len(tool_calls),
    )

    if not assistant_text and not images and not tool_calls:
        logger.warning("Gemini returned an empty response (no text, images, or tool calls)")
        if hasattr(response_or_stream, "candidates") and response_or_stream.candidates:
            logger.debug(f"Candidates: {response_or_stream.candidates}")

    image_generation_forced = (
        request.tool_choice is not None
        and isinstance(request.tool_choice, ResponseToolChoice)
        and request.tool_choice.type == "image_generation"
    )
    if image_generation_forced and not images:
        logger.warning(
            "Image generation was requested but Gemini Web did not return an image payload."
        )
        raise HTTPException(
            status_code=502,
            detail="Image generation request completed without returning an image.",
        )

    contents: list[ResponseOutputContent] = []
    image_calls: list[ResponseImageGenerationCall] = []
    seen_hashes: set[str] = set()

    for image in images:
        try:
            image_base64, width, height, filename, file_hash = await _image_to_base64(
                image,
                image_store,
            )
            if file_hash in seen_hashes:
                (image_store / filename).unlink(missing_ok=True)
                continue

            seen_hashes.add(file_hash)
            image_url = (
                f"![{filename}]({base_url}images/{filename}?token={deps.get_image_token(filename)})"
            )
            contents.append(ResponseOutputContent(type="output_text", text=image_url))
            image_calls.append(
                ResponseImageGenerationCall(
                    id=filename.rsplit(".", 1)[0],
                    result=image_base64,
                    output_format="png" if isinstance(image, GeneratedImage) else "jpeg",
                    size=f"{width}x{height}" if width and height else None,
                )
            )
        except Exception as exc:
            logger.warning(f"Image error: {exc}")

    if assistant_text:
        contents.append(ResponseOutputContent(type="output_text", text=assistant_text))
    if not contents:
        contents.append(ResponseOutputContent(type="output_text", text=""))

    image_markdown = ""
    for image_call in image_calls:
        filename = f"{image_call.id}.{image_call.output_format}"
        image_url = (
            f"![{filename}]({base_url}images/{filename}?token={deps.get_image_token(filename)})"
        )
        image_markdown += f"\n\n{image_url}"

    if image_markdown:
        storage_output += image_markdown

    input_tokens, output_tokens, total_tokens, reasoning_tokens = deps.calculate_usage(
        messages,
        assistant_text,
        tool_calls,
        thoughts,
    )
    usage = ResponseUsage(
        input_tokens=input_tokens,
        output_tokens=output_tokens,
        total_tokens=total_tokens,
        output_tokens_details={"reasoning_tokens": reasoning_tokens},
    )

    payload = deps.create_responses_standard_payload(
        response_id,
        created_time,
        model.model_name,
        None if direct_image_generation else tool_calls,
        image_calls,
        contents,
        usage,
        request,
        normalized_input,
        thoughts,
    )

    deps.persist_conversation(
        db,
        model.model_name,
        client.id,
        session.metadata,
        messages,
        storage_output,
        None if direct_image_generation else tool_calls,
        thoughts,
    )

    return payload