Spaces:
Runtime error
Runtime error
File size: 7,789 Bytes
5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 828be99 5eeb2b8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 | from __future__ import annotations
import reprlib
import uuid
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Awaitable, Callable
from fastapi import HTTPException, Request
from gemini_webapi import ModelOutput
from gemini_webapi.types.image import GeneratedImage, Image
from loguru import logger
from ..models import ChatCompletionRequest
from ..services.client import GeminiClientWrapper
from ..services.lmdb import LMDBConversationStore
from ..utils.helper import detect_image_extension, extract_image_dimensions
StreamFactory = Callable[..., Any]
PayloadFactory = Callable[..., dict[str, Any]]
PersistConversationFn = Callable[..., str | None]
ProcessOutputFn = Callable[..., tuple[str | None, str, str, list[Any]]]
CalculateUsageFn = Callable[..., tuple[int, int, int, int]]
ExecuteCoreSessionFn = Callable[
..., Awaitable[tuple[Any, str, int, Any, Any, LMDBConversationStore, Any, list[Any], Any]]
]
@dataclass(slots=True)
class OpenAIChatHandlerDependencies:
"""Injected runtime hooks extracted from the chat router."""
execute_core_chat_session: ExecuteCoreSessionFn
create_real_streaming_response: StreamFactory
create_chat_completion_standard_payload: PayloadFactory
process_llm_output: ProcessOutputFn
calculate_usage: CalculateUsageFn
persist_conversation: PersistConversationFn
get_image_token: Callable[[str], str]
async def _image_to_base64(
image: Image, temp_dir: Path
) -> tuple[str, int | None, int | None, str, str]:
"""Persist an image provided by gemini_webapi and return base64 plus dimensions, filename, and hash."""
import base64
import hashlib
if isinstance(image, GeneratedImage):
try:
saved_path = await image.save(path=str(temp_dir), full_size=True)
except Exception as e:
logger.warning(
f"Failed to download full-size GeneratedImage, retrying with default size: {e}"
)
saved_path = await image.save(path=str(temp_dir), full_size=False)
else:
saved_path = await image.save(path=str(temp_dir))
if not saved_path:
raise ValueError("Failed to save generated image")
original_path = Path(saved_path)
data = original_path.read_bytes()
suffix = original_path.suffix
if not suffix:
detected_ext = detect_image_extension(data)
suffix = detected_ext or (".png" if isinstance(image, GeneratedImage) else ".jpg")
random_name = f"img_{uuid.uuid4().hex}{suffix}"
new_path = temp_dir / random_name
original_path.rename(new_path)
width, height = extract_image_dimensions(data)
filename = random_name
file_hash = hashlib.sha256(data).hexdigest()
return base64.b64encode(data).decode("ascii"), width, height, filename, file_hash
async def handle_chat_completion(
request: ChatCompletionRequest,
raw_request: Request,
tmp_dir: Path,
image_store: Path,
deps: OpenAIChatHandlerDependencies,
) -> Any:
"""
Handle the OpenAI-compatible chat completions endpoint.
This module intentionally contains only OpenAI chat request/response orchestration.
Lower-level Gemini session execution, streaming builders, and persistence helpers are
injected so the existing runtime behavior can be preserved while the router is slimmed down.
"""
base_url = str(raw_request.base_url)
(
resp_or_stream,
completion_id,
created_time,
session,
client,
db,
model,
msgs,
struct_req,
) = await deps.execute_core_chat_session(
request_model=request.model,
messages=request.messages,
tmp_dir=tmp_dir,
tools=request.tools,
tool_choice=request.tool_choice,
stream=bool(request.stream),
response_format=request.response_format,
)
if request.stream:
return deps.create_real_streaming_response(
resp_or_stream,
completion_id,
created_time,
model.model_name,
msgs,
db,
model,
client,
session,
base_url,
struct_req,
)
if not isinstance(resp_or_stream, ModelOutput):
raise HTTPException(status_code=502, detail="Malformed non-stream response.")
try:
thoughts = getattr(resp_or_stream, "thoughts", None)
raw_clean = GeminiClientWrapper.extract_output(resp_or_stream, include_thoughts=False)
except Exception as exc: # pragma: no cover - defensive runtime boundary
logger.exception("Gemini output parsing failed.")
raise HTTPException(status_code=502, detail="Malformed response.") from exc
thoughts, visible_output, storage_output, tool_calls = deps.process_llm_output(
thoughts,
raw_clean,
struct_req,
)
images = resp_or_stream.images or []
logger.debug(
"Chat response: text_len=%s, images=%s, tool_calls=%s",
len(visible_output),
len(images),
len(tool_calls),
)
logger.debug(
"Raw response text (first 500 chars): %s",
raw_clean[:500] if raw_clean else "EMPTY",
)
if not visible_output and not images and not tool_calls:
logger.warning(
"Gemini returned an empty response for chat completion (no text, images, or tool calls)"
)
logger.debug(f"Response object type: {type(resp_or_stream)}")
logger.debug(f"Response has candidates: {hasattr(resp_or_stream, 'candidates')}")
if hasattr(resp_or_stream, "candidates"):
logger.debug(f"Candidates: {resp_or_stream.candidates}")
if hasattr(resp_or_stream, "text"):
logger.debug(
"Response text attribute: %s",
repr(resp_or_stream.text)[:200] if resp_or_stream.text else "None",
)
image_markdown = ""
seen_hashes: set[str] = set()
for image in images:
try:
_, _, _, filename, file_hash = await _image_to_base64(image, image_store)
if file_hash in seen_hashes:
(image_store / filename).unlink(missing_ok=True)
continue
seen_hashes.add(file_hash)
img_url = (
f"})"
)
image_markdown += f"\n\n{img_url}"
except Exception as exc: # pragma: no cover - defensive runtime boundary
logger.warning(f"Failed to process image in OpenAI response: {exc}")
if image_markdown:
visible_output += image_markdown
storage_output += image_markdown
tool_calls_payload = [call.model_dump(mode="json") for call in tool_calls]
if tool_calls_payload:
logger.debug(f"Detected tool calls: {reprlib.repr(tool_calls_payload)}")
prompt_tokens, completion_tokens, total_tokens, reasoning_tokens = deps.calculate_usage(
msgs,
visible_output,
tool_calls,
thoughts,
)
usage = {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": total_tokens,
"completion_tokens_details": {"reasoning_tokens": reasoning_tokens},
}
payload = deps.create_chat_completion_standard_payload(
completion_id,
created_time,
model.model_name,
visible_output,
tool_calls_payload,
"tool_calls" if tool_calls else "stop",
usage,
thoughts,
)
deps.persist_conversation(
db,
model.model_name,
client.id,
session.metadata,
msgs,
storage_output,
tool_calls,
thoughts,
)
return payload
|