Improve vision input handling and text sanitization

api/ai.py

@@ -6,7 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field
 
 from api.image_inputs import parse_image_edit_request, read_image_sources
 from api.support import require_identity, resolve_image_base_url
-from services.content_filter import check_request, request_text
+from services.content_filter import check_request, request_shape, request_text
 from services.log_service import LoggedCall
 from services.protocol import (
     anthropic_v1_messages,
@@ -114,7 +114,14 @@ def create_router() -> APIRouter:
         payload = body.model_dump(mode="python")
         model = str(payload.get("model") or "auto")
         request_preview = request_text(payload.get("prompt"), payload.get("messages"))
-        call = LoggedCall(identity, "/v1/chat/completions", model, "文本生成", request_text=request_preview)
+        call = LoggedCall(
+            identity,
+            "/v1/chat/completions",
+            model,
+            "文本生成",
+            request_text=request_preview,
+            request_shape=request_shape(payload.get("messages")),
+        )
         await filter_or_log(call, request_preview)
         return await call.run(openai_v1_chat_complete.handle, payload)
 
@@ -124,7 +131,14 @@ def create_router() -> APIRouter:
         payload = body.model_dump(mode="python")
         model = str(payload.get("model") or "auto")
         request_preview = request_text(payload.get("input"), payload.get("instructions"))
-        call = LoggedCall(identity, "/v1/responses", model, "Responses", request_text=request_preview)
+        call = LoggedCall(
+            identity,
+            "/v1/responses",
+            model,
+            "Responses",
+            request_text=request_preview,
+            request_shape=request_shape(payload.get("input")),
+        )
         await filter_or_log(call, request_preview)
         return await call.run(openai_v1_response.handle, payload)
 

services/content_filter.py

@@ -37,6 +37,52 @@ def request_text(*values: object) -> str:
     return "\n".join(part for value in values if (part := _text(value).strip()))
 
 
+def request_shape(*values: object) -> dict[str, int]:
+    """Return a safe structural summary without logging prompts or image bytes."""
+    stats = {
+        "response_message_items": 0,
+        "input_image_parts": 0,
+        "image_url_parts": 0,
+        "image_parts": 0,
+        "data_url_images": 0,
+        "remote_image_urls": 0,
+        "literal_image_placeholders": 0,
+    }
+
+    def walk(value: object, key: str = "") -> None:
+        if isinstance(value, str):
+            text = value.strip()
+            lower = text.lower()
+            if "<image>" in lower:
+                stats["literal_image_placeholders"] += 1
+            if lower.startswith("data:image/"):
+                stats["data_url_images"] += 1
+            elif key in {"image_url", "url"} and lower.startswith(("http://", "https://")):
+                stats["remote_image_urls"] += 1
+            return
+        if isinstance(value, list):
+            for item in value:
+                walk(item, key)
+            return
+        if not isinstance(value, dict):
+            return
+        item_type = str(value.get("type") or "").strip()
+        if item_type == "message":
+            stats["response_message_items"] += 1
+        elif item_type == "input_image":
+            stats["input_image_parts"] += 1
+        elif item_type == "image_url":
+            stats["image_url_parts"] += 1
+        elif item_type == "image":
+            stats["image_parts"] += 1
+        for child_key, child in value.items():
+            walk(child, str(child_key))
+
+    for value in values:
+        walk(value)
+    return {key: value for key, value in stats.items() if value}
+
+
 def _sanitize_for_review(text: str) -> tuple[str, dict[str, int]]:
     """Strip base64 data URIs and truncate to the review-service context limit.
 

services/log_service.py

@@ -208,6 +208,7 @@ class LoggedCall:
     summary: str
     started: float = field(default_factory=time.time)
     request_text: str = ""
+    request_shape: dict[str, int] | None = None
 
     async def run(self, handler, *args, sse: str = "openai"):
         from services.protocol.conversation import ImageGenerationError
@@ -294,6 +295,8 @@ class LoggedCall:
         request_excerpt = _request_excerpt(self.request_text)
         if request_excerpt:
             detail["request_text"] = request_excerpt
+        if self.request_shape:
+            detail["request_shape"] = self.request_shape
         if error:
             detail["error"] = error
         email = str(account_email or "").strip()

services/protocol/conversation.py

@@ -124,7 +124,7 @@ def normalize_messages(messages: object, system: Any = None) -> list[dict[str, A
                         if not isinstance(part, dict) or part.get("type") != "image":
                             continue
                         data = part.get("data")
-                        if isinstance(data, (bytes, bytearray)):
+                        if isinstance(data, (bytes, bytearray)) and all(existing[0] != bytes(data) for existing in images):
                             images.append((bytes(data), str(part.get("mime") or "image/png")))
             if images:
                 parts: list[Any] = []
@@ -316,20 +316,45 @@ def strip_history(text: str, history_text: str = "") -> str:
 def sanitize_output_text(text: str) -> str:
     text = str(text or "")
 
-    def replace_url(match: re.Match[str]) -> str:
-        label = match.group(1).strip()
-        url = match.group(2).strip()
-        if label and url.startswith(("http://", "https://")):
-            return f"{label} ({url})"
-        return label or url
+    def is_internal_annotation_part(part: str) -> bool:
+        value = part.strip()
+        if not value:
+            return True
+        lower = value.lower()
+        return bool(
+            re.fullmatch(r"turn\d+[a-z]*\d*", lower)
+            or re.fullmatch(r"turn\d+\w*", lower)
+            or lower.startswith(("turn", "source", "sources"))
+        )
+
+    def readable_annotation_part(parts: list[str]) -> str:
+        for part in parts:
+            value = part.strip()
+            if value and not is_internal_annotation_part(value):
+                return value
+        return ""
+
+    def replace_annotation(match: re.Match[str]) -> str:
+        payload = match.group(1)
+        parts = [part.strip() for part in payload.split("\ue202")]
+        kind = (parts[0] if parts else "").lower()
+        data = parts[1:]
+        if kind == "url":
+            label = data[0] if data else ""
+            url = data[1] if len(data) > 1 else ""
+            if label and url.startswith(("http://", "https://")):
+                return f"{label} ({url})"
+            return label or url
+        if kind == "cite":
+            return readable_annotation_part(data)
+        return readable_annotation_part(data)
 
     # ChatGPT web sometimes returns rich annotation markers using private-use
-    # characters. API clients cannot render those, so convert links and drop
-    # citations before emitting OpenAI-compatible text.
-    text = re.sub(r"\ue200url\ue202([^\ue202\ue201]*)\ue202([^\ue201]*)\ue201", replace_url, text)
-    text = re.sub(r"\ue200cite\ue202[^\ue201]*\ue201", "", text)
-    text = re.sub(r"\ue200[^\ue201]*\ue201", "", text)
+    # characters. API clients cannot render those. Preserve readable labels
+    # from entity/link annotations, while removing internal citation pointers.
+    text = re.sub(r"\ue200([^\ue201]*)\ue201", replace_annotation, text)
     text = re.sub(r"\ue200[^\ue201]*$", "", text)
+    text = re.sub(r"\s+([.,;:!?])", r"\1", text)
     return text
 
 

services/protocol/openai_v1_response.py

@@ -33,7 +33,7 @@ TOOL_UNAVAILABLE_SYSTEM_MESSAGE = (
     "If a user asks you to use a tool, say that tool execution is unavailable through this backend."
 )
 
-RESPONSE_CONTENT_PART_TYPES = {"text", "input_text", "output_text", "image_url", "input_image"}
+RESPONSE_CONTENT_PART_TYPES = {"text", "input_text", "output_text", "image_url", "input_image", "image"}
 
 
 def is_text_response_request(body: dict[str, Any]) -> bool:

test/test_chat_completion_cache.py

@@ -9,6 +9,7 @@ from services.config import config
 from services.protocol import openai_v1_chat_complete, openai_v1_response
 from services.protocol.chat_completion_cache import chat_completion_cache
 from services.protocol.conversation import iter_conversation_payloads, sanitize_output_text
+from utils.helper import extract_image_from_message_content
 
 
 PNG_1X1 = base64.b64decode(
@@ -190,9 +191,25 @@ class ChatCompletionCacheTests(unittest.TestCase):
 
         self.assertEqual(
             sanitize_output_text(text),
-            "Repo: basketikun/chatgpt2api (https://github.com/basketikun/chatgpt2api) details .",
+            "Repo: basketikun/chatgpt2api (https://github.com/basketikun/chatgpt2api) details.",
         )
 
+    def test_output_sanitizer_preserves_annotated_entity_text(self) -> None:
+        text = (
+            "The character is from \ue200entity\ue202Invincible\ue201, "
+            "which is based on the comic series \ue200entity\ue202Invincible\ue201."
+        )
+
+        self.assertEqual(
+            sanitize_output_text(text),
+            "The character is from Invincible, which is based on the comic series Invincible.",
+        )
+
+    def test_output_sanitizer_preserves_readable_cite_label(self) -> None:
+        text = "The character is \ue200cite\ue202Invincible\ue202turn0search0\ue201."
+
+        self.assertEqual(sanitize_output_text(text), "The character is Invincible.")
+
     def test_stream_sanitizer_does_not_emit_partial_annotation_or_repeat_prefix(self) -> None:
         events = [
             {"p": "/message/content/parts/0", "o": "append", "v": "Repo: \ue200url\ue202chat"},
@@ -206,7 +223,7 @@ class ChatCompletionCacheTests(unittest.TestCase):
             if event.get("type") == "conversation.delta"
         ]
 
-        self.assertEqual("".join(deltas), "Repo: chatgpt2api done .")
+        self.assertEqual("".join(deltas), "Repo: chatgpt2api done.")
         self.assertFalse(any("\ue200" in delta or "\ue202" in delta or "\ue201" in delta for delta in deltas))
 
     def test_responses_tools_add_honest_no_tool_guard(self) -> None:
@@ -301,6 +318,19 @@ class ChatCompletionCacheTests(unittest.TestCase):
         self.assertEqual(content[1]["data"], PNG_1X1)
         self.assertEqual(content[1]["mime"], "image/png")
 
+    def test_image_extractor_supports_extra_image_object_shapes(self) -> None:
+        encoded = base64.b64encode(PNG_1X1).decode("ascii")
+
+        images = extract_image_from_message_content([
+            {"type": "image", "data": PNG_1X1, "mime": "image/png"},
+            {"type": "input_image", "base64": encoded, "mime_type": "image/png"},
+            {"type": "input_image", "source": {"type": "base64", "data": encoded, "media_type": "image/png"}},
+        ])
+
+        self.assertEqual(len(images), 3)
+        self.assertEqual([image[1] for image in images], ["image/png", "image/png", "image/png"])
+        self.assertTrue(all(image[0] == PNG_1X1 for image in images))
+
 
 if __name__ == "__main__":
     unittest.main()

utils/helper.py

@@ -320,8 +320,14 @@ def extract_prompt_from_message_content(content: object) -> str:
     return "\n".join(parts).strip()
 
 
-def _decode_message_image_url(url: str) -> tuple[bytes, str] | None:
-    source = str(url or "").strip()
+def _message_image_url(value: object) -> str:
+    if isinstance(value, dict):
+        return str(value.get("url") or value.get("image_url") or "").strip()
+    return str(value or "").strip()
+
+
+def _decode_message_image_url(value: object) -> tuple[bytes, str] | None:
+    source = _message_image_url(value)
     if source.startswith("data:"):
         header, _, data = source.partition(",")
         mime = header.split(";")[0].removeprefix("data:") or "image/png"
@@ -363,6 +369,31 @@ def _decode_message_image_url(url: str) -> tuple[bytes, str] | None:
     return image_data, mime
 
 
+def _decode_message_image_object(item: dict[str, object]) -> tuple[bytes, str] | None:
+    data = item.get("data")
+    if isinstance(data, (bytes, bytearray)):
+        return bytes(data), str(item.get("mime") or item.get("mime_type") or "image/png")
+    for key in ("image_url", "url"):
+        image = _decode_message_image_url(item.get(key))
+        if image:
+            return image
+    value = item.get("b64_json") or item.get("base64")
+    if isinstance(value, str) and value.strip():
+        image_data, _, mime = _decode_json_image_string(
+            value,
+            1,
+            mime_type=str(item.get("mime") or item.get("mime_type") or item.get("mimeType") or "image/png"),
+        )
+        return image_data, mime
+    source = item.get("source")
+    if isinstance(source, dict) and str(source.get("type") or "") == "base64":
+        encoded = str(source.get("data") or "")
+        mime = str(source.get("media_type") or source.get("mime_type") or "image/png")
+        image_data, _, resolved_mime = _decode_json_image_string(encoded, 1, mime_type=mime)
+        return image_data, resolved_mime
+    return None
+
+
 def extract_image_from_message_content(content: object) -> list[tuple[bytes, str]]:
     if not isinstance(content, list):
         return []
@@ -372,15 +403,11 @@ def extract_image_from_message_content(content: object) -> list[tuple[bytes, str
             continue
         item_type = str(item.get("type") or "").strip()
         if item_type == "image_url":
-            url_obj = item.get("image_url") or item
-            url = str(url_obj.get("url") or "") if isinstance(url_obj, dict) else str(url_obj)
-            image = _decode_message_image_url(url)
+            image = _decode_message_image_url(item.get("image_url") or item.get("url") or item)
             if image:
                 images.append(image)
-        elif item_type == "input_image":
-            url_obj = item.get("image_url") or item.get("url") or ""
-            image_url = str(url_obj.get("url") or url_obj.get("image_url") or "") if isinstance(url_obj, dict) else str(url_obj)
-            image = _decode_message_image_url(image_url)
+        elif item_type in {"input_image", "image"}:
+            image = _decode_message_image_object(item)
             if image:
                 images.append(image)
     return images