File size: 5,123 Bytes
b891545 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 | """Base transformer interface with inference-backend request enhancement."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class BaseTransformer(ABC):
"""Abstract base class for API transformers.
Transforms requests from source API format to OpenAI format (for the
inference backend), and transforms responses back to source API format.
"""
@abstractmethod
def transform_request(self, body: dict[str, Any]) -> dict[str, Any]:
"""Transform request body to OpenAI format for the inference backend."""
pass
@abstractmethod
def transform_response(
self,
response: dict[str, Any],
original_request: dict[str, Any],
) -> dict[str, Any]:
"""Transform response back to source API format."""
pass
@abstractmethod
def transform_stream_chunk(
self,
chunk: dict[str, Any],
original_request: dict[str, Any],
is_first: bool = False,
) -> dict[str, Any] | list[dict[str, Any]]:
"""Transform a streaming chunk to source API format."""
pass
def is_streaming_request(self, body: dict[str, Any]) -> bool:
"""Check if request is for streaming response."""
return body.get("stream", False)
def create_stream_state(self, original_request: dict[str, Any]) -> Any | None:
"""Create per-request stream state when chunk transforms need memory."""
return None
@staticmethod
def _is_qwen35_model(model_name: str | None) -> bool:
if not model_name:
return False
return "qwen3.5" in model_name.lower()
@staticmethod
def _content_to_text(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict):
text = block.get("text")
if isinstance(text, str):
parts.append(text)
return "\n".join(parts)
return str(content) if content else ""
@classmethod
def _merge_developer_role(cls, request: dict[str, Any]) -> dict[str, Any]:
"""Rename 'developer' role to 'system' and merge all system messages into one."""
messages = request.get("messages")
if not isinstance(messages, list):
return request
# Rename developer -> system
normalized = [
{**msg, "role": "system"}
if isinstance(msg, dict) and msg.get("role") == "developer"
else msg
for msg in messages
]
# Merge multiple system messages into one at the top
system_parts: list[str] = []
non_system: list[Any] = []
for msg in normalized:
if isinstance(msg, dict) and msg.get("role") == "system":
content = msg.get("content", "")
text = cls._content_to_text(content)
if text:
system_parts.append(text)
elif content:
# A system message whose content is not reducible to text — an image part, an
# unfamiliar content type — used to vanish here: `_content_to_text` returned "",
# the falsy check skipped it, and the merged system prompt silently lost it. The
# prompt then differs from what the harness sent, which for a captured turn means
# the recorded prompt is not the one the model saw. Kept as its own message
# instead of merged, since it cannot be concatenated into the text block.
non_system.append(msg)
else:
non_system.append(msg)
if system_parts:
request["messages"] = [
{"role": "system", "content": "\n\n".join(system_parts)},
*non_system,
]
else:
request["messages"] = non_system
return request
def _normalize_request(
self,
request: dict[str, Any],
model_name: str | None = None,
) -> dict[str, Any]:
"""Normalize the OpenAI request: drop internal keys, merge system roles,
and apply per-model template fixes. Training-signal params (logprobs,
token ids) are added later by the inference engine.
"""
request.pop("_served_model", None)
request = self._merge_developer_role(request)
if self._is_qwen35_model(model_name):
# Qwen3.5 outputs tool calls inside thinking; disable thinking.
# https://www.reddit.com/r/LocalLLaMA/comments/1sccqt2/i_think_i_got_solutions_for_qwen_35_tool_call_in/
chat_template_kwargs = dict(request.get("chat_template_kwargs") or {})
chat_template_kwargs.setdefault("enable_thinking", False)
request["chat_template_kwargs"] = chat_template_kwargs
return request
|