Update app.py
Browse files
app.py
CHANGED
|
@@ -1,17 +1,7 @@
|
|
| 1 |
"""
|
| 2 |
-
|
| 3 |
-
=============================
|
| 4 |
-
|
| 5 |
-
OpenCode Zen غيّر بنيته كلياً — كل نموذج له endpoint مختلف:
|
| 6 |
-
|
| 7 |
-
/zen/v1/messages → نماذج Anthropic-format (Claude, Qwen, MiniMax M3, Nemotron)
|
| 8 |
-
/zen/v1/chat/completions → نماذج OpenAI-format (DeepSeek, MiniMax M2.x, GLM, MiMo, Kimi)
|
| 9 |
-
/zen/v1/responses → نماذج OpenAI Responses (GPT-5.x)
|
| 10 |
-
|
| 11 |
-
الكود القديم كان يُرسل الجميع إلى /chat/completions → صمت تام بلا استجابة.
|
| 12 |
-
|
| 13 |
-
الحل: جدول MODEL_REGISTRY يحمل (model_id, endpoint_type, upstream_url)
|
| 14 |
-
والـ gateway يختار الـ pipeline الصحيح تلقائياً.
|
| 15 |
"""
|
| 16 |
|
| 17 |
import os
|
|
@@ -20,9 +10,18 @@ import base64
|
|
| 20 |
import uuid
|
| 21 |
import asyncio
|
| 22 |
import logging
|
|
|
|
| 23 |
from contextlib import asynccontextmanager
|
| 24 |
from dataclasses import dataclass
|
| 25 |
-
from typing import Literal
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
|
| 27 |
try:
|
| 28 |
import orjson as _json_lib
|
|
@@ -50,138 +49,96 @@ logging.basicConfig(
|
|
| 50 |
level=logging.INFO,
|
| 51 |
format="%(asctime)s [%(levelname)s] %(message)s",
|
| 52 |
)
|
| 53 |
-
logger = logging.getLogger("
|
| 54 |
|
| 55 |
# =========================================================
|
| 56 |
# CONFIG
|
| 57 |
# =========================================================
|
| 58 |
-
|
| 59 |
-
MAX_RETRIES =
|
| 60 |
THINKING_MODE = os.getenv("THINKING_MODE", "enabled").strip().lower()
|
| 61 |
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
|
| 68 |
# =========================================================
|
| 69 |
# MODEL REGISTRY
|
| 70 |
# =========================================================
|
| 71 |
-
# مصدر: https://opencode.ai/docs/zen/ (جدول Endpoints)
|
| 72 |
-
# endpoint_type:
|
| 73 |
-
# "messages" → Anthropic /v1/messages protocol
|
| 74 |
-
# "chat" → OpenAI /v1/chat/completions protocol
|
| 75 |
-
# "responses" → OpenAI /v1/responses protocol (لم ندعمه بعد)
|
| 76 |
-
|
| 77 |
@dataclass
|
| 78 |
class ModelEntry:
|
| 79 |
-
upstream_id: str
|
| 80 |
-
endpoint_type: Literal["
|
| 81 |
upstream_url: str
|
| 82 |
|
| 83 |
MODEL_REGISTRY: dict[str, ModelEntry] = {
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
"claude-sonnet-4-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
"claude-opus-4-6": ModelEntry("mimo-v2.5-free", "chat", URL_CHAT),
|
| 94 |
-
"claude-sonnet-4-5": ModelEntry("north-mini-code-free", "chat", URL_CHAT),
|
| 95 |
-
|
| 96 |
-
# ── نماذج Kimi (OpenAI endpoint) ──────────────────────────────────
|
| 97 |
-
|
| 98 |
-
"claude-sonnet-4-7": ModelEntry("kimi-k2.5-free", "chat", URL_CHAT),
|
| 99 |
-
"claude-opus-4-7": ModelEntry("nemotron-3-ultra-free", "chat", URL_CHAT),
|
| 100 |
-
|
| 101 |
-
# ── نماذج Big Pickle (OpenAI endpoint) ────────────────────────────
|
| 102 |
-
"claude-sonnet-4-8": ModelEntry("big-pickle", "chat", URL_CHAT),
|
| 103 |
}
|
| 104 |
|
| 105 |
-
|
| 106 |
-
_DEFAULT_ENTRY = ModelEntry("deepseek-v4-flash", "chat", URL_CHAT)
|
| 107 |
-
|
| 108 |
-
# تحويل الأسماء القديمة (التي كانت في MODEL_MAP السابق) → أسماء Zen الجديدة
|
| 109 |
-
_LEGACY_MAP: dict[str, str] = {
|
| 110 |
-
# الأسماء الوهمية القديمة → الأسماء الحقيقية في Zen
|
| 111 |
-
"nemotron-3-super-free": "qwen3.7-max", # كان يُعطى لـ Sonnet/Opus → أفضل بديل
|
| 112 |
-
"nemotron-3-ultra-free": "qwen3.7-max",
|
| 113 |
-
"north-mini-code-free": "deepseek-v4-flash",
|
| 114 |
-
}
|
| 115 |
|
| 116 |
def _resolve(requested_model: str) -> ModelEntry:
|
| 117 |
-
"""إيجاد الـ ModelEntry المناسب للنموذج المطلوب."""
|
| 118 |
-
# 1. مباشرة في الـ registry
|
| 119 |
if requested_model in MODEL_REGISTRY:
|
| 120 |
return MODEL_REGISTRY[requested_model]
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
if alias and alias in MODEL_REGISTRY:
|
| 124 |
-
return MODEL_REGISTRY[alias]
|
| 125 |
-
# 3. fallback
|
| 126 |
-
logger.warning("Unknown model '%s' → fallback deepseek-v4-flash", requested_model)
|
| 127 |
return _DEFAULT_ENTRY
|
| 128 |
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
|
|
|
|
|
|
|
|
|
| 132 |
_MODELS_CACHE = {
|
| 133 |
"object": "list",
|
| 134 |
"data": [
|
| 135 |
-
{"id": mid, "object": "model", "created": 1710000000, "owned_by": "
|
| 136 |
for mid in MODEL_REGISTRY
|
| 137 |
],
|
| 138 |
}
|
| 139 |
|
| 140 |
# =========================================================
|
| 141 |
-
#
|
| 142 |
# =========================================================
|
| 143 |
-
_OPENCODE_VERSION = os.getenv("OPENCODE_VERSION", "1.15.4")
|
| 144 |
-
_USER_ID = os.getenv("OPENCODE_USER_ID", uuid.uuid4().hex)
|
| 145 |
-
_PROJECT_ID = os.getenv("OPENCODE_PROJECT_ID", uuid.uuid4().hex)
|
| 146 |
-
|
| 147 |
-
_STATIC_UPSTREAM_HEADERS = {
|
| 148 |
-
"Content-Type": "application/json",
|
| 149 |
-
"Accept-Encoding": "identity",
|
| 150 |
-
"Accept": "application/json",
|
| 151 |
-
"Origin": "https://opencode.ai",
|
| 152 |
-
"Referer": "https://opencode.ai/",
|
| 153 |
-
"User-Agent": f"opencode/{_OPENCODE_VERSION}",
|
| 154 |
-
"x-opencode-client": "cli",
|
| 155 |
-
}
|
| 156 |
-
|
| 157 |
def _make_headers(api_key: str) -> dict:
|
| 158 |
return {
|
| 159 |
-
|
| 160 |
-
"
|
| 161 |
-
"
|
| 162 |
-
"
|
| 163 |
-
"x-opencode-session": uuid.uuid4().hex,
|
| 164 |
}
|
| 165 |
|
| 166 |
STREAM_RESPONSE_HEADERS = {
|
| 167 |
-
"Cache-Control": "no-cache",
|
| 168 |
"Connection": "keep-alive",
|
| 169 |
"X-Accel-Buffering": "no",
|
|
|
|
| 170 |
}
|
| 171 |
|
| 172 |
# =========================================================
|
| 173 |
-
#
|
| 174 |
# =========================================================
|
| 175 |
@asynccontextmanager
|
| 176 |
async def lifespan(app: FastAPI):
|
|
|
|
| 177 |
app.state.client = httpx.AsyncClient(
|
| 178 |
http2=True,
|
| 179 |
-
timeout=httpx.Timeout(connect=
|
| 180 |
-
limits=httpx.Limits(max_connections=
|
| 181 |
follow_redirects=True,
|
| 182 |
)
|
| 183 |
-
logger.info("Gateway started — THINKING_MODE=%s
|
| 184 |
-
THINKING_MODE, _json_lib.__name__, _USER_ID[:8])
|
| 185 |
yield
|
| 186 |
await app.state.client.aclose()
|
| 187 |
|
|
@@ -190,11 +147,11 @@ app = FastAPI(lifespan=lifespan)
|
|
| 190 |
@app.get("/")
|
| 191 |
@app.head("/")
|
| 192 |
async def root():
|
| 193 |
-
return {"status": "ok", "service": "
|
| 194 |
|
| 195 |
@app.get("/health")
|
| 196 |
async def health():
|
| 197 |
-
return {"status": "healthy"}
|
| 198 |
|
| 199 |
@app.get("/v1/models")
|
| 200 |
async def get_models():
|
|
@@ -217,7 +174,7 @@ def _synthetic_signature(text: str) -> str:
|
|
| 217 |
return "synthetic_" + base64.urlsafe_b64encode(digest).decode().rstrip("=")
|
| 218 |
|
| 219 |
# =========================================================
|
| 220 |
-
#
|
| 221 |
# =========================================================
|
| 222 |
def normalize_messages(messages: list) -> list:
|
| 223 |
output = []
|
|
@@ -238,14 +195,17 @@ def normalize_messages(messages: list) -> list:
|
|
| 238 |
|
| 239 |
for item in content:
|
| 240 |
if not isinstance(item, dict):
|
| 241 |
-
text_parts.append(str(item))
|
|
|
|
| 242 |
t = item.get("type", "")
|
| 243 |
if t == "text":
|
| 244 |
txt = item.get("text", "")
|
| 245 |
-
if txt:
|
|
|
|
| 246 |
elif t == "thinking":
|
| 247 |
tk = item.get("thinking", "")
|
| 248 |
-
if tk:
|
|
|
|
| 249 |
elif t == "tool_use":
|
| 250 |
tool_calls.append({
|
| 251 |
"id": item.get("id", f"call_{uuid.uuid4().hex[:8]}"),
|
|
@@ -291,141 +251,45 @@ def normalize_messages(messages: list) -> list:
|
|
| 291 |
output.append(r)
|
| 292 |
return output
|
| 293 |
|
| 294 |
-
# =========================================================
|
| 295 |
-
# TOOLS
|
| 296 |
-
# =========================================================
|
| 297 |
def convert_tools_to_openai(tools: list) -> list:
|
| 298 |
return [{"type":"function","function":{"name":t.get("name",""),
|
| 299 |
"description":t.get("description",""),
|
| 300 |
"parameters":_fix_schema(t.get("input_schema",{}))}} for t in tools] if tools else []
|
| 301 |
|
| 302 |
def map_tool_choice(raw, has_tools: bool, thinking_on: bool):
|
| 303 |
-
if not raw or not has_tools:
|
|
|
|
| 304 |
if isinstance(raw, dict):
|
| 305 |
kind = raw.get("type","auto")
|
| 306 |
if thinking_on and kind in ("any","tool"):
|
| 307 |
-
logger.warning("tool_choice '%s' incompatible with thinking → auto", kind)
|
| 308 |
return "auto"
|
| 309 |
-
if kind == "any":
|
| 310 |
-
|
|
|
|
|
|
|
| 311 |
return kind
|
| 312 |
if thinking_on and raw in ("any","required"):
|
| 313 |
return "auto"
|
| 314 |
return raw
|
| 315 |
|
| 316 |
-
# =========================================================
|
| 317 |
-
# BUILD PAYLOAD
|
| 318 |
-
# =========================================================
|
| 319 |
def _build_chat_payload(body, messages, tools, tool_choice, upstream_model, stream):
|
| 320 |
-
"""Payload لـ /chat/completions (OpenAI format)."""
|
| 321 |
-
thinking_on = THINKING_MODE == "enabled"
|
| 322 |
payload = {
|
| 323 |
"model": upstream_model,
|
| 324 |
"messages": messages,
|
| 325 |
-
"max_tokens": min(body.get("max_tokens",
|
| 326 |
"stream": stream,
|
| 327 |
-
"thinking": {"type": "enabled"} if thinking_on else {"type": "disabled"},
|
| 328 |
}
|
| 329 |
-
if
|
| 330 |
payload["temperature"] = body.get("temperature", 0.7)
|
| 331 |
-
elif "temperature" in body:
|
| 332 |
-
payload["temperature"] = body["temperature"]
|
| 333 |
if tools:
|
| 334 |
payload["tools"] = tools
|
| 335 |
if tool_choice is not None:
|
| 336 |
payload["tool_choice"] = tool_choice
|
| 337 |
return payload
|
| 338 |
|
| 339 |
-
def _build_messages_payload(body, messages, tools, tool_choice, upstream_model, stream):
|
| 340 |
-
"""
|
| 341 |
-
Payload لـ /messages (Anthropic format).
|
| 342 |
-
هذا الـ endpoint يستقبل تنسيق Anthropic مباشرة — لذا نُعيد
|
| 343 |
-
بناء الرسائل بصيغتها الأصلية.
|
| 344 |
-
"""
|
| 345 |
-
thinking_on = THINKING_MODE == "enabled"
|
| 346 |
-
|
| 347 |
-
# استخرج system prompt من أول رسالة system
|
| 348 |
-
system_msg = None
|
| 349 |
-
filtered = []
|
| 350 |
-
for m in messages:
|
| 351 |
-
if m.get("role") == "system":
|
| 352 |
-
system_msg = m.get("content", "")
|
| 353 |
-
else:
|
| 354 |
-
filtered.append(m)
|
| 355 |
-
|
| 356 |
-
# حوّل OpenAI tool_calls → Anthropic content blocks
|
| 357 |
-
anthropic_messages = []
|
| 358 |
-
for m in filtered:
|
| 359 |
-
role = m.get("role", "user")
|
| 360 |
-
content = m.get("content")
|
| 361 |
-
tcs = m.get("tool_calls")
|
| 362 |
-
|
| 363 |
-
if tcs:
|
| 364 |
-
blocks = []
|
| 365 |
-
if content:
|
| 366 |
-
blocks.append({"type": "text", "text": content})
|
| 367 |
-
for tc in tcs:
|
| 368 |
-
fn = tc.get("function", {})
|
| 369 |
-
try:
|
| 370 |
-
inp = json.loads(fn.get("arguments", "{}"))
|
| 371 |
-
except Exception:
|
| 372 |
-
inp = {}
|
| 373 |
-
blocks.append({"type":"tool_use","id":tc["id"],
|
| 374 |
-
"name":fn.get("name",""),"input":inp})
|
| 375 |
-
anthropic_messages.append({"role": "assistant", "content": blocks})
|
| 376 |
-
|
| 377 |
-
elif role == "tool":
|
| 378 |
-
# tool result → user content block
|
| 379 |
-
tid = m.get("tool_call_id", "")
|
| 380 |
-
anthropic_messages.append({
|
| 381 |
-
"role": "user",
|
| 382 |
-
"content": [{"type":"tool_result","tool_use_id":tid,
|
| 383 |
-
"content": m.get("content","")}]
|
| 384 |
-
})
|
| 385 |
-
else:
|
| 386 |
-
cnt = content or ""
|
| 387 |
-
if isinstance(cnt, str):
|
| 388 |
-
anthropic_messages.append({"role": role, "content": cnt})
|
| 389 |
-
else:
|
| 390 |
-
anthropic_messages.append({"role": role, "content": cnt})
|
| 391 |
-
|
| 392 |
-
payload = {
|
| 393 |
-
"model": upstream_model,
|
| 394 |
-
"messages": anthropic_messages,
|
| 395 |
-
"max_tokens": min(body.get("max_tokens", 32000), 64000),
|
| 396 |
-
"stream": stream,
|
| 397 |
-
}
|
| 398 |
-
if system_msg:
|
| 399 |
-
payload["system"] = system_msg
|
| 400 |
-
if thinking_on:
|
| 401 |
-
payload["thinking"] = {"type": "enabled", "budget_tokens": 8000}
|
| 402 |
-
if tools:
|
| 403 |
-
# حوّل tools إلى تنسيق Anthropic
|
| 404 |
-
anthropic_tools = []
|
| 405 |
-
for t in tools:
|
| 406 |
-
fn = t.get("function", {})
|
| 407 |
-
anthropic_tools.append({
|
| 408 |
-
"name": fn.get("name", ""),
|
| 409 |
-
"description": fn.get("description", ""),
|
| 410 |
-
"input_schema": fn.get("parameters", {"type":"object","properties":{}}),
|
| 411 |
-
})
|
| 412 |
-
payload["tools"] = anthropic_tools
|
| 413 |
-
if tool_choice is not None and tool_choice != "required":
|
| 414 |
-
# Anthropic يقبل: {"type":"auto"} أو {"type":"any"} أو {"type":"tool","name":"..."}
|
| 415 |
-
if isinstance(tool_choice, str):
|
| 416 |
-
payload["tool_choice"] = {"type": tool_choice}
|
| 417 |
-
else:
|
| 418 |
-
payload["tool_choice"] = tool_choice
|
| 419 |
-
|
| 420 |
-
return payload
|
| 421 |
-
|
| 422 |
-
# =========================================================
|
| 423 |
-
# CONVERT NON-STREAM CHAT RESPONSE → Anthropic
|
| 424 |
-
# =========================================================
|
| 425 |
_FINISH_MAP = {"tool_calls": "tool_use", "length": "max_tokens"}
|
| 426 |
|
| 427 |
def _chat_to_anthropic(parsed: dict, requested_model: str) -> dict:
|
| 428 |
-
"""OpenAI /chat/completions response → Anthropic format."""
|
| 429 |
content_blocks = []
|
| 430 |
stop_reason = "end_turn"
|
| 431 |
try:
|
|
@@ -444,8 +308,10 @@ def _chat_to_anthropic(parsed: dict, requested_model: str) -> dict:
|
|
| 444 |
content_blocks.append({"type": "text", "text": text})
|
| 445 |
for tc in (message.get("tool_calls") or []):
|
| 446 |
fn = tc.get("function", {})
|
| 447 |
-
try:
|
| 448 |
-
|
|
|
|
|
|
|
| 449 |
content_blocks.append({
|
| 450 |
"type": "tool_use",
|
| 451 |
"id": tc.get("id", f"toolu_{uuid.uuid4().hex[:8]}"),
|
|
@@ -468,35 +334,8 @@ def _chat_to_anthropic(parsed: dict, requested_model: str) -> dict:
|
|
| 468 |
"output_tokens": usage.get("completion_tokens", 0)},
|
| 469 |
}
|
| 470 |
|
| 471 |
-
def _messages_passthrough(parsed: dict, requested_model: str) -> dict:
|
| 472 |
-
"""
|
| 473 |
-
Anthropic /messages response → مُعاد بنفس التنسيق مع تصحيح الـ model field.
|
| 474 |
-
الـ response من Zen هو بالفعل Anthropic format، لذا نمرره مباشرة.
|
| 475 |
-
"""
|
| 476 |
-
parsed["model"] = requested_model
|
| 477 |
-
# أضف signature للـ thinking blocks إذا غابت
|
| 478 |
-
for block in parsed.get("content", []):
|
| 479 |
-
if block.get("type") == "thinking" and not block.get("signature"):
|
| 480 |
-
block["signature"] = _synthetic_signature(block.get("thinking", ""))
|
| 481 |
-
return parsed
|
| 482 |
-
|
| 483 |
# =========================================================
|
| 484 |
-
#
|
| 485 |
-
# =========================================================
|
| 486 |
-
async def _iter_sse_lines(response):
|
| 487 |
-
buf = b""
|
| 488 |
-
async for chunk in response.aiter_bytes():
|
| 489 |
-
if not chunk: continue
|
| 490 |
-
buf += chunk
|
| 491 |
-
while True:
|
| 492 |
-
nl = buf.find(b"\n")
|
| 493 |
-
if nl == -1: break
|
| 494 |
-
line = buf[:nl].rstrip(b"\r").decode("utf-8", errors="replace")
|
| 495 |
-
buf = buf[nl + 1:]
|
| 496 |
-
yield line
|
| 497 |
-
|
| 498 |
-
# =========================================================
|
| 499 |
-
# STREAMING: /chat/completions SSE → Anthropic SSE
|
| 500 |
# =========================================================
|
| 501 |
_MSG_START = "event: message_start\ndata: "
|
| 502 |
_BLK_START = "event: content_block_start\ndata: "
|
|
@@ -505,8 +344,22 @@ _BLK_STOP = "event: content_block_stop\ndata: "
|
|
| 505 |
_MSG_DELTA = "event: message_delta\ndata: "
|
| 506 |
_MSG_STOP = "event: message_stop\ndata: "
|
| 507 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 508 |
async def _stream_chat_to_anthropic(response, requested_model: str):
|
| 509 |
-
"""OpenAI SSE stream → Anthropic SSE stream."""
|
| 510 |
msg_id = f"msg_{uuid.uuid4().hex}"
|
| 511 |
in_tok = out_tok = 0
|
| 512 |
|
|
@@ -516,17 +369,26 @@ async def _stream_chat_to_anthropic(response, requested_model: str):
|
|
| 516 |
"usage":{"input_tokens":0,"output_tokens":0}}}) + "\n\n")
|
| 517 |
yield 'event: ping\ndata: {"type":"ping"}\n\n'
|
| 518 |
|
| 519 |
-
tool_state = {}
|
| 520 |
-
|
| 521 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 522 |
|
| 523 |
try:
|
| 524 |
async for line in _iter_sse_lines(response):
|
| 525 |
-
if not line or not line.startswith("data:"):
|
|
|
|
| 526 |
data = line[5:].strip()
|
| 527 |
-
if data == "[DONE]":
|
| 528 |
-
|
| 529 |
-
|
|
|
|
|
|
|
|
|
|
| 530 |
|
| 531 |
if chunk.get("usage"):
|
| 532 |
u = chunk["usage"]
|
|
@@ -534,18 +396,22 @@ async def _stream_chat_to_anthropic(response, requested_model: str):
|
|
| 534 |
out_tok = u.get("completion_tokens", out_tok)
|
| 535 |
|
| 536 |
choices = chunk.get("choices")
|
| 537 |
-
if not choices:
|
| 538 |
-
|
|
|
|
|
|
|
| 539 |
fr = choice.get("finish_reason")
|
| 540 |
-
if fr == "tool_calls":
|
| 541 |
-
|
|
|
|
|
|
|
| 542 |
|
| 543 |
-
# reasoning
|
| 544 |
rd = delta.get("reasoning_content")
|
| 545 |
if rd:
|
| 546 |
reasoning_text_acc += rd
|
| 547 |
if reasoning_block_idx is None:
|
| 548 |
-
reasoning_block_idx = next_idx
|
|
|
|
| 549 |
yield (_BLK_START + _dumps({"type":"content_block_start",
|
| 550 |
"index":reasoning_block_idx,
|
| 551 |
"content_block":{"type":"thinking","thinking":""}}) + "\n\n")
|
|
@@ -553,7 +419,6 @@ async def _stream_chat_to_anthropic(response, requested_model: str):
|
|
| 553 |
"index":reasoning_block_idx,
|
| 554 |
"delta":{"type":"thinking_delta","thinking":rd}}) + "\n\n")
|
| 555 |
|
| 556 |
-
# text
|
| 557 |
td = delta.get("content")
|
| 558 |
if td:
|
| 559 |
if reasoning_block_idx is not None and not reasoning_closed:
|
|
@@ -565,7 +430,8 @@ async def _stream_chat_to_anthropic(response, requested_model: str):
|
|
| 565 |
"index":reasoning_block_idx}) + "\n\n")
|
| 566 |
reasoning_closed = True
|
| 567 |
if text_block_idx is None:
|
| 568 |
-
text_block_idx = next_idx
|
|
|
|
| 569 |
yield (_BLK_START + _dumps({"type":"content_block_start",
|
| 570 |
"index":text_block_idx,
|
| 571 |
"content_block":{"type":"text","text":""}}) + "\n\n")
|
|
@@ -573,7 +439,6 @@ async def _stream_chat_to_anthropic(response, requested_model: str):
|
|
| 573 |
"index":text_block_idx,
|
| 574 |
"delta":{"type":"text_delta","text":td}}) + "\n\n")
|
| 575 |
|
| 576 |
-
# tools
|
| 577 |
tcd = delta.get("tool_calls")
|
| 578 |
if tcd:
|
| 579 |
if reasoning_block_idx is not None and not reasoning_closed:
|
|
@@ -590,11 +455,14 @@ async def _stream_chat_to_anthropic(response, requested_model: str):
|
|
| 590 |
fn0 = tc.get("function", {})
|
| 591 |
tool_state[idx] = {"id":tc.get("id",f"toolu_{uuid.uuid4().hex[:8]}"),
|
| 592 |
"name":fn0.get("name",""),"args":""}
|
| 593 |
-
blk = next_idx
|
|
|
|
|
|
|
| 594 |
yield (_BLK_START + _dumps({"type":"content_block_start","index":blk,
|
| 595 |
"content_block":{"type":"tool_use","id":tool_state[idx]["id"],
|
| 596 |
"name":tool_state[idx]["name"],"input":{}}}) + "\n\n")
|
| 597 |
-
fn = tc.get("function", {})
|
|
|
|
| 598 |
if args:
|
| 599 |
tool_state[idx]["args"] += args
|
| 600 |
yield (_BLK_DELTA + _dumps({"type":"content_block_delta",
|
|
@@ -603,7 +471,6 @@ async def _stream_chat_to_anthropic(response, requested_model: str):
|
|
| 603 |
except Exception as exc:
|
| 604 |
logger.warning("stream_chat error: %s", exc)
|
| 605 |
|
| 606 |
-
# close blocks
|
| 607 |
if reasoning_block_idx is not None and not reasoning_closed:
|
| 608 |
sig = _synthetic_signature(reasoning_text_acc)
|
| 609 |
yield (_BLK_DELTA + _dumps({"type":"content_block_delta","index":reasoning_block_idx,
|
|
@@ -619,174 +486,146 @@ async def _stream_chat_to_anthropic(response, requested_model: str):
|
|
| 619 |
"usage":{"output_tokens":out_tok}}) + "\n\n")
|
| 620 |
yield f"{_MSG_STOP}" + '{"type":"message_stop"}' + "\n\n"
|
| 621 |
|
| 622 |
-
|
| 623 |
-
async def _stream_messages_passthrough(response, requested_model: str):
|
| 624 |
-
"""
|
| 625 |
-
Anthropic SSE stream → Anthropic SSE stream (passthrough).
|
| 626 |
-
الـ response من /zen/v1/messages هو بالفعل Anthropic SSE format.
|
| 627 |
-
نمرره مباشرة مع تصحيح بسيط لـ model field في message_start.
|
| 628 |
-
"""
|
| 629 |
-
first_chunk = True
|
| 630 |
-
async for line in _iter_sse_lines(response):
|
| 631 |
-
if not line:
|
| 632 |
-
yield "\n"
|
| 633 |
-
continue
|
| 634 |
-
if line.startswith("data:"):
|
| 635 |
-
data = line[5:].strip()
|
| 636 |
-
if data == "[DONE]":
|
| 637 |
-
yield "data: [DONE]\n\n"
|
| 638 |
-
break
|
| 639 |
-
# صحّح model في message_start
|
| 640 |
-
if first_chunk and '"message_start"' in data:
|
| 641 |
-
try:
|
| 642 |
-
obj = _loads(data)
|
| 643 |
-
if obj.get("type") == "message_start":
|
| 644 |
-
obj["message"]["model"] = requested_model
|
| 645 |
-
# أضف signature للـ thinking blocks لو ظهرت لاحقاً
|
| 646 |
-
yield f"event: message_start\ndata: {_dumps(obj)}\n\n"
|
| 647 |
-
first_chunk = False
|
| 648 |
-
continue
|
| 649 |
-
except Exception:
|
| 650 |
-
pass
|
| 651 |
-
first_chunk = False
|
| 652 |
-
# أضف signature لـ thinking block لو غابت
|
| 653 |
-
if '"thinking"' in data and '"signature"' not in data:
|
| 654 |
-
try:
|
| 655 |
-
obj = _loads(data)
|
| 656 |
-
cb = obj.get("content_block") or obj.get("delta") or {}
|
| 657 |
-
if cb.get("type") == "thinking" and not cb.get("signature"):
|
| 658 |
-
cb["signature"] = _synthetic_signature(cb.get("thinking",""))
|
| 659 |
-
data = _dumps(obj)
|
| 660 |
-
except Exception:
|
| 661 |
-
pass
|
| 662 |
-
# استخرج event type من السطر السابق إذا كان موجوداً
|
| 663 |
-
yield f"data: {data}\n\n"
|
| 664 |
-
elif line.startswith("event:"):
|
| 665 |
-
yield f"{line}\n"
|
| 666 |
-
else:
|
| 667 |
-
yield f"{line}\n"
|
| 668 |
-
|
| 669 |
# =========================================================
|
| 670 |
-
#
|
| 671 |
# =========================================================
|
| 672 |
@app.post("/v1/messages")
|
| 673 |
-
|
| 674 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 675 |
request_id = uuid.uuid4().hex[:8]
|
| 676 |
|
| 677 |
-
if not
|
| 678 |
-
return JSONResponse(status_code=401, content={"error": "
|
| 679 |
|
| 680 |
try:
|
| 681 |
body = await request.json()
|
| 682 |
except Exception:
|
| 683 |
-
return JSONResponse(status_code=400, content={"error": "Invalid JSON"})
|
| 684 |
|
| 685 |
-
requested_model = body.get("model", "claude-
|
| 686 |
-
entry
|
| 687 |
-
stream
|
| 688 |
-
thinking_on
|
| 689 |
|
| 690 |
-
anthropic_tools
|
| 691 |
-
openai_tools
|
| 692 |
openai_tool_choice = map_tool_choice(body.get("tool_choice"), bool(openai_tools), thinking_on)
|
| 693 |
|
| 694 |
-
# تحويل messages إلى OpenAI format (مشترك بين الـ pipeline-ين)
|
| 695 |
openai_messages = normalize_messages(body.get("messages", []))
|
| 696 |
-
|
|
|
|
| 697 |
if system_prompt:
|
| 698 |
if isinstance(system_prompt, list):
|
| 699 |
-
system_prompt = "\n".join(
|
| 700 |
-
b.get("text","") if isinstance(b,dict) else str(b) for b in system_prompt)
|
| 701 |
openai_messages.insert(0, {"role": "system", "content": system_prompt})
|
| 702 |
|
| 703 |
-
|
| 704 |
-
|
| 705 |
-
payload = _build_messages_payload(body, openai_messages, openai_tools,
|
| 706 |
-
openai_tool_choice, entry.upstream_id, stream)
|
| 707 |
-
else:
|
| 708 |
-
payload = _build_chat_payload(body, openai_messages, openai_tools,
|
| 709 |
-
openai_tool_choice, entry.upstream_id, stream)
|
| 710 |
|
| 711 |
-
|
| 712 |
|
| 713 |
-
|
| 714 |
-
request_id, requested_model, entry.upstream_id,
|
| 715 |
-
entry.endpoint_type, stream, len(openai_tools), THINKING_MODE)
|
| 716 |
-
|
| 717 |
-
client = request.app.state.client
|
| 718 |
last_error = None
|
|
|
|
| 719 |
|
|
|
|
| 720 |
for attempt in range(1, MAX_RETRIES + 1):
|
| 721 |
if attempt > 1:
|
| 722 |
-
|
| 723 |
-
|
|
|
|
|
|
|
|
|
|
| 724 |
try:
|
|
|
|
|
|
|
| 725 |
if stream:
|
| 726 |
-
req = client.build_request("POST", entry.upstream_url,
|
| 727 |
-
json=payload, headers=headers)
|
| 728 |
resp = await client.send(req, stream=True)
|
| 729 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 730 |
if resp.status_code == 429:
|
| 731 |
-
|
| 732 |
-
raw_err = (await resp.aread()).decode(errors="ignore")
|
| 733 |
-
logger.warning("[%s] 429 retry-after=%.0fs: %s",
|
| 734 |
-
request_id, retry_after, raw_err[:300])
|
| 735 |
await resp.aclose()
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
headers={"retry-after":str(int(retry_after))} if retry_after else {})
|
| 740 |
|
| 741 |
if resp.status_code != 200:
|
| 742 |
raw_err = (await resp.aread()).decode(errors="ignore")
|
| 743 |
-
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
|
| 747 |
-
# ا
|
| 748 |
-
|
| 749 |
-
|
| 750 |
-
|
| 751 |
-
|
| 752 |
-
|
| 753 |
-
|
| 754 |
-
|
| 755 |
-
|
| 756 |
-
|
| 757 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 758 |
resp = await client.post(entry.upstream_url, json=payload, headers=headers)
|
|
|
|
|
|
|
|
|
|
| 759 |
|
| 760 |
if resp.status_code == 429:
|
| 761 |
-
|
| 762 |
-
|
| 763 |
-
|
| 764 |
-
|
| 765 |
-
"message":f"Rate limited. retry-after={retry_after:.0f}s"}},
|
| 766 |
-
headers={"retry-after":str(int(retry_after))} if retry_after else {})
|
| 767 |
|
| 768 |
if resp.status_code != 200:
|
| 769 |
-
|
| 770 |
-
|
| 771 |
-
last_error = resp.text; continue
|
| 772 |
|
| 773 |
raw = resp.content
|
| 774 |
-
|
| 775 |
-
|
| 776 |
-
|
| 777 |
-
|
| 778 |
-
|
| 779 |
-
|
| 780 |
-
|
| 781 |
-
|
| 782 |
-
else:
|
| 783 |
-
result = _chat_to_anthropic(parsed, requested_model)
|
| 784 |
-
|
| 785 |
return JSONResponse(content=result)
|
| 786 |
|
| 787 |
except Exception as exc:
|
| 788 |
last_error = str(exc)
|
| 789 |
-
logger.exception("[%s] exception attempt %d", request_id, attempt)
|
| 790 |
|
| 791 |
-
|
| 792 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
"""
|
| 2 |
+
NVIDIA API Gateway for MiniMax M3 Model (Ultra-Performance & Resilient Edition)
|
| 3 |
+
===============================================================================
|
| 4 |
+
تعديل جراحي محافظ ومقاوم للـ Rate Limits (429). متوافق 100% مع بيئة Claude Code.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
"""
|
| 6 |
|
| 7 |
import os
|
|
|
|
| 10 |
import uuid
|
| 11 |
import asyncio
|
| 12 |
import logging
|
| 13 |
+
import time # مضاف لحساب المقاييس الزمنية (Performance Metrics)
|
| 14 |
from contextlib import asynccontextmanager
|
| 15 |
from dataclasses import dataclass
|
| 16 |
+
from typing import Literal, Optional, Dict, Any, List
|
| 17 |
+
|
| 18 |
+
# A. High-Concurrency Event Loop (uvloop) - دمج جراحي اختياري ومستقل
|
| 19 |
+
try:
|
| 20 |
+
import uvloop
|
| 21 |
+
uvloop.install()
|
| 22 |
+
logging.info("🚀 uvloop installed successfully for event loop optimization.")
|
| 23 |
+
except Exception:
|
| 24 |
+
pass
|
| 25 |
|
| 26 |
try:
|
| 27 |
import orjson as _json_lib
|
|
|
|
| 49 |
level=logging.INFO,
|
| 50 |
format="%(asctime)s [%(levelname)s] %(message)s",
|
| 51 |
)
|
| 52 |
+
logger = logging.getLogger("NVIDIAGateway")
|
| 53 |
|
| 54 |
# =========================================================
|
| 55 |
# CONFIG
|
| 56 |
# =========================================================
|
| 57 |
+
NVIDIA_API_KEY = os.getenv("NVIDIA_API_KEY", "").strip()
|
| 58 |
+
MAX_RETRIES = 3 # تحديث جراحي لرفع عدد المحاولات إلى 3 لمقاومة الـ 429
|
| 59 |
THINKING_MODE = os.getenv("THINKING_MODE", "enabled").strip().lower()
|
| 60 |
|
| 61 |
+
_BASE = "https://integrate.api.nvidia.com/v1"
|
| 62 |
+
URL_CHAT = f"{_BASE}/chat/completions"
|
| 63 |
+
|
| 64 |
+
if not NVIDIA_API_KEY:
|
| 65 |
+
logger.warning("⚠️ NVIDIA_API_KEY environment variable not set. Gateway will refuse requests.")
|
| 66 |
|
| 67 |
# =========================================================
|
| 68 |
# MODEL REGISTRY
|
| 69 |
# =========================================================
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
@dataclass
|
| 71 |
class ModelEntry:
|
| 72 |
+
upstream_id: str
|
| 73 |
+
endpoint_type: Literal["chat"]
|
| 74 |
upstream_url: str
|
| 75 |
|
| 76 |
MODEL_REGISTRY: dict[str, ModelEntry] = {
|
| 77 |
+
"claude-sonnet-4-6": ModelEntry("minimaxai/minimax-m3", "chat", URL_CHAT),
|
| 78 |
+
"claude-opus-4-6": ModelEntry("minimaxai/minimax-m3", "chat", URL_CHAT),
|
| 79 |
+
"claude-sonnet-4-5": ModelEntry("minimaxai/minimax-m3", "chat", URL_CHAT),
|
| 80 |
+
"claude-sonnet-4-7": ModelEntry("minimaxai/minimax-m3", "chat", URL_CHAT),
|
| 81 |
+
"claude-opus-4-7": ModelEntry("minimaxai/minimax-m3", "chat", URL_CHAT),
|
| 82 |
+
"claude-sonnet-4-8": ModelEntry("minimaxai/minimax-m3", "chat", URL_CHAT),
|
| 83 |
+
"claude-3-5-sonnet": ModelEntry("minimaxai/minimax-m3", "chat", URL_CHAT),
|
| 84 |
+
"claude-3-5-sonnet-latest": ModelEntry("minimaxai/minimax-m3", "chat", URL_CHAT),
|
| 85 |
+
"claude-3-7-sonnet": ModelEntry("minimaxai/minimax-m3", "chat", URL_CHAT),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
}
|
| 87 |
|
| 88 |
+
_DEFAULT_ENTRY = ModelEntry("minimaxai/minimax-m3", "chat", URL_CHAT)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
|
| 90 |
def _resolve(requested_model: str) -> ModelEntry:
|
|
|
|
|
|
|
| 91 |
if requested_model in MODEL_REGISTRY:
|
| 92 |
return MODEL_REGISTRY[requested_model]
|
| 93 |
+
if "sonnet" in requested_model.lower():
|
| 94 |
+
return MODEL_REGISTRY["claude-3-5-sonnet"]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
return _DEFAULT_ENTRY
|
| 96 |
|
| 97 |
+
_LEGACY_MAP: dict[str, str] = {
|
| 98 |
+
"nemotron-3-super-free": "minimaxai/minimax-m3",
|
| 99 |
+
"nemotron-3-ultra-free": "minimaxai/minimax-m3",
|
| 100 |
+
"north-mini-code-free": "minimaxai/minimax-m3",
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
_MODELS_CACHE = {
|
| 104 |
"object": "list",
|
| 105 |
"data": [
|
| 106 |
+
{"id": mid, "object": "model", "created": 1710000000, "owned_by": "anthropic"}
|
| 107 |
for mid in MODEL_REGISTRY
|
| 108 |
],
|
| 109 |
}
|
| 110 |
|
| 111 |
# =========================================================
|
| 112 |
+
# NVIDIA API HEADERS
|
| 113 |
# =========================================================
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
def _make_headers(api_key: str) -> dict:
|
| 115 |
return {
|
| 116 |
+
"Content-Type": "application/json",
|
| 117 |
+
"Accept": "application/json",
|
| 118 |
+
"Authorization": f"Bearer {api_key}",
|
| 119 |
+
"User-Agent": "NVIDIA-Gateway/1.0",
|
|
|
|
| 120 |
}
|
| 121 |
|
| 122 |
STREAM_RESPONSE_HEADERS = {
|
| 123 |
+
"Cache-Control": "no-cache, no-store, must-revalidate",
|
| 124 |
"Connection": "keep-alive",
|
| 125 |
"X-Accel-Buffering": "no",
|
| 126 |
+
"Content-Type": "text/event-stream",
|
| 127 |
}
|
| 128 |
|
| 129 |
# =========================================================
|
| 130 |
+
# C. Persistent Connection Pooling (httpx.AsyncClient)
|
| 131 |
# =========================================================
|
| 132 |
@asynccontextmanager
|
| 133 |
async def lifespan(app: FastAPI):
|
| 134 |
+
# تم تعديل max_keepalive_connections جراحياً ومحافظاً إلى 100 لتسريع الـ TTFB
|
| 135 |
app.state.client = httpx.AsyncClient(
|
| 136 |
http2=True,
|
| 137 |
+
timeout=httpx.Timeout(connect=5, read=120, write=30, pool=10),
|
| 138 |
+
limits=httpx.Limits(max_connections=200, max_keepalive_connections=100),
|
| 139 |
follow_redirects=True,
|
| 140 |
)
|
| 141 |
+
logger.info("✅ NVIDIA Claude-Code Gateway started — THINKING_MODE=%s", THINKING_MODE)
|
|
|
|
| 142 |
yield
|
| 143 |
await app.state.client.aclose()
|
| 144 |
|
|
|
|
| 147 |
@app.get("/")
|
| 148 |
@app.head("/")
|
| 149 |
async def root():
|
| 150 |
+
return {"status": "ok", "service": "NVIDIA Gateway for Claude Code (MiniMax M3)"}
|
| 151 |
|
| 152 |
@app.get("/health")
|
| 153 |
async def health():
|
| 154 |
+
return {"status": "healthy", "api_configured": bool(NVIDIA_API_KEY)}
|
| 155 |
|
| 156 |
@app.get("/v1/models")
|
| 157 |
async def get_models():
|
|
|
|
| 174 |
return "synthetic_" + base64.urlsafe_b64encode(digest).decode().rstrip("=")
|
| 175 |
|
| 176 |
# =========================================================
|
| 177 |
+
# CORE LOGIC (Inviolable Functional Constraints - ممنوع التعديل)
|
| 178 |
# =========================================================
|
| 179 |
def normalize_messages(messages: list) -> list:
|
| 180 |
output = []
|
|
|
|
| 195 |
|
| 196 |
for item in content:
|
| 197 |
if not isinstance(item, dict):
|
| 198 |
+
text_parts.append(str(item))
|
| 199 |
+
continue
|
| 200 |
t = item.get("type", "")
|
| 201 |
if t == "text":
|
| 202 |
txt = item.get("text", "")
|
| 203 |
+
if txt:
|
| 204 |
+
text_parts.append(txt)
|
| 205 |
elif t == "thinking":
|
| 206 |
tk = item.get("thinking", "")
|
| 207 |
+
if tk:
|
| 208 |
+
thinking_parts.append(tk)
|
| 209 |
elif t == "tool_use":
|
| 210 |
tool_calls.append({
|
| 211 |
"id": item.get("id", f"call_{uuid.uuid4().hex[:8]}"),
|
|
|
|
| 251 |
output.append(r)
|
| 252 |
return output
|
| 253 |
|
|
|
|
|
|
|
|
|
|
| 254 |
def convert_tools_to_openai(tools: list) -> list:
|
| 255 |
return [{"type":"function","function":{"name":t.get("name",""),
|
| 256 |
"description":t.get("description",""),
|
| 257 |
"parameters":_fix_schema(t.get("input_schema",{}))}} for t in tools] if tools else []
|
| 258 |
|
| 259 |
def map_tool_choice(raw, has_tools: bool, thinking_on: bool):
|
| 260 |
+
if not raw or not has_tools:
|
| 261 |
+
return None
|
| 262 |
if isinstance(raw, dict):
|
| 263 |
kind = raw.get("type","auto")
|
| 264 |
if thinking_on and kind in ("any","tool"):
|
|
|
|
| 265 |
return "auto"
|
| 266 |
+
if kind == "any":
|
| 267 |
+
return "required"
|
| 268 |
+
if kind == "tool":
|
| 269 |
+
return {"type":"function","function":{"name":raw.get("name","")}}
|
| 270 |
return kind
|
| 271 |
if thinking_on and raw in ("any","required"):
|
| 272 |
return "auto"
|
| 273 |
return raw
|
| 274 |
|
|
|
|
|
|
|
|
|
|
| 275 |
def _build_chat_payload(body, messages, tools, tool_choice, upstream_model, stream):
|
|
|
|
|
|
|
| 276 |
payload = {
|
| 277 |
"model": upstream_model,
|
| 278 |
"messages": messages,
|
| 279 |
+
"max_tokens": min(body.get("max_tokens", 4096), 64000),
|
| 280 |
"stream": stream,
|
|
|
|
| 281 |
}
|
| 282 |
+
if "temperature" in body:
|
| 283 |
payload["temperature"] = body.get("temperature", 0.7)
|
|
|
|
|
|
|
| 284 |
if tools:
|
| 285 |
payload["tools"] = tools
|
| 286 |
if tool_choice is not None:
|
| 287 |
payload["tool_choice"] = tool_choice
|
| 288 |
return payload
|
| 289 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 290 |
_FINISH_MAP = {"tool_calls": "tool_use", "length": "max_tokens"}
|
| 291 |
|
| 292 |
def _chat_to_anthropic(parsed: dict, requested_model: str) -> dict:
|
|
|
|
| 293 |
content_blocks = []
|
| 294 |
stop_reason = "end_turn"
|
| 295 |
try:
|
|
|
|
| 308 |
content_blocks.append({"type": "text", "text": text})
|
| 309 |
for tc in (message.get("tool_calls") or []):
|
| 310 |
fn = tc.get("function", {})
|
| 311 |
+
try:
|
| 312 |
+
parsed_args = _loads(fn.get("arguments", "{}"))
|
| 313 |
+
except:
|
| 314 |
+
parsed_args = {"raw": fn.get("arguments", "")}
|
| 315 |
content_blocks.append({
|
| 316 |
"type": "tool_use",
|
| 317 |
"id": tc.get("id", f"toolu_{uuid.uuid4().hex[:8]}"),
|
|
|
|
| 334 |
"output_tokens": usage.get("completion_tokens", 0)},
|
| 335 |
}
|
| 336 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
# =========================================================
|
| 338 |
+
# STREAM CONSTANTS & PARSER
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 339 |
# =========================================================
|
| 340 |
_MSG_START = "event: message_start\ndata: "
|
| 341 |
_BLK_START = "event: content_block_start\ndata: "
|
|
|
|
| 344 |
_MSG_DELTA = "event: message_delta\ndata: "
|
| 345 |
_MSG_STOP = "event: message_stop\ndata: "
|
| 346 |
|
| 347 |
+
# B. Memory-Efficient SSE Buffering (bytearray) - حقن جراحي
|
| 348 |
+
async def _iter_sse_lines(response):
|
| 349 |
+
buf = bytearray() # تهيئة مصفوفة بايتات ديناميكية مرنة
|
| 350 |
+
async for chunk in response.aiter_bytes():
|
| 351 |
+
if not chunk:
|
| 352 |
+
continue
|
| 353 |
+
buf.extend(chunk) # إلحاق البيانات بدون مجهود نسخ السلاسل الثابتة
|
| 354 |
+
while True:
|
| 355 |
+
nl = buf.find(b"\n")
|
| 356 |
+
if nl == -1:
|
| 357 |
+
break
|
| 358 |
+
line = buf[:nl].rstrip(b"\r").decode("utf-8", errors="replace")
|
| 359 |
+
del buf[:nl + 1] # تفريغ جراحي فوري للذاكرة المقروءة للسطر الحالي
|
| 360 |
+
yield line
|
| 361 |
+
|
| 362 |
async def _stream_chat_to_anthropic(response, requested_model: str):
|
|
|
|
| 363 |
msg_id = f"msg_{uuid.uuid4().hex}"
|
| 364 |
in_tok = out_tok = 0
|
| 365 |
|
|
|
|
| 369 |
"usage":{"input_tokens":0,"output_tokens":0}}}) + "\n\n")
|
| 370 |
yield 'event: ping\ndata: {"type":"ping"}\n\n'
|
| 371 |
|
| 372 |
+
tool_state = {}
|
| 373 |
+
tool_block_idx = {}
|
| 374 |
+
next_idx = 0
|
| 375 |
+
reasoning_block_idx = None
|
| 376 |
+
reasoning_text_acc = ""
|
| 377 |
+
reasoning_closed = False
|
| 378 |
+
text_block_idx = None
|
| 379 |
+
stop_reason = "end_turn"
|
| 380 |
|
| 381 |
try:
|
| 382 |
async for line in _iter_sse_lines(response):
|
| 383 |
+
if not line or not line.startswith("data:"):
|
| 384 |
+
continue
|
| 385 |
data = line[5:].strip()
|
| 386 |
+
if data == "[DONE]":
|
| 387 |
+
break
|
| 388 |
+
try:
|
| 389 |
+
chunk = _loads(data)
|
| 390 |
+
except:
|
| 391 |
+
continue
|
| 392 |
|
| 393 |
if chunk.get("usage"):
|
| 394 |
u = chunk["usage"]
|
|
|
|
| 396 |
out_tok = u.get("completion_tokens", out_tok)
|
| 397 |
|
| 398 |
choices = chunk.get("choices")
|
| 399 |
+
if not choices:
|
| 400 |
+
continue
|
| 401 |
+
choice = choices[0]
|
| 402 |
+
delta = choice.get("delta", {})
|
| 403 |
fr = choice.get("finish_reason")
|
| 404 |
+
if fr == "tool_calls":
|
| 405 |
+
stop_reason = "tool_use"
|
| 406 |
+
elif fr == "length":
|
| 407 |
+
stop_reason = "max_tokens"
|
| 408 |
|
|
|
|
| 409 |
rd = delta.get("reasoning_content")
|
| 410 |
if rd:
|
| 411 |
reasoning_text_acc += rd
|
| 412 |
if reasoning_block_idx is None:
|
| 413 |
+
reasoning_block_idx = next_idx
|
| 414 |
+
next_idx += 1
|
| 415 |
yield (_BLK_START + _dumps({"type":"content_block_start",
|
| 416 |
"index":reasoning_block_idx,
|
| 417 |
"content_block":{"type":"thinking","thinking":""}}) + "\n\n")
|
|
|
|
| 419 |
"index":reasoning_block_idx,
|
| 420 |
"delta":{"type":"thinking_delta","thinking":rd}}) + "\n\n")
|
| 421 |
|
|
|
|
| 422 |
td = delta.get("content")
|
| 423 |
if td:
|
| 424 |
if reasoning_block_idx is not None and not reasoning_closed:
|
|
|
|
| 430 |
"index":reasoning_block_idx}) + "\n\n")
|
| 431 |
reasoning_closed = True
|
| 432 |
if text_block_idx is None:
|
| 433 |
+
text_block_idx = next_idx
|
| 434 |
+
next_idx += 1
|
| 435 |
yield (_BLK_START + _dumps({"type":"content_block_start",
|
| 436 |
"index":text_block_idx,
|
| 437 |
"content_block":{"type":"text","text":""}}) + "\n\n")
|
|
|
|
| 439 |
"index":text_block_idx,
|
| 440 |
"delta":{"type":"text_delta","text":td}}) + "\n\n")
|
| 441 |
|
|
|
|
| 442 |
tcd = delta.get("tool_calls")
|
| 443 |
if tcd:
|
| 444 |
if reasoning_block_idx is not None and not reasoning_closed:
|
|
|
|
| 455 |
fn0 = tc.get("function", {})
|
| 456 |
tool_state[idx] = {"id":tc.get("id",f"toolu_{uuid.uuid4().hex[:8]}"),
|
| 457 |
"name":fn0.get("name",""),"args":""}
|
| 458 |
+
blk = next_idx
|
| 459 |
+
next_idx += 1
|
| 460 |
+
tool_block_idx[idx] = blk
|
| 461 |
yield (_BLK_START + _dumps({"type":"content_block_start","index":blk,
|
| 462 |
"content_block":{"type":"tool_use","id":tool_state[idx]["id"],
|
| 463 |
"name":tool_state[idx]["name"],"input":{}}}) + "\n\n")
|
| 464 |
+
fn = tc.get("function", {})
|
| 465 |
+
args = fn.get("arguments","")
|
| 466 |
if args:
|
| 467 |
tool_state[idx]["args"] += args
|
| 468 |
yield (_BLK_DELTA + _dumps({"type":"content_block_delta",
|
|
|
|
| 471 |
except Exception as exc:
|
| 472 |
logger.warning("stream_chat error: %s", exc)
|
| 473 |
|
|
|
|
| 474 |
if reasoning_block_idx is not None and not reasoning_closed:
|
| 475 |
sig = _synthetic_signature(reasoning_text_acc)
|
| 476 |
yield (_BLK_DELTA + _dumps({"type":"content_block_delta","index":reasoning_block_idx,
|
|
|
|
| 486 |
"usage":{"output_tokens":out_tok}}) + "\n\n")
|
| 487 |
yield f"{_MSG_STOP}" + '{"type":"message_stop"}' + "\n\n"
|
| 488 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 489 |
# =========================================================
|
| 490 |
+
# THE ANTHROPIC COMPATIBLE ENDPOINT (/v1/messages)
|
| 491 |
# =========================================================
|
| 492 |
@app.post("/v1/messages")
|
| 493 |
+
async def anthropic_messages(request: Request, x_api_key: Optional[str] = Header(default=None)):
|
| 494 |
+
t_request_start = time.time() # مقياس: المدة الإجمالية للدورة البرمجية للطلب
|
| 495 |
+
t_upstream_start = 0.0
|
| 496 |
+
t_upstream_duration = 0.0
|
| 497 |
+
t_stream_start = 0.0
|
| 498 |
+
|
| 499 |
request_id = uuid.uuid4().hex[:8]
|
| 500 |
|
| 501 |
+
if not NVIDIA_API_KEY:
|
| 502 |
+
return JSONResponse(status_code=401, content={"error": {"message": "NVIDIA_API_KEY missing on Gateway", "type": "authentication_error"}})
|
| 503 |
|
| 504 |
try:
|
| 505 |
body = await request.json()
|
| 506 |
except Exception:
|
| 507 |
+
return JSONResponse(status_code=400, content={"error": "Invalid JSON payload"})
|
| 508 |
|
| 509 |
+
requested_model = body.get("model", "claude-3-5-sonnet")
|
| 510 |
+
entry = _resolve(requested_model)
|
| 511 |
+
stream = bool(body.get("stream", False))
|
| 512 |
+
thinking_on = THINKING_MODE == "enabled"
|
| 513 |
|
| 514 |
+
anthropic_tools = body.get("tools", [])
|
| 515 |
+
openai_tools = convert_tools_to_openai(anthropic_tools)
|
| 516 |
openai_tool_choice = map_tool_choice(body.get("tool_choice"), bool(openai_tools), thinking_on)
|
| 517 |
|
|
|
|
| 518 |
openai_messages = normalize_messages(body.get("messages", []))
|
| 519 |
+
|
| 520 |
+
system_prompt = body.get("system", "")
|
| 521 |
if system_prompt:
|
| 522 |
if isinstance(system_prompt, list):
|
| 523 |
+
system_prompt = "\n".join(b.get("text","") if isinstance(b,dict) else str(b) for b in system_prompt)
|
|
|
|
| 524 |
openai_messages.insert(0, {"role": "system", "content": system_prompt})
|
| 525 |
|
| 526 |
+
payload = _build_chat_payload(body, openai_messages, openai_tools, openai_tool_choice, entry.upstream_id, stream)
|
| 527 |
+
headers = _make_headers(NVIDIA_API_KEY)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 528 |
|
| 529 |
+
logger.info("[%s] Claude Code Inbound: %s -> %s | stream=%s", request_id, requested_model, entry.upstream_id, stream)
|
| 530 |
|
| 531 |
+
client = request.app.state.client
|
|
|
|
|
|
|
|
|
|
|
|
|
| 532 |
last_error = None
|
| 533 |
+
retry_count = 0 # مقياس: عداد تتبع حلقة المحاولات
|
| 534 |
|
| 535 |
+
# D. Rate-Limit Resilience & Intelligent Backoff
|
| 536 |
for attempt in range(1, MAX_RETRIES + 1):
|
| 537 |
if attempt > 1:
|
| 538 |
+
retry_count += 1
|
| 539 |
+
# حساب زمن الـ Exponential Backoff الخفيف (الطلبات الناجحة فوراً تواجه 0 ثانية تأخير)
|
| 540 |
+
backoff_delay = 0.3 * (2 ** (attempt - 2))
|
| 541 |
+
await asyncio.sleep(backoff_delay)
|
| 542 |
+
|
| 543 |
try:
|
| 544 |
+
t_upstream_start = time.time() # مقياس: بداية وقت معالجة NVIDIA
|
| 545 |
+
|
| 546 |
if stream:
|
| 547 |
+
req = client.build_request("POST", entry.upstream_url, json=payload, headers=headers)
|
|
|
|
| 548 |
resp = await client.send(req, stream=True)
|
| 549 |
|
| 550 |
+
# تسجيل الـ HTTP Status من أجل التحليل الهيكلي المستقبلي
|
| 551 |
+
logger.info("[%s] NVIDIA STATUS=%s | Attempt=%d", request_id, resp.status_code, attempt)
|
| 552 |
+
|
| 553 |
+
# إذا واجه الكود الـ 429، نطبع الـ Headers للفحص جراحياً ولا نكسر اتصال المستخدم بل نكرر المحاولة
|
| 554 |
if resp.status_code == 429:
|
| 555 |
+
logger.warning("[%s] ⚠️ NVIDIA Rate Limit hit. Headers: %s", request_id, dict(resp.headers))
|
|
|
|
|
|
|
|
|
|
| 556 |
await resp.aclose()
|
| 557 |
+
if attempt < MAX_RETRIES:
|
| 558 |
+
continue # القفز فوراً للمحاولة التالية لإنقاذ الـ Session
|
| 559 |
+
return JSONResponse(status_code=429, content={"error": {"type": "rate_limit_error", "message": "NVIDIA API Rate Limit exhausted after retries"}})
|
|
|
|
| 560 |
|
| 561 |
if resp.status_code != 200:
|
| 562 |
raw_err = (await resp.aread()).decode(errors="ignore")
|
| 563 |
+
last_error = raw_err
|
| 564 |
+
await resp.aclose()
|
| 565 |
+
continue
|
| 566 |
+
|
| 567 |
+
t_upstream_duration = time.time() - t_upstream_start # حساب الـ TTFB
|
| 568 |
+
t_stream_start = time.time() # مقياس: بداية وقت انطلاق بث التوكنات للمستخدم
|
| 569 |
+
|
| 570 |
+
async def tracked_stream_generator():
|
| 571 |
+
try:
|
| 572 |
+
async for chunk in _stream_chat_to_anthropic(resp, requested_model):
|
| 573 |
+
yield chunk
|
| 574 |
+
finally:
|
| 575 |
+
t_stream_duration = time.time() - t_stream_start
|
| 576 |
+
t_total_duration = time.time() - t_request_start
|
| 577 |
+
# E. Non-Invasive Performance Metrics - تسجيل المقاييس الزمنية دون تسريب البيانات الحساسة
|
| 578 |
+
logger.info(
|
| 579 |
+
"[%s] Metrics -> total_duration: %.3fs | upstream_ttfb: %.3fs | stream_duration: %.3fs | retry_count: %d",
|
| 580 |
+
request_id, t_total_duration, t_upstream_duration, t_stream_duration, retry_count
|
| 581 |
+
)
|
| 582 |
+
|
| 583 |
+
return StreamingResponse(
|
| 584 |
+
tracked_stream_generator(),
|
| 585 |
+
media_type="text/event-stream",
|
| 586 |
+
headers=STREAM_RESPONSE_HEADERS,
|
| 587 |
+
background=BackgroundTask(resp.aclose)
|
| 588 |
+
)
|
| 589 |
+
|
| 590 |
+
# Non-stream block
|
| 591 |
resp = await client.post(entry.upstream_url, json=payload, headers=headers)
|
| 592 |
+
t_upstream_duration = time.time() - t_upstream_start
|
| 593 |
+
|
| 594 |
+
logger.info("[%s] NVIDIA STATUS=%s | Attempt=%d (Non-Stream)", request_id, resp.status_code, attempt)
|
| 595 |
|
| 596 |
if resp.status_code == 429:
|
| 597 |
+
logger.warning("[%s] ⚠️ NVIDIA Rate Limit hit (Non-Stream). Headers: %s", request_id, dict(resp.headers))
|
| 598 |
+
if attempt < MAX_RETRIES:
|
| 599 |
+
continue
|
| 600 |
+
return JSONResponse(status_code=429, content={"error": {"type": "rate_limit_error", "message": "NVIDIA API Rate Limit exhausted after retries"}})
|
|
|
|
|
|
|
| 601 |
|
| 602 |
if resp.status_code != 200:
|
| 603 |
+
last_error = resp.text
|
| 604 |
+
continue
|
|
|
|
| 605 |
|
| 606 |
raw = resp.content
|
| 607 |
+
parsed = _loads(raw)
|
| 608 |
+
result = _chat_to_anthropic(parsed, requested_model)
|
| 609 |
+
|
| 610 |
+
t_total_duration = time.time() - t_request_start
|
| 611 |
+
logger.info(
|
| 612 |
+
"[%s] Metrics -> total_duration: %.3fs | upstream_duration: %.3fs | stream_duration: N/A | retry_count: %d",
|
| 613 |
+
request_id, t_total_duration, t_upstream_duration, retry_count
|
| 614 |
+
)
|
|
|
|
|
|
|
|
|
|
| 615 |
return JSONResponse(content=result)
|
| 616 |
|
| 617 |
except Exception as exc:
|
| 618 |
last_error = str(exc)
|
|
|
|
| 619 |
|
| 620 |
+
t_total_duration = time.time() - t_request_start
|
| 621 |
+
logger.error("[%s] Request Failed -> total_duration: %.3fs | retry_count: %d | error: %s", request_id, t_total_duration, retry_count, last_error)
|
| 622 |
+
return JSONResponse(status_code=502, content={"error": "Upstream NVIDIA failed", "details": last_error})
|
| 623 |
+
|
| 624 |
+
@app.post("/v1/chat/completions")
|
| 625 |
+
async def chat_completions(request: Request):
|
| 626 |
+
return await anthropic_messages(request)
|
| 627 |
+
|
| 628 |
+
if __name__ == "__main__":
|
| 629 |
+
import uvicorn
|
| 630 |
+
port = int(os.getenv("PORT", "8000"))
|
| 631 |
+
uvicorn.run(app, host="0.0.0.0", port=port, log_level="info")
|