Upload 4 files

app/config.py

@@ -0,0 +1,38 @@
+import os
+
+
+def _int(name: str, default: int) -> int:
+    return int(os.environ.get(name, default))
+
+
+def _float(name: str, default: float) -> float:
+    return float(os.environ.get(name, default))
+
+
+MODEL_REPO = os.environ.get("MODEL_REPO", "tencent/Hy-MT2-7B-GGUF")
+MODEL_FILE = os.environ.get("MODEL_FILE", "Hy-MT2-7B-Q4_K_M.gguf")
+MODEL_DIR = os.environ.get("MODEL_DIR", "/app/models")
+
+LLAMA_SERVER_HOST = os.environ.get("LLAMA_SERVER_HOST", "127.0.0.1")
+LLAMA_SERVER_PORT = _int("LLAMA_SERVER_PORT", 8080)
+LLAMA_SERVER_URL = f"http://{LLAMA_SERVER_HOST}:{LLAMA_SERVER_PORT}"
+
+# Matches Hy-MT2 1.8B/7B recommended sampling params from the model card.
+TEMPERATURE = _float("TEMPERATURE", 0.7)
+TOP_P = _float("TOP_P", 0.6)
+TOP_K = _int("TOP_K", 20)
+REPEAT_PENALTY = _float("REPEAT_PENALTY", 1.05)
+MAX_TOKENS = _int("MAX_TOKENS", 512)
+
+# --parallel value the llama-server was started with; also used to bound
+# how many translation requests this gateway keeps in flight at once so
+# requests queue instead of overwhelming the server's slots.
+PARALLEL_SLOTS = _int("PARALLEL_SLOTS", 8)
+
+MAX_BATCH_SIZE = _int("MAX_BATCH_SIZE", 200)
+REQUEST_TIMEOUT_SECONDS = _float("REQUEST_TIMEOUT_SECONDS", 120.0)
+
+# Optional shared-secret auth for the public gateway. Empty = disabled.
+API_KEY = os.environ.get("API_KEY", "")
+
+PORT = _int("PORT", 7860)

app/llama_client.py

@@ -0,0 +1,75 @@
+"""Thin async client for the internal llama-server instance.
+
+llama-server is started with --parallel N --cont-batching (see entrypoint.sh),
+which makes it batch multiple in-flight requests' forward passes together
+instead of running them one at a time. This is where the actual CPU
+efficiency gain for many short strings comes from: we just need to keep
+enough requests in flight concurrently, and llama.cpp does the batching.
+"""
+
+import asyncio
+
+import httpx
+
+from . import config
+
+_client: httpx.AsyncClient | None = None
+# Bounds how many requests we hand to llama-server at once. Slightly above
+# PARALLEL_SLOTS would over-queue on the server side instead of here; we
+# keep it equal so backpressure is visible to callers via queueing time.
+_semaphore = asyncio.Semaphore(config.PARALLEL_SLOTS)
+
+
+def get_client() -> httpx.AsyncClient:
+    global _client
+    if _client is None:
+        _client = httpx.AsyncClient(
+            base_url=config.LLAMA_SERVER_URL,
+            timeout=httpx.Timeout(config.REQUEST_TIMEOUT_SECONDS),
+        )
+    return _client
+
+
+async def aclose() -> None:
+    global _client
+    if _client is not None:
+        await _client.aclose()
+        _client = None
+
+
+class LlamaServerError(RuntimeError):
+    pass
+
+
+async def chat_complete(prompt: str) -> str:
+    client = get_client()
+    payload = {
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": config.TEMPERATURE,
+        "top_p": config.TOP_P,
+        "top_k": config.TOP_K,
+        "repeat_penalty": config.REPEAT_PENALTY,
+        "max_tokens": config.MAX_TOKENS,
+        "stream": False,
+    }
+    async with _semaphore:
+        try:
+            resp = await client.post("/v1/chat/completions", json=payload)
+            resp.raise_for_status()
+        except httpx.HTTPError as exc:
+            raise LlamaServerError(str(exc)) from exc
+
+    data = resp.json()
+    try:
+        return data["choices"][0]["message"]["content"].strip()
+    except (KeyError, IndexError) as exc:
+        raise LlamaServerError(f"Unexpected llama-server response: {data}") from exc
+
+
+async def health() -> bool:
+    client = get_client()
+    try:
+        resp = await client.get("/health", timeout=5.0)
+        return resp.status_code == 200
+    except httpx.HTTPError:
+        return False

app/main.py

@@ -0,0 +1,147 @@
+import asyncio
+import time
+
+from fastapi import Depends, FastAPI, Header, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel, Field
+
+from . import config, llama_client
+from .translation import LANGUAGES, build_prompt
+
+app = FastAPI(title="Hy-MT2 Translation API", version="1.0.0")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+async def verify_api_key(x_api_key: str | None = Header(default=None)) -> None:
+    if config.API_KEY and x_api_key != config.API_KEY:
+        raise HTTPException(status_code=401, detail="Invalid or missing X-API-Key header")
+
+
+class TranslateRequest(BaseModel):
+    text: str
+    target_lang: str
+    source_lang: str | None = None
+    style: str | None = None
+    glossary: dict[str, str] | None = None
+    preserve_placeholders: bool = True
+
+
+class BatchTranslateRequest(BaseModel):
+    texts: list[str] = Field(..., min_length=1)
+    target_lang: str
+    source_lang: str | None = None
+    style: str | None = None
+    glossary: dict[str, str] | None = None
+    preserve_placeholders: bool = True
+
+
+class TranslateResponse(BaseModel):
+    translation: str
+    elapsed_seconds: float
+
+
+class BatchItem(BaseModel):
+    original: str
+    translation: str | None
+    error: str | None = None
+
+
+class BatchTranslateResponse(BaseModel):
+    translations: list[BatchItem]
+    count: int
+    elapsed_seconds: float
+    items_per_second: float | None
+
+
+@app.get("/health")
+async def health_check():
+    backend_ok = await llama_client.health()
+    return {
+        "status": "ok" if backend_ok else "starting",
+        "llama_server": backend_ok,
+        "parallel_slots": config.PARALLEL_SLOTS,
+        "model_file": config.MODEL_FILE,
+    }
+
+
+@app.get("/languages")
+async def languages():
+    return LANGUAGES
+
+
+@app.post("/translate", response_model=TranslateResponse, dependencies=[Depends(verify_api_key)])
+async def translate(req: TranslateRequest):
+    prompt = build_prompt(
+        req.text,
+        req.target_lang,
+        req.source_lang,
+        req.style,
+        req.glossary,
+        req.preserve_placeholders,
+    )
+    start = time.perf_counter()
+    try:
+        translation = await llama_client.chat_complete(prompt)
+    except llama_client.LlamaServerError as exc:
+        raise HTTPException(status_code=502, detail=str(exc)) from exc
+    elapsed = time.perf_counter() - start
+    return TranslateResponse(translation=translation, elapsed_seconds=round(elapsed, 4))
+
+
+@app.post(
+    "/translate/batch",
+    response_model=BatchTranslateResponse,
+    dependencies=[Depends(verify_api_key)],
+)
+async def translate_batch(req: BatchTranslateRequest):
+    if len(req.texts) > config.MAX_BATCH_SIZE:
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                f"Batch too large ({len(req.texts)} items). "
+                f"Split into chunks of at most {config.MAX_BATCH_SIZE} "
+                "(50-100 is a good size for many short strings)."
+            ),
+        )
+
+    async def _translate_one(text: str):
+        prompt = build_prompt(
+            text,
+            req.target_lang,
+            req.source_lang,
+            req.style,
+            req.glossary,
+            req.preserve_placeholders,
+        )
+        return await llama_client.chat_complete(prompt)
+
+    start = time.perf_counter()
+    results = await asyncio.gather(
+        *(_translate_one(t) for t in req.texts), return_exceptions=True
+    )
+    elapsed = time.perf_counter() - start
+
+    items: list[BatchItem] = []
+    for original, result in zip(req.texts, results):
+        if isinstance(result, Exception):
+            items.append(BatchItem(original=original, translation=None, error=str(result)))
+        else:
+            items.append(BatchItem(original=original, translation=result))
+
+    return BatchTranslateResponse(
+        translations=items,
+        count=len(req.texts),
+        elapsed_seconds=round(elapsed, 4),
+        items_per_second=round(len(req.texts) / elapsed, 2) if elapsed > 0 else None,
+    )
+
+
+@app.on_event("shutdown")
+async def shutdown():
+    await llama_client.aclose()

app/translation.py

@@ -0,0 +1,102 @@
+"""Prompt construction for Hy-MT2, following the instruction formats documented
+on https://huggingface.co/tencent/Hy-MT2-7B-GGUF ("Default Translation",
+"Terminology", "Style" and "Structured Data" instruction variants).
+"""
+
+# The 33 languages Hy-MT2 is documented to support, code -> English name.
+LANGUAGES = {
+    "zh": "Chinese",
+    "zh-hant": "Traditional Chinese",
+    "yue": "Cantonese",
+    "en": "English",
+    "fr": "French",
+    "pt": "Portuguese",
+    "es": "Spanish",
+    "ja": "Japanese",
+    "tr": "Turkish",
+    "ru": "Russian",
+    "ar": "Arabic",
+    "ko": "Korean",
+    "th": "Thai",
+    "it": "Italian",
+    "de": "German",
+    "vi": "Vietnamese",
+    "ms": "Malay",
+    "id": "Indonesian",
+    "tl": "Filipino",
+    "hi": "Hindi",
+    "pl": "Polish",
+    "cs": "Czech",
+    "nl": "Dutch",
+    "km": "Khmer",
+    "my": "Burmese",
+    "fa": "Persian",
+    "gu": "Gujarati",
+    "ur": "Urdu",
+    "te": "Telugu",
+    "mr": "Marathi",
+    "he": "Hebrew",
+    "bn": "Bengali",
+    "ta": "Tamil",
+    "uk": "Ukrainian",
+    "bo": "Tibetan",
+    "kk": "Kazakh",
+    "mn": "Mongolian",
+    "ug": "Uyghur",
+}
+
+# Placeholder styles commonly found in game strings (RPG Maker control codes,
+# printf-style format specifiers, ICU/handlebars-style variables) that the
+# model must copy through untouched rather than translate.
+_PLACEHOLDER_HINT = (
+    "You must NEVER translate or alter placeholders and control codes such as "
+    r"\N[..], \V[..], \C[..], \I[..], {variable}, ${variable}, %s, %d, %1, {{var}} "
+    "or similar tokens. Keep them exactly as they appear, in the same position."
+)
+
+
+def resolve_language(code_or_name: str) -> str:
+    """Map an ISO-ish code to its English name; pass through unknown values
+    (so callers can already supply a plain language name)."""
+    return LANGUAGES.get(code_or_name.strip().lower(), code_or_name)
+
+
+def build_prompt(
+    text: str,
+    target_lang: str,
+    source_lang: str | None = None,
+    style: str | None = None,
+    glossary: dict[str, str] | None = None,
+    preserve_placeholders: bool = True,
+) -> str:
+    target_name = resolve_language(target_lang)
+
+    header = (
+        f"Translate the following text into {target_name}. "
+        "Note that you should only output the translated result "
+        "without any additional explanation"
+    )
+    if source_lang:
+        source_name = resolve_language(source_lang)
+        header += f". The source text is in {source_name}"
+    header += "."
+
+    parts = [header]
+
+    if style:
+        parts.append(
+            f"Note that the translation style must strictly conform to [{style}]."
+        )
+
+    if glossary:
+        ref_lines = "\n".join(f"{k} translates to {v}" for k, v in glossary.items())
+        parts.insert(
+            0,
+            "Reference the following translations:\n" + ref_lines,
+        )
+
+    if preserve_placeholders:
+        parts.append(_PLACEHOLDER_HINT)
+
+    instruction = "\n".join(parts)
+    return f"{instruction}\n\n{text}"