Upload 4 files

Dockerfile

@@ -0,0 +1,79 @@
+# syntax=docker/dockerfile:1
+
+# ---------------------------------------------------------------------------
+# Stage 1: build llama-server from source.
+#
+# IMPORTANT: tencent/Hy-MT2-7B-GGUF depends on a custom "STQ" ggml kernel
+# that shipped in https://github.com/ggml-org/llama.cpp/pull/22836. A pip
+# install of llama-cpp-python pins an older llama.cpp commit and will very
+# likely NOT be able to load this GGUF (unknown tensor type / load error).
+# Building llama.cpp from source here (default: the `main` branch, which
+# should have the PR merged by now) avoids that. If it hasn't landed yet,
+# rebuild with:
+#   docker build --build-arg LLAMA_CPP_REF=refs/pull/22836/head -t hy-mt2-api .
+# ---------------------------------------------------------------------------
+FROM debian:bookworm-slim AS builder
+
+ARG LLAMA_CPP_REF=main
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential cmake git ca-certificates libcurl4-openssl-dev pkg-config \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+RUN git clone https://github.com/ggml-org/llama.cpp.git . \
+    && if [ "$LLAMA_CPP_REF" != "main" ]; then \
+         git fetch origin "$LLAMA_CPP_REF" && git checkout FETCH_HEAD; \
+       fi
+
+# GGML_NATIVE=OFF because the Docker build machine's CPU may differ from the
+# machine the image eventually runs on; GGML_AVX2=ON targets the common
+# baseline for modern cloud/Space CPUs. Drop it if you hit "illegal
+# instruction" at runtime on older hardware.
+RUN cmake -B build \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DLLAMA_CURL=OFF \
+      -DGGML_NATIVE=OFF \
+      -DGGML_AVX2=ON \
+    && cmake --build build --config Release -j"$(nproc)" --target llama-server
+
+# ---------------------------------------------------------------------------
+# Stage 2: runtime image (FastAPI gateway + llama-server binary)
+# ---------------------------------------------------------------------------
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libgomp1 libcurl4 curl \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY --from=builder /build/build/bin/llama-server /app/llama-server
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY app ./app
+COPY entrypoint.sh .
+RUN chmod +x /app/llama-server /app/entrypoint.sh
+
+ENV MODEL_REPO=tencent/Hy-MT2-7B-GGUF \
+    MODEL_FILE=Hy-MT2-7B-Q4_K_M.gguf \
+    MODEL_DIR=/app/models \
+    LLAMA_SERVER_HOST=127.0.0.1 \
+    LLAMA_SERVER_PORT=8080 \
+    THREADS=8 \
+    PARALLEL_SLOTS=8 \
+    CTX_SIZE=16384 \
+    MAX_TOKENS=512 \
+    TEMPERATURE=0.7 \
+    TOP_P=0.6 \
+    TOP_K=20 \
+    REPEAT_PENALTY=1.05 \
+    MAX_BATCH_SIZE=200 \
+    API_KEY="" \
+    PORT=7860
+
+EXPOSE 7860
+
+CMD ["./entrypoint.sh"]

README.md

@@ -1,10 +1,130 @@
 ---
-title: Mtapi
-emoji: 🐠
-colorFrom: pink
-colorTo: blue
+title: Hy-MT2 Translation API
+emoji: 🌐
+colorFrom: blue
+colorTo: purple
 sdk: docker
+app_port: 7860
 pinned: false
+license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Hy-MT2 Translation API
+
+A batching-optimized translation API for [tencent/Hy-MT2-7B-GGUF](https://huggingface.co/tencent/Hy-MT2-7B-GGUF),
+built to run as a Hugging Face Space Docker app on an 8 vCPU / 32 GB RAM
+instance.
+
+## How it works
+
+- **Inference engine**: `llama-server` (from [llama.cpp](https://github.com/ggml-org/llama.cpp)),
+  compiled from source in the Docker build (see the note in the `Dockerfile`
+  about the STQ kernel this GGUF depends on — it needs a recent llama.cpp).
+  It's started with `--threads 8 --parallel 8 --cont-batching`, which uses
+  llama.cpp's continuous batching to combine multiple in-flight requests'
+  forward passes into shared compute instead of running them one at a time.
+  This is the real mechanism behind the "batch instead of one-by-one"
+  optimization you asked for — the 8 cores get fed one bigger batched matmul
+  per step instead of 8 independent tiny ones.
+- **Gateway**: a small FastAPI app (`app/main.py`) in front of it, which
+  builds the Hy-MT2 instruction-format prompt (see `app/translation.py`) for
+  each string and fans batch requests out concurrently to `llama-server`,
+  bounded by `PARALLEL_SLOTS` so requests queue cleanly instead of
+  overloading the model.
+- Chat formatting uses the GGUF's embedded Jinja chat template
+  (`llama-server --jinja`), matching the model card's documented usage.
+
+## Endpoints
+
+### `GET /health`
+Backend status check.
+
+### `GET /languages`
+Returns the map of supported language codes → names (the 33 languages
+Hy-MT2 documents support for).
+
+### `POST /translate` — single string
+```json
+{
+  "text": "Where is the nearest potion shop?",
+  "target_lang": "tr",
+  "preserve_placeholders": true
+}
+```
+```json
+{ "translation": "En yakın iksir dükkanı nerede?", "elapsed_seconds": 0.41 }
+```
+
+### `POST /translate/batch` — many strings in one call (recommended: 50-100 per call)
+```json
+{
+  "texts": ["Hello!", "Welcome!", "Potion", "Attack"],
+  "target_lang": "tr"
+}
+```
+```json
+{
+  "translations": [
+    { "original": "Hello!", "translation": "Merhaba!" },
+    { "original": "Welcome!", "translation": "Hoş geldin!" },
+    { "original": "Potion", "translation": "İksir" },
+    { "original": "Attack", "translation": "Saldırı" }
+  ],
+  "count": 4,
+  "elapsed_seconds": 0.9,
+  "items_per_second": 4.44
+}
+```
+
+For 50,000 game strings: split them into chunks of ~50-100 and call
+`/translate/batch` per chunk (either sequentially or a few chunks at a time)
+rather than one string per HTTP call. `MAX_BATCH_SIZE` (default 200) is a
+server-side safety cap on a single call.
+
+**Request fields common to both endpoints:**
+| field | type | notes |
+|---|---|---|
+| `target_lang` | string | ISO code (`tr`, `en`, `ja`, ...) or full name |
+| `source_lang` | string, optional | omit to let the model auto-detect |
+| `style` | string, optional | e.g. `"casual, playful"` — injected as a style instruction |
+| `glossary` | object, optional | `{"HP": "Can Puanı"}` — forces specific term translations |
+| `preserve_placeholders` | bool, default `true` | instructs the model to never touch `\N[1]`, `\V[1]`, `{var}`, `%s`, `%1`, etc. — important for RPG Maker / game strings |
+
+### Auth
+Set the `API_KEY` env var on the Space to require an `X-API-Key` header on
+`/translate*`. Leave empty (default) to disable auth.
+
+## Configuration (env vars)
+
+| var | default | meaning |
+|---|---|---|
+| `MODEL_FILE` | `Hy-MT2-7B-Q4_K_M.gguf` | switch to `HY-MT2-7B-Q6_K.gguf` or `HY-MT2-7B-Q8_0.gguf` for higher quality / slower |
+| `THREADS` | `8` | CPU threads for llama-server |
+| `PARALLEL_SLOTS` | `8` | concurrent generation slots (continuous batching) |
+| `CTX_SIZE` | `16384` | total context, split across `PARALLEL_SLOTS` slots |
+| `MAX_TOKENS` | `512` | max output tokens per translation |
+| `MAX_BATCH_SIZE` | `200` | max items accepted per `/translate/batch` call |
+| `TEMPERATURE`/`TOP_P`/`TOP_K`/`REPEAT_PENALTY` | `0.7`/`0.6`/`20`/`1.05` | Hy-MT2 7B recommended sampling params |
+| `API_KEY` | *(empty)* | optional shared secret for `X-API-Key` |
+
+## Deploying
+
+1. On [huggingface.co/new-space](https://huggingface.co/new-space), choose
+   **Docker** as the SDK, then push this folder's contents (or upload via
+   the web UI / `huggingface_hub`).
+2. In the Space's **Settings → Hardware**, select a CPU tier with 8 vCPU /
+   32 GB RAM (this needs to be set manually — it's a paid tier and isn't
+   controlled by any file in this repo).
+3. First boot downloads the ~4.6 GB GGUF file, so the first request after a
+   cold start can take a few minutes; enable **persistent storage** in Space
+   settings to avoid re-downloading on every restart.
+
+## Local testing
+
+```bash
+docker build -t hy-mt2-api .
+docker run -p 7860:7860 --cpus 8 --memory 32g hy-mt2-api
+```
+
+Then open `test-ui/index.html` in a browser, point it at
+`http://localhost:7860`, and use it to benchmark single vs. batch requests.

entrypoint.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MODEL_PATH="${MODEL_DIR}/${MODEL_FILE}"
+
+if [ ! -f "$MODEL_PATH" ]; then
+  echo "[entrypoint] Downloading ${MODEL_FILE} from ${MODEL_REPO} ..."
+  python -c "
+from huggingface_hub import hf_hub_download
+hf_hub_download(repo_id='${MODEL_REPO}', filename='${MODEL_FILE}', local_dir='${MODEL_DIR}')
+"
+else
+  echo "[entrypoint] Found cached model at ${MODEL_PATH}"
+fi
+
+echo "[entrypoint] Starting llama-server (threads=${THREADS}, parallel=${PARALLEL_SLOTS}, ctx=${CTX_SIZE}) ..."
+/app/llama-server \
+  --model "$MODEL_PATH" \
+  --host "${LLAMA_SERVER_HOST}" \
+  --port "${LLAMA_SERVER_PORT}" \
+  --threads "${THREADS}" \
+  --threads-batch "${THREADS}" \
+  --ctx-size "${CTX_SIZE}" \
+  --parallel "${PARALLEL_SLOTS}" \
+  --cont-batching \
+  --jinja \
+  --no-webui &
+LLAMA_PID=$!
+
+echo "[entrypoint] Waiting for llama-server to become healthy ..."
+for _ in $(seq 1 150); do
+  if curl -sf "http://${LLAMA_SERVER_HOST}:${LLAMA_SERVER_PORT}/health" >/dev/null 2>&1; then
+    echo "[entrypoint] llama-server is ready."
+    break
+  fi
+  if ! kill -0 "$LLAMA_PID" 2>/dev/null; then
+    echo "[entrypoint] llama-server exited unexpectedly, aborting." >&2
+    exit 1
+  fi
+  sleep 2
+done
+
+echo "[entrypoint] Starting API gateway on port ${PORT} ..."
+uvicorn app.main:app --host 0.0.0.0 --port "${PORT}" &
+API_PID=$!
+
+wait -n "$LLAMA_PID" "$API_PID"

requirements.txt

@@ -0,0 +1,5 @@
+fastapi==0.115.6
+uvicorn[standard]==0.34.0
+httpx==0.28.1
+pydantic==2.10.5
+huggingface_hub==0.27.1