translator / server.py
Rodion111's picture
Lower latency: tuned Live VAD, stream audio without holding for language gate, parallel opus-mt+Gemini fallback with browser voice, fix model names, Live Translate as default mode, shorter PTT release
77bdfea verified
Raw History Blame Contribute Delete
23.6 kB
#!/usr/bin/env python3
"""Hebrew<->Russian live translator: browser <-> this server <-> Gemini Live API.
The API key stays here (env GEMINI_API_KEY); the browser only talks to /ws."""
import os, json, asyncio, base64, time, secrets, re, pathlib, logging
import aiohttp
try:
import opus_mt # local Helsinki-NLP opus-mt he<->ru (CTranslate2 int8); optional
if not opus_mt.available(): opus_mt = None
except Exception:
opus_mt = None
from aiohttp import web, ClientSession, WSMsgType
KEY = os.environ["GEMINI_API_KEY"]
TOKEN = os.environ.get("ACCESS_TOKEN") or pathlib.Path(__file__).with_name(".token").read_text().strip()
PORT = int(os.environ.get("PORT", "8787"))
LIVE_MODEL = os.environ.get("LIVE_MODEL", "gemini-3.8-live")
# tried in order when the live model refuses the session (quota / unavailable)
LIVE_MODELS = [LIVE_MODEL] + [m for m in os.environ.get("LIVE_FALLBACK_MODELS",
"gemini-3.1-flash-live-preview,gemini-2.5-flash-native-audio-preview-12-2025").split(",") if m and m != LIVE_MODEL]
def friendly(err):
e = str(err)
if "429" in e or "RESOURCE_EXHAUSTED" in e or "quota" in e.lower(): return "Лимит Gemini (квота исчерпана)"
if "1008" in e or "not found" in e.lower() or "not supported" in e.lower(): return "Модель Gemini недоступна"
return "Ошибка Gemini"
TRANSLATE_MODEL = os.environ.get("TRANSLATE_MODEL", "gemini-3.5-live-translate-preview")
URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent"
# server VAD (docs: ai.google.dev/gemini-api/docs/live-api/capabilities, realtimeInputConfig.automaticActivityDetection)
VAD = {"automaticActivityDetection": {"disabled": False,
"startOfSpeechSensitivity": os.environ.get("VAD_START", "START_SENSITIVITY_HIGH"),
"endOfSpeechSensitivity": os.environ.get("VAD_END", "END_SENSITIVITY_HIGH"),
"prefixPaddingMs": int(os.environ.get("VAD_PREFIX_MS", "100")),
"silenceDurationMs": int(os.environ.get("VAD_SILENCE_MS", "400"))}}
IN_TR = json.loads(os.environ.get("IN_TR", '{"languageCodes": ["he-IL", "ru-RU"]}'))
ROOT = pathlib.Path(__file__).parent
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
log = logging.getLogger("tr")
def system_prompt(glossary: str) -> str:
p = ("You are a real-time voice interpreter between EXACTLY two languages: Russian and Hebrew. You are NOT a conversation partner.\n"
"HARD RULES:\n"
"1. The input speech is ALWAYS either Hebrew or Russian. It is NEVER Arabic, English, Yiddish or any other language, "
"even if it sounds like one. Never treat or transcribe it as another language.\n"
"2. If the speech is Russian, say ONLY its faithful Hebrew translation.\n"
"3. Otherwise (Hebrew, or anything unclear, accented, mispronounced, mumbled or not clearly Russian) ASSUME IT IS HEBREW "
"spoken by a non-native speaker (often with a Russian or other accent): interpret it as the most plausible Hebrew words "
"and say ONLY its faithful Russian translation.\n"
"4. Your output is ALWAYS only Russian or only Hebrew. Never output any other language.\n"
"Never answer questions, never add comments, greetings, explanations or the original text. Keep names as they are (transliterate). "
"Preserve meaning, tone and register; be concise. If the audio is silence or noise, say nothing.")
if glossary.strip():
p += ("\nPersonal glossary (names/terms; always use these renderings, format 'source = target'):\n" + glossary.strip()[:3000])
return p
def check(req):
if req.query.get("t") != TOKEN:
raise web.HTTPForbidden(text="bad token")
async def index(req):
check(req)
return web.FileResponse(ROOT / "public" / "index.html", headers={"Cache-Control": "no-store"})
async def save(req):
check(req)
data = await req.json()
d = ROOT / "transcripts"; d.mkdir(exist_ok=True)
fn = d / time.strftime("transcript-%Y%m%d-%H%M%S.json")
fn.write_text(json.dumps(data, ensure_ascii=False, indent=1))
return web.json_response({"saved": fn.name})
GATE = os.environ.get("GATE", "1") == "1"
VOICE = {"he": os.environ.get("VOICE_HE", "Puck"), "ru": os.environ.get("VOICE_RU", "Kore")}
HE_RE, RU_RE, LET_RE = re.compile(r"[\u0590-\u05FF]"), re.compile(r"[\u0400-\u04FF]"), re.compile(r"[^\W\d_]")
def script(t):
"""'ru' if Cyrillic, 'he' if Hebrew, 'x' other letters, None if no letters yet."""
h, r = len(HE_RE.findall(t)), len(RU_RE.findall(t))
if r > h: return "ru"
if h: return "he"
return "x" if LET_RE.search(t) else None
def target_for(inp): # deterministic: Cyrillic input -> Hebrew, anything else -> Russian
return "he" if script(inp) == "ru" else "ru"
def valid(inp, out):
"""Output must be Hebrew or Russian and must be the OTHER language of a Hebrew/Cyrillic input.
Latin-transcribed input (e.g. 'da') is ambiguous, so either direction is accepted for it."""
si, so = script(inp), script(out)
if so not in ("he", "ru"): return False
return si not in ("he", "ru") or so != si
FIX_MODELS = os.environ.get("FIX_MODELS", "gemini-3.5-flash-lite,gemini-3.8-flash").split(",")
TTS_MODELS = os.environ.get("TTS_MODELS", "gemini-3.8-flash-lite-tts").split(",")
async def gen(http, models, body, pick, timeout=8):
"""generateContent with model fallback (quota/errors); returns pick(response)."""
err = None
for m in models:
try:
async with http.post(f"https://generativelanguage.googleapis.com/v1beta/models/{m}:generateContent", json=body,
headers={"x-goog-api-key": KEY}, timeout=aiohttp.ClientTimeout(total=timeout)) as r:
d = await r.json()
if r.status == 200: return pick(d)
err = f"{m}: {r.status} {d.get('error', {}).get('status')}"
except Exception as e:
err = f"{m}: {e!r}"
log.warning("gen fail %s", err)
raise RuntimeError(err)
async def fix_translate(http, text, target, glossary):
lang = {"he": "Hebrew", "ru": "Russian"}[target]
src = "Russian" if target == "he" else "Hebrew (possibly accented or mis-transcribed; interpret as Hebrew)"
prompt = (f"Translate this {src} utterance into {lang}. Output ONLY the {lang} translation, nothing else.\n"
+ (f"Glossary (source = target):\n{glossary[:3000]}\n" if glossary.strip() else "") + f"Utterance: {text}")
return await gen(http, FIX_MODELS, {"contents": [{"parts": [{"text": prompt}]}]},
lambda d: "".join(p.get("text", "") for p in d["candidates"][0]["content"]["parts"]).strip(), FIX_TIMEOUT)
FIX_TIMEOUT = float(os.environ.get("FIX_TIMEOUT", "4"))
FALLBACK_TTS = os.environ.get("FALLBACK_TTS", "browser") # browser (instant) | gemini
OPUS = os.environ.get("OPUS", "1") == "1"
async def local_translate(text, target, glossary=""):
"""opus-mt int8 on CPU, off the event loop -> (text, missing glossary terms, method). Raises if unavailable."""
if not (OPUS and opus_mt): raise RuntimeError("opus-mt not available")
return await asyncio.to_thread(opus_mt.translate_ex, text, target, glossary)
async def text_translate(http, text, target, glossary="", use_gemini=True):
"""Fallback text translation: opus-mt and Gemini text run IN PARALLEL; first valid result wins
(opus-mt result counts only if it is in the target script and hits all glossary terms). -> (text, engine)."""
jobs = {}
if script(text) in ("he", "ru"): # Latin-transcribed / garbled ASR goes to Gemini only
jobs[asyncio.create_task(local_translate(text, target, glossary))] = "opus"
if use_gemini:
jobs[asyncio.create_task(fix_translate(http, text, target, glossary))] = "gemini"
local = None
try:
for fut in asyncio.as_completed(list(jobs)):
try: r = await fut
except Exception as e: log.warning("fallback engine failed: %r", e); continue
if isinstance(r, tuple):
out, miss, meth = r
if script(out) != target: log.warning("opus-mt wrong script: %r", out); continue
if not miss: return out, f"opus-mt/{meth}"
local = (out, f"opus-mt/{meth}-glossary-miss"); log.info("opus-mt missed glossary %r", miss)
else:
if script(r) == target: return r, "gemini"
log.warning("gemini fallback wrong script: %r", r)
finally:
for t in jobs:
if not t.done(): t.cancel()
if local: return local
raise RuntimeError("no fallback translation")
TTS_COOLDOWN = float(os.environ.get("TTS_COOLDOWN", "90")); _tts_off_until = 0.0
async def tts(http, text, target):
"""Gemini TTS; after a quota error skip it for TTS_COOLDOWN s (the browser speaks instead, no wasted round-trips)."""
global _tts_off_until
if time.time() < _tts_off_until: raise RuntimeError("tts quota cooldown")
try:
return await _tts(http, text, target)
except Exception as e:
if "429" in str(e) or "RESOURCE_EXHAUSTED" in str(e): _tts_off_until = time.time() + TTS_COOLDOWN
raise
async def _tts(http, text, target):
body = {"contents": [{"parts": [{"text": text}]}], "generationConfig": {"responseModalities": ["AUDIO"],
"speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": VOICE[target]}}}}}
return await gen(http, TTS_MODELS, body, lambda d: d["candidates"][0]["content"]["parts"][0]["inlineData"]["data"], 6)
class Upstream:
"""One Gemini Live session; forwards events to the browser tagged with `tag`."""
def __init__(self, http, client, tag, setup, glossary=""):
self.http, self.client, self.tag, self.setup, self.glossary = http, client, tag, setup, glossary
self.ws = None; self.ready = asyncio.Event(); self.task = None; self.fail = None; self.bytes = 0
self.gate = GATE and tag == "live"; self.stats = {"ok": 0, "fixed": 0}
self.reset()
def reset(self):
self.inp = ""; self.out = ""; self.held = []; self.state = "wait" # wait | pass | drop
self.forced = None # target language of a re-request turn
def ok(self, final=False):
if self.forced: return script(self.out) == self.forced
return valid(self.inp, self.out)
def decide(self, final=False):
"""Validate output script against the target derived from the input script."""
if self.state != "wait": return
si, so = script(self.inp), script(self.out)
if so is None and not final: return
if si is None and not final and not self.forced:
if so == "x": self.state = "drop"; return # neither Hebrew nor Russian output: always wrong
return # wait for input transcript
self.state = "pass" if self.ok() else "drop"
async def flush(self):
if self.state == "pass":
held, self.held = self.held, []
for o in held: await self.send(o)
async def fallback(self, inp, bad):
"""Wrong-language output was discarded: re-translate the transcript as text, speak it with the target's voice."""
inp = inp.strip(); tgt = target_for(inp)
if not inp: return
t0 = time.perf_counter(); txt = None; engine = None
# 1) local opus-mt (no quota, ~30-100 ms); 2) Gemini text models; 3) re-ask the live session
try:
txt, engine = await text_translate(self.http, inp, tgt, self.glossary)
except Exception as e:
log.warning("fallback translate failed: %r", e)
if not txt: # text API unavailable (quota): re-ask the live session itself, as text, with an explicit target
lang = "Hebrew" if tgt == "he" else "Russian"
self.forced = tgt
await self.send({"type": "in", "text": inp})
await self.ws.send_str(json.dumps({"clientContent": {"turns": [{"role": "user", "parts": [{"text":
f"Translate into {lang} and say ONLY the {lang} translation: {inp}"}]}], "turnComplete": True}}))
log.info("RE-REQUEST via live in=%r -> %s", inp, tgt)
return
await self.send({"type": "in", "text": inp})
if txt:
t_txt = (time.perf_counter() - t0) * 1000
await self.send({"type": "out", "text": txt, "fixed": True, "engine": engine})
try:
if FALLBACK_TTS != "gemini": raise RuntimeError("browser voice")
audio = await tts(self.http, txt, tgt)
await self.send({"type": "audio", "data": audio, "voice": VOICE[tgt]})
except Exception as e: # server TTS unavailable: let the browser speak it in the target language
log.warning("fallback tts failed: %r", e)
await self.send({"type": "speak", "text": txt, "lang": "he-IL" if tgt == "he" else "ru-RU"})
self.stats["fixed"] += 1
log.info("FIX[%s] in=%r bad=%r -> %r (text %d ms, total %d ms)", engine, inp, bad, txt, t_txt, (time.perf_counter() - t0) * 1000)
await self.send({"type": "turn"})
async def emit(self, obj, kind=None):
"""Stream events to the browser immediately; validate the output language on the fly.
On wrong language: tell the browser to cut playback (interrupted+drop) and run the text fallback."""
if not self.gate: return await self.send(obj)
if kind == "in": self.inp += obj["text"]
if kind == "out": self.out += obj["text"]
if kind == "turn":
log.info("TURN state=%s in=%r out=%r", self.state, self.inp, self.out)
if self.state == "wait": self.decide(final=True)
if self.state == "pass" and not self.ok(): await self.drop()
if self.state != "drop":
self.stats["ok"] += 1; self.reset(); return await self.send(obj)
inp, bad, was_forced = self.inp, self.out, self.forced; self.reset()
log.info("DROP in=%r out=%r%s", inp, bad, " (re-request also wrong; silent)" if was_forced else "")
if was_forced: return await self.send({"type": "turn"})
return asyncio.create_task(self.fallback(inp, bad))
if self.state == "drop": return # rest of a wrong-language turn is swallowed
if kind != "in" or not self.forced: await self.send(obj)
self.decide()
if self.state == "drop": self.state = "pass"; await self.drop()
async def drop(self):
if self.state != "drop":
self.state = "drop"; await self.send({"type": "interrupted", "drop": True})
async def start(self):
self.ws = await self.http.ws_connect(URL, headers={"x-goog-api-key": KEY}, max_msg_size=0, heartbeat=20)
await self.ws.send_str(json.dumps({"setup": self.setup}))
self.task = asyncio.create_task(self.pump())
await asyncio.wait_for(self.ready.wait(), 15)
if self.fail: raise RuntimeError(self.fail)
async def pump(self):
try:
async for m in self.ws:
if m.type not in (WSMsgType.TEXT, WSMsgType.BINARY):
break
d = json.loads(m.data)
if "setupComplete" in d:
self.ready.set(); continue
sc = d.get("serverContent")
if not sc:
if "goAway" in d: await self.send({"type": "info", "text": "goAway: session will end soon"})
continue
if "inputTranscription" in sc and sc["inputTranscription"].get("text"):
await self.emit({"type": "in", "text": sc["inputTranscription"]["text"]}, "in")
if "outputTranscription" in sc and sc["outputTranscription"].get("text"):
await self.emit({"type": "out", "text": sc["outputTranscription"]["text"]}, "out")
for part in (sc.get("modelTurn") or {}).get("parts", []):
if "inlineData" in part:
await self.emit({"type": "audio", "data": part["inlineData"]["data"]}, "audio")
if sc.get("interrupted"): await self.emit({"type": "interrupted"}, "interrupted")
if sc.get("turnComplete"): await self.emit({"type": "turn"}, "turn")
cc = self.ws.close_code; msg = f"upstream closed {cc} {getattr(self.ws, '_close_message', '') or ''} {self.ws.exception() or ''}".strip()
log.warning("UPSTREAM %s %s (ready=%s, audio in %d KB)", self.tag, msg, self.ready.is_set(), self.bytes // 1024)
if not self.ready.is_set(): self.fail = msg; self.ready.set(); return
await self.send({"type": "error", "text": f"{friendly(msg)}: {msg}", "code": cc})
if not self.client.closed: await self.client.close() # browser reconnects -> new upstream session
except Exception as e:
log.warning("UPSTREAM %s exception %r", self.tag, e)
if not self.ready.is_set(): self.fail = repr(e); self.ready.set(); return
await self.send({"type": "error", "text": f"{friendly(e)}: {e!r}"})
async def send(self, obj):
obj["src"] = self.tag
if not self.client.closed:
await self.client.send_str(json.dumps(obj, ensure_ascii=False))
async def audio(self, b64):
self.bytes += len(b64) * 3 // 4
if self.ws and not self.ws.closed:
await self.ws.send_str(json.dumps({"realtimeInput": {"audio": {"data": b64, "mimeType": "audio/pcm;rate=16000"}}}))
async def end_stream(self):
if self.ws and not self.ws.closed:
await self.ws.send_str(json.dumps({"realtimeInput": {"audioStreamEnd": True}}))
async def close(self):
if self.ws: await self.ws.close()
if self.task: self.task.cancel()
async def ws_handler(req):
check(req)
client = web.WebSocketResponse(heartbeat=25, max_msg_size=0)
await client.prepare(req)
ups = []; nbytes = 0; nmsg = 0
async with ClientSession() as http:
try:
first = await client.receive_json(timeout=30) # {"type":"start","mode":"live|translate","glossary":"..."}
mode = first.get("mode", "live")
if mode == "translate":
for lang in ("ru", "he"):
ups.append(Upstream(http, client, lang, {
"model": f"models/{TRANSLATE_MODEL}",
"generationConfig": {"responseModalities": ["AUDIO"],
"translationConfig": {"targetLanguageCode": lang, "echoTargetLanguage": False}},
"inputAudioTranscription": {}, "outputAudioTranscription": {}}))
model = TRANSLATE_MODEL
if mode == "translate":
await asyncio.gather(*(u.start() for u in ups))
else:
last = None
for model in LIVE_MODELS:
u = Upstream(http, client, "live", {
"model": f"models/{model}",
"generationConfig": {"responseModalities": ["AUDIO"]},
"systemInstruction": {"parts": [{"text": os.environ.get("TEST_PROMPT") or system_prompt(first.get("glossary", ""))}]},
"realtimeInputConfig": VAD, "inputAudioTranscription": IN_TR, "outputAudioTranscription": {}}, first.get("glossary", ""))
try:
await u.start(); ups = [u]; break
except Exception as e:
last = e; await u.close()
log.warning("live model %s failed to start: %s", model, e)
await client.send_str(json.dumps({"type": "info", "text": f"{friendly(e)} ({model}) — пробую другую модель…"}))
else:
raise RuntimeError(f"{friendly(last)}: {last}")
await client.send_str(json.dumps({"type": "ready", "mode": mode, "model": model}))
log.info("session start mode=%s model=%s ua=%s", mode, model, req.headers.get("User-Agent", "")[:120])
async for m in client:
if m.type == WSMsgType.BINARY:
nbytes += len(m.data); nmsg += 1
if nmsg == 1: log.info("first client audio frame %d bytes", len(m.data))
b64 = base64.b64encode(m.data).decode()
for u in ups: await u.audio(b64)
elif m.type == WSMsgType.TEXT:
d = json.loads(m.data)
if d.get("type") == "diag": log.info("CLIENT %s", json.dumps(d, ensure_ascii=False)[:500])
if d.get("type") == "flush":
for u in ups: await u.end_stream()
else:
break
except Exception as e:
log.exception("session error")
if not client.closed:
await client.send_str(json.dumps({"type": "error", "text": f"{friendly(e)}: {type(e).__name__}: {e}"}))
finally:
for u in ups: await u.close()
log.info("session end: client audio %d KB (%.1f s), msgs=%d, stats=%s", nbytes // 1024, nbytes / 32000, nmsg,
[u.stats for u in ups])
return client
async def translate_http(req):
"""Text-only translation with local opus-mt: GET/POST /translate?t=TOKEN&text=...[&to=he|ru][&glossary=...][&gemini=0]
(POST may send JSON {"text","to","glossary"}). Gemini text is used only if opus-mt misses a glossary term
or fails (gemini=0 disables it). Target defaults to the other language of the input script."""
check(req)
q = dict(req.query)
if req.method == "POST" and req.can_read_body:
try: q.update(await req.json())
except Exception: pass
text = (q.get("text") or "").strip()[:2000]
if not text: raise web.HTTPBadRequest(text="text required")
to = q.get("to") or target_for(text)
if to not in ("he", "ru"): raise web.HTTPBadRequest(text="to must be he or ru")
t0 = time.perf_counter()
try:
if str(q.get("gemini", "1")) == "0":
out, miss, meth = await local_translate(text, to, q.get("glossary", "")); engine = f"opus-mt/{meth}"
else:
async with ClientSession() as http:
out, engine = await text_translate(http, text, to, q.get("glossary", ""))
except Exception as e:
return web.json_response({"error": f"{type(e).__name__}: {e}"}, status=503)
return web.json_response({"text": out, "to": to, "engine": engine, "ms": round((time.perf_counter() - t0) * 1000)},
dumps=lambda o: json.dumps(o, ensure_ascii=False))
async def on_start(app):
if OPUS and opus_mt: # load both models in the background so the first fallback is fast
t0 = time.perf_counter()
try:
await asyncio.to_thread(opus_mt.preload); log.info("opus-mt loaded in %d ms", (time.perf_counter() - t0) * 1000)
except Exception as e:
log.warning("opus-mt preload failed: %r", e)
else:
log.warning("opus-mt not available; fallback uses Gemini text only")
app = web.Application(client_max_size=20 * 2**20)
async def start_preload(app): app["preload"] = asyncio.create_task(on_start(app))
app.on_startup.append(start_preload)
app.add_routes([web.get("/", index), web.get("/ws", ws_handler), web.post("/save", save),
web.get("/translate", translate_http), web.post("/translate", translate_http)])
if __name__ == "__main__":
web.run_app(app, host=os.environ.get("HOST", "127.0.0.1"), port=PORT, access_log=None)