Spaces:
Sleeping
Sleeping
update
Browse files- public/index.html +3 -0
- server.py +132 -9
public/index.html
CHANGED
|
@@ -86,6 +86,9 @@ window.__m=[];function onMsg(d){if(d.type!=='audio'&&__m.length<2000)__m.push([M
|
|
| 86 |
if(d.type==='ready'){st(`Подключено: ${d.model} · говорите`);return}
|
| 87 |
if(d.type==='error'||d.type==='info'){st('⚠ '+d.text);return}
|
| 88 |
if(d.type==='interrupted'){stopPlay();return}
|
|
|
|
|
|
|
|
|
|
| 89 |
if(d.src&&d.src!=='live')return onTr(d);
|
| 90 |
if(d.type==='turn'){cur=null;return}
|
| 91 |
const u=cur||newUtt();const now=performance.now();
|
|
|
|
| 86 |
if(d.type==='ready'){st(`Подключено: ${d.model} · говорите`);return}
|
| 87 |
if(d.type==='error'||d.type==='info'){st('⚠ '+d.text);return}
|
| 88 |
if(d.type==='interrupted'){stopPlay();return}
|
| 89 |
+
if(d.type==='speak'){ // server-side fallback: speak corrected text in the TARGET language voice
|
| 90 |
+
if($('speak').checked&&window.speechSynthesis){const s=new SpeechSynthesisUtterance(d.text);s.lang=d.lang;const v=speechSynthesis.getVoices().find(v=>v.lang.replace('_','-').startsWith(d.lang.slice(0,2)));if(v)s.voice=v;speechSynthesis.speak(s)}
|
| 91 |
+
return}
|
| 92 |
if(d.src&&d.src!=='live')return onTr(d);
|
| 93 |
if(d.type==='turn'){cur=null;return}
|
| 94 |
const u=cur||newUtt();const now=performance.now();
|
server.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
| 2 |
"""Hebrew<->Russian live translator: browser <-> this server <-> Gemini Live API.
|
| 3 |
The API key stays here (env GEMINI_API_KEY); the browser only talks to /ws."""
|
| 4 |
import os, json, asyncio, base64, time, secrets, re, pathlib, logging
|
|
|
|
| 5 |
from aiohttp import web, ClientSession, WSMsgType
|
| 6 |
|
| 7 |
KEY = os.environ["GEMINI_API_KEY"]
|
|
@@ -47,11 +48,133 @@ async def save(req):
|
|
| 47 |
fn.write_text(json.dumps(data, ensure_ascii=False, indent=1))
|
| 48 |
return web.json_response({"saved": fn.name})
|
| 49 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
class Upstream:
|
| 51 |
"""One Gemini Live session; forwards events to the browser tagged with `tag`."""
|
| 52 |
-
def __init__(self, http, client, tag, setup):
|
| 53 |
-
self.http, self.client, self.tag, self.setup = http, client, tag, setup
|
| 54 |
self.ws = None; self.ready = asyncio.Event(); self.task = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
async def start(self):
|
| 56 |
self.ws = await self.http.ws_connect(URL, headers={"x-goog-api-key": KEY}, max_msg_size=0, heartbeat=20)
|
| 57 |
await self.ws.send_str(json.dumps({"setup": self.setup}))
|
|
@@ -70,14 +193,14 @@ class Upstream:
|
|
| 70 |
if "goAway" in d: await self.send({"type": "info", "text": "goAway: session will end soon"})
|
| 71 |
continue
|
| 72 |
if "inputTranscription" in sc and sc["inputTranscription"].get("text"):
|
| 73 |
-
await self.
|
| 74 |
if "outputTranscription" in sc and sc["outputTranscription"].get("text"):
|
| 75 |
-
await self.
|
| 76 |
for part in (sc.get("modelTurn") or {}).get("parts", []):
|
| 77 |
if "inlineData" in part:
|
| 78 |
-
await self.
|
| 79 |
-
if sc.get("interrupted"): await self.
|
| 80 |
-
if sc.get("turnComplete"): await self.
|
| 81 |
cc = self.ws.close_code
|
| 82 |
await self.send({"type": "error", "text": f"upstream closed {cc} {getattr(self.ws,'_close_message', '') or ''}"})
|
| 83 |
except Exception as e:
|
|
@@ -116,8 +239,8 @@ async def ws_handler(req):
|
|
| 116 |
ups.append(Upstream(http, client, "live", {
|
| 117 |
"model": f"models/{LIVE_MODEL}",
|
| 118 |
"generationConfig": {"responseModalities": ["AUDIO"]},
|
| 119 |
-
"systemInstruction": {"parts": [{"text": system_prompt(first.get("glossary", ""))}]},
|
| 120 |
-
"inputAudioTranscription": IN_TR, "outputAudioTranscription": {}}))
|
| 121 |
await asyncio.gather(*(u.start() for u in ups))
|
| 122 |
await client.send_str(json.dumps({"type": "ready", "mode": mode,
|
| 123 |
"model": TRANSLATE_MODEL if mode == "translate" else LIVE_MODEL}))
|
|
|
|
| 2 |
"""Hebrew<->Russian live translator: browser <-> this server <-> Gemini Live API.
|
| 3 |
The API key stays here (env GEMINI_API_KEY); the browser only talks to /ws."""
|
| 4 |
import os, json, asyncio, base64, time, secrets, re, pathlib, logging
|
| 5 |
+
import aiohttp
|
| 6 |
from aiohttp import web, ClientSession, WSMsgType
|
| 7 |
|
| 8 |
KEY = os.environ["GEMINI_API_KEY"]
|
|
|
|
| 48 |
fn.write_text(json.dumps(data, ensure_ascii=False, indent=1))
|
| 49 |
return web.json_response({"saved": fn.name})
|
| 50 |
|
| 51 |
+
GATE = os.environ.get("GATE", "1") == "1"
|
| 52 |
+
VOICE = {"he": os.environ.get("VOICE_HE", "Puck"), "ru": os.environ.get("VOICE_RU", "Kore")}
|
| 53 |
+
HE_RE, RU_RE, LET_RE = re.compile(r"[\u0590-\u05FF]"), re.compile(r"[\u0400-\u04FF]"), re.compile(r"[^\W\d_]")
|
| 54 |
+
def script(t):
|
| 55 |
+
"""'ru' if Cyrillic, 'he' if Hebrew, 'x' other letters, None if no letters yet."""
|
| 56 |
+
h, r = len(HE_RE.findall(t)), len(RU_RE.findall(t))
|
| 57 |
+
if r > h: return "ru"
|
| 58 |
+
if h: return "he"
|
| 59 |
+
return "x" if LET_RE.search(t) else None
|
| 60 |
+
def target_for(inp): # deterministic: Cyrillic input -> Hebrew, anything else -> Russian
|
| 61 |
+
return "he" if script(inp) == "ru" else "ru"
|
| 62 |
+
def valid(inp, out):
|
| 63 |
+
"""Output must be Hebrew or Russian and must be the OTHER language of a Hebrew/Cyrillic input.
|
| 64 |
+
Latin-transcribed input (e.g. 'da') is ambiguous, so either direction is accepted for it."""
|
| 65 |
+
si, so = script(inp), script(out)
|
| 66 |
+
if so not in ("he", "ru"): return False
|
| 67 |
+
return si not in ("he", "ru") or so != si
|
| 68 |
+
FIX_MODELS = os.environ.get("FIX_MODELS", "gemini-3.5-flash,gemini-3.8-flash,gemini-flash-latest,gemini-3.5-flash").split(",")
|
| 69 |
+
TTS_MODELS = os.environ.get("TTS_MODELS", "gemini-3.8-flash-lite-tts,gemini-3.8-flash-tts,gemini-3.8-flash-lite-tts").split(",")
|
| 70 |
+
async def gen(http, models, body, pick, timeout=8):
|
| 71 |
+
"""generateContent with model fallback (quota/errors); returns pick(response)."""
|
| 72 |
+
err = None
|
| 73 |
+
for m in models:
|
| 74 |
+
try:
|
| 75 |
+
async with http.post(f"https://generativelanguage.googleapis.com/v1beta/models/{m}:generateContent", json=body,
|
| 76 |
+
headers={"x-goog-api-key": KEY}, timeout=aiohttp.ClientTimeout(total=timeout)) as r:
|
| 77 |
+
d = await r.json()
|
| 78 |
+
if r.status == 200: return pick(d)
|
| 79 |
+
err = f"{m}: {r.status} {d.get('error', {}).get('status')}"
|
| 80 |
+
except Exception as e:
|
| 81 |
+
err = f"{m}: {e!r}"
|
| 82 |
+
log.warning("gen fail %s", err)
|
| 83 |
+
raise RuntimeError(err)
|
| 84 |
+
async def fix_translate(http, text, target, glossary):
|
| 85 |
+
lang = {"he": "Hebrew", "ru": "Russian"}[target]
|
| 86 |
+
src = "Russian" if target == "he" else "Hebrew (possibly accented or mis-transcribed; interpret as Hebrew)"
|
| 87 |
+
prompt = (f"Translate this {src} utterance into {lang}. Output ONLY the {lang} translation, nothing else.\n"
|
| 88 |
+
+ (f"Glossary (source = target):\n{glossary[:3000]}\n" if glossary.strip() else "") + f"Utterance: {text}")
|
| 89 |
+
return await gen(http, FIX_MODELS, {"contents": [{"parts": [{"text": prompt}]}]},
|
| 90 |
+
lambda d: "".join(p.get("text", "") for p in d["candidates"][0]["content"]["parts"]).strip())
|
| 91 |
+
async def tts(http, text, target):
|
| 92 |
+
body = {"contents": [{"parts": [{"text": text}]}], "generationConfig": {"responseModalities": ["AUDIO"],
|
| 93 |
+
"speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": VOICE[target]}}}}}
|
| 94 |
+
return await gen(http, TTS_MODELS, body, lambda d: d["candidates"][0]["content"]["parts"][0]["inlineData"]["data"], 12)
|
| 95 |
+
|
| 96 |
class Upstream:
|
| 97 |
"""One Gemini Live session; forwards events to the browser tagged with `tag`."""
|
| 98 |
+
def __init__(self, http, client, tag, setup, glossary=""):
|
| 99 |
+
self.http, self.client, self.tag, self.setup, self.glossary = http, client, tag, setup, glossary
|
| 100 |
self.ws = None; self.ready = asyncio.Event(); self.task = None
|
| 101 |
+
self.gate = GATE and tag == "live"; self.stats = {"ok": 0, "fixed": 0}
|
| 102 |
+
self.reset()
|
| 103 |
+
def reset(self):
|
| 104 |
+
self.inp = ""; self.out = ""; self.held = []; self.state = "wait" # wait | pass | drop
|
| 105 |
+
self.forced = None # target language of a re-request turn
|
| 106 |
+
def ok(self, final=False):
|
| 107 |
+
if self.forced: return script(self.out) == self.forced
|
| 108 |
+
return valid(self.inp, self.out)
|
| 109 |
+
def decide(self, final=False):
|
| 110 |
+
"""Validate output script against the target derived from the input script."""
|
| 111 |
+
if self.state != "wait": return
|
| 112 |
+
si, so = script(self.inp), script(self.out)
|
| 113 |
+
if so is None and not final: return
|
| 114 |
+
if si is None and not final and not self.forced:
|
| 115 |
+
if so == "x": self.state = "drop"; return # neither Hebrew nor Russian output: always wrong
|
| 116 |
+
return # wait for input transcript
|
| 117 |
+
self.state = "pass" if self.ok() else "drop"
|
| 118 |
+
async def flush(self):
|
| 119 |
+
if self.state == "pass":
|
| 120 |
+
held, self.held = self.held, []
|
| 121 |
+
for o in held: await self.send(o)
|
| 122 |
+
async def fallback(self, inp, bad):
|
| 123 |
+
"""Wrong-language output was discarded: re-translate the transcript as text, speak it with the target's voice."""
|
| 124 |
+
inp = inp.strip(); tgt = target_for(inp)
|
| 125 |
+
if not inp: return
|
| 126 |
+
t0 = time.perf_counter(); txt = None
|
| 127 |
+
try:
|
| 128 |
+
txt = await fix_translate(self.http, inp, tgt, self.glossary)
|
| 129 |
+
if script(txt) != tgt: log.warning("fix still wrong script: %r", txt); txt = None
|
| 130 |
+
except Exception as e:
|
| 131 |
+
log.warning("fallback translate failed: %r", e)
|
| 132 |
+
if not txt: # text API unavailable (quota): re-ask the live session itself, as text, with an explicit target
|
| 133 |
+
lang = "Hebrew" if tgt == "he" else "Russian"
|
| 134 |
+
self.forced = tgt
|
| 135 |
+
await self.send({"type": "in", "text": inp})
|
| 136 |
+
await self.ws.send_str(json.dumps({"clientContent": {"turns": [{"role": "user", "parts": [{"text":
|
| 137 |
+
f"Translate into {lang} and say ONLY the {lang} translation: {inp}"}]}], "turnComplete": True}}))
|
| 138 |
+
log.info("RE-REQUEST via live in=%r -> %s", inp, tgt)
|
| 139 |
+
return
|
| 140 |
+
await self.send({"type": "in", "text": inp})
|
| 141 |
+
if txt:
|
| 142 |
+
await self.send({"type": "out", "text": txt, "fixed": True})
|
| 143 |
+
try:
|
| 144 |
+
audio = await tts(self.http, txt, tgt)
|
| 145 |
+
await self.send({"type": "audio", "data": audio, "voice": VOICE[tgt]})
|
| 146 |
+
except Exception as e: # server TTS unavailable: let the browser speak it in the target language
|
| 147 |
+
log.warning("fallback tts failed: %r", e)
|
| 148 |
+
await self.send({"type": "speak", "text": txt, "lang": "he-IL" if tgt == "he" else "ru-RU"})
|
| 149 |
+
self.stats["fixed"] += 1
|
| 150 |
+
log.info("FIX in=%r bad=%r -> %r (%d ms)", inp, bad, txt, (time.perf_counter() - t0) * 1000)
|
| 151 |
+
await self.send({"type": "turn"})
|
| 152 |
+
async def emit(self, obj, kind=None):
|
| 153 |
+
"""Route an event through the per-turn validation gate."""
|
| 154 |
+
if not self.gate: return await self.send(obj)
|
| 155 |
+
if kind == "in": self.inp += obj["text"]
|
| 156 |
+
if kind == "out": self.out += obj["text"]
|
| 157 |
+
if kind == "turn":
|
| 158 |
+
log.info("TURN state=%s held=%d in=%r out=%r", self.state, sum(o["type"] == "audio" for o in self.held), self.inp, self.out)
|
| 159 |
+
# input transcript may trail the model turn a bit
|
| 160 |
+
for _ in range(15):
|
| 161 |
+
if self.state != "wait" or script(self.inp) or self.forced: break
|
| 162 |
+
await asyncio.sleep(0.1)
|
| 163 |
+
if self.state == "wait": self.decide(final=True)
|
| 164 |
+
# re-check a passed turn against the full texts
|
| 165 |
+
if self.state == "pass" and not self.ok(): self.state = "drop"
|
| 166 |
+
if self.state == "pass":
|
| 167 |
+
await self.flush(); self.stats["ok"] += 1; self.reset(); return await self.send(obj)
|
| 168 |
+
inp, bad, was_forced = self.inp, self.out, self.forced; self.reset()
|
| 169 |
+
log.info("DROP in=%r out=%r%s", inp, bad, " (re-request also wrong; silent)" if was_forced else "")
|
| 170 |
+
if was_forced: return await self.send({"type": "turn"})
|
| 171 |
+
return asyncio.create_task(self.fallback(inp, bad))
|
| 172 |
+
if kind == "interrupted":
|
| 173 |
+
if self.state == "pass": await self.send(obj)
|
| 174 |
+
return
|
| 175 |
+
self.held.append(obj)
|
| 176 |
+
self.decide()
|
| 177 |
+
await self.flush()
|
| 178 |
async def start(self):
|
| 179 |
self.ws = await self.http.ws_connect(URL, headers={"x-goog-api-key": KEY}, max_msg_size=0, heartbeat=20)
|
| 180 |
await self.ws.send_str(json.dumps({"setup": self.setup}))
|
|
|
|
| 193 |
if "goAway" in d: await self.send({"type": "info", "text": "goAway: session will end soon"})
|
| 194 |
continue
|
| 195 |
if "inputTranscription" in sc and sc["inputTranscription"].get("text"):
|
| 196 |
+
await self.emit({"type": "in", "text": sc["inputTranscription"]["text"]}, "in")
|
| 197 |
if "outputTranscription" in sc and sc["outputTranscription"].get("text"):
|
| 198 |
+
await self.emit({"type": "out", "text": sc["outputTranscription"]["text"]}, "out")
|
| 199 |
for part in (sc.get("modelTurn") or {}).get("parts", []):
|
| 200 |
if "inlineData" in part:
|
| 201 |
+
await self.emit({"type": "audio", "data": part["inlineData"]["data"]}, "audio")
|
| 202 |
+
if sc.get("interrupted"): await self.emit({"type": "interrupted"}, "interrupted")
|
| 203 |
+
if sc.get("turnComplete"): await self.emit({"type": "turn"}, "turn")
|
| 204 |
cc = self.ws.close_code
|
| 205 |
await self.send({"type": "error", "text": f"upstream closed {cc} {getattr(self.ws,'_close_message', '') or ''}"})
|
| 206 |
except Exception as e:
|
|
|
|
| 239 |
ups.append(Upstream(http, client, "live", {
|
| 240 |
"model": f"models/{LIVE_MODEL}",
|
| 241 |
"generationConfig": {"responseModalities": ["AUDIO"]},
|
| 242 |
+
"systemInstruction": {"parts": [{"text": os.environ.get("TEST_PROMPT") or system_prompt(first.get("glossary", ""))}]},
|
| 243 |
+
"inputAudioTranscription": IN_TR, "outputAudioTranscription": {}}, first.get("glossary", "")))
|
| 244 |
await asyncio.gather(*(u.start() for u in ups))
|
| 245 |
await client.send_str(json.dumps({"type": "ready", "mode": mode,
|
| 246 |
"model": TRANSLATE_MODEL if mode == "translate" else LIVE_MODEL}))
|