Rodion111 commited on
Commit
ad5e3da
·
verified ·
1 Parent(s): cda9c71
Files changed (2) hide show
  1. public/index.html +3 -0
  2. server.py +132 -9
public/index.html CHANGED
@@ -86,6 +86,9 @@ window.__m=[];function onMsg(d){if(d.type!=='audio'&&__m.length<2000)__m.push([M
86
  if(d.type==='ready'){st(`Подключено: ${d.model} · говорите`);return}
87
  if(d.type==='error'||d.type==='info'){st('⚠ '+d.text);return}
88
  if(d.type==='interrupted'){stopPlay();return}
 
 
 
89
  if(d.src&&d.src!=='live')return onTr(d);
90
  if(d.type==='turn'){cur=null;return}
91
  const u=cur||newUtt();const now=performance.now();
 
86
  if(d.type==='ready'){st(`Подключено: ${d.model} · говорите`);return}
87
  if(d.type==='error'||d.type==='info'){st('⚠ '+d.text);return}
88
  if(d.type==='interrupted'){stopPlay();return}
89
+ if(d.type==='speak'){ // server-side fallback: speak corrected text in the TARGET language voice
90
+ if($('speak').checked&&window.speechSynthesis){const s=new SpeechSynthesisUtterance(d.text);s.lang=d.lang;const v=speechSynthesis.getVoices().find(v=>v.lang.replace('_','-').startsWith(d.lang.slice(0,2)));if(v)s.voice=v;speechSynthesis.speak(s)}
91
+ return}
92
  if(d.src&&d.src!=='live')return onTr(d);
93
  if(d.type==='turn'){cur=null;return}
94
  const u=cur||newUtt();const now=performance.now();
server.py CHANGED
@@ -2,6 +2,7 @@
2
  """Hebrew<->Russian live translator: browser <-> this server <-> Gemini Live API.
3
  The API key stays here (env GEMINI_API_KEY); the browser only talks to /ws."""
4
  import os, json, asyncio, base64, time, secrets, re, pathlib, logging
 
5
  from aiohttp import web, ClientSession, WSMsgType
6
 
7
  KEY = os.environ["GEMINI_API_KEY"]
@@ -47,11 +48,133 @@ async def save(req):
47
  fn.write_text(json.dumps(data, ensure_ascii=False, indent=1))
48
  return web.json_response({"saved": fn.name})
49
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  class Upstream:
51
  """One Gemini Live session; forwards events to the browser tagged with `tag`."""
52
- def __init__(self, http, client, tag, setup):
53
- self.http, self.client, self.tag, self.setup = http, client, tag, setup
54
  self.ws = None; self.ready = asyncio.Event(); self.task = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
  async def start(self):
56
  self.ws = await self.http.ws_connect(URL, headers={"x-goog-api-key": KEY}, max_msg_size=0, heartbeat=20)
57
  await self.ws.send_str(json.dumps({"setup": self.setup}))
@@ -70,14 +193,14 @@ class Upstream:
70
  if "goAway" in d: await self.send({"type": "info", "text": "goAway: session will end soon"})
71
  continue
72
  if "inputTranscription" in sc and sc["inputTranscription"].get("text"):
73
- await self.send({"type": "in", "text": sc["inputTranscription"]["text"]})
74
  if "outputTranscription" in sc and sc["outputTranscription"].get("text"):
75
- await self.send({"type": "out", "text": sc["outputTranscription"]["text"]})
76
  for part in (sc.get("modelTurn") or {}).get("parts", []):
77
  if "inlineData" in part:
78
- await self.send({"type": "audio", "data": part["inlineData"]["data"]})
79
- if sc.get("interrupted"): await self.send({"type": "interrupted"})
80
- if sc.get("turnComplete"): await self.send({"type": "turn"})
81
  cc = self.ws.close_code
82
  await self.send({"type": "error", "text": f"upstream closed {cc} {getattr(self.ws,'_close_message', '') or ''}"})
83
  except Exception as e:
@@ -116,8 +239,8 @@ async def ws_handler(req):
116
  ups.append(Upstream(http, client, "live", {
117
  "model": f"models/{LIVE_MODEL}",
118
  "generationConfig": {"responseModalities": ["AUDIO"]},
119
- "systemInstruction": {"parts": [{"text": system_prompt(first.get("glossary", ""))}]},
120
- "inputAudioTranscription": IN_TR, "outputAudioTranscription": {}}))
121
  await asyncio.gather(*(u.start() for u in ups))
122
  await client.send_str(json.dumps({"type": "ready", "mode": mode,
123
  "model": TRANSLATE_MODEL if mode == "translate" else LIVE_MODEL}))
 
2
  """Hebrew<->Russian live translator: browser <-> this server <-> Gemini Live API.
3
  The API key stays here (env GEMINI_API_KEY); the browser only talks to /ws."""
4
  import os, json, asyncio, base64, time, secrets, re, pathlib, logging
5
+ import aiohttp
6
  from aiohttp import web, ClientSession, WSMsgType
7
 
8
  KEY = os.environ["GEMINI_API_KEY"]
 
48
  fn.write_text(json.dumps(data, ensure_ascii=False, indent=1))
49
  return web.json_response({"saved": fn.name})
50
 
51
+ GATE = os.environ.get("GATE", "1") == "1"
52
+ VOICE = {"he": os.environ.get("VOICE_HE", "Puck"), "ru": os.environ.get("VOICE_RU", "Kore")}
53
+ HE_RE, RU_RE, LET_RE = re.compile(r"[\u0590-\u05FF]"), re.compile(r"[\u0400-\u04FF]"), re.compile(r"[^\W\d_]")
54
+ def script(t):
55
+ """'ru' if Cyrillic, 'he' if Hebrew, 'x' other letters, None if no letters yet."""
56
+ h, r = len(HE_RE.findall(t)), len(RU_RE.findall(t))
57
+ if r > h: return "ru"
58
+ if h: return "he"
59
+ return "x" if LET_RE.search(t) else None
60
+ def target_for(inp): # deterministic: Cyrillic input -> Hebrew, anything else -> Russian
61
+ return "he" if script(inp) == "ru" else "ru"
62
+ def valid(inp, out):
63
+ """Output must be Hebrew or Russian and must be the OTHER language of a Hebrew/Cyrillic input.
64
+ Latin-transcribed input (e.g. 'da') is ambiguous, so either direction is accepted for it."""
65
+ si, so = script(inp), script(out)
66
+ if so not in ("he", "ru"): return False
67
+ return si not in ("he", "ru") or so != si
68
+ FIX_MODELS = os.environ.get("FIX_MODELS", "gemini-3.5-flash,gemini-3.8-flash,gemini-flash-latest,gemini-3.5-flash").split(",")
69
+ TTS_MODELS = os.environ.get("TTS_MODELS", "gemini-3.8-flash-lite-tts,gemini-3.8-flash-tts,gemini-3.8-flash-lite-tts").split(",")
70
+ async def gen(http, models, body, pick, timeout=8):
71
+ """generateContent with model fallback (quota/errors); returns pick(response)."""
72
+ err = None
73
+ for m in models:
74
+ try:
75
+ async with http.post(f"https://generativelanguage.googleapis.com/v1beta/models/{m}:generateContent", json=body,
76
+ headers={"x-goog-api-key": KEY}, timeout=aiohttp.ClientTimeout(total=timeout)) as r:
77
+ d = await r.json()
78
+ if r.status == 200: return pick(d)
79
+ err = f"{m}: {r.status} {d.get('error', {}).get('status')}"
80
+ except Exception as e:
81
+ err = f"{m}: {e!r}"
82
+ log.warning("gen fail %s", err)
83
+ raise RuntimeError(err)
84
+ async def fix_translate(http, text, target, glossary):
85
+ lang = {"he": "Hebrew", "ru": "Russian"}[target]
86
+ src = "Russian" if target == "he" else "Hebrew (possibly accented or mis-transcribed; interpret as Hebrew)"
87
+ prompt = (f"Translate this {src} utterance into {lang}. Output ONLY the {lang} translation, nothing else.\n"
88
+ + (f"Glossary (source = target):\n{glossary[:3000]}\n" if glossary.strip() else "") + f"Utterance: {text}")
89
+ return await gen(http, FIX_MODELS, {"contents": [{"parts": [{"text": prompt}]}]},
90
+ lambda d: "".join(p.get("text", "") for p in d["candidates"][0]["content"]["parts"]).strip())
91
+ async def tts(http, text, target):
92
+ body = {"contents": [{"parts": [{"text": text}]}], "generationConfig": {"responseModalities": ["AUDIO"],
93
+ "speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": VOICE[target]}}}}}
94
+ return await gen(http, TTS_MODELS, body, lambda d: d["candidates"][0]["content"]["parts"][0]["inlineData"]["data"], 12)
95
+
96
  class Upstream:
97
  """One Gemini Live session; forwards events to the browser tagged with `tag`."""
98
+ def __init__(self, http, client, tag, setup, glossary=""):
99
+ self.http, self.client, self.tag, self.setup, self.glossary = http, client, tag, setup, glossary
100
  self.ws = None; self.ready = asyncio.Event(); self.task = None
101
+ self.gate = GATE and tag == "live"; self.stats = {"ok": 0, "fixed": 0}
102
+ self.reset()
103
+ def reset(self):
104
+ self.inp = ""; self.out = ""; self.held = []; self.state = "wait" # wait | pass | drop
105
+ self.forced = None # target language of a re-request turn
106
+ def ok(self, final=False):
107
+ if self.forced: return script(self.out) == self.forced
108
+ return valid(self.inp, self.out)
109
+ def decide(self, final=False):
110
+ """Validate output script against the target derived from the input script."""
111
+ if self.state != "wait": return
112
+ si, so = script(self.inp), script(self.out)
113
+ if so is None and not final: return
114
+ if si is None and not final and not self.forced:
115
+ if so == "x": self.state = "drop"; return # neither Hebrew nor Russian output: always wrong
116
+ return # wait for input transcript
117
+ self.state = "pass" if self.ok() else "drop"
118
+ async def flush(self):
119
+ if self.state == "pass":
120
+ held, self.held = self.held, []
121
+ for o in held: await self.send(o)
122
+ async def fallback(self, inp, bad):
123
+ """Wrong-language output was discarded: re-translate the transcript as text, speak it with the target's voice."""
124
+ inp = inp.strip(); tgt = target_for(inp)
125
+ if not inp: return
126
+ t0 = time.perf_counter(); txt = None
127
+ try:
128
+ txt = await fix_translate(self.http, inp, tgt, self.glossary)
129
+ if script(txt) != tgt: log.warning("fix still wrong script: %r", txt); txt = None
130
+ except Exception as e:
131
+ log.warning("fallback translate failed: %r", e)
132
+ if not txt: # text API unavailable (quota): re-ask the live session itself, as text, with an explicit target
133
+ lang = "Hebrew" if tgt == "he" else "Russian"
134
+ self.forced = tgt
135
+ await self.send({"type": "in", "text": inp})
136
+ await self.ws.send_str(json.dumps({"clientContent": {"turns": [{"role": "user", "parts": [{"text":
137
+ f"Translate into {lang} and say ONLY the {lang} translation: {inp}"}]}], "turnComplete": True}}))
138
+ log.info("RE-REQUEST via live in=%r -> %s", inp, tgt)
139
+ return
140
+ await self.send({"type": "in", "text": inp})
141
+ if txt:
142
+ await self.send({"type": "out", "text": txt, "fixed": True})
143
+ try:
144
+ audio = await tts(self.http, txt, tgt)
145
+ await self.send({"type": "audio", "data": audio, "voice": VOICE[tgt]})
146
+ except Exception as e: # server TTS unavailable: let the browser speak it in the target language
147
+ log.warning("fallback tts failed: %r", e)
148
+ await self.send({"type": "speak", "text": txt, "lang": "he-IL" if tgt == "he" else "ru-RU"})
149
+ self.stats["fixed"] += 1
150
+ log.info("FIX in=%r bad=%r -> %r (%d ms)", inp, bad, txt, (time.perf_counter() - t0) * 1000)
151
+ await self.send({"type": "turn"})
152
+ async def emit(self, obj, kind=None):
153
+ """Route an event through the per-turn validation gate."""
154
+ if not self.gate: return await self.send(obj)
155
+ if kind == "in": self.inp += obj["text"]
156
+ if kind == "out": self.out += obj["text"]
157
+ if kind == "turn":
158
+ log.info("TURN state=%s held=%d in=%r out=%r", self.state, sum(o["type"] == "audio" for o in self.held), self.inp, self.out)
159
+ # input transcript may trail the model turn a bit
160
+ for _ in range(15):
161
+ if self.state != "wait" or script(self.inp) or self.forced: break
162
+ await asyncio.sleep(0.1)
163
+ if self.state == "wait": self.decide(final=True)
164
+ # re-check a passed turn against the full texts
165
+ if self.state == "pass" and not self.ok(): self.state = "drop"
166
+ if self.state == "pass":
167
+ await self.flush(); self.stats["ok"] += 1; self.reset(); return await self.send(obj)
168
+ inp, bad, was_forced = self.inp, self.out, self.forced; self.reset()
169
+ log.info("DROP in=%r out=%r%s", inp, bad, " (re-request also wrong; silent)" if was_forced else "")
170
+ if was_forced: return await self.send({"type": "turn"})
171
+ return asyncio.create_task(self.fallback(inp, bad))
172
+ if kind == "interrupted":
173
+ if self.state == "pass": await self.send(obj)
174
+ return
175
+ self.held.append(obj)
176
+ self.decide()
177
+ await self.flush()
178
  async def start(self):
179
  self.ws = await self.http.ws_connect(URL, headers={"x-goog-api-key": KEY}, max_msg_size=0, heartbeat=20)
180
  await self.ws.send_str(json.dumps({"setup": self.setup}))
 
193
  if "goAway" in d: await self.send({"type": "info", "text": "goAway: session will end soon"})
194
  continue
195
  if "inputTranscription" in sc and sc["inputTranscription"].get("text"):
196
+ await self.emit({"type": "in", "text": sc["inputTranscription"]["text"]}, "in")
197
  if "outputTranscription" in sc and sc["outputTranscription"].get("text"):
198
+ await self.emit({"type": "out", "text": sc["outputTranscription"]["text"]}, "out")
199
  for part in (sc.get("modelTurn") or {}).get("parts", []):
200
  if "inlineData" in part:
201
+ await self.emit({"type": "audio", "data": part["inlineData"]["data"]}, "audio")
202
+ if sc.get("interrupted"): await self.emit({"type": "interrupted"}, "interrupted")
203
+ if sc.get("turnComplete"): await self.emit({"type": "turn"}, "turn")
204
  cc = self.ws.close_code
205
  await self.send({"type": "error", "text": f"upstream closed {cc} {getattr(self.ws,'_close_message', '') or ''}"})
206
  except Exception as e:
 
239
  ups.append(Upstream(http, client, "live", {
240
  "model": f"models/{LIVE_MODEL}",
241
  "generationConfig": {"responseModalities": ["AUDIO"]},
242
+ "systemInstruction": {"parts": [{"text": os.environ.get("TEST_PROMPT") or system_prompt(first.get("glossary", ""))}]},
243
+ "inputAudioTranscription": IN_TR, "outputAudioTranscription": {}}, first.get("glossary", "")))
244
  await asyncio.gather(*(u.start() for u in ups))
245
  await client.send_str(json.dumps({"type": "ready", "mode": mode,
246
  "model": TRANSLATE_MODEL if mode == "translate" else LIVE_MODEL}))