3v324v23 commited on
Commit
722f0c9
·
1 Parent(s): b37e768

feat(tts): enhance TTS warm process with phase tracking and to_generate count

Browse files
Files changed (3) hide show
  1. app/main.py +38 -20
  2. static/index.html +8 -7
  3. tests/test_scoring.py +36 -1
app/main.py CHANGED
@@ -30,6 +30,7 @@ import zipfile
30
  from contextlib import asynccontextmanager
31
  from datetime import date, datetime, timedelta, timezone
32
  from pathlib import Path
 
33
  from zoneinfo import ZoneInfo
34
 
35
  from dotenv import load_dotenv
@@ -484,7 +485,7 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
484
  app.state.tts_warm = {"running": False, "voice_id": "", "total": 0, "done": 0, "generated": 0,
485
  "skipped": 0, "failed": 0, "current": "", "waiting": 0, "error": "",
486
  "bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": "", "stopped": False,
487
- "started_at": 0}
488
  app.state.tts_warm_lock = threading.Lock()
489
  app.state.tts_warm_stop = threading.Event() # set by POST /api/tts-cache/warm/stop
490
  app.state.tutor = tutor or load_tutor(app.state.keys.get("ANTHROPIC_API_KEY"), app.state.keys.get("TUTOR_MODEL"))
@@ -2669,7 +2670,8 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
2669
  app.state.tts_warm = {"running": True, "voice_id": voice, "total": len(jobs), "done": 0, "generated": 0,
2670
  "skipped": 0, "failed": 0, "current": "", "waiting": 0, "error": "",
2671
  "bucket": mirror.bucket_id if mirror else "", "uploaded": 0, "upload_failed": 0,
2672
- "upload_error": "", "stopped": False, "started_at": time.time()}
 
2673
  stop = app.state.tts_warm_stop
2674
  stop.clear()
2675
 
@@ -2749,34 +2751,32 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
2749
  return False
2750
 
2751
  def run() -> None:
 
2752
  if mirror is not None:
2753
  try:
2754
  remote.update(mirror.remote_sizes())
 
2755
  except Exception as e: # listing failed: upload everything; the uploads report their own errors
2756
  log.warning("tts bucket listing failed: %s", e)
2757
  with app.state.tts_warm_lock:
2758
  app.state.tts_warm["upload_error"] = f"listing failed: {str(e)[:300]}"
2759
- # Already cached: skip without reading it (the cache can sit on a bucket mount, where reading each WAV is
2760
- # a network fetch); the upload pass at the end still sends it if the bucket lacks it.
2761
- todo = []
2762
- for text in jobs:
2763
- if stop.is_set():
2764
- break
2765
- try:
2766
- cached = tts.cached(text, voice)
2767
- except OSError:
2768
- cached = False
2769
- if cached:
2770
- bump(skipped=1, done=1)
2771
- else:
2772
- todo.append(text)
2773
- work = iter(todo)
2774
- take = threading.Lock()
2775
 
2776
  def worker() -> None:
2777
  while not (stop.is_set() or halted.is_set()):
2778
- with take:
2779
- text = next(work, None)
2780
  if text is None:
2781
  return
2782
  with app.state.tts_warm_lock:
@@ -2795,6 +2795,22 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
2795
  for i in range(WARM_WORKERS)]
2796
  for w in workers:
2797
  w.start()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2798
  for w in workers:
2799
  w.join()
2800
  if mirror is not None and (stop.is_set() or halted.is_set()):
@@ -2803,6 +2819,7 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
2803
  # also everything cached earlier: other voices, playback misses, the legacy flat layout
2804
  with app.state.tts_warm_lock:
2805
  app.state.tts_warm["current"] = "uploading cached files"
 
2806
  try:
2807
  for path, rel in tts.cached_files():
2808
  queue_upload(path, rel)
@@ -2812,6 +2829,7 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
2812
  with app.state.tts_warm_lock:
2813
  app.state.tts_warm["running"] = False
2814
  app.state.tts_warm["current"] = ""
 
2815
  app.state.tts_warm["stopped"] = stop.is_set()
2816
 
2817
  threading.Thread(target=run, name="ihki-tts-warm", daemon=True).start()
 
30
  from contextlib import asynccontextmanager
31
  from datetime import date, datetime, timedelta, timezone
32
  from pathlib import Path
33
+ from queue import Queue
34
  from zoneinfo import ZoneInfo
35
 
36
  from dotenv import load_dotenv
 
485
  app.state.tts_warm = {"running": False, "voice_id": "", "total": 0, "done": 0, "generated": 0,
486
  "skipped": 0, "failed": 0, "current": "", "waiting": 0, "error": "",
487
  "bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": "", "stopped": False,
488
+ "started_at": 0, "phase": "", "to_generate": 0}
489
  app.state.tts_warm_lock = threading.Lock()
490
  app.state.tts_warm_stop = threading.Event() # set by POST /api/tts-cache/warm/stop
491
  app.state.tutor = tutor or load_tutor(app.state.keys.get("ANTHROPIC_API_KEY"), app.state.keys.get("TUTOR_MODEL"))
 
2670
  app.state.tts_warm = {"running": True, "voice_id": voice, "total": len(jobs), "done": 0, "generated": 0,
2671
  "skipped": 0, "failed": 0, "current": "", "waiting": 0, "error": "",
2672
  "bucket": mirror.bucket_id if mirror else "", "uploaded": 0, "upload_failed": 0,
2673
+ "upload_error": "", "stopped": False, "started_at": time.time(),
2674
+ "phase": "scanning", "to_generate": 0}
2675
  stop = app.state.tts_warm_stop
2676
  stop.clear()
2677
 
 
2751
  return False
2752
 
2753
  def run() -> None:
2754
+ listed = False
2755
  if mirror is not None:
2756
  try:
2757
  remote.update(mirror.remote_sizes())
2758
+ listed = True
2759
  except Exception as e: # listing failed: upload everything; the uploads report their own errors
2760
  log.warning("tts bucket listing failed: %s", e)
2761
  with app.state.tts_warm_lock:
2762
  app.state.tts_warm["upload_error"] = f"listing failed: {str(e)[:300]}"
2763
+
2764
+ def is_cached(text: str) -> bool:
2765
+ """With a bucket listing: the local staging folder or the listing, both instant. Asking the bucket
2766
+ mount instead costs a network lookup per missing file, minutes for the whole bank."""
2767
+ if not listed or getattr(tts, "staging_dir", None) is None:
2768
+ return tts.cached(text, voice)
2769
+ rel = tts.cache_key(text, voice)
2770
+ return ((tts.staging_dir / rel).is_file() or mirror.remote_path(rel) in remote
2771
+ or mirror.remote_path(rel.rsplit("/", 1)[-1]) in remote)
2772
+
2773
+ # Already cached: skip without reading it. The upload pass at the end still sends it if the bucket
2774
+ # lacks it. Workers start on the missing phrases while the scan goes on.
2775
+ work: Queue = Queue()
 
 
 
2776
 
2777
  def worker() -> None:
2778
  while not (stop.is_set() or halted.is_set()):
2779
+ text = work.get()
 
2780
  if text is None:
2781
  return
2782
  with app.state.tts_warm_lock:
 
2795
  for i in range(WARM_WORKERS)]
2796
  for w in workers:
2797
  w.start()
2798
+ for text in jobs:
2799
+ if stop.is_set() or halted.is_set():
2800
+ break
2801
+ try:
2802
+ cached = is_cached(text)
2803
+ except OSError:
2804
+ cached = False
2805
+ if cached:
2806
+ bump(skipped=1, done=1)
2807
+ else:
2808
+ bump(to_generate=1)
2809
+ work.put(text)
2810
+ with app.state.tts_warm_lock:
2811
+ app.state.tts_warm["phase"] = "generating"
2812
+ for _ in workers:
2813
+ work.put(None)
2814
  for w in workers:
2815
  w.join()
2816
  if mirror is not None and (stop.is_set() or halted.is_set()):
 
2819
  # also everything cached earlier: other voices, playback misses, the legacy flat layout
2820
  with app.state.tts_warm_lock:
2821
  app.state.tts_warm["current"] = "uploading cached files"
2822
+ app.state.tts_warm["phase"] = "uploading"
2823
  try:
2824
  for path, rel in tts.cached_files():
2825
  queue_upload(path, rel)
 
2829
  with app.state.tts_warm_lock:
2830
  app.state.tts_warm["running"] = False
2831
  app.state.tts_warm["current"] = ""
2832
+ app.state.tts_warm["phase"] = ""
2833
  app.state.tts_warm["stopped"] = stop.is_set()
2834
 
2835
  threading.Thread(target=run, name="ihki-tts-warm", daemon=True).start()
static/index.html CHANGED
@@ -4921,17 +4921,18 @@ function drawTtsCacheControls(box){
4921
  progress.max=100; progress.value=0; progress.hidden=true; progress.style.width="100%"; progress.setAttribute("aria-label","התקדמות הכנת הקלטות Sawtak"); progressLabel.hidden=true;
4922
  let polling=false, samples=[]; // samples: recent [time, done] while the sweep runs, for the time estimate
4923
  const duration=sec=>{ sec=Math.round(sec); if(sec<60) return "פחות מדקה"; const m=Math.round(sec/60); if(m<60) return `כ־${m} דקות`; const h=Math.floor(m/60),r=m%60; return r?`כ־${h} שעות ו־${r} דקות`:`כ־${h} שעות`; };
4924
- const estimate=s=>{ // from the pace of the last 3 minutes (cached phrases fly, new ones wait for Sawtak); else since the start
4925
  const now=Date.now(); if(!s.running){ samples=[]; return s.started_at&&s.total&&s.done===s.total?` · ההכנה ארכה ${duration(now/1000-s.started_at)}`:""; }
4926
- samples.push([now,s.done]); samples=samples.filter(([t])=>now-t<=180000);
4927
- const [t0,d0]=samples[0], left=s.total-s.done; let rate=0;
4928
- if(now-t0>=20000&&s.done>d0) rate=(s.done-d0)/((now-t0)/1000);
4929
- else if(s.started_at&&s.done) rate=s.done/Math.max(1,now/1000-s.started_at);
4930
- if(!left) return ""; if(!rate) return " · מחשב זמן משוער…";
 
4931
  const end=new Date(now+left/rate*1000).toLocaleTimeString("he-IL",{hour:"2-digit",minute:"2-digit"});
4932
  return ` · נותרו ${duration(left/rate)} (סיום משוער ${end})`;
4933
  };
4934
- const warmState=s=>{ if(!s) return; warm.disabled=button.disabled=!!s.running; halt.hidden=!s.running; const pct=s.total?Math.round(100*s.done/s.total):0,voice=s.voice_id?` · קול ${s.voice_id}`:""; progress.hidden=progressLabel.hidden=!s.total; progress.value=pct; progressLabel.textContent=`${pct}% · ${s.done} מתוך ${s.total} · נוצרו ${s.generated} · כבר היו ${s.skipped}`+(s.bucket?` · הועלו לדלי ${s.uploaded||0}`+(s.upload_failed?` · נכשלו בהעלאה ${s.upload_failed}`:""):" · אין דלי מוגדר (TTS_BUCKET)")+(s.upload_error?` · ${s.upload_error}`:"")+estimate(s); if(s.running) msg.textContent=s.waiting?`Sawtak לא ענה כרגע (הגבלת קצב או תקלת רשת); ניסיון נוסף בעוד ${s.waiting} שניות${voice}`:`מכין הקלטות מול Sawtak${voice}`; else if(s.stopped) msg.textContent=`ההכנה נעצרה לבקשתך אחרי ${s.done} הקלטות. הפעלה חוזרת תמשיך מאותה נקודה${voice}.`; else if(s.error) msg.textContent=`ההכנה נעצרה אחרי ${s.done} הקלטות: ${s.error}${voice}`; else if(s.total&&s.done===s.total) msg.textContent=`המאגר מוכן${voice}.`; if(s.running&&!polling){polling=true; setTimeout(poll,2000);} };
4935
  const show=data=>{ usage.replaceChildren(); const size=data.bytes<1024?`${data.bytes} B`:data.bytes<1048576?`${(data.bytes/1024).toFixed(1)} KB`:`${(data.bytes/1048576).toFixed(1)} MB`, rate=data.hit_rate==null?"–":`${Math.round(data.hit_rate*100)}%`; for(const [n,label] of [[data.files,"קבצי שמע במטמון"],[size,"נפח בדיסק"],[data.hits,"פגיעות מאז ההפעלה"],[data.misses,"יצירות מאז ההפעלה"],[rate,"שיעור פגיעה"]]){ const stat=mark("activity-stat",""); stat.append(mark("",n,"b"),mark("",label,"small")); usage.append(stat); } if(!data.available) msg.textContent="Sawtak אינו מוגדר בשרת."; };
4936
  const poll=()=>api("/api/tts-cache").then(async res=>{const data=await res.json().catch(()=>({}));if(!res.ok) throw new Error(data.detail||res.status);show(data);polling=false;warmState(data.warm);}).catch(err=>{polling=false;msg.textContent="נתוני המטמון לא זמינים: "+err.message;});
4937
  poll();
 
4921
  progress.max=100; progress.value=0; progress.hidden=true; progress.style.width="100%"; progress.setAttribute("aria-label","התקדמות הכנת הקלטות Sawtak"); progressLabel.hidden=true;
4922
  let polling=false, samples=[]; // samples: recent [time, done] while the sweep runs, for the time estimate
4923
  const duration=sec=>{ sec=Math.round(sec); if(sec<60) return "פחות מדקה"; const m=Math.round(sec/60); if(m<60) return `כ־${m} דקות`; const h=Math.floor(m/60),r=m%60; return r?`כ־${h} שעות ו־${r} דקות`:`כ־${h} שעות`; };
4924
+ const estimate=s=>{ // from the pace of new recordings over the last 3 minutes (skips are instant and would mislead); else since the start
4925
  const now=Date.now(); if(!s.running){ samples=[]; return s.started_at&&s.total&&s.done===s.total?` · ההכנה ארכה ${duration(now/1000-s.started_at)}`:""; }
4926
+ if(s.phase==="uploading") return " · מעלה לדלי קבצים שעוד לא שם…";
4927
+ samples.push([now,s.generated]); samples=samples.filter(([t])=>now-t<=180000);
4928
+ const [t0,g0]=samples[0], left=s.total-s.done; let rate=0;
4929
+ if(now-t0>=20000&&s.generated>g0) rate=(s.generated-g0)/((now-t0)/1000);
4930
+ else if(s.started_at&&s.generated) rate=s.generated/Math.max(1,now/1000-s.started_at);
4931
+ if(!left) return ""; if(!rate) return s.phase==="scanning"?" · בודק אילו הקלטות כבר קיימות…":" · מחשב זמן משוער…";
4932
  const end=new Date(now+left/rate*1000).toLocaleTimeString("he-IL",{hour:"2-digit",minute:"2-digit"});
4933
  return ` · נותרו ${duration(left/rate)} (סיום משוער ${end})`;
4934
  };
4935
+ const warmState=s=>{ if(!s) return; warm.disabled=button.disabled=!!s.running; halt.hidden=!s.running; const pct=s.total?Math.round(100*s.done/s.total):0,voice=s.voice_id?` · קול ${s.voice_id}`:""; progress.hidden=progressLabel.hidden=!s.total; progress.value=pct; progressLabel.textContent=`${pct}% · ${s.done} מתוך ${s.total} · נוצרו ${s.generated}`+(s.to_generate?` מתוך ${s.to_generate} חסרות`:"")+` · כבר היו ${s.skipped}`+(s.bucket?` · הועלו לדלי ${s.uploaded||0}`+(s.upload_failed?` · נכשלו בהעלאה ${s.upload_failed}`:""):" · אין דלי מוגדר (TTS_BUCKET)")+(s.upload_error?` · ${s.upload_error}`:"")+estimate(s); if(s.running) msg.textContent=s.waiting?`Sawtak לא ענה כרגע (הגבלת קצב או תקלת רשת); ניסיון נוסף בעוד ${s.waiting} שניות${voice}`:`מכין הקלטות מול Sawtak${voice}`; else if(s.stopped) msg.textContent=`ההכנה נעצרה לבקשתך אחרי ${s.done} הקלטות. הפעלה חוזרת תמשיך מאותה נקודה${voice}.`; else if(s.error) msg.textContent=`ההכנה נעצרה אחרי ${s.done} הקלטות: ${s.error}${voice}`; else if(s.total&&s.done===s.total) msg.textContent=`המאגר מוכן${voice}.`; if(s.running&&!polling){polling=true; setTimeout(poll,2000);} };
4936
  const show=data=>{ usage.replaceChildren(); const size=data.bytes<1024?`${data.bytes} B`:data.bytes<1048576?`${(data.bytes/1024).toFixed(1)} KB`:`${(data.bytes/1048576).toFixed(1)} MB`, rate=data.hit_rate==null?"–":`${Math.round(data.hit_rate*100)}%`; for(const [n,label] of [[data.files,"קבצי שמע במטמון"],[size,"נפח בדיסק"],[data.hits,"פגיעות מאז ההפעלה"],[data.misses,"יצירות מאז ההפעלה"],[rate,"שיעור פגיעה"]]){ const stat=mark("activity-stat",""); stat.append(mark("",n,"b"),mark("",label,"small")); usage.append(stat); } if(!data.available) msg.textContent="Sawtak אינו מוגדר בשרת."; };
4937
  const poll=()=>api("/api/tts-cache").then(async res=>{const data=await res.json().catch(()=>({}));if(!res.ok) throw new Error(data.detail||res.status);show(data);polling=false;warmState(data.warm);}).catch(err=>{polling=false;msg.textContent="נתוני המטמון לא זמינים: "+err.message;});
4938
  poll();
tests/test_scoring.py CHANGED
@@ -530,7 +530,8 @@ def test_monitoring_can_warm_the_unique_tts_bank(monkeypatch):
530
  assert state.pop("started_at") > 0
531
  assert state == {"running": False, "voice_id": RAEF_VOICE_ID, "total": 2, "done": 2, "generated": 1,
532
  "skipped": 1, "failed": 0, "current": "", "waiting": 0, "error": "",
533
- "bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": "", "stopped": False}
 
534
  assert [voice for _text, voice in tts.calls] == [RAEF_VOICE_ID] # the cached one is not read again
535
 
536
 
@@ -2477,3 +2478,37 @@ def test_the_activity_log_shows_each_sawtak_request_and_upload(monkeypatch, tmp_
2477
  assert failed["status"] == 429 and failed["source"] == "playback" and "slow down" in failed["message"]
2478
  assert ok["kind"] == "sawtak" and ok["status"] == 200 and ok["source"] == "sweep" and ok["audio_s"] == 1.0
2479
  assert ok["text"] == "مرحبا" and ok["ms"] >= 0 and ok["waited_ms"] >= 0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
530
  assert state.pop("started_at") > 0
531
  assert state == {"running": False, "voice_id": RAEF_VOICE_ID, "total": 2, "done": 2, "generated": 1,
532
  "skipped": 1, "failed": 0, "current": "", "waiting": 0, "error": "",
533
+ "bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": "", "stopped": False,
534
+ "phase": "", "to_generate": 1}
535
  assert [voice for _text, voice in tts.calls] == [RAEF_VOICE_ID] # the cached one is not read again
536
 
537
 
 
2478
  assert failed["status"] == 429 and failed["source"] == "playback" and "slow down" in failed["message"]
2479
  assert ok["kind"] == "sawtak" and ok["status"] == 200 and ok["source"] == "sweep" and ok["audio_s"] == 1.0
2480
  assert ok["text"] == "مرحبا" and ok["ms"] >= 0 and ok["waited_ms"] >= 0
2481
+
2482
+
2483
+ def test_the_sweep_decides_what_exists_from_the_bucket_listing_not_the_mount(monkeypatch, tmp_path):
2484
+ import time
2485
+
2486
+ from fastapi.testclient import TestClient
2487
+
2488
+ import app.main
2489
+ import app.tts as tts_module
2490
+ from app.main import create_app
2491
+ from app.tts import RAEF_VOICE_ID, BucketMirror, SawtakTTS
2492
+
2493
+ monkeypatch.setattr(tts_module, "MIN_REQUEST_INTERVAL", 0)
2494
+ words = ["واحد", "اثنين", "ثلاثة"]
2495
+ monkeypatch.setattr(app.main, "PHRASES", [{"variants": [(w, "a")]} for w in words])
2496
+ monkeypatch.setattr(app.main, "load_auth", lambda: None)
2497
+ tts = SawtakTTS("key", cache_dir=tmp_path / "mount")
2498
+ tts.staging_dir = tmp_path / "staging"
2499
+ monkeypatch.setattr(tts, "cached", lambda *a: (_ for _ in ()).throw(AssertionError("asked the slow mount")))
2500
+ monkeypatch.setattr(tts, "_generate_now", lambda text, voice: b"RIFF" + text.encode())
2501
+ keys = [tts.cache_key(app.main.point_arabic(w, "a"), RAEF_VOICE_ID) for w in words]
2502
+
2503
+ class Mirror(BucketMirror):
2504
+ def remote_sizes(self): # the first in the per-voice layout, the second in the legacy flat one
2505
+ return {f"ihki-tts/{keys[0]}": 10, f"ihki-tts/{keys[1].rsplit('/', 1)[-1]}": 10}
2506
+ def upload(self, files, source=""): pass
2507
+ tts.mirror = Mirror("user/bucket")
2508
+ with TestClient(create_app(scorer_for(emissions(["w"])), tts, tutor=object())) as c:
2509
+ c.post("/api/tts-cache/warm", json={"voice_id": RAEF_VOICE_ID})
2510
+ for _ in range(300):
2511
+ state = c.get("/api/tts-cache").json()["warm"]
2512
+ if not state["running"]: break
2513
+ time.sleep(.01)
2514
+ assert (state["skipped"], state["generated"], state["to_generate"], state["failed"]) == (2, 1, 1, 0)