3v324v23 commited on
Commit
0707fc8
·
1 Parent(s): b3ad06b

feat(tts): add endpoint for downloading TTS cache as a zip with manifest

Browse files
Files changed (3) hide show
  1. app/main.py +65 -2
  2. static/index.html +14 -2
  3. tests/test_scoring.py +30 -1
app/main.py CHANGED
@@ -13,6 +13,7 @@ A local .env file is loaded if present; real environment variables win over it.
13
  from __future__ import annotations
14
 
15
  import hashlib
 
16
  import io
17
  import json
18
  import logging
@@ -31,7 +32,7 @@ from zoneinfo import ZoneInfo
31
 
32
  from dotenv import load_dotenv
33
  from fastapi import BackgroundTasks, Body, Depends, FastAPI, File, Form, HTTPException, Query, Request, UploadFile
34
- from fastapi.responses import HTMLResponse, Response
35
  from fastapi.staticfiles import StaticFiles
36
 
37
  from .arabic_diacritics import point_arabic
@@ -2518,6 +2519,61 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
2518
  app.state.refs.invalidate()
2519
  return {"ok": True, "removed": removed}
2520
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2521
  @app.post("/api/tts-cache/warm")
2522
  def warm_tts_cache(body: dict = Body(...), _a: SupabaseAuth = Depends(teacher_only)):
2523
  """Generate the finite phrase bank for a selected voice, one request at a time."""
@@ -2588,7 +2644,14 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
2588
  with app.state.tts_warm_lock:
2589
  app.state.tts_warm["current"] = text
2590
  try:
2591
- was_cached = tts.cached(text, voice)
 
 
 
 
 
 
 
2592
  for attempt in range(5):
2593
  try:
2594
  data, _used = tts.wav_for_voice(text, voice) # serialised: one request in this replica
 
13
  from __future__ import annotations
14
 
15
  import hashlib
16
+ import csv
17
  import io
18
  import json
19
  import logging
 
32
 
33
  from dotenv import load_dotenv
34
  from fastapi import BackgroundTasks, Body, Depends, FastAPI, File, Form, HTTPException, Query, Request, UploadFile
35
+ from fastapi.responses import HTMLResponse, Response, StreamingResponse
36
  from fastapi.staticfiles import StaticFiles
37
 
38
  from .arabic_diacritics import point_arabic
 
2519
  app.state.refs.invalidate()
2520
  return {"ok": True, "removed": removed}
2521
 
2522
+ @app.get("/api/tts-cache/download")
2523
+ def download_tts_cache(_a: SupabaseAuth = Depends(teacher_only)):
2524
+ """An offline copy of the Sawtak recordings: every cached WAV, plus manifest.csv naming each file's phrase.
2525
+
2526
+ The zip is streamed (stored, not compressed: WAV barely compresses), so the whole cache never sits in memory."""
2527
+ tts = app.state.tts
2528
+ if tts is None:
2529
+ raise HTTPException(503, "tts not configured")
2530
+ try:
2531
+ files = tts.cached_files()
2532
+ except OSError as e:
2533
+ raise HTTPException(503, f"could not read the voice cache: {e}")
2534
+ names = {i: n for n, i in {**VOICES, **LINE_VOICES}.items()}
2535
+ known: dict[str, dict] = {} # file name (sha1 of version, voice and text) -> what it says
2536
+ for phrase_id, phrase in enumerate(PHRASES):
2537
+ text = point_arabic(*phrase["variants"][0])
2538
+ for voice in names:
2539
+ info = known.setdefault(Path(tts.cache_key(text, voice)).name,
2540
+ {"voice_id": voice, "voice": names[voice], "text": text,
2541
+ "he": phrase.get("he", ""), "tr": phrase.get("tr", ""), "phrase_ids": []})
2542
+ info["phrase_ids"].append(str(phrase_id))
2543
+
2544
+ def chunks():
2545
+ class Sink(io.RawIOBase): # unseekable: zipfile writes data descriptors instead of seeking back
2546
+ def __init__(self): self.parts: list[bytes] = []
2547
+ def writable(self): return True
2548
+ def write(self, b): self.parts.append(bytes(b)); return len(b)
2549
+ sink = Sink()
2550
+
2551
+ def drain() -> bytes:
2552
+ out = b"".join(sink.parts); sink.parts.clear(); return out
2553
+
2554
+ rows = io.StringIO()
2555
+ writer = csv.writer(rows)
2556
+ writer.writerow(["file", "voice_id", "voice", "phrase_ids", "arabic", "hebrew", "transliteration"])
2557
+ with zipfile.ZipFile(sink, "w", zipfile.ZIP_STORED, allowZip64=True) as zf:
2558
+ for path, rel in files:
2559
+ try:
2560
+ zf.write(path, f"ihki-tts/{rel}")
2561
+ except OSError as e: # removed or unreadable meanwhile: leave it out, keep the rest
2562
+ log.warning("tts download skipped %s: %s", rel, e)
2563
+ continue
2564
+ info = known.get(path.name, {})
2565
+ writer.writerow([f"ihki-tts/{rel}", info.get("voice_id", rel.split("/")[0] if "/" in rel else ""),
2566
+ info.get("voice", ""), " ".join(info.get("phrase_ids", [])), info.get("text", ""),
2567
+ info.get("he", ""), info.get("tr", "")])
2568
+ yield drain()
2569
+ zf.writestr("manifest.csv", "\ufeff" + rows.getvalue()) # BOM: Excel opens the Hebrew/Arabic right
2570
+ yield drain()
2571
+
2572
+ name = f"ihki-tts-{datetime.now(timezone.utc):%Y-%m-%d_%H-%M}.zip"
2573
+ return StreamingResponse(chunks(), media_type="application/zip", headers={
2574
+ "Content-Disposition": f'attachment; filename="{name}"', "Cache-Control": "no-store",
2575
+ "X-File-Count": str(len(files))})
2576
+
2577
  @app.post("/api/tts-cache/warm")
2578
  def warm_tts_cache(body: dict = Body(...), _a: SupabaseAuth = Depends(teacher_only)):
2579
  """Generate the finite phrase bank for a selected voice, one request at a time."""
 
2644
  with app.state.tts_warm_lock:
2645
  app.state.tts_warm["current"] = text
2646
  try:
2647
+ # Already cached: skip without reading it. The cache can sit on a bucket mount, where reading
2648
+ # each WAV is a network fetch; the upload pass at the end still sends it if the bucket lacks it.
2649
+ if tts.cached(text, voice):
2650
+ with app.state.tts_warm_lock:
2651
+ app.state.tts_warm["skipped"] += 1
2652
+ app.state.tts_warm["done"] += 1
2653
+ continue
2654
+ was_cached = False
2655
  for attempt in range(5):
2656
  try:
2657
  data, _used = tts.wav_for_voice(text, voice) # serialised: one request in this replica
static/index.html CHANGED
@@ -4865,7 +4865,7 @@ function drawSystemActivity(box){
4865
  }
4866
  function drawTtsCacheControls(box){
4867
  const card=mark("card",""), title=mark("","מטמון הקלטות Sawtak","h2"), text=mark("","אפשר להכין מראש את כל מאגר ההקלטות. התהליך מדלג על קבצים קיימים וממשיך מהם בהפעלה הבאה. מחיקה גורמת ליצירה מחדש.","p");
4868
- const usage=mark("activity-stats",""), progress=document.createElement("progress"), progressLabel=mark("msg",""), row=mark("row",""), warm=mark("btn primary","הכנת כל ההקלטות","button"), button=mark("btn","מחיקת מטמון ההקלטות","button"), msg=mark("msg","");
4869
  progress.max=100; progress.value=0; progress.hidden=true; progress.style.width="100%"; progress.setAttribute("aria-label","התקדמות הכנת הקלטות Sawtak"); progressLabel.hidden=true;
4870
  let polling=false;
4871
  const warmState=s=>{ if(!s) return; warm.disabled=button.disabled=!!s.running; const pct=s.total?Math.round(100*s.done/s.total):0,voice=s.voice_id?` · קול ${s.voice_id}`:""; progress.hidden=progressLabel.hidden=!s.total; progress.value=pct; progressLabel.textContent=`${pct}% · ${s.done} מתוך ${s.total} · נוצרו ${s.generated} · כבר היו ${s.skipped}`+(s.bucket?` · הועלו לדלי ${s.uploaded||0}`+(s.upload_failed?` · נכשלו בהעלאה ${s.upload_failed}`:""):" · אין דלי מוגדר (TTS_BUCKET)")+(s.upload_error?` · ${s.upload_error}`:""); if(s.running) msg.textContent=s.waiting?`Sawtak לא ענה כרגע (הגבלת קצב או תקלת רשת); ניסיון נוסף בעוד ${s.waiting} שניות${voice}`:`מכין הקלטות מול Sawtak${voice}`; else if(s.error) msg.textContent=`ההכנה נעצרה אחרי ${s.done} הקלטות: ${s.error}${voice}`; else if(s.total&&s.done===s.total) msg.textContent=`המאגר מוכן${voice}.`; if(s.running&&!polling){polling=true; setTimeout(poll,2000);} };
@@ -4873,6 +4873,18 @@ function drawTtsCacheControls(box){
4873
  const poll=()=>api("/api/tts-cache").then(async res=>{const data=await res.json().catch(()=>({}));if(!res.ok) throw new Error(data.detail||res.status);show(data);polling=false;warmState(data.warm);}).catch(err=>{polling=false;msg.textContent="נתוני המטמון לא זמינים: "+err.message;});
4874
  poll();
4875
  warm.onclick=async()=>{ const voice=(prompt("מזהה הקול של Sawtak:","273d81b2-991b-5b76-8525-eeacc6a3d759")||"").trim(); if(!voice) return; if(!confirm(`להכין עכשיו את כל הקלטות Sawtak החסרות לקול ${voice}? התהליך משתמש ביתרה בחשבון וממשיך ברקע.`)) return; warm.disabled=true; msg.textContent="מתחיל…"; try{const res=await api("/api/tts-cache/warm",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({voice_id:voice})}),data=await res.json().catch(()=>({}));if(!res.ok) throw new Error(data.detail||res.status);warmState(data);}catch(err){warm.disabled=false;msg.textContent="ההכנה לא התחילה: "+err.message;} };
 
 
 
 
 
 
 
 
 
 
 
 
4876
  button.onclick=async()=>{
4877
  if(!confirm("למחוק את כל הקלטות התשובה מהמטמון? גם הקלטות שנבחרו ידנית יימחקו, ולא ניתן לבטל את הפעולה.")) return;
4878
  button.disabled=true; msg.textContent="מוחק…";
@@ -4886,7 +4898,7 @@ function drawTtsCacheControls(box){
4886
  }catch(err){ msg.textContent="המחיקה נכשלה: "+err.message; }
4887
  button.disabled=false;
4888
  };
4889
- row.append(warm,button,msg); card.append(title,text,usage,progress,progressLabel,row); box.append(card);
4890
  }
4891
  // how to get each key, step by step: a step is its parts, text or [text, link]
4892
  const KEY_GUIDES={
 
4865
  }
4866
  function drawTtsCacheControls(box){
4867
  const card=mark("card",""), title=mark("","מטמון הקלטות Sawtak","h2"), text=mark("","אפשר להכין מראש את כל מאגר ההקלטות. התהליך מדלג על קבצים קיימים וממשיך מהם בהפעלה הבאה. מחיקה גורמת ליצירה מחדש.","p");
4868
+ const usage=mark("activity-stats",""), progress=document.createElement("progress"), progressLabel=mark("msg",""), row=mark("row",""), warm=mark("btn primary","הכנת כל ההקלטות","button"), button=mark("btn","מחיקת מטמון ההקלטות","button"), offline=mark("btn","הורדת עותק אופליין","button"), msg=mark("msg","");
4869
  progress.max=100; progress.value=0; progress.hidden=true; progress.style.width="100%"; progress.setAttribute("aria-label","התקדמות הכנת הקלטות Sawtak"); progressLabel.hidden=true;
4870
  let polling=false;
4871
  const warmState=s=>{ if(!s) return; warm.disabled=button.disabled=!!s.running; const pct=s.total?Math.round(100*s.done/s.total):0,voice=s.voice_id?` · קול ${s.voice_id}`:""; progress.hidden=progressLabel.hidden=!s.total; progress.value=pct; progressLabel.textContent=`${pct}% · ${s.done} מתוך ${s.total} · נוצרו ${s.generated} · כבר היו ${s.skipped}`+(s.bucket?` · הועלו לדלי ${s.uploaded||0}`+(s.upload_failed?` · נכשלו בהעלאה ${s.upload_failed}`:""):" · אין דלי מוגדר (TTS_BUCKET)")+(s.upload_error?` · ${s.upload_error}`:""); if(s.running) msg.textContent=s.waiting?`Sawtak לא ענה כרגע (הגבלת קצב או תקלת רשת); ניסיון נוסף בעוד ${s.waiting} שניות${voice}`:`מכין הקלטות מול Sawtak${voice}`; else if(s.error) msg.textContent=`ההכנה נעצרה אחרי ${s.done} הקלטות: ${s.error}${voice}`; else if(s.total&&s.done===s.total) msg.textContent=`המאגר מוכן${voice}.`; if(s.running&&!polling){polling=true; setTimeout(poll,2000);} };
 
4873
  const poll=()=>api("/api/tts-cache").then(async res=>{const data=await res.json().catch(()=>({}));if(!res.ok) throw new Error(data.detail||res.status);show(data);polling=false;warmState(data.warm);}).catch(err=>{polling=false;msg.textContent="נתוני המטמון לא זמינים: "+err.message;});
4874
  poll();
4875
  warm.onclick=async()=>{ const voice=(prompt("מזהה הקול של Sawtak:","273d81b2-991b-5b76-8525-eeacc6a3d759")||"").trim(); if(!voice) return; if(!confirm(`להכין עכשיו את כל הקלטות Sawtak החסרות לקול ${voice}? התהליך משתמש ביתרה בחשבון וממשיך ברקע.`)) return; warm.disabled=true; msg.textContent="מתחיל…"; try{const res=await api("/api/tts-cache/warm",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({voice_id:voice})}),data=await res.json().catch(()=>({}));if(!res.ok) throw new Error(data.detail||res.status);warmState(data);}catch(err){warm.disabled=false;msg.textContent="ההכנה לא התחילה: "+err.message;} };
4876
+ offline.onclick=async()=>{ // every cached WAV + manifest.csv (which phrase each file is), streamed as one zip
4877
+ offline.disabled=true; msg.textContent="מכין עותק אופליין…";
4878
+ try{
4879
+ const res=await api("/api/tts-cache/download"); if(!res.ok) throw new Error((await res.json().catch(()=>({}))).detail||res.status);
4880
+ const count=res.headers.get("X-File-Count")||"", name=(/filename="([^"]+)"/.exec(res.headers.get("Content-Disposition")||"")||[])[1]||"ihki-tts.zip";
4881
+ const reader=res.body.getReader(), parts=[]; let got=0;
4882
+ for(;;){ const {done,value}=await reader.read(); if(done) break; parts.push(value); got+=value.length; msg.textContent=`מוריד ${count} הקלטות… ${(got/1048576).toFixed(1)} MB`; }
4883
+ const url=URL.createObjectURL(new Blob(parts,{type:"application/zip"})), a=document.createElement("a"); a.href=url; a.download=name; a.click(); setTimeout(()=>URL.revokeObjectURL(url),60000);
4884
+ msg.textContent=`העותק הורד: ${count} הקלטות · ${(got/1048576).toFixed(1)} MB · הקובץ manifest.csv בתוכו מפרט איזה משפט בכל קובץ.`;
4885
+ }catch(err){ msg.textContent="ההורדה נכשלה: "+err.message; }
4886
+ offline.disabled=false;
4887
+ };
4888
  button.onclick=async()=>{
4889
  if(!confirm("למחוק את כל הקלטות התשובה מהמטמון? גם הקלטות שנבחרו ידנית יימחקו, ולא ניתן לבטל את הפעולה.")) return;
4890
  button.disabled=true; msg.textContent="מוחק…";
 
4898
  }catch(err){ msg.textContent="המחיקה נכשלה: "+err.message; }
4899
  button.disabled=false;
4900
  };
4901
+ row.append(warm,offline,button,msg); card.append(title,text,usage,progress,progressLabel,row); box.append(card);
4902
  }
4903
  // how to get each key, step by step: a step is its parts, text or [text, link]
4904
  const KEY_GUIDES={
tests/test_scoring.py CHANGED
@@ -530,7 +530,7 @@ def test_monitoring_can_warm_the_unique_tts_bank(monkeypatch):
530
  assert state == {"running": False, "voice_id": RAEF_VOICE_ID, "total": 2, "done": 2, "generated": 1,
531
  "skipped": 1, "failed": 0, "current": "", "waiting": 0, "error": "",
532
  "bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": ""}
533
- assert [voice for _text, voice in tts.calls] == [RAEF_VOICE_ID, RAEF_VOICE_ID]
534
 
535
 
536
  def test_warm_sweep_uploads_every_cached_wav_to_the_bucket(monkeypatch, tmp_path):
@@ -2257,3 +2257,32 @@ def test_an_account_is_sent_to_its_subjects_pages():
2257
  assert put("חשבון").status_code == 400 and auth.access["u2"]["subject"] == "מתמטיקה"
2258
  assert put(" ערבית ").json() == {"ok": True, "subject": "ערבית"} and auth.access["u2"]["subject"] == "ערבית"
2259
  assert c.get("/api/me", headers=tok("t")).json()["subject"] == "arabic"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
530
  assert state == {"running": False, "voice_id": RAEF_VOICE_ID, "total": 2, "done": 2, "generated": 1,
531
  "skipped": 1, "failed": 0, "current": "", "waiting": 0, "error": "",
532
  "bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": ""}
533
+ assert [voice for _text, voice in tts.calls] == [RAEF_VOICE_ID] # the cached one is not read again
534
 
535
 
536
  def test_warm_sweep_uploads_every_cached_wav_to_the_bucket(monkeypatch, tmp_path):
 
2257
  assert put("חשבון").status_code == 400 and auth.access["u2"]["subject"] == "מתמטיקה"
2258
  assert put(" ערבית ").json() == {"ok": True, "subject": "ערבית"} and auth.access["u2"]["subject"] == "ערבית"
2259
  assert c.get("/api/me", headers=tok("t")).json()["subject"] == "arabic"
2260
+
2261
+
2262
+ def test_the_tts_cache_downloads_as_an_offline_zip_with_a_manifest(monkeypatch, tmp_path):
2263
+ import io
2264
+ import zipfile
2265
+
2266
+ from fastapi.testclient import TestClient
2267
+
2268
+ import app.main
2269
+ from app.main import create_app
2270
+ from app.tts import RAEF_VOICE_ID, SawtakTTS
2271
+
2272
+ phrases = [{"he": "שלום", "tr": "מרחבא", "variants": [("مرحبا", "m a r ħ a b a")]}]
2273
+ monkeypatch.setattr(app.main, "PHRASES", phrases)
2274
+ monkeypatch.setattr(app.main, "load_auth", lambda: None)
2275
+ tts = SawtakTTS("key", cache_dir=tmp_path)
2276
+ monkeypatch.setattr(tts, "_generate", lambda text, voice: b"RIFF-" + text.encode())
2277
+ tts.wav(app.main.point_arabic(*phrases[0]["variants"][0]))
2278
+ (tmp_path / "stray.wav").write_bytes(b"old") # a file no phrase names is still included
2279
+ with TestClient(create_app(scorer_for(emissions(["w"])), tts, tutor=object())) as c:
2280
+ res = c.get("/api/tts-cache/download")
2281
+ assert res.status_code == 200 and res.headers["x-file-count"] == "2"
2282
+ zf = zipfile.ZipFile(io.BytesIO(res.content))
2283
+ names = sorted(zf.namelist())
2284
+ key = "ihki-tts/" + tts.cache_key(app.main.point_arabic(*phrases[0]["variants"][0]), RAEF_VOICE_ID)
2285
+ assert names == sorted([key, "ihki-tts/stray.wav", "manifest.csv"])
2286
+ assert zf.read(key).startswith(b"RIFF-")
2287
+ manifest = zf.read("manifest.csv").decode("utf-8-sig")
2288
+ assert "שלום" in manifest and RAEF_VOICE_ID in manifest and "ihki-tts/stray.wav" in manifest