feat(tts): implement local staging for new recordings and add offline cache import functionality
Browse files- README.md +1 -1
- app/main.py +65 -3
- app/tts.py +75 -28
- static/index.html +30 -4
- tests/test_scoring.py +73 -0
README.md
CHANGED
|
@@ -625,7 +625,7 @@ recordings may be retried because they never became a scored attempt. Session st
|
|
| 625 |
the application process: ending a session, restarting the server, or two hours without a teacher heartbeat erases them.
|
| 626 |
No live-session audio or result history is stored.
|
| 627 |
|
| 628 |
-
The **ניטור** activity dashboard can switch between the last 5 minutes, 24 hours and 7 days. It also reports Sawtak cache usage: cached WAV count, disk size, cache hits, generated misses, and the hit rate since the current server process started. **הכנת כל ההקלטות** asks for a Sawtak voice id and starts a resumable, sequential background pass over every unique phrase for that voice; existing files are skipped and progress is returned by `GET /api/tts-cache`. Recordings are stored below `TTS_CACHE_DIR/<voice-id>/`. When `TTS_BUCKET` is set (a Hugging Face Storage Bucket id, e.g. `wzender/ihki-storage`, with an `HF_TOKEN` secret that can write to it), the sweep also uploads every WAV through the Bucket API to `TTS_BUCKET_PREFIX/<voice-id>/` (default prefix `ihki-tts`), and at the end uploads every other WAV already on disk; files already in the bucket with the same size are skipped. Writing to a mounted `/data` folder alone was not reaching the bucket. Clearing the cache resets the process counters and leaves the bucket untouched.
|
| 629 |
|
| 630 |
The panel reads through `GET /api/class`, which answers teachers only. Row-level security hides other students' rows from the teacher's own token, so the server reads them with `SUPABASE_SECRET_KEY` (Supabase → Project Settings → API Keys → secret key). Unlike the publishable key it must stay private: set it as a Space **secret**. Without it the panel opens and says the data is not available.
|
| 631 |
|
|
|
|
| 625 |
the application process: ending a session, restarting the server, or two hours without a teacher heartbeat erases them.
|
| 626 |
No live-session audio or result history is stored.
|
| 627 |
|
| 628 |
+
The **ניטור** activity dashboard can switch between the last 5 minutes, 24 hours and 7 days. It also reports Sawtak cache usage: cached WAV count, disk size, cache hits, generated misses, and the hit rate since the current server process started. **הכנת כל ההקלטות** asks for a Sawtak voice id and starts a resumable, sequential background pass over every unique phrase for that voice; existing files are skipped and progress is returned by `GET /api/tts-cache`. Recordings are stored below `TTS_CACHE_DIR/<voice-id>/`. When `TTS_BUCKET` is set (a Hugging Face Storage Bucket id, e.g. `wzender/ihki-storage`, with an `HF_TOKEN` secret that can write to it), the sweep also uploads every WAV through the Bucket API to `TTS_BUCKET_PREFIX/<voice-id>/` (default prefix `ihki-tts`), and at the end uploads every other WAV already on disk; files already in the bucket with the same size are skipped. Writing to a mounted `/data` folder alone was not reaching the bucket, and was slow (the mount flushes a few files every ~30 s), so with `TTS_BUCKET` set new recordings are written to a local staging folder (`TTS_STAGING_DIR`, default `/tmp/ihki-tts-staging`) and uploaded through the API: the sweep in batches of 25 (at least every 30 s), playback misses one by one in the background. Lookups check the staging folder first, then `TTS_CACHE_DIR`. Clearing the cache resets the process counters and leaves the bucket untouched.
|
| 629 |
|
| 630 |
The panel reads through `GET /api/class`, which answers teachers only. Row-level security hides other students' rows from the teacher's own token, so the server reads them with `SUPABASE_SECRET_KEY` (Supabase → Project Settings → API Keys → secret key). Unlike the publishable key it must stay private: set it as a Space **secret**. Without it the panel opens and says the data is not available.
|
| 631 |
|
app/main.py
CHANGED
|
@@ -19,6 +19,8 @@ import json
|
|
| 19 |
import logging
|
| 20 |
import os
|
| 21 |
import re
|
|
|
|
|
|
|
| 22 |
import subprocess
|
| 23 |
import threading
|
| 24 |
import time
|
|
@@ -480,7 +482,8 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
|
|
| 480 |
app.state.tts = tts or load_tts(app.state.keys.get("SAWTAK_API_KEY"))
|
| 481 |
app.state.tts_warm = {"running": False, "voice_id": "", "total": 0, "done": 0, "generated": 0,
|
| 482 |
"skipped": 0, "failed": 0, "current": "", "waiting": 0, "error": "",
|
| 483 |
-
"bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": "", "stopped": False
|
|
|
|
| 484 |
app.state.tts_warm_lock = threading.Lock()
|
| 485 |
app.state.tts_warm_stop = threading.Event() # set by POST /api/tts-cache/warm/stop
|
| 486 |
app.state.tutor = tutor or load_tutor(app.state.keys.get("ANTHROPIC_API_KEY"), app.state.keys.get("TUTOR_MODEL"))
|
|
@@ -2520,6 +2523,62 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
|
|
| 2520 |
app.state.refs.invalidate()
|
| 2521 |
return {"ok": True, "removed": removed}
|
| 2522 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2523 |
@app.get("/api/tts-cache/download")
|
| 2524 |
def download_tts_cache(_a: SupabaseAuth = Depends(teacher_only)):
|
| 2525 |
"""An offline copy of the Sawtak recordings: every cached WAV, plus manifest.csv naming each file's phrase.
|
|
@@ -2597,7 +2656,7 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
|
|
| 2597 |
app.state.tts_warm = {"running": True, "voice_id": voice, "total": len(jobs), "done": 0, "generated": 0,
|
| 2598 |
"skipped": 0, "failed": 0, "current": "", "waiting": 0, "error": "",
|
| 2599 |
"bucket": mirror.bucket_id if mirror else "", "uploaded": 0, "upload_failed": 0,
|
| 2600 |
-
"upload_error": "", "stopped": False}
|
| 2601 |
stop = app.state.tts_warm_stop
|
| 2602 |
stop.clear()
|
| 2603 |
|
|
@@ -2605,6 +2664,7 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
|
|
| 2605 |
# mounted /data folder alone did not reach the bucket. Files already there with the same size are not resent.
|
| 2606 |
remote: dict[str, int] = {}
|
| 2607 |
pending: list[tuple[Path | bytes, str, int]] = []
|
|
|
|
| 2608 |
|
| 2609 |
def flush() -> None:
|
| 2610 |
if not pending:
|
|
@@ -2631,8 +2691,10 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
|
|
| 2631 |
dest = mirror.remote_path(rel)
|
| 2632 |
if remote.get(dest) == size or any(d == dest for _s, d, _z in pending):
|
| 2633 |
return
|
|
|
|
|
|
|
| 2634 |
pending.append((src, dest, size))
|
| 2635 |
-
if len(pending) >= 25:
|
| 2636 |
flush()
|
| 2637 |
|
| 2638 |
def run() -> None:
|
|
|
|
| 19 |
import logging
|
| 20 |
import os
|
| 21 |
import re
|
| 22 |
+
import shutil
|
| 23 |
+
import tempfile
|
| 24 |
import subprocess
|
| 25 |
import threading
|
| 26 |
import time
|
|
|
|
| 482 |
app.state.tts = tts or load_tts(app.state.keys.get("SAWTAK_API_KEY"))
|
| 483 |
app.state.tts_warm = {"running": False, "voice_id": "", "total": 0, "done": 0, "generated": 0,
|
| 484 |
"skipped": 0, "failed": 0, "current": "", "waiting": 0, "error": "",
|
| 485 |
+
"bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": "", "stopped": False,
|
| 486 |
+
"started_at": 0}
|
| 487 |
app.state.tts_warm_lock = threading.Lock()
|
| 488 |
app.state.tts_warm_stop = threading.Event() # set by POST /api/tts-cache/warm/stop
|
| 489 |
app.state.tutor = tutor or load_tutor(app.state.keys.get("ANTHROPIC_API_KEY"), app.state.keys.get("TUTOR_MODEL"))
|
|
|
|
| 2523 |
app.state.refs.invalidate()
|
| 2524 |
return {"ok": True, "removed": removed}
|
| 2525 |
|
| 2526 |
+
@app.post("/api/tts-cache/import")
|
| 2527 |
+
def import_tts_cache(archive: UploadFile = File(...), _a: SupabaseAuth = Depends(teacher_only)):
|
| 2528 |
+
"""Load an offline copy (a zip from GET /api/tts-cache/download) back into the cache: each WAV that is not
|
| 2529 |
+
cached yet is kept, then uploaded to the bucket in the background. manifest.csv and anything else is ignored.
|
| 2530 |
+
|
| 2531 |
+
Answers {"added", "skipped", "invalid", "uploading"}."""
|
| 2532 |
+
tts = app.state.tts
|
| 2533 |
+
if tts is None:
|
| 2534 |
+
raise HTTPException(503, "tts not configured")
|
| 2535 |
+
part = r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}"
|
| 2536 |
+
name_ok = re.compile(rf"(?:ihki-tts/)?((?:{part}/)?[0-9a-f]{{40}}\.wav)")
|
| 2537 |
+
added: list[tuple[Path, str]] = []
|
| 2538 |
+
skipped = invalid = 0
|
| 2539 |
+
spool = tempfile.TemporaryFile() # a plain file: zipfile seeks, and Python 3.10's spooled upload cannot
|
| 2540 |
+
try:
|
| 2541 |
+
shutil.copyfileobj(archive.file, spool, 1024 * 1024)
|
| 2542 |
+
spool.seek(0)
|
| 2543 |
+
with zipfile.ZipFile(spool) as zf:
|
| 2544 |
+
for info in zf.infolist():
|
| 2545 |
+
if info.is_dir() or info.filename == "manifest.csv":
|
| 2546 |
+
continue
|
| 2547 |
+
m = name_ok.fullmatch(info.filename)
|
| 2548 |
+
if not m or ".." in info.filename or info.file_size > 20 * 1024 * 1024:
|
| 2549 |
+
invalid += 1
|
| 2550 |
+
continue
|
| 2551 |
+
rel = m.group(1)
|
| 2552 |
+
existing = tts.has(rel)
|
| 2553 |
+
if existing is not None and existing.stat().st_size == info.file_size:
|
| 2554 |
+
skipped += 1
|
| 2555 |
+
continue
|
| 2556 |
+
data = zf.read(info)
|
| 2557 |
+
if data[:4] != b"RIFF" or data[8:12] != b"WAVE":
|
| 2558 |
+
invalid += 1
|
| 2559 |
+
continue
|
| 2560 |
+
added.append((tts.store(rel, data), rel))
|
| 2561 |
+
except zipfile.BadZipFile:
|
| 2562 |
+
raise HTTPException(400, "זה לא קובץ zip של עותק אופליין")
|
| 2563 |
+
except OSError as e:
|
| 2564 |
+
raise HTTPException(503, f"could not write the voice cache: {e}")
|
| 2565 |
+
finally:
|
| 2566 |
+
spool.close()
|
| 2567 |
+
mirror = getattr(tts, "mirror", None)
|
| 2568 |
+
if mirror is not None and added:
|
| 2569 |
+
def upload() -> None:
|
| 2570 |
+
for i in range(0, len(added), 25):
|
| 2571 |
+
batch = added[i:i + 25]
|
| 2572 |
+
try:
|
| 2573 |
+
mirror.upload([(path, mirror.remote_path(rel)) for path, rel in batch])
|
| 2574 |
+
except Exception as e: # the next cache-warm sweep uploads what is still missing
|
| 2575 |
+
log.warning("tts import: bucket upload of %d files failed: %s", len(batch), e)
|
| 2576 |
+
threading.Thread(target=upload, name="ihki-tts-import", daemon=True).start()
|
| 2577 |
+
if app.state.refs and added:
|
| 2578 |
+
app.state.refs.invalidate()
|
| 2579 |
+
return {"added": len(added), "skipped": skipped, "invalid": invalid,
|
| 2580 |
+
"uploading": bool(mirror is not None and added)}
|
| 2581 |
+
|
| 2582 |
@app.get("/api/tts-cache/download")
|
| 2583 |
def download_tts_cache(_a: SupabaseAuth = Depends(teacher_only)):
|
| 2584 |
"""An offline copy of the Sawtak recordings: every cached WAV, plus manifest.csv naming each file's phrase.
|
|
|
|
| 2656 |
app.state.tts_warm = {"running": True, "voice_id": voice, "total": len(jobs), "done": 0, "generated": 0,
|
| 2657 |
"skipped": 0, "failed": 0, "current": "", "waiting": 0, "error": "",
|
| 2658 |
"bucket": mirror.bucket_id if mirror else "", "uploaded": 0, "upload_failed": 0,
|
| 2659 |
+
"upload_error": "", "stopped": False, "started_at": time.time()}
|
| 2660 |
stop = app.state.tts_warm_stop
|
| 2661 |
stop.clear()
|
| 2662 |
|
|
|
|
| 2664 |
# mounted /data folder alone did not reach the bucket. Files already there with the same size are not resent.
|
| 2665 |
remote: dict[str, int] = {}
|
| 2666 |
pending: list[tuple[Path | bytes, str, int]] = []
|
| 2667 |
+
oldest = [0.0] # when the oldest pending file was queued: a batch waits at most 30 s for its 25th file
|
| 2668 |
|
| 2669 |
def flush() -> None:
|
| 2670 |
if not pending:
|
|
|
|
| 2691 |
dest = mirror.remote_path(rel)
|
| 2692 |
if remote.get(dest) == size or any(d == dest for _s, d, _z in pending):
|
| 2693 |
return
|
| 2694 |
+
if not pending:
|
| 2695 |
+
oldest[0] = time.monotonic()
|
| 2696 |
pending.append((src, dest, size))
|
| 2697 |
+
if len(pending) >= 25 or time.monotonic() - oldest[0] > 30:
|
| 2698 |
flush()
|
| 2699 |
|
| 2700 |
def run() -> None:
|
app/tts.py
CHANGED
|
@@ -8,6 +8,8 @@ Env: SAWTAK_API_KEY required; without it /api/tts answers 503 and the page fa
|
|
| 8 |
TTS_BUCKET optional Hugging Face Storage Bucket id (e.g. "user/ihki-storage"); the cache-warm sweep
|
| 9 |
uploads every cached WAV to it through the Bucket API (needs HF_TOKEN with write access)
|
| 10 |
TTS_BUCKET_PREFIX folder inside the bucket; default "ihki-tts"
|
|
|
|
|
|
|
| 11 |
"""
|
| 12 |
from __future__ import annotations
|
| 13 |
|
|
@@ -17,6 +19,7 @@ import io
|
|
| 17 |
import json
|
| 18 |
import logging
|
| 19 |
import os
|
|
|
|
| 20 |
import threading
|
| 21 |
import time
|
| 22 |
import urllib.error
|
|
@@ -88,6 +91,9 @@ class SawtakTTS:
|
|
| 88 |
self._last_request = 0.0
|
| 89 |
self._cache_hits = self._cache_misses = 0
|
| 90 |
self.mirror: BucketMirror | None = None # set by load_tts when TTS_BUCKET is configured
|
|
|
|
|
|
|
|
|
|
| 91 |
|
| 92 |
def _request(self, path: str, body: dict | None = None):
|
| 93 |
req = urllib.request.Request(
|
|
@@ -127,24 +133,47 @@ class SawtakTTS:
|
|
| 127 |
"""The recording's path relative to the cache folder ("<voice>/<sha1>.wav"), also its path in the bucket."""
|
| 128 |
return self._path(text, voice).relative_to(self.cache_dir).as_posix()
|
| 129 |
|
| 130 |
-
def
|
| 131 |
-
"""
|
|
|
|
| 132 |
path = self._path(text, voice)
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
|
|
|
|
|
|
| 137 |
|
| 138 |
def cached_files(self) -> list[tuple[Path, str]]:
|
| 139 |
-
"""Every cached WAV on disk
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
|
| 144 |
def cached(self, text: str, voice: str) -> bool:
|
| 145 |
"""Has this voice said `text` before? Then wav() answers from disk, without the provider."""
|
| 146 |
-
|
| 147 |
-
return path.exists() or self._legacy_path(path).exists()
|
| 148 |
|
| 149 |
def _generate(self, text: str, voice: str) -> bytes:
|
| 150 |
"""Generate one uncached WAV recording."""
|
|
@@ -184,11 +213,9 @@ class SawtakTTS:
|
|
| 184 |
def clear_cache(self) -> int:
|
| 185 |
"""Delete only generated audio files from this instance's dedicated cache directory."""
|
| 186 |
with self._lock:
|
| 187 |
-
if not self.cache_dir.is_dir():
|
| 188 |
-
self._cache_hits = self._cache_misses = 0
|
| 189 |
-
return 0
|
| 190 |
removed = 0
|
| 191 |
-
for
|
|
|
|
| 192 |
if path.is_file() and (path.suffix == ".wav" or path.name.endswith(".tmp")):
|
| 193 |
try:
|
| 194 |
path.unlink()
|
|
@@ -204,25 +231,28 @@ class SawtakTTS:
|
|
| 204 |
"""Anonymous disk and process-lifetime cache usage for the monitoring panel."""
|
| 205 |
with self._lock:
|
| 206 |
files = size = 0
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
except OSError as e:
|
| 214 |
-
raise TTSUnavailable(f"could not inspect voice cache: {e}") from None
|
| 215 |
requests = self._cache_hits + self._cache_misses
|
| 216 |
return {"files": files, "bytes": size, "hits": self._cache_hits,
|
| 217 |
"misses": self._cache_misses,
|
| 218 |
"hit_rate": round(self._cache_hits / requests, 4) if requests else None}
|
| 219 |
|
| 220 |
-
def _wav(self, text: str, voice: str) -> tuple[bytes, str]:
|
| 221 |
-
"""Generate or read `text` for an already validated voice id.
|
|
|
|
| 222 |
with self._lock:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
path = self._path(text, voice)
|
| 224 |
legacy = self._legacy_path(path)
|
| 225 |
-
if not path.exists() and legacy.exists():
|
| 226 |
try:
|
| 227 |
path.parent.mkdir(parents=True, exist_ok=True)
|
| 228 |
legacy.replace(path)
|
|
@@ -236,6 +266,8 @@ class SawtakTTS:
|
|
| 236 |
# The caller supplies the phrase's Palestinian vowel marks. Sawtak preserves existing
|
| 237 |
# marks; enhancement fills remaining pronunciation cues using the selected voice's dialect.
|
| 238 |
data = self._generate(text, voice)
|
|
|
|
|
|
|
| 239 |
try:
|
| 240 |
path.parent.mkdir(parents=True, exist_ok=True)
|
| 241 |
tmp = path.with_suffix(".tmp")
|
|
@@ -243,12 +275,25 @@ class SawtakTTS:
|
|
| 243 |
tmp.replace(path)
|
| 244 |
except OSError as e: # read-only or full cache dir: the audio is still good, it just isn't kept
|
| 245 |
log.warning("tts cache write failed: %s: %s", type(e).__name__, e)
|
|
|
|
|
|
|
| 246 |
return data, voice
|
| 247 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 248 |
def wav(self, text: str, voice: str = "") -> tuple[bytes, str]:
|
| 249 |
"""(WAV, voice id) in Raef's voice, or a dialogue line's configured voice."""
|
| 250 |
voice = voice if voice in LINE_VOICES.values() else RAEF_VOICE_ID
|
| 251 |
-
return self._wav(text, voice)
|
| 252 |
|
| 253 |
def wav_for_voice(self, text: str, voice: str) -> tuple[bytes, str]:
|
| 254 |
"""(WAV, voice id) for a teacher-selected, validated Sawtak voice."""
|
|
@@ -267,4 +312,6 @@ def load_tts(key: str | None = None) -> SawtakTTS | None:
|
|
| 267 |
if bucket:
|
| 268 |
tts.mirror = BucketMirror(bucket, os.environ.get("TTS_BUCKET_PREFIX", "ihki-tts"),
|
| 269 |
os.environ.get("HF_TOKEN", "").strip() or None)
|
|
|
|
|
|
|
| 270 |
return tts
|
|
|
|
| 8 |
TTS_BUCKET optional Hugging Face Storage Bucket id (e.g. "user/ihki-storage"); the cache-warm sweep
|
| 9 |
uploads every cached WAV to it through the Bucket API (needs HF_TOKEN with write access)
|
| 10 |
TTS_BUCKET_PREFIX folder inside the bucket; default "ihki-tts"
|
| 11 |
+
TTS_STAGING_DIR with TTS_BUCKET: a fast local folder new recordings are written to before their upload;
|
| 12 |
+
default <tmp>/ihki-tts-staging. A bucket mounted at /data takes ~10 s per written file.
|
| 13 |
"""
|
| 14 |
from __future__ import annotations
|
| 15 |
|
|
|
|
| 19 |
import json
|
| 20 |
import logging
|
| 21 |
import os
|
| 22 |
+
import tempfile
|
| 23 |
import threading
|
| 24 |
import time
|
| 25 |
import urllib.error
|
|
|
|
| 91 |
self._last_request = 0.0
|
| 92 |
self._cache_hits = self._cache_misses = 0
|
| 93 |
self.mirror: BucketMirror | None = None # set by load_tts when TTS_BUCKET is configured
|
| 94 |
+
# With a mirror, new recordings are written here (local disk) and uploaded through the Bucket API, instead
|
| 95 |
+
# of into cache_dir: a bucket mounted there only flushes a few files every ~30 s, so each write crawled.
|
| 96 |
+
self.staging_dir: Path | None = None
|
| 97 |
|
| 98 |
def _request(self, path: str, body: dict | None = None):
|
| 99 |
req = urllib.request.Request(
|
|
|
|
| 133 |
"""The recording's path relative to the cache folder ("<voice>/<sha1>.wav"), also its path in the bucket."""
|
| 134 |
return self._path(text, voice).relative_to(self.cache_dir).as_posix()
|
| 135 |
|
| 136 |
+
def _candidates(self, text: str, voice: str) -> list[Path]:
|
| 137 |
+
"""Where a recording may be: the local staging folder first (fast), then the cache folder, then its
|
| 138 |
+
legacy flat layout."""
|
| 139 |
path = self._path(text, voice)
|
| 140 |
+
staged = [self.staging_dir / self.cache_key(text, voice)] if self.staging_dir else []
|
| 141 |
+
return staged + [path, self._legacy_path(path)]
|
| 142 |
+
|
| 143 |
+
def cache_file(self, text: str, voice: str) -> Path | None:
|
| 144 |
+
"""The cached WAV on disk, wherever it is; None when not cached."""
|
| 145 |
+
return next((c for c in self._candidates(text, voice) if c.is_file()), None)
|
| 146 |
|
| 147 |
def cached_files(self) -> list[tuple[Path, str]]:
|
| 148 |
+
"""Every cached WAV on disk with its path relative to its folder; a staged copy wins over the cache's."""
|
| 149 |
+
found: dict[str, Path] = {}
|
| 150 |
+
for root in (self.cache_dir, self.staging_dir): # staging last, so it overrides
|
| 151 |
+
if root is not None and root.is_dir():
|
| 152 |
+
for p in root.rglob("*.wav"):
|
| 153 |
+
if p.is_file():
|
| 154 |
+
found[p.relative_to(root).as_posix()] = p
|
| 155 |
+
return [(found[rel], rel) for rel in sorted(found)]
|
| 156 |
+
|
| 157 |
+
def store(self, rel: str, data: bytes) -> Path:
|
| 158 |
+
"""Keep a recording from an offline copy at `rel` ("<voice>/<sha1>.wav" or a legacy "<sha1>.wav"): in the
|
| 159 |
+
staging folder with a mirror (the slow mount is not written), else in the cache folder."""
|
| 160 |
+
path = (self.staging_dir or self.cache_dir) / rel
|
| 161 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 162 |
+
tmp = path.with_suffix(".tmp")
|
| 163 |
+
tmp.write_bytes(data)
|
| 164 |
+
tmp.replace(path)
|
| 165 |
+
return path
|
| 166 |
+
|
| 167 |
+
def has(self, rel: str) -> Path | None:
|
| 168 |
+
"""The cached file at `rel`, staged or in the cache folder; None when neither has it."""
|
| 169 |
+
for root in (self.staging_dir, self.cache_dir):
|
| 170 |
+
if root is not None and (root / rel).is_file():
|
| 171 |
+
return root / rel
|
| 172 |
+
return None
|
| 173 |
|
| 174 |
def cached(self, text: str, voice: str) -> bool:
|
| 175 |
"""Has this voice said `text` before? Then wav() answers from disk, without the provider."""
|
| 176 |
+
return any(c.exists() for c in self._candidates(text, voice))
|
|
|
|
| 177 |
|
| 178 |
def _generate(self, text: str, voice: str) -> bytes:
|
| 179 |
"""Generate one uncached WAV recording."""
|
|
|
|
| 213 |
def clear_cache(self) -> int:
|
| 214 |
"""Delete only generated audio files from this instance's dedicated cache directory."""
|
| 215 |
with self._lock:
|
|
|
|
|
|
|
|
|
|
| 216 |
removed = 0
|
| 217 |
+
roots = [r for r in (self.cache_dir, self.staging_dir) if r is not None and r.is_dir()]
|
| 218 |
+
for path in (p for root in roots for p in root.rglob("*")):
|
| 219 |
if path.is_file() and (path.suffix == ".wav" or path.name.endswith(".tmp")):
|
| 220 |
try:
|
| 221 |
path.unlink()
|
|
|
|
| 231 |
"""Anonymous disk and process-lifetime cache usage for the monitoring panel."""
|
| 232 |
with self._lock:
|
| 233 |
files = size = 0
|
| 234 |
+
try:
|
| 235 |
+
for path, _rel in self.cached_files():
|
| 236 |
+
files += 1
|
| 237 |
+
size += path.stat().st_size
|
| 238 |
+
except OSError as e:
|
| 239 |
+
raise TTSUnavailable(f"could not inspect voice cache: {e}") from None
|
|
|
|
|
|
|
| 240 |
requests = self._cache_hits + self._cache_misses
|
| 241 |
return {"files": files, "bytes": size, "hits": self._cache_hits,
|
| 242 |
"misses": self._cache_misses,
|
| 243 |
"hit_rate": round(self._cache_hits / requests, 4) if requests else None}
|
| 244 |
|
| 245 |
+
def _wav(self, text: str, voice: str, upload: bool = False) -> tuple[bytes, str]:
|
| 246 |
+
"""Generate or read `text` for an already validated voice id. `upload`: send a new recording to the bucket
|
| 247 |
+
in the background (playback); the cache-warm sweep uploads its own in batches."""
|
| 248 |
with self._lock:
|
| 249 |
+
found = self.cache_file(text, voice) if self.staging_dir else None
|
| 250 |
+
if found is not None: # with a mirror the cache folder may be a slow mount: no migration there
|
| 251 |
+
self._cache_hits += 1
|
| 252 |
+
return found.read_bytes(), voice
|
| 253 |
path = self._path(text, voice)
|
| 254 |
legacy = self._legacy_path(path)
|
| 255 |
+
if self.staging_dir is None and not path.exists() and legacy.exists():
|
| 256 |
try:
|
| 257 |
path.parent.mkdir(parents=True, exist_ok=True)
|
| 258 |
legacy.replace(path)
|
|
|
|
| 266 |
# The caller supplies the phrase's Palestinian vowel marks. Sawtak preserves existing
|
| 267 |
# marks; enhancement fills remaining pronunciation cues using the selected voice's dialect.
|
| 268 |
data = self._generate(text, voice)
|
| 269 |
+
if self.staging_dir is not None:
|
| 270 |
+
path = self.staging_dir / self.cache_key(text, voice)
|
| 271 |
try:
|
| 272 |
path.parent.mkdir(parents=True, exist_ok=True)
|
| 273 |
tmp = path.with_suffix(".tmp")
|
|
|
|
| 275 |
tmp.replace(path)
|
| 276 |
except OSError as e: # read-only or full cache dir: the audio is still good, it just isn't kept
|
| 277 |
log.warning("tts cache write failed: %s: %s", type(e).__name__, e)
|
| 278 |
+
if upload and self.mirror is not None:
|
| 279 |
+
self._upload_soon(data, self.cache_key(text, voice))
|
| 280 |
return data, voice
|
| 281 |
|
| 282 |
+
def _upload_soon(self, data: bytes, rel: str) -> None:
|
| 283 |
+
"""One playback recording to the bucket, off the request: the student does not wait for it."""
|
| 284 |
+
mirror = self.mirror
|
| 285 |
+
|
| 286 |
+
def go() -> None:
|
| 287 |
+
try:
|
| 288 |
+
mirror.upload([(data, mirror.remote_path(rel))])
|
| 289 |
+
except Exception as e: # the next cache-warm sweep uploads what is still missing
|
| 290 |
+
log.warning("tts bucket upload of %s failed: %s", rel, e)
|
| 291 |
+
threading.Thread(target=go, name="ihki-tts-upload", daemon=True).start()
|
| 292 |
+
|
| 293 |
def wav(self, text: str, voice: str = "") -> tuple[bytes, str]:
|
| 294 |
"""(WAV, voice id) in Raef's voice, or a dialogue line's configured voice."""
|
| 295 |
voice = voice if voice in LINE_VOICES.values() else RAEF_VOICE_ID
|
| 296 |
+
return self._wav(text, voice, upload=True)
|
| 297 |
|
| 298 |
def wav_for_voice(self, text: str, voice: str) -> tuple[bytes, str]:
|
| 299 |
"""(WAV, voice id) for a teacher-selected, validated Sawtak voice."""
|
|
|
|
| 312 |
if bucket:
|
| 313 |
tts.mirror = BucketMirror(bucket, os.environ.get("TTS_BUCKET_PREFIX", "ihki-tts"),
|
| 314 |
os.environ.get("HF_TOKEN", "").strip() or None)
|
| 315 |
+
staging = os.environ.get("TTS_STAGING_DIR", "").strip()
|
| 316 |
+
tts.staging_dir = Path(staging) if staging else Path(tempfile.gettempdir()) / "ihki-tts-staging"
|
| 317 |
return tts
|
static/index.html
CHANGED
|
@@ -4865,10 +4865,22 @@ function drawSystemActivity(box){
|
|
| 4865 |
}
|
| 4866 |
function drawTtsCacheControls(box){
|
| 4867 |
const card=mark("card",""), title=mark("","מטמון הקלטות Sawtak","h2"), text=mark("","אפשר להכין מראש את כל מאגר ההקלטות. התהליך מדלג על קבצים קיימים וממשיך מהם בהפעלה הבאה. מחיקה גורמת ליצירה מחדש.","p");
|
| 4868 |
-
const usage=mark("activity-stats",""), progress=document.createElement("progress"), progressLabel=mark("msg",""), row=mark("row",""), warm=mark("btn primary","הכנת כל ההקלטות","button"), button=mark("btn","מחיקת מטמון ההקלטות","button"), offline=mark("btn","הורדת עותק אופליין","button"), halt=mark("btn","עצירת ההכנה","button"), msg=mark("msg","");
|
|
|
|
| 4869 |
progress.max=100; progress.value=0; progress.hidden=true; progress.style.width="100%"; progress.setAttribute("aria-label","התקדמות הכנת הקלטות Sawtak"); progressLabel.hidden=true;
|
| 4870 |
-
let polling=false;
|
| 4871 |
-
const
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4872 |
const show=data=>{ usage.replaceChildren(); const size=data.bytes<1024?`${data.bytes} B`:data.bytes<1048576?`${(data.bytes/1024).toFixed(1)} KB`:`${(data.bytes/1048576).toFixed(1)} MB`, rate=data.hit_rate==null?"–":`${Math.round(data.hit_rate*100)}%`; for(const [n,label] of [[data.files,"קבצי שמע במטמון"],[size,"נפח בדיסק"],[data.hits,"פגיעות מאז ההפעלה"],[data.misses,"יצירות מאז ההפעלה"],[rate,"שיעור פגיעה"]]){ const stat=mark("activity-stat",""); stat.append(mark("",n,"b"),mark("",label,"small")); usage.append(stat); } if(!data.available) msg.textContent="Sawtak אינו מוגדר בשרת."; };
|
| 4873 |
const poll=()=>api("/api/tts-cache").then(async res=>{const data=await res.json().catch(()=>({}));if(!res.ok) throw new Error(data.detail||res.status);show(data);polling=false;warmState(data.warm);}).catch(err=>{polling=false;msg.textContent="נתוני המטמון לא זמינים: "+err.message;});
|
| 4874 |
poll();
|
|
@@ -4880,6 +4892,20 @@ function drawTtsCacheControls(box){
|
|
| 4880 |
catch(err){ msg.textContent="העצירה נכשלה: "+err.message; }
|
| 4881 |
halt.disabled=false;
|
| 4882 |
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4883 |
offline.onclick=async()=>{ // every cached WAV + manifest.csv (which phrase each file is), streamed as one zip
|
| 4884 |
offline.disabled=true; msg.textContent="מכין עותק אופליין…";
|
| 4885 |
try{
|
|
@@ -4905,7 +4931,7 @@ function drawTtsCacheControls(box){
|
|
| 4905 |
}catch(err){ msg.textContent="המחיקה נכשלה: "+err.message; }
|
| 4906 |
button.disabled=false;
|
| 4907 |
};
|
| 4908 |
-
row.append(warm,halt,offline,button,msg); card.append(title,text,usage,progress,progressLabel,row); box.append(card);
|
| 4909 |
}
|
| 4910 |
// how to get each key, step by step: a step is its parts, text or [text, link]
|
| 4911 |
const KEY_GUIDES={
|
|
|
|
| 4865 |
}
|
| 4866 |
function drawTtsCacheControls(box){
|
| 4867 |
const card=mark("card",""), title=mark("","מטמון הקלטות Sawtak","h2"), text=mark("","אפשר להכין מראש את כל מאגר ההקלטות. התהליך מדלג על קבצים קיימים וממשיך מהם בהפעלה הבאה. מחיקה גורמת ליצירה מחדש.","p");
|
| 4868 |
+
const usage=mark("activity-stats",""), progress=document.createElement("progress"), progressLabel=mark("msg",""), row=mark("row",""), warm=mark("btn primary","הכנת כל ההקלטות","button"), button=mark("btn","מחיקת מטמון ההקלטות","button"), offline=mark("btn","הורדת עותק אופליין","button"), halt=mark("btn","עצירת ההכנה","button"), load=mark("btn","טעינת עותק אופליין","button"), loadFile=document.createElement("input"), msg=mark("msg","");
|
| 4869 |
+
loadFile.type="file"; loadFile.accept=".zip,application/zip"; loadFile.hidden=true;
|
| 4870 |
progress.max=100; progress.value=0; progress.hidden=true; progress.style.width="100%"; progress.setAttribute("aria-label","התקדמות הכנת הקלטות Sawtak"); progressLabel.hidden=true;
|
| 4871 |
+
let polling=false, samples=[]; // samples: recent [time, done] while the sweep runs, for the time estimate
|
| 4872 |
+
const duration=sec=>{ sec=Math.round(sec); if(sec<60) return "פחות מדקה"; const m=Math.round(sec/60); if(m<60) return `כ־${m} דקות`; const h=Math.floor(m/60),r=m%60; return r?`כ־${h} שעות ו־${r} דקות`:`כ־${h} שעות`; };
|
| 4873 |
+
const estimate=s=>{ // from the pace of the last 3 minutes (cached phrases fly, new ones wait for Sawtak); else since the start
|
| 4874 |
+
const now=Date.now(); if(!s.running){ samples=[]; return s.started_at&&s.total&&s.done===s.total?` · ההכנה ארכה ${duration(now/1000-s.started_at)}`:""; }
|
| 4875 |
+
samples.push([now,s.done]); samples=samples.filter(([t])=>now-t<=180000);
|
| 4876 |
+
const [t0,d0]=samples[0], left=s.total-s.done; let rate=0;
|
| 4877 |
+
if(now-t0>=20000&&s.done>d0) rate=(s.done-d0)/((now-t0)/1000);
|
| 4878 |
+
else if(s.started_at&&s.done) rate=s.done/Math.max(1,now/1000-s.started_at);
|
| 4879 |
+
if(!left) return ""; if(!rate) return " · מחשב זמן משוער…";
|
| 4880 |
+
const end=new Date(now+left/rate*1000).toLocaleTimeString("he-IL",{hour:"2-digit",minute:"2-digit"});
|
| 4881 |
+
return ` · נותרו ${duration(left/rate)} (סיום משוער ${end})`;
|
| 4882 |
+
};
|
| 4883 |
+
const warmState=s=>{ if(!s) return; warm.disabled=button.disabled=!!s.running; halt.hidden=!s.running; const pct=s.total?Math.round(100*s.done/s.total):0,voice=s.voice_id?` · קול ${s.voice_id}`:""; progress.hidden=progressLabel.hidden=!s.total; progress.value=pct; progressLabel.textContent=`${pct}% · ${s.done} מתוך ${s.total} · נוצרו ${s.generated} · כבר היו ${s.skipped}`+(s.bucket?` · הועלו לדלי ${s.uploaded||0}`+(s.upload_failed?` · נכשלו בהעלאה ${s.upload_failed}`:""):" · אין דלי מוגדר (TTS_BUCKET)")+(s.upload_error?` · ${s.upload_error}`:"")+estimate(s); if(s.running) msg.textContent=s.waiting?`Sawtak לא ענה כרגע (הגבלת קצב או תקלת רשת); ניסיון נוסף בעוד ${s.waiting} שניות${voice}`:`מכין הקלטות מול Sawtak${voice}`; else if(s.stopped) msg.textContent=`ההכנה נעצרה לבקשתך אחרי ${s.done} הקלטות. הפעלה חוזרת תמשיך מאותה נקודה${voice}.`; else if(s.error) msg.textContent=`ההכנה נעצרה אחרי ${s.done} הקלטות: ${s.error}${voice}`; else if(s.total&&s.done===s.total) msg.textContent=`המאגר מוכן${voice}.`; if(s.running&&!polling){polling=true; setTimeout(poll,2000);} };
|
| 4884 |
const show=data=>{ usage.replaceChildren(); const size=data.bytes<1024?`${data.bytes} B`:data.bytes<1048576?`${(data.bytes/1024).toFixed(1)} KB`:`${(data.bytes/1048576).toFixed(1)} MB`, rate=data.hit_rate==null?"–":`${Math.round(data.hit_rate*100)}%`; for(const [n,label] of [[data.files,"קבצי שמע במטמון"],[size,"נפח בדיסק"],[data.hits,"פגיעות מאז ההפעלה"],[data.misses,"יצירות מאז ההפעלה"],[rate,"שיעור פגיעה"]]){ const stat=mark("activity-stat",""); stat.append(mark("",n,"b"),mark("",label,"small")); usage.append(stat); } if(!data.available) msg.textContent="Sawtak אינו מוגדר בשרת."; };
|
| 4885 |
const poll=()=>api("/api/tts-cache").then(async res=>{const data=await res.json().catch(()=>({}));if(!res.ok) throw new Error(data.detail||res.status);show(data);polling=false;warmState(data.warm);}).catch(err=>{polling=false;msg.textContent="נתוני המטמון לא זמינים: "+err.message;});
|
| 4886 |
poll();
|
|
|
|
| 4892 |
catch(err){ msg.textContent="העצירה נכשלה: "+err.message; }
|
| 4893 |
halt.disabled=false;
|
| 4894 |
};
|
| 4895 |
+
load.onclick=()=>{ loadFile.value=""; loadFile.click(); };
|
| 4896 |
+
loadFile.onchange=async()=>{ // a zip from "הורדת עותק אופליין": its recordings back into the cache, then to the bucket
|
| 4897 |
+
const file=loadFile.files[0]; if(!file) return;
|
| 4898 |
+
if(!confirm(`לטעון ${file.name} (${(file.size/1048576).toFixed(1)} MB)? הקלטות שכבר קיימות ידולגו.`)) return;
|
| 4899 |
+
load.disabled=true; msg.textContent=`מעלה ${file.name}…`;
|
| 4900 |
+
try{
|
| 4901 |
+
const form=new FormData(); form.append("archive",file);
|
| 4902 |
+
const res=await api("/api/tts-cache/import",{method:"POST",body:form}),data=await res.json().catch(()=>({}));
|
| 4903 |
+
if(!res.ok) throw new Error(data.detail||res.status);
|
| 4904 |
+
msg.textContent=`נטענו ${data.added} הקלטות · ${data.skipped} כבר היו`+(data.invalid?` · ${data.invalid} קבצים לא תקינים דולגו`:"")+(data.uploading?" · ההעלאה לדלי ממשיכה ברקע.":".");
|
| 4905 |
+
poll();
|
| 4906 |
+
}catch(err){ msg.textContent="הטעינה נכשלה: "+err.message; }
|
| 4907 |
+
load.disabled=false;
|
| 4908 |
+
};
|
| 4909 |
offline.onclick=async()=>{ // every cached WAV + manifest.csv (which phrase each file is), streamed as one zip
|
| 4910 |
offline.disabled=true; msg.textContent="מכין עותק אופליין…";
|
| 4911 |
try{
|
|
|
|
| 4931 |
}catch(err){ msg.textContent="המחיקה נכשלה: "+err.message; }
|
| 4932 |
button.disabled=false;
|
| 4933 |
};
|
| 4934 |
+
row.append(warm,halt,offline,load,loadFile,button,msg); card.append(title,text,usage,progress,progressLabel,row); box.append(card);
|
| 4935 |
}
|
| 4936 |
// how to get each key, step by step: a step is its parts, text or [text, link]
|
| 4937 |
const KEY_GUIDES={
|
tests/test_scoring.py
CHANGED
|
@@ -527,6 +527,7 @@ def test_monitoring_can_warm_the_unique_tts_bank(monkeypatch):
|
|
| 527 |
state = c.get("/api/tts-cache").json()["warm"]
|
| 528 |
if not state["running"]: break
|
| 529 |
import time; time.sleep(.01)
|
|
|
|
| 530 |
assert state == {"running": False, "voice_id": RAEF_VOICE_ID, "total": 2, "done": 2, "generated": 1,
|
| 531 |
"skipped": 1, "failed": 0, "current": "", "waiting": 0, "error": "",
|
| 532 |
"bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": "", "stopped": False}
|
|
@@ -2321,3 +2322,75 @@ def test_the_warm_sweep_stops_on_request_and_keeps_what_it_made(monkeypatch):
|
|
| 2321 |
if not state["running"]: break
|
| 2322 |
time.sleep(.01)
|
| 2323 |
assert state["stopped"] is True and state["done"] == state["generated"] == 1 and len(tts.calls) == 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 527 |
state = c.get("/api/tts-cache").json()["warm"]
|
| 528 |
if not state["running"]: break
|
| 529 |
import time; time.sleep(.01)
|
| 530 |
+
assert state.pop("started_at") > 0
|
| 531 |
assert state == {"running": False, "voice_id": RAEF_VOICE_ID, "total": 2, "done": 2, "generated": 1,
|
| 532 |
"skipped": 1, "failed": 0, "current": "", "waiting": 0, "error": "",
|
| 533 |
"bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": "", "stopped": False}
|
|
|
|
| 2322 |
if not state["running"]: break
|
| 2323 |
time.sleep(.01)
|
| 2324 |
assert state["stopped"] is True and state["done"] == state["generated"] == 1 and len(tts.calls) == 1
|
| 2325 |
+
|
| 2326 |
+
|
| 2327 |
+
def test_with_a_bucket_new_recordings_are_staged_locally_not_written_to_the_mount(monkeypatch, tmp_path):
|
| 2328 |
+
from app.tts import RAEF_VOICE_ID, BucketMirror, SawtakTTS
|
| 2329 |
+
|
| 2330 |
+
mount, staging = tmp_path / "mount", tmp_path / "staging"
|
| 2331 |
+
tts = SawtakTTS("key", cache_dir=mount)
|
| 2332 |
+
tts.staging_dir = staging
|
| 2333 |
+
monkeypatch.setattr(tts, "_generate", lambda text, voice: b"RIFF-" + text.encode())
|
| 2334 |
+
uploaded = []
|
| 2335 |
+
|
| 2336 |
+
class Mirror(BucketMirror):
|
| 2337 |
+
def upload(self, files): uploaded.extend(dest for _src, dest in files)
|
| 2338 |
+
tts.mirror = Mirror("user/bucket", "ihki-tts")
|
| 2339 |
+
|
| 2340 |
+
data, _ = tts.wav("مرحبا")
|
| 2341 |
+
key = tts.cache_key("مرحبا", RAEF_VOICE_ID)
|
| 2342 |
+
assert (staging / key).read_bytes() == data and not mount.exists() # the slow mount is not written
|
| 2343 |
+
assert tts.cached("مرحبا", RAEF_VOICE_ID) and tts.cache_file("مرحبا", RAEF_VOICE_ID) == staging / key
|
| 2344 |
+
import time
|
| 2345 |
+
for _ in range(100):
|
| 2346 |
+
if uploaded: break
|
| 2347 |
+
time.sleep(.01)
|
| 2348 |
+
assert uploaded == [f"ihki-tts/{key}"] # playback uploads in the background
|
| 2349 |
+
(mount / RAEF_VOICE_ID).mkdir(parents=True) # the bucket's copy appears in the mount
|
| 2350 |
+
(mount / key).write_bytes(data)
|
| 2351 |
+
assert [rel for _p, rel in tts.cached_files()] == [key] # counted once
|
| 2352 |
+
assert tts.cache_stats()["files"] == 1
|
| 2353 |
+
assert tts.wav("مرحبا")[0] == data and tts.cache_stats()["hits"] == 1
|
| 2354 |
+
assert tts.clear_cache() == 2 and not tts.cached("مرحبا", RAEF_VOICE_ID)
|
| 2355 |
+
|
| 2356 |
+
|
| 2357 |
+
def test_an_offline_copy_loads_back_into_the_cache_and_the_bucket(monkeypatch, tmp_path):
|
| 2358 |
+
import io
|
| 2359 |
+
import time
|
| 2360 |
+
import zipfile
|
| 2361 |
+
|
| 2362 |
+
from fastapi.testclient import TestClient
|
| 2363 |
+
|
| 2364 |
+
import app.main
|
| 2365 |
+
from app.main import create_app
|
| 2366 |
+
from app.tts import RAEF_VOICE_ID, BucketMirror, SawtakTTS
|
| 2367 |
+
|
| 2368 |
+
monkeypatch.setattr(app.main, "load_auth", lambda: None)
|
| 2369 |
+
tts = SawtakTTS("key", cache_dir=tmp_path / "mount")
|
| 2370 |
+
tts.staging_dir = tmp_path / "staging"
|
| 2371 |
+
uploaded = []
|
| 2372 |
+
|
| 2373 |
+
class Mirror(BucketMirror):
|
| 2374 |
+
def upload(self, files): uploaded.extend(dest for _src, dest in files)
|
| 2375 |
+
tts.mirror = Mirror("user/bucket", "ihki-tts")
|
| 2376 |
+
wav = b"RIFF\0\0\0\0WAVEfmt " + b"\0" * 40
|
| 2377 |
+
new, old, legacy = f"{RAEF_VOICE_ID}/{'a' * 40}.wav", f"{RAEF_VOICE_ID}/{'b' * 40}.wav", f"{'c' * 40}.wav"
|
| 2378 |
+
tts.store(old, wav) # already cached: skipped
|
| 2379 |
+
buf = io.BytesIO()
|
| 2380 |
+
with zipfile.ZipFile(buf, "w") as zf:
|
| 2381 |
+
zf.writestr(f"ihki-tts/{new}", wav)
|
| 2382 |
+
zf.writestr(f"ihki-tts/{old}", wav)
|
| 2383 |
+
zf.writestr(f"ihki-tts/{legacy}", wav)
|
| 2384 |
+
zf.writestr("manifest.csv", "file\n") # ignored
|
| 2385 |
+
zf.writestr("ihki-tts/../../evil.wav", wav) # refused: outside the cache
|
| 2386 |
+
zf.writestr(f"ihki-tts/{RAEF_VOICE_ID}/{'d' * 40}.wav", b"not audio") # refused: not a WAV
|
| 2387 |
+
with TestClient(create_app(scorer_for(emissions(["w"])), tts, tutor=object())) as c:
|
| 2388 |
+
res = c.post("/api/tts-cache/import", files={"archive": ("copy.zip", buf.getvalue(), "application/zip")})
|
| 2389 |
+
assert res.status_code == 200, res.text
|
| 2390 |
+
assert res.json() == {"added": 2, "skipped": 1, "invalid": 2, "uploading": True}
|
| 2391 |
+
assert c.post("/api/tts-cache/import", files={"archive": ("x.zip", b"nope", "application/zip")}).status_code == 400
|
| 2392 |
+
assert tts.has(new) == tts.staging_dir / new and tts.has(legacy) and not (tmp_path / "evil.wav").exists()
|
| 2393 |
+
for _ in range(100):
|
| 2394 |
+
if len(uploaded) == 2: break
|
| 2395 |
+
time.sleep(.01)
|
| 2396 |
+
assert sorted(uploaded) == sorted([f"ihki-tts/{new}", f"ihki-tts/{legacy}"])
|