3v324v23 commited on
Commit
075de80
·
1 Parent(s): 3d230e1

feat(tts): implement local staging for new recordings and add offline cache import functionality

Browse files
Files changed (5) hide show
  1. README.md +1 -1
  2. app/main.py +65 -3
  3. app/tts.py +75 -28
  4. static/index.html +30 -4
  5. tests/test_scoring.py +73 -0
README.md CHANGED
@@ -625,7 +625,7 @@ recordings may be retried because they never became a scored attempt. Session st
625
  the application process: ending a session, restarting the server, or two hours without a teacher heartbeat erases them.
626
  No live-session audio or result history is stored.
627
 
628
- The **ניטור** activity dashboard can switch between the last 5 minutes, 24 hours and 7 days. It also reports Sawtak cache usage: cached WAV count, disk size, cache hits, generated misses, and the hit rate since the current server process started. **הכנת כל ההקלטות** asks for a Sawtak voice id and starts a resumable, sequential background pass over every unique phrase for that voice; existing files are skipped and progress is returned by `GET /api/tts-cache`. Recordings are stored below `TTS_CACHE_DIR/<voice-id>/`. When `TTS_BUCKET` is set (a Hugging Face Storage Bucket id, e.g. `wzender/ihki-storage`, with an `HF_TOKEN` secret that can write to it), the sweep also uploads every WAV through the Bucket API to `TTS_BUCKET_PREFIX/<voice-id>/` (default prefix `ihki-tts`), and at the end uploads every other WAV already on disk; files already in the bucket with the same size are skipped. Writing to a mounted `/data` folder alone was not reaching the bucket. Clearing the cache resets the process counters and leaves the bucket untouched.
629
 
630
  The panel reads through `GET /api/class`, which answers teachers only. Row-level security hides other students' rows from the teacher's own token, so the server reads them with `SUPABASE_SECRET_KEY` (Supabase → Project Settings → API Keys → secret key). Unlike the publishable key it must stay private: set it as a Space **secret**. Without it the panel opens and says the data is not available.
631
 
 
625
  the application process: ending a session, restarting the server, or two hours without a teacher heartbeat erases them.
626
  No live-session audio or result history is stored.
627
 
628
+ The **ניטור** activity dashboard can switch between the last 5 minutes, 24 hours and 7 days. It also reports Sawtak cache usage: cached WAV count, disk size, cache hits, generated misses, and the hit rate since the current server process started. **הכנת כל ההקלטות** asks for a Sawtak voice id and starts a resumable, sequential background pass over every unique phrase for that voice; existing files are skipped and progress is returned by `GET /api/tts-cache`. Recordings are stored below `TTS_CACHE_DIR/<voice-id>/`. When `TTS_BUCKET` is set (a Hugging Face Storage Bucket id, e.g. `wzender/ihki-storage`, with an `HF_TOKEN` secret that can write to it), the sweep also uploads every WAV through the Bucket API to `TTS_BUCKET_PREFIX/<voice-id>/` (default prefix `ihki-tts`), and at the end uploads every other WAV already on disk; files already in the bucket with the same size are skipped. Writing to a mounted `/data` folder alone was not reaching the bucket, and was slow (the mount flushes a few files every ~30 s), so with `TTS_BUCKET` set new recordings are written to a local staging folder (`TTS_STAGING_DIR`, default `/tmp/ihki-tts-staging`) and uploaded through the API: the sweep in batches of 25 (at least every 30 s), playback misses one by one in the background. Lookups check the staging folder first, then `TTS_CACHE_DIR`. Clearing the cache resets the process counters and leaves the bucket untouched.
629
 
630
  The panel reads through `GET /api/class`, which answers teachers only. Row-level security hides other students' rows from the teacher's own token, so the server reads them with `SUPABASE_SECRET_KEY` (Supabase → Project Settings → API Keys → secret key). Unlike the publishable key it must stay private: set it as a Space **secret**. Without it the panel opens and says the data is not available.
631
 
app/main.py CHANGED
@@ -19,6 +19,8 @@ import json
19
  import logging
20
  import os
21
  import re
 
 
22
  import subprocess
23
  import threading
24
  import time
@@ -480,7 +482,8 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
480
  app.state.tts = tts or load_tts(app.state.keys.get("SAWTAK_API_KEY"))
481
  app.state.tts_warm = {"running": False, "voice_id": "", "total": 0, "done": 0, "generated": 0,
482
  "skipped": 0, "failed": 0, "current": "", "waiting": 0, "error": "",
483
- "bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": "", "stopped": False}
 
484
  app.state.tts_warm_lock = threading.Lock()
485
  app.state.tts_warm_stop = threading.Event() # set by POST /api/tts-cache/warm/stop
486
  app.state.tutor = tutor or load_tutor(app.state.keys.get("ANTHROPIC_API_KEY"), app.state.keys.get("TUTOR_MODEL"))
@@ -2520,6 +2523,62 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
2520
  app.state.refs.invalidate()
2521
  return {"ok": True, "removed": removed}
2522
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2523
  @app.get("/api/tts-cache/download")
2524
  def download_tts_cache(_a: SupabaseAuth = Depends(teacher_only)):
2525
  """An offline copy of the Sawtak recordings: every cached WAV, plus manifest.csv naming each file's phrase.
@@ -2597,7 +2656,7 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
2597
  app.state.tts_warm = {"running": True, "voice_id": voice, "total": len(jobs), "done": 0, "generated": 0,
2598
  "skipped": 0, "failed": 0, "current": "", "waiting": 0, "error": "",
2599
  "bucket": mirror.bucket_id if mirror else "", "uploaded": 0, "upload_failed": 0,
2600
- "upload_error": "", "stopped": False}
2601
  stop = app.state.tts_warm_stop
2602
  stop.clear()
2603
 
@@ -2605,6 +2664,7 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
2605
  # mounted /data folder alone did not reach the bucket. Files already there with the same size are not resent.
2606
  remote: dict[str, int] = {}
2607
  pending: list[tuple[Path | bytes, str, int]] = []
 
2608
 
2609
  def flush() -> None:
2610
  if not pending:
@@ -2631,8 +2691,10 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
2631
  dest = mirror.remote_path(rel)
2632
  if remote.get(dest) == size or any(d == dest for _s, d, _z in pending):
2633
  return
 
 
2634
  pending.append((src, dest, size))
2635
- if len(pending) >= 25:
2636
  flush()
2637
 
2638
  def run() -> None:
 
19
  import logging
20
  import os
21
  import re
22
+ import shutil
23
+ import tempfile
24
  import subprocess
25
  import threading
26
  import time
 
482
  app.state.tts = tts or load_tts(app.state.keys.get("SAWTAK_API_KEY"))
483
  app.state.tts_warm = {"running": False, "voice_id": "", "total": 0, "done": 0, "generated": 0,
484
  "skipped": 0, "failed": 0, "current": "", "waiting": 0, "error": "",
485
+ "bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": "", "stopped": False,
486
+ "started_at": 0}
487
  app.state.tts_warm_lock = threading.Lock()
488
  app.state.tts_warm_stop = threading.Event() # set by POST /api/tts-cache/warm/stop
489
  app.state.tutor = tutor or load_tutor(app.state.keys.get("ANTHROPIC_API_KEY"), app.state.keys.get("TUTOR_MODEL"))
 
2523
  app.state.refs.invalidate()
2524
  return {"ok": True, "removed": removed}
2525
 
2526
+ @app.post("/api/tts-cache/import")
2527
+ def import_tts_cache(archive: UploadFile = File(...), _a: SupabaseAuth = Depends(teacher_only)):
2528
+ """Load an offline copy (a zip from GET /api/tts-cache/download) back into the cache: each WAV that is not
2529
+ cached yet is kept, then uploaded to the bucket in the background. manifest.csv and anything else is ignored.
2530
+
2531
+ Answers {"added", "skipped", "invalid", "uploading"}."""
2532
+ tts = app.state.tts
2533
+ if tts is None:
2534
+ raise HTTPException(503, "tts not configured")
2535
+ part = r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}"
2536
+ name_ok = re.compile(rf"(?:ihki-tts/)?((?:{part}/)?[0-9a-f]{{40}}\.wav)")
2537
+ added: list[tuple[Path, str]] = []
2538
+ skipped = invalid = 0
2539
+ spool = tempfile.TemporaryFile() # a plain file: zipfile seeks, and Python 3.10's spooled upload cannot
2540
+ try:
2541
+ shutil.copyfileobj(archive.file, spool, 1024 * 1024)
2542
+ spool.seek(0)
2543
+ with zipfile.ZipFile(spool) as zf:
2544
+ for info in zf.infolist():
2545
+ if info.is_dir() or info.filename == "manifest.csv":
2546
+ continue
2547
+ m = name_ok.fullmatch(info.filename)
2548
+ if not m or ".." in info.filename or info.file_size > 20 * 1024 * 1024:
2549
+ invalid += 1
2550
+ continue
2551
+ rel = m.group(1)
2552
+ existing = tts.has(rel)
2553
+ if existing is not None and existing.stat().st_size == info.file_size:
2554
+ skipped += 1
2555
+ continue
2556
+ data = zf.read(info)
2557
+ if data[:4] != b"RIFF" or data[8:12] != b"WAVE":
2558
+ invalid += 1
2559
+ continue
2560
+ added.append((tts.store(rel, data), rel))
2561
+ except zipfile.BadZipFile:
2562
+ raise HTTPException(400, "זה לא קובץ zip של עותק אופליין")
2563
+ except OSError as e:
2564
+ raise HTTPException(503, f"could not write the voice cache: {e}")
2565
+ finally:
2566
+ spool.close()
2567
+ mirror = getattr(tts, "mirror", None)
2568
+ if mirror is not None and added:
2569
+ def upload() -> None:
2570
+ for i in range(0, len(added), 25):
2571
+ batch = added[i:i + 25]
2572
+ try:
2573
+ mirror.upload([(path, mirror.remote_path(rel)) for path, rel in batch])
2574
+ except Exception as e: # the next cache-warm sweep uploads what is still missing
2575
+ log.warning("tts import: bucket upload of %d files failed: %s", len(batch), e)
2576
+ threading.Thread(target=upload, name="ihki-tts-import", daemon=True).start()
2577
+ if app.state.refs and added:
2578
+ app.state.refs.invalidate()
2579
+ return {"added": len(added), "skipped": skipped, "invalid": invalid,
2580
+ "uploading": bool(mirror is not None and added)}
2581
+
2582
  @app.get("/api/tts-cache/download")
2583
  def download_tts_cache(_a: SupabaseAuth = Depends(teacher_only)):
2584
  """An offline copy of the Sawtak recordings: every cached WAV, plus manifest.csv naming each file's phrase.
 
2656
  app.state.tts_warm = {"running": True, "voice_id": voice, "total": len(jobs), "done": 0, "generated": 0,
2657
  "skipped": 0, "failed": 0, "current": "", "waiting": 0, "error": "",
2658
  "bucket": mirror.bucket_id if mirror else "", "uploaded": 0, "upload_failed": 0,
2659
+ "upload_error": "", "stopped": False, "started_at": time.time()}
2660
  stop = app.state.tts_warm_stop
2661
  stop.clear()
2662
 
 
2664
  # mounted /data folder alone did not reach the bucket. Files already there with the same size are not resent.
2665
  remote: dict[str, int] = {}
2666
  pending: list[tuple[Path | bytes, str, int]] = []
2667
+ oldest = [0.0] # when the oldest pending file was queued: a batch waits at most 30 s for its 25th file
2668
 
2669
  def flush() -> None:
2670
  if not pending:
 
2691
  dest = mirror.remote_path(rel)
2692
  if remote.get(dest) == size or any(d == dest for _s, d, _z in pending):
2693
  return
2694
+ if not pending:
2695
+ oldest[0] = time.monotonic()
2696
  pending.append((src, dest, size))
2697
+ if len(pending) >= 25 or time.monotonic() - oldest[0] > 30:
2698
  flush()
2699
 
2700
  def run() -> None:
app/tts.py CHANGED
@@ -8,6 +8,8 @@ Env: SAWTAK_API_KEY required; without it /api/tts answers 503 and the page fa
8
  TTS_BUCKET optional Hugging Face Storage Bucket id (e.g. "user/ihki-storage"); the cache-warm sweep
9
  uploads every cached WAV to it through the Bucket API (needs HF_TOKEN with write access)
10
  TTS_BUCKET_PREFIX folder inside the bucket; default "ihki-tts"
 
 
11
  """
12
  from __future__ import annotations
13
 
@@ -17,6 +19,7 @@ import io
17
  import json
18
  import logging
19
  import os
 
20
  import threading
21
  import time
22
  import urllib.error
@@ -88,6 +91,9 @@ class SawtakTTS:
88
  self._last_request = 0.0
89
  self._cache_hits = self._cache_misses = 0
90
  self.mirror: BucketMirror | None = None # set by load_tts when TTS_BUCKET is configured
 
 
 
91
 
92
  def _request(self, path: str, body: dict | None = None):
93
  req = urllib.request.Request(
@@ -127,24 +133,47 @@ class SawtakTTS:
127
  """The recording's path relative to the cache folder ("<voice>/<sha1>.wav"), also its path in the bucket."""
128
  return self._path(text, voice).relative_to(self.cache_dir).as_posix()
129
 
130
- def cache_file(self, text: str, voice: str) -> Path | None:
131
- """The cached WAV on disk, in the current or the legacy flat layout; None when not cached."""
 
132
  path = self._path(text, voice)
133
- for candidate in (path, self._legacy_path(path)):
134
- if candidate.is_file():
135
- return candidate
136
- return None
 
 
137
 
138
  def cached_files(self) -> list[tuple[Path, str]]:
139
- """Every cached WAV on disk, with its path relative to the cache folder."""
140
- if not self.cache_dir.is_dir():
141
- return []
142
- return [(p, p.relative_to(self.cache_dir).as_posix()) for p in sorted(self.cache_dir.rglob("*.wav")) if p.is_file()]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
143
 
144
  def cached(self, text: str, voice: str) -> bool:
145
  """Has this voice said `text` before? Then wav() answers from disk, without the provider."""
146
- path = self._path(text, voice)
147
- return path.exists() or self._legacy_path(path).exists()
148
 
149
  def _generate(self, text: str, voice: str) -> bytes:
150
  """Generate one uncached WAV recording."""
@@ -184,11 +213,9 @@ class SawtakTTS:
184
  def clear_cache(self) -> int:
185
  """Delete only generated audio files from this instance's dedicated cache directory."""
186
  with self._lock:
187
- if not self.cache_dir.is_dir():
188
- self._cache_hits = self._cache_misses = 0
189
- return 0
190
  removed = 0
191
- for path in self.cache_dir.rglob("*"):
 
192
  if path.is_file() and (path.suffix == ".wav" or path.name.endswith(".tmp")):
193
  try:
194
  path.unlink()
@@ -204,25 +231,28 @@ class SawtakTTS:
204
  """Anonymous disk and process-lifetime cache usage for the monitoring panel."""
205
  with self._lock:
206
  files = size = 0
207
- if self.cache_dir.is_dir():
208
- try:
209
- for path in self.cache_dir.rglob("*.wav"):
210
- if path.is_file() and path.suffix == ".wav":
211
- files += 1
212
- size += path.stat().st_size
213
- except OSError as e:
214
- raise TTSUnavailable(f"could not inspect voice cache: {e}") from None
215
  requests = self._cache_hits + self._cache_misses
216
  return {"files": files, "bytes": size, "hits": self._cache_hits,
217
  "misses": self._cache_misses,
218
  "hit_rate": round(self._cache_hits / requests, 4) if requests else None}
219
 
220
- def _wav(self, text: str, voice: str) -> tuple[bytes, str]:
221
- """Generate or read `text` for an already validated voice id."""
 
222
  with self._lock:
 
 
 
 
223
  path = self._path(text, voice)
224
  legacy = self._legacy_path(path)
225
- if not path.exists() and legacy.exists():
226
  try:
227
  path.parent.mkdir(parents=True, exist_ok=True)
228
  legacy.replace(path)
@@ -236,6 +266,8 @@ class SawtakTTS:
236
  # The caller supplies the phrase's Palestinian vowel marks. Sawtak preserves existing
237
  # marks; enhancement fills remaining pronunciation cues using the selected voice's dialect.
238
  data = self._generate(text, voice)
 
 
239
  try:
240
  path.parent.mkdir(parents=True, exist_ok=True)
241
  tmp = path.with_suffix(".tmp")
@@ -243,12 +275,25 @@ class SawtakTTS:
243
  tmp.replace(path)
244
  except OSError as e: # read-only or full cache dir: the audio is still good, it just isn't kept
245
  log.warning("tts cache write failed: %s: %s", type(e).__name__, e)
 
 
246
  return data, voice
247
 
 
 
 
 
 
 
 
 
 
 
 
248
  def wav(self, text: str, voice: str = "") -> tuple[bytes, str]:
249
  """(WAV, voice id) in Raef's voice, or a dialogue line's configured voice."""
250
  voice = voice if voice in LINE_VOICES.values() else RAEF_VOICE_ID
251
- return self._wav(text, voice)
252
 
253
  def wav_for_voice(self, text: str, voice: str) -> tuple[bytes, str]:
254
  """(WAV, voice id) for a teacher-selected, validated Sawtak voice."""
@@ -267,4 +312,6 @@ def load_tts(key: str | None = None) -> SawtakTTS | None:
267
  if bucket:
268
  tts.mirror = BucketMirror(bucket, os.environ.get("TTS_BUCKET_PREFIX", "ihki-tts"),
269
  os.environ.get("HF_TOKEN", "").strip() or None)
 
 
270
  return tts
 
8
  TTS_BUCKET optional Hugging Face Storage Bucket id (e.g. "user/ihki-storage"); the cache-warm sweep
9
  uploads every cached WAV to it through the Bucket API (needs HF_TOKEN with write access)
10
  TTS_BUCKET_PREFIX folder inside the bucket; default "ihki-tts"
11
+ TTS_STAGING_DIR with TTS_BUCKET: a fast local folder new recordings are written to before their upload;
12
+ default <tmp>/ihki-tts-staging. A bucket mounted at /data takes ~10 s per written file.
13
  """
14
  from __future__ import annotations
15
 
 
19
  import json
20
  import logging
21
  import os
22
+ import tempfile
23
  import threading
24
  import time
25
  import urllib.error
 
91
  self._last_request = 0.0
92
  self._cache_hits = self._cache_misses = 0
93
  self.mirror: BucketMirror | None = None # set by load_tts when TTS_BUCKET is configured
94
+ # With a mirror, new recordings are written here (local disk) and uploaded through the Bucket API, instead
95
+ # of into cache_dir: a bucket mounted there only flushes a few files every ~30 s, so each write crawled.
96
+ self.staging_dir: Path | None = None
97
 
98
  def _request(self, path: str, body: dict | None = None):
99
  req = urllib.request.Request(
 
133
  """The recording's path relative to the cache folder ("<voice>/<sha1>.wav"), also its path in the bucket."""
134
  return self._path(text, voice).relative_to(self.cache_dir).as_posix()
135
 
136
+ def _candidates(self, text: str, voice: str) -> list[Path]:
137
+ """Where a recording may be: the local staging folder first (fast), then the cache folder, then its
138
+ legacy flat layout."""
139
  path = self._path(text, voice)
140
+ staged = [self.staging_dir / self.cache_key(text, voice)] if self.staging_dir else []
141
+ return staged + [path, self._legacy_path(path)]
142
+
143
+ def cache_file(self, text: str, voice: str) -> Path | None:
144
+ """The cached WAV on disk, wherever it is; None when not cached."""
145
+ return next((c for c in self._candidates(text, voice) if c.is_file()), None)
146
 
147
  def cached_files(self) -> list[tuple[Path, str]]:
148
+ """Every cached WAV on disk with its path relative to its folder; a staged copy wins over the cache's."""
149
+ found: dict[str, Path] = {}
150
+ for root in (self.cache_dir, self.staging_dir): # staging last, so it overrides
151
+ if root is not None and root.is_dir():
152
+ for p in root.rglob("*.wav"):
153
+ if p.is_file():
154
+ found[p.relative_to(root).as_posix()] = p
155
+ return [(found[rel], rel) for rel in sorted(found)]
156
+
157
+ def store(self, rel: str, data: bytes) -> Path:
158
+ """Keep a recording from an offline copy at `rel` ("<voice>/<sha1>.wav" or a legacy "<sha1>.wav"): in the
159
+ staging folder with a mirror (the slow mount is not written), else in the cache folder."""
160
+ path = (self.staging_dir or self.cache_dir) / rel
161
+ path.parent.mkdir(parents=True, exist_ok=True)
162
+ tmp = path.with_suffix(".tmp")
163
+ tmp.write_bytes(data)
164
+ tmp.replace(path)
165
+ return path
166
+
167
+ def has(self, rel: str) -> Path | None:
168
+ """The cached file at `rel`, staged or in the cache folder; None when neither has it."""
169
+ for root in (self.staging_dir, self.cache_dir):
170
+ if root is not None and (root / rel).is_file():
171
+ return root / rel
172
+ return None
173
 
174
  def cached(self, text: str, voice: str) -> bool:
175
  """Has this voice said `text` before? Then wav() answers from disk, without the provider."""
176
+ return any(c.exists() for c in self._candidates(text, voice))
 
177
 
178
  def _generate(self, text: str, voice: str) -> bytes:
179
  """Generate one uncached WAV recording."""
 
213
  def clear_cache(self) -> int:
214
  """Delete only generated audio files from this instance's dedicated cache directory."""
215
  with self._lock:
 
 
 
216
  removed = 0
217
+ roots = [r for r in (self.cache_dir, self.staging_dir) if r is not None and r.is_dir()]
218
+ for path in (p for root in roots for p in root.rglob("*")):
219
  if path.is_file() and (path.suffix == ".wav" or path.name.endswith(".tmp")):
220
  try:
221
  path.unlink()
 
231
  """Anonymous disk and process-lifetime cache usage for the monitoring panel."""
232
  with self._lock:
233
  files = size = 0
234
+ try:
235
+ for path, _rel in self.cached_files():
236
+ files += 1
237
+ size += path.stat().st_size
238
+ except OSError as e:
239
+ raise TTSUnavailable(f"could not inspect voice cache: {e}") from None
 
 
240
  requests = self._cache_hits + self._cache_misses
241
  return {"files": files, "bytes": size, "hits": self._cache_hits,
242
  "misses": self._cache_misses,
243
  "hit_rate": round(self._cache_hits / requests, 4) if requests else None}
244
 
245
+ def _wav(self, text: str, voice: str, upload: bool = False) -> tuple[bytes, str]:
246
+ """Generate or read `text` for an already validated voice id. `upload`: send a new recording to the bucket
247
+ in the background (playback); the cache-warm sweep uploads its own in batches."""
248
  with self._lock:
249
+ found = self.cache_file(text, voice) if self.staging_dir else None
250
+ if found is not None: # with a mirror the cache folder may be a slow mount: no migration there
251
+ self._cache_hits += 1
252
+ return found.read_bytes(), voice
253
  path = self._path(text, voice)
254
  legacy = self._legacy_path(path)
255
+ if self.staging_dir is None and not path.exists() and legacy.exists():
256
  try:
257
  path.parent.mkdir(parents=True, exist_ok=True)
258
  legacy.replace(path)
 
266
  # The caller supplies the phrase's Palestinian vowel marks. Sawtak preserves existing
267
  # marks; enhancement fills remaining pronunciation cues using the selected voice's dialect.
268
  data = self._generate(text, voice)
269
+ if self.staging_dir is not None:
270
+ path = self.staging_dir / self.cache_key(text, voice)
271
  try:
272
  path.parent.mkdir(parents=True, exist_ok=True)
273
  tmp = path.with_suffix(".tmp")
 
275
  tmp.replace(path)
276
  except OSError as e: # read-only or full cache dir: the audio is still good, it just isn't kept
277
  log.warning("tts cache write failed: %s: %s", type(e).__name__, e)
278
+ if upload and self.mirror is not None:
279
+ self._upload_soon(data, self.cache_key(text, voice))
280
  return data, voice
281
 
282
+ def _upload_soon(self, data: bytes, rel: str) -> None:
283
+ """One playback recording to the bucket, off the request: the student does not wait for it."""
284
+ mirror = self.mirror
285
+
286
+ def go() -> None:
287
+ try:
288
+ mirror.upload([(data, mirror.remote_path(rel))])
289
+ except Exception as e: # the next cache-warm sweep uploads what is still missing
290
+ log.warning("tts bucket upload of %s failed: %s", rel, e)
291
+ threading.Thread(target=go, name="ihki-tts-upload", daemon=True).start()
292
+
293
  def wav(self, text: str, voice: str = "") -> tuple[bytes, str]:
294
  """(WAV, voice id) in Raef's voice, or a dialogue line's configured voice."""
295
  voice = voice if voice in LINE_VOICES.values() else RAEF_VOICE_ID
296
+ return self._wav(text, voice, upload=True)
297
 
298
  def wav_for_voice(self, text: str, voice: str) -> tuple[bytes, str]:
299
  """(WAV, voice id) for a teacher-selected, validated Sawtak voice."""
 
312
  if bucket:
313
  tts.mirror = BucketMirror(bucket, os.environ.get("TTS_BUCKET_PREFIX", "ihki-tts"),
314
  os.environ.get("HF_TOKEN", "").strip() or None)
315
+ staging = os.environ.get("TTS_STAGING_DIR", "").strip()
316
+ tts.staging_dir = Path(staging) if staging else Path(tempfile.gettempdir()) / "ihki-tts-staging"
317
  return tts
static/index.html CHANGED
@@ -4865,10 +4865,22 @@ function drawSystemActivity(box){
4865
  }
4866
  function drawTtsCacheControls(box){
4867
  const card=mark("card",""), title=mark("","מטמון הקלטות Sawtak","h2"), text=mark("","אפשר להכין מראש את כל מאגר ההקלטות. התהליך מדלג על קבצים קיימים וממשיך מהם בהפעלה הבאה. מחיקה גורמת ליצירה מחדש.","p");
4868
- const usage=mark("activity-stats",""), progress=document.createElement("progress"), progressLabel=mark("msg",""), row=mark("row",""), warm=mark("btn primary","הכנת כל ההקלטות","button"), button=mark("btn","מחיקת מטמון ההקלטות","button"), offline=mark("btn","הורדת עותק אופליין","button"), halt=mark("btn","עצירת ההכנה","button"), msg=mark("msg","");
 
4869
  progress.max=100; progress.value=0; progress.hidden=true; progress.style.width="100%"; progress.setAttribute("aria-label","התקדמות הכנת הקלטות Sawtak"); progressLabel.hidden=true;
4870
- let polling=false;
4871
- const warmState=s=>{ if(!s) return; warm.disabled=button.disabled=!!s.running; halt.hidden=!s.running; const pct=s.total?Math.round(100*s.done/s.total):0,voice=s.voice_id?` · קול ${s.voice_id}`:""; progress.hidden=progressLabel.hidden=!s.total; progress.value=pct; progressLabel.textContent=`${pct}% · ${s.done} מתוך ${s.total} · נוצרו ${s.generated} · כבר היו ${s.skipped}`+(s.bucket?` · הועלו לדלי ${s.uploaded||0}`+(s.upload_failed?` · נכשלו בהעלאה ${s.upload_failed}`:""):" · אין דלי מוגדר (TTS_BUCKET)")+(s.upload_error?` · ${s.upload_error}`:""); if(s.running) msg.textContent=s.waiting?`Sawtak לא ענה כרגע (הגבלת קצב או תקלת רשת); ניסיון נוסף בעוד ${s.waiting} שניות${voice}`:`מכין הקלטות מול Sawtak${voice}`; else if(s.stopped) msg.textContent=`ההכנה נעצרה לבקשתך אחרי ${s.done} הקלטות. הפעלה חוזרת תמשיך מאותה נקודה${voice}.`; else if(s.error) msg.textContent=`ההכנה נעצרה אחרי ${s.done} הקלטות: ${s.error}${voice}`; else if(s.total&&s.done===s.total) msg.textContent=`המאגר מוכן${voice}.`; if(s.running&&!polling){polling=true; setTimeout(poll,2000);} };
 
 
 
 
 
 
 
 
 
 
 
4872
  const show=data=>{ usage.replaceChildren(); const size=data.bytes<1024?`${data.bytes} B`:data.bytes<1048576?`${(data.bytes/1024).toFixed(1)} KB`:`${(data.bytes/1048576).toFixed(1)} MB`, rate=data.hit_rate==null?"–":`${Math.round(data.hit_rate*100)}%`; for(const [n,label] of [[data.files,"קבצי שמע במטמון"],[size,"נפח בדיסק"],[data.hits,"פגיעות מאז ההפעלה"],[data.misses,"יצירות מאז ההפעלה"],[rate,"שיעור פגיעה"]]){ const stat=mark("activity-stat",""); stat.append(mark("",n,"b"),mark("",label,"small")); usage.append(stat); } if(!data.available) msg.textContent="Sawtak אינו מוגדר בשרת."; };
4873
  const poll=()=>api("/api/tts-cache").then(async res=>{const data=await res.json().catch(()=>({}));if(!res.ok) throw new Error(data.detail||res.status);show(data);polling=false;warmState(data.warm);}).catch(err=>{polling=false;msg.textContent="נתוני המטמון לא זמינים: "+err.message;});
4874
  poll();
@@ -4880,6 +4892,20 @@ function drawTtsCacheControls(box){
4880
  catch(err){ msg.textContent="העצירה נכשלה: "+err.message; }
4881
  halt.disabled=false;
4882
  };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4883
  offline.onclick=async()=>{ // every cached WAV + manifest.csv (which phrase each file is), streamed as one zip
4884
  offline.disabled=true; msg.textContent="מכין עותק אופליין…";
4885
  try{
@@ -4905,7 +4931,7 @@ function drawTtsCacheControls(box){
4905
  }catch(err){ msg.textContent="המחיקה נכשלה: "+err.message; }
4906
  button.disabled=false;
4907
  };
4908
- row.append(warm,halt,offline,button,msg); card.append(title,text,usage,progress,progressLabel,row); box.append(card);
4909
  }
4910
  // how to get each key, step by step: a step is its parts, text or [text, link]
4911
  const KEY_GUIDES={
 
4865
  }
4866
  function drawTtsCacheControls(box){
4867
  const card=mark("card",""), title=mark("","מטמון הקלטות Sawtak","h2"), text=mark("","אפשר להכין מראש את כל מאגר ההקלטות. התהליך מדלג על קבצים קיימים וממשיך מהם בהפעלה הבאה. מחיקה גורמת ליצירה מחדש.","p");
4868
+ const usage=mark("activity-stats",""), progress=document.createElement("progress"), progressLabel=mark("msg",""), row=mark("row",""), warm=mark("btn primary","הכנת כל ההקלטות","button"), button=mark("btn","מחיקת מטמון ההקלטות","button"), offline=mark("btn","הורדת עותק אופליין","button"), halt=mark("btn","עצירת ההכנה","button"), load=mark("btn","טעינת עותק אופליין","button"), loadFile=document.createElement("input"), msg=mark("msg","");
4869
+ loadFile.type="file"; loadFile.accept=".zip,application/zip"; loadFile.hidden=true;
4870
  progress.max=100; progress.value=0; progress.hidden=true; progress.style.width="100%"; progress.setAttribute("aria-label","התקדמות הכנת הקלטות Sawtak"); progressLabel.hidden=true;
4871
+ let polling=false, samples=[]; // samples: recent [time, done] while the sweep runs, for the time estimate
4872
+ const duration=sec=>{ sec=Math.round(sec); if(sec<60) return "פחות מדקה"; const m=Math.round(sec/60); if(m<60) return `כ־${m} דקות`; const h=Math.floor(m/60),r=m%60; return r?`כ־${h} שעות ו־${r} דקות`:`כ־${h} שעות`; };
4873
+ const estimate=s=>{ // from the pace of the last 3 minutes (cached phrases fly, new ones wait for Sawtak); else since the start
4874
+ const now=Date.now(); if(!s.running){ samples=[]; return s.started_at&&s.total&&s.done===s.total?` · ההכנה ארכה ${duration(now/1000-s.started_at)}`:""; }
4875
+ samples.push([now,s.done]); samples=samples.filter(([t])=>now-t<=180000);
4876
+ const [t0,d0]=samples[0], left=s.total-s.done; let rate=0;
4877
+ if(now-t0>=20000&&s.done>d0) rate=(s.done-d0)/((now-t0)/1000);
4878
+ else if(s.started_at&&s.done) rate=s.done/Math.max(1,now/1000-s.started_at);
4879
+ if(!left) return ""; if(!rate) return " · מחשב זמן משוער…";
4880
+ const end=new Date(now+left/rate*1000).toLocaleTimeString("he-IL",{hour:"2-digit",minute:"2-digit"});
4881
+ return ` · נותרו ${duration(left/rate)} (סיום משוער ${end})`;
4882
+ };
4883
+ const warmState=s=>{ if(!s) return; warm.disabled=button.disabled=!!s.running; halt.hidden=!s.running; const pct=s.total?Math.round(100*s.done/s.total):0,voice=s.voice_id?` · קול ${s.voice_id}`:""; progress.hidden=progressLabel.hidden=!s.total; progress.value=pct; progressLabel.textContent=`${pct}% · ${s.done} מתוך ${s.total} · נוצרו ${s.generated} · כבר היו ${s.skipped}`+(s.bucket?` · הועלו לדלי ${s.uploaded||0}`+(s.upload_failed?` · נכשלו בהעלאה ${s.upload_failed}`:""):" · אין דלי מוגדר (TTS_BUCKET)")+(s.upload_error?` · ${s.upload_error}`:"")+estimate(s); if(s.running) msg.textContent=s.waiting?`Sawtak לא ענה כרגע (הגבלת קצב או תקלת רשת); ניסיון נוסף בעוד ${s.waiting} שניות${voice}`:`מכין הקלטות מול Sawtak${voice}`; else if(s.stopped) msg.textContent=`ההכנה נעצרה לבקשתך אחרי ${s.done} הקלטות. הפעלה חוזרת תמשיך מאותה נקודה${voice}.`; else if(s.error) msg.textContent=`ההכנה נעצרה אחרי ${s.done} הקלטות: ${s.error}${voice}`; else if(s.total&&s.done===s.total) msg.textContent=`המאגר מוכן${voice}.`; if(s.running&&!polling){polling=true; setTimeout(poll,2000);} };
4884
  const show=data=>{ usage.replaceChildren(); const size=data.bytes<1024?`${data.bytes} B`:data.bytes<1048576?`${(data.bytes/1024).toFixed(1)} KB`:`${(data.bytes/1048576).toFixed(1)} MB`, rate=data.hit_rate==null?"–":`${Math.round(data.hit_rate*100)}%`; for(const [n,label] of [[data.files,"קבצי שמע במטמון"],[size,"נפח בדיסק"],[data.hits,"פגיעות מאז ההפעלה"],[data.misses,"יצירות מאז ההפעלה"],[rate,"שיעור פגיעה"]]){ const stat=mark("activity-stat",""); stat.append(mark("",n,"b"),mark("",label,"small")); usage.append(stat); } if(!data.available) msg.textContent="Sawtak אינו מוגדר בשרת."; };
4885
  const poll=()=>api("/api/tts-cache").then(async res=>{const data=await res.json().catch(()=>({}));if(!res.ok) throw new Error(data.detail||res.status);show(data);polling=false;warmState(data.warm);}).catch(err=>{polling=false;msg.textContent="נתוני המטמון לא זמינים: "+err.message;});
4886
  poll();
 
4892
  catch(err){ msg.textContent="העצירה נכשלה: "+err.message; }
4893
  halt.disabled=false;
4894
  };
4895
+ load.onclick=()=>{ loadFile.value=""; loadFile.click(); };
4896
+ loadFile.onchange=async()=>{ // a zip from "הורדת עותק אופליין": its recordings back into the cache, then to the bucket
4897
+ const file=loadFile.files[0]; if(!file) return;
4898
+ if(!confirm(`לטעון ${file.name} (${(file.size/1048576).toFixed(1)} MB)? הקלטות שכבר קיימות ידולגו.`)) return;
4899
+ load.disabled=true; msg.textContent=`מעלה ${file.name}…`;
4900
+ try{
4901
+ const form=new FormData(); form.append("archive",file);
4902
+ const res=await api("/api/tts-cache/import",{method:"POST",body:form}),data=await res.json().catch(()=>({}));
4903
+ if(!res.ok) throw new Error(data.detail||res.status);
4904
+ msg.textContent=`נטענו ${data.added} הקלטות · ${data.skipped} כבר היו`+(data.invalid?` · ${data.invalid} קבצים לא תקינים דולגו`:"")+(data.uploading?" · ההעלאה לדלי ממשיכה ברקע.":".");
4905
+ poll();
4906
+ }catch(err){ msg.textContent="הטעינה נכשלה: "+err.message; }
4907
+ load.disabled=false;
4908
+ };
4909
  offline.onclick=async()=>{ // every cached WAV + manifest.csv (which phrase each file is), streamed as one zip
4910
  offline.disabled=true; msg.textContent="מכין עותק אופליין…";
4911
  try{
 
4931
  }catch(err){ msg.textContent="המחיקה נכשלה: "+err.message; }
4932
  button.disabled=false;
4933
  };
4934
+ row.append(warm,halt,offline,load,loadFile,button,msg); card.append(title,text,usage,progress,progressLabel,row); box.append(card);
4935
  }
4936
  // how to get each key, step by step: a step is its parts, text or [text, link]
4937
  const KEY_GUIDES={
tests/test_scoring.py CHANGED
@@ -527,6 +527,7 @@ def test_monitoring_can_warm_the_unique_tts_bank(monkeypatch):
527
  state = c.get("/api/tts-cache").json()["warm"]
528
  if not state["running"]: break
529
  import time; time.sleep(.01)
 
530
  assert state == {"running": False, "voice_id": RAEF_VOICE_ID, "total": 2, "done": 2, "generated": 1,
531
  "skipped": 1, "failed": 0, "current": "", "waiting": 0, "error": "",
532
  "bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": "", "stopped": False}
@@ -2321,3 +2322,75 @@ def test_the_warm_sweep_stops_on_request_and_keeps_what_it_made(monkeypatch):
2321
  if not state["running"]: break
2322
  time.sleep(.01)
2323
  assert state["stopped"] is True and state["done"] == state["generated"] == 1 and len(tts.calls) == 1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
527
  state = c.get("/api/tts-cache").json()["warm"]
528
  if not state["running"]: break
529
  import time; time.sleep(.01)
530
+ assert state.pop("started_at") > 0
531
  assert state == {"running": False, "voice_id": RAEF_VOICE_ID, "total": 2, "done": 2, "generated": 1,
532
  "skipped": 1, "failed": 0, "current": "", "waiting": 0, "error": "",
533
  "bucket": "", "uploaded": 0, "upload_failed": 0, "upload_error": "", "stopped": False}
 
2322
  if not state["running"]: break
2323
  time.sleep(.01)
2324
  assert state["stopped"] is True and state["done"] == state["generated"] == 1 and len(tts.calls) == 1
2325
+
2326
+
2327
+ def test_with_a_bucket_new_recordings_are_staged_locally_not_written_to_the_mount(monkeypatch, tmp_path):
2328
+ from app.tts import RAEF_VOICE_ID, BucketMirror, SawtakTTS
2329
+
2330
+ mount, staging = tmp_path / "mount", tmp_path / "staging"
2331
+ tts = SawtakTTS("key", cache_dir=mount)
2332
+ tts.staging_dir = staging
2333
+ monkeypatch.setattr(tts, "_generate", lambda text, voice: b"RIFF-" + text.encode())
2334
+ uploaded = []
2335
+
2336
+ class Mirror(BucketMirror):
2337
+ def upload(self, files): uploaded.extend(dest for _src, dest in files)
2338
+ tts.mirror = Mirror("user/bucket", "ihki-tts")
2339
+
2340
+ data, _ = tts.wav("مرحبا")
2341
+ key = tts.cache_key("مرحبا", RAEF_VOICE_ID)
2342
+ assert (staging / key).read_bytes() == data and not mount.exists() # the slow mount is not written
2343
+ assert tts.cached("مرحبا", RAEF_VOICE_ID) and tts.cache_file("مرحبا", RAEF_VOICE_ID) == staging / key
2344
+ import time
2345
+ for _ in range(100):
2346
+ if uploaded: break
2347
+ time.sleep(.01)
2348
+ assert uploaded == [f"ihki-tts/{key}"] # playback uploads in the background
2349
+ (mount / RAEF_VOICE_ID).mkdir(parents=True) # the bucket's copy appears in the mount
2350
+ (mount / key).write_bytes(data)
2351
+ assert [rel for _p, rel in tts.cached_files()] == [key] # counted once
2352
+ assert tts.cache_stats()["files"] == 1
2353
+ assert tts.wav("مرحبا")[0] == data and tts.cache_stats()["hits"] == 1
2354
+ assert tts.clear_cache() == 2 and not tts.cached("مرحبا", RAEF_VOICE_ID)
2355
+
2356
+
2357
+ def test_an_offline_copy_loads_back_into_the_cache_and_the_bucket(monkeypatch, tmp_path):
2358
+ import io
2359
+ import time
2360
+ import zipfile
2361
+
2362
+ from fastapi.testclient import TestClient
2363
+
2364
+ import app.main
2365
+ from app.main import create_app
2366
+ from app.tts import RAEF_VOICE_ID, BucketMirror, SawtakTTS
2367
+
2368
+ monkeypatch.setattr(app.main, "load_auth", lambda: None)
2369
+ tts = SawtakTTS("key", cache_dir=tmp_path / "mount")
2370
+ tts.staging_dir = tmp_path / "staging"
2371
+ uploaded = []
2372
+
2373
+ class Mirror(BucketMirror):
2374
+ def upload(self, files): uploaded.extend(dest for _src, dest in files)
2375
+ tts.mirror = Mirror("user/bucket", "ihki-tts")
2376
+ wav = b"RIFF\0\0\0\0WAVEfmt " + b"\0" * 40
2377
+ new, old, legacy = f"{RAEF_VOICE_ID}/{'a' * 40}.wav", f"{RAEF_VOICE_ID}/{'b' * 40}.wav", f"{'c' * 40}.wav"
2378
+ tts.store(old, wav) # already cached: skipped
2379
+ buf = io.BytesIO()
2380
+ with zipfile.ZipFile(buf, "w") as zf:
2381
+ zf.writestr(f"ihki-tts/{new}", wav)
2382
+ zf.writestr(f"ihki-tts/{old}", wav)
2383
+ zf.writestr(f"ihki-tts/{legacy}", wav)
2384
+ zf.writestr("manifest.csv", "file\n") # ignored
2385
+ zf.writestr("ihki-tts/../../evil.wav", wav) # refused: outside the cache
2386
+ zf.writestr(f"ihki-tts/{RAEF_VOICE_ID}/{'d' * 40}.wav", b"not audio") # refused: not a WAV
2387
+ with TestClient(create_app(scorer_for(emissions(["w"])), tts, tutor=object())) as c:
2388
+ res = c.post("/api/tts-cache/import", files={"archive": ("copy.zip", buf.getvalue(), "application/zip")})
2389
+ assert res.status_code == 200, res.text
2390
+ assert res.json() == {"added": 2, "skipped": 1, "invalid": 2, "uploading": True}
2391
+ assert c.post("/api/tts-cache/import", files={"archive": ("x.zip", b"nope", "application/zip")}).status_code == 400
2392
+ assert tts.has(new) == tts.staging_dir / new and tts.has(legacy) and not (tmp_path / "evil.wav").exists()
2393
+ for _ in range(100):
2394
+ if len(uploaded) == 2: break
2395
+ time.sleep(.01)
2396
+ assert sorted(uploaded) == sorted([f"ihki-tts/{new}", f"ihki-tts/{legacy}"])