Two-level TTS cache: raw samples + encoded per format

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

services/tts_service.py

@@ -49,6 +49,13 @@ _cache: "OrderedDict[tuple, tuple[bytes, str]]" = OrderedDict()
 _cache_lock = threading.Lock()
 _CACHE_MAX = int(os.getenv("TTS_CACHE_SIZE", "256"))
 
+# Second cache level: raw float32 samples per (text, voice). Synthesis is the
+# expensive step (seconds); encoding is milliseconds — so a request for the
+# same text in a different container (e.g. Safari asking for wav after the
+# cache was warmed with ogg) re-encodes instead of re-synthesizing.
+_sample_cache: "OrderedDict[tuple, object]" = OrderedDict()
+_SAMPLE_CACHE_MAX = int(os.getenv("TTS_SAMPLE_CACHE_SIZE", "128"))
+
 
 def _onnx_paths() -> tuple[Path, Path]:
     model = Path(os.getenv("KOKORO_ONNX_MODEL", "models/kokoro-v1.0.onnx"))
@@ -218,6 +225,22 @@ def _cache_put(key: tuple, value: tuple[bytes, str]) -> None:
             _cache.popitem(last=False)
 
 
+def _samples_get(key: tuple):
+    with _cache_lock:
+        hit = _sample_cache.get(key)
+        if hit is not None:
+            _sample_cache.move_to_end(key)
+        return hit
+
+
+def _samples_put(key: tuple, samples) -> None:
+    with _cache_lock:
+        _sample_cache[key] = samples
+        _sample_cache.move_to_end(key)
+        while len(_sample_cache) > _SAMPLE_CACHE_MAX:
+            _sample_cache.popitem(last=False)
+
+
 def _encode(samples, fmt: str) -> tuple[bytes, str]:
     """Encode float32 samples → (bytes, mime). OGG/Opus is ~10x smaller than
     WAV for speech; falls back to WAV if libsndfile lacks Opus support."""
@@ -244,22 +267,28 @@ def synthesize(text: str, voice: str | None = None, fmt: str = "wav") -> tuple[b
     voice = voice or DEFAULT_VOICE
     fmt = "ogg" if str(fmt).lower() in ("ogg", "opus") else "wav"
     key = (text, voice, fmt)
+    skey = (text, voice)
 
     cached = _cache_get(key)
     if cached is not None:
         return cached
 
-    with _synth_sem:
-        cached = _cache_get(key)  # may have been filled while we waited
-        if cached is not None:
-            return cached
-        backend = _get_backend()
-        chunks = list(backend.segments(text, voice))
-        if not chunks:
-            raise RuntimeError("Kokoro produced no audio")
-        samples = np.concatenate(chunks) if len(chunks) > 1 else chunks[0]
-        result = _encode(samples, fmt)
-
+    samples = _samples_get(skey)
+    if samples is None:
+        with _synth_sem:
+            cached = _cache_get(key)  # may have been filled while we waited
+            if cached is not None:
+                return cached
+            samples = _samples_get(skey)
+            if samples is None:
+                backend = _get_backend()
+                chunks = list(backend.segments(text, voice))
+                if not chunks:
+                    raise RuntimeError("Kokoro produced no audio")
+                samples = np.concatenate(chunks) if len(chunks) > 1 else chunks[0]
+                _samples_put(skey, samples)
+
+    result = _encode(samples, fmt)
     _cache_put(key, result)
     return result