kokoro-sima / engine.py
florianvoss's picture
Simplify setup to the existing Neat environment
eca98a1
Raw History Blame Contribute Delete
7.65 kB
"""Persistent, single-worker Modalix inference with immediate PCM callbacks."""
from __future__ import annotations
import json
import logging
import time
import threading
import tempfile
from pathlib import Path
from runtime import pipeline
import numpy as np
import torch
from runtime.neat_backend import ModalixBackend
from runtime.common import MODELS, LANGUAGES, SAMPLES, options, split_sentences, validate_request
from runtime.text import create_g2p
from runtime.audio import pcm16, wav_writer
class Cancelled(Exception):
pass
class Kokoro:
def __init__(self, *, languages=None, threads=16):
if not 1 <= threads <= 16:
raise ValueError('threads must be between 1 and 16')
languages = list(LANGUAGES) if languages is None else list(dict.fromkeys(languages))
if not languages or any(language not in LANGUAGES for language in languages):
raise ValueError('Select supported language tags from runtime.common.LANGUAGES')
self.lock = threading.Lock()
torch.set_num_threads(threads)
try:
torch.set_num_interop_threads(1)
except RuntimeError:
if torch.get_num_interop_threads() != 1:
raise
self.backend = None
self.g2p = {}
self.language_errors = {}
self.vocab = json.loads((MODELS / "vocab.json").read_text())
self.styles = {}
try:
for language in languages:
try:
processor = create_g2p(language)
processor(SAMPLES[language]) # Resolve dictionaries during startup.
self.g2p[language] = processor
except Exception as error:
self.language_errors[language] = str(error)
if not self.g2p:
raise RuntimeError('No text frontend initialized: ' + str(self.language_errors))
# Initialize native text libraries before Neat loads its runtime libraries.
# Loading spaCy/srsly afterward crashes with the validated target SDK.
logging.getLogger("kokoro.web").info("Text frontends ready: %s", ", ".join(self.g2p))
language = next(iter(self.g2p))
voice = next(v['id'] for v in options()['voices'] if v['language'] == language)
ids, style = self._inputs(split_sentences(SAMPLES[language])[0], language, voice)
self.backend = ModalixBackend(MODELS, threads)
pipeline.synthesize(self.backend, ids, style)
except BaseException:
self.close()
raise
def style(self, voice, index):
if voice not in self.styles:
self.styles[voice] = np.load(MODELS / "voices" / (voice + ".npy"), allow_pickle=False)
pack = self.styles[voice]
if not 0 <= index < len(pack):
raise ValueError("This sentence is too long. Split it with . ? or ! and try again.")
return pack[index].reshape(1, 256).astype(np.float32, copy=False)
def _inputs(self, text, language, voice):
ps, _ = self.g2p[language](text)
ids = [self.vocab[p] for p in ps if p in self.vocab]
if not ids:
raise ValueError('This sentence produced no supported speech sounds.')
if len(ids) + 2 > 480:
raise ValueError('This sentence is too long. Add . ? or ! to split it.')
return np.asarray([[0, *ids, 0]], dtype=np.int64), self.style(voice, len(ps) - 1)
def generate(self, request, emit, cancel):
"""Generate validated sentence requests; callbacks receive PCM immediately."""
with self.lock:
return self._generate(request, emit, cancel)
def _generate(self, request, emit, cancel):
language = request["language"]
if language not in self.g2p:
raise ValueError(f"{LANGUAGES[language]} is unavailable: {self.language_errors.get(language, 'G2P not initialized')}")
started = time.perf_counter()
reports, first = [], None
for index, text in enumerate(request["sentences"]):
if cancel.is_set():
raise Cancelled()
emit("sentence", {"index": index, "text": text, "total": len(request["sentences"])})
g2p_started = time.perf_counter()
ids, style = self._inputs(text, language, request["voice"])
g2p_seconds = time.perf_counter() - g2p_started
if cancel.is_set():
raise Cancelled()
self.backend.timings = {}
def pcm(value):
nonlocal first
if cancel.is_set():
raise Cancelled()
if first is None:
first = time.perf_counter() - started
emit("first_audio", {"seconds": first})
emit("pcm", np.asarray(value, dtype="<f4"))
try:
_, report = pipeline.synthesize(self.backend, ids, style, request["speed"],
(request["seed"] + index) % 2**32, on_audio_chunk=pcm)
except ValueError as error:
raise ValueError(f"Sentence {index + 1}: {error}") from error
report["g2p_seconds"] = g2p_seconds
reports.append(report)
emit("sentence_done", {"index": index, "audio_seconds": report["audio_seconds"],
"completed": index + 1, "total": len(request["sentences"])})
seconds = time.perf_counter() - started
audio_seconds = sum(r["audio_seconds"] for r in reports)
return {"sentences": reports, "generation_seconds": seconds, "audio_seconds": audio_seconds,
"first_pcm_seconds": first, "rtf": seconds / audio_seconds if audio_seconds else None}
def synthesize(self, text, *, voice='af_heart', language=None, speed=1.0, seed=1234,
on_audio_chunk=None):
"""Return (float32 mono audio, timing report); optionally receive live blocks."""
request = validate_request(dict(text=text, voice=voice, language=language or voice[:1],
speed=speed, seed=seed))
chunks = []
def emit(event, value):
if event == 'pcm':
chunks.append(value)
if on_audio_chunk is not None:
on_audio_chunk(value)
report = self.generate(request, emit, threading.Event())
return np.concatenate(chunks), report
def save_wav(self, text, path, *, voice='af_heart', language=None, speed=1.0, seed=1234):
"""Write a PCM16 WAV as blocks arrive; publish the file after success."""
request = validate_request(dict(text=text, voice=voice, language=language or voice[:1],
speed=speed, seed=seed))
destination = Path(path)
destination.parent.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(dir=destination.parent, suffix='.wav', delete=False) as temp:
temporary = Path(temp.name)
try:
with wav_writer(temporary) as writer:
def emit(event, value):
if event == 'pcm':
writer.writeframesraw(pcm16(value))
report = self.generate(request, emit, threading.Event())
temporary.replace(destination)
return report
finally:
temporary.unlink(missing_ok=True)
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def close(self):
if self.backend is not None:
self.backend.close()