"""Public Python API. from maya3 import Maya3TTS tts = Maya3TTS.from_pretrained("path/to/model.pt", backbone_dir=..., mimi_dir=..., ecapa_dir=...) wav = tts.generate("नमस्ते, आप कैसे हैं?", lang="hi") # plain voice tts.register_voice("asha", ref_audio="asha.wav", ref_text="...", lang="hi") # clone once, reuse wav = tts.generate("Hello there.", lang="en", voice="asha") # cached prefix KV + ECAPA wav = tts.generate("Hello there.", lang="en", ref_audio="x.wav", ref_text="...") # one-off clone for pcm in tts.stream("Long text ...", lang="te", voice="asha"): ... # int16 chunks, 24 kHz async for pcm in tts.astream(...): ... # asyncio Audio is 24 kHz mono. generate() returns float32 in [-1, 1]; stream()/astream() yield int16 numpy chunks. Concurrency: any number of threads / coroutines may call generate/stream at once; the engine batches them. """ from __future__ import annotations import asyncio import hashlib import os import queue import threading from pathlib import Path import numpy as np import torch from maya3.engine import SR, Engine, EngineConfig from maya3.prompt import Voice, norm_mode class GenerationError(RuntimeError): def __init__(self, code, msg): super().__init__(f"{code}: {msg}") self.code = code def load_audio(x, sr=None): """path | (array, sr) | array (assumed 24 kHz) -> float32 mono 24 kHz.""" if isinstance(x, (str, os.PathLike)): import soundfile as sf w, sr = sf.read(str(x), dtype="float32") elif isinstance(x, tuple): w, sr = x else: w = x w = np.asarray(w, np.float32) if w.ndim > 1: w = w.mean(1) if w.shape[1] < w.shape[0] else w.mean(0) sr = sr or SR if sr != SR: from math import gcd from scipy.signal import resample_poly g = gcd(int(sr), SR) w = resample_poly(w, SR // g, int(sr) // g).astype(np.float32) return w class Ecapa: """ECAPA-TDNN speaker embedding, corpus recipe (speechbrain spkrec-ecapa-voxceleb; 16 kHz fbank-80, sentence norm).""" def __init__(self, ecapa_dir, device="cuda"): from speechbrain.lobes.features import Fbank from speechbrain.processing.features import InputNormalization from speechbrain.lobes.models.ECAPA_TDNN import ECAPA_TDNN self.fbank = Fbank(n_mels=80).eval().to(device) self.norm = InputNormalization(norm_type="sentence", std_norm=False).eval().to(device) emb = ECAPA_TDNN(input_size=80, channels=[1024, 1024, 1024, 1024, 3072], kernel_sizes=[5, 3, 3, 3, 1], dilations=[1, 2, 3, 4, 1], attention_channels=128, lin_neurons=192).eval() emb.load_state_dict(torch.load(Path(ecapa_dir) / "embedding_model.ckpt", map_location="cpu", weights_only=True), strict=True) self.emb = emb.to(device) self.device = device @torch.inference_mode() def __call__(self, w24): from scipy.signal import resample_poly if w24 is None or len(w24) < 2400: raise ValueError("reference audio is too short (need >= 0.1 s; 3-10 s recommended)") w16 = torch.from_numpy(resample_poly(np.asarray(w24, np.float32), 2, 3).astype(np.float32)).to(self.device)[None] L = torch.ones(1, device=self.device) e = self.emb(self.norm(self.fbank(w16), L), L).reshape(-1) return torch.nn.functional.normalize(e, dim=0).cpu().numpy() class Maya3TTS: def __init__(self, engine: Engine, ecapa_dir=None): self.engine = engine self.ecapa_dir = ecapa_dir self._ecapa = None self._oneoff = {} engine.start() @classmethod def from_pretrained(cls, ckpt, backbone_dir=None, mimi_dir=None, ecapa_dir=None, device="cuda", voices=True, **config): """from_pretrained("") -- a Maya3 release directory (model.safetensors + backbone/ + voices/), resolved by maya3.hub.resolve (Mimi / ECAPA from the Hugging Face cache); or explicit (ckpt, backbone_dir, mimi_dir, ecapa_dir). voices: register the release's named voices (voices/voices.json). config: EngineConfig fields.""" named = None if backbone_dir is None: from maya3 import hub r = hub.resolve(ckpt) ckpt, backbone_dir = r["ckpt"], r["backbone_dir"] mimi_dir, ecapa_dir = mimi_dir or r["mimi_dir"], ecapa_dir or r.get("ecapa_dir") named = r.get("voices_json") cfg = EngineConfig(**config) tts = cls(Engine(ckpt, backbone_dir, mimi_dir, cfg, device=device), ecapa_dir=ecapa_dir) if voices and named and os.path.exists(named): tts.load_named_voices(named) return tts def load_named_voices(self, voices_json, **kw): """Register the named (trained) voices listed in voices.json; see maya3.prompt.load_named_voices.""" from maya3.prompt import load_named_voices ids = load_named_voices(self.engine.registry, voices_json, **kw) for vid in ids: v = self.engine.registry.get(vid) self.engine.call(lambda v=v: self.engine.prepare_voice(v)) return ids # ------------------------------------------------------------------ voices @property def voices(self): return self.engine.registry.ids() def _embed(self, w): if self._ecapa is None: if self.ecapa_dir is None: raise ValueError("ecapa_dir is required for reference-audio voices") self._ecapa = Ecapa(self.ecapa_dir, self.engine.device) return self._ecapa(w) def make_voice(self, voice_id, ref_audio=None, ref_text=None, lang=None, mode="icl_spk", ref_codes=None, ecapa=None, builder=None, meta=None): """Build a Voice from a reference clip (audio + its transcript + its language) or precomputed codes/ECAPA.""" mode = norm_mode(mode) clips = [] if ref_audio is not None or ref_codes is not None: if ref_codes is None: w = load_audio(ref_audio) if len(w) > 30 * SR: raise ValueError("reference audio longer than 30 s; 3-10 s works best") def enc(): x = torch.as_tensor(w, device=self.engine.device)[None, None] return self.engine.mimi.encode(x, num_quantizers=32).audio_codes[0].cpu().numpy().astype(np.int64) ref_codes = self.engine.call(enc) if ecapa is None and mode in ("spk", "icl_spk"): ecapa = self.engine.call(lambda: self._embed(w)) if mode in ("icl", "icl_spk"): if not ref_text or not lang: raise ValueError("ref_text and lang (the reference clip's transcript and language) are required for cloning") clips = [{"text": ref_text, "codes": np.asarray(ref_codes, np.int64), "lang": lang}] v = Voice(voice_id, mode, clips=clips, ecapa=ecapa, builder=builder, meta=meta or {}) self.engine.call(lambda: self.engine.prepare_voice(v)) return v def register_voice(self, voice_id, ref_audio=None, ref_text=None, lang=None, mode="icl_spk", **kw): """Register (or replace) a named voice; its prefix KV and speaker memory are computed now and reused.""" v = self.make_voice(voice_id, ref_audio, ref_text, lang, mode, **kw) self.engine.registry.add(v) return voice_id def remove_voice(self, voice_id): return self.engine.registry.remove(voice_id) is not None def _voice(self, voice, ref_audio, ref_text, ref_lang, mode): if ref_audio is None: return self.engine.registry.get(voice or "plain") if voice is not None and voice != "plain": raise ValueError("pass either voice= or ref_audio=, not both (a named voice with a reference " "clip was never trained)") h = hashlib.sha1() if isinstance(ref_audio, (str, os.PathLike)): st = os.stat(ref_audio) h.update(f"{os.path.abspath(ref_audio)}|{st.st_size}|{st.st_mtime_ns}".encode()) else: h.update(np.asarray(load_audio(ref_audio)).tobytes()) h.update(f"|{ref_text}|{ref_lang}|{mode}".encode()) key = h.hexdigest() if key not in self._oneoff: if len(self._oneoff) > 64: self._oneoff.pop(next(iter(self._oneoff))) self._oneoff[key] = self.make_voice(f"oneoff-{key[:12]}", ref_audio, ref_text, ref_lang, mode) return self._oneoff[key] # ------------------------------------------------------------------ synthesis SAMPLING_KEYS = ("seed", "temperature", "top_k", "top_p", "repetition_penalty") def _prep_text(self, text, lang, normalize): if not normalize: return text from maya3.text import normalise_text # the bundled normaliser (digits, currency, URLs, ...) if isinstance(text, (list, tuple)): return [normalise_text(t, lang) for t in text] return normalise_text(text, lang) def stream(self, text, lang="en", voice=None, ref_audio=None, ref_text=None, ref_lang=None, mode="icl_spk", max_frames=None, timeout=300, normalize=False, info=None, seed=None, temperature=None, top_k=None, top_p=None, repetition_penalty=None): """Yield int16 PCM chunks (24 kHz) as they are generated. seed/temperature/top_k/top_p/repetition_penalty: per-request sampler settings (None = engine defaults). normalize: run the bundled text normaliser first (digits, currency, dates, URLs, ALL-CAPS names). info: optional dict that receives the end payload (reason, frames, ttfa_s, stops, ...).""" v = self._voice(voice, ref_audio, ref_text, ref_lang or lang, mode) sampling = {"seed": seed, "temperature": temperature, "top_k": top_k, "top_p": top_p, "repetition_penalty": repetition_penalty} q = queue.Queue() req = self.engine.submit(self._prep_text(text, lang, normalize), lang, v, on_event=lambda k, p: q.put((k, p)), max_frames=max_frames, sampling=sampling) try: while True: kind, payload = q.get(timeout=timeout) if kind == "audio": yield payload elif kind == "end": if info is not None: info.update(payload) self.last_info = payload return else: raise GenerationError(payload.get("code"), payload.get("error")) finally: req.cancel() def generate(self, text, lang="en", voice=None, ref_audio=None, ref_text=None, ref_lang=None, mode="icl_spk", max_frames=None, return_info=False, **kw): """Synthesise text -> float32 waveform (24 kHz). kw: normalize, seed, temperature, top_k, top_p, repetition_penalty.""" info = {} chunks = list(self.stream(text, lang, voice, ref_audio, ref_text, ref_lang, mode, max_frames, info=info, **kw)) wav = (np.concatenate(chunks).astype(np.float32) / 32767.0) if chunks else np.zeros(0, np.float32) return (wav, info) if return_info else wav async def astream(self, text, lang="en", voice=None, ref_audio=None, ref_text=None, ref_lang=None, mode="icl_spk", max_frames=None, info=None, normalize=False, **sampling): """asyncio variant of stream(); `info` (a dict) receives the end payload.""" loop = asyncio.get_running_loop() q = asyncio.Queue() if ref_audio is not None: v = await asyncio.to_thread(self._voice, voice, ref_audio, ref_text, ref_lang or lang, mode) else: v = self.engine.registry.get(voice or "plain") req = self.engine.submit(self._prep_text(text, lang, normalize), lang, v, on_event=lambda k, p: loop.call_soon_threadsafe(q.put_nowait, (k, p)), max_frames=max_frames, sampling=sampling) try: while True: kind, payload = await q.get() if kind == "audio": yield payload elif kind == "end": if info is not None: info.update(payload) return else: raise GenerationError(payload.get("code"), payload.get("error")) finally: req.cancel() def close(self): self.engine.close()