import json import re import numpy as np import soundfile as sf import torch import torchaudio import gradio as gr try: import spaces # required on ZeroGPU Spaces gpu = spaces.GPU except ImportError: def gpu(fn=None, **kwargs): return fn if fn is not None else (lambda f: f) from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan from datasets import load_dataset MODEL_ID = "AndreaNotes1/xqieech" FALLBACK_PROCESSOR_ID = "microsoft/speecht5_tts" VOCODER_ID = "microsoft/speecht5_hifigan" SAMPLE_RATE = 16000 MAX_TOKENS = 600 device = "cuda" if torch.cuda.is_available() else "cpu" # ---------- model ---------- try: processor = SpeechT5Processor.from_pretrained(MODEL_ID) except Exception: processor = SpeechT5Processor.from_pretrained(FALLBACK_PROCESSOR_ID) model = SpeechT5ForTextToSpeech.from_pretrained(MODEL_ID).to(device).eval() vocoder = SpeechT5HifiGan.from_pretrained(VOCODER_ID).to(device).eval() # ---------- built-in voices (CMU Arctic x-vectors, looked up by speaker code) ---------- VOICE_META = { "slt": "Female - US (slt)", "clb": "Female - US (clb)", "bdl": "Male - US (bdl)", "rms": "Male - US (rms)", "awb": "Male - Scottish (awb)", "jmk": "Male - Canadian (jmk)", "ksp": "Male - Indian (ksp)", } N_AVG = 40 BUILTIN = {} # label -> np.ndarray (512,) try: ds = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation") names = ds["filename"] for code, label in VOICE_META.items(): idx = [i for i, f in enumerate(names) if f"cmu_us_{code}_" in f][:N_AVG] if not idx: continue v = torch.tensor(np.array(ds.select(idx)["xvector"]), dtype=torch.float32).mean(0) BUILTIN[label] = (v / v.norm()).numpy() except Exception as e: print("Could not load x-vectors:", e) BUILTIN["Neutral (zeros)"] = np.zeros(512, dtype=np.float32) VOICE_NAMES = sorted(BUILTIN, key=lambda n: (not n.startswith("Female"), n)) DEFAULT_VOICE = VOICE_NAMES[0] # ---------- speaker encoder for "record your own voice" ---------- encoder = None try: try: from speechbrain.inference.speaker import EncoderClassifier except ImportError: from speechbrain.pretrained import EncoderClassifier encoder = EncoderClassifier.from_hparams( source="speechbrain/spkrec-xvect-voxceleb", savedir="pretrained_models/spkrec-xvect-voxceleb", run_opts={"device": "cpu"}, ) except Exception as e: print("Speaker encoder unavailable, custom voices disabled:", e) def embed_recording(audio): """audio: (sample_rate, np.ndarray) from gr.Audio(type='numpy') -> (512,) x-vector.""" sr, data = audio data = np.asarray(data) if data.dtype.kind == "i": data = data.astype(np.float32) / np.iinfo(data.dtype).max else: data = data.astype(np.float32) if data.ndim > 1: data = data.mean(axis=1) wav = torch.from_numpy(data).unsqueeze(0) if sr != SAMPLE_RATE: wav = torchaudio.functional.resample(wav, sr, SAMPLE_RATE) if wav.shape[1] < SAMPLE_RATE * 3: raise gr.Error("Please record at least 3 seconds (10-20 seconds works best).") with torch.no_grad(): emb = encoder.encode_batch(wav) emb = torch.nn.functional.normalize(emb, dim=2) return emb.squeeze().cpu().numpy() def add_voice(audio, name, custom): if encoder is None: raise gr.Error("Custom voices are unavailable in this Space.") if audio is None: raise gr.Error("Record or upload a voice sample first.") custom = dict(custom or {}) base = (name or "").strip() or "My voice" label, n = f"{base} (custom)", 2 while label in custom or label in BUILTIN: label, n = f"{base} {n} (custom)", n + 1 custom[label] = embed_recording(audio) choices = VOICE_NAMES + list(custom) return custom, gr.update(choices=choices, value=label), f"Added '{label}'. It's selected in the Voice list." # ---------- synthesis ---------- def chunk_text(text, max_chars=250): sentences = re.split(r"(?<=[.!?])\s+", text) chunks, cur = [], "" for s in sentences: if len(cur) + len(s) + 1 <= max_chars: cur = f"{cur} {s}".strip() else: if cur: chunks.append(cur) while len(s) > max_chars: chunks.append(s[:max_chars]) s = s[max_chars:] cur = s if cur: chunks.append(cur) return chunks @gpu @torch.no_grad() def synthesize(text, voice, speed, custom): text = re.sub(r"\s+", " ", (text or "").strip()) if not text: raise gr.Error("Please enter some text.") vec = (custom or {}).get(voice) if vec is None: vec = BUILTIN.get(voice) if vec is None: raise gr.Error("Voice not found. Please pick one from the list.") emb = torch.tensor(vec, dtype=torch.float32).unsqueeze(0).to(device) gap = np.zeros(int(SAMPLE_RATE * 0.15), dtype=np.float32) pieces = [] for chunk in chunk_text(text): inputs = processor(text=chunk, return_tensors="pt", truncation=True, max_length=MAX_TOKENS) speech = model.generate_speech(inputs["input_ids"].to(device), emb, vocoder=vocoder) pieces += [speech.cpu().numpy().astype(np.float32), gap] audio = np.concatenate(pieces) if speed != 1.0: idx = np.arange(0, len(audio), speed) audio = np.interp(idx, np.arange(len(audio)), audio).astype(np.float32) return SAMPLE_RATE, audio # ---------- stateless endpoints for the website ---------- def embed_json(audio_path): """Audio file -> voice profile (512 numbers) as a JSON string.""" if encoder is None: raise gr.Error("Custom voices are unavailable in this Space.") if not audio_path: raise gr.Error("No audio received.") data, sr = sf.read(audio_path, dtype="float32", always_2d=False) return json.dumps(embed_recording((sr, data)).tolist()) def synthesize_api(text, voice, speed, embedding_json): """Like synthesize(), but the voice profile arrives with the request (nothing is stored).""" custom = None if embedding_json and embedding_json.strip(): try: v = np.asarray(json.loads(embedding_json), dtype=np.float32) except Exception: raise gr.Error("Invalid voice data.") if v.shape != (512,) or not np.all(np.isfinite(v)): raise gr.Error("Invalid voice data.") norm = float(np.linalg.norm(v)) if norm == 0: raise gr.Error("Invalid voice data.") custom = {"__custom__": v / norm} voice = "__custom__" return synthesize(text, voice, speed, custom) # ---------- UI ---------- with gr.Blocks(title="xqieech TTS") as demo: custom_state = gr.State({}) # per-session custom voices only gr.Markdown( "# 🎙️ xqieech TTS\n" "English text-to-speech. Pick a male or female voice, or add your own below. " "Long text is split by sentence automatically." ) with gr.Row(): with gr.Column(): text = gr.Textbox(label="Text", lines=5, value="Hello! This is a demo of my text to speech model.") voice = gr.Dropdown(VOICE_NAMES, value=DEFAULT_VOICE, label="Voice") speed = gr.Slider(0.7, 1.4, value=1.0, step=0.05, label="Speed") btn = gr.Button("Generate", variant="primary") with gr.Column(): out = gr.Audio(label="Output", type="numpy") if encoder is not None: with gr.Accordion("Add your own voice", open=False): gr.Markdown( "Record 10-20 seconds of clear speech, or upload a clip. " "**Only use your own voice, or one you have permission to use.** " "The voice lives only in your current session and disappears when you close the page. " "Results are approximate and may not sound exactly like you." ) rec = gr.Audio(sources=["microphone", "upload"], type="numpy", label="Voice sample") vname = gr.Textbox(label="Name for this voice", placeholder="e.g. Andrea") add_btn = gr.Button("Add voice") status = gr.Markdown() add_btn.click(add_voice, [rec, vname, custom_state], [custom_state, voice, status]) btn.click(synthesize, [text, voice, speed, custom_state], out, api_name="synthesize") # API-only endpoints used by the website (not visible in the Space UI) with gr.Column(visible=False): _a_in = gr.Audio(type="filepath") _a_json = gr.Textbox() _a_btn = gr.Button() _s_text = gr.Textbox() _s_voice = gr.Textbox() _s_speed = gr.Number(value=1.0) _s_emb = gr.Textbox() _s_out = gr.Audio(type="numpy") _s_btn = gr.Button() _a_btn.click(embed_json, _a_in, _a_json, api_name="embed") _s_btn.click(synthesize_api, [_s_text, _s_voice, _s_speed, _s_emb], _s_out, api_name="synthesize_custom") if __name__ == "__main__": demo.queue().launch()