Spaces:
Sleeping
Sleeping
| import json | |
| import re | |
| import numpy as np | |
| import soundfile as sf | |
| import torch | |
| import torchaudio | |
| import gradio as gr | |
| try: | |
| import spaces # required on ZeroGPU Spaces | |
| gpu = spaces.GPU | |
| except ImportError: | |
| def gpu(fn=None, **kwargs): | |
| return fn if fn is not None else (lambda f: f) | |
| from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan | |
| from datasets import load_dataset | |
| MODEL_ID = "AndreaNotes1/xqieech" | |
| FALLBACK_PROCESSOR_ID = "microsoft/speecht5_tts" | |
| VOCODER_ID = "microsoft/speecht5_hifigan" | |
| SAMPLE_RATE = 16000 | |
| MAX_TOKENS = 600 | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| # ---------- model ---------- | |
| try: | |
| processor = SpeechT5Processor.from_pretrained(MODEL_ID) | |
| except Exception: | |
| processor = SpeechT5Processor.from_pretrained(FALLBACK_PROCESSOR_ID) | |
| model = SpeechT5ForTextToSpeech.from_pretrained(MODEL_ID).to(device).eval() | |
| vocoder = SpeechT5HifiGan.from_pretrained(VOCODER_ID).to(device).eval() | |
| # ---------- built-in voices (CMU Arctic x-vectors, looked up by speaker code) ---------- | |
| VOICE_META = { | |
| "slt": "Female - US (slt)", | |
| "clb": "Female - US (clb)", | |
| "bdl": "Male - US (bdl)", | |
| "rms": "Male - US (rms)", | |
| "awb": "Male - Scottish (awb)", | |
| "jmk": "Male - Canadian (jmk)", | |
| "ksp": "Male - Indian (ksp)", | |
| } | |
| N_AVG = 40 | |
| BUILTIN = {} # label -> np.ndarray (512,) | |
| try: | |
| ds = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation") | |
| names = ds["filename"] | |
| for code, label in VOICE_META.items(): | |
| idx = [i for i, f in enumerate(names) if f"cmu_us_{code}_" in f][:N_AVG] | |
| if not idx: | |
| continue | |
| v = torch.tensor(np.array(ds.select(idx)["xvector"]), dtype=torch.float32).mean(0) | |
| BUILTIN[label] = (v / v.norm()).numpy() | |
| except Exception as e: | |
| print("Could not load x-vectors:", e) | |
| BUILTIN["Neutral (zeros)"] = np.zeros(512, dtype=np.float32) | |
| VOICE_NAMES = sorted(BUILTIN, key=lambda n: (not n.startswith("Female"), n)) | |
| DEFAULT_VOICE = VOICE_NAMES[0] | |
| # ---------- speaker encoder for "record your own voice" ---------- | |
| encoder = None | |
| try: | |
| try: | |
| from speechbrain.inference.speaker import EncoderClassifier | |
| except ImportError: | |
| from speechbrain.pretrained import EncoderClassifier | |
| encoder = EncoderClassifier.from_hparams( | |
| source="speechbrain/spkrec-xvect-voxceleb", | |
| savedir="pretrained_models/spkrec-xvect-voxceleb", | |
| run_opts={"device": "cpu"}, | |
| ) | |
| except Exception as e: | |
| print("Speaker encoder unavailable, custom voices disabled:", e) | |
| def embed_recording(audio): | |
| """audio: (sample_rate, np.ndarray) from gr.Audio(type='numpy') -> (512,) x-vector.""" | |
| sr, data = audio | |
| data = np.asarray(data) | |
| if data.dtype.kind == "i": | |
| data = data.astype(np.float32) / np.iinfo(data.dtype).max | |
| else: | |
| data = data.astype(np.float32) | |
| if data.ndim > 1: | |
| data = data.mean(axis=1) | |
| wav = torch.from_numpy(data).unsqueeze(0) | |
| if sr != SAMPLE_RATE: | |
| wav = torchaudio.functional.resample(wav, sr, SAMPLE_RATE) | |
| if wav.shape[1] < SAMPLE_RATE * 3: | |
| raise gr.Error("Please record at least 3 seconds (10-20 seconds works best).") | |
| with torch.no_grad(): | |
| emb = encoder.encode_batch(wav) | |
| emb = torch.nn.functional.normalize(emb, dim=2) | |
| return emb.squeeze().cpu().numpy() | |
| def add_voice(audio, name, custom): | |
| if encoder is None: | |
| raise gr.Error("Custom voices are unavailable in this Space.") | |
| if audio is None: | |
| raise gr.Error("Record or upload a voice sample first.") | |
| custom = dict(custom or {}) | |
| base = (name or "").strip() or "My voice" | |
| label, n = f"{base} (custom)", 2 | |
| while label in custom or label in BUILTIN: | |
| label, n = f"{base} {n} (custom)", n + 1 | |
| custom[label] = embed_recording(audio) | |
| choices = VOICE_NAMES + list(custom) | |
| return custom, gr.update(choices=choices, value=label), f"Added '{label}'. It's selected in the Voice list." | |
| # ---------- synthesis ---------- | |
| def chunk_text(text, max_chars=250): | |
| sentences = re.split(r"(?<=[.!?])\s+", text) | |
| chunks, cur = [], "" | |
| for s in sentences: | |
| if len(cur) + len(s) + 1 <= max_chars: | |
| cur = f"{cur} {s}".strip() | |
| else: | |
| if cur: | |
| chunks.append(cur) | |
| while len(s) > max_chars: | |
| chunks.append(s[:max_chars]) | |
| s = s[max_chars:] | |
| cur = s | |
| if cur: | |
| chunks.append(cur) | |
| return chunks | |
| def synthesize(text, voice, speed, custom): | |
| text = re.sub(r"\s+", " ", (text or "").strip()) | |
| if not text: | |
| raise gr.Error("Please enter some text.") | |
| vec = (custom or {}).get(voice) | |
| if vec is None: | |
| vec = BUILTIN.get(voice) | |
| if vec is None: | |
| raise gr.Error("Voice not found. Please pick one from the list.") | |
| emb = torch.tensor(vec, dtype=torch.float32).unsqueeze(0).to(device) | |
| gap = np.zeros(int(SAMPLE_RATE * 0.15), dtype=np.float32) | |
| pieces = [] | |
| for chunk in chunk_text(text): | |
| inputs = processor(text=chunk, return_tensors="pt", truncation=True, max_length=MAX_TOKENS) | |
| speech = model.generate_speech(inputs["input_ids"].to(device), emb, vocoder=vocoder) | |
| pieces += [speech.cpu().numpy().astype(np.float32), gap] | |
| audio = np.concatenate(pieces) | |
| if speed != 1.0: | |
| idx = np.arange(0, len(audio), speed) | |
| audio = np.interp(idx, np.arange(len(audio)), audio).astype(np.float32) | |
| return SAMPLE_RATE, audio | |
| # ---------- stateless endpoints for the website ---------- | |
| def embed_json(audio_path): | |
| """Audio file -> voice profile (512 numbers) as a JSON string.""" | |
| if encoder is None: | |
| raise gr.Error("Custom voices are unavailable in this Space.") | |
| if not audio_path: | |
| raise gr.Error("No audio received.") | |
| data, sr = sf.read(audio_path, dtype="float32", always_2d=False) | |
| return json.dumps(embed_recording((sr, data)).tolist()) | |
| def synthesize_api(text, voice, speed, embedding_json): | |
| """Like synthesize(), but the voice profile arrives with the request (nothing is stored).""" | |
| custom = None | |
| if embedding_json and embedding_json.strip(): | |
| try: | |
| v = np.asarray(json.loads(embedding_json), dtype=np.float32) | |
| except Exception: | |
| raise gr.Error("Invalid voice data.") | |
| if v.shape != (512,) or not np.all(np.isfinite(v)): | |
| raise gr.Error("Invalid voice data.") | |
| norm = float(np.linalg.norm(v)) | |
| if norm == 0: | |
| raise gr.Error("Invalid voice data.") | |
| custom = {"__custom__": v / norm} | |
| voice = "__custom__" | |
| return synthesize(text, voice, speed, custom) | |
| # ---------- UI ---------- | |
| with gr.Blocks(title="xqieech TTS") as demo: | |
| custom_state = gr.State({}) # per-session custom voices only | |
| gr.Markdown( | |
| "# 🎙️ xqieech TTS\n" | |
| "English text-to-speech. Pick a male or female voice, or add your own below. " | |
| "Long text is split by sentence automatically." | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| text = gr.Textbox(label="Text", lines=5, | |
| value="Hello! This is a demo of my text to speech model.") | |
| voice = gr.Dropdown(VOICE_NAMES, value=DEFAULT_VOICE, label="Voice") | |
| speed = gr.Slider(0.7, 1.4, value=1.0, step=0.05, label="Speed") | |
| btn = gr.Button("Generate", variant="primary") | |
| with gr.Column(): | |
| out = gr.Audio(label="Output", type="numpy") | |
| if encoder is not None: | |
| with gr.Accordion("Add your own voice", open=False): | |
| gr.Markdown( | |
| "Record 10-20 seconds of clear speech, or upload a clip. " | |
| "**Only use your own voice, or one you have permission to use.** " | |
| "The voice lives only in your current session and disappears when you close the page. " | |
| "Results are approximate and may not sound exactly like you." | |
| ) | |
| rec = gr.Audio(sources=["microphone", "upload"], type="numpy", label="Voice sample") | |
| vname = gr.Textbox(label="Name for this voice", placeholder="e.g. Andrea") | |
| add_btn = gr.Button("Add voice") | |
| status = gr.Markdown() | |
| add_btn.click(add_voice, [rec, vname, custom_state], [custom_state, voice, status]) | |
| btn.click(synthesize, [text, voice, speed, custom_state], out, api_name="synthesize") | |
| # API-only endpoints used by the website (not visible in the Space UI) | |
| with gr.Column(visible=False): | |
| _a_in = gr.Audio(type="filepath") | |
| _a_json = gr.Textbox() | |
| _a_btn = gr.Button() | |
| _s_text = gr.Textbox() | |
| _s_voice = gr.Textbox() | |
| _s_speed = gr.Number(value=1.0) | |
| _s_emb = gr.Textbox() | |
| _s_out = gr.Audio(type="numpy") | |
| _s_btn = gr.Button() | |
| _a_btn.click(embed_json, _a_in, _a_json, api_name="embed") | |
| _s_btn.click(synthesize_api, [_s_text, _s_voice, _s_speed, _s_emb], _s_out, | |
| api_name="synthesize_custom") | |
| if __name__ == "__main__": | |
| demo.queue().launch() | |