"""Routyo Space: which language is spoken, Kreyòl, English, French or Spanish? One sentence, or a whole conversation.""" import json import os import sys import time os.environ.setdefault("CACTUS_TELEMETRY", "0") os.environ.setdefault("NEEDLE_TELEMETRY", "0") import gradio as gr # noqa: E402 import numpy as np # noqa: E402 import spaces # noqa: E402 from huggingface_hub import hf_hub_download, snapshot_download # noqa: E402 # Download everything up front, but load the Cactus engine only inside the ZeroGPU call: its worker threads start when # a model loads and do not survive the fork ZeroGPU makes for each call. PKG = snapshot_download("jsbeaudry/Routyo", allow_patterns=["routyo.py", "routyo.npz", "whistle-kreyol.cact", "word_check.json"]) STOCK = hf_hub_download("Cactus-Compute/whistle", "whistle.cact") from cactus.agent.runtime import _library_path # noqa: E402 _library_path() # fetch the engine library now (no load) sys.path.insert(0, PKG) from routyo import LABELS, onset # noqa: E402 CHECK = json.load(open(os.path.join(PKG, "word_check.json"))) _ENGINE = {} def cpus(): """CPUs this container may use (cgroup quota), not the host's count.""" try: quota, period = open("/sys/fs/cgroup/cpu.max").read().split() if quota != "max": return max(1, int(int(quota) / int(period))) except (OSError, ValueError): pass try: return len(os.sched_getaffinity(0)) except AttributeError: return os.cpu_count() or 1 def engine(): """Routyo and the two decoders, loaded once per process (so inside the GPU worker).""" if not _ENGINE: t = time.perf_counter() import cactus from routyo import Routyo, whistle_module n = cpus() cactus.set_threads(n) r = Routyo(PKG, stock=STOCK) W = whistle_module() _ENGINE.update(r=r, dec={"kreyol": W.Whistle(r.kreyol.model.weights), "stock": W.Whistle(STOCK)}, threads=n, load_ms=(time.perf_counter() - t) * 1000, fresh=True) print(f"engine loaded: {n} threads, {_ENGINE['load_ms']:.0f} ms", flush=True) else: _ENGINE["fresh"] = False return _ENGINE RATE = 16000 NAME = {"ht": "Kreyòl", "en": "English", "fr": "Français", "es": "Español"} COLOR = {"Kreyòl": "#1f6feb", "English": "#d1242f", "Français": "#8250df", "Español": "#bf8700"} MODES = {"All four": None, "Kreyòl ↔ English": ("ht", "en"), "Kreyòl ↔ Français": ("ht", "fr"), "Kreyòl ↔ Español": ("ht", "es")} HERE = os.path.dirname(os.path.abspath(__file__)) EX = json.load(open(os.path.join(HERE, "examples", "examples.json"))) def to16k(audio): if audio is None: raise gr.Error("Record or upload some speech first.") sr, x = audio x = np.asarray(x) if np.issubdtype(x.dtype, np.integer): x = x.astype(np.float32) / np.iinfo(x.dtype).max x = x.astype(np.float32) if x.ndim == 2: x = x.mean(1) if sr != RATE: import soxr x = soxr.resample(x, sr, RATE) return np.ascontiguousarray(x, np.float32) def judge(x, pair, words): """Routyo's rates for one stretch of speech, plus the transcript in each language decoded.""" e = engine() t = time.perf_counter() s = e["r"].log_probs(x) ms_sound = (time.perf_counter() - t) * 1000 texts, ms_words = {}, 0.0 if words: t = time.perf_counter() wp = np.zeros(4) for i, k in enumerate(LABELS): if pair and k not in pair: continue res = e["dec"][CHECK["engines"][k]].transcribe(x, language=k, word_timestamps=True) ws = res.get("words") or [] wp[i] = float(np.mean([w["probability"] for w in ws])) if ws else 0.0 texts[k] = (res.get("text") or "").strip() s = (s + CHECK["beta"] * wp + np.array([CHECK["offsets"][k] for k in LABELS])) / CHECK["temperature"] ms_words = (time.perf_counter() - t) * 1000 if pair: s = np.where([k in pair for k in LABELS], s, -np.inf) p = np.exp(s - s.max()) p /= p.sum() return p, texts, ms_sound, ms_words @spaces.GPU(duration=30) def one(audio, mode, words, first): x = to16k(audio) if len(x) < 0.3 * RATE: raise gr.Error("That is too short: say a few words.") x = x[:30 * RATE] if first: a, b = onset(x, 1.5) x = x[a:b] pair = MODES[mode] p, texts, ms_sound, ms_words = judge(x, pair, words) k = int(np.argmax(p)) rates = {NAME[LABELS[i]]: float(p[i]) for i in range(4) if not pair or LABELS[i] in pair} lines = [f"### {NAME[LABELS[k]]} · {p[k] * 100:.0f} %", f"Sound: **{ms_sound:.0f} ms**" + (f" · word check: **{ms_words:.0f} ms**" if words else "") + f" · {len(x) / RATE:.1f} s of audio" + (" (first 1.5 s of speech)" if first else "") + (f" · engine started in {_ENGINE['load_ms']:.0f} ms" if _ENGINE.get("fresh") else "")] if texts.get(LABELS[k]): lines.append(f"Heard ({NAME[LABELS[k]]}): *{texts[LABELS[k]]}*") return rates, "\n\n".join(lines) def pieces(x, gap=0.8, shortest=0.5): """Split a recording at its pauses (20 ms frames quieter than 35 dB under the loudest).""" hop = 320 n = len(x) // hop if n == 0: return [] rms = np.sqrt((x[:n * hop].reshape(n, hop) ** 2).mean(1) + 1e-12) db = 20 * np.log10(rms) on = db > max(db.max() - 35, -60) runs, start = [], None for i, v in enumerate(list(on) + [False]): if v and start is None: start = i elif not v and start is not None: runs.append([start, i]) start = None merged = [] for a, b in runs: if merged and (a - merged[-1][1]) * hop < gap * RATE: merged[-1][1] = b else: merged.append([a, b]) pad = int(0.1 * RATE) return [(max(a * hop - pad, 0), min(b * hop + pad, len(x))) for a, b in merged if (b - a) * hop >= shortest * RATE] @spaces.GPU(duration=90) def conversation(audio, mode, words): x = to16k(audio)[:90 * RATE] pair = MODES[mode] marks, rows = [], [] t0 = time.perf_counter() for i, (a, b) in enumerate(pieces(x), 1): p, texts, ms_sound, _ = judge(x[a:b], pair, words) k = LABELS[int(np.argmax(p))] said = texts.get(k) or f"sentence {i}" marks.append((said + " ", NAME[k])) rows.append([i, f"{a / RATE:.1f}–{b / RATE:.1f} s", NAME[k], f"{p.max() * 100:.0f} %", f"{ms_sound:.0f} ms", said]) if not rows: raise gr.Error("No speech found in that recording.") note = f"{len(rows)} sentences routed in {(time.perf_counter() - t0):.1f} s" + (" (with the word check)" if words else "") return marks, rows, note HEAD = """ # 🧭 Routyo **Which language is spoken: Kreyòl, English, French or Spanish?** Routyo listens to one sentence and says, in tens of milliseconds on a CPU, so each sentence can go to the right small model: speech recognition, translation, a voice. *Routyo koute yon fraz epi li di ki lang li ye: kreyòl, anglè, franse oswa panyòl.* Any voice, any accent: a Haitian speaking English is English; a foreigner speaking Kreyòl is Kreyòl. Model: [jsbeaudry/Routyo](https://huggingface.co/jsbeaudry/Routyo) · built on [Whistle](https://huggingface.co/Cactus-Compute/whistle) · 99.7 % on real speech, 96.6 % on voices never heard in training (Whisper large-v3-turbo: 91.7 % and 51.5 %). """ FOOT = """ **About the examples.** Every example is new to Routyo: voices kept out of training and sentences from topics it never saw. Most are synthetic (Fatiman V3, our Kreyòl voice model, and [Phonon-40m-multilingual](https://huggingface.co/gradium/phonon-40m-multilingual), CC BY-NC 4.0) so that one voice can speak several languages; the real recordings come from FLEURS (CC BY 4.0) and VoxPopuli (CC0). They were chosen among clips Routyo routes right; the [model card](https://huggingface.co/jsbeaudry/Routyo) gives its accuracy on thousands of clips, misses included. The **word check** adds a short transcription in each language and is more accurate (~0.3 s); turn it off to see the sound-only speed. Audio you record is processed in memory and not kept. """ with gr.Blocks(title="Routyo", theme=gr.themes.Soft(primary_hue="blue")) as demo: gr.Markdown(HEAD) with gr.Tab("One sentence"): with gr.Row(): with gr.Column(): audio = gr.Audio(sources=["microphone", "upload"], type="numpy", label="Speak or upload a sentence") mode = gr.Radio(list(MODES), value="All four", label="Languages to choose from") with gr.Row(): words = gr.Checkbox(value=True, label="Word check (more accurate)") first = gr.Checkbox(value=False, label="First 1.5 s only") go = gr.Button("Which language?", variant="primary") with gr.Column(): rates = gr.Label(num_top_classes=4, label="Language") info = gr.Markdown() go.click(one, [audio, mode, words, first], [rates, info]) for lang in LABELS: rows = [e for e in EX["examples"] if e["language"] == lang] gr.Examples([[os.path.join(HERE, "examples", e["file"]), "All four", True, False] for e in rows], [audio, mode, words, first], [rates, info], fn=one, run_on_click=True, cache_examples=False, label=f"{NAME[lang]} examples", example_labels=[e["label"].split(" · ", 1)[1] for e in rows]) with gr.Tab("A whole conversation"): gr.Markdown("Upload or record several sentences, with a pause of about a second between them, in any mix of " "the four languages. Routyo splits at the pauses and routes each sentence on its own.") with gr.Row(): with gr.Column(): c_audio = gr.Audio(sources=["upload", "microphone"], type="numpy", label="A conversation (up to 90 s)") c_mode = gr.Radio(list(MODES), value="All four", label="Languages to choose from") c_words = gr.Checkbox(value=True, label="Word check (shows what was heard)") c_go = gr.Button("Route each sentence", variant="primary") with gr.Column(): c_marks = gr.HighlightedText(label="Each sentence, coloured by language", color_map=COLOR, show_legend=True, combine_adjacent=False) c_note = gr.Markdown() c_rows = gr.Dataframe(headers=["#", "time", "language", "rate", "sound only", "heard"], wrap=True, label="Sentences") c_go.click(conversation, [c_audio, c_mode, c_words], [c_marks, c_rows, c_note]) gr.Examples([[os.path.join(HERE, "examples", c["file"]), "All four", True] for c in EX["conversations"]], [c_audio, c_mode, c_words], [c_marks, c_rows, c_note], fn=conversation, run_on_click=True, cache_examples=False, label="Conversation examples", example_labels=[c["title"].split(" (")[0] for c in EX["conversations"]]) gr.Markdown(FOOT) demo.queue(default_concurrency_limit=1, max_size=32) if __name__ == "__main__": demo.launch()