Routyo / app.py
jsbeaudry's picture
Run on ZeroGPU: engine loads inside the GPU call, threads sized to the container
1aec157 verified
Raw History Blame Contribute Delete
11.4 kB
"""Routyo Space: which language is spoken, Kreyòl, English, French or Spanish? One sentence, or a whole conversation."""
import json
import os
import sys
import time
os.environ.setdefault("CACTUS_TELEMETRY", "0")
os.environ.setdefault("NEEDLE_TELEMETRY", "0")
import gradio as gr # noqa: E402
import numpy as np # noqa: E402
import spaces # noqa: E402
from huggingface_hub import hf_hub_download, snapshot_download # noqa: E402
# Download everything up front, but load the Cactus engine only inside the ZeroGPU call: its worker threads start when
# a model loads and do not survive the fork ZeroGPU makes for each call.
PKG = snapshot_download("jsbeaudry/Routyo", allow_patterns=["routyo.py", "routyo.npz", "whistle-kreyol.cact",
"word_check.json"])
STOCK = hf_hub_download("Cactus-Compute/whistle", "whistle.cact")
from cactus.agent.runtime import _library_path # noqa: E402
_library_path() # fetch the engine library now (no load)
sys.path.insert(0, PKG)
from routyo import LABELS, onset # noqa: E402
CHECK = json.load(open(os.path.join(PKG, "word_check.json")))
_ENGINE = {}
def cpus():
"""CPUs this container may use (cgroup quota), not the host's count."""
try:
quota, period = open("/sys/fs/cgroup/cpu.max").read().split()
if quota != "max":
return max(1, int(int(quota) / int(period)))
except (OSError, ValueError):
pass
try:
return len(os.sched_getaffinity(0))
except AttributeError:
return os.cpu_count() or 1
def engine():
"""Routyo and the two decoders, loaded once per process (so inside the GPU worker)."""
if not _ENGINE:
t = time.perf_counter()
import cactus
from routyo import Routyo, whistle_module
n = cpus()
cactus.set_threads(n)
r = Routyo(PKG, stock=STOCK)
W = whistle_module()
_ENGINE.update(r=r, dec={"kreyol": W.Whistle(r.kreyol.model.weights), "stock": W.Whistle(STOCK)},
threads=n, load_ms=(time.perf_counter() - t) * 1000, fresh=True)
print(f"engine loaded: {n} threads, {_ENGINE['load_ms']:.0f} ms", flush=True)
else:
_ENGINE["fresh"] = False
return _ENGINE
RATE = 16000
NAME = {"ht": "Kreyòl", "en": "English", "fr": "Français", "es": "Español"}
COLOR = {"Kreyòl": "#1f6feb", "English": "#d1242f", "Français": "#8250df", "Español": "#bf8700"}
MODES = {"All four": None, "Kreyòl ↔ English": ("ht", "en"), "Kreyòl ↔ Français": ("ht", "fr"),
"Kreyòl ↔ Español": ("ht", "es")}
HERE = os.path.dirname(os.path.abspath(__file__))
EX = json.load(open(os.path.join(HERE, "examples", "examples.json")))
def to16k(audio):
if audio is None:
raise gr.Error("Record or upload some speech first.")
sr, x = audio
x = np.asarray(x)
if np.issubdtype(x.dtype, np.integer):
x = x.astype(np.float32) / np.iinfo(x.dtype).max
x = x.astype(np.float32)
if x.ndim == 2:
x = x.mean(1)
if sr != RATE:
import soxr
x = soxr.resample(x, sr, RATE)
return np.ascontiguousarray(x, np.float32)
def judge(x, pair, words):
"""Routyo's rates for one stretch of speech, plus the transcript in each language decoded."""
e = engine()
t = time.perf_counter()
s = e["r"].log_probs(x)
ms_sound = (time.perf_counter() - t) * 1000
texts, ms_words = {}, 0.0
if words:
t = time.perf_counter()
wp = np.zeros(4)
for i, k in enumerate(LABELS):
if pair and k not in pair:
continue
res = e["dec"][CHECK["engines"][k]].transcribe(x, language=k, word_timestamps=True)
ws = res.get("words") or []
wp[i] = float(np.mean([w["probability"] for w in ws])) if ws else 0.0
texts[k] = (res.get("text") or "").strip()
s = (s + CHECK["beta"] * wp + np.array([CHECK["offsets"][k] for k in LABELS])) / CHECK["temperature"]
ms_words = (time.perf_counter() - t) * 1000
if pair:
s = np.where([k in pair for k in LABELS], s, -np.inf)
p = np.exp(s - s.max())
p /= p.sum()
return p, texts, ms_sound, ms_words
@spaces.GPU(duration=30)
def one(audio, mode, words, first):
x = to16k(audio)
if len(x) < 0.3 * RATE:
raise gr.Error("That is too short: say a few words.")
x = x[:30 * RATE]
if first:
a, b = onset(x, 1.5)
x = x[a:b]
pair = MODES[mode]
p, texts, ms_sound, ms_words = judge(x, pair, words)
k = int(np.argmax(p))
rates = {NAME[LABELS[i]]: float(p[i]) for i in range(4) if not pair or LABELS[i] in pair}
lines = [f"### {NAME[LABELS[k]]} · {p[k] * 100:.0f} %",
f"Sound: **{ms_sound:.0f} ms**" + (f" · word check: **{ms_words:.0f} ms**" if words else "") +
f" · {len(x) / RATE:.1f} s of audio" + (" (first 1.5 s of speech)" if first else "") +
(f" · engine started in {_ENGINE['load_ms']:.0f} ms" if _ENGINE.get("fresh") else "")]
if texts.get(LABELS[k]):
lines.append(f"Heard ({NAME[LABELS[k]]}): *{texts[LABELS[k]]}*")
return rates, "\n\n".join(lines)
def pieces(x, gap=0.8, shortest=0.5):
"""Split a recording at its pauses (20 ms frames quieter than 35 dB under the loudest)."""
hop = 320
n = len(x) // hop
if n == 0:
return []
rms = np.sqrt((x[:n * hop].reshape(n, hop) ** 2).mean(1) + 1e-12)
db = 20 * np.log10(rms)
on = db > max(db.max() - 35, -60)
runs, start = [], None
for i, v in enumerate(list(on) + [False]):
if v and start is None:
start = i
elif not v and start is not None:
runs.append([start, i])
start = None
merged = []
for a, b in runs:
if merged and (a - merged[-1][1]) * hop < gap * RATE:
merged[-1][1] = b
else:
merged.append([a, b])
pad = int(0.1 * RATE)
return [(max(a * hop - pad, 0), min(b * hop + pad, len(x))) for a, b in merged if (b - a) * hop >= shortest * RATE]
@spaces.GPU(duration=90)
def conversation(audio, mode, words):
x = to16k(audio)[:90 * RATE]
pair = MODES[mode]
marks, rows = [], []
t0 = time.perf_counter()
for i, (a, b) in enumerate(pieces(x), 1):
p, texts, ms_sound, _ = judge(x[a:b], pair, words)
k = LABELS[int(np.argmax(p))]
said = texts.get(k) or f"sentence {i}"
marks.append((said + " ", NAME[k]))
rows.append([i, f"{a / RATE:.1f}–{b / RATE:.1f} s", NAME[k], f"{p.max() * 100:.0f} %", f"{ms_sound:.0f} ms", said])
if not rows:
raise gr.Error("No speech found in that recording.")
note = f"{len(rows)} sentences routed in {(time.perf_counter() - t0):.1f} s" + (" (with the word check)" if words else "")
return marks, rows, note
HEAD = """
# 🧭 Routyo
**Which language is spoken: Kreyòl, English, French or Spanish?** Routyo listens to one sentence and says, in tens of
milliseconds on a CPU, so each sentence can go to the right small model: speech recognition, translation, a voice.
*Routyo koute yon fraz epi li di ki lang li ye: kreyòl, anglè, franse oswa panyòl.*
Any voice, any accent: a Haitian speaking English is English; a foreigner speaking Kreyòl is Kreyòl.
Model: [jsbeaudry/Routyo](https://huggingface.co/jsbeaudry/Routyo) · built on
[Whistle](https://huggingface.co/Cactus-Compute/whistle) · 99.7 % on real speech, 96.6 % on voices never heard in
training (Whisper large-v3-turbo: 91.7 % and 51.5 %).
"""
FOOT = """
**About the examples.** Every example is new to Routyo: voices kept out of training and sentences from topics it never
saw. Most are synthetic (Fatiman V3, our Kreyòl voice model, and
[Phonon-40m-multilingual](https://huggingface.co/gradium/phonon-40m-multilingual), CC BY-NC 4.0) so that one voice can
speak several languages; the real recordings come from FLEURS (CC BY 4.0) and VoxPopuli (CC0). They were chosen
among clips Routyo routes right; the [model card](https://huggingface.co/jsbeaudry/Routyo) gives its accuracy on
thousands of clips, misses included. The **word check** adds a short transcription in each language and is more accurate (~0.3 s); turn it off to see
the sound-only speed. Audio you record is processed in memory and not kept.
"""
with gr.Blocks(title="Routyo", theme=gr.themes.Soft(primary_hue="blue")) as demo:
gr.Markdown(HEAD)
with gr.Tab("One sentence"):
with gr.Row():
with gr.Column():
audio = gr.Audio(sources=["microphone", "upload"], type="numpy", label="Speak or upload a sentence")
mode = gr.Radio(list(MODES), value="All four", label="Languages to choose from")
with gr.Row():
words = gr.Checkbox(value=True, label="Word check (more accurate)")
first = gr.Checkbox(value=False, label="First 1.5 s only")
go = gr.Button("Which language?", variant="primary")
with gr.Column():
rates = gr.Label(num_top_classes=4, label="Language")
info = gr.Markdown()
go.click(one, [audio, mode, words, first], [rates, info])
for lang in LABELS:
rows = [e for e in EX["examples"] if e["language"] == lang]
gr.Examples([[os.path.join(HERE, "examples", e["file"]), "All four", True, False] for e in rows],
[audio, mode, words, first], [rates, info], fn=one, run_on_click=True, cache_examples=False,
label=f"{NAME[lang]} examples", example_labels=[e["label"].split(" · ", 1)[1] for e in rows])
with gr.Tab("A whole conversation"):
gr.Markdown("Upload or record several sentences, with a pause of about a second between them, in any mix of "
"the four languages. Routyo splits at the pauses and routes each sentence on its own.")
with gr.Row():
with gr.Column():
c_audio = gr.Audio(sources=["upload", "microphone"], type="numpy", label="A conversation (up to 90 s)")
c_mode = gr.Radio(list(MODES), value="All four", label="Languages to choose from")
c_words = gr.Checkbox(value=True, label="Word check (shows what was heard)")
c_go = gr.Button("Route each sentence", variant="primary")
with gr.Column():
c_marks = gr.HighlightedText(label="Each sentence, coloured by language", color_map=COLOR,
show_legend=True, combine_adjacent=False)
c_note = gr.Markdown()
c_rows = gr.Dataframe(headers=["#", "time", "language", "rate", "sound only", "heard"], wrap=True,
label="Sentences")
c_go.click(conversation, [c_audio, c_mode, c_words], [c_marks, c_rows, c_note])
gr.Examples([[os.path.join(HERE, "examples", c["file"]), "All four", True] for c in EX["conversations"]],
[c_audio, c_mode, c_words], [c_marks, c_rows, c_note], fn=conversation, run_on_click=True,
cache_examples=False, label="Conversation examples",
example_labels=[c["title"].split(" (")[0] for c in EX["conversations"]])
gr.Markdown(FOOT)
demo.queue(default_concurrency_limit=1, max_size=32)
if __name__ == "__main__":
demo.launch()