File size: 15,500 Bytes
4c7fa41 4e26536 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 0f54627 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 4e26536 4c7fa41 90e4342 4c7fa41 4e26536 0f54627 4e26536 0f54627 90e4342 4c7fa41 90e4342 4c7fa41 84bee02 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 90e4342 4c7fa41 84bee02 4c7fa41 90e4342 4e26536 90e4342 4c7fa41 90e4342 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 | """rumblex demo: transcription with word timestamps and subtitles from Reson8/Rumble-2-Flash."""
from __future__ import annotations
import html
import json
import os
import tempfile
import time
import wave
from pathlib import Path
from urllib.parse import quote
import gradio as gr
import numpy as np
import torch
import rumblex
from rumblex.audio import load_audio
from rumblex.writers import WriterOptions
ZERO_GPU = os.environ.get("SPACES_ZERO_GPU", "").lower() in ("1", "true")
ON_GPU = ZERO_GPU or torch.cuda.is_available()
MAX_SECONDS = float(os.environ.get("MAX_SECONDS", "600" if ON_GPU else "120"))
SR = 16000
PX_PER_SECOND = 110 # width of one second on the word timeline
LANGUAGES = ["Auto-detect", "English", "Dutch", "German", "French", "Spanish", "Italian", "Portuguese"]
SPEAKER_COLOURS = ["#6366f1", "#d97706", "#059669", "#db2777"]
OUT_ROOT = Path(tempfile.gettempdir()) / "rumblex_demo" # served to the browser (allowed_paths below)
OUT_ROOT.mkdir(exist_ok=True)
def gpu_seconds(audio_path, language=None, line_width=None, progress=None) -> int:
"""GPU time to reserve on ZeroGPU. Each call uses up this much of the visitor's daily quota, so short
clips ask for little (logged-out visitors only get a few minutes)."""
try:
seconds = min(len(load_audio(audio_path)) / SR, MAX_SECONDS)
except Exception:
return 60
return int(min(120, 20 + seconds / 10))
try: # Hugging Face's ZeroGPU helper; a no-op outside ZeroGPU hardware
import spaces
gpu = spaces.GPU(duration=gpu_seconds)
except ImportError:
def gpu(fn):
return fn
# On ZeroGPU the model is placed on cuda at import time (the GPU is attached inside @gpu calls). The dtype is
# given explicitly so nothing queries the device before a GPU is attached.
DEVICE = "cuda" if ON_GPU else "cpu"
MODEL = rumblex.load_model(
device=DEVICE, compute_type="bfloat16" if ON_GPU else "float32", batch_size=16 if ON_GPU else 4
)
# Player, word timeline and karaoke transcript. Event delegation on the document, so it works for every
# result Gradio renders. A requestAnimationFrame loop repaints while playing (timeupdate is only ~4 Hz).
HEAD = """
<script>
(() => {
const origin = (e) => (e.composedPath ? e.composedPath()[0] : e.target);
const clock = (t) => `${Math.floor(t / 60)}:${String(Math.floor(t % 60)).padStart(2, "0")}`;
function paint(box) {
const a = box.querySelector("audio");
const t = a.currentTime, d = a.duration || +box.dataset.dur;
box.querySelector(".rx-fill").style.width = `${Math.min(100, (100 * t) / d)}%`;
box.querySelector(".rx-time").textContent = `${clock(t)} / ${clock(d)}`;
box.querySelector(".rx-play").textContent = a.paused ? "▶" : "❚❚";
const px = +box.dataset.px, head = box.querySelector(".rx-head"), tl = box.querySelector(".rx-tl");
head.style.left = `${t * px}px`;
if (!a.paused) tl.scrollLeft = Math.max(0, t * px - tl.clientWidth / 3);
let current = null;
for (const w of box.querySelectorAll("[data-s]")) {
const on = t >= +w.dataset.s && t < +w.dataset.e;
w.classList.toggle("on", on);
w.classList.toggle("done", t >= +w.dataset.e);
if (on && w.classList.contains("w")) current = w;
}
const text = box.querySelector(".rx-text");
if (current && !a.paused) {
const y = current.offsetTop - text.offsetTop;
if (y < text.scrollTop || y > text.scrollTop + text.clientHeight - 40) text.scrollTop = y - 40;
}
}
function loop(box) {
paint(box);
if (!box.querySelector("audio").paused) requestAnimationFrame(() => loop(box));
}
for (const ev of ["play", "pause", "seeked", "loadedmetadata", "ended"]) {
document.addEventListener(ev, (e) => {
const a = origin(e);
const box = a && a.closest && a.closest(".rx");
if (box) ev === "play" ? loop(box) : paint(box);
}, true);
}
document.addEventListener("click", (e) => {
const el = origin(e);
if (!el.closest) return;
const box = el.closest(".rx");
if (!box) return;
const a = box.querySelector("audio");
if (el.closest(".rx-play")) { a.paused ? a.play() : a.pause(); return; }
const prog = el.closest(".rx-prog");
if (prog) {
const r = prog.getBoundingClientRect();
a.currentTime = ((e.clientX - r.left) / r.width) * (a.duration || +box.dataset.dur);
return;
}
const w = el.closest("[data-s]");
if (w) { a.currentTime = +w.dataset.s + 0.001; a.play(); }
}, true);
})();
</script>
<style>
.rx-hero h1 { font-size: 2.1em; font-weight: 700; margin: 0 0 .1em; letter-spacing: -.02em; }
.rx-hero p { margin: .2em 0 .7em; opacity: .8; font-size: 1.05em; max-width: 60em; }
.rx-links { display: flex; gap: .5em; flex-wrap: wrap; margin-bottom: .3em; }
.rx-links a { text-decoration: none !important; border: 1px solid var(--border-color-primary);
border-radius: 999px; padding: .25em .85em; font-size: .9em; color: var(--body-text-color) !important; }
.rx-links a:hover { border-color: #6366f1; }
.rx-empty { border: 1px dashed var(--border-color-primary); border-radius: 12px; padding: 3em 1.5em;
text-align: center; opacity: .65; }
.rx { border: 1px solid var(--border-color-primary); border-radius: 12px; padding: 1em 1.1em;
background: var(--block-background-fill); }
.rx-stats { display: flex; gap: .45em; flex-wrap: wrap; margin-bottom: .8em; }
.rx-stats span { background: rgba(99,102,241,.14); border-radius: 999px; padding: .2em .75em; font-size: .85em; }
.rx-bar { display: flex; align-items: center; gap: .8em; }
.rx-play { width: 2.5em; height: 2.5em; border-radius: 50%; border: none; background: #6366f1; color: #fff;
font-size: .95em; cursor: pointer; flex: none; }
.rx-play:hover { background: #4f46e5; }
.rx-time { font-family: ui-monospace, monospace; font-size: .85em; opacity: .7; flex: none; min-width: 6.5em; }
.rx-prog { flex: 1; height: 6px; border-radius: 3px; background: rgba(127,127,127,.25); cursor: pointer;
position: relative; }
.rx-fill { position: absolute; inset: 0 auto 0 0; width: 0; border-radius: 3px; background: #6366f1; }
.rx-tl { overflow-x: auto; margin: .9em 0 .6em; padding-bottom: 4px; }
.rx-tl-inner { position: relative; height: 34px; }
.rx-tl .tw { position: absolute; top: 4px; height: 26px; line-height: 26px; font-size: .78em; overflow: hidden;
white-space: nowrap; text-overflow: ellipsis; padding: 0 3px; border-radius: 4px; cursor: pointer;
background: rgba(99,102,241,.18); border-left: 2px solid rgba(99,102,241,.7); box-sizing: border-box; }
.rx-tl .tw.done { opacity: .55; }
.rx-tl .tw.on { background: #6366f1; color: #fff; opacity: 1; }
.rx-head { position: absolute; top: 0; bottom: 0; width: 2px; background: #f43f5e; left: 0; pointer-events: none; }
.rx-text { max-height: 380px; overflow-y: auto; }
.rx .seg { display: flex; gap: .9em; padding: .35em 0; line-height: 1.85; font-size: 1.08em; }
.rx .t { font-family: ui-monospace, monospace; font-size: .75em; opacity: .5; padding-top: .45em; flex: none;
width: 3.4em; }
.rx .w { padding: 2px 3px; border-radius: 4px; cursor: pointer; transition: background .08s; }
.rx .w:hover { outline: 1px solid rgba(99,102,241,.6); }
.rx .w.done { opacity: .6; }
.rx .w.on { background: #6366f1; color: #fff; opacity: 1; }
</style>
"""
HERO = """
<div class="rx-hero">
<h1>rumblex</h1>
<p>Word-level timestamps from <b>Reson8/Rumble-2-Flash</b>, aligned with the model's own CTC head. Upload a
file or record yourself, then press play and watch each word light up as it is spoken.</p>
<div class="rx-links">
<a href="https://github.com/EdwardGlockner/rumblex" target="_blank">GitHub</a>
<a href="https://pypi.org/project/rumblex/" target="_blank">pip install rumblex</a>
<a href="https://huggingface.co/Reson8/Rumble-2-Flash" target="_blank">Rumble-2-Flash</a>
</div>
</div>
"""
EXAMPLE = str(Path(__file__).with_name("librispeech_mr_quilter.wav"))
EMPTY = (
'<div class="rx-empty">Your transcript, word timeline and player will appear here.<br>'
"No audio at hand? Click <b>Try an example clip</b> on the left.</div>"
)
FOOTER = """
The demo transcribes the first {limit} of each file. Speaker labels (pyannote) are in the package, not in
this demo. rumblex is an independent project and is not affiliated with Reson8.
""".replace("{limit}", f"{MAX_SECONDS / 60:.0f} minutes")
def fmt(t: float) -> str:
return f"{int(t // 60)}:{int(t % 60):02d}"
def write_wav(path: Path, audio: np.ndarray) -> None:
pcm = (np.clip(audio, -1, 1) * 32767).astype("<i2")
with wave.open(str(path), "wb") as f:
f.setnchannels(1)
f.setsampwidth(2)
f.setframerate(SR)
f.writeframes(pcm.tobytes())
def result_html(result, audio_url: str, duration: float, stats: list[str]) -> str:
"""Player, word timeline and transcript; every word carries its times and lights up while spoken."""
speakers = sorted({s.speaker for s in result.segments if s.speaker})
colour = {s: SPEAKER_COLOURS[i % len(SPEAKER_COLOURS)] for i, s in enumerate(speakers)}
rows, boxes = [], []
for seg in result.segments:
words = seg.words or []
spans = " ".join(
f'<span class="w" data-s="{w.start:.3f}" data-e="{w.end:.3f}" '
f'title="{w.start:.2f} – {w.end:.2f} s">{html.escape(w.word.strip())}</span>'
for w in words
) or html.escape(seg.text)
label = (
f'<b style="color:{colour[seg.speaker]}">{html.escape(seg.speaker)}</b> ' if seg.speaker else ""
)
rows.append(f'<div class="seg"><span class="t">{fmt(seg.start)}</span><span>{label}{spans}</span></div>')
for w in words:
left, width = w.start * PX_PER_SECOND, max((w.end - w.start) * PX_PER_SECOND, 6)
boxes.append(
f'<span class="tw" data-s="{w.start:.3f}" data-e="{w.end:.3f}" '
f'style="left:{left:.1f}px;width:{width:.1f}px" title="{w.start:.2f} – {w.end:.2f} s">'
f"{html.escape(w.word.strip())}</span>"
)
chips = "".join(f"<span>{html.escape(s)}</span>" for s in stats)
width = max(duration * PX_PER_SECOND, 1)
return (
f'<div class="rx" data-px="{PX_PER_SECOND}" data-dur="{duration:.3f}">'
f'<div class="rx-stats">{chips}</div>'
f'<audio preload="auto" src="{audio_url}"></audio>'
f'<div class="rx-bar"><button class="rx-play" type="button">▶</button>'
f'<span class="rx-time">0:00 / {fmt(duration)}</span>'
f'<div class="rx-prog"><div class="rx-fill"></div></div></div>'
f'<div class="rx-tl"><div class="rx-tl-inner" style="width:{width:.0f}px">{"".join(boxes)}'
f'<div class="rx-head"></div></div></div>'
f'<div class="rx-text">{"".join(rows)}</div></div>'
)
@gpu
def run(audio_path, language, line_width, progress=gr.Progress()):
if not audio_path:
raise gr.Error("Upload or record some audio first.")
wav = load_audio(audio_path)
note = ""
if len(wav) > MAX_SECONDS * SR:
wav = wav[: int(MAX_SECONDS * SR)]
where = "a shared GPU" if ON_GPU else "a shared CPU"
note = f"Only the first {MAX_SECONDS / 60:.0f} min were transcribed (this demo runs on {where})."
kwargs = {} if language == "Auto-detect" else {"language": language}
progress(0, desc="Transcribing")
t0 = time.perf_counter()
result = rumblex.transcribe(
np.asarray(wav, dtype=np.float32),
MODEL,
progress=lambda done, total: progress(done / max(total, 1), desc="Transcribing"),
**kwargs,
)
took = time.perf_counter() - t0
audio_s = len(wav) / SR
# On short clips the fixed start-up cost dominates, so a real-time factor would mislead.
speed = f"done in {took:.1f} s"
if audio_s >= 30:
speed += f" ({audio_s / max(took, 1e-6):.0f}× real time)"
return show(result, wav, Path(audio_path).stem or "audio", line_width, speed, note)
def show(result, wav, stem: str, line_width, speed: str, note: str = ""):
"""Write the subtitle files and build the player, word table and status for a transcript."""
out_dir = Path(tempfile.mkdtemp(dir=OUT_ROOT))
opts = WriterOptions(max_line_width=int(line_width) if line_width else None)
files = rumblex.write_result(result, f"{stem}.wav", out_dir, ["srt", "vtt", "json"], opts)
played = out_dir / "audio.wav"
write_wav(played, wav)
words = [
[w.word.strip(), round(w.start, 2), round(w.end, 2)] for s in result.segments for w in (s.words or [])
]
audio_s = len(wav) / SR
stats = [f"{audio_s:.1f} s of audio", speed, f"language: {result.language}", f"{len(words)} words"]
status = note or "Press ▶: each word lights up as it is spoken. Click any word to jump to it."
view = result_html(result, f"/gradio_api/file={quote(str(played))}", audio_s, stats)
return status, view, words, [str(p) for p in files]
# The example's transcript ships with the Space (made with make_example.py on CPU), so trying the example
# needs no GPU: it works for every visitor and uses none of their ZeroGPU quota.
EXAMPLE_RESULT = Path(__file__).with_name("example_result.json")
def run_example(line_width):
result = rumblex.TranscriptResult.from_dict(json.loads(EXAMPLE_RESULT.read_text(encoding="utf-8")))
wav = load_audio(EXAMPLE)
return show(result, wav, Path(EXAMPLE).stem, line_width, "precomputed, no GPU needed")
THEME = gr.themes.Soft(
primary_hue="indigo", font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"]
)
with gr.Blocks(title="rumblex: word timestamps for Rumble-2-Flash") as demo:
gr.HTML(HERO)
with gr.Row(equal_height=False):
with gr.Column(scale=2, min_width=300):
audio = gr.Audio(sources=["upload", "microphone"], type="filepath", label="Audio")
language = gr.Dropdown(LANGUAGES, value="Auto-detect", label="Language", allow_custom_value=True)
example = gr.Button("▶ No audio handy? Try an example clip", variant="primary", size="lg")
go = gr.Button("Transcribe again (e.g. after changing the language)", variant="secondary", size="sm")
gr.Markdown("<small>The example is a 6 s LibriSpeech recording (CC BY 4.0).</small>")
with gr.Column(scale=3, min_width=360):
status = gr.Markdown()
transcript = gr.HTML(EMPTY)
with gr.Accordion("Download subtitles and JSON", open=False):
width = gr.Slider(
0, 80, value=42, step=1, label="Subtitle line width (0 = one cue per segment)"
)
files = gr.File(label="SRT, VTT and JSON", file_count="multiple")
table = gr.Dataframe(headers=["word", "start (s)", "end (s)"], label="Word timestamps")
gr.Markdown(FOOTER)
outputs = [status, transcript, table, files]
go.click(run, [audio, language, width], outputs)
# Uploads and microphone recordings are transcribed straight away, without pressing a button.
audio.upload(run, [audio, language, width], outputs)
audio.stop_recording(run, [audio, language, width], outputs)
# One click: load the example clip into the audio box and transcribe it.
example.click(lambda: (EXAMPLE, "Auto-detect"), None, [audio, language]).then(
run_example, [width], outputs
)
if __name__ == "__main__":
demo.queue(default_concurrency_limit=1).launch(head=HEAD, allowed_paths=[str(OUT_ROOT)], theme=THEME)
|