edwardgl commited on
Commit
4c7fa41
·
verified ·
1 Parent(s): 4d0ec94

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +223 -130
app.py CHANGED
@@ -1,130 +1,223 @@
1
- """rumblex demo: transcription with word timestamps and subtitles from Reson8/Rumble-2-Flash."""
2
-
3
- from __future__ import annotations
4
-
5
- import html
6
- import os
7
- import tempfile
8
- from pathlib import Path
9
-
10
- import gradio as gr
11
- import numpy as np
12
- import torch
13
-
14
- import rumblex
15
- from rumblex.audio import load_audio
16
- from rumblex.writers import WriterOptions
17
-
18
- try: # Hugging Face's ZeroGPU helper; a no-op outside ZeroGPU hardware
19
- import spaces
20
-
21
- gpu = spaces.GPU(duration=60)
22
- except ImportError:
23
-
24
- def gpu(fn):
25
- return fn
26
-
27
-
28
- ZERO_GPU = os.environ.get("SPACES_ZERO_GPU", "").lower() in ("1", "true")
29
- ON_GPU = ZERO_GPU or torch.cuda.is_available()
30
- MAX_SECONDS = float(os.environ.get("MAX_SECONDS", 600 if ON_GPU else 120))
31
- SR = 16000
32
- LANGUAGES = ["Auto-detect", "English", "Dutch", "German", "French", "Spanish", "Italian", "Portuguese"]
33
- SPEAKER_COLOURS = ["#2563eb", "#d97706", "#059669", "#9333ea"]
34
-
35
- # On ZeroGPU the model is placed on cuda at import time (the GPU is attached inside @gpu calls). The dtype is
36
- # given explicitly so nothing queries the device before a GPU is attached.
37
- DEVICE = "cuda" if ON_GPU else "cpu"
38
- MODEL = rumblex.load_model(
39
- device=DEVICE, compute_type="bfloat16" if ON_GPU else "float32", batch_size=16 if ON_GPU else 4
40
- )
41
-
42
-
43
- def fmt(t: float) -> str:
44
- return f"{int(t // 60):02d}:{t % 60:05.2f}"
45
-
46
-
47
- def transcript_html(result) -> str:
48
- """One line per segment; every word carries its start and end time (hover to see them)."""
49
- speakers = sorted({s.speaker for s in result.segments if s.speaker})
50
- colour = {s: SPEAKER_COLOURS[i % len(SPEAKER_COLOURS)] for i, s in enumerate(speakers)}
51
- rows = []
52
- for seg in result.segments:
53
- words = seg.words or []
54
- spans = " ".join(
55
- f'<span class="w" title="{w.start:.2f} – {w.end:.2f} s">{html.escape(w.word.strip())}</span>'
56
- for w in words
57
- ) or html.escape(seg.text)
58
- label = (
59
- f'<b style="color:{colour[seg.speaker]}">{html.escape(seg.speaker)}</b> ' if seg.speaker else ""
60
- )
61
- rows.append(f'<div class="seg"><span class="t">{fmt(seg.start)} → {fmt(seg.end)}</span> {label}{spans}</div>')
62
- return (
63
- "<style>.seg{margin:.35em 0;line-height:1.7}.t{font-family:monospace;opacity:.6;margin-right:.6em}"
64
- ".w{padding:1px 2px;border-radius:3px;cursor:help}.w:hover{background:rgba(37,99,235,.25)}</style>"
65
- + "".join(rows)
66
- )
67
-
68
-
69
- @gpu
70
- def run(audio_path, language, line_width, progress=gr.Progress()):
71
- if not audio_path:
72
- raise gr.Error("Upload or record some audio first.")
73
- wav = load_audio(audio_path)
74
- note = ""
75
- if len(wav) > MAX_SECONDS * SR:
76
- wav = wav[: int(MAX_SECONDS * SR)]
77
- where = "a shared GPU" if ON_GPU else "a shared CPU"
78
- note = f"Only the first {MAX_SECONDS / 60:.0f} min were transcribed (this demo runs on {where}). "
79
- kwargs = {} if language == "Auto-detect" else {"language": language}
80
- progress(0, desc="Transcribing")
81
- result = rumblex.transcribe(
82
- np.asarray(wav, dtype=np.float32),
83
- MODEL,
84
- progress=lambda done, total: progress(done / max(total, 1), desc="Transcribing"),
85
- **kwargs,
86
- )
87
- out_dir = Path(tempfile.mkdtemp())
88
- stem = Path(audio_path).stem or "audio"
89
- opts = WriterOptions(max_line_width=int(line_width) if line_width else None)
90
- files = rumblex.write_result(result, f"{stem}.wav", out_dir, ["srt", "vtt", "json"], opts)
91
- words = [
92
- [w.word.strip(), round(w.start, 2), round(w.end, 2)] for s in result.segments for w in (s.words or [])
93
- ]
94
- audio_s = len(wav) / SR
95
- status = f"{note}{audio_s:.1f} s of audio, language: {result.language}, {len(words)} words."
96
- return status, transcript_html(result), words, [str(p) for p in files]
97
-
98
-
99
- DESCRIPTION = """
100
- Transcription with **word-level timestamps** and subtitles, using
101
- [Reson8/Rumble-2-Flash](https://huggingface.co/Reson8/Rumble-2-Flash) and its own CTC head for alignment.
102
- This is the [rumblex](https://github.com/EdwardGlockner/rumblex) pipeline (`pip install rumblex`).
103
- Hover over a word to see its start and end time.
104
-
105
- This free demo transcribes the first {limit} of each file. Speaker labels (pyannote) are available in the
106
- package, not in this demo. rumblex is an independent project and is not affiliated with Reson8.
107
- """.replace("{limit}", f"{MAX_SECONDS / 60:.0f} minutes")
108
-
109
- with gr.Blocks(title="rumblex: word timestamps for Rumble-2-Flash") as demo:
110
- gr.Markdown("# rumblex\n" + DESCRIPTION)
111
- with gr.Row():
112
- with gr.Column(scale=1):
113
- audio = gr.Audio(sources=["upload", "microphone"], type="filepath", label="Audio")
114
- language = gr.Dropdown(LANGUAGES, value="Auto-detect", label="Language", allow_custom_value=True)
115
- width = gr.Slider(0, 80, value=42, step=1, label="Subtitle line width (0 = one cue per segment)")
116
- go = gr.Button("Transcribe", variant="primary")
117
- gr.Examples(
118
- [["librispeech_mr_quilter.wav", "Auto-detect", 42]],
119
- inputs=[audio, language, width],
120
- label="Example (LibriSpeech, CC BY 4.0)",
121
- )
122
- with gr.Column(scale=2):
123
- status = gr.Markdown()
124
- transcript = gr.HTML(label="Transcript")
125
- files = gr.File(label="Subtitles and JSON", file_count="multiple")
126
- table = gr.Dataframe(headers=["word", "start (s)", "end (s)"], label="Word timestamps")
127
- go.click(run, [audio, language, width], [status, transcript, table, files])
128
-
129
- if __name__ == "__main__":
130
- demo.queue(default_concurrency_limit=1).launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """rumblex demo: transcription with word timestamps and subtitles from Reson8/Rumble-2-Flash."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import html
6
+ import os
7
+ import tempfile
8
+ import time
9
+ import wave
10
+ from pathlib import Path
11
+ from urllib.parse import quote
12
+
13
+ import gradio as gr
14
+ import numpy as np
15
+ import torch
16
+
17
+ import rumblex
18
+ from rumblex.audio import load_audio
19
+ from rumblex.writers import WriterOptions
20
+
21
+ try: # Hugging Face's ZeroGPU helper; a no-op outside ZeroGPU hardware
22
+ import spaces
23
+
24
+ gpu = spaces.GPU(duration=60)
25
+ except ImportError:
26
+
27
+ def gpu(fn):
28
+ return fn
29
+
30
+
31
+ ZERO_GPU = os.environ.get("SPACES_ZERO_GPU", "").lower() in ("1", "true")
32
+ ON_GPU = ZERO_GPU or torch.cuda.is_available()
33
+ MAX_SECONDS = float(os.environ.get("MAX_SECONDS", "600" if ON_GPU else "120"))
34
+ SR = 16000
35
+ LANGUAGES = ["Auto-detect", "English", "Dutch", "German", "French", "Spanish", "Italian", "Portuguese"]
36
+ SPEAKER_COLOURS = ["#2563eb", "#d97706", "#059669", "#9333ea"]
37
+ OUT_ROOT = Path(tempfile.gettempdir()) / "rumblex_demo" # served to the browser (allowed_paths below)
38
+ OUT_ROOT.mkdir(exist_ok=True)
39
+
40
+ # On ZeroGPU the model is placed on cuda at import time (the GPU is attached inside @gpu calls). The dtype is
41
+ # given explicitly so nothing queries the device before a GPU is attached.
42
+ DEVICE = "cuda" if ON_GPU else "cpu"
43
+ MODEL = rumblex.load_model(
44
+ device=DEVICE, compute_type="bfloat16" if ON_GPU else "float32", batch_size=16 if ON_GPU else 4
45
+ )
46
+
47
+ # Karaoke playback: while the audio plays, the word under the playhead is highlighted; clicking a word seeks
48
+ # to it. Event delegation on the document, so it works for every transcript Gradio renders.
49
+ HEAD = """
50
+ <script>
51
+ (() => {
52
+ const target = (e) => (e.composedPath ? e.composedPath()[0] : e.target);
53
+ function paint(audio) {
54
+ const box = audio.closest(".rx");
55
+ if (!box) return;
56
+ const t = audio.currentTime;
57
+ let current = null;
58
+ for (const w of box.querySelectorAll(".w")) {
59
+ const s = +w.dataset.s, e = +w.dataset.e;
60
+ const on = t >= s && t < e;
61
+ w.classList.toggle("on", on);
62
+ w.classList.toggle("done", t >= e);
63
+ if (on) current = w;
64
+ }
65
+ const text = box.querySelector(".rx-text");
66
+ if (current && text) {
67
+ const y = current.offsetTop - text.offsetTop;
68
+ if (y < text.scrollTop || y > text.scrollTop + text.clientHeight - 30) text.scrollTop = y - 30;
69
+ }
70
+ }
71
+ function loop(audio) {
72
+ paint(audio);
73
+ if (!audio.paused && !audio.ended) requestAnimationFrame(() => loop(audio));
74
+ }
75
+ for (const ev of ["play", "seeked", "timeupdate"]) {
76
+ document.addEventListener(ev, (e) => {
77
+ const a = target(e);
78
+ if (a && a.matches && a.matches("audio.rx-audio")) (ev === "play" ? loop(a) : paint(a));
79
+ }, true);
80
+ }
81
+ document.addEventListener("click", (e) => {
82
+ const w = target(e).closest && target(e).closest(".rx .w");
83
+ if (!w) return;
84
+ const a = w.closest(".rx").querySelector("audio.rx-audio");
85
+ a.currentTime = +w.dataset.s + 0.001;
86
+ a.play();
87
+ }, true);
88
+ })();
89
+ </script>
90
+ <style>
91
+ .rx-audio { width: 100%; margin: .2em 0 .6em; }
92
+ .rx-text { max-height: 420px; overflow-y: auto; padding-right: .4em; }
93
+ .rx .seg { margin: .3em 0; line-height: 1.9; font-size: 1.05em; }
94
+ .rx .t { font-family: monospace; font-size: .8em; opacity: .55; margin-right: .6em; }
95
+ .rx .w { padding: 2px 3px; border-radius: 4px; cursor: pointer; transition: background .08s; }
96
+ .rx .w:hover { outline: 1px solid rgba(99,102,241,.6); }
97
+ .rx .w.done { opacity: .6; }
98
+ .rx .w.on { background: #6366f1; color: #fff; opacity: 1; }
99
+ .rx-stats { display: flex; gap: .6em; flex-wrap: wrap; margin: .2em 0 .4em; }
100
+ .rx-stats span { background: rgba(99,102,241,.12); border-radius: 999px; padding: .15em .7em; font-size: .9em; }
101
+ </style>
102
+ """
103
+
104
+
105
+ def fmt(t: float) -> str:
106
+ return f"{int(t // 60):02d}:{t % 60:05.2f}"
107
+
108
+
109
+ def write_wav(path: Path, audio: np.ndarray) -> None:
110
+ pcm = (np.clip(audio, -1, 1) * 32767).astype("<i2")
111
+ with wave.open(str(path), "wb") as f:
112
+ f.setnchannels(1)
113
+ f.setsampwidth(2)
114
+ f.setframerate(SR)
115
+ f.writeframes(pcm.tobytes())
116
+
117
+
118
+ def transcript_html(result, audio_url: str, stats: list[str]) -> str:
119
+ """Audio player plus the transcript; every word carries its times and lights up while it is spoken."""
120
+ speakers = sorted({s.speaker for s in result.segments if s.speaker})
121
+ colour = {s: SPEAKER_COLOURS[i % len(SPEAKER_COLOURS)] for i, s in enumerate(speakers)}
122
+ rows = []
123
+ for seg in result.segments:
124
+ words = seg.words or []
125
+ spans = " ".join(
126
+ f'<span class="w" data-s="{w.start:.3f}" data-e="{w.end:.3f}" '
127
+ f'title="{w.start:.2f} – {w.end:.2f} s">{html.escape(w.word.strip())}</span>'
128
+ for w in words
129
+ ) or html.escape(seg.text)
130
+ label = (
131
+ f'<b style="color:{colour[seg.speaker]}">{html.escape(seg.speaker)}</b> ' if seg.speaker else ""
132
+ )
133
+ rows.append(f'<div class="seg"><span class="t">{fmt(seg.start)}</span>{label}{spans}</div>')
134
+ chips = "".join(f"<span>{html.escape(s)}</span>" for s in stats)
135
+ return (
136
+ f'<div class="rx"><div class="rx-stats">{chips}</div>'
137
+ f'<audio class="rx-audio" controls preload="auto" src="{audio_url}"></audio>'
138
+ f'<div class="rx-text">{"".join(rows)}</div></div>'
139
+ )
140
+
141
+
142
+ @gpu
143
+ def run(audio_path, language, line_width, progress=gr.Progress()):
144
+ if not audio_path:
145
+ raise gr.Error("Upload or record some audio first.")
146
+ wav = load_audio(audio_path)
147
+ note = ""
148
+ if len(wav) > MAX_SECONDS * SR:
149
+ wav = wav[: int(MAX_SECONDS * SR)]
150
+ where = "a shared GPU" if ON_GPU else "a shared CPU"
151
+ note = f"Only the first {MAX_SECONDS / 60:.0f} min were transcribed (this demo runs on {where})."
152
+ kwargs = {} if language == "Auto-detect" else {"language": language}
153
+ progress(0, desc="Transcribing")
154
+ t0 = time.perf_counter()
155
+ result = rumblex.transcribe(
156
+ np.asarray(wav, dtype=np.float32),
157
+ MODEL,
158
+ progress=lambda done, total: progress(done / max(total, 1), desc="Transcribing"),
159
+ **kwargs,
160
+ )
161
+ took = time.perf_counter() - t0
162
+ out_dir = Path(tempfile.mkdtemp(dir=OUT_ROOT))
163
+ stem = Path(audio_path).stem or "audio"
164
+ opts = WriterOptions(max_line_width=int(line_width) if line_width else None)
165
+ files = rumblex.write_result(result, f"{stem}.wav", out_dir, ["srt", "vtt", "json"], opts)
166
+ played = out_dir / "audio.wav"
167
+ write_wav(played, wav)
168
+ words = [
169
+ [w.word.strip(), round(w.start, 2), round(w.end, 2)] for s in result.segments for w in (s.words or [])
170
+ ]
171
+ audio_s = len(wav) / SR
172
+ stats = [
173
+ f"{audio_s:.1f} s of audio",
174
+ f"transcribed and aligned in {took:.1f} s ({audio_s / max(took, 1e-6):.0f}× real time)",
175
+ f"language: {result.language}",
176
+ f"{len(words)} words",
177
+ ]
178
+ status = note or "Press play: each word lights up as it is spoken. Click a word to jump to it."
179
+ view = transcript_html(result, f"/gradio_api/file={quote(str(played))}", stats)
180
+ return status, view, words, [str(p) for p in files]
181
+
182
+
183
+ DESCRIPTION = """
184
+ Transcription with **word-level timestamps**, from
185
+ [Reson8/Rumble-2-Flash](https://huggingface.co/Reson8/Rumble-2-Flash) and the model's own CTC head, using
186
+ [rumblex](https://github.com/EdwardGlockner/rumblex) (`pip install rumblex`). Upload a file or record from your
187
+ microphone (it transcribes as soon as you stop), then press play and watch the words light up as they are
188
+ spoken.
189
+
190
+ The demo transcribes the first {limit} of each file. Speaker labels (pyannote) are in the package, not in
191
+ this demo. rumblex is an independent project and is not affiliated with Reson8.
192
+ """.replace("{limit}", f"{MAX_SECONDS / 60:.0f} minutes")
193
+
194
+ with gr.Blocks(title="rumblex: word timestamps for Rumble-2-Flash") as demo:
195
+ gr.Markdown("# rumblex\n" + DESCRIPTION)
196
+ with gr.Row():
197
+ with gr.Column(scale=1):
198
+ audio = gr.Audio(sources=["upload", "microphone"], type="filepath", label="Audio")
199
+ language = gr.Dropdown(LANGUAGES, value="Auto-detect", label="Language", allow_custom_value=True)
200
+ go = gr.Button("Transcribe", variant="primary")
201
+ gr.Examples(
202
+ [["librispeech_mr_quilter.wav", "Auto-detect"]],
203
+ inputs=[audio, language],
204
+ label="Example (LibriSpeech, CC BY 4.0)",
205
+ )
206
+ with gr.Column(scale=2):
207
+ status = gr.Markdown()
208
+ transcript = gr.HTML()
209
+ with gr.Accordion("Subtitles, JSON and the word table", open=False):
210
+ width = gr.Slider(
211
+ 0, 80, value=42, step=1, label="Subtitle line width (0 = one cue per segment)"
212
+ )
213
+ files = gr.File(label="Subtitles and JSON", file_count="multiple")
214
+ table = gr.Dataframe(headers=["word", "start (s)", "end (s)"], label="Word timestamps")
215
+ outputs = [status, transcript, table, files]
216
+ go.click(run, [audio, language, width], outputs)
217
+ # A microphone recording is transcribed as soon as the recording stops, without pressing the button.
218
+ audio.stop_recording(run, [audio, language, width], outputs)
219
+
220
+ if __name__ == "__main__":
221
+ demo.queue(default_concurrency_limit=1).launch(
222
+ head=HEAD, allowed_paths=[str(OUT_ROOT)], theme=gr.themes.Soft(primary_hue="indigo")
223
+ )