edwardgl commited on
Commit
90e4342
·
verified ·
1 Parent(s): 4c7fa41

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +145 -73
app.py CHANGED
@@ -32,8 +32,9 @@ ZERO_GPU = os.environ.get("SPACES_ZERO_GPU", "").lower() in ("1", "true")
32
  ON_GPU = ZERO_GPU or torch.cuda.is_available()
33
  MAX_SECONDS = float(os.environ.get("MAX_SECONDS", "600" if ON_GPU else "120"))
34
  SR = 16000
 
35
  LANGUAGES = ["Auto-detect", "English", "Dutch", "German", "French", "Spanish", "Italian", "Portuguese"]
36
- SPEAKER_COLOURS = ["#2563eb", "#d97706", "#059669", "#9333ea"]
37
  OUT_ROOT = Path(tempfile.gettempdir()) / "rumblex_demo" # served to the browser (allowed_paths below)
38
  OUT_ROOT.mkdir(exist_ok=True)
39
 
@@ -44,66 +45,131 @@ MODEL = rumblex.load_model(
44
  device=DEVICE, compute_type="bfloat16" if ON_GPU else "float32", batch_size=16 if ON_GPU else 4
45
  )
46
 
47
- # Karaoke playback: while the audio plays, the word under the playhead is highlighted; clicking a word seeks
48
- # to it. Event delegation on the document, so it works for every transcript Gradio renders.
49
  HEAD = """
50
  <script>
51
  (() => {
52
- const target = (e) => (e.composedPath ? e.composedPath()[0] : e.target);
53
- function paint(audio) {
54
- const box = audio.closest(".rx");
55
- if (!box) return;
56
- const t = audio.currentTime;
 
 
 
 
 
 
57
  let current = null;
58
- for (const w of box.querySelectorAll(".w")) {
59
- const s = +w.dataset.s, e = +w.dataset.e;
60
- const on = t >= s && t < e;
61
  w.classList.toggle("on", on);
62
- w.classList.toggle("done", t >= e);
63
- if (on) current = w;
64
  }
65
  const text = box.querySelector(".rx-text");
66
- if (current && text) {
67
  const y = current.offsetTop - text.offsetTop;
68
- if (y < text.scrollTop || y > text.scrollTop + text.clientHeight - 30) text.scrollTop = y - 30;
69
  }
70
  }
71
- function loop(audio) {
72
- paint(audio);
73
- if (!audio.paused && !audio.ended) requestAnimationFrame(() => loop(audio));
74
  }
75
- for (const ev of ["play", "seeked", "timeupdate"]) {
76
  document.addEventListener(ev, (e) => {
77
- const a = target(e);
78
- if (a && a.matches && a.matches("audio.rx-audio")) (ev === "play" ? loop(a) : paint(a));
 
79
  }, true);
80
  }
81
  document.addEventListener("click", (e) => {
82
- const w = target(e).closest && target(e).closest(".rx .w");
83
- if (!w) return;
84
- const a = w.closest(".rx").querySelector("audio.rx-audio");
85
- a.currentTime = +w.dataset.s + 0.001;
86
- a.play();
 
 
 
 
 
 
 
 
 
87
  }, true);
88
  })();
89
  </script>
90
  <style>
91
- .rx-audio { width: 100%; margin: .2em 0 .6em; }
92
- .rx-text { max-height: 420px; overflow-y: auto; padding-right: .4em; }
93
- .rx .seg { margin: .3em 0; line-height: 1.9; font-size: 1.05em; }
94
- .rx .t { font-family: monospace; font-size: .8em; opacity: .55; margin-right: .6em; }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  .rx .w { padding: 2px 3px; border-radius: 4px; cursor: pointer; transition: background .08s; }
96
  .rx .w:hover { outline: 1px solid rgba(99,102,241,.6); }
97
  .rx .w.done { opacity: .6; }
98
  .rx .w.on { background: #6366f1; color: #fff; opacity: 1; }
99
- .rx-stats { display: flex; gap: .6em; flex-wrap: wrap; margin: .2em 0 .4em; }
100
- .rx-stats span { background: rgba(99,102,241,.12); border-radius: 999px; padding: .15em .7em; font-size: .9em; }
101
  </style>
102
  """
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
 
105
  def fmt(t: float) -> str:
106
- return f"{int(t // 60):02d}:{t % 60:05.2f}"
107
 
108
 
109
  def write_wav(path: Path, audio: np.ndarray) -> None:
@@ -115,11 +181,11 @@ def write_wav(path: Path, audio: np.ndarray) -> None:
115
  f.writeframes(pcm.tobytes())
116
 
117
 
118
- def transcript_html(result, audio_url: str, stats: list[str]) -> str:
119
- """Audio player plus the transcript; every word carries its times and lights up while it is spoken."""
120
  speakers = sorted({s.speaker for s in result.segments if s.speaker})
121
  colour = {s: SPEAKER_COLOURS[i % len(SPEAKER_COLOURS)] for i, s in enumerate(speakers)}
122
- rows = []
123
  for seg in result.segments:
124
  words = seg.words or []
125
  spans = " ".join(
@@ -130,11 +196,25 @@ def transcript_html(result, audio_url: str, stats: list[str]) -> str:
130
  label = (
131
  f'<b style="color:{colour[seg.speaker]}">{html.escape(seg.speaker)}</b> ' if seg.speaker else ""
132
  )
133
- rows.append(f'<div class="seg"><span class="t">{fmt(seg.start)}</span>{label}{spans}</div>')
 
 
 
 
 
 
 
134
  chips = "".join(f"<span>{html.escape(s)}</span>" for s in stats)
 
135
  return (
136
- f'<div class="rx"><div class="rx-stats">{chips}</div>'
137
- f'<audio class="rx-audio" controls preload="auto" src="{audio_url}"></audio>'
 
 
 
 
 
 
138
  f'<div class="rx-text">{"".join(rows)}</div></div>'
139
  )
140
 
@@ -169,55 +249,47 @@ def run(audio_path, language, line_width, progress=gr.Progress()):
169
  [w.word.strip(), round(w.start, 2), round(w.end, 2)] for s in result.segments for w in (s.words or [])
170
  ]
171
  audio_s = len(wav) / SR
172
- stats = [
173
- f"{audio_s:.1f} s of audio",
174
- f"transcribed and aligned in {took:.1f} s ({audio_s / max(took, 1e-6):.0f}× real time)",
175
- f"language: {result.language}",
176
- f"{len(words)} words",
177
- ]
178
- status = note or "Press play: each word lights up as it is spoken. Click a word to jump to it."
179
- view = transcript_html(result, f"/gradio_api/file={quote(str(played))}", stats)
180
  return status, view, words, [str(p) for p in files]
181
 
182
 
183
- DESCRIPTION = """
184
- Transcription with **word-level timestamps**, from
185
- [Reson8/Rumble-2-Flash](https://huggingface.co/Reson8/Rumble-2-Flash) and the model's own CTC head, using
186
- [rumblex](https://github.com/EdwardGlockner/rumblex) (`pip install rumblex`). Upload a file or record from your
187
- microphone (it transcribes as soon as you stop), then press play and watch the words light up as they are
188
- spoken.
189
-
190
- The demo transcribes the first {limit} of each file. Speaker labels (pyannote) are in the package, not in
191
- this demo. rumblex is an independent project and is not affiliated with Reson8.
192
- """.replace("{limit}", f"{MAX_SECONDS / 60:.0f} minutes")
193
 
194
  with gr.Blocks(title="rumblex: word timestamps for Rumble-2-Flash") as demo:
195
- gr.Markdown("# rumblex\n" + DESCRIPTION)
196
- with gr.Row():
197
- with gr.Column(scale=1):
198
  audio = gr.Audio(sources=["upload", "microphone"], type="filepath", label="Audio")
199
  language = gr.Dropdown(LANGUAGES, value="Auto-detect", label="Language", allow_custom_value=True)
200
- go = gr.Button("Transcribe", variant="primary")
201
- gr.Examples(
202
- [["librispeech_mr_quilter.wav", "Auto-detect"]],
203
- inputs=[audio, language],
204
- label="Example (LibriSpeech, CC BY 4.0)",
205
- )
206
- with gr.Column(scale=2):
207
  status = gr.Markdown()
208
- transcript = gr.HTML()
209
- with gr.Accordion("Subtitles, JSON and the word table", open=False):
210
  width = gr.Slider(
211
  0, 80, value=42, step=1, label="Subtitle line width (0 = one cue per segment)"
212
  )
213
- files = gr.File(label="Subtitles and JSON", file_count="multiple")
214
  table = gr.Dataframe(headers=["word", "start (s)", "end (s)"], label="Word timestamps")
 
215
  outputs = [status, transcript, table, files]
216
  go.click(run, [audio, language, width], outputs)
217
  # A microphone recording is transcribed as soon as the recording stops, without pressing the button.
218
  audio.stop_recording(run, [audio, language, width], outputs)
 
 
 
 
219
 
220
  if __name__ == "__main__":
221
- demo.queue(default_concurrency_limit=1).launch(
222
- head=HEAD, allowed_paths=[str(OUT_ROOT)], theme=gr.themes.Soft(primary_hue="indigo")
223
- )
 
32
  ON_GPU = ZERO_GPU or torch.cuda.is_available()
33
  MAX_SECONDS = float(os.environ.get("MAX_SECONDS", "600" if ON_GPU else "120"))
34
  SR = 16000
35
+ PX_PER_SECOND = 110 # width of one second on the word timeline
36
  LANGUAGES = ["Auto-detect", "English", "Dutch", "German", "French", "Spanish", "Italian", "Portuguese"]
37
+ SPEAKER_COLOURS = ["#6366f1", "#d97706", "#059669", "#db2777"]
38
  OUT_ROOT = Path(tempfile.gettempdir()) / "rumblex_demo" # served to the browser (allowed_paths below)
39
  OUT_ROOT.mkdir(exist_ok=True)
40
 
 
45
  device=DEVICE, compute_type="bfloat16" if ON_GPU else "float32", batch_size=16 if ON_GPU else 4
46
  )
47
 
48
+ # Player, word timeline and karaoke transcript. Event delegation on the document, so it works for every
49
+ # result Gradio renders. A requestAnimationFrame loop repaints while playing (timeupdate is only ~4 Hz).
50
  HEAD = """
51
  <script>
52
  (() => {
53
+ const origin = (e) => (e.composedPath ? e.composedPath()[0] : e.target);
54
+ const clock = (t) => `${Math.floor(t / 60)}:${String(Math.floor(t % 60)).padStart(2, "0")}`;
55
+ function paint(box) {
56
+ const a = box.querySelector("audio");
57
+ const t = a.currentTime, d = a.duration || +box.dataset.dur;
58
+ box.querySelector(".rx-fill").style.width = `${Math.min(100, (100 * t) / d)}%`;
59
+ box.querySelector(".rx-time").textContent = `${clock(t)} / ${clock(d)}`;
60
+ box.querySelector(".rx-play").textContent = a.paused ? "▶" : "❚❚";
61
+ const px = +box.dataset.px, head = box.querySelector(".rx-head"), tl = box.querySelector(".rx-tl");
62
+ head.style.left = `${t * px}px`;
63
+ if (!a.paused) tl.scrollLeft = Math.max(0, t * px - tl.clientWidth / 3);
64
  let current = null;
65
+ for (const w of box.querySelectorAll("[data-s]")) {
66
+ const on = t >= +w.dataset.s && t < +w.dataset.e;
 
67
  w.classList.toggle("on", on);
68
+ w.classList.toggle("done", t >= +w.dataset.e);
69
+ if (on && w.classList.contains("w")) current = w;
70
  }
71
  const text = box.querySelector(".rx-text");
72
+ if (current && !a.paused) {
73
  const y = current.offsetTop - text.offsetTop;
74
+ if (y < text.scrollTop || y > text.scrollTop + text.clientHeight - 40) text.scrollTop = y - 40;
75
  }
76
  }
77
+ function loop(box) {
78
+ paint(box);
79
+ if (!box.querySelector("audio").paused) requestAnimationFrame(() => loop(box));
80
  }
81
+ for (const ev of ["play", "pause", "seeked", "loadedmetadata", "ended"]) {
82
  document.addEventListener(ev, (e) => {
83
+ const a = origin(e);
84
+ const box = a && a.closest && a.closest(".rx");
85
+ if (box) ev === "play" ? loop(box) : paint(box);
86
  }, true);
87
  }
88
  document.addEventListener("click", (e) => {
89
+ const el = origin(e);
90
+ if (!el.closest) return;
91
+ const box = el.closest(".rx");
92
+ if (!box) return;
93
+ const a = box.querySelector("audio");
94
+ if (el.closest(".rx-play")) { a.paused ? a.play() : a.pause(); return; }
95
+ const prog = el.closest(".rx-prog");
96
+ if (prog) {
97
+ const r = prog.getBoundingClientRect();
98
+ a.currentTime = ((e.clientX - r.left) / r.width) * (a.duration || +box.dataset.dur);
99
+ return;
100
+ }
101
+ const w = el.closest("[data-s]");
102
+ if (w) { a.currentTime = +w.dataset.s + 0.001; a.play(); }
103
  }, true);
104
  })();
105
  </script>
106
  <style>
107
+ .rx-hero h1 { font-size: 2.1em; font-weight: 700; margin: 0 0 .1em; letter-spacing: -.02em; }
108
+ .rx-hero p { margin: .2em 0 .7em; opacity: .8; font-size: 1.05em; max-width: 60em; }
109
+ .rx-links { display: flex; gap: .5em; flex-wrap: wrap; margin-bottom: .3em; }
110
+ .rx-links a { text-decoration: none !important; border: 1px solid var(--border-color-primary);
111
+ border-radius: 999px; padding: .25em .85em; font-size: .9em; color: var(--body-text-color) !important; }
112
+ .rx-links a:hover { border-color: #6366f1; }
113
+ .rx-empty { border: 1px dashed var(--border-color-primary); border-radius: 12px; padding: 3em 1.5em;
114
+ text-align: center; opacity: .65; }
115
+ .rx { border: 1px solid var(--border-color-primary); border-radius: 12px; padding: 1em 1.1em;
116
+ background: var(--block-background-fill); }
117
+ .rx-stats { display: flex; gap: .45em; flex-wrap: wrap; margin-bottom: .8em; }
118
+ .rx-stats span { background: rgba(99,102,241,.14); border-radius: 999px; padding: .2em .75em; font-size: .85em; }
119
+ .rx-bar { display: flex; align-items: center; gap: .8em; }
120
+ .rx-play { width: 2.5em; height: 2.5em; border-radius: 50%; border: none; background: #6366f1; color: #fff;
121
+ font-size: .95em; cursor: pointer; flex: none; }
122
+ .rx-play:hover { background: #4f46e5; }
123
+ .rx-time { font-family: ui-monospace, monospace; font-size: .85em; opacity: .7; flex: none; min-width: 6.5em; }
124
+ .rx-prog { flex: 1; height: 6px; border-radius: 3px; background: rgba(127,127,127,.25); cursor: pointer;
125
+ position: relative; }
126
+ .rx-fill { position: absolute; inset: 0 auto 0 0; width: 0; border-radius: 3px; background: #6366f1; }
127
+ .rx-tl { overflow-x: auto; margin: .9em 0 .6em; padding-bottom: 4px; }
128
+ .rx-tl-inner { position: relative; height: 34px; }
129
+ .rx-tl .tw { position: absolute; top: 4px; height: 26px; line-height: 26px; font-size: .78em; overflow: hidden;
130
+ white-space: nowrap; text-overflow: ellipsis; padding: 0 3px; border-radius: 4px; cursor: pointer;
131
+ background: rgba(99,102,241,.18); border-left: 2px solid rgba(99,102,241,.7); box-sizing: border-box; }
132
+ .rx-tl .tw.done { opacity: .55; }
133
+ .rx-tl .tw.on { background: #6366f1; color: #fff; opacity: 1; }
134
+ .rx-head { position: absolute; top: 0; bottom: 0; width: 2px; background: #f43f5e; left: 0; pointer-events: none; }
135
+ .rx-text { max-height: 380px; overflow-y: auto; }
136
+ .rx .seg { display: flex; gap: .9em; padding: .35em 0; line-height: 1.85; font-size: 1.08em; }
137
+ .rx .t { font-family: ui-monospace, monospace; font-size: .75em; opacity: .5; padding-top: .45em; flex: none;
138
+ width: 3.4em; }
139
  .rx .w { padding: 2px 3px; border-radius: 4px; cursor: pointer; transition: background .08s; }
140
  .rx .w:hover { outline: 1px solid rgba(99,102,241,.6); }
141
  .rx .w.done { opacity: .6; }
142
  .rx .w.on { background: #6366f1; color: #fff; opacity: 1; }
 
 
143
  </style>
144
  """
145
 
146
+ HERO = """
147
+ <div class="rx-hero">
148
+ <h1>rumblex</h1>
149
+ <p>Word-level timestamps from <b>Reson8/Rumble-2-Flash</b>, aligned with the model's own CTC head. Upload a
150
+ file or record yourself, then press play and watch each word light up as it is spoken.</p>
151
+ <div class="rx-links">
152
+ <a href="https://github.com/EdwardGlockner/rumblex" target="_blank">GitHub</a>
153
+ <a href="https://pypi.org/project/rumblex/" target="_blank">pip install rumblex</a>
154
+ <a href="https://huggingface.co/Reson8/Rumble-2-Flash" target="_blank">Rumble-2-Flash</a>
155
+ </div>
156
+ </div>
157
+ """
158
+
159
+ EXAMPLE = str(Path(__file__).with_name("librispeech_mr_quilter.wav"))
160
+ EMPTY = (
161
+ '<div class="rx-empty">Your transcript, word timeline and player will appear here.<br>'
162
+ "No audio at hand? Click <b>Try an example clip</b> on the left.</div>"
163
+ )
164
+
165
+ FOOTER = """
166
+ The demo transcribes the first {limit} of each file. Speaker labels (pyannote) are in the package, not in
167
+ this demo. rumblex is an independent project and is not affiliated with Reson8.
168
+ """.replace("{limit}", f"{MAX_SECONDS / 60:.0f} minutes")
169
+
170
 
171
  def fmt(t: float) -> str:
172
+ return f"{int(t // 60)}:{int(t % 60):02d}"
173
 
174
 
175
  def write_wav(path: Path, audio: np.ndarray) -> None:
 
181
  f.writeframes(pcm.tobytes())
182
 
183
 
184
+ def result_html(result, audio_url: str, duration: float, stats: list[str]) -> str:
185
+ """Player, word timeline and transcript; every word carries its times and lights up while spoken."""
186
  speakers = sorted({s.speaker for s in result.segments if s.speaker})
187
  colour = {s: SPEAKER_COLOURS[i % len(SPEAKER_COLOURS)] for i, s in enumerate(speakers)}
188
+ rows, boxes = [], []
189
  for seg in result.segments:
190
  words = seg.words or []
191
  spans = " ".join(
 
196
  label = (
197
  f'<b style="color:{colour[seg.speaker]}">{html.escape(seg.speaker)}</b> ' if seg.speaker else ""
198
  )
199
+ rows.append(f'<div class="seg"><span class="t">{fmt(seg.start)}</span><span>{label}{spans}</span></div>')
200
+ for w in words:
201
+ left, width = w.start * PX_PER_SECOND, max((w.end - w.start) * PX_PER_SECOND, 6)
202
+ boxes.append(
203
+ f'<span class="tw" data-s="{w.start:.3f}" data-e="{w.end:.3f}" '
204
+ f'style="left:{left:.1f}px;width:{width:.1f}px" title="{w.start:.2f} – {w.end:.2f} s">'
205
+ f"{html.escape(w.word.strip())}</span>"
206
+ )
207
  chips = "".join(f"<span>{html.escape(s)}</span>" for s in stats)
208
+ width = max(duration * PX_PER_SECOND, 1)
209
  return (
210
+ f'<div class="rx" data-px="{PX_PER_SECOND}" data-dur="{duration:.3f}">'
211
+ f'<div class="rx-stats">{chips}</div>'
212
+ f'<audio preload="auto" src="{audio_url}"></audio>'
213
+ f'<div class="rx-bar"><button class="rx-play" type="button">▶</button>'
214
+ f'<span class="rx-time">0:00 / {fmt(duration)}</span>'
215
+ f'<div class="rx-prog"><div class="rx-fill"></div></div></div>'
216
+ f'<div class="rx-tl"><div class="rx-tl-inner" style="width:{width:.0f}px">{"".join(boxes)}'
217
+ f'<div class="rx-head"></div></div></div>'
218
  f'<div class="rx-text">{"".join(rows)}</div></div>'
219
  )
220
 
 
249
  [w.word.strip(), round(w.start, 2), round(w.end, 2)] for s in result.segments for w in (s.words or [])
250
  ]
251
  audio_s = len(wav) / SR
252
+ # On short clips the fixed start-up cost dominates, so a real-time factor would mislead.
253
+ speed = f"done in {took:.1f} s"
254
+ if audio_s >= 30:
255
+ speed += f" ({audio_s / max(took, 1e-6):.0f}× real time)"
256
+ stats = [f"{audio_s:.1f} s of audio", speed, f"language: {result.language}", f"{len(words)} words"]
257
+ status = note or "Press ▶: each word lights up as it is spoken. Click any word to jump to it."
258
+ view = result_html(result, f"/gradio_api/file={quote(str(played))}", audio_s, stats)
 
259
  return status, view, words, [str(p) for p in files]
260
 
261
 
262
+ THEME = gr.themes.Soft(
263
+ primary_hue="indigo", font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"]
264
+ )
 
 
 
 
 
 
 
265
 
266
  with gr.Blocks(title="rumblex: word timestamps for Rumble-2-Flash") as demo:
267
+ gr.HTML(HERO)
268
+ with gr.Row(equal_height=False):
269
+ with gr.Column(scale=2, min_width=300):
270
  audio = gr.Audio(sources=["upload", "microphone"], type="filepath", label="Audio")
271
  language = gr.Dropdown(LANGUAGES, value="Auto-detect", label="Language", allow_custom_value=True)
272
+ go = gr.Button("Transcribe", variant="primary", size="lg")
273
+ example = gr.Button("▶ No audio handy? Try an example clip", variant="secondary")
274
+ gr.Markdown("<small>The example is a 6 s LibriSpeech recording (CC BY 4.0).</small>")
275
+ with gr.Column(scale=3, min_width=360):
 
 
 
276
  status = gr.Markdown()
277
+ transcript = gr.HTML(EMPTY)
278
+ with gr.Accordion("Download subtitles and JSON", open=False):
279
  width = gr.Slider(
280
  0, 80, value=42, step=1, label="Subtitle line width (0 = one cue per segment)"
281
  )
282
+ files = gr.File(label="SRT, VTT and JSON", file_count="multiple")
283
  table = gr.Dataframe(headers=["word", "start (s)", "end (s)"], label="Word timestamps")
284
+ gr.Markdown(FOOTER)
285
  outputs = [status, transcript, table, files]
286
  go.click(run, [audio, language, width], outputs)
287
  # A microphone recording is transcribed as soon as the recording stops, without pressing the button.
288
  audio.stop_recording(run, [audio, language, width], outputs)
289
+ # One click: load the example clip into the audio box and transcribe it.
290
+ example.click(lambda: (EXAMPLE, "Auto-detect"), None, [audio, language]).then(
291
+ run, [audio, language, width], outputs
292
+ )
293
 
294
  if __name__ == "__main__":
295
+ demo.queue(default_concurrency_limit=1).launch(head=HEAD, allowed_paths=[str(OUT_ROOT)], theme=THEME)