Spaces:
Running on Zero
Running on Zero
| """TTS Arena: blind side-by-side battles between small text-to-speech models. | |
| Two randomly chosen models read the same text, you vote, and their ratings move. Names stay | |
| hidden until the vote. Every text and vote is written to the arena's Hugging Face bucket. | |
| """ | |
| import random | |
| import uuid | |
| import gradio as gr | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| import elo | |
| import engines | |
| import registry | |
| import storage | |
| try: # ZeroGPU is only present on the Space | |
| import spaces | |
| except ImportError: # pragma: no cover - local dev | |
| class _Spaces: | |
| def GPU(*args, **kwargs): | |
| def wrap(fn): | |
| return fn | |
| return wrap if not (args and callable(args[0])) else args[0] | |
| spaces = _Spaces() | |
| ARENA = "tts" | |
| GPU_DURATION = 30 | |
| MAX_CHARS = 300 | |
| SENTENCES = [ | |
| "The museum opens at nine, but the best exhibits are upstairs, past the old clock.", | |
| "If the train is late again, we should probably walk the rest of the way.", | |
| "She folded the letter twice, slipped it into her coat, and never mentioned it again.", | |
| "Fresh bread, a little butter, and a quiet morning: that is all I really need.", | |
| "Please remember to water the plants on the balcony while I am away.", | |
| "The storm knocked out the power, so we told stories by candlelight until midnight.", | |
| "Honestly, I did not expect the sequel to be better than the original.", | |
| "Turn left at the bakery, then keep going until you see the blue door.", | |
| "Our team spent three weeks on the design, and it still needs another review.", | |
| "Why would anyone leave a perfectly good umbrella on the bus?", | |
| "The river was calm that evening, and the whole town came out to watch the lanterns.", | |
| "Could you read that number back to me, slowly this time?", | |
| ] | |
| def bootstrap(): | |
| """Rebuild ratings from the vote log (the authoritative history); never overwrite history after a failed read.""" | |
| store = elo.EloStore() | |
| try: | |
| applied = store.replay(storage.load_vote_log()) | |
| return store, True, f"Ratings rebuilt from {applied} logged votes." | |
| except Exception as exc: | |
| print(f"[arena] vote-log replay failed: {type(exc).__name__}: {exc}") | |
| try: | |
| cached = storage.load_state() | |
| if cached is not None: | |
| return elo.EloStore(cached), True, "Ratings loaded from state/elo.json." | |
| return store, True, "No history yet." | |
| except Exception as exc: | |
| return store, False, (f"Could not read history ({type(exc).__name__}). Ratings shown are for this session only " | |
| "and are not saved. Votes are still logged.") | |
| STORE, STATE_WRITABLE, LOAD_STATUS = bootstrap() | |
| print(f"[arena] {LOAD_STATUS}") | |
| engines.warm([s.repo_id for s in registry.MODELS]) | |
| import logging | |
| for _n in ("httpx", "httpcore", "urllib3", "matplotlib", "numba", "phonemizer"): | |
| logging.getLogger(_n).setLevel(logging.WARNING) | |
| VOTE_LABELS = {"a": "A is better", "b": "B is better", "tie": "Tie", "both_bad": "Both bad"} | |
| # --------------------------------------------------------------- generation | |
| def _run_gpu(repo_ids, text, seed): | |
| import torch | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| return [engines.synthesize(r, text, seed, device) for r in repo_ids] | |
| def _run_battle(repo_a, repo_b, text, seed): | |
| """Torch models share one GPU grant; ONNX / Phonon run on the CPU without waiting for a GPU.""" | |
| gpu = [r for r in (repo_a, repo_b) if engines.needs_gpu(r)] | |
| out = dict(zip(gpu, _run_gpu(gpu, text, seed))) if gpu else {} | |
| for r in (repo_a, repo_b): | |
| if r not in out: | |
| out[r] = engines.synthesize(r, text, seed, "cpu") | |
| return out[repo_a], out[repo_b] | |
| def battle(text, profile: gr.OAuthProfile | None = None): | |
| if profile is None: | |
| raise gr.Error("Sign in with Hugging Face to run battles.") | |
| text = " ".join((text or "").split()) | |
| if not text: | |
| raise gr.Error("Type something for the models to read first.") | |
| if len(text) > MAX_CHARS: | |
| raise gr.Error(f"Keep it under {MAX_CHARS} characters.") | |
| spec_a, spec_b = random.sample(registry.arena_models(ARENA), 2) | |
| seed = random.randint(0, 2**31 - 1) | |
| battle_id = uuid.uuid4().hex | |
| audio_a, audio_b = _run_battle(spec_a.repo_id, spec_b.repo_id, text, seed) | |
| storage.log_generation({ | |
| "battle_id": battle_id, "arena": ARENA, "created_at": storage.utc_now(), "hf_user": profile.username, "prompt": text, | |
| "settings": {"seed": seed}, | |
| "model_a": {"repo_id": spec_a.repo_id, "params": spec_a.params, "seconds": len(audio_a[1]) / audio_a[0]}, | |
| "model_b": {"repo_id": spec_b.repo_id, "params": spec_b.params, "seconds": len(audio_b[1]) / audio_b[0]}, | |
| }) | |
| state = {"battle_id": battle_id, "prompt": text, "repo_a": spec_a.repo_id, "repo_b": spec_b.repo_id, | |
| "settings": {"seed": seed}, "voted": False, "hf_user": profile.username} | |
| reveal = '<div class="note">Listen to both, then vote. Names are hidden until you do.</div>' | |
| return [audio_a, audio_b, state, reveal, *[gr.update(interactive=True) for _ in VOTE_LABELS]] | |
| # --------------------------------------------------------------- voting | |
| def _name_html(s): | |
| return f'<a href="{s.url}" target="_blank">{s.name} ↗</a>' if s.url else f"<span>{s.name}</span>" | |
| def _reveal_html(state, deltas): | |
| rows = [] | |
| for side, repo_id in (("A", state["repo_a"]), ("B", state["repo_b"])): | |
| s, d = registry.spec(repo_id), deltas[repo_id] | |
| diff = d["after"] - d["before"] | |
| sign = "up" if diff > 0 else ("down" if diff < 0 else "flat") | |
| rows.append(f'<div class="rev-row"><span class="side">{side}</span>{_name_html(s)}' | |
| f'<span class="muted">{s.params_m:.1f}M</span>' | |
| f'<span class="elo {sign}">{d["before"]:.1f} → {d["after"]:.1f}</span></div>') | |
| return '<div class="reveal"><div class="crumb">REVEALED</div>' + "".join(rows) + \ | |
| '<div class="muted small">Vote recorded. Type new text or draw a sentence for the next battle.</div></div>' | |
| def vote(state, choice, profile: gr.OAuthProfile | None = None): | |
| if profile is None: | |
| raise gr.Error("Sign in with Hugging Face to vote.") | |
| if not state or not state.get("battle_id"): | |
| raise gr.Error("Run a battle first.") | |
| if state.get("voted"): | |
| raise gr.Error("Already voted on this one. Start a new battle.") | |
| if state.get("hf_user") != profile.username: | |
| raise gr.Error("This battle was started by a different account. Start a new battle.") | |
| deltas = STORE.record_vote(ARENA, state["repo_a"], state["repo_b"], choice) | |
| if STATE_WRITABLE: | |
| storage.save_state(STORE.to_dict()) | |
| storage.log_vote({ | |
| "battle_id": state["battle_id"], "arena": ARENA, "created_at": storage.utc_now(), "hf_user": profile.username, "vote": choice, | |
| "prompt": state["prompt"], "settings": state["settings"], | |
| "model_a": {"repo_id": state["repo_a"], "elo": deltas[state["repo_a"]]}, | |
| "model_b": {"repo_id": state["repo_b"], "elo": deltas[state["repo_b"]]}, | |
| }) | |
| state = dict(state, voted=True) | |
| return [state, _reveal_html(state, deltas), *[gr.update(interactive=False) for _ in VOTE_LABELS], | |
| leaderboard_html(), leaderboard_plot()] | |
| # --------------------------------------------------------------- leaderboard | |
| def leaderboard_html(): | |
| rows = STORE.rows(ARENA) | |
| total = STORE.total_battles(ARENA) | |
| body = [] | |
| for i, r in enumerate(rows, start=1): | |
| s = registry.spec(r["repo_id"]) | |
| wr = f"{r['win_rate'] * 100:.0f}%" if r["battles"] else "—" | |
| body.append(f"<tr><td class='rank'>{i:02d}</td><td class='name'>{_name_html(s)}<div class='org'>{s.org}</div></td>" | |
| f"<td class='num strong'>{r['elo']:.1f}</td><td class='num'>{r['params_m']:.1f}</td>" | |
| f"<td class='num'>{r['elo_per_m']:.0f}</td><td class='num'>{r['battles']}</td>" | |
| f"<td class='num'>{r['wins']}</td><td class='num'>{r['losses']}</td><td class='num'>{r['ties']}</td>" | |
| f"<td class='num'>{r['both_bad']}</td><td class='num'>{wr}</td></tr>") | |
| status = "" if STATE_WRITABLE else f"<div class='note'>{LOAD_STATUS}</div>" | |
| return (f"<div class='lb-meta'>{len(rows)} models · {total} rated battles</div>{status}" | |
| "<div class='table-wrap'><table class='lb'><thead><tr><th>#</th><th>Model</th><th>Elo</th><th>Params (M)</th>" | |
| "<th>Elo / M</th><th>Battles</th><th>W</th><th>L</th><th>T</th><th>Both bad</th><th>Win rate</th></tr></thead>" | |
| f"<tbody>{''.join(body)}</tbody></table></div>") | |
| def leaderboard_plot(): | |
| rows = STORE.rows(ARENA) | |
| plt.rcParams["font.family"] = "DejaVu Sans Mono" | |
| fig, ax = plt.subplots(figsize=(9, 4.6), dpi=120) | |
| fig.patch.set_facecolor("#0a0a0a") | |
| ax.set_facecolor("#0a0a0a") | |
| for r in rows: | |
| ax.scatter(r["params_m"], r["elo"], s=40 + 5 * min(r["battles"], 40), c="#eeeeee", edgecolors="#0a0a0a", zorder=3) | |
| ax.annotate(r["model"], (r["params_m"], r["elo"]), textcoords="offset points", xytext=(7, 5), fontsize=7.5, color="#9a9a9a") | |
| ax.axhline(elo.ANCHOR, color="#333333", linestyle="--", linewidth=1, zorder=1) | |
| ax.set_xscale("log") | |
| ax.set_xlabel("Parameters (millions, log scale)", color="#9a9a9a") | |
| ax.set_ylabel("Elo", color="#9a9a9a") | |
| ax.tick_params(colors="#9a9a9a") | |
| for sp in ax.spines.values(): | |
| sp.set_color("#2a2a2a") | |
| ax.grid(True, which="both", color="#1c1c1c", zorder=0) | |
| fig.tight_layout() | |
| return fig | |
| def load_page(): | |
| """Every page load replays the vote log so the leaderboard is current.""" | |
| global STATE_WRITABLE, LOAD_STATUS | |
| try: | |
| applied = STORE.replay(storage.load_vote_log()) | |
| STATE_WRITABLE, LOAD_STATUS = True, f"Ratings rebuilt from {applied} logged votes." | |
| except Exception as exc: | |
| print(f"[arena] replay on load failed: {type(exc).__name__}: {exc}") | |
| return leaderboard_html(), leaderboard_plot() | |
| # --------------------------------------------------------------- UI | |
| CSS = """ | |
| :root, .dark { --bg:#0a0a0a; --fg:#eeeeee; --muted:#9a9a9a; --line:#2a2a2a; --field:#141414; } | |
| body, gradio-app, .gradio-container, .main, .wrap, .contain { background: var(--bg) !important; color: var(--fg) !important; } | |
| .gradio-container { max-width: 980px !important; margin: 0 auto !important; padding: 0 16px !important; | |
| font-family: "Courier New", ui-monospace, "DejaVu Sans Mono", monospace !important; } | |
| .gradio-container * { font-family: inherit; } | |
| footer { display: none !important; } | |
| .block, .form, .panel, .gr-group, .gr-box { background: transparent !important; border: none !important; box-shadow: none !important; } | |
| #topbar { display:flex; justify-content:space-between; align-items:center; padding:22px 0 22px; border-bottom:1px solid var(--line); } | |
| #topbar .brand { white-space:nowrap; font-family: Helvetica, Arial, sans-serif; font-weight:700; font-size:22px; letter-spacing:-0.04em; color:var(--fg); } | |
| #topbar .mark { display:inline-flex; width:34px; height:34px; background:#000; border:1px solid var(--line); margin-right:12px; | |
| align-items:center; justify-content:center; vertical-align:middle; font-size:15px; } | |
| #topbar a { color: var(--fg); text-decoration:none; font-size:13px; } | |
| .crumb { font-size:11px; letter-spacing:0.25em; color:var(--muted); text-transform:uppercase; margin: 6px 0 14px; } | |
| #hero { padding: 64px 0 10px; } | |
| #hero h1 { font-size: 46px; font-weight: 800; letter-spacing:-0.03em; margin: 0 0 16px; line-height:1.05; color:var(--fg); } | |
| #hero p { color: var(--muted); font-size: 14px; margin:0; max-width: 640px; line-height:1.6; } | |
| .tabs > .tab-nav, div[role="tablist"] { border-bottom: 1px solid var(--line) !important; gap:0 !important; } | |
| button[role="tab"] { background:transparent !important; color:var(--muted) !important; border:none !important; border-bottom:2px solid transparent !important; | |
| border-radius:0 !important; font-size:15px !important; padding:12px 26px !important; flex:1; } | |
| button[role="tab"][aria-selected="true"], button[role="tab"].selected { color:var(--fg) !important; border-bottom:2px solid var(--fg) !important; } | |
| label span, .label-wrap span, span[data-testid="block-info"] { color: var(--fg) !important; font-size:13px !important; background:transparent !important; } | |
| textarea, input[type=text] { background: var(--field) !important; color: var(--fg) !important; border:1px solid var(--line) !important; | |
| border-radius:0 !important; font-size:15px !important; padding:14px 16px !important; } | |
| textarea:focus { border-color:#6b6b6b !important; outline:none !important; box-shadow:none !important; } | |
| button.primary, #go { position:relative; background:#eeeeee !important; color:#0a0a0a !important; border:none !important; border-radius:0 !important; | |
| font-weight:700 !important; font-size:15px !important; padding:16px !important; | |
| clip-path: polygon(8px 0, calc(100% - 8px) 0, calc(100% - 8px) 8px, 100% 8px, 100% calc(100% - 8px), calc(100% - 8px) calc(100% - 8px), | |
| calc(100% - 8px) 100%, 8px 100%, 8px calc(100% - 8px), 0 calc(100% - 8px), 0 8px, 8px 8px); } | |
| button.primary:hover { background:#ffffff !important; } | |
| button.secondary { background:transparent !important; color:var(--fg) !important; border:1px solid var(--line) !important; border-radius:0 !important; | |
| font-size:14px !important; padding:14px !important; } | |
| button.secondary:hover:not(:disabled) { border-color:var(--fg) !important; } | |
| button:disabled { opacity:0.35 !important; cursor:not-allowed; } | |
| .linkbtn, .linkbtn button { white-space:nowrap !important; width:auto !important; flex:none !important; } | |
| .linkbtn { background:transparent !important; border:none !important; color:var(--muted) !important; text-decoration:underline !important; | |
| text-underline-offset:4px; font-size:13px !important; padding:4px 0 !important; min-width:0 !important; box-shadow:none !important; } | |
| .linkbtn:hover { color:var(--fg) !important; } | |
| .side-label { font-size:11px; letter-spacing:0.25em; color:var(--muted); margin:18px 0 6px; } | |
| .audio-box, .audio-box * { background: var(--field) !important; border-color: var(--line) !important; border-radius:0 !important; color:var(--fg) !important; } | |
| .note { color: var(--muted); font-size: 13px; margin: 14px 0; } | |
| .reveal { border-top:1px solid var(--line); margin-top:18px; padding-top:14px; } | |
| .rev-row { display:flex; gap:18px; align-items:baseline; padding:8px 0; border-bottom:1px solid #161616; font-size:15px; flex-wrap:wrap; } | |
| .rev-row .side { color:var(--muted); width:16px; } | |
| .rev-row a, .name a { color:var(--fg); text-decoration:underline; text-underline-offset:4px; text-decoration-color:#555; } | |
| .muted { color: var(--muted); } .small { font-size: 12px; margin-top:12px; } | |
| .elo.up { color:#d8d8d8; } .elo.down { color:#7a7a7a; } .elo { margin-left:auto; } | |
| .lb-meta { color:var(--muted); font-size:13px; margin:22px 0 12px; } | |
| .table-wrap { overflow-x:auto; } | |
| table.lb { width:100%; border-collapse:collapse; font-size:14px; } | |
| table.lb th { text-align:left; font-weight:400; color:var(--muted); font-size:11px; letter-spacing:0.12em; text-transform:uppercase; | |
| padding:10px 8px; border-bottom:1px solid var(--line); white-space:nowrap; } | |
| table.lb td { padding:12px 8px; border-bottom:1px solid #161616; color:var(--fg); } | |
| table.lb td.num, table.lb th:nth-child(n+3) { text-align:right; font-variant-numeric: tabular-nums; } | |
| table.lb td.rank { color:var(--muted); } table.lb td.strong { font-weight:700; } | |
| table.lb .org { color:var(--muted); font-size:11px; margin-top:3px; } | |
| .prose { color: var(--muted); font-size:13.5px; line-height:1.7; max-width: 720px; } | |
| .prose h3 { color: var(--fg); font-size:15px; margin: 26px 0 8px; } | |
| .prose a { color: var(--fg); } | |
| #foot { border-top:1px solid var(--line); margin-top:56px; padding:22px 0 40px; color:var(--muted); font-size:12px; line-height:1.6; } | |
| #votes > * { min-width: 0 !important; } | |
| @media (max-width: 640px) { #votes { display:grid !important; grid-template-columns:1fr 1fr; gap:8px; } | |
| #topbar .muted, .charnote { display:none !important; } #hero { padding-top:36px; } #hero h1 { font-size:34px; } button[role="tab"] { padding:12px 10px !important; } } | |
| """ | |
| FORCE_DARK = """ | |
| () => { document.documentElement.classList.add('dark'); document.body.classList.add('dark'); } | |
| """ | |
| THEME = gr.themes.Base(primary_hue="neutral", neutral_hue="neutral", radius_size="none", | |
| font=["Courier New", "ui-monospace", "monospace"], | |
| font_mono=["Courier New", "ui-monospace", "monospace"]).set( | |
| body_background_fill="#0a0a0a", body_background_fill_dark="#0a0a0a", body_text_color="#eeeeee", | |
| body_text_color_dark="#eeeeee", block_background_fill="transparent", block_background_fill_dark="transparent", | |
| block_border_width="0px", input_background_fill_dark="#141414", input_background_fill="#141414", | |
| border_color_primary="#2a2a2a", border_color_primary_dark="#2a2a2a") | |
| ABOUT = f""" | |
| <div class="prose"> | |
| <h3>How it works</h3> | |
| Type a sentence (or draw one), and two of the {len(registry.MODELS)} models below are picked at random to read it. | |
| Every clip is loudness-matched before playback, so nobody wins by being louder. Each model reads with one fixed | |
| voice. You vote, then the names are revealed. | |
| <h3>Rating</h3> | |
| Ratings are Elo (400 points per factor of 10 in odds), replayed vote by vote. Each model starts from a rating set by | |
| its automatic benchmark score (TTS Index: WER, UTMOS, SCOREQ, DNSMOS). How much a vote moves a rating depends on the gap: | |
| beating a stronger model gains more than beating a weaker one. Each model also has its own step size, large for its | |
| first battles and shrinking as it plays more, so two models rarely move by the same amount. | |
| <i>Both bad</i> counts as a draw but is tallied separately. | |
| <h3>Contenders</h3> | |
| {"<br>".join(f"{_name_html(s)} <span class='muted'>{s.params_m:.2f}M · {s.org}</span>" for s in registry.MODELS)} | |
| <br><br>Codename models are unreleased checkpoints, rated blind like everyone else. | |
| <h3>Logging</h3> | |
| Battles and votes need a Hugging Face sign-in; the leaderboard is open to everyone. Every text you submit and every vote is written to the arena's Hugging Face bucket, together with your Hugging Face username. Don't type personal or sensitive information. | |
| </div> | |
| """ | |
| with gr.Blocks(title="TTS Arena", theme=THEME, css=CSS, js=FORCE_DARK) as demo: | |
| gr.HTML('<div id="topbar"><div class="brand"><span class="mark">▶</span>TTS Arena</div>' | |
| '<span class="muted" style="font-size:13px">small models · blind votes</span></div>' | |
| '<div id="hero"><div class="crumb">ARENA / TEXT-TO-SPEECH</div><h1>Which voice<br>sounds better?</h1>' | |
| '<p>Blind battles between small text-to-speech models. Two read the same text, you pick the better one.</p></div>') | |
| with gr.Tabs(): | |
| with gr.Tab("Arena"): | |
| gr.LoginButton(value="Sign in with Hugging Face to battle and vote") | |
| text = gr.Textbox(label="Text", placeholder="Type a sentence for both models to read…", lines=3, max_lines=6, | |
| value=random.choice(SENTENCES)) | |
| with gr.Row(): | |
| draw = gr.Button("Draw a random sentence", elem_classes="linkbtn", scale=0) | |
| gr.HTML(f'<div class="muted small" style="text-align:right;margin:0">max {MAX_CHARS} characters</div>', elem_classes="charnote") | |
| go = gr.Button("Battle ↗", variant="primary", elem_id="go") | |
| with gr.Row(equal_height=True): | |
| with gr.Column(): | |
| gr.HTML('<div class="side-label">MODEL A</div>') | |
| out_a = gr.Audio(show_label=False, interactive=False, elem_classes="audio-box", show_download_button=False) | |
| with gr.Column(): | |
| gr.HTML('<div class="side-label">MODEL B</div>') | |
| out_b = gr.Audio(show_label=False, interactive=False, elem_classes="audio-box", show_download_button=False) | |
| with gr.Row(elem_id="votes"): | |
| btn_a = gr.Button("← " + VOTE_LABELS["a"], interactive=False, variant="secondary") | |
| btn_tie = gr.Button(VOTE_LABELS["tie"], interactive=False, variant="secondary") | |
| btn_bad = gr.Button(VOTE_LABELS["both_bad"], interactive=False, variant="secondary") | |
| btn_b = gr.Button(VOTE_LABELS["b"] + " →", interactive=False, variant="secondary") | |
| reveal = gr.HTML('<div class="note">Press Battle to hear two random models.</div>') | |
| with gr.Tab("Leaderboard"): | |
| board = gr.HTML(leaderboard_html()) | |
| plot = gr.Plot(value=leaderboard_plot(), show_label=False) | |
| refresh = gr.Button("Refresh", elem_classes="linkbtn") | |
| with gr.Tab("About"): | |
| gr.HTML(ABOUT) | |
| gr.HTML('<div id="foot">Texts and votes are logged to a Hugging Face bucket with your username. ' | |
| 'Audio is generated on demand; no clip is stored.</div>') | |
| state = gr.State({}) | |
| votes = [btn_a, btn_b, btn_tie, btn_bad] | |
| draw.click(lambda: random.choice(SENTENCES), outputs=text) | |
| go.click(battle, inputs=text, outputs=[out_a, out_b, state, reveal, *votes], api_name="battle") | |
| def _voter(c): | |
| def fn(s, profile: gr.OAuthProfile | None = None): | |
| return vote(s, c, profile) | |
| return fn | |
| for choice, b in (("a", btn_a), ("b", btn_b), ("tie", btn_tie), ("both_bad", btn_bad)): | |
| b.click(_voter(choice), inputs=state, outputs=[state, reveal, *votes, board, plot]) | |
| refresh.click(load_page, outputs=[board, plot]) | |
| demo.load(load_page, outputs=[board, plot]) | |
| if __name__ == "__main__": | |
| demo.queue(max_size=32).launch() | |