"""TTS Arena: blind side-by-side battles between small text-to-speech models.
Two randomly chosen models read the same text, you vote, and their ratings move. Names stay
hidden until the vote. Every text and vote is written to the arena's Hugging Face bucket.
"""
import random
import uuid
import gradio as gr
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import elo
import engines
import registry
import storage
try: # ZeroGPU is only present on the Space
import spaces
except ImportError: # pragma: no cover - local dev
class _Spaces:
@staticmethod
def GPU(*args, **kwargs):
def wrap(fn):
return fn
return wrap if not (args and callable(args[0])) else args[0]
spaces = _Spaces()
ARENA = "tts"
GPU_DURATION = 30
MAX_CHARS = 300
SENTENCES = [
"The museum opens at nine, but the best exhibits are upstairs, past the old clock.",
"If the train is late again, we should probably walk the rest of the way.",
"She folded the letter twice, slipped it into her coat, and never mentioned it again.",
"Fresh bread, a little butter, and a quiet morning: that is all I really need.",
"Please remember to water the plants on the balcony while I am away.",
"The storm knocked out the power, so we told stories by candlelight until midnight.",
"Honestly, I did not expect the sequel to be better than the original.",
"Turn left at the bakery, then keep going until you see the blue door.",
"Our team spent three weeks on the design, and it still needs another review.",
"Why would anyone leave a perfectly good umbrella on the bus?",
"The river was calm that evening, and the whole town came out to watch the lanterns.",
"Could you read that number back to me, slowly this time?",
]
def bootstrap():
"""Rebuild ratings from the vote log (the authoritative history); never overwrite history after a failed read."""
store = elo.EloStore()
try:
applied = store.replay(storage.load_vote_log())
return store, True, f"Ratings rebuilt from {applied} logged votes."
except Exception as exc:
print(f"[arena] vote-log replay failed: {type(exc).__name__}: {exc}")
try:
cached = storage.load_state()
if cached is not None:
return elo.EloStore(cached), True, "Ratings loaded from state/elo.json."
return store, True, "No history yet."
except Exception as exc:
return store, False, (f"Could not read history ({type(exc).__name__}). Ratings shown are for this session only "
"and are not saved. Votes are still logged.")
STORE, STATE_WRITABLE, LOAD_STATUS = bootstrap()
print(f"[arena] {LOAD_STATUS}")
engines.warm([s.repo_id for s in registry.MODELS])
import logging
for _n in ("httpx", "httpcore", "urllib3", "matplotlib", "numba", "phonemizer"):
logging.getLogger(_n).setLevel(logging.WARNING)
VOTE_LABELS = {"a": "A is better", "b": "B is better", "tie": "Tie", "both_bad": "Both bad"}
# --------------------------------------------------------------- generation
@spaces.GPU(duration=GPU_DURATION)
def _run_gpu(repo_ids, text, seed):
import torch
device = "cuda" if torch.cuda.is_available() else "cpu"
return [engines.synthesize(r, text, seed, device) for r in repo_ids]
def _run_battle(repo_a, repo_b, text, seed):
"""Torch models share one GPU grant; ONNX / Phonon run on the CPU without waiting for a GPU."""
gpu = [r for r in (repo_a, repo_b) if engines.needs_gpu(r)]
out = dict(zip(gpu, _run_gpu(gpu, text, seed))) if gpu else {}
for r in (repo_a, repo_b):
if r not in out:
out[r] = engines.synthesize(r, text, seed, "cpu")
return out[repo_a], out[repo_b]
def battle(text, profile: gr.OAuthProfile | None = None):
if profile is None:
raise gr.Error("Sign in with Hugging Face to run battles.")
text = " ".join((text or "").split())
if not text:
raise gr.Error("Type something for the models to read first.")
if len(text) > MAX_CHARS:
raise gr.Error(f"Keep it under {MAX_CHARS} characters.")
spec_a, spec_b = random.sample(registry.arena_models(ARENA), 2)
seed = random.randint(0, 2**31 - 1)
battle_id = uuid.uuid4().hex
audio_a, audio_b = _run_battle(spec_a.repo_id, spec_b.repo_id, text, seed)
storage.log_generation({
"battle_id": battle_id, "arena": ARENA, "created_at": storage.utc_now(), "hf_user": profile.username, "prompt": text,
"settings": {"seed": seed},
"model_a": {"repo_id": spec_a.repo_id, "params": spec_a.params, "seconds": len(audio_a[1]) / audio_a[0]},
"model_b": {"repo_id": spec_b.repo_id, "params": spec_b.params, "seconds": len(audio_b[1]) / audio_b[0]},
})
state = {"battle_id": battle_id, "prompt": text, "repo_a": spec_a.repo_id, "repo_b": spec_b.repo_id,
"settings": {"seed": seed}, "voted": False, "hf_user": profile.username}
reveal = 'Listen to both, then vote. Names are hidden until you do.
'
return [audio_a, audio_b, state, reveal, *[gr.update(interactive=True) for _ in VOTE_LABELS]]
# --------------------------------------------------------------- voting
def _name_html(s):
return f'{s.name} ↗' if s.url else f"{s.name}"
def _reveal_html(state, deltas):
rows = []
for side, repo_id in (("A", state["repo_a"]), ("B", state["repo_b"])):
s, d = registry.spec(repo_id), deltas[repo_id]
diff = d["after"] - d["before"]
sign = "up" if diff > 0 else ("down" if diff < 0 else "flat")
rows.append(f'{side}{_name_html(s)}'
f'{s.params_m:.1f}M'
f'{d["before"]:.1f} → {d["after"]:.1f}
')
return 'REVEALED
' + "".join(rows) + \
'
Vote recorded. Type new text or draw a sentence for the next battle.
| {i:02d} | {_name_html(s)} {s.org} | "
f"{r['elo']:.1f} | {r['params_m']:.1f} | "
f"{r['elo_per_m']:.0f} | {r['battles']} | "
f"{r['wins']} | {r['losses']} | {r['ties']} | "
f"{r['both_bad']} | {wr} |
")
status = "" if STATE_WRITABLE else f"{LOAD_STATUS}
"
return (f"{len(rows)} models · {total} rated battles
{status}"
"| # | Model | Elo | Params (M) | "
"Elo / M | Battles | W | L | T | Both bad | Win rate |
|---|
"
f"{''.join(body)}
How it works
Type a sentence (or draw one), and two of the {len(registry.MODELS)} models below are picked at random to read it.
Every clip is loudness-matched before playback, so nobody wins by being louder. Each model reads with one fixed
voice. You vote, then the names are revealed.
Rating
Ratings are Elo (400 points per factor of 10 in odds), replayed vote by vote. Each model starts from a rating set by
its automatic benchmark score (TTS Index: WER, UTMOS, SCOREQ, DNSMOS). How much a vote moves a rating depends on the gap:
beating a stronger model gains more than beating a weaker one. Each model also has its own step size, large for its
first battles and shrinking as it plays more, so two models rarely move by the same amount.
Both bad counts as a draw but is tallied separately.
Contenders
{"
".join(f"{_name_html(s)} {s.params_m:.2f}M · {s.org}" for s in registry.MODELS)}
Codename models are unreleased checkpoints, rated blind like everyone else.
Logging
Battles and votes need a Hugging Face sign-in; the leaderboard is open to everyone. Every text you submit and every vote is written to the arena's Hugging Face bucket, together with your Hugging Face username. Don't type personal or sensitive information.
"""
with gr.Blocks(title="TTS Arena", theme=THEME, css=CSS, js=FORCE_DARK) as demo:
gr.HTML('▶TTS Arena
'
'
small models · blind votesARENA / TEXT-TO-SPEECH
Which voice
sounds better?
'
'
Blind battles between small text-to-speech models. Two read the same text, you pick the better one.
max {MAX_CHARS} characters
', elem_classes="charnote")
go = gr.Button("Battle ↗", variant="primary", elem_id="go")
with gr.Row(equal_height=True):
with gr.Column():
gr.HTML('MODEL A
')
out_a = gr.Audio(show_label=False, interactive=False, elem_classes="audio-box", show_download_button=False)
with gr.Column():
gr.HTML('MODEL B
')
out_b = gr.Audio(show_label=False, interactive=False, elem_classes="audio-box", show_download_button=False)
with gr.Row(elem_id="votes"):
btn_a = gr.Button("← " + VOTE_LABELS["a"], interactive=False, variant="secondary")
btn_tie = gr.Button(VOTE_LABELS["tie"], interactive=False, variant="secondary")
btn_bad = gr.Button(VOTE_LABELS["both_bad"], interactive=False, variant="secondary")
btn_b = gr.Button(VOTE_LABELS["b"] + " →", interactive=False, variant="secondary")
reveal = gr.HTML('Press Battle to hear two random models.
')
with gr.Tab("Leaderboard"):
board = gr.HTML(leaderboard_html())
plot = gr.Plot(value=leaderboard_plot(), show_label=False)
refresh = gr.Button("Refresh", elem_classes="linkbtn")
with gr.Tab("About"):
gr.HTML(ABOUT)
gr.HTML('')
state = gr.State({})
votes = [btn_a, btn_b, btn_tie, btn_bad]
draw.click(lambda: random.choice(SENTENCES), outputs=text)
go.click(battle, inputs=text, outputs=[out_a, out_b, state, reveal, *votes], api_name="battle")
def _voter(c):
def fn(s, profile: gr.OAuthProfile | None = None):
return vote(s, c, profile)
return fn
for choice, b in (("a", btn_a), ("b", btn_b), ("tie", btn_tie), ("both_bad", btn_bad)):
b.click(_voter(choice), inputs=state, outputs=[state, reveal, *votes, board, plot])
refresh.click(load_page, outputs=[board, plot])
demo.load(load_page, outputs=[board, plot])
if __name__ == "__main__":
demo.queue(max_size=32).launch()