:kenlm-a0.6-b100 »)
plutôt que par une colonne dédiée : c'est déjà l'identité exacte de la
configuration évaluée, et ça évite une migration de schéma.
Trois écritures coexistent : « :kenlm-… » (soumissions FonBench),
« #kenlm… » (lignes importées du benchmark JML v2.1) et « + KenLM » dans
le nom d'un système JML v2.1. Même règle que `avecKenlm` du Space statique.
"""
return bool(_AVEC_LM.search(r.get("model_revision") or ""))
_AVEC_LM = re.compile(r"[:#]kenlm|\+\s*kenlm", re.IGNORECASE)
LM_VUES = {"All decoding": None, "With LM": True, "No LM": False}
_SUFFIXE_LM = re.compile(r":kenlm-[^:]*$")
def apparier(rows: list[dict], rank_by_wer: bool = False) -> list[dict]:
"""Une ligne par modèle, portant les deux WER de décodage.
Les configurations d'un même modèle ne diffèrent que par le suffixe de
révision (« …:kenlm-a0.6-b100 ») : on regroupe là-dessus. La ligne
affichée reste celle retenue au classement (non masquée) ; l'autre ne
sert qu'à renseigner la colonne de WER qui lui manque.
Un groupe dont AUCUNE ligne n'est visible reste absent : c'est ce qui
permet d'écarter durablement un checkpoint sans perdre ses mesures.
"""
groupes: dict[tuple, list[dict]] = {}
for r in rows:
base = _SUFFIXE_LM.sub("", r.get("model_revision") or "")
groupes.setdefault((r["model_id"], base), []).append(r)
sortie = []
for membres in groupes.values():
visibles = [r for r in membres if not r.get("is_hidden")]
if not visibles:
continue
# Configuration retenue = la meilleure selon la métrique qui classe
# (T-WER sur un corpus annoté en tons, WER_seg sinon). La choisir au
# WER alors que le tableau trie au T-WER donnerait des lignes dont le
# rang contredit le contenu.
def rang(r):
# Un benchmark peut imposer le WER avec tons (ranking_metric).
for cle in (("wer",) if rank_by_wer
else ("twer", "wer_notone", "wer")):
if r.get(cle) is not None:
return r[cle]
return 9e9
canonique = min(visibles, key=rang)
ligne = dict(canonique)
glouton = next((r for r in membres if not avec_kenlm(r)), None)
avec = next((r for r in membres if avec_kenlm(r)), None)
ligne["wer_greedy"] = (glouton or canonique).get("wer")
ligne["wer_lm"] = avec.get("wer") if avec else None
ligne["a_variante_lm"] = avec is not None
sortie.append(ligne)
return sortie
def build_table(benchmark_id: str, vue: str, archs: list[str],
decoders: list[str], hide_contaminated: bool, sort_by: str,
lm: str = "All decoding"):
try:
bench = next((b for b in get_benchmarks() if b["id"] == benchmark_id), {})
rank_by_wer = bench.get("ranking_metric") == "wer"
rows = apparier(get_results(benchmark_id), rank_by_wer)
except Exception as exc: # noqa: BLE001
return (pd.DataFrame({"Error": [f"Database unreachable: {exc}"]}),
"", "", gr.update(), gr.update())
all_archs = sorted({r["architecture"] for r in rows if r.get("architecture")})
all_decs = sorted({r["decoder_type"] for r in rows if r.get("decoder_type")})
sel = rows
vises = VUES.get(vue)
if vises:
sel = [r for r in sel if categorie(r) in vises]
if archs:
sel = [r for r in sel if r.get("architecture") in archs]
if decoders:
sel = [r for r in sel if r.get("decoder_type") in decoders]
vise_lm = LM_VUES.get(lm)
if vise_lm is not None:
sel = [r for r in sel if bool(r.get("a_variante_lm")) is vise_lm]
if hide_contaminated:
sel = [r for r in sel if not r.get("contamination_flag")]
# Le corpus annote-t-il les tons ? Si oui le T-WER fait foi, sinon on
# classe sur le WER_seg — comparer des WER bruts entre corpus dont les
# conventions tonales diffèrent n'aurait aucun sens.
tonal = any(r.get("twer") is not None for r in rows)
metric = "wer" if rank_by_wer else ("twer" if tonal else "wer_notone")
if sort_by == "Model size":
sel = sorted(sel, key=lambda r: (r.get("model_params") or 0))
else:
sel = sorted(sel,
key=lambda r: (r.get(metric) is None, r.get(metric) or 9e9))
data = []
for i, r in enumerate(sel, 1):
medaille = {1: "🥇", 2: "🥈", 3: "🥉"}.get(i, "")
nom = lien(r["model_id"])
# Plusieurs configurations d'un meme depot (checkpoints, decodages)
# portent le meme identifiant : la revision les distingue.
# Separateur « # » (benchmarks importes) ou « : » (checkpoint, kenlm).
rev = r.get("model_revision") or ""
sep = "#" if "#" in rev else ":"
ck = rev.split(sep, 1)
if len(ck) > 1 and ck[1]:
nom += f"
↳ {ck[1]}"
if r.get("contamination_flag"):
nom += " ⚠️"
data.append({
"#": f"{i} {medaille}".strip(),
"Model": nom,
"Type": TYPES[categorie(r)],
"WER": pct(r.get("wer")),
"WER (no tones)": pct(r.get("wer_notone")),
"CER": pct(r.get("cer")),
"CER (no tones)": pct(r.get("cer_notone")),
"Size": params_txt(r.get("model_params")),
"Base model": lien(r.get("base_model")),
"Trained on": train_txt(r),
"Architecture": r.get("architecture") or "—",
# Le décodeur seul ne dirait pas qu'un LM externe est branché :
# deux lignes « ctc » peuvent être deux systèmes différents.
"Decoder": ((r.get("decoder_type") or "—")
+ (" + kenlm" if avec_kenlm(r) else "")),
})
compte = {c: sum(1 for r in sel if categorie(r) == c)
for c in ("maison", "tiers", "base")}
meta = (
""
f"{flag(13)} {bench.get('name', benchmark_id)}"
f"{bench.get('num_utterances', '?')} utterances"
f"{bench.get('duration_hours', '?')} hours"
+ ("private test set"
if bench.get("is_private") else "")
+ f"ranked by {'WER' if rank_by_wer else ('T-WER' if tonal else 'WER_seg')}"
+ f"{len(sel)} of {len(rows)} models shown"
# Un compteur nu (« 16 community ») ne dit pas ce qu'il compte :
# on écrit la phrase en entier.
+ (""
f"🔧 {compte['maison']} fine-tuned by FonBench"
" · "
f"👥 {compte['tiers']} fine-tuned by the community"
" · "
f"🌍 {compte['base']} general-purpose"
"")
+ "
"
)
legende = (
"All error rates in **%**, lower is better. **⚠️** marks a model "
"likely trained on this test set: its score is not comparable."
)
return (pd.DataFrame(data, columns=COLONNES), meta, legende,
gr.update(choices=all_archs), gr.update(choices=all_decs))
NOTE_METRIQUES = """
### Reading the table
Fon is a **tonal** language: tones are written with diacritics (á, ɔ́, ě…)
and change the meaning of words. But Fon corpora don't follow the same
convention — some mark no tone at all. A raw WER therefore isn't comparable
from one corpus to the next. Hence two views of the same errors.
| Metric | What it tells you |
|---|---|
| **WER** | Word error rate, **tones included**: a wrong tone makes the word wrong. |
| **WER (no tones)** | The same word errors after **stripping tone marks**. Measures phonetic accuracy and stays comparable across every corpus. |
| **CER** | Character error rate, tones included. |
| **CER (no tones)** | The same character errors after stripping tone marks. |
**Model types.** 🌍 *General-purpose* models were not built for Fon: some
are multilingual systems that do cover it among a thousand other languages
(the MMS family ships a Fon adapter), others are multilingual without Fon,
and others again are monolingual systems for English or French. They are not
*base* models in the pretrained sense — `wav2vec2-large-960h-lv60-self` is a
finished English recogniser, not a starting point. The **Trained on** column
says exactly what each one saw. 🔧 *FonBench* and 👥 *Community* models were
fine-tuned on Fon; comparing them against a general-purpose model that shares
their base is what actually measures the value of fine-tuning.
Truly *base* models — raw pretrained checkpoints such as
`facebook/wav2vec2-large-xlsr-53` — cannot appear here at all: without a CTC
head or a vocabulary they transcribe nothing. They show up only in the
**Base model** column, as the starting point of the models that were
fine-tuned from them.
"""
# --- file d'attente -----------------------------------------------------
def build_queue():
try:
rows = fetch("public_queue", {"select": "*", "order": "created_at.desc",
"limit": "50"}, ttl=15)
except Exception as exc: # noqa: BLE001
return pd.DataFrame({"Error": [f"Database unreachable: {exc}"]}), ""
etats = {"pending": "⏳ pending", "running": "⚙️ running",
"done": "✅ done", "failed": "❌ failed",
"rejected": "🚫 rejected"}
data = []
for r in rows:
total = r.get("progress_total") or 0
done = r.get("progress_done") or 0
avance = f"{done}/{total}" if total else "—"
if total and r["status"] == "running":
avance += f" ({done * 100 // total}%)"
data.append({
"Model": r["model_id"],
"Status": etats.get(r["status"], r["status"]),
"Progress": avance,
"Details": (r.get("error_message") or "")[:120],
"Submitted": (r.get("created_at") or "")[:10],
})
etat = evaluator.status()
ligne = f"**Evaluator:** {etat['message']}"
if etat.get("model"):
ligne += f" — {etat['model']} ({etat['done']}/{etat['total']})"
return pd.DataFrame(data), ligne
# --- soumission ---------------------------------------------------------
def submit(model_id: str, hf_username: str, contact: str, note: str,
benchmark_id: str, train_data: str, train_hours, base_model: str):
if not benchmark_id:
return ("### Submissions are closed\n"
"No test set currently accepts submissions: the active "
"benchmarks are evaluated by the JML team with their own protocol.")
model_id = (model_id or "").strip()
if "/" not in model_id or len(model_id.split("/")) != 2 or \
not all(model_id.split("/")):
return ("### ❌ Invalid format\n"
"The identifier must look like `organisation/name`, exactly as "
"it appears in the model's URL on Hugging Face.")
payload = {"model_id": model_id, "benchmark_id": benchmark_id}
for champ, valeur in (("hf_username", hf_username), ("contact", contact),
("note", note), ("train_data", train_data),
("base_model", base_model)):
if (valeur or "").strip():
payload[champ] = valeur.strip()
if train_hours:
payload["train_hours"] = float(train_hours)
try:
r = requests.post(
f"{REST}/public_requests",
headers={**HEADERS, "Content-Type": "application/json",
"Prefer": "return=minimal"},
json=payload, timeout=30,
)
except Exception as exc: # noqa: BLE001
return f"### ❌ Could not submit\nDatabase unreachable: {exc}"
if r.status_code >= 400:
# La base applique elle-même les garde-fous (doublon, débit, file
# pleine) et renvoie un message déjà rédigé.
try:
detail = r.json().get("message") or r.text
except Exception: # noqa: BLE001
detail = r.text
return f"### ❌ Submission rejected\n{detail}"
_cache.clear()
return (
f"### ✅ `{model_id}` is queued\n\n"
"Evaluation runs in slices on the Space's shared GPU. Depending on the "
"available quota, expect anywhere from a few minutes to a few hours. "
"The score will appear in the leaderboard once computed — follow "
"progress in the **Queue** tab."
)
FINE_TUNING = """
## The FonBench fine-tuning runs
> These four runs and their scores belong to the former test set (JML test v1.0,
> 2,555 utterances). They are not comparable with the ranking of the current
> benchmark.
Four pretrained models were fine-tuned on Fon under strictly identical
conditions, so that any gap between them comes from the starting model
alone.
### The data
The training corpus holds **44,225 utterances**, of which **13,716 were
kept — 30.00 hours from 471 speakers**. The sample is not drawn at random:
it is built by taking turns across speakers, each contributing a little
before any one of them dominates. For generalisation, the diversity of
voices matters more than raw volume.
The 471 training speakers and the 45 test speakers are **strictly
disjoint** — verified, zero in common. A temporal cutoff separates the
transcripts; 20 sentences out of 2,555 (0.8%) do appear in both, too few to
move a score but worth stating rather than claiming a perfect separation.
### The protocol
Identical across all four: **3 epochs**, i.e. 2,574 steps with 257 of
warmup. CTC decoding, character vocabulary built from the training corpus,
feature extractor taken from the base model. A single 24 GB L4, `fp16`,
gradient checkpointing.
| Resulting model | Base | Learning rate | Batch | Wall time | Final dev loss | WER_seg |
|---|---|---|---|---|---|---|
| `wav2vec2-large-xlsr-53-fon-30h` | facebook/wav2vec2-large-xlsr-53 | 3·10⁻⁴ | 8×2 | 54 min | **0.468** | **38.6%** |
| `mms-300m-fon-30h` | facebook/mms-300m | 3·10⁻⁴ | 8×2 | not retained | not retained | **42.8%** |
| `w2v-bert-2.0-fon-30h` | facebook/w2v-bert-2.0 | 3·10⁻⁵ | 4×4 | 116 min | 3.102 | 96.9% |
| `AfriHuBERT-fon-30h` | ajesujoba/AfriHuBERT | 3·10⁻⁵ | 8×2 | 24 min | 3.078 | 100.0% |
### What this shows
**Fine-tuning makes the difference, not the base.** `chrisjay/fonxlsr`
starts from exactly the same model as our best run —
`facebook/wav2vec2-large-xlsr-53`, confirmed in its configuration — and
reaches 69.3% where we reach 38.6%. The error is cut by a factor of 1.8
with 30 hours of well-chosen data.
**Two runs out of four failed**, and that deserves saying plainly.
`w2v-bert-2.0` and `AfriHuBERT` do learn: their loss drops clearly during
training. But it plateaus around 3.1 against 0.47 for the run that
succeeds — the signature of settling into CTC's trivial solution, emitting
the blank symbol everywhere. This is not a learning-rate problem: both were
first run at 3·10⁻⁴, then rerun at 3·10⁻⁵ — the figures in the table — for
exactly the same outcome. They would need far more epochs, or more data.
The `mms-300m` run predates per-model logging, so its wall time and final
loss are genuinely lost. They are shown as *not retained* rather than
reconstructed from memory.
Both failures stay in the leaderboard. Removing them would paint a
flattering and false picture of what fine-tuning guarantees.
### Reproducing
`train_ctc.py` is published in the **Files** tab of this Space, under
`repro/`, together with the evaluation script and the scoring code. See the
**Reproduce** tab.
"""
REPRODUIRE = """
## Reproduce every number
A leaderboard nobody can check is just a poster. The scoring code of the
current benchmark is published in the **Files** tab of this Space, so any row
can be recomputed from a model's outputs.
| File | What it is |
|---|---|
| `jml_v21/texte.py`, `jml_v21/metriques.py` | The scoring code of **JML Fon v2.1**: text normalisation, WER and CER with and without tones. Copied **unchanged** from the JML reference code (`benchmark-jml-v2.1/code/`); `jml_v21/SOURCE.txt` gives the commit and the SHA-256 of each file. |
| `evaluator.py` | The code that evaluates each submission on this Space. |
| `repro/` | Standalone scripts of the previous protocol (see below). |
### Recomputing a score
```
cd jml_v21
python -c "import metriques as m; print(m.evaluer(['mǐ ná yi'], ['mi ná yi'], tons=True))"
```
This prints WER 33.3 and CER 10.0 (with tones, one word out of three is wrong).
With `tons=False` both are 0.0. Errors are summed over the whole corpus (not
averaged per utterance), after both texts are normalised the same way: lower
case, punctuation and digits removed. A tone mark counts as a character. The
test set, `JMLdata/fon-test-v2-1` (revision
`7f247395d423177a0ba89803a8b033e3f07fc4cc`), is private: its owner can grant
access, and its `transcription` column holds the references.
**Checked:** applied to the published outputs of four models, this code
returns exactly the scores that JML imported (WER, WER without tones, CER and
CER without tones).
### The previous protocol
The scripts in `repro/` and their reference values belong to the former test
set (JML test v1.0, 2,555 utterances), which is no longer ranked. They use
FonBench's own text normalisation, which is **not** the one used by the
current benchmark: on the same outputs of four models, the two scorers differ
by up to 6.5 points of CER. Do not compare their figures with the current
leaderboard.
RTFx is a hardware measurement and only reproduces on the same GPU; the
hardware is recorded with every score.
If you get a materially different number, tell us. That is the point of
publishing this.
"""
A_PROPOS = """
## FonBench
The first public speech-recognition leaderboard for **Fon**, a tonal
language spoken by over two million people in Benin. The project answers
a concrete difficulty: until now, published Fon results were not comparable
with one another, for lack of a shared protocol and a shared test set.
### Why the test set is not published
Public Fon corpora have been circulating since 2016, and several released
models were trained on them. Their WER on that data then looks remarkable —
while saying nothing about their real ability to transcribe an unseen
voice. We measured it: one of the test sets we were using shared **83% of
its utterances** with the training split of a public dataset.
The test set of the current benchmark, **JML Fon v2.1**, holds **6,382
utterances, 13.97 hours and 60 speakers**. It is therefore not distributed:
only aggregate scores are made public, and nobody — not even through this
Space — can download its audio or its transcriptions. Its clips, speakers and
sentences are disjoint from the v2.1 training data (checked on the published
identifier lists); 259 of its clips come from the former official test, none
of them in the v2.1 training data. A model whose training predates v2.1 may
still have heard some of its voices: such rows carry a **⚠ contamination**
mark, and their scores are not comparable.
### The protocol
- **Pinned revision.** Every score is tied to the exact commit hash of the
repository evaluated. Republishing a model under the same name does not
change a score already obtained.
- **One scorer for every row.** WER and CER, with and without tones, come from
the JML reference code (`jml_v21/` in the Files tab, copied unchanged). The
rows imported from JML and the rows evaluated here are scored by the same
code. The ranking is on the **WER with tones**.
- **No arbitrary code, with two documented exceptions.** Models are loaded
with `trust_remote_code=False`, so custom code shipped in a repository is
never executed. The exceptions are pinned to a revision that was reviewed
before being allowed: `asrDIL/fonglab-asr-w2vbert`, whose own decoding
package is executed (syllable units, language model and rescoring), and two
private `Fongbe-ASR` repositories in standard transformers format, read with
an access token and run without any repository code.
- **Where a row comes from.** Evaluated on this Space; imported from JML
(computed by JML with its own pipeline); or outputs supplied by the authors
and scored here (stated in the row's revision text, as for
`Fongbe-ASR/DePal-asr-fon-omni-v1`).
- **No duplicate work.** A (model, revision, corpus) triple is never
re-evaluated: the existing score is reused.
### Submitting a model
The repository must be **public** on the Hub. CTC models are supported
(wav2vec2, wav2vec2-BERT, HuBERT, MMS); a run takes about 3 to 10 minutes of
GPU. Whisper and other encoder-decoder models **cannot be evaluated on this
Space at the moment** (a GPU-allocation error, fix pending). No account
required.
"""
# --- interface ----------------------------------------------------------
def build_ui() -> gr.Blocks:
try:
benchs = get_benchmarks()
except Exception: # noqa: BLE001 — l'interface doit s'afficher malgré tout
benchs = []
choix = [(f"{b['name']} ({b.get('num_utterances', '?')} utterances)",
b["id"]) for b in benchs]
# Un benchmark peut fermer ses soumissions (résultats importés, évaluation
# faite par JML) : visible au classement, absent du formulaire.
choix_soumission = [(f"{b['name']} ({b.get('num_utterances', '?')} utterances)",
b["id"]) for b in benchs
if b.get("accepts_submissions") is not False]
# Le classement s'ouvre sur le benchmark le plus complet ; le formulaire,
# lui, garde un benchmark qui accepte encore les soumissions.
defaut_classement = "jml-v2-1" if any(b["id"] == "jml-v2-1" for b in benchs) else None
defaut = "jml-test-v1" if any(b["id"] == "jml-test-v1" for b in benchs) \
else (benchs[0]["id"] if benchs else "")
defaut_classement = defaut_classement or defaut
# Gradio 6 : `theme` et `css` se passent à launch(), plus au constructeur.
with gr.Blocks(title="FonBench — Fon ASR leaderboard") as demo:
gr.HTML(
""
"
FonBench "
f"{flag(26)}
"
"
The public speech-recognition leaderboard for "
"Fon, a tonal language of Benin
"
"