| """FonBench — leaderboard ASR pour le fongbe. |
| |
| Vitrine publique du classement, formulaire de soumission, et lancement de |
| l'évaluateur de fond (voir evaluator.py). Cette interface ne lit que des |
| scores agrégés : ni l'audio ni les transcriptions du corpus de test n'y |
| transitent jamais. |
| |
| Interface en anglais (public international) ; commentaires en français, |
| comme le reste du dépôt. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import re |
| import time |
|
|
| import gradio as gr |
| import pandas as pd |
| import requests |
|
|
| import evaluator |
|
|
| |
| |
| SUPABASE_URL = evaluator.SUPABASE_URL |
| ANON_KEY = evaluator.ANON_KEY |
| REST = f"{SUPABASE_URL}/rest/v1" |
| HEADERS = {"apikey": ANON_KEY} |
|
|
| CACHE_SECONDS = 60 |
| _cache: dict = {} |
|
|
| |
| CSS = """ |
| :root { |
| --fb-indigo:#3b3b7a; --fb-ocre:#c8873b; --fb-ink:#1b1b2e; |
| --fb-muted:#6b6b7b; --fb-line:rgba(120,120,160,.22); |
| } |
| .dark { --fb-indigo:#a9a9f0; --fb-ink:#ececf6; --fb-muted:#9a9ab0; } |
| |
| .fb-head { padding:1.6rem 0 .5rem; border-bottom:1px solid var(--fb-line); |
| margin-bottom:1rem; } |
| .fb-title { display:flex; align-items:center; gap:.55rem; |
| font-size:2.4rem; font-weight:800; letter-spacing:-.03em; |
| color:var(--fb-indigo); line-height:1.05; margin:0; } |
| .fb-title span { color:var(--fb-ocre); } |
| .fb-title svg { flex:0 0 auto; box-shadow:0 0 0 1px rgba(0,0,0,.12); } |
| .fb-sub { color:var(--fb-muted); margin:.35rem 0 0; font-size:1.02rem; } |
| |
| .fb-note { font-size:.9rem; line-height:1.6; color:var(--fb-ink); } |
| .fb-note h3 { margin-top:1.4rem; font-size:1.05rem; letter-spacing:-.01em; } |
| .fb-note table { font-size:.86rem; } |
| |
| .fb-badge { display:inline-block; padding:.12rem .55rem; border-radius:999px; |
| font-size:.72rem; font-weight:700; letter-spacing:.02em; |
| background:var(--fb-ocre); color:#fff; vertical-align:middle; } |
| |
| /* Bandeau de tête du classement : chiffres clés, pas un paragraphe. */ |
| .fb-meta { display:flex; flex-wrap:wrap; gap:.45rem 1.4rem; align-items:center; |
| padding:.7rem .9rem; margin:.2rem 0 .7rem; |
| border:1px solid var(--fb-line); border-radius:12px; |
| font-size:.88rem; color:var(--fb-muted); } |
| /* Chaque item est lui-même un flex : sans ça le SVG du drapeau se cale sur |
| la ligne de base du texte et paraît remonté. */ |
| .fb-meta > span { display:inline-flex; align-items:center; gap:.35rem; } |
| .fb-meta b { color:var(--fb-ink); font-weight:650; } |
| .fb-meta .fb-sep { color:var(--fb-line); } |
| |
| .fb-legend { font-size:.82rem; color:var(--fb-muted); margin:.15rem 0 .9rem; } |
| |
| /* Filtres : chaque menu est un champ à part, avec son étiquette au-dessus. |
| Sans ça, Gradio fusionne les champs voisins d'une rangée en un seul bloc |
| gris où l'on ne distingue plus où commence quoi. */ |
| .fb-filters { gap:.7rem !important; margin-bottom:.6rem; } |
| .fb-filters .block { padding:0 !important; } |
| .fb-filters label > span:first-child, |
| .fb-filters span[data-testid='block-info'] { |
| display:block; margin-bottom:.28rem; |
| font-size:.7rem !important; font-weight:700 !important; |
| letter-spacing:.07em; text-transform:uppercase; |
| color:var(--fb-muted) !important; } |
| .fb-filters input, .fb-filters .wrap-inner, .fb-filters .secondary-wrap { |
| border-radius:9px !important; } |
| .fb-row2 { align-items:center; gap:.6rem !important; margin-bottom:.4rem; } |
| |
| /* Sélecteur de vue : des pastilles cliquables, pas des boutons radio. */ |
| .fb-views .wrap { gap:.35rem !important; } |
| .fb-views label { border-radius:999px !important; |
| padding:.32rem .9rem !important; font-weight:600; |
| font-size:.86rem; border:1px solid var(--fb-line) !important; |
| cursor:pointer; transition:background .12s; } |
| .fb-views label:hover { background:rgba(120,120,180,.12) !important; } |
| /* Le cercle du radio n'apporte rien à côté de l'état sélectionné : on le |
| masque sans le retirer du parcours clavier. */ |
| .fb-views input[type=radio] { position:absolute; opacity:0; |
| width:1px; height:1px; } |
| |
| /* Le tableau est large : on le laisse défiler plutôt que compresser. */ |
| .fb-table table { font-size:.86rem; } |
| .fb-table td, .fb-table th { padding:.42rem .55rem !important; } |
| |
| footer { display:none !important; } |
| """ |
|
|
|
|
| |
|
|
| def fetch(path: str, params: dict, ttl: int = CACHE_SECONDS): |
| key = (path, tuple(sorted(params.items()))) |
| hit = _cache.get(key) |
| if hit and time.time() - hit[0] < ttl: |
| return hit[1] |
| r = requests.get(f"{REST}/{path}", headers=HEADERS, params=params, timeout=30) |
| r.raise_for_status() |
| data = r.json() |
| _cache[key] = (time.time(), data) |
| return data |
|
|
|
|
| def get_benchmarks() -> list[dict]: |
| return fetch("benchmarks", {"is_active": "eq.true", "select": "*", |
| "order": "id"}, ttl=600) |
|
|
|
|
| def get_results(benchmark_id: str) -> list[dict]: |
| """Toutes les lignes, masquées comprises. |
| |
| Un modèle peut avoir été évalué en plusieurs configurations de décodage |
| (glouton, puis avec un n-gram externe). Le classement n'affiche qu'une |
| ligne par modèle, mais il a besoin des deux pour remplir les colonnes |
| « WER greedy » et « WER +LM » : on récupère donc tout et c'est |
| `apparier()` qui décide de ce qui s'affiche. |
| """ |
| return fetch("results", {"benchmark_id": f"eq.{benchmark_id}", |
| "select": "*"}) |
|
|
|
|
| |
|
|
| TYPES = { |
| "base": "🌍 General", |
| "maison": "🔧 FonBench", |
| "tiers": "👥 Community", |
| } |
| |
| |
| VUES: dict[str, set[str] | None] = { |
| "All models": None, |
| "🎯 Fine-tuned on Fon": {"maison", "tiers"}, |
| "🔧 By FonBench": {"maison"}, |
| "👥 By the community": {"tiers"}, |
| "🌍 General-purpose": {"base"}, |
| } |
|
|
|
|
| def categorie(r: dict) -> str: |
| """Catégorie d'un modèle, tolérante aux schémas intermédiaires. |
| |
| `model_kind` a d'abord valu 'fongbe' ou 'base' avant de passer à trois |
| valeurs. Entre les deux migrations, une lecture au pied de la lettre |
| ferait disparaître tous les modèles fongbe : on retombe alors sur le |
| préfixe du dépôt, qui dit la même chose. |
| """ |
| k = r.get("model_kind") |
| if k in ("base", "maison", "tiers"): |
| return k |
| if k == "fongbe": |
| return "maison" if r["model_id"].startswith("fonbench/") else "tiers" |
| return "base" |
|
|
|
|
| |
| |
| def flag(h: int = 15) -> str: |
| w = round(h * 1.5) |
| return ( |
| f"<svg viewBox='0 0 30 20' width='{w}' height='{h}' " |
| "style='vertical-align:-.12em;border-radius:2px' " |
| "role='img' aria-label='Benin'>" |
| "<rect width='30' height='20' fill='#FCD116'/>" |
| "<rect y='10' width='30' height='10' fill='#E8112D'/>" |
| "<rect width='12' height='20' fill='#008751'/></svg>" |
| ) |
|
|
|
|
| def pct(x) -> str: |
| return "—" if x is None else f"{float(x) * 100:.2f}" |
|
|
|
|
| def params_txt(n) -> str: |
| if not n: |
| return "—" |
| n = int(n) |
| return f"{n / 1e9:.1f}B" if n >= 1e9 else f"{n / 1e6:.0f}M" |
|
|
|
|
| def train_txt(r: dict) -> str: |
| data = r.get("train_data") |
| if not data: |
| return "not declared" |
| if data == "aucun fongbe": |
| return "no Fon data" |
| h = r.get("train_hours") |
| return f"{data} · {float(h):g}h" if h else data |
|
|
|
|
| def lien(model_id: str | None) -> str: |
| if not model_id: |
| return "—" |
| return f"[{model_id}](https://huggingface.co/{model_id})" |
|
|
|
|
| COLONNES = ["#", "Model", "Type", "WER", "WER (no tones)", "CER", |
| "CER (no tones)", "Size", "Base model", "Trained on", |
| "Architecture", "Decoder"] |
| |
| DTYPES = (["str", "markdown"] + ["str"] * 6 + ["markdown"] + ["str"] * 3) |
|
|
|
|
| def avec_kenlm(r: dict) -> bool: |
| """Cette ligne a-t-elle été produite avec fusion d'un n-gram kenlm ? |
| |
| La variante est marquée dans la révision (« <sha>:kenlm-a0.6-b100 ») |
| plutôt que par une colonne dédiée : c'est déjà l'identité exacte de la |
| configuration évaluée, et ça évite une migration de schéma. |
| |
| Trois écritures coexistent : « :kenlm-… » (soumissions FonBench), |
| « #kenlm… » (lignes importées du benchmark JML v2.1) et « + KenLM » dans |
| le nom d'un système JML v2.1. Même règle que `avecKenlm` du Space statique. |
| """ |
| return bool(_AVEC_LM.search(r.get("model_revision") or "")) |
|
|
|
|
| _AVEC_LM = re.compile(r"[:#]kenlm|\+\s*kenlm", re.IGNORECASE) |
|
|
|
|
| LM_VUES = {"All decoding": None, "With LM": True, "No LM": False} |
|
|
| _SUFFIXE_LM = re.compile(r":kenlm-[^:]*$") |
|
|
|
|
| def apparier(rows: list[dict], rank_by_wer: bool = False) -> list[dict]: |
| """Une ligne par modèle, portant les deux WER de décodage. |
| |
| Les configurations d'un même modèle ne diffèrent que par le suffixe de |
| révision (« …:kenlm-a0.6-b100 ») : on regroupe là-dessus. La ligne |
| affichée reste celle retenue au classement (non masquée) ; l'autre ne |
| sert qu'à renseigner la colonne de WER qui lui manque. |
| |
| Un groupe dont AUCUNE ligne n'est visible reste absent : c'est ce qui |
| permet d'écarter durablement un checkpoint sans perdre ses mesures. |
| """ |
| groupes: dict[tuple, list[dict]] = {} |
| for r in rows: |
| base = _SUFFIXE_LM.sub("", r.get("model_revision") or "") |
| groupes.setdefault((r["model_id"], base), []).append(r) |
|
|
| sortie = [] |
| for membres in groupes.values(): |
| visibles = [r for r in membres if not r.get("is_hidden")] |
| if not visibles: |
| continue |
| |
| |
| |
| |
| def rang(r): |
| |
| for cle in (("wer",) if rank_by_wer |
| else ("twer", "wer_notone", "wer")): |
| if r.get(cle) is not None: |
| return r[cle] |
| return 9e9 |
|
|
| canonique = min(visibles, key=rang) |
| ligne = dict(canonique) |
| glouton = next((r for r in membres if not avec_kenlm(r)), None) |
| avec = next((r for r in membres if avec_kenlm(r)), None) |
| ligne["wer_greedy"] = (glouton or canonique).get("wer") |
| ligne["wer_lm"] = avec.get("wer") if avec else None |
| ligne["a_variante_lm"] = avec is not None |
| sortie.append(ligne) |
| return sortie |
|
|
|
|
| def build_table(benchmark_id: str, vue: str, archs: list[str], |
| decoders: list[str], hide_contaminated: bool, sort_by: str, |
| lm: str = "All decoding"): |
| try: |
| bench = next((b for b in get_benchmarks() if b["id"] == benchmark_id), {}) |
| rank_by_wer = bench.get("ranking_metric") == "wer" |
| rows = apparier(get_results(benchmark_id), rank_by_wer) |
| except Exception as exc: |
| return (pd.DataFrame({"Error": [f"Database unreachable: {exc}"]}), |
| "", "", gr.update(), gr.update()) |
|
|
| all_archs = sorted({r["architecture"] for r in rows if r.get("architecture")}) |
| all_decs = sorted({r["decoder_type"] for r in rows if r.get("decoder_type")}) |
|
|
| sel = rows |
| vises = VUES.get(vue) |
| if vises: |
| sel = [r for r in sel if categorie(r) in vises] |
| if archs: |
| sel = [r for r in sel if r.get("architecture") in archs] |
| if decoders: |
| sel = [r for r in sel if r.get("decoder_type") in decoders] |
| vise_lm = LM_VUES.get(lm) |
| if vise_lm is not None: |
| sel = [r for r in sel if bool(r.get("a_variante_lm")) is vise_lm] |
| if hide_contaminated: |
| sel = [r for r in sel if not r.get("contamination_flag")] |
|
|
| |
| |
| |
| tonal = any(r.get("twer") is not None for r in rows) |
| metric = "wer" if rank_by_wer else ("twer" if tonal else "wer_notone") |
| if sort_by == "Model size": |
| sel = sorted(sel, key=lambda r: (r.get("model_params") or 0)) |
| else: |
| sel = sorted(sel, |
| key=lambda r: (r.get(metric) is None, r.get(metric) or 9e9)) |
|
|
| data = [] |
| for i, r in enumerate(sel, 1): |
| medaille = {1: "🥇", 2: "🥈", 3: "🥉"}.get(i, "") |
| nom = lien(r["model_id"]) |
| |
| |
| |
| rev = r.get("model_revision") or "" |
| sep = "#" if "#" in rev else ":" |
| ck = rev.split(sep, 1) |
| if len(ck) > 1 and ck[1]: |
| nom += f"<br><small>↳ {ck[1]}</small>" |
| if r.get("contamination_flag"): |
| nom += " ⚠️" |
| data.append({ |
| "#": f"{i} {medaille}".strip(), |
| "Model": nom, |
| "Type": TYPES[categorie(r)], |
| "WER": pct(r.get("wer")), |
| "WER (no tones)": pct(r.get("wer_notone")), |
| "CER": pct(r.get("cer")), |
| "CER (no tones)": pct(r.get("cer_notone")), |
| "Size": params_txt(r.get("model_params")), |
| "Base model": lien(r.get("base_model")), |
| "Trained on": train_txt(r), |
| "Architecture": r.get("architecture") or "—", |
| |
| |
| "Decoder": ((r.get("decoder_type") or "—") |
| + (" + kenlm" if avec_kenlm(r) else "")), |
| }) |
|
|
| compte = {c: sum(1 for r in sel if categorie(r) == c) |
| for c in ("maison", "tiers", "base")} |
| meta = ( |
| "<div class='fb-meta'>" |
| f"<span>{flag(13)} <b>{bench.get('name', benchmark_id)}</b></span>" |
| f"<span><b>{bench.get('num_utterances', '?')}</b> utterances</span>" |
| f"<span><b>{bench.get('duration_hours', '?')}</b> hours</span>" |
| + ("<span class='fb-badge'>private test set</span>" |
| if bench.get("is_private") else "") |
| + f"<span>ranked by <b>{'WER' if rank_by_wer else ('T-WER' if tonal else 'WER_seg')}</b></span>" |
| + f"<span><b>{len(sel)}</b> of {len(rows)} models shown</span>" |
| |
| |
| + ("<span>" |
| f"🔧 <b>{compte['maison']}</b> fine-tuned by FonBench" |
| "<span class='fb-sep'> · </span>" |
| f"👥 <b>{compte['tiers']}</b> fine-tuned by the community" |
| "<span class='fb-sep'> · </span>" |
| f"🌍 <b>{compte['base']}</b> general-purpose" |
| "</span>") |
| + "</div>" |
| ) |
| legende = ( |
| "All error rates in **%**, lower is better. **⚠️** marks a model " |
| "likely trained on this test set: its score is not comparable." |
| ) |
| return (pd.DataFrame(data, columns=COLONNES), meta, legende, |
| gr.update(choices=all_archs), gr.update(choices=all_decs)) |
|
|
|
|
| NOTE_METRIQUES = """ |
| ### Reading the table |
| |
| Fon is a **tonal** language: tones are written with diacritics (á, ɔ́, ě…) |
| and change the meaning of words. But Fon corpora don't follow the same |
| convention — some mark no tone at all. A raw WER therefore isn't comparable |
| from one corpus to the next. Hence two views of the same errors. |
| |
| | Metric | What it tells you | |
| |---|---| |
| | **WER** | Word error rate, **tones included**: a wrong tone makes the word wrong. | |
| | **WER (no tones)** | The same word errors after **stripping tone marks**. Measures phonetic accuracy and stays comparable across every corpus. | |
| | **CER** | Character error rate, tones included. | |
| | **CER (no tones)** | The same character errors after stripping tone marks. | |
| |
| **Model types.** 🌍 *General-purpose* models were not built for Fon: some |
| are multilingual systems that do cover it among a thousand other languages |
| (the MMS family ships a Fon adapter), others are multilingual without Fon, |
| and others again are monolingual systems for English or French. They are not |
| *base* models in the pretrained sense — `wav2vec2-large-960h-lv60-self` is a |
| finished English recogniser, not a starting point. The **Trained on** column |
| says exactly what each one saw. 🔧 *FonBench* and 👥 *Community* models were |
| fine-tuned on Fon; comparing them against a general-purpose model that shares |
| their base is what actually measures the value of fine-tuning. |
| |
| Truly *base* models — raw pretrained checkpoints such as |
| `facebook/wav2vec2-large-xlsr-53` — cannot appear here at all: without a CTC |
| head or a vocabulary they transcribe nothing. They show up only in the |
| **Base model** column, as the starting point of the models that were |
| fine-tuned from them. |
| |
| """ |
|
|
|
|
| |
|
|
| def build_queue(): |
| try: |
| rows = fetch("public_queue", {"select": "*", "order": "created_at.desc", |
| "limit": "50"}, ttl=15) |
| except Exception as exc: |
| return pd.DataFrame({"Error": [f"Database unreachable: {exc}"]}), "" |
|
|
| etats = {"pending": "⏳ pending", "running": "⚙️ running", |
| "done": "✅ done", "failed": "❌ failed", |
| "rejected": "🚫 rejected"} |
| data = [] |
| for r in rows: |
| total = r.get("progress_total") or 0 |
| done = r.get("progress_done") or 0 |
| avance = f"{done}/{total}" if total else "—" |
| if total and r["status"] == "running": |
| avance += f" ({done * 100 // total}%)" |
| data.append({ |
| "Model": r["model_id"], |
| "Status": etats.get(r["status"], r["status"]), |
| "Progress": avance, |
| "Details": (r.get("error_message") or "")[:120], |
| "Submitted": (r.get("created_at") or "")[:10], |
| }) |
|
|
| etat = evaluator.status() |
| ligne = f"**Evaluator:** {etat['message']}" |
| if etat.get("model"): |
| ligne += f" — {etat['model']} ({etat['done']}/{etat['total']})" |
| return pd.DataFrame(data), ligne |
|
|
|
|
| |
|
|
| def submit(model_id: str, hf_username: str, contact: str, note: str, |
| benchmark_id: str, train_data: str, train_hours, base_model: str): |
| if not benchmark_id: |
| return ("### Submissions are closed\n" |
| "No test set currently accepts submissions: the active " |
| "benchmarks are evaluated by the JML team with their own protocol.") |
| model_id = (model_id or "").strip() |
| if "/" not in model_id or len(model_id.split("/")) != 2 or \ |
| not all(model_id.split("/")): |
| return ("### ❌ Invalid format\n" |
| "The identifier must look like `organisation/name`, exactly as " |
| "it appears in the model's URL on Hugging Face.") |
|
|
| payload = {"model_id": model_id, "benchmark_id": benchmark_id} |
| for champ, valeur in (("hf_username", hf_username), ("contact", contact), |
| ("note", note), ("train_data", train_data), |
| ("base_model", base_model)): |
| if (valeur or "").strip(): |
| payload[champ] = valeur.strip() |
| if train_hours: |
| payload["train_hours"] = float(train_hours) |
|
|
| try: |
| r = requests.post( |
| f"{REST}/public_requests", |
| headers={**HEADERS, "Content-Type": "application/json", |
| "Prefer": "return=minimal"}, |
| json=payload, timeout=30, |
| ) |
| except Exception as exc: |
| return f"### ❌ Could not submit\nDatabase unreachable: {exc}" |
|
|
| if r.status_code >= 400: |
| |
| |
| try: |
| detail = r.json().get("message") or r.text |
| except Exception: |
| detail = r.text |
| return f"### ❌ Submission rejected\n{detail}" |
|
|
| _cache.clear() |
| return ( |
| f"### ✅ `{model_id}` is queued\n\n" |
| "Evaluation runs in slices on the Space's shared GPU. Depending on the " |
| "available quota, expect anywhere from a few minutes to a few hours. " |
| "The score will appear in the leaderboard once computed — follow " |
| "progress in the **Queue** tab." |
| ) |
|
|
|
|
| FINE_TUNING = """ |
| ## The FonBench fine-tuning runs |
| |
| > These four runs and their scores belong to the former test set (JML test v1.0, |
| > 2,555 utterances). They are not comparable with the ranking of the current |
| > benchmark. |
| |
| Four pretrained models were fine-tuned on Fon under strictly identical |
| conditions, so that any gap between them comes from the starting model |
| alone. |
| |
| ### The data |
| |
| The training corpus holds **44,225 utterances**, of which **13,716 were |
| kept — 30.00 hours from 471 speakers**. The sample is not drawn at random: |
| it is built by taking turns across speakers, each contributing a little |
| before any one of them dominates. For generalisation, the diversity of |
| voices matters more than raw volume. |
| |
| The 471 training speakers and the 45 test speakers are **strictly |
| disjoint** — verified, zero in common. A temporal cutoff separates the |
| transcripts; 20 sentences out of 2,555 (0.8%) do appear in both, too few to |
| move a score but worth stating rather than claiming a perfect separation. |
| |
| ### The protocol |
| |
| Identical across all four: **3 epochs**, i.e. 2,574 steps with 257 of |
| warmup. CTC decoding, character vocabulary built from the training corpus, |
| feature extractor taken from the base model. A single 24 GB L4, `fp16`, |
| gradient checkpointing. |
| |
| | Resulting model | Base | Learning rate | Batch | Wall time | Final dev loss | WER_seg | |
| |---|---|---|---|---|---|---| |
| | `wav2vec2-large-xlsr-53-fon-30h` | facebook/wav2vec2-large-xlsr-53 | 3·10⁻⁴ | 8×2 | 54 min | **0.468** | **38.6%** | |
| | `mms-300m-fon-30h` | facebook/mms-300m | 3·10⁻⁴ | 8×2 | not retained | not retained | **42.8%** | |
| | `w2v-bert-2.0-fon-30h` | facebook/w2v-bert-2.0 | 3·10⁻⁵ | 4×4 | 116 min | 3.102 | 96.9% | |
| | `AfriHuBERT-fon-30h` | ajesujoba/AfriHuBERT | 3·10⁻⁵ | 8×2 | 24 min | 3.078 | 100.0% | |
| |
| ### What this shows |
| |
| **Fine-tuning makes the difference, not the base.** `chrisjay/fonxlsr` |
| starts from exactly the same model as our best run — |
| `facebook/wav2vec2-large-xlsr-53`, confirmed in its configuration — and |
| reaches 69.3% where we reach 38.6%. The error is cut by a factor of 1.8 |
| with 30 hours of well-chosen data. |
| |
| **Two runs out of four failed**, and that deserves saying plainly. |
| `w2v-bert-2.0` and `AfriHuBERT` do learn: their loss drops clearly during |
| training. But it plateaus around 3.1 against 0.47 for the run that |
| succeeds — the signature of settling into CTC's trivial solution, emitting |
| the blank symbol everywhere. This is not a learning-rate problem: both were |
| first run at 3·10⁻⁴, then rerun at 3·10⁻⁵ — the figures in the table — for |
| exactly the same outcome. They would need far more epochs, or more data. |
| |
| The `mms-300m` run predates per-model logging, so its wall time and final |
| loss are genuinely lost. They are shown as *not retained* rather than |
| reconstructed from memory. |
| |
| Both failures stay in the leaderboard. Removing them would paint a |
| flattering and false picture of what fine-tuning guarantees. |
| |
| ### Reproducing |
| |
| `train_ctc.py` is published in the **Files** tab of this Space, under |
| `repro/`, together with the evaluation script and the scoring code. See the |
| **Reproduce** tab. |
| """ |
|
|
|
|
| REPRODUIRE = """ |
| ## Reproduce every number |
| |
| A leaderboard nobody can check is just a poster. The scoring code of the |
| current benchmark is published in the **Files** tab of this Space, so any row |
| can be recomputed from a model's outputs. |
| |
| | File | What it is | |
| |---|---| |
| | `jml_v21/texte.py`, `jml_v21/metriques.py` | The scoring code of **JML Fon v2.1**: text normalisation, WER and CER with and without tones. Copied **unchanged** from the JML reference code (`benchmark-jml-v2.1/code/`); `jml_v21/SOURCE.txt` gives the commit and the SHA-256 of each file. | |
| | `evaluator.py` | The code that evaluates each submission on this Space. | |
| | `repro/` | Standalone scripts of the previous protocol (see below). | |
| |
| ### Recomputing a score |
| |
| ``` |
| cd jml_v21 |
| python -c "import metriques as m; print(m.evaluer(['mǐ ná yi'], ['mi ná yi'], tons=True))" |
| ``` |
| |
| This prints WER 33.3 and CER 10.0 (with tones, one word out of three is wrong). |
| With `tons=False` both are 0.0. Errors are summed over the whole corpus (not |
| averaged per utterance), after both texts are normalised the same way: lower |
| case, punctuation and digits removed. A tone mark counts as a character. The |
| test set, `JMLdata/fon-test-v2-1` (revision |
| `7f247395d423177a0ba89803a8b033e3f07fc4cc`), is private: its owner can grant |
| access, and its `transcription` column holds the references. |
| |
| **Checked:** applied to the published outputs of four models, this code |
| returns exactly the scores that JML imported (WER, WER without tones, CER and |
| CER without tones). |
| |
| ### The previous protocol |
| |
| The scripts in `repro/` and their reference values belong to the former test |
| set (JML test v1.0, 2,555 utterances), which is no longer ranked. They use |
| FonBench's own text normalisation, which is **not** the one used by the |
| current benchmark: on the same outputs of four models, the two scorers differ |
| by up to 6.5 points of CER. Do not compare their figures with the current |
| leaderboard. |
| |
| RTFx is a hardware measurement and only reproduces on the same GPU; the |
| hardware is recorded with every score. |
| |
| If you get a materially different number, tell us. That is the point of |
| publishing this. |
| """ |
|
|
|
|
| A_PROPOS = """ |
| ## FonBench |
| |
| The first public speech-recognition leaderboard for **Fon**, a tonal |
| language spoken by over two million people in Benin. The project answers |
| a concrete difficulty: until now, published Fon results were not comparable |
| with one another, for lack of a shared protocol and a shared test set. |
| |
| ### Why the test set is not published |
| |
| Public Fon corpora have been circulating since 2016, and several released |
| models were trained on them. Their WER on that data then looks remarkable — |
| while saying nothing about their real ability to transcribe an unseen |
| voice. We measured it: one of the test sets we were using shared **83% of |
| its utterances** with the training split of a public dataset. |
| |
| The test set of the current benchmark, **JML Fon v2.1**, holds **6,382 |
| utterances, 13.97 hours and 60 speakers**. It is therefore not distributed: |
| only aggregate scores are made public, and nobody — not even through this |
| Space — can download its audio or its transcriptions. Its clips, speakers and |
| sentences are disjoint from the v2.1 training data (checked on the published |
| identifier lists); 259 of its clips come from the former official test, none |
| of them in the v2.1 training data. A model whose training predates v2.1 may |
| still have heard some of its voices: such rows carry a **⚠ contamination** |
| mark, and their scores are not comparable. |
| |
| ### The protocol |
| |
| - **Pinned revision.** Every score is tied to the exact commit hash of the |
| repository evaluated. Republishing a model under the same name does not |
| change a score already obtained. |
| - **One scorer for every row.** WER and CER, with and without tones, come from |
| the JML reference code (`jml_v21/` in the Files tab, copied unchanged). The |
| rows imported from JML and the rows evaluated here are scored by the same |
| code. The ranking is on the **WER with tones**. |
| - **No arbitrary code, with two documented exceptions.** Models are loaded |
| with `trust_remote_code=False`, so custom code shipped in a repository is |
| never executed. The exceptions are pinned to a revision that was reviewed |
| before being allowed: `asrDIL/fonglab-asr-w2vbert`, whose own decoding |
| package is executed (syllable units, language model and rescoring), and two |
| private `Fongbe-ASR` repositories in standard transformers format, read with |
| an access token and run without any repository code. |
| - **Where a row comes from.** Evaluated on this Space; imported from JML |
| (computed by JML with its own pipeline); or outputs supplied by the authors |
| and scored here (stated in the row's revision text, as for |
| `Fongbe-ASR/DePal-asr-fon-omni-v1`). |
| - **No duplicate work.** A (model, revision, corpus) triple is never |
| re-evaluated: the existing score is reused. |
| |
| ### Submitting a model |
| |
| The repository must be **public** on the Hub. CTC models are supported |
| (wav2vec2, wav2vec2-BERT, HuBERT, MMS); a run takes about 3 to 10 minutes of |
| GPU. Whisper and other encoder-decoder models **cannot be evaluated on this |
| Space at the moment** (a GPU-allocation error, fix pending). No account |
| required. |
| """ |
|
|
|
|
| |
|
|
| def build_ui() -> gr.Blocks: |
| try: |
| benchs = get_benchmarks() |
| except Exception: |
| benchs = [] |
| choix = [(f"{b['name']} ({b.get('num_utterances', '?')} utterances)", |
| b["id"]) for b in benchs] |
| |
| |
| choix_soumission = [(f"{b['name']} ({b.get('num_utterances', '?')} utterances)", |
| b["id"]) for b in benchs |
| if b.get("accepts_submissions") is not False] |
| |
| |
| defaut_classement = "jml-v2-1" if any(b["id"] == "jml-v2-1" for b in benchs) else None |
| defaut = "jml-test-v1" if any(b["id"] == "jml-test-v1" for b in benchs) \ |
| else (benchs[0]["id"] if benchs else "") |
| defaut_classement = defaut_classement or defaut |
|
|
| |
| with gr.Blocks(title="FonBench — Fon ASR leaderboard") as demo: |
| gr.HTML( |
| "<div class='fb-head'>" |
| "<p class='fb-title'>Fon<span>Bench</span> " |
| f"{flag(26)}</p>" |
| "<p class='fb-sub'>The public speech-recognition leaderboard for " |
| "Fon, a tonal language of Benin</p>" |
| "</div>" |
| ) |
|
|
| with gr.Tabs(): |
| with gr.Tab("Leaderboard"): |
| |
| |
| |
| with gr.Column(elem_classes="fb-filters"): |
| with gr.Row(equal_height=True): |
| with gr.Column(scale=5, min_width=190): |
| b_sel = gr.Dropdown(choix, value=defaut_classement, |
| label="Test set") |
| with gr.Column(scale=3, min_width=160): |
| tri = gr.Dropdown( |
| ["Quality (ranking metric)", "Model size"], |
| value="Quality (ranking metric)", |
| label="Sort by") |
| with gr.Column(scale=3, min_width=160): |
| f_arch = gr.Dropdown([], multiselect=True, |
| label="Architecture") |
| with gr.Column(scale=3, min_width=160): |
| f_dec = gr.Dropdown([], multiselect=True, |
| label="Decoder") |
| with gr.Row(elem_classes="fb-row2"): |
| f_type = gr.Radio(list(VUES), value="All models", |
| show_label=False, container=False, |
| elem_classes="fb-views", scale=6) |
| f_lm = gr.Radio(list(LM_VUES), value="All decoding", |
| show_label=False, container=False, |
| elem_classes="fb-views", scale=3) |
| f_cont = gr.Checkbox(label="Hide contaminated", |
| container=False, scale=2) |
| rafraichir = gr.Button("Refresh", size="sm", scale=1) |
|
|
| meta = gr.HTML() |
| legende = gr.Markdown(elem_classes="fb-legend") |
| table = gr.Dataframe(interactive=False, wrap=False, |
| datatype=DTYPES, elem_classes="fb-table") |
| gr.Markdown(NOTE_METRIQUES, elem_classes="fb-note") |
|
|
| entrees = [b_sel, f_type, f_arch, f_dec, f_cont, tri, f_lm] |
| sorties = [table, meta, legende, f_arch, f_dec] |
| for widget in (b_sel, tri, f_type, f_arch, f_dec, f_cont, f_lm): |
| widget.change(build_table, entrees, sorties) |
| rafraichir.click(lambda: _cache.clear(), None, None).then( |
| build_table, entrees, sorties) |
| demo.load(build_table, entrees, sorties) |
|
|
| with gr.Tab("Queue"): |
| etat_line = gr.Markdown() |
| q_table = gr.Dataframe(interactive=False, wrap=True, |
| elem_classes="fb-table") |
| gr.Button("Refresh").click(build_queue, None, |
| [q_table, etat_line]) |
| gr.Markdown( |
| "Evaluation runs in slices on the Space's shared GPU. The " |
| "compute itself is fast — a few minutes for a CTC model — " |
| "but the daily GPU quota is limited, so a run may spread " |
| "over several hours, pausing between slices. An " |
| "interrupted run resumes exactly where it stopped.", |
| elem_classes="fb-note") |
| demo.load(build_queue, None, [q_table, etat_line]) |
|
|
| with gr.Tab("Submit a model"): |
| gr.Markdown( |
| "The repository must be **public** and must not rely on " |
| "custom code: `trust_remote_code` is disabled. Both " |
| "`safetensors` and `.bin` weights are accepted. CTC models " |
| "only for now: Whisper and other encoder-decoder models " |
| "currently fail on this Space (a GPU-allocation error). " |
| "No account required.", elem_classes="fb-note") |
| s_model = gr.Textbox(label="Hugging Face model ID", |
| placeholder="organisation/model-name") |
| with gr.Row(): |
| s_user = gr.Textbox(label="Your HF username (optional)") |
| s_contact = gr.Textbox(label="Contact (optional)") |
| with gr.Row(): |
| s_base = gr.Textbox( |
| label="Base model (optional)", |
| placeholder="facebook/wav2vec2-large-xlsr-53") |
| s_data = gr.Textbox( |
| label="Training data (optional)", |
| placeholder="ALFFA, Zenodo, private corpus…", |
| info="Leave empty for a model never fine-tuned on Fon.") |
| s_hours = gr.Number(label="Hours of Fon audio (optional)", |
| precision=1, minimum=0) |
| s_note = gr.Textbox(label="Note (optional)", lines=2) |
| s_bench = gr.Dropdown(choix_soumission, |
| value=(choix_soumission[0][1] if choix_soumission else None), |
| label="Test set") |
| s_out = gr.Markdown() |
| gr.Button("Submit", variant="primary").click( |
| submit, |
| [s_model, s_user, s_contact, s_note, s_bench, |
| s_data, s_hours, s_base], |
| s_out) |
|
|
| with gr.Tab("Fine-tuning"): |
| gr.Markdown(FINE_TUNING, elem_classes="fb-note") |
|
|
| with gr.Tab("Reproduce"): |
| gr.Markdown(REPRODUIRE, elem_classes="fb-note") |
|
|
| with gr.Tab("About"): |
| gr.Markdown(A_PROPOS, elem_classes="fb-note") |
|
|
| return demo |
|
|
|
|
| |
| |
| |
| THEME = gr.themes.Soft(primary_hue="indigo", secondary_hue="amber").set( |
| block_label_background_fill="transparent", |
| block_label_background_fill_dark="transparent", |
| block_label_border_width="0px", |
| block_label_text_color="*neutral_500", |
| block_label_text_color_dark="*neutral_400", |
| block_label_text_size="*text_xs", |
| block_label_text_weight="600", |
| block_background_fill="transparent", |
| block_background_fill_dark="transparent", |
| block_border_width="0px", |
| block_shadow="none", |
| panel_background_fill="transparent", |
| panel_background_fill_dark="transparent", |
| panel_border_width="0px", |
| form_gap_width="0px", |
| input_background_fill="*neutral_50", |
| input_background_fill_dark="*neutral_800", |
| input_border_width="1px", |
| input_border_color="*neutral_200", |
| input_border_color_dark="*neutral_700", |
| ) |
|
|
| |
| |
| evaluator.start() |
| demo = build_ui() |
|
|
| if __name__ == "__main__": |
| demo.launch(css=CSS, theme=THEME) |
|
|
