| """Score both shipped prompts on ONE sentence set, so the card's table is real. |
| |
| The two voices were confirmed in separate runs against different held-out sets |
| -- male on the v4 corpus holdout, female on the v3 one. Each of those runs was |
| internally sound: the new prompt and the one it replaced were paired on the same |
| sentences, which is what makes +0.728 and +0.527 trustworthy. But the card shows |
| the two voices side by side, and that comparison was never measured. |
| |
| So this re-scores both against the same 120 held-out sentences, three seeds, |
| n=360 each. Nothing else changes: same checkpoint, same sampler constants, same |
| prompts that ship today. The output is a table whose columns can honestly be |
| read against one another. |
| |
| Sentences come from the preserved v3 text holdout, withheld from the corpus the |
| model trained on, so they measure intelligibility rather than recall. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import json |
| import pathlib |
| import random |
| import statistics |
| import sys |
|
|
| REPO = pathlib.Path("/workspace/oron-tts") |
| sys.path.insert(0, str(REPO)) |
| sys.path.insert(0, str(REPO / "scripts")) |
|
|
| SEEDS = (0, 1, 2) |
| N_SENTENCES = 120 |
| HUMAN_CER_BASELINE = 0.123 |
|
|
|
|
| def cer_counts(ref, hyp): |
| r, h = list(ref), list(hyp) |
| if not r: |
| return (len(h), 0) |
| prev = list(range(len(h) + 1)) |
| for i, rc in enumerate(r, 1): |
| cur = [i] |
| for j, hc in enumerate(h, 1): |
| cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (rc != hc))) |
| prev = cur |
| return prev[-1], len(r) |
|
|
|
|
| def micro_cer(c): |
| e = sum(a for a, _ in c) |
| n = sum(b for _, b in c) |
| return e / n if n else 0.0 |
|
|
|
|
| def boot(vals, stat, resamples=2000, seed=0): |
| items = list(vals) |
| if len(items) < 2: |
| v = stat(items) if items else float("nan") |
| return v, v, v |
| rng = random.Random(seed) |
| n = len(items) |
| d = sorted(stat([items[rng.randrange(n)] for _ in range(n)]) for _ in range(resamples)) |
| return stat(items), d[int(0.025 * resamples)], d[int(0.975 * resamples)] |
|
|
|
|
| def paired(a, b, stat, resamples=2000, seed=0): |
| xs, ys = list(a), list(b) |
| n = min(len(xs), len(ys)) |
| if n < 2: |
| return None |
| xs, ys = xs[:n], ys[:n] |
| rng = random.Random(seed) |
| d = [] |
| for _ in range(resamples): |
| idx = [rng.randrange(n) for _ in range(n)] |
| d.append(stat([xs[i] for i in idx]) - stat([ys[i] for i in idx])) |
| d.sort() |
| lo, hi = d[int(0.025 * resamples)], d[int(0.975 * resamples)] |
| return {"diff": stat(xs) - stat(ys), "lo": lo, "hi": hi, |
| "separated": lo > 0 or hi < 0, "n": n} |
|
|
|
|
| def main() -> None: |
| import os |
|
|
| from f5_tts.api import F5TTS |
| from huggingface_hub import HfApi, hf_hub_download |
|
|
| from eval_mn import synthesise |
| from oron_tts.eval import MongolianASR, bandwidth_hz, utmos |
| from oron_tts.eval.metrics import normalize_for_scoring |
|
|
| lines = [ln.strip() for ln in pathlib.Path( |
| hf_hub_download("btsee/oron-tts", "eval/cv_eval_sentences.txt") |
| ).read_text(encoding="utf-8").splitlines() if ln.strip()] |
| sents = lines[1::2][:N_SENTENCES] |
| print(f"one common set: {len(sents)} sentences x {len(SEEDS)} seeds " |
| f"= {len(sents) * len(SEEDS)} per voice") |
|
|
| ckpt = hf_hub_download("btsee/oron-tts", "model.safetensors") |
| vocab = hf_hub_download("btsee/oron-tts", "vocab.txt") |
| asr = MongolianASR(device="cuda") |
|
|
| out = {} |
| for gender in ("male", "female"): |
| wav = pathlib.Path(hf_hub_download("btsee/oron-tts", f"voices/{gender}.wav", |
| force_download=True)) |
| rtxt = pathlib.Path(hf_hub_download("btsee/oron-tts", f"voices/{gender}.txt", |
| force_download=True) |
| ).read_text(encoding="utf-8").strip() |
| f5 = F5TTS(model="F5TTS_v1_Base", ckpt_file=ckpt, vocab_file=vocab, |
| device="cuda", use_ema=False) |
| counts, moses, bws = [], [], [] |
| for k, text in enumerate(sents): |
| for seed in SEEDS: |
| try: |
| w, sr = synthesise(f5, text, wav, rtxt, seed) |
| except Exception as exc: |
| print(f" synth failed: {str(exc)[:70]}") |
| continue |
| counts.append(cer_counts(normalize_for_scoring(text), |
| normalize_for_scoring(asr.transcribe(w, sr)))) |
| bws.append(bandwidth_hz(w, sr)) |
| try: |
| moses.append(utmos(w, sr)) |
| except Exception: |
| pass |
| if k % 30 == 0: |
| print(f" {gender} {k}/{len(sents)}", flush=True) |
| cp, clo, chi = boot(counts, micro_cer) |
| up, ulo, uhi = boot(moses, statistics.fmean) |
| out[gender] = {"cer_micro": cp, "cer_lo": clo, "cer_hi": chi, |
| "utmos": up, "utmos_lo": ulo, "utmos_hi": uhi, |
| "utmos_values": moses, "n": len(counts), |
| "bandwidth_median": statistics.median(bws) if bws else None} |
| print(f" {gender:<7} UTMOS {up:.3f} [{ulo:.3f}, {uhi:.3f}] " |
| f"CER {cp:.4f} [{clo:.4f}, {chi:.4f}] n={len(counts)}") |
| del f5 |
| import torch |
| torch.cuda.empty_cache() |
|
|
| |
| d = paired(out["male"]["utmos_values"], out["female"]["utmos_values"], |
| statistics.fmean) |
| verdict = ("separated" if d["separated"] |
| else "NOT separated -- the two voices score alike on this set") |
| print(f"\n male - female UTMOS {d['diff']:+.3f} [{d['lo']:+.3f}, {d['hi']:+.3f}] " |
| f"n={d['n']} {verdict}") |
| out["_male_vs_female"] = d |
|
|
| payload = {"shipped-v4-common-set": { |
| g: {k: v for k, v in out[g].items() if k != "utmos_values"} |
| for g in ("male", "female")}, "male_vs_female": d, |
| "sentences": len(sents), "seeds": list(SEEDS)} |
| p = pathlib.Path("/workspace/common_eval.json") |
| p.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") |
| HfApi(token=os.environ["HF_TOKEN"]).upload_file( |
| path_or_fileobj=str(p), path_in_repo="eval/common_eval.json", |
| repo_id="btsee/oron-tts", |
| commit_message="Both prompts on one sentence set, so the columns compare") |
| print("published eval/common_eval.json") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
