"""Pick the Space's example clips from material Routyo never trained on, and build two mixed conversations. python -I tools/pick_examples.py Synthetic: the test split only (voices held out, everyday sentences from the 10 test-only situations), gate passed. Real: FLEURS (CC BY 4.0) and VoxPopuli (CC0) test clips for en/fr/es. Each candidate must be called right by Routyo with the word check; the first that is wins (fixed seed order). Writes examples/*.wav (16 kHz mono) and examples/examples.json. """ import json import os import random import sys import numpy as np import soundfile as sf sys.path.insert(0, sys.argv[1]) from routyo import Routyo, load_audio # noqa: E402 DATA = os.path.expanduser("~/Documents/Youtube-Ht-Data/data/langid") WORLD = os.path.join(DATA, "world") OUT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") os.makedirs(OUT, exist_ok=True) LOCALE = {"en-us": "American", "en-gb": "British", "fr-fr": "French", "es-es": "Spanish", "es-mx": "Mexican", "de-de": "German", "pt-br": "Brazilian"} voices = json.load(open(os.path.join(WORLD, "voices.json"))) catalogue = {v["id"]: v for v in voices["catalogue"]} gate = {} for line in open(os.path.join(WORLD, "gate.jsonl")): g = json.loads(line) gate[g["id"]] = g["cer"] made = [] for name in sorted(os.listdir(WORLD)): if name.startswith("made_") and "convo" in name: for line in open(os.path.join(WORLD, name)): m = json.loads(line) if m["split"] == "test" and gate.get(m["id"], 1) <= 0.35 and 1.8 <= m["duration"] <= 7: made.append(m) situation = {} for line in open(os.path.join(WORLD, "convo.jsonl")): c = json.loads(line) if c["split"] == "test": for k in ("ht", "en", "fr", "es"): situation[(k, c[k])] = c["situation"] real = [json.loads(line) for line in open(os.path.join(DATA, "clips.jsonl"))] real = [r for r in real if r["split"] == "test" and r["source"] in ("fleurs", "voxpopuli_en_accented") and 3 <= r["duration"] <= 9] print(len(made), "synthetic candidates,", len(real), "real") r = Routyo() rng = random.Random(7) def voice_of(m): if m["kind"] == "fatiman_h": return "Haitian voice" v = catalogue.get(m["voice"], {}) return f"{LOCALE.get(v.get('locale'), 'foreign')} voice" def take(pool, lang, label, note, source): pool = list(pool) rng.shuffle(pool) for m in pool: x = load_audio(m["path"]) rates = r.rates(x, words=True) if rates[0]["language"] == lang: label = label.replace("{voice}", voice_of(m)) if "kind" in m else label name = f"{len(picked):02d}_{lang}_{source}.wav" sf.write(os.path.join(OUT, name), x, 16000, subtype="PCM_16") picked.append({"file": name, "language": lang, "label": label, "note": note, "text": m.get("text", ""), "source": source, "seconds": round(len(x) / 16000, 2), "rate": rates[0]["rate"]}) print(f" {label:45} {rates[0]['language']} {rates[0]['rate']:.3f} {m.get('text', '')[:60]}") return m print(" none for", label) def syn(kind, lang, locale=None, gender=None): out = [] for m in made: if m["kind"] != kind or m["language"] != lang: continue if locale and catalogue.get(m["voice"], {}).get("locale") not in locale: continue out.append({**m, "path": os.path.join(DATA, m["audio"])}) return out def rl(source, lang): return [{**x, "path": os.path.join(DATA, x["audio"]) if not os.path.isabs(x["audio"]) else x["audio"]} for x in real if x["source"] == source and x["language"] == lang] picked = [] SYN = "synthetic, voice never heard in training" # Kreyòl take(syn("fatiman_h", "ht"), "ht", "Kreyòl · Haitian voice", SYN, "fatiman") take(syn("fatiman_c", "ht", {"en-us", "en-gb"}), "ht", "Kreyòl · {voice}", SYN, "fatiman-clone") take(syn("fatiman_c", "ht", {"fr-fr"}), "ht", "Kreyòl · {voice}", SYN, "fatiman-clone") take(syn("fatiman_c", "ht", {"es-mx", "es-es"}), "ht", "Kreyòl · {voice}", SYN, "fatiman-clone") take(syn("phonon", "ht"), "ht", "Kreyòl · {voice} (another engine)", SYN, "phonon") # English take(syn("fatiman_h", "en"), "en", "English · Haitian voice", SYN, "fatiman") take(syn("phonon", "en"), "en", "English · {voice}", SYN, "phonon") take(syn("fatiman_c", "en", {"de-de", "pt-br"}), "en", "English · {voice}", SYN, "fatiman-clone") take(rl("fleurs", "en"), "en", "English · real speaker (FLEURS)", "real recording, FLEURS (CC BY 4.0)", "fleurs") take(rl("voxpopuli_en_accented", "en"), "en", "English · real non-native speaker (VoxPopuli)", "real recording, VoxPopuli (CC0)", "voxpopuli") # French take(syn("fatiman_h", "fr"), "fr", "Français · Haitian voice", SYN, "fatiman") take(syn("phonon", "fr"), "fr", "Français · {voice}", SYN, "phonon") take(syn("fatiman_c", "fr", {"en-us", "en-gb"}), "fr", "Français · {voice}", SYN, "fatiman-clone") take(rl("fleurs", "fr"), "fr", "Français · real speaker (FLEURS)", "real recording, FLEURS (CC BY 4.0)", "fleurs") # Spanish take(syn("fatiman_h", "es"), "es", "Español · Haitian voice", SYN, "fatiman") take(syn("phonon", "es"), "es", "Español · {voice}", SYN, "phonon") take(syn("fatiman_c", "es", {"fr-fr"}), "es", "Español · {voice}", SYN, "fatiman-clone") take(rl("fleurs", "es"), "es", "Español · real speaker (FLEURS)", "real recording, FLEURS (CC BY 4.0)", "fleurs") # Conversations: lines of one test situation, glued with pauses; every line must also be routed right def right(m): return r.rates(load_audio(os.path.join(DATA, m["audio"])), words=True)[0]["language"] == m["language"] def first_right(c): c = list(c) rng.shuffle(c) return next((m for m in c if right(m)), None) def glue(lines, name, title): gap = np.zeros(int(1.5 * 16000), np.float32) parts, script = [], [] for m in lines: x = load_audio(os.path.join(DATA, m["audio"])) x = x / (np.abs(x).max() + 1e-6) * 0.8 parts += [x, gap] script.append({"language": m["language"], "text": m["text"], "voice": voice_of(m)}) y = np.concatenate(parts[:-1]).astype(np.float32) sf.write(os.path.join(OUT, name), y, 16000, subtype="PCM_16") convos.append({"file": name, "title": title, "lines": script, "seconds": round(len(y) / 16000, 1)}) print(" ", title, [s["language"] for s in script]) convos = [] by_sit = {} for m in made: s = situation.get((m["language"], m["text"])) if s: by_sit.setdefault(s, []).append(m) # 1: many voices, languages alternating, one situation order = ["ht", "en", "fr", "ht", "es", "en", "ht"] for sit in sorted(by_sit): pool, lines, used = by_sit[sit], [], set() for k in order: m = first_right(m for m in pool if m["language"] == k and m["text"] not in used) if m is None: break used.add(m["text"]) lines.append(m) if len(lines) == len(order): glue(lines, "convo_many_voices.wav", f"Many voices, four languages ({sit})") break # 2: one Haitian voice switching language every sentence haitian = {} for m in made: if m["kind"] == "fatiman_h": haitian.setdefault(m["voice"], set()).add(m["language"]) for voice in sorted(haitian, key=lambda v: -sum(m["voice"] == v for m in made)): pool = [m for m in made if m["kind"] == "fatiman_h" and m["voice"] == voice] lines = [] for k in ["ht", "en", "fr", "es", "ht", "fr"]: m = first_right(m for m in pool if m["language"] == k and m not in lines) if m is None: break lines.append(m) if len(lines) == 6: glue(lines, "convo_one_voice.wav", "One Haitian voice switching languages") break json.dump({"examples": picked, "conversations": convos}, open(os.path.join(OUT, "examples.json"), "w"), indent=1, ensure_ascii=False) print(len(picked), "examples,", len(convos), "conversations ->", OUT)