Routyo / tools /pick_examples.py
jsbeaudry's picture
Routyo demo: one sentence or a whole conversation, 18 examples in four languages
c6eaeea verified
Raw History Blame Contribute Delete
8.04 kB
"""Pick the Space's example clips from material Routyo never trained on, and build two mixed conversations.
python -I tools/pick_examples.py <routyo package folder>
Synthetic: the test split only (voices held out, everyday sentences from the 10 test-only situations), gate passed.
Real: FLEURS (CC BY 4.0) and VoxPopuli (CC0) test clips for en/fr/es.
Each candidate must be called right by Routyo with the word check; the first that is wins (fixed seed order).
Writes examples/*.wav (16 kHz mono) and examples/examples.json.
"""
import json
import os
import random
import sys
import numpy as np
import soundfile as sf
sys.path.insert(0, sys.argv[1])
from routyo import Routyo, load_audio # noqa: E402
DATA = os.path.expanduser("~/Documents/Youtube-Ht-Data/data/langid")
WORLD = os.path.join(DATA, "world")
OUT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples")
os.makedirs(OUT, exist_ok=True)
LOCALE = {"en-us": "American", "en-gb": "British", "fr-fr": "French", "es-es": "Spanish", "es-mx": "Mexican",
"de-de": "German", "pt-br": "Brazilian"}
voices = json.load(open(os.path.join(WORLD, "voices.json")))
catalogue = {v["id"]: v for v in voices["catalogue"]}
gate = {}
for line in open(os.path.join(WORLD, "gate.jsonl")):
g = json.loads(line)
gate[g["id"]] = g["cer"]
made = []
for name in sorted(os.listdir(WORLD)):
if name.startswith("made_") and "convo" in name:
for line in open(os.path.join(WORLD, name)):
m = json.loads(line)
if m["split"] == "test" and gate.get(m["id"], 1) <= 0.35 and 1.8 <= m["duration"] <= 7:
made.append(m)
situation = {}
for line in open(os.path.join(WORLD, "convo.jsonl")):
c = json.loads(line)
if c["split"] == "test":
for k in ("ht", "en", "fr", "es"):
situation[(k, c[k])] = c["situation"]
real = [json.loads(line) for line in open(os.path.join(DATA, "clips.jsonl"))]
real = [r for r in real if r["split"] == "test" and r["source"] in ("fleurs", "voxpopuli_en_accented") and 3 <= r["duration"] <= 9]
print(len(made), "synthetic candidates,", len(real), "real")
r = Routyo()
rng = random.Random(7)
def voice_of(m):
if m["kind"] == "fatiman_h":
return "Haitian voice"
v = catalogue.get(m["voice"], {})
return f"{LOCALE.get(v.get('locale'), 'foreign')} voice"
def take(pool, lang, label, note, source):
pool = list(pool)
rng.shuffle(pool)
for m in pool:
x = load_audio(m["path"])
rates = r.rates(x, words=True)
if rates[0]["language"] == lang:
label = label.replace("{voice}", voice_of(m)) if "kind" in m else label
name = f"{len(picked):02d}_{lang}_{source}.wav"
sf.write(os.path.join(OUT, name), x, 16000, subtype="PCM_16")
picked.append({"file": name, "language": lang, "label": label, "note": note, "text": m.get("text", ""),
"source": source, "seconds": round(len(x) / 16000, 2), "rate": rates[0]["rate"]})
print(f" {label:45} {rates[0]['language']} {rates[0]['rate']:.3f} {m.get('text', '')[:60]}")
return m
print(" none for", label)
def syn(kind, lang, locale=None, gender=None):
out = []
for m in made:
if m["kind"] != kind or m["language"] != lang:
continue
if locale and catalogue.get(m["voice"], {}).get("locale") not in locale:
continue
out.append({**m, "path": os.path.join(DATA, m["audio"])})
return out
def rl(source, lang):
return [{**x, "path": os.path.join(DATA, x["audio"]) if not os.path.isabs(x["audio"]) else x["audio"]}
for x in real if x["source"] == source and x["language"] == lang]
picked = []
SYN = "synthetic, voice never heard in training"
# Krey貌l
take(syn("fatiman_h", "ht"), "ht", "Krey貌l 路 Haitian voice", SYN, "fatiman")
take(syn("fatiman_c", "ht", {"en-us", "en-gb"}), "ht", "Krey貌l 路 {voice}", SYN, "fatiman-clone")
take(syn("fatiman_c", "ht", {"fr-fr"}), "ht", "Krey貌l 路 {voice}", SYN, "fatiman-clone")
take(syn("fatiman_c", "ht", {"es-mx", "es-es"}), "ht", "Krey貌l 路 {voice}", SYN, "fatiman-clone")
take(syn("phonon", "ht"), "ht", "Krey貌l 路 {voice} (another engine)", SYN, "phonon")
# English
take(syn("fatiman_h", "en"), "en", "English 路 Haitian voice", SYN, "fatiman")
take(syn("phonon", "en"), "en", "English 路 {voice}", SYN, "phonon")
take(syn("fatiman_c", "en", {"de-de", "pt-br"}), "en", "English 路 {voice}", SYN, "fatiman-clone")
take(rl("fleurs", "en"), "en", "English 路 real speaker (FLEURS)", "real recording, FLEURS (CC BY 4.0)", "fleurs")
take(rl("voxpopuli_en_accented", "en"), "en", "English 路 real non-native speaker (VoxPopuli)",
"real recording, VoxPopuli (CC0)", "voxpopuli")
# French
take(syn("fatiman_h", "fr"), "fr", "Fran莽ais 路 Haitian voice", SYN, "fatiman")
take(syn("phonon", "fr"), "fr", "Fran莽ais 路 {voice}", SYN, "phonon")
take(syn("fatiman_c", "fr", {"en-us", "en-gb"}), "fr", "Fran莽ais 路 {voice}", SYN, "fatiman-clone")
take(rl("fleurs", "fr"), "fr", "Fran莽ais 路 real speaker (FLEURS)", "real recording, FLEURS (CC BY 4.0)", "fleurs")
# Spanish
take(syn("fatiman_h", "es"), "es", "Espa帽ol 路 Haitian voice", SYN, "fatiman")
take(syn("phonon", "es"), "es", "Espa帽ol 路 {voice}", SYN, "phonon")
take(syn("fatiman_c", "es", {"fr-fr"}), "es", "Espa帽ol 路 {voice}", SYN, "fatiman-clone")
take(rl("fleurs", "es"), "es", "Espa帽ol 路 real speaker (FLEURS)", "real recording, FLEURS (CC BY 4.0)", "fleurs")
# Conversations: lines of one test situation, glued with pauses; every line must also be routed right
def right(m):
return r.rates(load_audio(os.path.join(DATA, m["audio"])), words=True)[0]["language"] == m["language"]
def first_right(c):
c = list(c)
rng.shuffle(c)
return next((m for m in c if right(m)), None)
def glue(lines, name, title):
gap = np.zeros(int(1.5 * 16000), np.float32)
parts, script = [], []
for m in lines:
x = load_audio(os.path.join(DATA, m["audio"]))
x = x / (np.abs(x).max() + 1e-6) * 0.8
parts += [x, gap]
script.append({"language": m["language"], "text": m["text"], "voice": voice_of(m)})
y = np.concatenate(parts[:-1]).astype(np.float32)
sf.write(os.path.join(OUT, name), y, 16000, subtype="PCM_16")
convos.append({"file": name, "title": title, "lines": script, "seconds": round(len(y) / 16000, 1)})
print(" ", title, [s["language"] for s in script])
convos = []
by_sit = {}
for m in made:
s = situation.get((m["language"], m["text"]))
if s:
by_sit.setdefault(s, []).append(m)
# 1: many voices, languages alternating, one situation
order = ["ht", "en", "fr", "ht", "es", "en", "ht"]
for sit in sorted(by_sit):
pool, lines, used = by_sit[sit], [], set()
for k in order:
m = first_right(m for m in pool if m["language"] == k and m["text"] not in used)
if m is None:
break
used.add(m["text"])
lines.append(m)
if len(lines) == len(order):
glue(lines, "convo_many_voices.wav", f"Many voices, four languages ({sit})")
break
# 2: one Haitian voice switching language every sentence
haitian = {}
for m in made:
if m["kind"] == "fatiman_h":
haitian.setdefault(m["voice"], set()).add(m["language"])
for voice in sorted(haitian, key=lambda v: -sum(m["voice"] == v for m in made)):
pool = [m for m in made if m["kind"] == "fatiman_h" and m["voice"] == voice]
lines = []
for k in ["ht", "en", "fr", "es", "ht", "fr"]:
m = first_right(m for m in pool if m["language"] == k and m not in lines)
if m is None:
break
lines.append(m)
if len(lines) == 6:
glue(lines, "convo_one_voice.wav", "One Haitian voice switching languages")
break
json.dump({"examples": picked, "conversations": convos}, open(os.path.join(OUT, "examples.json"), "w"),
indent=1, ensure_ascii=False)
print(len(picked), "examples,", len(convos), "conversations ->", OUT)