File size: 8,035 Bytes
c6eaeea | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 | """Pick the Space's example clips from material Routyo never trained on, and build two mixed conversations.
python -I tools/pick_examples.py <routyo package folder>
Synthetic: the test split only (voices held out, everyday sentences from the 10 test-only situations), gate passed.
Real: FLEURS (CC BY 4.0) and VoxPopuli (CC0) test clips for en/fr/es.
Each candidate must be called right by Routyo with the word check; the first that is wins (fixed seed order).
Writes examples/*.wav (16 kHz mono) and examples/examples.json.
"""
import json
import os
import random
import sys
import numpy as np
import soundfile as sf
sys.path.insert(0, sys.argv[1])
from routyo import Routyo, load_audio # noqa: E402
DATA = os.path.expanduser("~/Documents/Youtube-Ht-Data/data/langid")
WORLD = os.path.join(DATA, "world")
OUT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples")
os.makedirs(OUT, exist_ok=True)
LOCALE = {"en-us": "American", "en-gb": "British", "fr-fr": "French", "es-es": "Spanish", "es-mx": "Mexican",
"de-de": "German", "pt-br": "Brazilian"}
voices = json.load(open(os.path.join(WORLD, "voices.json")))
catalogue = {v["id"]: v for v in voices["catalogue"]}
gate = {}
for line in open(os.path.join(WORLD, "gate.jsonl")):
g = json.loads(line)
gate[g["id"]] = g["cer"]
made = []
for name in sorted(os.listdir(WORLD)):
if name.startswith("made_") and "convo" in name:
for line in open(os.path.join(WORLD, name)):
m = json.loads(line)
if m["split"] == "test" and gate.get(m["id"], 1) <= 0.35 and 1.8 <= m["duration"] <= 7:
made.append(m)
situation = {}
for line in open(os.path.join(WORLD, "convo.jsonl")):
c = json.loads(line)
if c["split"] == "test":
for k in ("ht", "en", "fr", "es"):
situation[(k, c[k])] = c["situation"]
real = [json.loads(line) for line in open(os.path.join(DATA, "clips.jsonl"))]
real = [r for r in real if r["split"] == "test" and r["source"] in ("fleurs", "voxpopuli_en_accented") and 3 <= r["duration"] <= 9]
print(len(made), "synthetic candidates,", len(real), "real")
r = Routyo()
rng = random.Random(7)
def voice_of(m):
if m["kind"] == "fatiman_h":
return "Haitian voice"
v = catalogue.get(m["voice"], {})
return f"{LOCALE.get(v.get('locale'), 'foreign')} voice"
def take(pool, lang, label, note, source):
pool = list(pool)
rng.shuffle(pool)
for m in pool:
x = load_audio(m["path"])
rates = r.rates(x, words=True)
if rates[0]["language"] == lang:
label = label.replace("{voice}", voice_of(m)) if "kind" in m else label
name = f"{len(picked):02d}_{lang}_{source}.wav"
sf.write(os.path.join(OUT, name), x, 16000, subtype="PCM_16")
picked.append({"file": name, "language": lang, "label": label, "note": note, "text": m.get("text", ""),
"source": source, "seconds": round(len(x) / 16000, 2), "rate": rates[0]["rate"]})
print(f" {label:45} {rates[0]['language']} {rates[0]['rate']:.3f} {m.get('text', '')[:60]}")
return m
print(" none for", label)
def syn(kind, lang, locale=None, gender=None):
out = []
for m in made:
if m["kind"] != kind or m["language"] != lang:
continue
if locale and catalogue.get(m["voice"], {}).get("locale") not in locale:
continue
out.append({**m, "path": os.path.join(DATA, m["audio"])})
return out
def rl(source, lang):
return [{**x, "path": os.path.join(DATA, x["audio"]) if not os.path.isabs(x["audio"]) else x["audio"]}
for x in real if x["source"] == source and x["language"] == lang]
picked = []
SYN = "synthetic, voice never heard in training"
# Krey貌l
take(syn("fatiman_h", "ht"), "ht", "Krey貌l 路 Haitian voice", SYN, "fatiman")
take(syn("fatiman_c", "ht", {"en-us", "en-gb"}), "ht", "Krey貌l 路 {voice}", SYN, "fatiman-clone")
take(syn("fatiman_c", "ht", {"fr-fr"}), "ht", "Krey貌l 路 {voice}", SYN, "fatiman-clone")
take(syn("fatiman_c", "ht", {"es-mx", "es-es"}), "ht", "Krey貌l 路 {voice}", SYN, "fatiman-clone")
take(syn("phonon", "ht"), "ht", "Krey貌l 路 {voice} (another engine)", SYN, "phonon")
# English
take(syn("fatiman_h", "en"), "en", "English 路 Haitian voice", SYN, "fatiman")
take(syn("phonon", "en"), "en", "English 路 {voice}", SYN, "phonon")
take(syn("fatiman_c", "en", {"de-de", "pt-br"}), "en", "English 路 {voice}", SYN, "fatiman-clone")
take(rl("fleurs", "en"), "en", "English 路 real speaker (FLEURS)", "real recording, FLEURS (CC BY 4.0)", "fleurs")
take(rl("voxpopuli_en_accented", "en"), "en", "English 路 real non-native speaker (VoxPopuli)",
"real recording, VoxPopuli (CC0)", "voxpopuli")
# French
take(syn("fatiman_h", "fr"), "fr", "Fran莽ais 路 Haitian voice", SYN, "fatiman")
take(syn("phonon", "fr"), "fr", "Fran莽ais 路 {voice}", SYN, "phonon")
take(syn("fatiman_c", "fr", {"en-us", "en-gb"}), "fr", "Fran莽ais 路 {voice}", SYN, "fatiman-clone")
take(rl("fleurs", "fr"), "fr", "Fran莽ais 路 real speaker (FLEURS)", "real recording, FLEURS (CC BY 4.0)", "fleurs")
# Spanish
take(syn("fatiman_h", "es"), "es", "Espa帽ol 路 Haitian voice", SYN, "fatiman")
take(syn("phonon", "es"), "es", "Espa帽ol 路 {voice}", SYN, "phonon")
take(syn("fatiman_c", "es", {"fr-fr"}), "es", "Espa帽ol 路 {voice}", SYN, "fatiman-clone")
take(rl("fleurs", "es"), "es", "Espa帽ol 路 real speaker (FLEURS)", "real recording, FLEURS (CC BY 4.0)", "fleurs")
# Conversations: lines of one test situation, glued with pauses; every line must also be routed right
def right(m):
return r.rates(load_audio(os.path.join(DATA, m["audio"])), words=True)[0]["language"] == m["language"]
def first_right(c):
c = list(c)
rng.shuffle(c)
return next((m for m in c if right(m)), None)
def glue(lines, name, title):
gap = np.zeros(int(1.5 * 16000), np.float32)
parts, script = [], []
for m in lines:
x = load_audio(os.path.join(DATA, m["audio"]))
x = x / (np.abs(x).max() + 1e-6) * 0.8
parts += [x, gap]
script.append({"language": m["language"], "text": m["text"], "voice": voice_of(m)})
y = np.concatenate(parts[:-1]).astype(np.float32)
sf.write(os.path.join(OUT, name), y, 16000, subtype="PCM_16")
convos.append({"file": name, "title": title, "lines": script, "seconds": round(len(y) / 16000, 1)})
print(" ", title, [s["language"] for s in script])
convos = []
by_sit = {}
for m in made:
s = situation.get((m["language"], m["text"]))
if s:
by_sit.setdefault(s, []).append(m)
# 1: many voices, languages alternating, one situation
order = ["ht", "en", "fr", "ht", "es", "en", "ht"]
for sit in sorted(by_sit):
pool, lines, used = by_sit[sit], [], set()
for k in order:
m = first_right(m for m in pool if m["language"] == k and m["text"] not in used)
if m is None:
break
used.add(m["text"])
lines.append(m)
if len(lines) == len(order):
glue(lines, "convo_many_voices.wav", f"Many voices, four languages ({sit})")
break
# 2: one Haitian voice switching language every sentence
haitian = {}
for m in made:
if m["kind"] == "fatiman_h":
haitian.setdefault(m["voice"], set()).add(m["language"])
for voice in sorted(haitian, key=lambda v: -sum(m["voice"] == v for m in made)):
pool = [m for m in made if m["kind"] == "fatiman_h" and m["voice"] == voice]
lines = []
for k in ["ht", "en", "fr", "es", "ht", "fr"]:
m = first_right(m for m in pool if m["language"] == k and m not in lines)
if m is None:
break
lines.append(m)
if len(lines) == 6:
glue(lines, "convo_one_voice.wav", "One Haitian voice switching languages")
break
json.dump({"examples": picked, "conversations": convos}, open(os.path.join(OUT, "examples.json"), "w"),
indent=1, ensure_ascii=False)
print(len(picked), "examples,", len(convos), "conversations ->", OUT)
|