pingo-ptbr / training /features.py
itxtoledo's picture
Add pingo-ptbr: 0.87 MB pt-BR TTS (MLX + ONNX + web demo)
27d8618 verified
Raw History Blame Contribute Delete
2.92 kB
"""Opus features (20 per 10 ms frame) and copy synthesis with PiVo's FARGAN vocoder.
fargan_demo -features (xiph/opus bafd40d, BSD-3) writes 20 floats per frame:
18 cepstra, pitch (log2(256/period) - 1.5) and pitch correlation.
"""
import subprocess, tempfile
from pathlib import Path
import numpy as np
import os
ROOT = Path(__file__).resolve().parent.parent # repository root
DUMP = Path(os.environ.get("FARGAN_DEMO", ROOT / "bin" / "fargan_demo")) # see docs/HOW-WE-TRAINED.md, section 0
HOP = 160
DELAY = 80 # native extractor delay: frame k describes [160k-80, 160k+80)
def extract(wave16k):
"""wave16k float32 em [-1,1] a 16 kHz -> (quadros, 20) float32."""
pcm = (np.clip(wave16k, -1, 1) * 32767).round().astype("<i2")
with tempfile.TemporaryDirectory() as d:
a, b = Path(d, "in.s16"), Path(d, "f.f32")
a.write_bytes(pcm.tobytes())
subprocess.run([str(DUMP), "-features", str(a), str(b)], check=True, capture_output=True)
f = np.fromfile(b, dtype=np.float32).reshape(-1, 20)
return f.copy()
def target_audio(wave16k, frames):
"""Alvo alinhado às features (compensa o atraso de 80 amostras)."""
w = np.asarray(wave16k, dtype=np.float32)
if not frames * HOP <= len(w) < (frames + 1) * HOP + HOP:
w = np.pad(w, (0, max(0, frames * HOP - len(w))))
return np.pad(w, (DELAY, 0))[: frames * HOP]
def periods(features):
return np.rint(256 / np.power(2.0, np.clip(features[..., 18], -10, 10) + 1.5)).clip(32, 255).astype(np.int64)
class Vocoder:
STATE = ("memory", "gru1", "gru2", "gru3", "frame_state")
WIDTH = (256, 160, 128, 128, 164)
def __init__(self, directory=ROOT / "onnx"):
import onnxruntime as ort
o = ort.SessionOptions(); o.intra_op_num_threads = 1; o.inter_op_num_threads = 1
mk = lambda n: ort.InferenceSession(str(Path(directory) / f"{n}.onnx"), o, providers=["CPUExecutionProvider"])
self.cond, self.step = mk("conditioning"), mk("vocoder_step")
def __call__(self, feats):
state = {n: np.zeros((1, w), np.float32) for n, w in zip(self.STATE, self.WIDTH)}
out, deemph = [], 0.0
prev = feats[0]
for k in range(len(feats)):
cur = feats[k]; nxt = feats[k + 1] if k + 1 < len(feats) else cur
ctx = np.stack((prev, cur, nxt))[None].astype(np.float32)
per = periods(ctx)
c = self.cond.run(None, {"features": ctx, "periods": per})[0]
frame = []
for s in range(4):
r = self.step.run(None, {"conditioning": c[:, 0, s * 80:(s + 1) * 80], "period": per[:, 1], **state})
state = dict(zip(self.STATE, r[1:])); frame.append(r[0][0])
for x in np.concatenate(frame): # de-emphasis 0.85
deemph = float(x) + 0.85 * deemph; out.append(deemph)
prev = cur
return np.asarray(out, np.float32)