File size: 2,923 Bytes
27d8618
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
"""Opus features (20 per 10 ms frame) and copy synthesis with PiVo's FARGAN vocoder.

fargan_demo -features (xiph/opus bafd40d, BSD-3) writes 20 floats per frame:
18 cepstra, pitch (log2(256/period) - 1.5) and pitch correlation.
"""
import subprocess, tempfile
from pathlib import Path
import numpy as np

import os
ROOT = Path(__file__).resolve().parent.parent          # repository root
DUMP = Path(os.environ.get("FARGAN_DEMO", ROOT / "bin" / "fargan_demo"))   # see docs/HOW-WE-TRAINED.md, section 0
HOP = 160
DELAY = 80   # native extractor delay: frame k describes [160k-80, 160k+80)


def extract(wave16k):
    """wave16k float32 em [-1,1] a 16 kHz -> (quadros, 20) float32."""
    pcm = (np.clip(wave16k, -1, 1) * 32767).round().astype("<i2")
    with tempfile.TemporaryDirectory() as d:
        a, b = Path(d, "in.s16"), Path(d, "f.f32")
        a.write_bytes(pcm.tobytes())
        subprocess.run([str(DUMP), "-features", str(a), str(b)], check=True, capture_output=True)
        f = np.fromfile(b, dtype=np.float32).reshape(-1, 20)
    return f.copy()


def target_audio(wave16k, frames):
    """Alvo alinhado às features (compensa o atraso de 80 amostras)."""
    w = np.asarray(wave16k, dtype=np.float32)
    if not frames * HOP <= len(w) < (frames + 1) * HOP + HOP:
        w = np.pad(w, (0, max(0, frames * HOP - len(w))))
    return np.pad(w, (DELAY, 0))[: frames * HOP]


def periods(features):
    return np.rint(256 / np.power(2.0, np.clip(features[..., 18], -10, 10) + 1.5)).clip(32, 255).astype(np.int64)


class Vocoder:
    STATE = ("memory", "gru1", "gru2", "gru3", "frame_state")
    WIDTH = (256, 160, 128, 128, 164)

    def __init__(self, directory=ROOT / "onnx"):
        import onnxruntime as ort
        o = ort.SessionOptions(); o.intra_op_num_threads = 1; o.inter_op_num_threads = 1
        mk = lambda n: ort.InferenceSession(str(Path(directory) / f"{n}.onnx"), o, providers=["CPUExecutionProvider"])
        self.cond, self.step = mk("conditioning"), mk("vocoder_step")

    def __call__(self, feats):
        state = {n: np.zeros((1, w), np.float32) for n, w in zip(self.STATE, self.WIDTH)}
        out, deemph = [], 0.0
        prev = feats[0]
        for k in range(len(feats)):
            cur = feats[k]; nxt = feats[k + 1] if k + 1 < len(feats) else cur
            ctx = np.stack((prev, cur, nxt))[None].astype(np.float32)
            per = periods(ctx)
            c = self.cond.run(None, {"features": ctx, "periods": per})[0]
            frame = []
            for s in range(4):
                r = self.step.run(None, {"conditioning": c[:, 0, s * 80:(s + 1) * 80], "period": per[:, 1], **state})
                state = dict(zip(self.STATE, r[1:])); frame.append(r[0][0])
            for x in np.concatenate(frame):      # de-emphasis 0.85
                deemph = float(x) + 0.85 * deemph; out.append(deemph)
            prev = cur
        return np.asarray(out, np.float32)