fiato / tests /test_model.py
cfahlgren1's picture
cfahlgren1 HF Staff
rename the project to fiato
4744deb verified
Raw History Blame Contribute Delete
21.2 kB
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
import torch
from fiato.config import CHECKPOINT_FORMAT, Config
from fiato.data import collate
from fiato.infer import load_model
from fiato.inspect import parameter_report
from fiato.losses import Discriminators
from fiato.model import SourceFilter, Fiato, cepstrum_response, filtered, pulse_train
from fiato.train import (WEIGHTS, TrainingSystem, atomic_save, export_model, learning_rate, load_compatible, optimize_step,
teacher_probability)
from helpers import SYMBOLS, sample, small_config, write_annotations
RECIPE = {"learning_rate": 1e-3, "warmup": 2, "lr_half_life": 1000.0, "grad_clip": 10.0, "gan_start": 1, "crop_samples": 4096, "crops": 1,
"teacher_hold": 1, "teacher_ramp": 1, "teacher_final": 0.0, "weights": WEIGHTS, "gan_ratio": 0.7, "gan_ramp": 2,
"feature_matching": 30.0, "gan_scale_max": 2000.0, "branched_from": None}
class ModelTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
torch.set_num_threads(1)
def setUp(self):
torch.manual_seed(23)
def test_default_model_is_under_five_million_parameters(self):
report = parameter_report(Config())
self.assertEqual(sum(report["inference_parts"].values()), report["inference_parameters"])
self.assertLess(report["inference_parameters"], 5_000_000)
self.assertEqual(report["inference_parameters"], sum(p.numel() for p in Fiato().parameters()))
self.assertGreater(report["inference_parts"]["frame transformer B"], report["inference_parts"]["source-filter waveform head"])
def test_pulse_train_has_the_requested_pitch_unit_power_and_band_limit(self):
rate, hop = 24000, 256
for hertz in (110.0, 220.0):
wave = pulse_train(torch.full((1, 200), hertz), rate, hop)[0].double()
self.assertEqual(wave.numel(), 200 * hop)
self.assertAlmostEqual(float(wave.square().mean()), 1.0, delta=0.02)
spectrum = torch.fft.rfft(wave * torch.hann_window(wave.numel(), dtype=torch.float64)).abs()
frequency = torch.fft.rfftfreq(wave.numel(), 1 / rate)
self.assertAlmostEqual(float(frequency[spectrum[:int(1.5 * hertz * wave.numel() / rate)].argmax()]), hertz, delta=1.0)
self.assertLess(float(spectrum[frequency > 0.48 * rate].max() / spectrum.max()), 1e-3) # nothing above the limit
harmonic = spectrum[(frequency - 10 * hertz).abs().argmin()]
self.assertAlmostEqual(float(harmonic / spectrum[(frequency - hertz).abs().argmin()]), 1.0, delta=0.05) # flat source
glide = pulse_train(torch.linspace(90, 300, 100)[None], rate, hop)
self.assertTrue(torch.isfinite(glide).all())
self.assertLess(float(glide.abs().max()), 40)
shifted = pulse_train(torch.full((1, 20), 150.0), rate, hop, phase=torch.tensor([1.0]))
self.assertFalse(torch.allclose(shifted, pulse_train(torch.full((1, 20), 150.0), rate, hop)))
def test_stft_filtering_is_transparent_for_a_flat_response(self):
source = torch.randn(2, 4096)
window = torch.hann_window(512)
frames = 4096 // 128 + 1
flat = torch.ones(2, frames, 257, dtype=torch.complex64)
torch.testing.assert_close(filtered(source, flat, 512, 128, window), source, atol=1e-4, rtol=1e-4)
torch.testing.assert_close(filtered(source, 0.5 * flat, 512, 128, window), 0.5 * source, atol=1e-4, rtol=1e-4)
with self.assertRaises(ValueError):
filtered(source, flat[:, :-1], 512, 128, window)
def test_cepstrum_gives_gain_and_minimum_phase(self):
zero = cepstrum_response(torch.zeros(1, 8), torch.zeros(1, 2), 64)
torch.testing.assert_close(zero, torch.ones(1, 33, dtype=torch.complex64))
gain = cepstrum_response(torch.tensor([[math.log(0.5), 0, 0, 0]]), None, 64)
torch.testing.assert_close(gain.abs(), torch.full((1, 33), 0.5), atol=1e-6, rtol=1e-6)
# A causal cepstrum is a minimum-phase filter: its impulse response is causal and front-loaded.
response = cepstrum_response(torch.tensor([[0.0, 0.8, -0.3, 0.1]]), None, 64)
impulse = torch.fft.irfft(response, n=64)[0]
self.assertGreater(float(impulse[:8].square().sum() / impulse.square().sum()), 0.999)
def test_expand_skips_zero_duration_tokens_and_marks_positions(self):
model = Fiato(small_config()).eval()
hidden = torch.arange(4, dtype=torch.float32)[None, :, None].expand(1, 4, 32).contiguous()
durations = torch.tensor([[2, 0, 3, 1]])
with torch.no_grad():
model.frame_position.weight.zero_()
model.frame_position.bias.zero_()
model.frame_voices.weight.zero_()
frames, mask, lengths = model.expand(hidden, durations, torch.ones(1, 4, dtype=torch.bool), torch.tensor([0]))
self.assertEqual(lengths.tolist(), [6])
self.assertEqual(frames[0, :, 0].tolist(), [0, 0, 2, 2, 2, 3]) # token 1 has no frame
with self.assertRaises(ValueError):
model.expand(hidden, torch.tensor([[1, -1, 1, 1]]), torch.ones(1, 4, dtype=torch.bool), torch.tensor([0]))
with self.assertRaises(ValueError):
model.expand(hidden, torch.zeros(1, 4, dtype=torch.long), torch.ones(1, 4, dtype=torch.bool), torch.tensor([0]))
def test_padding_does_not_change_the_short_example(self):
c = small_config()
model = Fiato(c).eval()
with torch.no_grad(): # make the heads non-trivial so the test has teeth
for head in (model.vocoder.harmonic, model.vocoder.noise):
head.weight.normal_(std=0.05)
model.duration.out.weight.zero_() # three frames per token: rows differ in length
model.duration.out.bias.fill_(math.log1p(3))
a, b = sample(c), sample(c, voice=1, durations=(5, 6, 7, 6, 12, 3, 6, 14, 7, 6, 2, 8))
tokens = torch.stack((a["tokens"], b["tokens"]))
stress = torch.stack((a["stress"], b["stress"]))
lengths = torch.tensor([7, len(SYMBOLS)]) # first row uses only 7 tokens: genuine token padding
tokens[0, 7:], stress[0, 7:] = 0, 0
wave, samples, details = model.synthesize(tokens, stress, lengths, torch.tensor([0, 1]))
alone, n, _ = model.synthesize(tokens[:1, :7], stress[:1, :7], lengths[:1], torch.tensor([0]))
self.assertEqual(int(n[0]), int(samples[0]))
self.assertLess(int(samples[0]), int(samples[1]))
# Float32 rounding through exp() and two STFT round trips reaches ~1e-4; the
# padding bugs this guards against (pitch glide into padding, shared noise
# stream, missing end frames) each moved samples by more than 1e-2.
torch.testing.assert_close(wave[0, :int(n[0])], alone[0], atol=5e-4, rtol=1e-3)
self.assertGreater(float(alone.abs().max()), 0.01)
self.assertEqual(float(wave[0, int(n[0]):].abs().max()), 0.0)
# Every phoneme is audible; pause tokens may get zero frames.
phones = model.phone_tokens[tokens[1]]
self.assertTrue(bool((details["durations"][1][phones] >= 1).all()))
for speed in (0, float("nan"), 5):
with self.assertRaises(ValueError):
model.synthesize(tokens, stress, lengths, torch.tensor([0, 1]), speed)
def test_source_filter_is_differentiable_and_respects_lengths(self):
c = small_config()
head = SourceFilter(c)
hidden = torch.randn(2, 12, c.channels, requires_grad=True)
with torch.no_grad():
head.harmonic.weight.normal_(std=0.05)
head.noise.weight.normal_(std=0.05)
wave = head(hidden, torch.full((2, 12), 180.0), torch.tensor([12, 7]))
self.assertEqual(wave.shape, (2, 12 * c.hop))
self.assertEqual(float(wave[1, 7 * c.hop:].abs().max()), 0.0)
wave.square().mean().backward()
self.assertTrue(torch.isfinite(hidden.grad).all())
self.assertGreater(float(hidden.grad[1, :7].abs().sum()), 0)
self.assertEqual(float(hidden.grad[1, 7:].abs().sum()), 0.0) # padded frames cannot influence audio
def test_both_training_phases_update_every_inference_module(self):
c = small_config()
system, critics = TrainingSystem(c), Discriminators(periods=(2, 3), resolutions=((256, 64),))
optimizer = torch.optim.AdamW(system.parameters(), lr=1e-3)
d_optimizer = torch.optim.AdamW(critics.parameters(), lr=1e-3)
batch = collate([sample(c), sample(c, voice=1, durations=(5, 6, 7, 6, 12, 3, 6, 14, 7, 6, 0, 8))])
before = {name: value.detach().clone() for name, value in system.model.named_parameters()}
for step in (0, 1, 2):
metrics = optimize_step(system, critics, optimizer, d_optimizer, batch, step, RECIPE)
self.assertTrue(all(math.isfinite(v) for v in metrics.values()), metrics)
self.assertIn("adversarial", metrics)
self.assertGreater(metrics["gan_scale"], 0)
unchanged = [name for name, value in system.model.named_parameters() if torch.equal(value, before[name])]
self.assertEqual(unchanged, [])
self.assertTrue(all(p.grad is None or torch.isfinite(p.grad).all() for p in system.parameters()))
def test_adversarial_gradient_is_balanced_against_reconstruction(self):
# Regression for the "GAN phase that does nothing": with fixed weights the adversarial
# gradient at the waveform was under 1% of the spectral one on the real run.
c = small_config()
torch.manual_seed(5)
system, critics = TrainingSystem(c), Discriminators(periods=(2, 3), resolutions=((256, 64),))
batch = collate([sample(c), sample(c, voice=1)])
loss, _, auxiliary = system.objective(batch, 1.0, random_phase=False)
from fiato.losses import generator_gan_loss
from fiato.train import crops
wave = auxiliary["wave"]
fake, real, index = crops(wave, batch["wave"], batch["frame_lengths"] * c.hop, 4096, 1)
self.assertEqual(index.shape, (2, 4096))
torch.testing.assert_close(fake, wave.gather(1, index))
adversarial, matching = generator_gan_loss(critics(real), critics(fake))
gan = adversarial + 30.0 * matching
reconstruction = torch.autograd.grad(auxiliary["reconstruction"], wave, retain_graph=True)[0].gather(1, index)
raw = torch.autograd.grad(gan, wave, retain_graph=True)[0].gather(1, index)
scale = 0.7 * reconstruction.norm() / raw.norm()
balanced = torch.autograd.grad(scale.detach() * gan, wave)[0].gather(1, index)
self.assertAlmostEqual(float(balanced.norm() / reconstruction.norm()), 0.7, places=4)
def test_pause_tokens_vanish_unless_predicted_present_and_pitch_range_scales_movement(self):
model = Fiato(small_config()).eval()
tokens = sample(small_config())["tokens"][None]
mask = torch.ones_like(tokens, dtype=torch.bool)
hidden = torch.zeros(1, tokens.shape[1], 32)
phones = model.phone_tokens[tokens[0]]
with torch.no_grad():
model.duration.out.weight.zero_()
model.duration.out.bias.copy_(torch.tensor([math.log1p(5), -3.0])) # 5 frames, "no pause"
absent = model.predict_durations(hidden, mask, tokens)[0]
model.duration.out.bias.copy_(torch.tensor([math.log1p(5), 3.0])) # 5 frames, "pause"
present = model.predict_durations(hidden, mask, tokens)[0]
model.duration.out.bias.copy_(torch.tensor([-2.0, -3.0])) # shorter than a frame
floor = model.predict_durations(hidden, mask, tokens)[0]
self.assertEqual(absent[phones].tolist(), [5] * int(phones.sum()))
self.assertEqual(absent[~phones].tolist(), [0] * int((~phones).sum()))
self.assertEqual(present.tolist(), [5] * tokens.shape[1])
self.assertEqual(floor[phones].tolist(), [1] * int(phones.sum())) # a phoneme is never dropped
self.assertEqual(model.predict_durations(hidden, mask, tokens, speed=2.5)[0][phones].tolist(), [1] * int(phones.sum()))
prosody = torch.tensor([[[0.0, 1.0, 0.0], [0.4, 1.0, 0.0], [9.0, -1.0, 0.0], [-0.4, 1.0, 0.0]]])
frame_mask = torch.ones(1, 4, dtype=torch.bool)
wide = Fiato.widen_pitch(prosody, frame_mask, 1.5)
torch.testing.assert_close(wide[0, [0, 1, 3], 0], torch.tensor([0.0, 0.6, -0.6])) # around the voiced mean (0)
torch.testing.assert_close(wide[..., 1:], prosody[..., 1:])
self.assertIs(Fiato.widen_pitch(prosody, frame_mask, 1.0), prosody)
def test_branching_keeps_old_weights_when_a_layer_grows(self):
old = {"weight": torch.ones(1, 3), "bias": torch.full((1,), 2.0), "gone": torch.zeros(1)}
layer = torch.nn.Linear(3, 2)
fresh = layer.weight.detach().clone()
report = load_compatible(layer, old)
self.assertEqual(report, {"grown": ["weight", "bias"], "new": [], "dropped": ["gone"]})
torch.testing.assert_close(layer.weight[0], torch.ones(3))
torch.testing.assert_close(layer.weight[1], fresh[1]) # the new row keeps its initialisation
self.assertEqual(float(layer.bias[0]), 2.0)
with self.assertRaisesRegex(ValueError, "changed shape"):
load_compatible(torch.nn.Linear(2, 2), {"weight": torch.ones(2, 3), "bias": torch.ones(2)})
def test_teacher_schedule_and_learning_rate(self):
self.assertEqual(teacher_probability(0, 100, 200, 0.5), 1.0)
self.assertAlmostEqual(teacher_probability(200, 100, 200, 0.5), 0.75)
self.assertEqual(teacher_probability(10_000, 100, 200, 0.5), 0.5)
self.assertAlmostEqual(learning_rate(0, 1e-3, 10, 1000), 1e-4)
self.assertAlmostEqual(learning_rate(1000, 1e-3, 10, 1000), 5e-4)
self.assertAlmostEqual(learning_rate(1000, 1e-3, 10, 1000, start=1000), 5e-5) # a branched run warms up again
c = small_config()
system = TrainingSystem(c).eval()
batch = collate([sample(c), sample(c, voice=1)])
with torch.no_grad():
teacher = system.objective(batch, 1.0, random_phase=False)[1]
free = system.objective(batch, 0.0, random_phase=False)[1]
self.assertEqual(float(teacher["teacher_fraction"]), 1.0)
self.assertEqual(float(free["teacher_fraction"]), 0.0)
torch.testing.assert_close(teacher["pitch"], free["pitch"]) # predictors do not depend on the conditioning choice
# The pitch-spread term compares per-utterance standard deviations over trusted voiced frames.
from fiato.train import masked_std
values = torch.tensor([[1.0, 3.0, 100.0, 5.0], [2.0, 2.0, 2.0, 2.0]])
mask = torch.tensor([[1.0, 1.0, 0.0, 1.0], [1.0, 1.0, 1.0, 0.0]])
torch.testing.assert_close(masked_std(values, mask), torch.tensor([(8 / 3) ** 0.5, 1e-4]), atol=1e-5, rtol=1e-4)
self.assertGreater(float(teacher["pitch_spread"]), 0)
# An untrained model's ratio depends on its random weights (2 to 9 across seeds): only its sanity is checked.
self.assertTrue(0 <= float(teacher["pitch_spread_ratio"]) < 100)
def test_objective_rejects_durations_that_do_not_cover_the_frames(self):
c = small_config()
system = TrainingSystem(c).eval()
item = sample(c)
item["durations"] = item["durations"].clone()
item["durations"][4] -= 1
with self.assertRaisesRegex(ValueError, "frame count"), torch.no_grad():
system.objective(collate([item]), 1.0)
def test_export_roundtrip_and_explicit_rejection_of_old_checkpoints(self):
model = Fiato(small_config()).eval()
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "model.pt"
export_model(model, path, 7, "annotation")
restored = load_model(path)
self.assertEqual(set(model.state_dict()), set(restored.state_dict()))
for name, value in model.state_dict().items():
torch.testing.assert_close(value, restored.state_dict()[name], rtol=0, atol=0)
self.assertFalse(any("mel_head" in key or "critic" in key for key in restored.state_dict()))
state = torch.load(path, weights_only=True)
self.assertEqual(state["format_version"], CHECKPOINT_FORMAT)
for change, message in (({"format_version": 3}, "byte-token"),
({"frontend": {**state["frontend"], "fingerprint": "other"}}, "front end")):
atomic_save({**state, **change}, path)
with self.assertRaisesRegex(ValueError, message):
load_model(path)
def test_training_cli_resumes_same_as_uninterrupted_run(self):
c = small_config()
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "config.json").write_text(json.dumps(c.to_dict()))
train = write_annotations(root, c, "train")
validation = write_annotations(root, c, "validation")
base = [sys.executable, "-m", "fiato.train", "--train", str(train), "--validation", str(validation),
"--config", str(root / "config.json"), "--device", "cpu", "--batch-frames", "200", "--max-batch", "2",
"--save-every", "1", "--gan-start", "1", "--warmup", "2", "--teacher-hold", "1", "--teacher-ramp", "1",
"--crop-samples", "4096", "--crops", "1", "--validation-size", "2", "--max-seconds", "300", "--keep-every", "2"]
env = {**os.environ, "OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1"}
def run(arguments, expect=0):
result = subprocess.run(base + arguments, capture_output=True, text=True, env=env, timeout=600)
self.assertEqual(result.returncode != 0, expect != 0, result.stdout + result.stderr)
return result
run(["--output", str(root / "resumed"), "--steps", "1"])
run(["--output", str(root / "resumed"), "--steps", "2", "--resume", str(root / "resumed/last.pt")])
run(["--output", str(root / "continuous"), "--steps", "2"])
resumed = torch.load(root / "resumed/last.pt", weights_only=True)
continuous = torch.load(root / "continuous/last.pt", weights_only=True)
self.assertEqual(resumed["step"], 2)
self.assertEqual(resumed["format_version"], CHECKPOINT_FORMAT)
self.assertEqual(torch.load(root / "resumed/step-00000001.pt", weights_only=True)["step"], 1)
self.assertTrue((root / "resumed/step-00000002.pt").is_file())
for key, value in resumed["system"].items():
torch.testing.assert_close(value, continuous["system"][key], rtol=0, atol=0)
record = json.loads((root / "resumed/metrics.jsonl").read_text().splitlines()[-1])
self.assertIn("validation_predicted_mel", record)
self.assertIn("steps_per_second", record)
self.assertEqual(load_model(root / "resumed/inference.pt").config, c)
# A deadline that has already passed refuses to start; a stale format refuses to resume.
(root / "budget.json").write_text(json.dumps({"deadline": 1.0}))
failed = run(["--output", str(root / "late"), "--steps", "1", "--deadline-file", str(root / "budget.json")], expect=1)
self.assertIn("deadline", failed.stderr)
atomic_save({**resumed, "format_version": 3}, root / "old.pt")
failed = run(["--output", str(root / "resumed"), "--steps", "3", "--resume", str(root / "old.pt")], expect=1)
self.assertIn("cannot be resumed", failed.stderr)
# A changed recipe cannot be resumed, but it can be branched into a new run, which then resumes normally.
changed = ["--gan-ratio", "0.4", "--weights", '{"mel": 5}']
failed = run(["--output", str(root / "resumed"), "--steps", "3", "--resume", str(root / "resumed/last.pt")] + changed, expect=1)
self.assertIn("--init-from", failed.stderr)
run(["--output", str(root / "branch"), "--steps", "3", "--init-from", str(root / "resumed/last.pt")] + changed)
branch = torch.load(root / "branch/last.pt", weights_only=True)
self.assertEqual(branch["step"], 3)
self.assertEqual(branch["recipe"]["branched_from"]["step"], 2)
self.assertEqual(branch["recipe"]["weights"]["mel"], 5.0)
# Same architecture, source already in its adversarial phase: critics and optimizer moments carry over.
self.assertEqual(branch["recipe"]["branched_from"]["loaded"],
{"grown": [], "new": [], "dropped": [], "critics": True, "optimizer": True})
run(["--output", str(root / "branch"), "--steps", "4", "--resume", str(root / "branch/last.pt")] + changed)
self.assertEqual(torch.load(root / "branch/last.pt", weights_only=True)["step"], 4)
if __name__ == "__main__":
unittest.main()