Download tests/test_model.py from cfahlgren1/fiato: direct link, hf CLI and curl.
- Browser
- Download file 21.2 kB
-
https://huggingface.co/cfahlgren1/fiato/resolve/main/tests/test_model.py
- Command line
-
hf download hf://cfahlgren1/fiato/tests/test_model.py
-
curl -L -o test_model.py https://huggingface.co/cfahlgren1/fiato/resolve/main/tests/test_model.py
21.2 kB
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import subprocess | |
| import sys | |
| import tempfile | |
| import unittest | |
| import torch | |
| from fiato.config import CHECKPOINT_FORMAT, Config | |
| from fiato.data import collate | |
| from fiato.infer import load_model | |
| from fiato.inspect import parameter_report | |
| from fiato.losses import Discriminators | |
| from fiato.model import SourceFilter, Fiato, cepstrum_response, filtered, pulse_train | |
| from fiato.train import (WEIGHTS, TrainingSystem, atomic_save, export_model, learning_rate, load_compatible, optimize_step, | |
| teacher_probability) | |
| from helpers import SYMBOLS, sample, small_config, write_annotations | |
| RECIPE = {"learning_rate": 1e-3, "warmup": 2, "lr_half_life": 1000.0, "grad_clip": 10.0, "gan_start": 1, "crop_samples": 4096, "crops": 1, | |
| "teacher_hold": 1, "teacher_ramp": 1, "teacher_final": 0.0, "weights": WEIGHTS, "gan_ratio": 0.7, "gan_ramp": 2, | |
| "feature_matching": 30.0, "gan_scale_max": 2000.0, "branched_from": None} | |
| class ModelTests(unittest.TestCase): | |
| def setUpClass(cls): | |
| torch.set_num_threads(1) | |
| def setUp(self): | |
| torch.manual_seed(23) | |
| def test_default_model_is_under_five_million_parameters(self): | |
| report = parameter_report(Config()) | |
| self.assertEqual(sum(report["inference_parts"].values()), report["inference_parameters"]) | |
| self.assertLess(report["inference_parameters"], 5_000_000) | |
| self.assertEqual(report["inference_parameters"], sum(p.numel() for p in Fiato().parameters())) | |
| self.assertGreater(report["inference_parts"]["frame transformer B"], report["inference_parts"]["source-filter waveform head"]) | |
| def test_pulse_train_has_the_requested_pitch_unit_power_and_band_limit(self): | |
| rate, hop = 24000, 256 | |
| for hertz in (110.0, 220.0): | |
| wave = pulse_train(torch.full((1, 200), hertz), rate, hop)[0].double() | |
| self.assertEqual(wave.numel(), 200 * hop) | |
| self.assertAlmostEqual(float(wave.square().mean()), 1.0, delta=0.02) | |
| spectrum = torch.fft.rfft(wave * torch.hann_window(wave.numel(), dtype=torch.float64)).abs() | |
| frequency = torch.fft.rfftfreq(wave.numel(), 1 / rate) | |
| self.assertAlmostEqual(float(frequency[spectrum[:int(1.5 * hertz * wave.numel() / rate)].argmax()]), hertz, delta=1.0) | |
| self.assertLess(float(spectrum[frequency > 0.48 * rate].max() / spectrum.max()), 1e-3) # nothing above the limit | |
| harmonic = spectrum[(frequency - 10 * hertz).abs().argmin()] | |
| self.assertAlmostEqual(float(harmonic / spectrum[(frequency - hertz).abs().argmin()]), 1.0, delta=0.05) # flat source | |
| glide = pulse_train(torch.linspace(90, 300, 100)[None], rate, hop) | |
| self.assertTrue(torch.isfinite(glide).all()) | |
| self.assertLess(float(glide.abs().max()), 40) | |
| shifted = pulse_train(torch.full((1, 20), 150.0), rate, hop, phase=torch.tensor([1.0])) | |
| self.assertFalse(torch.allclose(shifted, pulse_train(torch.full((1, 20), 150.0), rate, hop))) | |
| def test_stft_filtering_is_transparent_for_a_flat_response(self): | |
| source = torch.randn(2, 4096) | |
| window = torch.hann_window(512) | |
| frames = 4096 // 128 + 1 | |
| flat = torch.ones(2, frames, 257, dtype=torch.complex64) | |
| torch.testing.assert_close(filtered(source, flat, 512, 128, window), source, atol=1e-4, rtol=1e-4) | |
| torch.testing.assert_close(filtered(source, 0.5 * flat, 512, 128, window), 0.5 * source, atol=1e-4, rtol=1e-4) | |
| with self.assertRaises(ValueError): | |
| filtered(source, flat[:, :-1], 512, 128, window) | |
| def test_cepstrum_gives_gain_and_minimum_phase(self): | |
| zero = cepstrum_response(torch.zeros(1, 8), torch.zeros(1, 2), 64) | |
| torch.testing.assert_close(zero, torch.ones(1, 33, dtype=torch.complex64)) | |
| gain = cepstrum_response(torch.tensor([[math.log(0.5), 0, 0, 0]]), None, 64) | |
| torch.testing.assert_close(gain.abs(), torch.full((1, 33), 0.5), atol=1e-6, rtol=1e-6) | |
| # A causal cepstrum is a minimum-phase filter: its impulse response is causal and front-loaded. | |
| response = cepstrum_response(torch.tensor([[0.0, 0.8, -0.3, 0.1]]), None, 64) | |
| impulse = torch.fft.irfft(response, n=64)[0] | |
| self.assertGreater(float(impulse[:8].square().sum() / impulse.square().sum()), 0.999) | |
| def test_expand_skips_zero_duration_tokens_and_marks_positions(self): | |
| model = Fiato(small_config()).eval() | |
| hidden = torch.arange(4, dtype=torch.float32)[None, :, None].expand(1, 4, 32).contiguous() | |
| durations = torch.tensor([[2, 0, 3, 1]]) | |
| with torch.no_grad(): | |
| model.frame_position.weight.zero_() | |
| model.frame_position.bias.zero_() | |
| model.frame_voices.weight.zero_() | |
| frames, mask, lengths = model.expand(hidden, durations, torch.ones(1, 4, dtype=torch.bool), torch.tensor([0])) | |
| self.assertEqual(lengths.tolist(), [6]) | |
| self.assertEqual(frames[0, :, 0].tolist(), [0, 0, 2, 2, 2, 3]) # token 1 has no frame | |
| with self.assertRaises(ValueError): | |
| model.expand(hidden, torch.tensor([[1, -1, 1, 1]]), torch.ones(1, 4, dtype=torch.bool), torch.tensor([0])) | |
| with self.assertRaises(ValueError): | |
| model.expand(hidden, torch.zeros(1, 4, dtype=torch.long), torch.ones(1, 4, dtype=torch.bool), torch.tensor([0])) | |
| def test_padding_does_not_change_the_short_example(self): | |
| c = small_config() | |
| model = Fiato(c).eval() | |
| with torch.no_grad(): # make the heads non-trivial so the test has teeth | |
| for head in (model.vocoder.harmonic, model.vocoder.noise): | |
| head.weight.normal_(std=0.05) | |
| model.duration.out.weight.zero_() # three frames per token: rows differ in length | |
| model.duration.out.bias.fill_(math.log1p(3)) | |
| a, b = sample(c), sample(c, voice=1, durations=(5, 6, 7, 6, 12, 3, 6, 14, 7, 6, 2, 8)) | |
| tokens = torch.stack((a["tokens"], b["tokens"])) | |
| stress = torch.stack((a["stress"], b["stress"])) | |
| lengths = torch.tensor([7, len(SYMBOLS)]) # first row uses only 7 tokens: genuine token padding | |
| tokens[0, 7:], stress[0, 7:] = 0, 0 | |
| wave, samples, details = model.synthesize(tokens, stress, lengths, torch.tensor([0, 1])) | |
| alone, n, _ = model.synthesize(tokens[:1, :7], stress[:1, :7], lengths[:1], torch.tensor([0])) | |
| self.assertEqual(int(n[0]), int(samples[0])) | |
| self.assertLess(int(samples[0]), int(samples[1])) | |
| # Float32 rounding through exp() and two STFT round trips reaches ~1e-4; the | |
| # padding bugs this guards against (pitch glide into padding, shared noise | |
| # stream, missing end frames) each moved samples by more than 1e-2. | |
| torch.testing.assert_close(wave[0, :int(n[0])], alone[0], atol=5e-4, rtol=1e-3) | |
| self.assertGreater(float(alone.abs().max()), 0.01) | |
| self.assertEqual(float(wave[0, int(n[0]):].abs().max()), 0.0) | |
| # Every phoneme is audible; pause tokens may get zero frames. | |
| phones = model.phone_tokens[tokens[1]] | |
| self.assertTrue(bool((details["durations"][1][phones] >= 1).all())) | |
| for speed in (0, float("nan"), 5): | |
| with self.assertRaises(ValueError): | |
| model.synthesize(tokens, stress, lengths, torch.tensor([0, 1]), speed) | |
| def test_source_filter_is_differentiable_and_respects_lengths(self): | |
| c = small_config() | |
| head = SourceFilter(c) | |
| hidden = torch.randn(2, 12, c.channels, requires_grad=True) | |
| with torch.no_grad(): | |
| head.harmonic.weight.normal_(std=0.05) | |
| head.noise.weight.normal_(std=0.05) | |
| wave = head(hidden, torch.full((2, 12), 180.0), torch.tensor([12, 7])) | |
| self.assertEqual(wave.shape, (2, 12 * c.hop)) | |
| self.assertEqual(float(wave[1, 7 * c.hop:].abs().max()), 0.0) | |
| wave.square().mean().backward() | |
| self.assertTrue(torch.isfinite(hidden.grad).all()) | |
| self.assertGreater(float(hidden.grad[1, :7].abs().sum()), 0) | |
| self.assertEqual(float(hidden.grad[1, 7:].abs().sum()), 0.0) # padded frames cannot influence audio | |
| def test_both_training_phases_update_every_inference_module(self): | |
| c = small_config() | |
| system, critics = TrainingSystem(c), Discriminators(periods=(2, 3), resolutions=((256, 64),)) | |
| optimizer = torch.optim.AdamW(system.parameters(), lr=1e-3) | |
| d_optimizer = torch.optim.AdamW(critics.parameters(), lr=1e-3) | |
| batch = collate([sample(c), sample(c, voice=1, durations=(5, 6, 7, 6, 12, 3, 6, 14, 7, 6, 0, 8))]) | |
| before = {name: value.detach().clone() for name, value in system.model.named_parameters()} | |
| for step in (0, 1, 2): | |
| metrics = optimize_step(system, critics, optimizer, d_optimizer, batch, step, RECIPE) | |
| self.assertTrue(all(math.isfinite(v) for v in metrics.values()), metrics) | |
| self.assertIn("adversarial", metrics) | |
| self.assertGreater(metrics["gan_scale"], 0) | |
| unchanged = [name for name, value in system.model.named_parameters() if torch.equal(value, before[name])] | |
| self.assertEqual(unchanged, []) | |
| self.assertTrue(all(p.grad is None or torch.isfinite(p.grad).all() for p in system.parameters())) | |
| def test_adversarial_gradient_is_balanced_against_reconstruction(self): | |
| # Regression for the "GAN phase that does nothing": with fixed weights the adversarial | |
| # gradient at the waveform was under 1% of the spectral one on the real run. | |
| c = small_config() | |
| torch.manual_seed(5) | |
| system, critics = TrainingSystem(c), Discriminators(periods=(2, 3), resolutions=((256, 64),)) | |
| batch = collate([sample(c), sample(c, voice=1)]) | |
| loss, _, auxiliary = system.objective(batch, 1.0, random_phase=False) | |
| from fiato.losses import generator_gan_loss | |
| from fiato.train import crops | |
| wave = auxiliary["wave"] | |
| fake, real, index = crops(wave, batch["wave"], batch["frame_lengths"] * c.hop, 4096, 1) | |
| self.assertEqual(index.shape, (2, 4096)) | |
| torch.testing.assert_close(fake, wave.gather(1, index)) | |
| adversarial, matching = generator_gan_loss(critics(real), critics(fake)) | |
| gan = adversarial + 30.0 * matching | |
| reconstruction = torch.autograd.grad(auxiliary["reconstruction"], wave, retain_graph=True)[0].gather(1, index) | |
| raw = torch.autograd.grad(gan, wave, retain_graph=True)[0].gather(1, index) | |
| scale = 0.7 * reconstruction.norm() / raw.norm() | |
| balanced = torch.autograd.grad(scale.detach() * gan, wave)[0].gather(1, index) | |
| self.assertAlmostEqual(float(balanced.norm() / reconstruction.norm()), 0.7, places=4) | |
| def test_pause_tokens_vanish_unless_predicted_present_and_pitch_range_scales_movement(self): | |
| model = Fiato(small_config()).eval() | |
| tokens = sample(small_config())["tokens"][None] | |
| mask = torch.ones_like(tokens, dtype=torch.bool) | |
| hidden = torch.zeros(1, tokens.shape[1], 32) | |
| phones = model.phone_tokens[tokens[0]] | |
| with torch.no_grad(): | |
| model.duration.out.weight.zero_() | |
| model.duration.out.bias.copy_(torch.tensor([math.log1p(5), -3.0])) # 5 frames, "no pause" | |
| absent = model.predict_durations(hidden, mask, tokens)[0] | |
| model.duration.out.bias.copy_(torch.tensor([math.log1p(5), 3.0])) # 5 frames, "pause" | |
| present = model.predict_durations(hidden, mask, tokens)[0] | |
| model.duration.out.bias.copy_(torch.tensor([-2.0, -3.0])) # shorter than a frame | |
| floor = model.predict_durations(hidden, mask, tokens)[0] | |
| self.assertEqual(absent[phones].tolist(), [5] * int(phones.sum())) | |
| self.assertEqual(absent[~phones].tolist(), [0] * int((~phones).sum())) | |
| self.assertEqual(present.tolist(), [5] * tokens.shape[1]) | |
| self.assertEqual(floor[phones].tolist(), [1] * int(phones.sum())) # a phoneme is never dropped | |
| self.assertEqual(model.predict_durations(hidden, mask, tokens, speed=2.5)[0][phones].tolist(), [1] * int(phones.sum())) | |
| prosody = torch.tensor([[[0.0, 1.0, 0.0], [0.4, 1.0, 0.0], [9.0, -1.0, 0.0], [-0.4, 1.0, 0.0]]]) | |
| frame_mask = torch.ones(1, 4, dtype=torch.bool) | |
| wide = Fiato.widen_pitch(prosody, frame_mask, 1.5) | |
| torch.testing.assert_close(wide[0, [0, 1, 3], 0], torch.tensor([0.0, 0.6, -0.6])) # around the voiced mean (0) | |
| torch.testing.assert_close(wide[..., 1:], prosody[..., 1:]) | |
| self.assertIs(Fiato.widen_pitch(prosody, frame_mask, 1.0), prosody) | |
| def test_branching_keeps_old_weights_when_a_layer_grows(self): | |
| old = {"weight": torch.ones(1, 3), "bias": torch.full((1,), 2.0), "gone": torch.zeros(1)} | |
| layer = torch.nn.Linear(3, 2) | |
| fresh = layer.weight.detach().clone() | |
| report = load_compatible(layer, old) | |
| self.assertEqual(report, {"grown": ["weight", "bias"], "new": [], "dropped": ["gone"]}) | |
| torch.testing.assert_close(layer.weight[0], torch.ones(3)) | |
| torch.testing.assert_close(layer.weight[1], fresh[1]) # the new row keeps its initialisation | |
| self.assertEqual(float(layer.bias[0]), 2.0) | |
| with self.assertRaisesRegex(ValueError, "changed shape"): | |
| load_compatible(torch.nn.Linear(2, 2), {"weight": torch.ones(2, 3), "bias": torch.ones(2)}) | |
| def test_teacher_schedule_and_learning_rate(self): | |
| self.assertEqual(teacher_probability(0, 100, 200, 0.5), 1.0) | |
| self.assertAlmostEqual(teacher_probability(200, 100, 200, 0.5), 0.75) | |
| self.assertEqual(teacher_probability(10_000, 100, 200, 0.5), 0.5) | |
| self.assertAlmostEqual(learning_rate(0, 1e-3, 10, 1000), 1e-4) | |
| self.assertAlmostEqual(learning_rate(1000, 1e-3, 10, 1000), 5e-4) | |
| self.assertAlmostEqual(learning_rate(1000, 1e-3, 10, 1000, start=1000), 5e-5) # a branched run warms up again | |
| c = small_config() | |
| system = TrainingSystem(c).eval() | |
| batch = collate([sample(c), sample(c, voice=1)]) | |
| with torch.no_grad(): | |
| teacher = system.objective(batch, 1.0, random_phase=False)[1] | |
| free = system.objective(batch, 0.0, random_phase=False)[1] | |
| self.assertEqual(float(teacher["teacher_fraction"]), 1.0) | |
| self.assertEqual(float(free["teacher_fraction"]), 0.0) | |
| torch.testing.assert_close(teacher["pitch"], free["pitch"]) # predictors do not depend on the conditioning choice | |
| # The pitch-spread term compares per-utterance standard deviations over trusted voiced frames. | |
| from fiato.train import masked_std | |
| values = torch.tensor([[1.0, 3.0, 100.0, 5.0], [2.0, 2.0, 2.0, 2.0]]) | |
| mask = torch.tensor([[1.0, 1.0, 0.0, 1.0], [1.0, 1.0, 1.0, 0.0]]) | |
| torch.testing.assert_close(masked_std(values, mask), torch.tensor([(8 / 3) ** 0.5, 1e-4]), atol=1e-5, rtol=1e-4) | |
| self.assertGreater(float(teacher["pitch_spread"]), 0) | |
| # An untrained model's ratio depends on its random weights (2 to 9 across seeds): only its sanity is checked. | |
| self.assertTrue(0 <= float(teacher["pitch_spread_ratio"]) < 100) | |
| def test_objective_rejects_durations_that_do_not_cover_the_frames(self): | |
| c = small_config() | |
| system = TrainingSystem(c).eval() | |
| item = sample(c) | |
| item["durations"] = item["durations"].clone() | |
| item["durations"][4] -= 1 | |
| with self.assertRaisesRegex(ValueError, "frame count"), torch.no_grad(): | |
| system.objective(collate([item]), 1.0) | |
| def test_export_roundtrip_and_explicit_rejection_of_old_checkpoints(self): | |
| model = Fiato(small_config()).eval() | |
| with tempfile.TemporaryDirectory() as directory: | |
| path = Path(directory) / "model.pt" | |
| export_model(model, path, 7, "annotation") | |
| restored = load_model(path) | |
| self.assertEqual(set(model.state_dict()), set(restored.state_dict())) | |
| for name, value in model.state_dict().items(): | |
| torch.testing.assert_close(value, restored.state_dict()[name], rtol=0, atol=0) | |
| self.assertFalse(any("mel_head" in key or "critic" in key for key in restored.state_dict())) | |
| state = torch.load(path, weights_only=True) | |
| self.assertEqual(state["format_version"], CHECKPOINT_FORMAT) | |
| for change, message in (({"format_version": 3}, "byte-token"), | |
| ({"frontend": {**state["frontend"], "fingerprint": "other"}}, "front end")): | |
| atomic_save({**state, **change}, path) | |
| with self.assertRaisesRegex(ValueError, message): | |
| load_model(path) | |
| def test_training_cli_resumes_same_as_uninterrupted_run(self): | |
| c = small_config() | |
| with tempfile.TemporaryDirectory() as directory: | |
| root = Path(directory) | |
| (root / "config.json").write_text(json.dumps(c.to_dict())) | |
| train = write_annotations(root, c, "train") | |
| validation = write_annotations(root, c, "validation") | |
| base = [sys.executable, "-m", "fiato.train", "--train", str(train), "--validation", str(validation), | |
| "--config", str(root / "config.json"), "--device", "cpu", "--batch-frames", "200", "--max-batch", "2", | |
| "--save-every", "1", "--gan-start", "1", "--warmup", "2", "--teacher-hold", "1", "--teacher-ramp", "1", | |
| "--crop-samples", "4096", "--crops", "1", "--validation-size", "2", "--max-seconds", "300", "--keep-every", "2"] | |
| env = {**os.environ, "OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1"} | |
| def run(arguments, expect=0): | |
| result = subprocess.run(base + arguments, capture_output=True, text=True, env=env, timeout=600) | |
| self.assertEqual(result.returncode != 0, expect != 0, result.stdout + result.stderr) | |
| return result | |
| run(["--output", str(root / "resumed"), "--steps", "1"]) | |
| run(["--output", str(root / "resumed"), "--steps", "2", "--resume", str(root / "resumed/last.pt")]) | |
| run(["--output", str(root / "continuous"), "--steps", "2"]) | |
| resumed = torch.load(root / "resumed/last.pt", weights_only=True) | |
| continuous = torch.load(root / "continuous/last.pt", weights_only=True) | |
| self.assertEqual(resumed["step"], 2) | |
| self.assertEqual(resumed["format_version"], CHECKPOINT_FORMAT) | |
| self.assertEqual(torch.load(root / "resumed/step-00000001.pt", weights_only=True)["step"], 1) | |
| self.assertTrue((root / "resumed/step-00000002.pt").is_file()) | |
| for key, value in resumed["system"].items(): | |
| torch.testing.assert_close(value, continuous["system"][key], rtol=0, atol=0) | |
| record = json.loads((root / "resumed/metrics.jsonl").read_text().splitlines()[-1]) | |
| self.assertIn("validation_predicted_mel", record) | |
| self.assertIn("steps_per_second", record) | |
| self.assertEqual(load_model(root / "resumed/inference.pt").config, c) | |
| # A deadline that has already passed refuses to start; a stale format refuses to resume. | |
| (root / "budget.json").write_text(json.dumps({"deadline": 1.0})) | |
| failed = run(["--output", str(root / "late"), "--steps", "1", "--deadline-file", str(root / "budget.json")], expect=1) | |
| self.assertIn("deadline", failed.stderr) | |
| atomic_save({**resumed, "format_version": 3}, root / "old.pt") | |
| failed = run(["--output", str(root / "resumed"), "--steps", "3", "--resume", str(root / "old.pt")], expect=1) | |
| self.assertIn("cannot be resumed", failed.stderr) | |
| # A changed recipe cannot be resumed, but it can be branched into a new run, which then resumes normally. | |
| changed = ["--gan-ratio", "0.4", "--weights", '{"mel": 5}'] | |
| failed = run(["--output", str(root / "resumed"), "--steps", "3", "--resume", str(root / "resumed/last.pt")] + changed, expect=1) | |
| self.assertIn("--init-from", failed.stderr) | |
| run(["--output", str(root / "branch"), "--steps", "3", "--init-from", str(root / "resumed/last.pt")] + changed) | |
| branch = torch.load(root / "branch/last.pt", weights_only=True) | |
| self.assertEqual(branch["step"], 3) | |
| self.assertEqual(branch["recipe"]["branched_from"]["step"], 2) | |
| self.assertEqual(branch["recipe"]["weights"]["mel"], 5.0) | |
| # Same architecture, source already in its adversarial phase: critics and optimizer moments carry over. | |
| self.assertEqual(branch["recipe"]["branched_from"]["loaded"], | |
| {"grown": [], "new": [], "dropped": [], "critics": True, "optimizer": True}) | |
| run(["--output", str(root / "branch"), "--steps", "4", "--resume", str(root / "branch/last.pt")] + changed) | |
| self.assertEqual(torch.load(root / "branch/last.pt", weights_only=True)["step"], 4) | |
| if __name__ == "__main__": | |
| unittest.main() | |