fiato / tests /test_data.py
cfahlgren1's picture
cfahlgren1 HF Staff
rename the project to fiato
4744deb verified
Raw History Blame Contribute Delete
14 kB
from dataclasses import replace
import importlib.util
import io
import json
import math
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import numpy as np
import torch
from fiato import text
from fiato.annotate import load_acoustic_rows
from fiato.config import Config
from fiato.data import AnnotatedDataset, BucketSampler, collate
from fiato.journal import append, recover
from fiato.losses import SpectralLoss, mel_filterbank
from fiato.train import balanced_indices, check_leakage
from helpers import small_config, write_annotations
# Fingerprint stored in every feature manifest written by the byte-token preparation
# on the real corpus (n4ze3m/fiato @ 93dfa15b). Those files remain usable.
class DataTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
torch.set_num_threads(1)
def test_existing_acoustic_features_stay_compatible_while_tokens_are_versioned_separately(self):
c = Config()
changed = replace(c, channels=128, dropout=0.2, acoustic_layers=3, cep_causal=64)
self.assertEqual(c.acoustic_fingerprint, changed.acoustic_fingerprint) # model changes reuse features
self.assertNotEqual(c.fingerprint, changed.fingerprint)
self.assertNotEqual(c.acoustic_fingerprint, replace(c, n_mels=40).acoustic_fingerprint)
baseline = c.acoustic_fingerprint
with patch.object(text, "FRONTEND_VERSION", 999):
self.assertEqual(c.acoustic_fingerprint, baseline) # a front-end change does not touch acoustics
def test_dataset_reads_annotations_and_ignores_stale_byte_tokens(self):
c = small_config()
with tempfile.TemporaryDirectory() as directory:
manifest = write_annotations(directory, c, "train")
data = AnnotatedDataset(manifest, c)
item = data[1]
self.assertEqual(item["tokens"].tolist(), data.rows[1]["tokens"]) # not the [1, 60, 75, 2] stored in the file
self.assertEqual(int(item["durations"].sum()), item["mel"].shape[0])
self.assertEqual(item["wave"].numel(), item["mel"].shape[0] * c.hop)
self.assertEqual(item["prosody"].shape, (item["mel"].shape[0], 3))
# Pitch is continuous through unvoiced edges; energy is normalized.
self.assertTrue(bool((item["prosody"][:3, 0] == item["prosody"][data.rows[1]["durations"][0], 0]).all()))
self.assertAlmostEqual(float(item["prosody"][0, 2]), (-3.0 - c.energy_mean) / c.energy_scale, places=6)
# Vowel frames are marked voiced when asked; the pitch trust mask is left alone.
from fiato.data import VOWEL_IDS
stored = torch.load(Path(directory) / "features" / data.rows[1]["path"], weights_only=True)
stored["prosody"][:, 1] = 0 # a tracker that heard no voicing at all
torch.save(stored, Path(directory) / "features" / data.rows[1]["path"])
plain = AnnotatedDataset(manifest, c, cache=False)[1]
fixed = AnnotatedDataset(manifest, c, cache=False, voiced_vowels=True)[1]
vowel = torch.repeat_interleave(torch.tensor([t in VOWEL_IDS for t in data.rows[1]["tokens"]]), torch.tensor(data.rows[1]["durations"]))
self.assertEqual(float(plain["prosody"][:, 1].sum()), 0.0)
self.assertTrue(bool((fixed["prosody"][:, 1].bool() == vowel).all()))
self.assertGreater(int(vowel.sum()), 0)
torch.testing.assert_close(fixed["pitch_trusted"], plain["pitch_trusted"])
batch = collate([data[0], data[3]])
self.assertEqual(batch["frame_lengths"].tolist(), [data.rows[0]["frames"], data.rows[3]["frames"]])
self.assertEqual(batch["durations"].sum(1).tolist(), batch["frame_lengths"].tolist())
def test_stale_or_inconsistent_annotations_are_rejected_explicitly(self):
c = small_config()
cases = [
(dict(edit=lambda r: r.update(frontend="front-end-2-byte-tokens")), "different front end"),
(dict(edit=lambda r: r.pop("durations")), "different front end"),
(dict(edit=lambda r: r.update(acoustic="other")), "acoustic feature settings"),
(dict(edit=lambda r: r.update(annotation="another-run")), "different annotation run"),
(dict(edit=lambda r: r["durations"].__setitem__(4, r["durations"][4] - 1)), "do not cover the frames"),
(dict(edit=lambda r: (r["durations"].__setitem__(1, 0), r["durations"].__setitem__(4, r["durations"][4] + 4))), "phoneme without frames"),
(dict(edit=lambda r: r["tokens"].__setitem__(1, 200)), "invalid token"),
(dict(complete=False), "incomplete"),
]
for options, message in cases:
with tempfile.TemporaryDirectory() as directory, self.subTest(message=message):
manifest = write_annotations(directory, c, "train", **options)
with self.assertRaisesRegex(ValueError, message):
AnnotatedDataset(manifest, c)
with tempfile.TemporaryDirectory() as directory:
# A manifest from the old preparation (no annotation layer next to it) is refused outright.
legacy = Path(directory) / "train.jsonl"
legacy.write_text(json.dumps({"id": "a", "features": "byte-token-preparation", "path": "a.pt"}) + "\n")
with self.assertRaisesRegex(ValueError, "legacy byte-token manifests are stale"):
AnnotatedDataset(legacy, c)
# The annotation step accepts feature files made with the current audio settings only.
(Path(directory) / "train.preparation.json").write_text(json.dumps({"complete": True}))
rows = [{"id": "a", "acoustic": Config().acoustic_fingerprint, "speaker": "F1"}, {"id": "b", "acoustic": Config().acoustic_fingerprint, "speaker": "M1"}]
legacy.write_text("".join(json.dumps(r) + "\n" for r in rows))
self.assertEqual(len(load_acoustic_rows(Path(directory), "train", Config())), 2)
legacy.write_text(json.dumps({"id": "a", "features": "something-else", "speaker": "F1"}) + "\n")
with self.assertRaisesRegex(ValueError, "compatible audio/DSP"):
load_acoustic_rows(Path(directory), "train", Config())
def test_unicode_separators_are_not_record_boundaries_and_journals_recover(self):
c = small_config()
with tempfile.TemporaryDirectory() as directory:
manifest = write_annotations(directory, c, "train", edit=lambda r: r.update(text="one
two
three\u0085four"))
data = AnnotatedDataset(manifest, c)
self.assertEqual(len(data), 4)
self.assertEqual(data.rows[0]["text"], "one
two
three\u0085four")
journal = Path(directory) / "journal.jsonl"
with journal.open("a", encoding="utf-8") as stream:
append(stream, {"id": 1}, sync=False)
append(stream, {"id": 2})
stream.write('{"id": 3, "torn":')
self.assertEqual([r["id"] for r in recover(journal)], [1, 2])
self.assertTrue(journal.read_text().endswith("\n"))
def test_bucket_sampler_respects_the_frame_budget_and_restores_exactly(self):
lengths = [50, 60, 400, 410, 55, 900, 65, 420]
sampler = BucketSampler(lengths, frame_budget=1000, max_batch=3, seed=5, pool=16)
batches = [sampler.next() for _ in range(40)]
for batch in batches:
self.assertLessEqual(len(batch), 3)
self.assertTrue(len(batch) == 1 or len(batch) * max(lengths[i] for i in batch) <= 1000)
self.assertEqual({i for b in batches for i in b}, set(range(8)))
state = json.loads(json.dumps({"pending": sampler.state()["pending"]}))
generator = sampler.state()["generator"].clone()
following = [sampler.next() for _ in range(25)]
restored = BucketSampler(lengths, frame_budget=1000, max_batch=3, seed=999, pool=16)
restored.load_state({"generator": generator, "pending": state["pending"]})
self.assertEqual([restored.next() for _ in range(25)], following)
def test_balanced_selection_and_leakage_report(self):
rows = [{"speaker": "F1"}] * 8 + [{"speaker": "M1"}] * 8
indices = balanced_indices(SimpleNamespace(rows=rows), 4)
self.assertEqual([rows[i]["speaker"] for i in indices], ["F1", "M1", "F1", "M1"])
train = SimpleNamespace(rows=[{"id": "a_F2", "text_hash": "same", "text": "Hello."}])
validation = SimpleNamespace(rows=[{"id": "a_M2", "text_hash": "same", "text": "Hello."}])
with self.assertRaisesRegex(ValueError, "a_M2.*a_F2"):
check_leakage(train, validation)
def test_spectral_loss_masks_padding_and_survives_silence(self):
loss = SpectralLoss(24000)
torch.manual_seed(1)
target = 0.1 * torch.randn(2, 12000)
predicted = (target + 0.05 * torch.randn(2, 12000)).requires_grad_()
lengths = torch.tensor([12000, 7936])
stft, mel = loss(predicted, target, lengths)
(stft + mel).backward()
self.assertTrue(torch.isfinite(predicted.grad).all())
self.assertEqual(float(predicted.grad[1, 9500:].abs().max()), 0.0) # beyond the short utterance (plus one window)
alone = loss(predicted[1:, :7936].detach(), target[1:, :7936], lengths[1:])
padded = loss(torch.cat((predicted[1:, :7936].detach(), torch.zeros(1, 4064)), 1),
torch.cat((target[1:, :7936], torch.zeros(1, 4064)), 1), lengths[1:])
torch.testing.assert_close(torch.stack(alone), torch.stack(padded), atol=1e-5, rtol=1e-5)
same = loss(target, target, lengths)
self.assertLess(float(same[0]), 1e-3)
self.assertLess(float(same[1]), 1e-4)
for amplitude in (0.0, 1e-7):
output = (amplitude * torch.randn(1, 4096)).requires_grad_()
stft, mel = loss(output, torch.zeros(1, 4096), torch.tensor([4096]))
(stft + mel).backward()
self.assertTrue(torch.isfinite(output.grad).all())
self.assertTrue(math.isfinite(float(stft)) and float(stft) < 100)
filters = mel_filterbank(24000, 1024, 80)
self.assertEqual(filters.shape, (80, 513))
self.assertTrue(bool((filters >= 0).all()) and bool((filters.sum(1) > 0).all()))
@unittest.skipUnless(importlib.util.find_spec("librosa") and importlib.util.find_spec("pyarrow"), "requires preparation extras")
def test_acoustic_preparation_interrupt_resume_provenance_and_bad_rows(self):
import pyarrow as pa
import pyarrow.parquet as pq
import soundfile as sf
from fiato.prepare import extract, prepare_split
c = Config()
time = np.arange(12000) / 24000
def audio(rate):
stream = io.BytesIO()
sf.write(stream, 0.1 * np.sin(2 * np.pi * 200 * time), rate, format="WAV")
return {"bytes": stream.getvalue()}
rows = [{"id": str(i), "text": "Hello.", "voice_id": "F1", "audio": audio(24000), "reference_audio": b"reference bytes",
"source_license": "test-synthetic"} for i in range(5)]
rows[2]["text"] = None
rows[3]["audio"] = None
rows[4]["audio"] = audio(16000)
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
source = root / "source.parquet"
pq.write_table(pa.Table.from_pylist(rows), source)
output = root / "features"
output.mkdir()
args = SimpleNamespace(output=output, inputs=[source], split="train", text_column="text", speaker_column="voice_id", resume=False, workers=1)
calls = {"count": 0}
def interrupted(wave, config):
calls["count"] += 1
if calls["count"] == 2:
raise KeyboardInterrupt
return extract(wave, config)
with patch("fiato.prepare.extract", side_effect=interrupted):
with self.assertRaises(KeyboardInterrupt):
prepare_split(args, c)
committed = recover(output / "train.jsonl")
self.assertEqual(len(committed), 1)
self.assertNotIn("phonemes", committed[0]) # acoustic only: no tokens to go stale
self.assertEqual(committed[0]["acoustic"], c.acoustic_fingerprint)
self.assertEqual(committed[0]["provenance"]["reference_audio"]["binary_bytes"], 15)
self.assertEqual(committed[0]["provenance"]["source_license"], "test-synthetic")
first_path = output / committed[0]["path"]
first_time = first_path.stat().st_mtime_ns
with (output / "train.jsonl").open("ab") as stream:
stream.write(b'{"interrupted":')
args.resume = True
with patch("fiato.prepare.extract", wraps=extract) as dsp:
prepare_split(args, c)
self.assertEqual(dsp.call_count, 1)
self.assertEqual(first_path.stat().st_mtime_ns, first_time)
self.assertEqual(len(recover(output / "train.jsonl")), 2)
rejected = recover(output / "train.rejected.jsonl")
self.assertEqual(len(rejected), 3)
self.assertTrue(any("24000" in r["reason"] for r in rejected))
self.assertTrue(json.loads((output / "train.preparation.json").read_text())["complete"])
stored = torch.load(first_path, weights_only=True)
self.assertEqual(set(stored), {"wave", "mel", "prosody", "speaker"})
voiced = stored["prosody"][:, 1].bool()
self.assertGreater(int(voiced.sum()), 10)
self.assertLess(float(stored["prosody"][voiced, 0].abs().median()), 0.1) # 200 Hz -> log(F0/200) = 0
self.assertEqual(len(load_acoustic_rows(output, "train", c)), 2)
if __name__ == "__main__":
unittest.main()