import itertools import unittest import numpy as np import torch from fiato.align import AlignmentSettings, assemble, ctc_viterbi, frame_energy_db, silence_mask, spans_from_states, token_spans from fiato.annotate import boundary_offsets, gate, summarize_offsets from fiato.data import continuous_pitch from fiato.refine import banded_viterbi, frame_labels, symbol_class SETTINGS = AlignmentSettings() HOP, RATE = 256, 24000 def collapse(path, blank=0): out, previous = [], blank for label in path: if label != blank and label != previous: out.append(label) previous = label return out class AlignmentTests(unittest.TestCase): def test_ctc_viterbi_matches_exhaustive_search_and_torchaudio(self): rng = np.random.default_rng(3) for labels in ([1, 2], [1, 1, 2], [2, 1, 2]): log_probs = np.log(rng.dirichlet(np.ones(3), size=7)).astype(np.float32) best, best_path = -np.inf, None for path in itertools.product(range(3), repeat=7): if collapse(path) == labels: score = sum(log_probs[t, s] for t, s in enumerate(path)) if score > best: best, best_path = score, path states = ctc_viterbi(log_probs, np.array(labels)) extended = [0 if s % 2 == 0 else labels[s // 2] for s in states] self.assertEqual(tuple(extended), best_path) # The production path (torchaudio when installed) gives the same emission spans. self.assertEqual(token_spans(log_probs, labels), spans_from_states(states, len(labels))) with self.assertRaises(ValueError): token_spans(np.zeros((2, 3), dtype=np.float32), [1, 1]) # a repeat needs a blank between def test_durations_follow_emission_onsets_and_cover_every_frame(self): symbols = ("", "h", "aɪ", "", "ð", "ɛɹ", ",", "", "t", "uː", ".", "") # Emission frames (20 ms each) of the six phones; a long gap before "t" holds a pause. spans = [(10, 10), (15, 16), (30, 30), (34, 34), (70, 70), (75, 75)] frames = 190 silent = np.zeros(frames, dtype=bool) silent[:16] = True # leading silence silent[90:120] = True # pause after the comma silent[170:] = True # trailing silence durations = assemble(symbols, spans, frames, silent, HOP, RATE, SETTINGS) self.assertEqual(int(durations.sum()), frames) self.assertEqual(len(durations), len(symbols)) self.assertTrue(all(durations[i] >= 1 for i in (1, 2, 4, 5, 8, 9))) self.assertEqual(durations[0], 16) # BOS takes the leading silence self.assertEqual(durations[3], 0) # word boundary without a pause self.assertEqual(durations[6], 30) # the comma carries the pause, not the after it self.assertEqual(durations[7], 0) self.assertEqual(durations[10], 0) # final punctuation: EOS owns trailing silence self.assertEqual(durations[11], 20) # "aɪ" starts 19 ms before its first emission frame (15 * 20 ms) and lasts to the next onset. start = int(np.ceil((15 * 0.02 + SETTINGS.onset_offset_seconds) * RATE / HOP)) self.assertEqual(int(durations[:2].sum()), start) self.assertEqual(int(durations[:3].sum()), int(np.ceil((30 * 0.02 + SETTINGS.onset_offset_seconds) * RATE / HOP))) def test_short_silences_stay_with_phones_and_bad_input_is_rejected(self): symbols = ("", "æ", "t", "", "eɪ", "t", "") spans = [(5, 5), (12, 12), (22, 22), (30, 30)] silent = np.zeros(80, dtype=bool) silent[28:32] = True # a 4-frame stop closure is not a pause durations = assemble(symbols, spans, 80, silent, HOP, RATE, SETTINGS) self.assertEqual(durations[3], 0) self.assertEqual(int(durations.sum()), 80) with self.assertRaises(ValueError): assemble(symbols, spans[:-1], 80, silent, HOP, RATE, SETTINGS) with self.assertRaises(ValueError): # more phones than frames can hold assemble(symbols, [(0, 0), (1, 1), (2, 2), (3, 3)], 3, np.zeros(3, dtype=bool), HOP, RATE, SETTINGS) def test_energy_silence_detection(self): wave = np.zeros(HOP * 100) wave[HOP * 30:HOP * 70] = 0.3 * np.sin(np.arange(HOP * 40) * 0.2) wave += 1e-4 * np.random.default_rng(0).standard_normal(len(wave)) level = frame_energy_db(wave, HOP) silent, threshold = silence_mask(level, SETTINGS) self.assertEqual(len(level), 100) self.assertTrue(silent[:28].all() and silent[72:].all()) self.assertFalse(silent[31:69].any()) self.assertLess(threshold, -30) def test_banded_refinement_moves_boundaries_only_within_the_band(self): truth = [6, 9, 5] classes = np.repeat([0, 1, 2], truth) scores = np.full((20, 3), -5.0) scores[np.arange(20), classes] = 0.0 coarse_starts, coarse_ends = np.array([0, 8, 13]), np.array([8, 13, 20]) # first boundary 2 late, second 2 early self.assertEqual(banded_viterbi(scores, coarse_starts, coarse_ends, band=3).tolist(), truth) limited = banded_viterbi(scores, coarse_starts, coarse_ends, band=1) self.assertEqual(limited.tolist(), [7, 7, 6]) # moves by at most one frame self.assertEqual(int(limited.sum()), 20) self.assertEqual(frame_labels(("", "t", "", "aɪ"), [2, 1, 0, 3]).tolist(), [symbol_class("")] * 2 + [symbol_class("t")] + [symbol_class("aɪ")] * 3) def test_voicing_probe_measures_boundary_offsets(self): symbols = ("", "s", "iː", "t", "") durations = [2, 6, 10, 5, 2] voiced = np.zeros(25, dtype=bool) voiced[7:19] = True # vowel is voiced one frame early and ends one frame late found = boundary_offsets(symbols, durations, voiced) self.assertEqual(found, {"vowel_to_voiceless": [1], "fricative_to_vowel": [-1]}) summary = summarize_offsets({"vowel_to_voiceless": [1, 0, None], "fricative_to_vowel": []}) self.assertEqual(summary["vowel_to_voiceless"]["boundaries"], 3) self.assertAlmostEqual(summary["vowel_to_voiceless"]["flip_found"], 0.6667, places=3) self.assertEqual(summary["vowel_to_voiceless"]["within_1_frame"], 1.0) def test_quality_gate_names_every_reason(self): limits = {"max_weak_phone_fraction": 0.25, "min_token_probability": 0.5, "max_phone_frames": 70, "min_vowel_voiced": 0.5, "min_phones_per_second": 4.0, "max_phones_per_second": 30.0, "min_expression_frames": 15} spoken = {"symbols": " h aɪ , j uː ", "durations": [0, 6, 9, 3, 40, 5, 12, 4]} good = {**spoken, "unknown": [], "qc": {"weak_phone_fraction": 0.02, "token_probability_mean": 0.85, "max_phone_frames": 30, "vowel_voiced": 0.93, "phones_per_second": 12.5}} self.assertEqual(gate(good, limits), []) silent = {**good, "durations": [0, 6, 9, 43, 0, 5, 12, 4]} self.assertEqual(gate(silent, limits), ["expression_missing"]) bad = {**spoken, "unknown": ["ʘ"], "qc": {"weak_phone_fraction": 0.4, "token_probability_mean": 0.3, "max_phone_frames": 90, "vowel_voiced": 0.2, "phones_per_second": 40.0}} self.assertEqual(gate(bad, limits), ["unknown_phoneme", "weak_alignment", "low_confidence", "overlong_phone", "vowels_not_voiced", "implausible_rate"]) def test_pitch_is_made_continuous_and_octave_errors_are_distrusted(self): pitch = torch.tensor([0.0, 0.0, 0.1, 0.1, 0.8, 0.1, 0.1, 0.0, 0.0, 0.3, 0.3, 0.3, 0.0]) voiced = torch.tensor([0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0.0]) curve, trusted = continuous_pitch(pitch, voiced) self.assertEqual(trusted.tolist(), [0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0]) torch.testing.assert_close(curve[:7], torch.full((7,), 0.1)) # outlier replaced, start extended torch.testing.assert_close(curve[6:10], torch.tensor([0.1, 0.1 + 0.2 / 3, 0.1 + 0.4 / 3, 0.3])) self.assertAlmostEqual(float(curve[-1]), 0.3, places=6) flat, none = continuous_pitch(torch.zeros(5), torch.zeros(5), default=-0.5) self.assertEqual(flat.tolist(), [-0.5] * 5) self.assertEqual(float(none.sum()), 0.0) if __name__ == "__main__": unittest.main()