WO-3 T7: SIMD blend shape solver from raw audio energy features
Browse filesEight bandpass resonators (RBJ constant-skirt biquads, transposed direct
form II) evaluated in f32x4 across BANDS, not samples: the filters are
recursive so the sample axis cannot be vectorised, while eight bands see the
identical input sample. f32x4 rather than f32x8 because this machine reports
sse4_2 with no AVX/AVX2/FMA, where wide emulates 256-bit as two halves.
Spectral shape (loudness removed) is mapped onto mouth channels by a fixed
basis with inhibitory terms: rounding is energy at 1.2 kHz IN THE ABSENCE OF
energy at 2 kHz, and without the negative weights every vowel would both
round and smile a little and converge on one mush shape.
Documented as a heuristic, not a trained regressor. TONGUE_OUT is never
driven -- tongue position behind closed lips has no acoustic signature.
23 tests. Measured on release:
solve: 2.686 us/frame for 267 samples (8 bands, f32x4) -- budget 200 us
jawOpen: /a/ = 0.6038, /i/ = 0.1996
/u/: funnel 0.1428 pucker 0.1048 smile 0.0000
/i/: funnel 0.0000 pucker 0.0000 smile 0.2293
/s/: jawOpen 0.0000 shrugLower 0.6530 rollLower 0.2435
/m/: mouthClose 0.8396 jawOpen 0.2055
jawOpen rise 3 frames, fall 8 frames
- miranda-nodes/src/lib.rs +1 -0
- miranda-nodes/src/solver.rs +1230 -0
|
@@ -30,4 +30,5 @@ pub mod blink;
|
|
| 30 |
pub mod breath;
|
| 31 |
pub mod compositor;
|
| 32 |
pub mod gaze;
|
|
|
|
| 33 |
pub mod viseme;
|
|
|
|
| 30 |
pub mod breath;
|
| 31 |
pub mod compositor;
|
| 32 |
pub mod gaze;
|
| 33 |
+
pub mod solver;
|
| 34 |
pub mod viseme;
|
|
@@ -0,0 +1,1230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! WO-3 T7 — SIMD blend shape solver driven by raw audio energy features.
|
| 2 |
+
//!
|
| 3 |
+
//! This is the **Pipeline 2 (local)** speech-to-mouth path. It takes raw PCM
|
| 4 |
+
//! and produces ARKit mouth weights directly, with no ASR, no phoneme
|
| 5 |
+
//! alignment, and no network round trip. It is the counterpart to
|
| 6 |
+
//! [`crate::viseme`], which is the **Pipeline 1 (cloud)** path driven by
|
| 7 |
+
//! Amazon Polly viseme events. Both feed the same compositor.
|
| 8 |
+
//!
|
| 9 |
+
//! # What this is, and what it is not
|
| 10 |
+
//!
|
| 11 |
+
//! It is an **acoustic-to-articulator heuristic**: a small bandpass filter
|
| 12 |
+
//! bank measuring energy in eight speech-relevant frequency bands, followed
|
| 13 |
+
//! by a fixed linear basis mapping that spectral shape onto mouth channels.
|
| 14 |
+
//! The mapping is grounded in formant phonetics — jaw opening tracks F1,
|
| 15 |
+
//! lip rounding versus spreading tracks F2, sibilance lives above 3.5 kHz —
|
| 16 |
+
//! but the coefficients are hand-authored, not fitted to data.
|
| 17 |
+
//!
|
| 18 |
+
//! It is **not** a trained audio-to-face regressor. A learned model
|
| 19 |
+
//! (Audio2Face-class) will produce better coarticulation and better
|
| 20 |
+
//! consonant detail than any hand-authored basis can, because much of what
|
| 21 |
+
//! the mouth does is not recoverable from band energies at all: /p/, /b/ and
|
| 22 |
+
//! /m/ are acoustically near-identical during closure and differ only in
|
| 23 |
+
//! what follows, and tongue position behind closed lips has no acoustic
|
| 24 |
+
//! signature whatsoever. `TONGUE_OUT` is therefore never driven here — a
|
| 25 |
+
//! plausible-looking guess would be fabrication.
|
| 26 |
+
//!
|
| 27 |
+
//! What this path does buy, and why it exists: it is deterministic, it runs
|
| 28 |
+
//! offline, it adds no latency beyond the analysis window, and its cost is
|
| 29 |
+
//! bounded and measurable. That makes it the honest floor of the system —
|
| 30 |
+
//! the mouth still moves correctly in amplitude and broad shape when the
|
| 31 |
+
//! network is gone.
|
| 32 |
+
//!
|
| 33 |
+
//! # Why the SIMD axis is *bands*, not samples
|
| 34 |
+
//!
|
| 35 |
+
//! The filter bank is recursive: each output sample depends on the previous
|
| 36 |
+
//! two, so the sample axis cannot be vectorised without restructuring the
|
| 37 |
+
//! filters. The band axis is perfectly parallel — eight independent filters
|
| 38 |
+
//! see the identical input sample. So the layout is eight bands across two
|
| 39 |
+
//! [`f32x4`] vectors, and the inner loop advances all eight filters per
|
| 40 |
+
//! sample with a handful of vector operations.
|
| 41 |
+
//!
|
| 42 |
+
//! Four-wide (`f32x4`, 128-bit SSE) is deliberate, not a default. The
|
| 43 |
+
//! development machine (Celeron N4500) reports `sse sse2 sse3 sse4_1 sse4_2`
|
| 44 |
+
//! and **no AVX, AVX2, or FMA**. `f32x8` on such a target is emulated as two
|
| 45 |
+
//! 128-bit halves by `wide`, so widening the vectors would not widen the
|
| 46 |
+
//! hardware — it would only make the band count harder to change. Eight
|
| 47 |
+
//! bands over two `f32x4` also happens to be the right *acoustic* resolution
|
| 48 |
+
//! for the formant structure being measured, so the two constraints agree.
|
| 49 |
+
|
| 50 |
+
use miranda_core::{arkit, AUDIO_SAMPLE_RATE_HZ, BLENDSHAPE_COUNT};
|
| 51 |
+
use wide::f32x4;
|
| 52 |
+
|
| 53 |
+
/// Number of analysis bands. Two [`f32x4`] vectors' worth.
|
| 54 |
+
pub const BAND_COUNT: usize = 8;
|
| 55 |
+
|
| 56 |
+
/// Number of SIMD vectors the band state occupies.
|
| 57 |
+
const VEC_COUNT: usize = BAND_COUNT / 4;
|
| 58 |
+
|
| 59 |
+
/// Band edges in Hz, chosen for speech structure at a 16 kHz sample rate
|
| 60 |
+
/// (Nyquist 8 kHz).
|
| 61 |
+
///
|
| 62 |
+
/// | band | range Hz | what it measures |
|
| 63 |
+
/// |------|----------|------------------|
|
| 64 |
+
/// | 0 | 100–300 | voicing / F0 — vocal fold activity |
|
| 65 |
+
/// | 1 | 300–600 | F1 low — close vowels /i/ /u/ |
|
| 66 |
+
/// | 2 | 600–1000 | F1 high — open vowels /a/ /ɑ/ |
|
| 67 |
+
/// | 3 | 1000–1600 | F2 low — back, rounded vowels /o/ /u/ |
|
| 68 |
+
/// | 4 | 1600–2400 | F2 high — front, spread vowels /i/ /e/ |
|
| 69 |
+
/// | 5 | 2400–3600 | F3 — /r/ colouring, vowel brightness |
|
| 70 |
+
/// | 6 | 3600–5500 | sibilant onset /ʃ/ /tʃ/ |
|
| 71 |
+
/// | 7 | 5500–7800 | high fricative /s/ /z/ /f/ |
|
| 72 |
+
///
|
| 73 |
+
/// The top edge stops at 7800 Hz rather than 8000: a resonator centred at
|
| 74 |
+
/// Nyquist is unstable, and the bilinear transform's frequency warping is
|
| 75 |
+
/// worst there. Leaving 200 Hz of headroom costs no meaningful sibilant
|
| 76 |
+
/// energy.
|
| 77 |
+
pub const BAND_EDGES_HZ: [(f32, f32); BAND_COUNT] = [
|
| 78 |
+
(100.0, 300.0),
|
| 79 |
+
(300.0, 600.0),
|
| 80 |
+
(600.0, 1000.0),
|
| 81 |
+
(1000.0, 1600.0),
|
| 82 |
+
(1600.0, 2400.0),
|
| 83 |
+
(2400.0, 3600.0),
|
| 84 |
+
(3600.0, 5500.0),
|
| 85 |
+
(5500.0, 7800.0),
|
| 86 |
+
];
|
| 87 |
+
|
| 88 |
+
// Named band indices. Used by the basis table so the phonetic reasoning is
|
| 89 |
+
// readable at the point of use rather than encoded in positional literals.
|
| 90 |
+
const B_VOICING: usize = 0;
|
| 91 |
+
const B_F1_LOW: usize = 1;
|
| 92 |
+
const B_F1_HIGH: usize = 2;
|
| 93 |
+
const B_F2_LOW: usize = 3;
|
| 94 |
+
const B_F2_HIGH: usize = 4;
|
| 95 |
+
const B_F3: usize = 5;
|
| 96 |
+
const B_SIB_LOW: usize = 6;
|
| 97 |
+
const B_SIB_HIGH: usize = 7;
|
| 98 |
+
|
| 99 |
+
/// Level below which the input is treated as silence, in dBFS of RMS.
|
| 100 |
+
pub const SILENCE_FLOOR_DBFS: f32 = -60.0;
|
| 101 |
+
|
| 102 |
+
/// Level at which the mouth is considered fully driven, in dBFS of RMS.
|
| 103 |
+
/// Above this, extra loudness does not open the mouth further — shouting
|
| 104 |
+
/// does not detach the jaw.
|
| 105 |
+
pub const FULL_DRIVE_DBFS: f32 = -14.0;
|
| 106 |
+
|
| 107 |
+
/// Attack smoothing coefficient, per 16.67 ms frame. Mouth shapes are
|
| 108 |
+
/// allowed to form quickly.
|
| 109 |
+
pub const ATTACK_ALPHA: f32 = 0.55;
|
| 110 |
+
|
| 111 |
+
/// Release smoothing coefficient, per 16.67 ms frame. Deliberately slower
|
| 112 |
+
/// than attack: articulators are pulled into position by muscle and fall
|
| 113 |
+
/// back passively, so closing lags opening. Symmetric smoothing is the
|
| 114 |
+
/// single most recognisable tell of a cheap lip-sync — the mouth snaps shut
|
| 115 |
+
/// between syllables and reads as a chattering puppet.
|
| 116 |
+
pub const RELEASE_ALPHA: f32 = 0.25;
|
| 117 |
+
|
| 118 |
+
// ---------------------------------------------------------------------
|
| 119 |
+
// Filter bank
|
| 120 |
+
// ---------------------------------------------------------------------
|
| 121 |
+
|
| 122 |
+
/// Eight constant-skirt bandpass resonators evaluated in SIMD across bands.
|
| 123 |
+
///
|
| 124 |
+
/// Each band is an RBJ biquad bandpass in transposed direct form II, which
|
| 125 |
+
/// needs only two state values per band (rather than direct form I's four)
|
| 126 |
+
/// and keeps the per-sample work to five vector multiply/adds.
|
| 127 |
+
#[derive(Clone)]
|
| 128 |
+
pub struct BandBank {
|
| 129 |
+
/// Numerator gain. For a constant-skirt bandpass `b1 = 0` and
|
| 130 |
+
/// `b2 = -b0`, so one coefficient covers the whole numerator.
|
| 131 |
+
b0: [f32x4; VEC_COUNT],
|
| 132 |
+
a1: [f32x4; VEC_COUNT],
|
| 133 |
+
a2: [f32x4; VEC_COUNT],
|
| 134 |
+
s1: [f32x4; VEC_COUNT],
|
| 135 |
+
s2: [f32x4; VEC_COUNT],
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
impl BandBank {
|
| 139 |
+
/// Builds the bank for [`AUDIO_SAMPLE_RATE_HZ`].
|
| 140 |
+
pub fn new() -> Self {
|
| 141 |
+
Self::with_sample_rate(AUDIO_SAMPLE_RATE_HZ as f32)
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
/// Builds the bank for an arbitrary sample rate.
|
| 145 |
+
///
|
| 146 |
+
/// Coefficients are computed once here, never in the hot loop. The
|
| 147 |
+
/// trigonometry and division below are exactly what must not appear in
|
| 148 |
+
/// the per-sample path.
|
| 149 |
+
pub fn with_sample_rate(sample_rate_hz: f32) -> Self {
|
| 150 |
+
let mut b0 = [0.0f32; BAND_COUNT];
|
| 151 |
+
let mut a1 = [0.0f32; BAND_COUNT];
|
| 152 |
+
let mut a2 = [0.0f32; BAND_COUNT];
|
| 153 |
+
|
| 154 |
+
let nyquist = sample_rate_hz * 0.5;
|
| 155 |
+
|
| 156 |
+
for (i, &(lo, hi)) in BAND_EDGES_HZ.iter().enumerate() {
|
| 157 |
+
// Geometric centre, not arithmetic: the bilinear-transform
|
| 158 |
+
// bandpass is symmetric on a log frequency axis, so the
|
| 159 |
+
// geometric centre is the frequency actually sitting mid-band.
|
| 160 |
+
let centre = (lo * hi).sqrt();
|
| 161 |
+
// Keep the resonator away from Nyquist, where the biquad
|
| 162 |
+
// degenerates. Bands are authored inside range for 16 kHz; this
|
| 163 |
+
// guard is what keeps `with_sample_rate` honest if someone
|
| 164 |
+
// instantiates the bank at 8 kHz.
|
| 165 |
+
let centre = centre.min(nyquist * 0.95);
|
| 166 |
+
let bandwidth = (hi - lo).min(nyquist * 0.9);
|
| 167 |
+
let q = (centre / bandwidth).max(0.3);
|
| 168 |
+
|
| 169 |
+
let w0 = std::f32::consts::TAU * centre / sample_rate_hz;
|
| 170 |
+
let (sin_w0, cos_w0) = w0.sin_cos();
|
| 171 |
+
let alpha = sin_w0 / (2.0 * q);
|
| 172 |
+
|
| 173 |
+
// RBJ constant-skirt bandpass, peak gain = Q.
|
| 174 |
+
let norm = 1.0 + alpha;
|
| 175 |
+
b0[i] = alpha / norm;
|
| 176 |
+
a1[i] = (-2.0 * cos_w0) / norm;
|
| 177 |
+
a2[i] = (1.0 - alpha) / norm;
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
Self {
|
| 181 |
+
b0: pack(&b0),
|
| 182 |
+
a1: pack(&a1),
|
| 183 |
+
a2: pack(&a2),
|
| 184 |
+
s1: [f32x4::ZERO; VEC_COUNT],
|
| 185 |
+
s2: [f32x4::ZERO; VEC_COUNT],
|
| 186 |
+
}
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
/// Clears filter state. Used between utterances so one turn's tail does
|
| 190 |
+
/// not ring into the next.
|
| 191 |
+
pub fn reset(&mut self) {
|
| 192 |
+
self.s1 = [f32x4::ZERO; VEC_COUNT];
|
| 193 |
+
self.s2 = [f32x4::ZERO; VEC_COUNT];
|
| 194 |
+
}
|
| 195 |
+
|
| 196 |
+
/// Runs `samples` through the bank and returns per-band RMS.
|
| 197 |
+
///
|
| 198 |
+
/// Allocation-free: state lives in `self`, the accumulators are on the
|
| 199 |
+
/// stack, and the return value is a fixed array.
|
| 200 |
+
pub fn analyse(&mut self, samples: &[f32]) -> [f32; BAND_COUNT] {
|
| 201 |
+
if samples.is_empty() {
|
| 202 |
+
return [0.0; BAND_COUNT];
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
let mut energy = [f32x4::ZERO; VEC_COUNT];
|
| 206 |
+
|
| 207 |
+
for &x in samples {
|
| 208 |
+
// One scalar sample broadcast to every band — this is why the
|
| 209 |
+
// band axis is the vectorisable one.
|
| 210 |
+
let xv = f32x4::splat(x);
|
| 211 |
+
|
| 212 |
+
for v in 0..VEC_COUNT {
|
| 213 |
+
// Transposed direct form II:
|
| 214 |
+
// y = b0*x + s1
|
| 215 |
+
// s1 = -a1*y + s2 (b1 == 0)
|
| 216 |
+
// s2 = -b0*x - a2*y (b2 == -b0)
|
| 217 |
+
let bx = self.b0[v] * xv;
|
| 218 |
+
let y = bx + self.s1[v];
|
| 219 |
+
self.s1[v] = self.s2[v] - self.a1[v] * y;
|
| 220 |
+
self.s2[v] = -bx - self.a2[v] * y;
|
| 221 |
+
energy[v] += y * y;
|
| 222 |
+
}
|
| 223 |
+
}
|
| 224 |
+
|
| 225 |
+
let inv_n = 1.0 / samples.len() as f32;
|
| 226 |
+
let mut out = [0.0f32; BAND_COUNT];
|
| 227 |
+
for v in 0..VEC_COUNT {
|
| 228 |
+
let e = (energy[v] * f32x4::splat(inv_n)).sqrt().to_array();
|
| 229 |
+
out[v * 4..v * 4 + 4].copy_from_slice(&e);
|
| 230 |
+
}
|
| 231 |
+
// A NaN or infinity here would propagate into every blend shape and
|
| 232 |
+
// then into the ring buffer, where it becomes a permanently broken
|
| 233 |
+
// face rather than one bad frame. Denormalised or absurd input is
|
| 234 |
+
// cheaper to neutralise than to trace later.
|
| 235 |
+
for e in out.iter_mut() {
|
| 236 |
+
if !e.is_finite() {
|
| 237 |
+
*e = 0.0;
|
| 238 |
+
}
|
| 239 |
+
}
|
| 240 |
+
out
|
| 241 |
+
}
|
| 242 |
+
}
|
| 243 |
+
|
| 244 |
+
impl Default for BandBank {
|
| 245 |
+
fn default() -> Self {
|
| 246 |
+
Self::new()
|
| 247 |
+
}
|
| 248 |
+
}
|
| 249 |
+
|
| 250 |
+
fn pack(values: &[f32; BAND_COUNT]) -> [f32x4; VEC_COUNT] {
|
| 251 |
+
let mut out = [f32x4::ZERO; VEC_COUNT];
|
| 252 |
+
for v in 0..VEC_COUNT {
|
| 253 |
+
out[v] = f32x4::new([
|
| 254 |
+
values[v * 4],
|
| 255 |
+
values[v * 4 + 1],
|
| 256 |
+
values[v * 4 + 2],
|
| 257 |
+
values[v * 4 + 3],
|
| 258 |
+
]);
|
| 259 |
+
}
|
| 260 |
+
out
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
/// Broadband RMS of a sample block, vectorised over samples.
|
| 264 |
+
///
|
| 265 |
+
/// Unlike the filter bank this *is* parallel over samples, since squaring
|
| 266 |
+
/// and summing carries no state.
|
| 267 |
+
pub fn rms(samples: &[f32]) -> f32 {
|
| 268 |
+
if samples.is_empty() {
|
| 269 |
+
return 0.0;
|
| 270 |
+
}
|
| 271 |
+
let chunks = samples.chunks_exact(4);
|
| 272 |
+
let tail = chunks.remainder();
|
| 273 |
+
|
| 274 |
+
let mut acc = f32x4::ZERO;
|
| 275 |
+
for c in chunks {
|
| 276 |
+
let v = f32x4::new([c[0], c[1], c[2], c[3]]);
|
| 277 |
+
acc += v * v;
|
| 278 |
+
}
|
| 279 |
+
let mut sum = acc.reduce_add();
|
| 280 |
+
for &x in tail {
|
| 281 |
+
sum += x * x;
|
| 282 |
+
}
|
| 283 |
+
let value = (sum / samples.len() as f32).sqrt();
|
| 284 |
+
if value.is_finite() {
|
| 285 |
+
value
|
| 286 |
+
} else {
|
| 287 |
+
0.0
|
| 288 |
+
}
|
| 289 |
+
}
|
| 290 |
+
|
| 291 |
+
// ---------------------------------------------------------------------
|
| 292 |
+
// Acoustic features
|
| 293 |
+
// ---------------------------------------------------------------------
|
| 294 |
+
|
| 295 |
+
/// Intermediate features extracted from one analysis window. Exposed
|
| 296 |
+
/// because the solver's behaviour is only auditable if the features feeding
|
| 297 |
+
/// it are inspectable — a wrong weight and a wrong band energy look
|
| 298 |
+
/// identical from the outside.
|
| 299 |
+
#[derive(Debug, Clone, Copy, PartialEq)]
|
| 300 |
+
pub struct AudioFeatures {
|
| 301 |
+
/// Per-band RMS, unnormalised.
|
| 302 |
+
pub band_rms: [f32; BAND_COUNT],
|
| 303 |
+
/// Per-band share of total band energy, summing to 1.0 (or all zero for
|
| 304 |
+
/// silence). This is the *shape* of the spectrum with loudness removed,
|
| 305 |
+
/// which is what determines mouth shape.
|
| 306 |
+
pub band_shape: [f32; BAND_COUNT],
|
| 307 |
+
/// Broadband RMS.
|
| 308 |
+
pub rms: f32,
|
| 309 |
+
/// Loudness mapped to `[0, 1]` between [`SILENCE_FLOOR_DBFS`] and
|
| 310 |
+
/// [`FULL_DRIVE_DBFS`]. This is what scales mouth motion.
|
| 311 |
+
pub drive: f32,
|
| 312 |
+
}
|
| 313 |
+
|
| 314 |
+
impl AudioFeatures {
|
| 315 |
+
/// Extracts features from one window. `bank` carries filter state
|
| 316 |
+
/// across calls, so windows must be fed in order.
|
| 317 |
+
pub fn extract(bank: &mut BandBank, samples: &[f32]) -> Self {
|
| 318 |
+
let band_rms = bank.analyse(samples);
|
| 319 |
+
let level = rms(samples);
|
| 320 |
+
|
| 321 |
+
let total: f32 = band_rms.iter().sum();
|
| 322 |
+
let mut band_shape = [0.0f32; BAND_COUNT];
|
| 323 |
+
if total > 1e-9 {
|
| 324 |
+
let inv = 1.0 / total;
|
| 325 |
+
for i in 0..BAND_COUNT {
|
| 326 |
+
band_shape[i] = band_rms[i] * inv;
|
| 327 |
+
}
|
| 328 |
+
}
|
| 329 |
+
|
| 330 |
+
Self {
|
| 331 |
+
band_rms,
|
| 332 |
+
band_shape,
|
| 333 |
+
rms: level,
|
| 334 |
+
drive: drive_from_rms(level),
|
| 335 |
+
}
|
| 336 |
+
}
|
| 337 |
+
}
|
| 338 |
+
|
| 339 |
+
/// Maps RMS to a `[0, 1]` drive factor on a decibel scale.
|
| 340 |
+
///
|
| 341 |
+
/// Decibels rather than linear amplitude because loudness perception and
|
| 342 |
+
/// articulatory effort are both roughly logarithmic. A linear map spends
|
| 343 |
+
/// almost all of its range on the loudest few decibels, so normal speech
|
| 344 |
+
/// would sit near zero and the mouth would barely move until someone
|
| 345 |
+
/// shouted.
|
| 346 |
+
pub fn drive_from_rms(level: f32) -> f32 {
|
| 347 |
+
if !(level > 0.0) || !level.is_finite() {
|
| 348 |
+
return 0.0;
|
| 349 |
+
}
|
| 350 |
+
let dbfs = 20.0 * level.log10();
|
| 351 |
+
((dbfs - SILENCE_FLOOR_DBFS) / (FULL_DRIVE_DBFS - SILENCE_FLOOR_DBFS)).clamp(0.0, 1.0)
|
| 352 |
+
}
|
| 353 |
+
|
| 354 |
+
// ---------------------------------------------------------------------
|
| 355 |
+
// The basis
|
| 356 |
+
// ---------------------------------------------------------------------
|
| 357 |
+
|
| 358 |
+
/// One row of the solver basis: a mouth channel, its output gain, and its
|
| 359 |
+
/// sensitivity to each band's share of the spectrum.
|
| 360 |
+
///
|
| 361 |
+
/// Negative weights are *inhibitory* and are the load-bearing part. Lip
|
| 362 |
+
/// rounding is not "energy at 1.2 kHz"; it is energy at 1.2 kHz **in the
|
| 363 |
+
/// absence of** energy at 2 kHz. Without the negative term every vowel
|
| 364 |
+
/// would round a little and every vowel would smile a little, and the mouth
|
| 365 |
+
/// would converge on one mush shape for all speech.
|
| 366 |
+
struct BasisRow {
|
| 367 |
+
channel: usize,
|
| 368 |
+
gain: f32,
|
| 369 |
+
weights: [f32; BAND_COUNT],
|
| 370 |
+
}
|
| 371 |
+
|
| 372 |
+
/// Builds a weight array with named bands, so a row reads as phonetics
|
| 373 |
+
/// rather than as eight positional numbers.
|
| 374 |
+
const fn w(pairs: &[(usize, f32)]) -> [f32; BAND_COUNT] {
|
| 375 |
+
let mut out = [0.0; BAND_COUNT];
|
| 376 |
+
let mut i = 0;
|
| 377 |
+
while i < pairs.len() {
|
| 378 |
+
out[pairs[i].0] = pairs[i].1;
|
| 379 |
+
i += 1;
|
| 380 |
+
}
|
| 381 |
+
out
|
| 382 |
+
}
|
| 383 |
+
|
| 384 |
+
/// The solver basis.
|
| 385 |
+
///
|
| 386 |
+
/// Every channel is referenced through [`arkit`] constants. No bare index
|
| 387 |
+
/// literals: a wrong index here moves the wrong muscle and would pass every
|
| 388 |
+
/// range check in this file.
|
| 389 |
+
fn basis() -> [BasisRow; 11] {
|
| 390 |
+
[
|
| 391 |
+
// Jaw opening tracks F1. High F1 (600–1000 Hz) means an open vocal
|
| 392 |
+
// tract; energy concentrated at low F1 means a close vowel with the
|
| 393 |
+
// jaw nearly shut. F2 contributes nothing to aperture.
|
| 394 |
+
BasisRow {
|
| 395 |
+
channel: arkit::JAW_OPEN,
|
| 396 |
+
gain: 1.6,
|
| 397 |
+
weights: w(&[(B_F1_HIGH, 1.0), (B_F1_LOW, 0.25), (B_SIB_HIGH, -0.5)]),
|
| 398 |
+
},
|
| 399 |
+
// Bilabial closure /p/ /b/ /m/: voicing present, nothing above it.
|
| 400 |
+
// The negative terms are the whole test — during closure the lips
|
| 401 |
+
// block the high frequencies, so their absence *is* the signal.
|
| 402 |
+
BasisRow {
|
| 403 |
+
channel: arkit::MOUTH_CLOSE,
|
| 404 |
+
gain: 1.3,
|
| 405 |
+
weights: w(&[
|
| 406 |
+
(B_VOICING, 1.0),
|
| 407 |
+
(B_F1_LOW, 0.2),
|
| 408 |
+
(B_F2_HIGH, -0.7),
|
| 409 |
+
(B_SIB_LOW, -0.7),
|
| 410 |
+
(B_SIB_HIGH, -0.7),
|
| 411 |
+
]),
|
| 412 |
+
},
|
| 413 |
+
// Rounding /o/ /u/ /w/: F2 pulled down. Inhibited by high F2, which
|
| 414 |
+
// is the spread-lip signature and the acoustic opposite.
|
| 415 |
+
BasisRow {
|
| 416 |
+
channel: arkit::MOUTH_FUNNEL,
|
| 417 |
+
gain: 1.7,
|
| 418 |
+
weights: w(&[(B_F2_LOW, 1.0), (B_F2_HIGH, -0.8), (B_SIB_HIGH, -0.4)]),
|
| 419 |
+
},
|
| 420 |
+
BasisRow {
|
| 421 |
+
channel: arkit::MOUTH_PUCKER,
|
| 422 |
+
gain: 1.5,
|
| 423 |
+
weights: w(&[
|
| 424 |
+
(B_F2_LOW, 0.9),
|
| 425 |
+
(B_F1_LOW, 0.3),
|
| 426 |
+
(B_F2_HIGH, -0.7),
|
| 427 |
+
(B_F1_HIGH, -0.4),
|
| 428 |
+
]),
|
| 429 |
+
},
|
| 430 |
+
// Spreading /i/ /e/: F2 high, and high F3 brightness with it.
|
| 431 |
+
BasisRow {
|
| 432 |
+
channel: arkit::MOUTH_SMILE_LEFT,
|
| 433 |
+
gain: 1.4,
|
| 434 |
+
weights: w(&[(B_F2_HIGH, 1.0), (B_F3, 0.3), (B_F2_LOW, -0.6)]),
|
| 435 |
+
},
|
| 436 |
+
BasisRow {
|
| 437 |
+
channel: arkit::MOUTH_SMILE_RIGHT,
|
| 438 |
+
gain: 1.4,
|
| 439 |
+
weights: w(&[(B_F2_HIGH, 1.0), (B_F3, 0.3), (B_F2_LOW, -0.6)]),
|
| 440 |
+
},
|
| 441 |
+
BasisRow {
|
| 442 |
+
channel: arkit::MOUTH_STRETCH_LEFT,
|
| 443 |
+
gain: 1.1,
|
| 444 |
+
weights: w(&[(B_F2_HIGH, 0.8), (B_F3, 0.5), (B_F1_HIGH, -0.3)]),
|
| 445 |
+
},
|
| 446 |
+
BasisRow {
|
| 447 |
+
channel: arkit::MOUTH_STRETCH_RIGHT,
|
| 448 |
+
gain: 1.1,
|
| 449 |
+
weights: w(&[(B_F2_HIGH, 0.8), (B_F3, 0.5), (B_F1_HIGH, -0.3)]),
|
| 450 |
+
},
|
| 451 |
+
// Labiodental /f/ /v/: high-frequency friction with little voicing.
|
| 452 |
+
BasisRow {
|
| 453 |
+
channel: arkit::MOUTH_SHRUG_LOWER,
|
| 454 |
+
gain: 1.5,
|
| 455 |
+
weights: w(&[(B_SIB_LOW, 0.8), (B_SIB_HIGH, 0.5), (B_VOICING, -0.8)]),
|
| 456 |
+
},
|
| 457 |
+
BasisRow {
|
| 458 |
+
channel: arkit::MOUTH_SHRUG_UPPER,
|
| 459 |
+
gain: 1.2,
|
| 460 |
+
weights: w(&[(B_SIB_HIGH, 0.7), (B_VOICING, -0.6)]),
|
| 461 |
+
},
|
| 462 |
+
// Alveolar sibilants /s/ /z/ draw the lips slightly back over the
|
| 463 |
+
// teeth; the roll is small and driven only by the top band.
|
| 464 |
+
BasisRow {
|
| 465 |
+
channel: arkit::MOUTH_ROLL_LOWER,
|
| 466 |
+
gain: 0.7,
|
| 467 |
+
weights: w(&[(B_SIB_HIGH, 0.8), (B_VOICING, -0.5)]),
|
| 468 |
+
},
|
| 469 |
+
]
|
| 470 |
+
}
|
| 471 |
+
|
| 472 |
+
// ---------------------------------------------------------------------
|
| 473 |
+
// The solver
|
| 474 |
+
// ---------------------------------------------------------------------
|
| 475 |
+
|
| 476 |
+
/// Raw-audio-driven mouth blend shape solver.
|
| 477 |
+
///
|
| 478 |
+
/// Feed it one window of PCM per rendered frame; it returns the full
|
| 479 |
+
/// 52-channel weight array with only its own mouth channels non-zero. Like
|
| 480 |
+
/// the oscillators, it never touches the autonomic channels — the
|
| 481 |
+
/// compositor owns combining the two layers.
|
| 482 |
+
pub struct AudioBlendshapeSolver {
|
| 483 |
+
bank: BandBank,
|
| 484 |
+
basis: [BasisRow; 11],
|
| 485 |
+
/// Smoothed output, held across frames.
|
| 486 |
+
current: [f32; BLENDSHAPE_COUNT],
|
| 487 |
+
/// Untouched scratch for the pre-smoothing solve. Owned so the hot path
|
| 488 |
+
/// allocates nothing.
|
| 489 |
+
target: [f32; BLENDSHAPE_COUNT],
|
| 490 |
+
attack_alpha: f32,
|
| 491 |
+
release_alpha: f32,
|
| 492 |
+
last_features: AudioFeatures,
|
| 493 |
+
}
|
| 494 |
+
|
| 495 |
+
impl AudioBlendshapeSolver {
|
| 496 |
+
pub fn new() -> Self {
|
| 497 |
+
Self::with_sample_rate(AUDIO_SAMPLE_RATE_HZ as f32)
|
| 498 |
+
}
|
| 499 |
+
|
| 500 |
+
pub fn with_sample_rate(sample_rate_hz: f32) -> Self {
|
| 501 |
+
Self {
|
| 502 |
+
bank: BandBank::with_sample_rate(sample_rate_hz),
|
| 503 |
+
basis: basis(),
|
| 504 |
+
current: [0.0; BLENDSHAPE_COUNT],
|
| 505 |
+
target: [0.0; BLENDSHAPE_COUNT],
|
| 506 |
+
// Both coefficients are clamped into (0, 1]. Zero would freeze
|
| 507 |
+
// the mouth forever and values above 1 would overshoot and
|
| 508 |
+
// oscillate — both are silent visual failures, so they are
|
| 509 |
+
// rejected here rather than trusted.
|
| 510 |
+
attack_alpha: ATTACK_ALPHA.clamp(f32::EPSILON, 1.0),
|
| 511 |
+
release_alpha: RELEASE_ALPHA.clamp(f32::EPSILON, 1.0),
|
| 512 |
+
last_features: AudioFeatures {
|
| 513 |
+
band_rms: [0.0; BAND_COUNT],
|
| 514 |
+
band_shape: [0.0; BAND_COUNT],
|
| 515 |
+
rms: 0.0,
|
| 516 |
+
drive: 0.0,
|
| 517 |
+
},
|
| 518 |
+
}
|
| 519 |
+
}
|
| 520 |
+
|
| 521 |
+
/// Features from the most recent [`Self::solve`]. For telemetry and for
|
| 522 |
+
/// the tests that need to distinguish a filter-bank error from a basis
|
| 523 |
+
/// error.
|
| 524 |
+
pub fn features(&self) -> &AudioFeatures {
|
| 525 |
+
&self.last_features
|
| 526 |
+
}
|
| 527 |
+
|
| 528 |
+
/// Clears filter and smoothing state.
|
| 529 |
+
pub fn reset(&mut self) {
|
| 530 |
+
self.bank.reset();
|
| 531 |
+
self.current = [0.0; BLENDSHAPE_COUNT];
|
| 532 |
+
self.last_features = AudioFeatures {
|
| 533 |
+
band_rms: [0.0; BAND_COUNT],
|
| 534 |
+
band_shape: [0.0; BAND_COUNT],
|
| 535 |
+
rms: 0.0,
|
| 536 |
+
drive: 0.0,
|
| 537 |
+
};
|
| 538 |
+
}
|
| 539 |
+
|
| 540 |
+
/// Solves one frame from `samples` and returns the smoothed weights.
|
| 541 |
+
///
|
| 542 |
+
/// Allocation-free, branch-light, and free of I/O and locking, so it is
|
| 543 |
+
/// safe to call from the 60 FPS thread.
|
| 544 |
+
pub fn solve(&mut self, samples: &[f32]) -> &[f32; BLENDSHAPE_COUNT] {
|
| 545 |
+
let features = AudioFeatures::extract(&mut self.bank, samples);
|
| 546 |
+
self.last_features = features;
|
| 547 |
+
|
| 548 |
+
self.target = [0.0; BLENDSHAPE_COUNT];
|
| 549 |
+
|
| 550 |
+
// Silence is an explicit case, not an emergent one. With drive == 0
|
| 551 |
+
// every row would multiply out to zero anyway, but relying on that
|
| 552 |
+
// means the *shape* is still computed from filter ringing during
|
| 553 |
+
// silence, and any later change to the drive term would let that
|
| 554 |
+
// ringing leak onto the face.
|
| 555 |
+
if features.drive > 0.0 {
|
| 556 |
+
for row in self.basis.iter() {
|
| 557 |
+
let mut acc = 0.0f32;
|
| 558 |
+
for b in 0..BAND_COUNT {
|
| 559 |
+
acc += row.weights[b] * features.band_shape[b];
|
| 560 |
+
}
|
| 561 |
+
// Rectify before scaling: a negative activation means "this
|
| 562 |
+
// articulator is not indicated", which is zero, not a
|
| 563 |
+
// negative weight.
|
| 564 |
+
let activation = (acc * row.gain).max(0.0);
|
| 565 |
+
self.target[row.channel] = (activation * features.drive).clamp(0.0, 1.0);
|
| 566 |
+
}
|
| 567 |
+
}
|
| 568 |
+
|
| 569 |
+
// Asymmetric smoothing, applied per channel on its own direction of
|
| 570 |
+
// travel.
|
| 571 |
+
for i in 0..BLENDSHAPE_COUNT {
|
| 572 |
+
let t = self.target[i];
|
| 573 |
+
let c = self.current[i];
|
| 574 |
+
let alpha = if t > c {
|
| 575 |
+
self.attack_alpha
|
| 576 |
+
} else {
|
| 577 |
+
self.release_alpha
|
| 578 |
+
};
|
| 579 |
+
self.current[i] = c + (t - c) * alpha;
|
| 580 |
+
}
|
| 581 |
+
|
| 582 |
+
&self.current
|
| 583 |
+
}
|
| 584 |
+
|
| 585 |
+
/// Current smoothed weights without advancing the solver.
|
| 586 |
+
pub fn weights(&self) -> &[f32; BLENDSHAPE_COUNT] {
|
| 587 |
+
&self.current
|
| 588 |
+
}
|
| 589 |
+
|
| 590 |
+
/// Adds the solved weights into `out`, additively, matching the
|
| 591 |
+
/// oscillators' contract so the compositor can layer all sources
|
| 592 |
+
/// uniformly and own the final clamp.
|
| 593 |
+
pub fn add_into(&self, out: &mut [f32; BLENDSHAPE_COUNT]) {
|
| 594 |
+
for i in 0..BLENDSHAPE_COUNT {
|
| 595 |
+
out[i] += self.current[i];
|
| 596 |
+
}
|
| 597 |
+
}
|
| 598 |
+
}
|
| 599 |
+
|
| 600 |
+
impl Default for AudioBlendshapeSolver {
|
| 601 |
+
fn default() -> Self {
|
| 602 |
+
Self::new()
|
| 603 |
+
}
|
| 604 |
+
}
|
| 605 |
+
|
| 606 |
+
#[cfg(test)]
|
| 607 |
+
mod tests {
|
| 608 |
+
use super::*;
|
| 609 |
+
use std::time::Instant;
|
| 610 |
+
|
| 611 |
+
const SR: f32 = AUDIO_SAMPLE_RATE_HZ as f32;
|
| 612 |
+
/// Samples per rendered frame at 60 FPS and 16 kHz: 16.67 ms.
|
| 613 |
+
const FRAME_SAMPLES: usize = 267;
|
| 614 |
+
|
| 615 |
+
fn sine(freq: f32, amp: f32, n: usize, phase_offset: f32) -> Vec<f32> {
|
| 616 |
+
(0..n)
|
| 617 |
+
.map(|i| {
|
| 618 |
+
(std::f32::consts::TAU * freq * i as f32 / SR + phase_offset).sin() * amp
|
| 619 |
+
})
|
| 620 |
+
.collect()
|
| 621 |
+
}
|
| 622 |
+
|
| 623 |
+
/// Two summed sinusoids at F1 and F2. Not a real vocal tract — a real
|
| 624 |
+
/// one has a harmonic source and a full formant envelope — but the
|
| 625 |
+
/// solver reads band energy, and a formant pair puts energy in the same
|
| 626 |
+
/// bands a real one would.
|
| 627 |
+
fn vowel(f1: f32, f2: f32, amp: f32, n: usize) -> Vec<f32> {
|
| 628 |
+
let a = sine(f1, amp * 0.6, n, 0.0);
|
| 629 |
+
let b = sine(f2, amp * 0.4, n, 1.0);
|
| 630 |
+
a.iter().zip(b.iter()).map(|(x, y)| x + y).collect()
|
| 631 |
+
}
|
| 632 |
+
|
| 633 |
+
/// Deterministic pseudo-noise band-limited by summing many sines across
|
| 634 |
+
/// a range. Used for fricatives.
|
| 635 |
+
fn noise_band(lo: f32, hi: f32, amp: f32, n: usize) -> Vec<f32> {
|
| 636 |
+
let mut out = vec![0.0f32; n];
|
| 637 |
+
let lines = 24;
|
| 638 |
+
for k in 0..lines {
|
| 639 |
+
let f = lo + (hi - lo) * (k as f32 / (lines - 1) as f32);
|
| 640 |
+
let phase = k as f32 * 1.7;
|
| 641 |
+
for (i, o) in out.iter_mut().enumerate() {
|
| 642 |
+
*o += (std::f32::consts::TAU * f * i as f32 / SR + phase).sin();
|
| 643 |
+
}
|
| 644 |
+
}
|
| 645 |
+
let scale = amp / (lines as f32).sqrt();
|
| 646 |
+
for o in out.iter_mut() {
|
| 647 |
+
*o *= scale;
|
| 648 |
+
}
|
| 649 |
+
out
|
| 650 |
+
}
|
| 651 |
+
|
| 652 |
+
/// Runs enough frames for the smoother to settle, then returns the
|
| 653 |
+
/// weights. A single frame only shows `attack_alpha` of the target, so
|
| 654 |
+
/// asserting on one frame would test the smoother, not the basis.
|
| 655 |
+
fn settle(solver: &mut AudioBlendshapeSolver, block: &[f32], frames: usize) -> [f32; BLENDSHAPE_COUNT] {
|
| 656 |
+
let mut last = [0.0; BLENDSHAPE_COUNT];
|
| 657 |
+
for _ in 0..frames {
|
| 658 |
+
last = *solver.solve(block);
|
| 659 |
+
}
|
| 660 |
+
last
|
| 661 |
+
}
|
| 662 |
+
|
| 663 |
+
// -----------------------------------------------------------------
|
| 664 |
+
// Filter bank correctness — the foundation everything else rests on
|
| 665 |
+
// -----------------------------------------------------------------
|
| 666 |
+
|
| 667 |
+
/// **The load-bearing test of this module.** Each band must respond
|
| 668 |
+
/// most strongly to its own centre frequency.
|
| 669 |
+
///
|
| 670 |
+
/// If the biquad coefficients are wrong, or the geometric centre is
|
| 671 |
+
/// miscomputed, or the bands are packed into the SIMD lanes in the
|
| 672 |
+
/// wrong order, every downstream weight is driven by the wrong part of
|
| 673 |
+
/// the spectrum. Nothing about that is visible from range checks: the
|
| 674 |
+
/// weights stay in `[0, 1]`, they move with the audio, and the mouth is
|
| 675 |
+
/// simply wrong. Testing tone-to-band correspondence directly is the
|
| 676 |
+
/// only way to catch it.
|
| 677 |
+
#[test]
|
| 678 |
+
fn each_band_responds_most_to_its_own_centre_frequency() {
|
| 679 |
+
for (band, &(lo, hi)) in BAND_EDGES_HZ.iter().enumerate() {
|
| 680 |
+
let centre = (lo * hi).sqrt();
|
| 681 |
+
let mut bank = BandBank::new();
|
| 682 |
+
// Long enough for the resonators to reach steady state.
|
| 683 |
+
let tone = sine(centre, 0.5, 4096, 0.0);
|
| 684 |
+
let energies = bank.analyse(&tone);
|
| 685 |
+
|
| 686 |
+
let winner = energies
|
| 687 |
+
.iter()
|
| 688 |
+
.enumerate()
|
| 689 |
+
.max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
|
| 690 |
+
.map(|(i, _)| i)
|
| 691 |
+
.unwrap();
|
| 692 |
+
|
| 693 |
+
assert_eq!(
|
| 694 |
+
winner, band,
|
| 695 |
+
"a {centre:.0} Hz tone (centre of band {band}, {lo}-{hi} Hz) \
|
| 696 |
+
produced peak energy in band {winner} instead. Band \
|
| 697 |
+
energies: {energies:?}"
|
| 698 |
+
);
|
| 699 |
+
}
|
| 700 |
+
}
|
| 701 |
+
|
| 702 |
+
/// Bands must also *reject* distant frequencies, not merely prefer
|
| 703 |
+
/// their own. A bank where every band passes everything would still
|
| 704 |
+
/// win the test above while carrying no spectral information at all.
|
| 705 |
+
#[test]
|
| 706 |
+
fn bands_reject_distant_frequencies() {
|
| 707 |
+
let mut bank = BandBank::new();
|
| 708 |
+
let tone = sine(200.0, 0.5, 4096, 0.0);
|
| 709 |
+
let e = bank.analyse(&tone);
|
| 710 |
+
assert!(
|
| 711 |
+
e[B_SIB_HIGH] < e[B_VOICING] * 0.05,
|
| 712 |
+
"a 200 Hz tone leaked {} into the 5.5-7.8 kHz band against {} \
|
| 713 |
+
in its own band — the bank is not selective",
|
| 714 |
+
e[B_SIB_HIGH],
|
| 715 |
+
e[B_VOICING]
|
| 716 |
+
);
|
| 717 |
+
|
| 718 |
+
let mut bank = BandBank::new();
|
| 719 |
+
let tone = sine(6500.0, 0.5, 4096, 0.0);
|
| 720 |
+
let e = bank.analyse(&tone);
|
| 721 |
+
assert!(
|
| 722 |
+
e[B_VOICING] < e[B_SIB_HIGH] * 0.05,
|
| 723 |
+
"a 6.5 kHz tone leaked {} into the 100-300 Hz band",
|
| 724 |
+
e[B_VOICING]
|
| 725 |
+
);
|
| 726 |
+
}
|
| 727 |
+
|
| 728 |
+
/// Pathological input must not destabilise the recursive filters. A
|
| 729 |
+
/// biquad that blows up produces infinities that, once written to the
|
| 730 |
+
/// ring buffer, break the face permanently rather than for one frame.
|
| 731 |
+
#[test]
|
| 732 |
+
fn bank_is_stable_under_pathological_input() {
|
| 733 |
+
let cases: Vec<(&str, Vec<f32>)> = vec![
|
| 734 |
+
("dc", vec![1.0; 4096]),
|
| 735 |
+
(
|
| 736 |
+
"nyquist square",
|
| 737 |
+
(0..4096).map(|i| if i % 2 == 0 { 1.0 } else { -1.0 }).collect(),
|
| 738 |
+
),
|
| 739 |
+
("full scale step", {
|
| 740 |
+
let mut v = vec![0.0; 4096];
|
| 741 |
+
v[2048..].fill(1.0);
|
| 742 |
+
v
|
| 743 |
+
}),
|
| 744 |
+
("loud", vec![100.0; 4096]),
|
| 745 |
+
];
|
| 746 |
+
|
| 747 |
+
for (name, input) in cases {
|
| 748 |
+
let mut bank = BandBank::new();
|
| 749 |
+
let e = bank.analyse(&input);
|
| 750 |
+
for (i, v) in e.iter().enumerate() {
|
| 751 |
+
assert!(
|
| 752 |
+
v.is_finite(),
|
| 753 |
+
"{name}: band {i} energy is not finite ({v}) — filter diverged"
|
| 754 |
+
);
|
| 755 |
+
}
|
| 756 |
+
}
|
| 757 |
+
}
|
| 758 |
+
|
| 759 |
+
#[test]
|
| 760 |
+
fn empty_input_is_handled_without_panicking() {
|
| 761 |
+
let mut bank = BandBank::new();
|
| 762 |
+
assert_eq!(bank.analyse(&[]), [0.0; BAND_COUNT]);
|
| 763 |
+
assert_eq!(rms(&[]), 0.0);
|
| 764 |
+
}
|
| 765 |
+
|
| 766 |
+
/// A block whose length is not a multiple of four must be fully
|
| 767 |
+
/// accounted for. `chunks_exact` silently drops the remainder, so a
|
| 768 |
+
/// missing tail loop is easy to write and hard to notice — the error is
|
| 769 |
+
/// a small level offset, not a crash.
|
| 770 |
+
#[test]
|
| 771 |
+
fn rms_includes_the_non_multiple_of_four_tail() {
|
| 772 |
+
// Nine samples: two full vectors and a one-sample remainder.
|
| 773 |
+
let mut v = vec![0.0f32; 9];
|
| 774 |
+
v[8] = 3.0;
|
| 775 |
+
let expected = (9.0f32 / 9.0).sqrt();
|
| 776 |
+
assert!(
|
| 777 |
+
(rms(&v) - expected).abs() < 1e-6,
|
| 778 |
+
"tail sample was dropped: got {}, expected {expected}",
|
| 779 |
+
rms(&v)
|
| 780 |
+
);
|
| 781 |
+
}
|
| 782 |
+
|
| 783 |
+
#[test]
|
| 784 |
+
fn rms_matches_the_scalar_definition() {
|
| 785 |
+
let block = vowel(730.0, 1090.0, 0.4, 267);
|
| 786 |
+
let scalar =
|
| 787 |
+
(block.iter().map(|x| x * x).sum::<f32>() / block.len() as f32).sqrt();
|
| 788 |
+
assert!(
|
| 789 |
+
(rms(&block) - scalar).abs() < 1e-5,
|
| 790 |
+
"SIMD rms {} disagrees with scalar {}",
|
| 791 |
+
rms(&block),
|
| 792 |
+
scalar
|
| 793 |
+
);
|
| 794 |
+
}
|
| 795 |
+
|
| 796 |
+
// -----------------------------------------------------------------
|
| 797 |
+
// Drive curve
|
| 798 |
+
// -----------------------------------------------------------------
|
| 799 |
+
|
| 800 |
+
#[test]
|
| 801 |
+
fn silence_produces_zero_drive_and_loud_speech_produces_full_drive() {
|
| 802 |
+
assert_eq!(drive_from_rms(0.0), 0.0);
|
| 803 |
+
// -80 dBFS, below the floor.
|
| 804 |
+
assert_eq!(drive_from_rms(0.0001), 0.0);
|
| 805 |
+
// 0 dBFS, above full drive.
|
| 806 |
+
assert_eq!(drive_from_rms(1.0), 1.0);
|
| 807 |
+
// Conversational speech around -30 dBFS should land mid-range, not
|
| 808 |
+
// pinned at either end. A linear (non-dB) map would put this at
|
| 809 |
+
// about 0.03 and the mouth would barely move.
|
| 810 |
+
let d = drive_from_rms(10f32.powf(-30.0 / 20.0));
|
| 811 |
+
assert!(
|
| 812 |
+
(0.3..0.8).contains(&d),
|
| 813 |
+
"-30 dBFS mapped to drive {d}, outside a usable mid-range"
|
| 814 |
+
);
|
| 815 |
+
}
|
| 816 |
+
|
| 817 |
+
#[test]
|
| 818 |
+
fn drive_is_monotonic_in_level() {
|
| 819 |
+
let mut prev = -1.0;
|
| 820 |
+
for i in 0..=100 {
|
| 821 |
+
let dbfs = -70.0 + i as f32 * 0.7;
|
| 822 |
+
let d = drive_from_rms(10f32.powf(dbfs / 20.0));
|
| 823 |
+
assert!(d >= prev, "drive fell from {prev} to {d} at {dbfs} dBFS");
|
| 824 |
+
prev = d;
|
| 825 |
+
}
|
| 826 |
+
}
|
| 827 |
+
|
| 828 |
+
// -----------------------------------------------------------------
|
| 829 |
+
// Solver behaviour
|
| 830 |
+
// -----------------------------------------------------------------
|
| 831 |
+
|
| 832 |
+
#[test]
|
| 833 |
+
fn silence_produces_a_rest_pose() {
|
| 834 |
+
let mut solver = AudioBlendshapeSolver::new();
|
| 835 |
+
let out = settle(&mut solver, &vec![0.0; FRAME_SAMPLES], 60);
|
| 836 |
+
for (i, w) in out.iter().enumerate() {
|
| 837 |
+
assert_eq!(*w, 0.0, "channel {i} = {w} during silence");
|
| 838 |
+
}
|
| 839 |
+
}
|
| 840 |
+
|
| 841 |
+
/// Layer ownership, enforced. Speech owns the mouth; the autonomic
|
| 842 |
+
/// oscillators own eyes, brows, cheeks and nose. If the solver ever
|
| 843 |
+
/// wrote an eye channel it would fight the blink and gaze generators
|
| 844 |
+
/// through the compositor's additive sum, and the result would look
|
| 845 |
+
/// like a rendering glitch rather than a layering bug.
|
| 846 |
+
#[test]
|
| 847 |
+
fn solver_never_touches_autonomic_channels() {
|
| 848 |
+
let forbidden = [
|
| 849 |
+
arkit::EYE_BLINK_LEFT,
|
| 850 |
+
arkit::EYE_BLINK_RIGHT,
|
| 851 |
+
arkit::EYE_SQUINT_LEFT,
|
| 852 |
+
arkit::EYE_SQUINT_RIGHT,
|
| 853 |
+
arkit::EYE_LOOK_IN_LEFT,
|
| 854 |
+
arkit::EYE_LOOK_OUT_LEFT,
|
| 855 |
+
arkit::EYE_LOOK_UP_LEFT,
|
| 856 |
+
arkit::EYE_LOOK_DOWN_LEFT,
|
| 857 |
+
arkit::EYE_LOOK_IN_RIGHT,
|
| 858 |
+
arkit::EYE_LOOK_OUT_RIGHT,
|
| 859 |
+
arkit::EYE_LOOK_UP_RIGHT,
|
| 860 |
+
arkit::EYE_LOOK_DOWN_RIGHT,
|
| 861 |
+
arkit::EYE_WIDE_LEFT,
|
| 862 |
+
arkit::EYE_WIDE_RIGHT,
|
| 863 |
+
arkit::BROW_DOWN_LEFT,
|
| 864 |
+
arkit::BROW_DOWN_RIGHT,
|
| 865 |
+
arkit::BROW_INNER_UP,
|
| 866 |
+
arkit::BROW_OUTER_UP_LEFT,
|
| 867 |
+
arkit::BROW_OUTER_UP_RIGHT,
|
| 868 |
+
arkit::CHEEK_PUFF,
|
| 869 |
+
arkit::CHEEK_SQUINT_LEFT,
|
| 870 |
+
arkit::CHEEK_SQUINT_RIGHT,
|
| 871 |
+
arkit::NOSE_SNEER_LEFT,
|
| 872 |
+
arkit::NOSE_SNEER_RIGHT,
|
| 873 |
+
// Never inferred from audio: nothing about band energy
|
| 874 |
+
// distinguishes tongue position behind closed lips.
|
| 875 |
+
arkit::TONGUE_OUT,
|
| 876 |
+
];
|
| 877 |
+
|
| 878 |
+
let inputs: Vec<Vec<f32>> = vec![
|
| 879 |
+
vowel(730.0, 1090.0, 0.5, FRAME_SAMPLES),
|
| 880 |
+
vowel(270.0, 2290.0, 0.5, FRAME_SAMPLES),
|
| 881 |
+
noise_band(5000.0, 7500.0, 0.5, FRAME_SAMPLES),
|
| 882 |
+
noise_band(80.0, 8000.0, 0.5, FRAME_SAMPLES),
|
| 883 |
+
vec![0.0; FRAME_SAMPLES],
|
| 884 |
+
];
|
| 885 |
+
|
| 886 |
+
let mut solver = AudioBlendshapeSolver::new();
|
| 887 |
+
for input in &inputs {
|
| 888 |
+
for _ in 0..30 {
|
| 889 |
+
let out = solver.solve(input);
|
| 890 |
+
for &c in forbidden.iter() {
|
| 891 |
+
assert_eq!(
|
| 892 |
+
out[c], 0.0,
|
| 893 |
+
"solver wrote {} to autonomic channel {c}",
|
| 894 |
+
out[c]
|
| 895 |
+
);
|
| 896 |
+
}
|
| 897 |
+
}
|
| 898 |
+
}
|
| 899 |
+
}
|
| 900 |
+
|
| 901 |
+
#[test]
|
| 902 |
+
fn weights_stay_in_unit_range_for_every_input() {
|
| 903 |
+
let inputs: Vec<Vec<f32>> = vec![
|
| 904 |
+
vowel(730.0, 1090.0, 1.0, FRAME_SAMPLES),
|
| 905 |
+
vowel(270.0, 2290.0, 1.0, FRAME_SAMPLES),
|
| 906 |
+
vowel(300.0, 870.0, 1.0, FRAME_SAMPLES),
|
| 907 |
+
noise_band(3600.0, 7800.0, 1.0, FRAME_SAMPLES),
|
| 908 |
+
noise_band(80.0, 8000.0, 1.0, FRAME_SAMPLES),
|
| 909 |
+
// Deliberately over unity — a clipped capture is a real thing
|
| 910 |
+
// that happens, and it must not produce out-of-range weights.
|
| 911 |
+
vec![50.0; FRAME_SAMPLES],
|
| 912 |
+
];
|
| 913 |
+
|
| 914 |
+
let mut solver = AudioBlendshapeSolver::new();
|
| 915 |
+
for (n, input) in inputs.iter().enumerate() {
|
| 916 |
+
for _ in 0..40 {
|
| 917 |
+
let out = solver.solve(input);
|
| 918 |
+
for (i, w) in out.iter().enumerate() {
|
| 919 |
+
assert!(
|
| 920 |
+
(0.0..=1.0).contains(w),
|
| 921 |
+
"input {n}: channel {i} = {w} out of range"
|
| 922 |
+
);
|
| 923 |
+
}
|
| 924 |
+
}
|
| 925 |
+
}
|
| 926 |
+
}
|
| 927 |
+
|
| 928 |
+
/// Loud input must not drive every channel to 1.0. A solver that
|
| 929 |
+
/// saturates is producing an expressionless rictus, which passes every
|
| 930 |
+
/// range check while carrying zero information about the speech.
|
| 931 |
+
#[test]
|
| 932 |
+
fn loud_input_does_not_saturate_every_channel() {
|
| 933 |
+
let mut solver = AudioBlendshapeSolver::new();
|
| 934 |
+
let out = settle(
|
| 935 |
+
&mut solver,
|
| 936 |
+
&noise_band(80.0, 8000.0, 0.9, FRAME_SAMPLES),
|
| 937 |
+
60,
|
| 938 |
+
);
|
| 939 |
+
let saturated = out.iter().filter(|w| **w > 0.99).count();
|
| 940 |
+
assert!(
|
| 941 |
+
saturated <= 2,
|
| 942 |
+
"{saturated} channels saturated at 1.0 on loud broadband input — \
|
| 943 |
+
the basis is not discriminating"
|
| 944 |
+
);
|
| 945 |
+
}
|
| 946 |
+
|
| 947 |
+
/// **Open versus close vowel.** /a/ (F1 730) must open the jaw more
|
| 948 |
+
/// than /i/ (F1 270). This is the single most visible property of
|
| 949 |
+
/// audio-driven lip sync, and it is what a band-order or F1/F2 mix-up
|
| 950 |
+
/// would break.
|
| 951 |
+
#[test]
|
| 952 |
+
fn open_vowel_opens_the_jaw_more_than_a_close_vowel() {
|
| 953 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 954 |
+
let ah = settle(&mut s, &vowel(730.0, 1090.0, 0.5, FRAME_SAMPLES), 60)
|
| 955 |
+
[arkit::JAW_OPEN];
|
| 956 |
+
|
| 957 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 958 |
+
let ee = settle(&mut s, &vowel(270.0, 2290.0, 0.5, FRAME_SAMPLES), 60)
|
| 959 |
+
[arkit::JAW_OPEN];
|
| 960 |
+
|
| 961 |
+
println!("jawOpen: /a/ = {ah:.4}, /i/ = {ee:.4}");
|
| 962 |
+
assert!(
|
| 963 |
+
ah > ee * 1.5,
|
| 964 |
+
"/a/ (F1 730) gave jawOpen {ah} but /i/ (F1 270) gave {ee} — the \
|
| 965 |
+
jaw is not tracking F1"
|
| 966 |
+
);
|
| 967 |
+
assert!(ah > 0.15, "/a/ barely opened the jaw ({ah}) — visually dead");
|
| 968 |
+
}
|
| 969 |
+
|
| 970 |
+
/// **Rounded versus spread.** /u/ must round the lips and /i/ must
|
| 971 |
+
/// spread them, and each must suppress the other. Driving both on every
|
| 972 |
+
/// vowel is the failure the inhibitory basis weights exist to prevent.
|
| 973 |
+
#[test]
|
| 974 |
+
fn rounded_and_spread_vowels_produce_opposite_lip_shapes() {
|
| 975 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 976 |
+
// /u/ — F1 300, F2 870: both formants low.
|
| 977 |
+
let oo = settle(&mut s, &vowel(300.0, 870.0, 0.5, FRAME_SAMPLES), 60);
|
| 978 |
+
|
| 979 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 980 |
+
// /i/ — F1 270, F2 2290: F2 far up.
|
| 981 |
+
let ee = settle(&mut s, &vowel(270.0, 2290.0, 0.5, FRAME_SAMPLES), 60);
|
| 982 |
+
|
| 983 |
+
println!(
|
| 984 |
+
"/u/: funnel {:.4} pucker {:.4} smile {:.4}",
|
| 985 |
+
oo[arkit::MOUTH_FUNNEL],
|
| 986 |
+
oo[arkit::MOUTH_PUCKER],
|
| 987 |
+
oo[arkit::MOUTH_SMILE_LEFT]
|
| 988 |
+
);
|
| 989 |
+
println!(
|
| 990 |
+
"/i/: funnel {:.4} pucker {:.4} smile {:.4}",
|
| 991 |
+
ee[arkit::MOUTH_FUNNEL],
|
| 992 |
+
ee[arkit::MOUTH_PUCKER],
|
| 993 |
+
ee[arkit::MOUTH_SMILE_LEFT]
|
| 994 |
+
);
|
| 995 |
+
|
| 996 |
+
assert!(
|
| 997 |
+
oo[arkit::MOUTH_PUCKER] > oo[arkit::MOUTH_SMILE_LEFT],
|
| 998 |
+
"/u/ smiled ({}) more than it puckered ({})",
|
| 999 |
+
oo[arkit::MOUTH_SMILE_LEFT],
|
| 1000 |
+
oo[arkit::MOUTH_PUCKER]
|
| 1001 |
+
);
|
| 1002 |
+
assert!(
|
| 1003 |
+
ee[arkit::MOUTH_SMILE_LEFT] > ee[arkit::MOUTH_PUCKER],
|
| 1004 |
+
"/i/ puckered ({}) more than it smiled ({})",
|
| 1005 |
+
ee[arkit::MOUTH_PUCKER],
|
| 1006 |
+
ee[arkit::MOUTH_SMILE_LEFT]
|
| 1007 |
+
);
|
| 1008 |
+
assert!(
|
| 1009 |
+
ee[arkit::MOUTH_SMILE_LEFT] > 0.1,
|
| 1010 |
+
"/i/ barely spread the lips ({}) — visually dead",
|
| 1011 |
+
ee[arkit::MOUTH_SMILE_LEFT]
|
| 1012 |
+
);
|
| 1013 |
+
}
|
| 1014 |
+
|
| 1015 |
+
/// Unvoiced friction must engage the lip/teeth channels rather than
|
| 1016 |
+
/// dropping the jaw. A solver keyed on loudness alone gapes at every
|
| 1017 |
+
/// /s/.
|
| 1018 |
+
#[test]
|
| 1019 |
+
fn sibilance_engages_lips_rather_than_opening_the_jaw() {
|
| 1020 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 1021 |
+
let ess = settle(
|
| 1022 |
+
&mut s,
|
| 1023 |
+
&noise_band(5000.0, 7500.0, 0.5, FRAME_SAMPLES),
|
| 1024 |
+
60,
|
| 1025 |
+
);
|
| 1026 |
+
println!(
|
| 1027 |
+
"/s/: jawOpen {:.4} shrugLower {:.4} rollLower {:.4}",
|
| 1028 |
+
ess[arkit::JAW_OPEN],
|
| 1029 |
+
ess[arkit::MOUTH_SHRUG_LOWER],
|
| 1030 |
+
ess[arkit::MOUTH_ROLL_LOWER]
|
| 1031 |
+
);
|
| 1032 |
+
assert!(
|
| 1033 |
+
ess[arkit::MOUTH_SHRUG_LOWER] > ess[arkit::JAW_OPEN],
|
| 1034 |
+
"/s/ opened the jaw ({}) more than it engaged the lips ({})",
|
| 1035 |
+
ess[arkit::JAW_OPEN],
|
| 1036 |
+
ess[arkit::MOUTH_SHRUG_LOWER]
|
| 1037 |
+
);
|
| 1038 |
+
}
|
| 1039 |
+
|
| 1040 |
+
/// Voiced low-frequency energy with nothing above it is bilabial
|
| 1041 |
+
/// closure, and must close rather than open the mouth.
|
| 1042 |
+
#[test]
|
| 1043 |
+
fn voiced_low_energy_reads_as_closure_not_aperture() {
|
| 1044 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 1045 |
+
let mm = settle(&mut s, &sine(150.0, 0.5, FRAME_SAMPLES, 0.0), 60);
|
| 1046 |
+
println!(
|
| 1047 |
+
"/m/: mouthClose {:.4} jawOpen {:.4}",
|
| 1048 |
+
mm[arkit::MOUTH_CLOSE],
|
| 1049 |
+
mm[arkit::JAW_OPEN]
|
| 1050 |
+
);
|
| 1051 |
+
assert!(
|
| 1052 |
+
mm[arkit::MOUTH_CLOSE] > mm[arkit::JAW_OPEN],
|
| 1053 |
+
"150 Hz voicing gave mouthClose {} vs jawOpen {} — closure is not \
|
| 1054 |
+
being detected",
|
| 1055 |
+
mm[arkit::MOUTH_CLOSE],
|
| 1056 |
+
mm[arkit::JAW_OPEN]
|
| 1057 |
+
);
|
| 1058 |
+
}
|
| 1059 |
+
|
| 1060 |
+
// -----------------------------------------------------------------
|
| 1061 |
+
// Temporal behaviour
|
| 1062 |
+
// -----------------------------------------------------------------
|
| 1063 |
+
|
| 1064 |
+
/// Attack must be faster than release. Symmetric smoothing makes the
|
| 1065 |
+
/// mouth snap shut between syllables — the classic cheap-lip-sync
|
| 1066 |
+
/// chatter.
|
| 1067 |
+
#[test]
|
| 1068 |
+
fn attack_is_faster_than_release() {
|
| 1069 |
+
let loud = vowel(730.0, 1090.0, 0.5, FRAME_SAMPLES);
|
| 1070 |
+
let quiet = vec![0.0f32; FRAME_SAMPLES];
|
| 1071 |
+
|
| 1072 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 1073 |
+
// Reach steady state, note it, then measure how many frames it took
|
| 1074 |
+
// to get most of the way there from rest.
|
| 1075 |
+
let settled = settle(&mut s, &loud, 80)[arkit::JAW_OPEN];
|
| 1076 |
+
assert!(settled > 0.1, "no signal to measure ({settled})");
|
| 1077 |
+
|
| 1078 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 1079 |
+
let mut rise_frames = 0;
|
| 1080 |
+
for f in 1..=200 {
|
| 1081 |
+
if s.solve(&loud)[arkit::JAW_OPEN] >= settled * 0.9 {
|
| 1082 |
+
rise_frames = f;
|
| 1083 |
+
break;
|
| 1084 |
+
}
|
| 1085 |
+
}
|
| 1086 |
+
|
| 1087 |
+
let mut fall_frames = 0;
|
| 1088 |
+
for f in 1..=200 {
|
| 1089 |
+
if s.solve(&quiet)[arkit::JAW_OPEN] <= settled * 0.1 {
|
| 1090 |
+
fall_frames = f;
|
| 1091 |
+
break;
|
| 1092 |
+
}
|
| 1093 |
+
}
|
| 1094 |
+
|
| 1095 |
+
println!("jawOpen rise {rise_frames} frames, fall {fall_frames} frames");
|
| 1096 |
+
assert!(rise_frames > 0 && fall_frames > 0, "never settled either way");
|
| 1097 |
+
assert!(
|
| 1098 |
+
fall_frames > rise_frames,
|
| 1099 |
+
"release ({fall_frames} frames) is not slower than attack \
|
| 1100 |
+
({rise_frames} frames)"
|
| 1101 |
+
);
|
| 1102 |
+
}
|
| 1103 |
+
|
| 1104 |
+
/// A steady input must reach a steady output. A solver that never
|
| 1105 |
+
/// settles has replaced silence with jitter.
|
| 1106 |
+
#[test]
|
| 1107 |
+
fn steady_input_converges_to_a_steady_output() {
|
| 1108 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 1109 |
+
let block = vowel(730.0, 1090.0, 0.5, FRAME_SAMPLES);
|
| 1110 |
+
let settled = settle(&mut s, &block, 120);
|
| 1111 |
+
for _ in 0..30 {
|
| 1112 |
+
let now = *s.solve(&block);
|
| 1113 |
+
for i in 0..BLENDSHAPE_COUNT {
|
| 1114 |
+
assert!(
|
| 1115 |
+
(now[i] - settled[i]).abs() < 1e-3,
|
| 1116 |
+
"channel {i} drifted from {} to {}",
|
| 1117 |
+
settled[i],
|
| 1118 |
+
now[i]
|
| 1119 |
+
);
|
| 1120 |
+
}
|
| 1121 |
+
}
|
| 1122 |
+
}
|
| 1123 |
+
|
| 1124 |
+
/// Reproducibility is required by T9's capture workflow: the same audio
|
| 1125 |
+
/// must yield the same weights.
|
| 1126 |
+
#[test]
|
| 1127 |
+
fn solving_is_deterministic_after_reset() {
|
| 1128 |
+
let block = vowel(500.0, 1500.0, 0.4, FRAME_SAMPLES);
|
| 1129 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 1130 |
+
let first = settle(&mut s, &block, 50);
|
| 1131 |
+
s.reset();
|
| 1132 |
+
let second = settle(&mut s, &block, 50);
|
| 1133 |
+
assert_eq!(first, second, "solver output is not reproducible after reset");
|
| 1134 |
+
}
|
| 1135 |
+
|
| 1136 |
+
#[test]
|
| 1137 |
+
fn reset_clears_filter_ringing_between_utterances() {
|
| 1138 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 1139 |
+
settle(&mut s, &vowel(730.0, 1090.0, 0.8, FRAME_SAMPLES), 40);
|
| 1140 |
+
s.reset();
|
| 1141 |
+
for (i, w) in s.weights().iter().enumerate() {
|
| 1142 |
+
assert_eq!(*w, 0.0, "channel {i} survived reset with {w}");
|
| 1143 |
+
}
|
| 1144 |
+
assert_eq!(s.features().drive, 0.0);
|
| 1145 |
+
}
|
| 1146 |
+
|
| 1147 |
+
#[test]
|
| 1148 |
+
fn add_into_is_additive_not_assigning() {
|
| 1149 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 1150 |
+
settle(&mut s, &vowel(730.0, 1090.0, 0.5, FRAME_SAMPLES), 60);
|
| 1151 |
+
|
| 1152 |
+
let mut out = [0.0f32; BLENDSHAPE_COUNT];
|
| 1153 |
+
out[arkit::JAW_OPEN] = 0.04; // as the breath oscillator would leave it
|
| 1154 |
+
let before = out[arkit::JAW_OPEN];
|
| 1155 |
+
s.add_into(&mut out);
|
| 1156 |
+
assert!(
|
| 1157 |
+
out[arkit::JAW_OPEN] > before,
|
| 1158 |
+
"add_into clobbered an existing contribution ({before} -> {})",
|
| 1159 |
+
out[arkit::JAW_OPEN]
|
| 1160 |
+
);
|
| 1161 |
+
assert!(
|
| 1162 |
+
(out[arkit::JAW_OPEN] - (before + s.weights()[arkit::JAW_OPEN])).abs() < 1e-6
|
| 1163 |
+
);
|
| 1164 |
+
}
|
| 1165 |
+
|
| 1166 |
+
#[test]
|
| 1167 |
+
fn solver_is_send() {
|
| 1168 |
+
fn assert_send<T: Send>() {}
|
| 1169 |
+
assert_send::<AudioBlendshapeSolver>();
|
| 1170 |
+
assert_send::<BandBank>();
|
| 1171 |
+
}
|
| 1172 |
+
|
| 1173 |
+
// -----------------------------------------------------------------
|
| 1174 |
+
// Performance budget
|
| 1175 |
+
// -----------------------------------------------------------------
|
| 1176 |
+
|
| 1177 |
+
/// **Acceptance criterion: solve time under 0.2 ms.**
|
| 1178 |
+
///
|
| 1179 |
+
/// Measured on a real block of one frame's worth of audio (267 samples
|
| 1180 |
+
/// at 16 kHz), averaged over many iterations to get above timer
|
| 1181 |
+
/// resolution.
|
| 1182 |
+
///
|
| 1183 |
+
/// The assertion is gated on `debug_assertions` because an unoptimised
|
| 1184 |
+
/// build is 10-40x slower and the number would say nothing about the
|
| 1185 |
+
/// shipped code. In a debug build the measurement is printed and the
|
| 1186 |
+
/// budget is not enforced; in release it is enforced. Run
|
| 1187 |
+
/// `cargo test --release -p miranda-nodes solve_time` for the number
|
| 1188 |
+
/// that counts.
|
| 1189 |
+
#[test]
|
| 1190 |
+
fn solve_time_is_within_the_frame_budget() {
|
| 1191 |
+
const BUDGET_US: f64 = 200.0;
|
| 1192 |
+
const ITERS: usize = 20_000;
|
| 1193 |
+
|
| 1194 |
+
let block = vowel(730.0, 1090.0, 0.5, FRAME_SAMPLES);
|
| 1195 |
+
let mut s = AudioBlendshapeSolver::new();
|
| 1196 |
+
|
| 1197 |
+
// Warm up so the measurement is not dominated by first-touch page
|
| 1198 |
+
// faults and cold caches.
|
| 1199 |
+
for _ in 0..2_000 {
|
| 1200 |
+
s.solve(&block);
|
| 1201 |
+
}
|
| 1202 |
+
|
| 1203 |
+
let start = Instant::now();
|
| 1204 |
+
for _ in 0..ITERS {
|
| 1205 |
+
let out = s.solve(&block);
|
| 1206 |
+
// Consume the result so the loop cannot be optimised away.
|
| 1207 |
+
std::hint::black_box(out[arkit::JAW_OPEN]);
|
| 1208 |
+
}
|
| 1209 |
+
let per_solve_us = start.elapsed().as_secs_f64() * 1e6 / ITERS as f64;
|
| 1210 |
+
|
| 1211 |
+
println!(
|
| 1212 |
+
"solve: {per_solve_us:.3} us/frame for {FRAME_SAMPLES} samples \
|
| 1213 |
+
({BAND_COUNT} bands, f32x4) — budget {BUDGET_US} us, \
|
| 1214 |
+
{}% of a 16.67 ms frame",
|
| 1215 |
+
(per_solve_us / 16_670.0 * 100.0).round()
|
| 1216 |
+
);
|
| 1217 |
+
|
| 1218 |
+
if cfg!(debug_assertions) {
|
| 1219 |
+
println!(
|
| 1220 |
+
"NOTE: debug build — budget NOT enforced. Run with --release \
|
| 1221 |
+
for the number that counts."
|
| 1222 |
+
);
|
| 1223 |
+
} else {
|
| 1224 |
+
assert!(
|
| 1225 |
+
per_solve_us < BUDGET_US,
|
| 1226 |
+
"solve took {per_solve_us:.3} us, over the {BUDGET_US} us budget"
|
| 1227 |
+
);
|
| 1228 |
+
}
|
| 1229 |
+
}
|
| 1230 |
+
}
|