using System; namespace Audio2Face { /// /// Per-frame skin post-process + ARKit-52 blendshape solve, ported from the Audio2Face-3D SDK (MIT): /// multitrack_animator (two-pole EMA, lower/upper face mask) and blendshape_solver_base (L1/L2/temporal/symmetry /// regularised least squares, box [0,1], then multipliers/offsets). Stateful: call per utterance. /// public sealed class A2FSolver { readonly A2FIdentityData d; readonly int P, K; readonly float[] lower1, lower2, upper1, upper2, tgt; readonly double[] prev, b, c; bool initialized; /// /// false (default): the solve target is the motion relative to the model's neutral, so a face at rest solves to 0. /// true: SDK-exact, relative to the bs_skin neutral. The two neutrals differ by up to 0.2 cm, so a face at rest /// solves to a constant offset (Claire: mouthFrown ~0.42, mouthRollLower ~0.24). /// public bool SdkNeutralReference { get; set; } /// Called after each solve with (b, x) of the active poses: verification hook. public Action OnSolved; public A2FIdentityData Data => d; readonly double[] pl1, pl2, pu1, pu2; // EMA state in pose space (projected path) public A2FSolver(A2FIdentityData data) { d = data; P = data.RowCount; K = data.ActiveCount; if (K > 64) throw new ArgumentException("at most 64 active poses"); lower1 = new float[P]; lower2 = new float[P]; upper1 = new float[P]; upper2 = new float[P]; tgt = new float[P]; prev = new double[K]; b = new double[K]; c = new double[K]; pl1 = new double[K]; pl2 = new double[K]; pu1 = new double[K]; pu2 = new double[K]; } public void Reset() { initialized = false; Array.Clear(prev, 0, K); } /// /// One output frame from the projected network output (A2FInference.Output.Projected): 2K values per frame, /// [delta W_lower | delta W_upper]. The EMA is linear and identical for every row, so filtering in pose space is /// exactly the SDK's per-vertex filter followed by D^T. weights: 52 ARKit weights (SDK order) written in place. /// public void Solve(float[] projected, int offset, float[] weights) { double aL = d.AlphaLower, aU = d.AlphaUpper; var c0 = SdkNeutralReference ? d.C0Sdk : d.C0Model; for (int k = 0; k < K; k++) { double yl = projected[offset + k], yu = projected[offset + K + k]; if (initialized && aL > 0) { pl1[k] += (yl - pl1[k]) * aL; pl2[k] += (pl1[k] - pl2[k]) * aL; } else pl1[k] = pl2[k] = yl; if (initialized && aU > 0) { pu1[k] += (yu - pu1[k]) * aU; pu2[k] += (pu1[k] - pu2[k]) * aU; } else pu1[k] = pu2[k] = yu; b[k] = c0[k] + pl2[k] + pu2[k] + d.TemporalB * prev[k]; } initialized = true; Finish(weights); } /// /// Reference path, SDK order of operations (A2FInference.Output.Rows): skinRows are the network's skin deltas at /// (offset into a [frames * P] buffer). /// public void SolveRows(float[] skinRows, int offset, float[] weights) { float aL = d.AlphaLower, aU = d.AlphaUpper, s = d.SkinStrength; var reference = SdkNeutralReference ? d.BsNeutral : d.Neutral; for (int r = 0; r < P; r++) { float delta = s * skinRows[offset + r]; if (initialized && aL > 0) { lower1[r] += (delta - lower1[r]) * aL; lower2[r] += (lower1[r] - lower2[r]) * aL; } else lower1[r] = lower2[r] = delta; if (initialized && aU > 0) { upper1[r] += (delta - upper1[r]) * aU; upper2[r] += (upper1[r] - upper2[r]) * aU; } else upper1[r] = upper2[r] = delta; float m = d.FaceMask[r]; float abs = d.Neutral[r] + upper2[r] * d.UpperStrength * (1 - m) + lower2[r] * d.LowerStrength * m; tgt[r] = abs - reference[r]; } initialized = true; // b = D^T tgt + T*scale*prev (SDK: blendshape_solver.cpp, no x100 here although A uses 100*T*scale) var Dt = d.Dt; for (int k = 0; k < K; k++) { double acc = 0; int o = k * P; for (int r = 0; r < P; r++) acc += Dt[o + r] * tgt[r]; b[k] = acc + d.TemporalB * prev[k]; } Finish(weights); } void Finish(float[] weights) { // min ||A x - b||^2, 0 <= x <= 1 == min 1/2 x^T Q x - c^T x with Q = A^T A, c = A^T b var A = d.A; for (int i = 0; i < K; i++) { double acc = 0; for (int k = 0; k < K; k++) acc += (double)A[k * K + i] * b[k]; c[i] = acc; } BoxQp(d.QD, c, prev, K); // warm start from the previous frame OnSolved?.Invoke(b, prev); Array.Clear(weights, 0, A2FConstants.ArkitCount); for (int k = 0; k < K; k++) weights[d.Active[k]] = (float)prev[k]; for (int j = 0; j < A2FConstants.ArkitCount; j++) weights[j] = weights[j] * d.Multipliers[j] + d.Offsets[j]; } // active-set workspace readonly int[] m_State = new int[64]; // -1 at lower, +1 at upper, 0 free readonly double[] m_L = new double[64 * 64], m_Rhs = new double[64], m_Sol = new double[64], m_G = new double[64]; readonly int[] m_Free = new int[64]; /// /// min 1/2 x^T Q x - c^T x, 0 <= x <= 1 (== min ||A x - b||^2 with Q = A^T A, c = A^T b): primal active-set /// method, the same algorithm family as the SDK's BVLS. Exact up to round-off; warm-started from x. /// void BoxQp(double[] Q, double[] c, double[] x, int K) { for (int i = 0; i < K; i++) { x[i] = x[i] <= 0 ? 0 : x[i] >= 1 ? 1 : x[i]; m_State[i] = x[i] <= 0 ? -1 : x[i] >= 1 ? 1 : 0; } for (int iter = 0; iter < 8 * K + 16; iter++) { // equality-constrained minimiser on the free set (bound variables held at their bounds) int nf = 0; for (int i = 0; i < K; i++) if (m_State[i] == 0) m_Free[nf++] = i; for (int a = 0; a < nf; a++) { int i = m_Free[a]; double r = c[i]; for (int j = 0; j < K; j++) if (m_State[j] != 0) r -= Q[i * K + j] * x[j]; m_Rhs[a] = r; for (int bb = 0; bb < nf; bb++) m_L[a * nf + bb] = Q[i * K + m_Free[bb]]; } CholeskySolve(m_L, m_Rhs, m_Sol, nf); // step toward it, stopping at the first bound hit double alpha = 1; int hit = -1; for (int a = 0; a < nf; a++) { int i = m_Free[a]; double d = m_Sol[a] - x[i]; if (d < 0 && m_Sol[a] < 0) { double t = x[i] / -d; if (t < alpha) { alpha = t; hit = i; } } else if (d > 0 && m_Sol[a] > 1) { double t = (1 - x[i]) / d; if (t < alpha) { alpha = t; hit = i; } } } for (int a = 0; a < nf; a++) { int i = m_Free[a]; x[i] += alpha * (m_Sol[a] - x[i]); } if (hit >= 0) { m_State[hit] = m_Sol[Array.IndexOf(m_Free, hit, 0, nf)] < 0 ? -1 : 1; x[hit] = m_State[hit] < 0 ? 0 : 1; continue; } // KKT: release the bound variable whose gradient points most into the box int release = -1; double worst = 1e-12; for (int i = 0; i < K; i++) { if (m_State[i] == 0) continue; double g = -c[i]; for (int j = 0; j < K; j++) g += Q[i * K + j] * x[j]; double v = m_State[i] < 0 ? -g : g; // at 0: g<0 wants up; at 1: g>0 wants down if (v > worst) { worst = v; release = i; } } if (release < 0) return; m_State[release] = 0; } } static void CholeskySolve(double[] M, double[] rhs, double[] sol, int n) { // in-place LL^T (M is SPD: Q restricted to the free set) for (int j = 0; j < n; j++) { double s = M[j * n + j]; for (int k = 0; k < j; k++) s -= M[j * n + k] * M[j * n + k]; double d = Math.Sqrt(Math.Max(s, 1e-300)); M[j * n + j] = d; for (int i = j + 1; i < n; i++) { double t = M[i * n + j]; for (int k = 0; k < j; k++) t -= M[i * n + k] * M[j * n + k]; M[i * n + j] = t / d; } } for (int i = 0; i < n; i++) { double t = rhs[i]; for (int k = 0; k < i; k++) t -= M[i * n + k] * sol[k]; sol[i] = t / M[i * n + i]; } for (int i = n - 1; i >= 0; i--) { double t = sol[i]; for (int k = i + 1; k < n; k++) t -= M[k * n + i] * sol[k]; sol[i] = t / M[i * n + i]; } } } }