Audio-to-Audio
unity-sentis
ONNX
unity
sentis
inference-engine
on-device
lipsync
facial-animation
arkit
blendshapes
audio2face
Instructions to use Sky-Kim/com.sky.sentis.audio2face with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- unity-sentis
How to use Sky-Kim/com.sky.sentis.audio2face with unity-sentis:
string modelName = "[Your model name here].sentis"; Model model = ModelLoader.Load(Application.streamingAssetsPath + "/" + modelName); IWorker engine = WorkerFactory.CreateWorker(BackendType.GPUCompute, model); // Please see provided C# file for more details
- Notebooks
- Google Colab
- Kaggle
Download Runtime/A2FSolver.cs from Sky-Kim/com.sky.sentis.audio2face: direct link, hf CLI and curl.
- Browser
- Download file 9.62 kB
-
https://huggingface.co/Sky-Kim/com.sky.sentis.audio2face/resolve/main/Runtime/A2FSolver.cs
- Command line
-
hf download hf://Sky-Kim/com.sky.sentis.audio2face/Runtime/A2FSolver.cs
-
curl -L -o A2FSolver.cs https://huggingface.co/Sky-Kim/com.sky.sentis.audio2face/resolve/main/Runtime/A2FSolver.cs
9.62 kB
| using System; | |
| namespace Audio2Face | |
| { | |
| /// <summary> | |
| /// Per-frame skin post-process + ARKit-52 blendshape solve, ported from the Audio2Face-3D SDK (MIT): | |
| /// multitrack_animator (two-pole EMA, lower/upper face mask) and blendshape_solver_base (L1/L2/temporal/symmetry | |
| /// regularised least squares, box [0,1], then multipliers/offsets). Stateful: call <see cref="Reset"/> per utterance. | |
| /// </summary> | |
| public sealed class A2FSolver | |
| { | |
| readonly A2FIdentityData d; | |
| readonly int P, K; | |
| readonly float[] lower1, lower2, upper1, upper2, tgt; | |
| readonly double[] prev, b, c; | |
| bool initialized; | |
| /// <summary> | |
| /// false (default): the solve target is the motion relative to the model's neutral, so a face at rest solves to 0. | |
| /// true: SDK-exact, relative to the bs_skin neutral. The two neutrals differ by up to 0.2 cm, so a face at rest | |
| /// solves to a constant offset (Claire: mouthFrown ~0.42, mouthRollLower ~0.24). | |
| /// </summary> | |
| public bool SdkNeutralReference { get; set; } | |
| /// <summary>Called after each solve with (b, x) of the active poses: verification hook.</summary> | |
| public Action<double[], double[]> OnSolved; | |
| public A2FIdentityData Data => d; | |
| readonly double[] pl1, pl2, pu1, pu2; // EMA state in pose space (projected path) | |
| public A2FSolver(A2FIdentityData data) | |
| { | |
| d = data; P = data.RowCount; K = data.ActiveCount; | |
| if (K > 64) throw new ArgumentException("at most 64 active poses"); | |
| lower1 = new float[P]; lower2 = new float[P]; upper1 = new float[P]; upper2 = new float[P]; tgt = new float[P]; | |
| prev = new double[K]; b = new double[K]; c = new double[K]; | |
| pl1 = new double[K]; pl2 = new double[K]; pu1 = new double[K]; pu2 = new double[K]; | |
| } | |
| public void Reset() | |
| { | |
| initialized = false; | |
| Array.Clear(prev, 0, K); | |
| } | |
| /// <summary> | |
| /// One output frame from the projected network output (A2FInference.Output.Projected): 2K values per frame, | |
| /// [delta W_lower | delta W_upper]. The EMA is linear and identical for every row, so filtering in pose space is | |
| /// exactly the SDK's per-vertex filter followed by D^T. weights: 52 ARKit weights (SDK order) written in place. | |
| /// </summary> | |
| public void Solve(float[] projected, int offset, float[] weights) | |
| { | |
| double aL = d.AlphaLower, aU = d.AlphaUpper; | |
| var c0 = SdkNeutralReference ? d.C0Sdk : d.C0Model; | |
| for (int k = 0; k < K; k++) | |
| { | |
| double yl = projected[offset + k], yu = projected[offset + K + k]; | |
| if (initialized && aL > 0) { pl1[k] += (yl - pl1[k]) * aL; pl2[k] += (pl1[k] - pl2[k]) * aL; } | |
| else pl1[k] = pl2[k] = yl; | |
| if (initialized && aU > 0) { pu1[k] += (yu - pu1[k]) * aU; pu2[k] += (pu1[k] - pu2[k]) * aU; } | |
| else pu1[k] = pu2[k] = yu; | |
| b[k] = c0[k] + pl2[k] + pu2[k] + d.TemporalB * prev[k]; | |
| } | |
| initialized = true; | |
| Finish(weights); | |
| } | |
| /// <summary> | |
| /// Reference path, SDK order of operations (A2FInference.Output.Rows): skinRows are the network's skin deltas at | |
| /// <see cref="A2FIdentityData.Rows"/> (offset into a [frames * P] buffer). | |
| /// </summary> | |
| public void SolveRows(float[] skinRows, int offset, float[] weights) | |
| { | |
| float aL = d.AlphaLower, aU = d.AlphaUpper, s = d.SkinStrength; | |
| var reference = SdkNeutralReference ? d.BsNeutral : d.Neutral; | |
| for (int r = 0; r < P; r++) | |
| { | |
| float delta = s * skinRows[offset + r]; | |
| if (initialized && aL > 0) { lower1[r] += (delta - lower1[r]) * aL; lower2[r] += (lower1[r] - lower2[r]) * aL; } | |
| else lower1[r] = lower2[r] = delta; | |
| if (initialized && aU > 0) { upper1[r] += (delta - upper1[r]) * aU; upper2[r] += (upper1[r] - upper2[r]) * aU; } | |
| else upper1[r] = upper2[r] = delta; | |
| float m = d.FaceMask[r]; | |
| float abs = d.Neutral[r] + upper2[r] * d.UpperStrength * (1 - m) + lower2[r] * d.LowerStrength * m; | |
| tgt[r] = abs - reference[r]; | |
| } | |
| initialized = true; | |
| // b = D^T tgt + T*scale*prev (SDK: blendshape_solver.cpp, no x100 here although A uses 100*T*scale) | |
| var Dt = d.Dt; | |
| for (int k = 0; k < K; k++) | |
| { | |
| double acc = 0; int o = k * P; | |
| for (int r = 0; r < P; r++) acc += Dt[o + r] * tgt[r]; | |
| b[k] = acc + d.TemporalB * prev[k]; | |
| } | |
| Finish(weights); | |
| } | |
| void Finish(float[] weights) | |
| { | |
| // min ||A x - b||^2, 0 <= x <= 1 == min 1/2 x^T Q x - c^T x with Q = A^T A, c = A^T b | |
| var A = d.A; | |
| for (int i = 0; i < K; i++) { double acc = 0; for (int k = 0; k < K; k++) acc += (double)A[k * K + i] * b[k]; c[i] = acc; } | |
| BoxQp(d.QD, c, prev, K); // warm start from the previous frame | |
| OnSolved?.Invoke(b, prev); | |
| Array.Clear(weights, 0, A2FConstants.ArkitCount); | |
| for (int k = 0; k < K; k++) weights[d.Active[k]] = (float)prev[k]; | |
| for (int j = 0; j < A2FConstants.ArkitCount; j++) weights[j] = weights[j] * d.Multipliers[j] + d.Offsets[j]; | |
| } | |
| // active-set workspace | |
| readonly int[] m_State = new int[64]; // -1 at lower, +1 at upper, 0 free | |
| readonly double[] m_L = new double[64 * 64], m_Rhs = new double[64], m_Sol = new double[64], m_G = new double[64]; | |
| readonly int[] m_Free = new int[64]; | |
| /// <summary> | |
| /// min 1/2 x^T Q x - c^T x, 0 <= x <= 1 (== min ||A x - b||^2 with Q = A^T A, c = A^T b): primal active-set | |
| /// method, the same algorithm family as the SDK's BVLS. Exact up to round-off; warm-started from x. | |
| /// </summary> | |
| void BoxQp(double[] Q, double[] c, double[] x, int K) | |
| { | |
| for (int i = 0; i < K; i++) | |
| { | |
| x[i] = x[i] <= 0 ? 0 : x[i] >= 1 ? 1 : x[i]; | |
| m_State[i] = x[i] <= 0 ? -1 : x[i] >= 1 ? 1 : 0; | |
| } | |
| for (int iter = 0; iter < 8 * K + 16; iter++) | |
| { | |
| // equality-constrained minimiser on the free set (bound variables held at their bounds) | |
| int nf = 0; | |
| for (int i = 0; i < K; i++) if (m_State[i] == 0) m_Free[nf++] = i; | |
| for (int a = 0; a < nf; a++) | |
| { | |
| int i = m_Free[a]; | |
| double r = c[i]; | |
| for (int j = 0; j < K; j++) if (m_State[j] != 0) r -= Q[i * K + j] * x[j]; | |
| m_Rhs[a] = r; | |
| for (int bb = 0; bb < nf; bb++) m_L[a * nf + bb] = Q[i * K + m_Free[bb]]; | |
| } | |
| CholeskySolve(m_L, m_Rhs, m_Sol, nf); | |
| // step toward it, stopping at the first bound hit | |
| double alpha = 1; int hit = -1; | |
| for (int a = 0; a < nf; a++) | |
| { | |
| int i = m_Free[a]; double d = m_Sol[a] - x[i]; | |
| if (d < 0 && m_Sol[a] < 0) { double t = x[i] / -d; if (t < alpha) { alpha = t; hit = i; } } | |
| else if (d > 0 && m_Sol[a] > 1) { double t = (1 - x[i]) / d; if (t < alpha) { alpha = t; hit = i; } } | |
| } | |
| for (int a = 0; a < nf; a++) { int i = m_Free[a]; x[i] += alpha * (m_Sol[a] - x[i]); } | |
| if (hit >= 0) | |
| { | |
| m_State[hit] = m_Sol[Array.IndexOf(m_Free, hit, 0, nf)] < 0 ? -1 : 1; | |
| x[hit] = m_State[hit] < 0 ? 0 : 1; | |
| continue; | |
| } | |
| // KKT: release the bound variable whose gradient points most into the box | |
| int release = -1; double worst = 1e-12; | |
| for (int i = 0; i < K; i++) | |
| { | |
| if (m_State[i] == 0) continue; | |
| double g = -c[i]; | |
| for (int j = 0; j < K; j++) g += Q[i * K + j] * x[j]; | |
| double v = m_State[i] < 0 ? -g : g; // at 0: g<0 wants up; at 1: g>0 wants down | |
| if (v > worst) { worst = v; release = i; } | |
| } | |
| if (release < 0) return; | |
| m_State[release] = 0; | |
| } | |
| } | |
| static void CholeskySolve(double[] M, double[] rhs, double[] sol, int n) | |
| { | |
| // in-place LL^T (M is SPD: Q restricted to the free set) | |
| for (int j = 0; j < n; j++) | |
| { | |
| double s = M[j * n + j]; | |
| for (int k = 0; k < j; k++) s -= M[j * n + k] * M[j * n + k]; | |
| double d = Math.Sqrt(Math.Max(s, 1e-300)); | |
| M[j * n + j] = d; | |
| for (int i = j + 1; i < n; i++) | |
| { | |
| double t = M[i * n + j]; | |
| for (int k = 0; k < j; k++) t -= M[i * n + k] * M[j * n + k]; | |
| M[i * n + j] = t / d; | |
| } | |
| } | |
| for (int i = 0; i < n; i++) { double t = rhs[i]; for (int k = 0; k < i; k++) t -= M[i * n + k] * sol[k]; sol[i] = t / M[i * n + i]; } | |
| for (int i = n - 1; i >= 0; i--) { double t = sol[i]; for (int k = i + 1; k < n; k++) t -= M[k * n + i] * sol[k]; sol[i] = t / M[i * n + i]; } | |
| } | |
| } | |
| } | |