Sky-Kim's picture
Initial release: NVIDIA Audio2Face-3D v3.0 for Unity Sentis 2.6.1 (ARKit-52, streaming, Claire/James/Mark)
575c51d
Raw History Blame Contribute Delete
9.62 kB
using System;
namespace Audio2Face
{
/// <summary>
/// Per-frame skin post-process + ARKit-52 blendshape solve, ported from the Audio2Face-3D SDK (MIT):
/// multitrack_animator (two-pole EMA, lower/upper face mask) and blendshape_solver_base (L1/L2/temporal/symmetry
/// regularised least squares, box [0,1], then multipliers/offsets). Stateful: call <see cref="Reset"/> per utterance.
/// </summary>
public sealed class A2FSolver
{
readonly A2FIdentityData d;
readonly int P, K;
readonly float[] lower1, lower2, upper1, upper2, tgt;
readonly double[] prev, b, c;
bool initialized;
/// <summary>
/// false (default): the solve target is the motion relative to the model's neutral, so a face at rest solves to 0.
/// true: SDK-exact, relative to the bs_skin neutral. The two neutrals differ by up to 0.2 cm, so a face at rest
/// solves to a constant offset (Claire: mouthFrown ~0.42, mouthRollLower ~0.24).
/// </summary>
public bool SdkNeutralReference { get; set; }
/// <summary>Called after each solve with (b, x) of the active poses: verification hook.</summary>
public Action<double[], double[]> OnSolved;
public A2FIdentityData Data => d;
readonly double[] pl1, pl2, pu1, pu2; // EMA state in pose space (projected path)
public A2FSolver(A2FIdentityData data)
{
d = data; P = data.RowCount; K = data.ActiveCount;
if (K > 64) throw new ArgumentException("at most 64 active poses");
lower1 = new float[P]; lower2 = new float[P]; upper1 = new float[P]; upper2 = new float[P]; tgt = new float[P];
prev = new double[K]; b = new double[K]; c = new double[K];
pl1 = new double[K]; pl2 = new double[K]; pu1 = new double[K]; pu2 = new double[K];
}
public void Reset()
{
initialized = false;
Array.Clear(prev, 0, K);
}
/// <summary>
/// One output frame from the projected network output (A2FInference.Output.Projected): 2K values per frame,
/// [delta W_lower | delta W_upper]. The EMA is linear and identical for every row, so filtering in pose space is
/// exactly the SDK's per-vertex filter followed by D^T. weights: 52 ARKit weights (SDK order) written in place.
/// </summary>
public void Solve(float[] projected, int offset, float[] weights)
{
double aL = d.AlphaLower, aU = d.AlphaUpper;
var c0 = SdkNeutralReference ? d.C0Sdk : d.C0Model;
for (int k = 0; k < K; k++)
{
double yl = projected[offset + k], yu = projected[offset + K + k];
if (initialized && aL > 0) { pl1[k] += (yl - pl1[k]) * aL; pl2[k] += (pl1[k] - pl2[k]) * aL; }
else pl1[k] = pl2[k] = yl;
if (initialized && aU > 0) { pu1[k] += (yu - pu1[k]) * aU; pu2[k] += (pu1[k] - pu2[k]) * aU; }
else pu1[k] = pu2[k] = yu;
b[k] = c0[k] + pl2[k] + pu2[k] + d.TemporalB * prev[k];
}
initialized = true;
Finish(weights);
}
/// <summary>
/// Reference path, SDK order of operations (A2FInference.Output.Rows): skinRows are the network's skin deltas at
/// <see cref="A2FIdentityData.Rows"/> (offset into a [frames * P] buffer).
/// </summary>
public void SolveRows(float[] skinRows, int offset, float[] weights)
{
float aL = d.AlphaLower, aU = d.AlphaUpper, s = d.SkinStrength;
var reference = SdkNeutralReference ? d.BsNeutral : d.Neutral;
for (int r = 0; r < P; r++)
{
float delta = s * skinRows[offset + r];
if (initialized && aL > 0) { lower1[r] += (delta - lower1[r]) * aL; lower2[r] += (lower1[r] - lower2[r]) * aL; }
else lower1[r] = lower2[r] = delta;
if (initialized && aU > 0) { upper1[r] += (delta - upper1[r]) * aU; upper2[r] += (upper1[r] - upper2[r]) * aU; }
else upper1[r] = upper2[r] = delta;
float m = d.FaceMask[r];
float abs = d.Neutral[r] + upper2[r] * d.UpperStrength * (1 - m) + lower2[r] * d.LowerStrength * m;
tgt[r] = abs - reference[r];
}
initialized = true;
// b = D^T tgt + T*scale*prev (SDK: blendshape_solver.cpp, no x100 here although A uses 100*T*scale)
var Dt = d.Dt;
for (int k = 0; k < K; k++)
{
double acc = 0; int o = k * P;
for (int r = 0; r < P; r++) acc += Dt[o + r] * tgt[r];
b[k] = acc + d.TemporalB * prev[k];
}
Finish(weights);
}
void Finish(float[] weights)
{
// min ||A x - b||^2, 0 <= x <= 1 == min 1/2 x^T Q x - c^T x with Q = A^T A, c = A^T b
var A = d.A;
for (int i = 0; i < K; i++) { double acc = 0; for (int k = 0; k < K; k++) acc += (double)A[k * K + i] * b[k]; c[i] = acc; }
BoxQp(d.QD, c, prev, K); // warm start from the previous frame
OnSolved?.Invoke(b, prev);
Array.Clear(weights, 0, A2FConstants.ArkitCount);
for (int k = 0; k < K; k++) weights[d.Active[k]] = (float)prev[k];
for (int j = 0; j < A2FConstants.ArkitCount; j++) weights[j] = weights[j] * d.Multipliers[j] + d.Offsets[j];
}
// active-set workspace
readonly int[] m_State = new int[64]; // -1 at lower, +1 at upper, 0 free
readonly double[] m_L = new double[64 * 64], m_Rhs = new double[64], m_Sol = new double[64], m_G = new double[64];
readonly int[] m_Free = new int[64];
/// <summary>
/// min 1/2 x^T Q x - c^T x, 0 &lt;= x &lt;= 1 (== min ||A x - b||^2 with Q = A^T A, c = A^T b): primal active-set
/// method, the same algorithm family as the SDK's BVLS. Exact up to round-off; warm-started from x.
/// </summary>
void BoxQp(double[] Q, double[] c, double[] x, int K)
{
for (int i = 0; i < K; i++)
{
x[i] = x[i] <= 0 ? 0 : x[i] >= 1 ? 1 : x[i];
m_State[i] = x[i] <= 0 ? -1 : x[i] >= 1 ? 1 : 0;
}
for (int iter = 0; iter < 8 * K + 16; iter++)
{
// equality-constrained minimiser on the free set (bound variables held at their bounds)
int nf = 0;
for (int i = 0; i < K; i++) if (m_State[i] == 0) m_Free[nf++] = i;
for (int a = 0; a < nf; a++)
{
int i = m_Free[a];
double r = c[i];
for (int j = 0; j < K; j++) if (m_State[j] != 0) r -= Q[i * K + j] * x[j];
m_Rhs[a] = r;
for (int bb = 0; bb < nf; bb++) m_L[a * nf + bb] = Q[i * K + m_Free[bb]];
}
CholeskySolve(m_L, m_Rhs, m_Sol, nf);
// step toward it, stopping at the first bound hit
double alpha = 1; int hit = -1;
for (int a = 0; a < nf; a++)
{
int i = m_Free[a]; double d = m_Sol[a] - x[i];
if (d < 0 && m_Sol[a] < 0) { double t = x[i] / -d; if (t < alpha) { alpha = t; hit = i; } }
else if (d > 0 && m_Sol[a] > 1) { double t = (1 - x[i]) / d; if (t < alpha) { alpha = t; hit = i; } }
}
for (int a = 0; a < nf; a++) { int i = m_Free[a]; x[i] += alpha * (m_Sol[a] - x[i]); }
if (hit >= 0)
{
m_State[hit] = m_Sol[Array.IndexOf(m_Free, hit, 0, nf)] < 0 ? -1 : 1;
x[hit] = m_State[hit] < 0 ? 0 : 1;
continue;
}
// KKT: release the bound variable whose gradient points most into the box
int release = -1; double worst = 1e-12;
for (int i = 0; i < K; i++)
{
if (m_State[i] == 0) continue;
double g = -c[i];
for (int j = 0; j < K; j++) g += Q[i * K + j] * x[j];
double v = m_State[i] < 0 ? -g : g; // at 0: g<0 wants up; at 1: g>0 wants down
if (v > worst) { worst = v; release = i; }
}
if (release < 0) return;
m_State[release] = 0;
}
}
static void CholeskySolve(double[] M, double[] rhs, double[] sol, int n)
{
// in-place LL^T (M is SPD: Q restricted to the free set)
for (int j = 0; j < n; j++)
{
double s = M[j * n + j];
for (int k = 0; k < j; k++) s -= M[j * n + k] * M[j * n + k];
double d = Math.Sqrt(Math.Max(s, 1e-300));
M[j * n + j] = d;
for (int i = j + 1; i < n; i++)
{
double t = M[i * n + j];
for (int k = 0; k < j; k++) t -= M[i * n + k] * M[j * n + k];
M[i * n + j] = t / d;
}
}
for (int i = 0; i < n; i++) { double t = rhs[i]; for (int k = 0; k < i; k++) t -= M[i * n + k] * sol[k]; sol[i] = t / M[i * n + i]; }
for (int i = n - 1; i >= 0; i--) { double t = sol[i]; for (int k = i + 1; k < n; k++) t -= M[k * n + i] * sol[k]; sol[i] = t / M[i * n + i]; }
}
}
}