using System;
using Unity.InferenceEngine;
using UnityEngine;
namespace Audio2Face
{
///
/// One network pass per 1 s window. The packaged graph is wrapped (Sentis Functional API) so that
/// - the diffusion noise [1,3,60,88831] is drawn on the device (seeded RandomNormal; no 64 MB upload per window),
/// - the 30 kept frames are reduced on the device to what the solver needs: either the skin rows of the frontal
/// mask ([30, P], ) or their projection onto the active poses ([30, 2K],
/// , default: 30 x 78 floats read back instead of 21 MB),
/// - the GRU latents stay on the device between windows.
///
public sealed class A2FInference : IDisposable
{
public enum Output { Projected, Rows }
static readonly System.Runtime.CompilerServices.ConditionalWeakTable s_Consumed =
new System.Runtime.CompilerServices.ConditionalWeakTable();
public const string OutKept = "kept", OutLatents = "latents";
readonly Worker m_Worker;
readonly Tensor m_Window, m_Identity, m_Emotion;
readonly Tensor m_Latents = new Tensor(new TensorShape(2, 2, 1, 256));
readonly float[] m_LatentValues = new float[2 * 2 * 256];
readonly bool m_ExternalNoise;
readonly Tensor m_Noise;
public BackendType Backend { get; }
public Output Mode { get; }
/// Values per kept frame in the read-back buffer (2K projected, P rows).
public int ValuesPerFrame { get; }
/// Tests only: noise becomes an input (SetNoise) instead of being drawn in the graph.
public A2FInference(Model network, A2FIdentityData data, BackendType backend, int seed = 0,
Output mode = Output.Projected, bool externalNoise = false)
{
// Functional.Forward modifies the source model in place (its weights end up shared with this graph), so a
// second graph built from the same Model object corrupts the first one's weights. Measured: the first of two
// instances sharing a Model intermittently produced wrong or empty output on GPUCompute.
if (network == null) throw new ArgumentNullException(nameof(network));
lock (s_Consumed)
{
if (s_Consumed.TryGetValue(network, out _))
throw new ArgumentException("This Model is already used by another A2FInference (Functional.Forward modifies it). " +
"Load a new Model per instance: A2FModel.LoadAsync() or ModelLoader.Load(asset).", nameof(network));
s_Consumed.Add(network, null);
}
Backend = backend == BackendType.GPUCompute && !SystemInfo.supportsComputeShaders ? BackendType.CPU : backend;
Mode = mode;
m_ExternalNoise = externalNoise;
var graph = new FunctionalGraph();
var window = graph.AddInput(new TensorShape(1, A2FConstants.WindowSamples), "window");
var identity = graph.AddInput(new TensorShape(1, 3), "identity");
var emotion = graph.AddInput(new TensorShape(1, A2FConstants.KeptFrames, A2FConstants.EmotionCount), "emotion");
var latents = graph.AddInput(new TensorShape(2, 2, 1, 256), "input_latents");
var noiseShape = new[] { 1, A2FConstants.DiffusionNoiseSteps, A2FConstants.PredictedFrames, A2FConstants.PredictionSize };
var noise = externalNoise
? graph.AddInput(new TensorShape(noiseShape), "noise")
: ChunkedNormal(noiseShape, seed);
var outs = Functional.ForwardWithCopy(network, window, identity, emotion, latents, noise);
var kept = outs[0].Narrow(1, A2FConstants.FirstKeptFrame, A2FConstants.KeptFrames) // [1,30,88831]
.IndexSelect(2, Functional.Constant(data.Rows)); // [1,30,P]
if (mode == Output.Projected)
{
var w = Functional.Constant(new TensorShape(data.RowCount, 2 * data.ActiveCount), data.Projection);
kept = Functional.MatMul(kept, w); // [1,30,2K]
ValuesPerFrame = 2 * data.ActiveCount;
}
else ValuesPerFrame = data.RowCount;
graph.AddOutput(kept, OutKept);
graph.AddOutput(outs[1], OutLatents);
m_Worker = new Worker(graph.Compile(), Backend);
m_Window = new Tensor(new TensorShape(1, A2FConstants.WindowSamples));
m_Identity = new Tensor(new TensorShape(1, 3));
m_Emotion = new Tensor(new TensorShape(1, A2FConstants.KeptFrames, A2FConstants.EmotionCount));
if (externalNoise) m_Noise = new Tensor(new TensorShape(noiseShape));
SetIdentity(data.Identity);
SetEmotion(null);
ResetLatents();
// Warm-up: run the graph once now. Measured on Sentis 2.6.1: a worker that has not executed yet returns
// all-zero results if another A2FInference graph is compiled before its first run (any backend); once it
// has run it is unaffected. The pass also front-loads kernel compilation and allocation.
Schedule(new float[A2FConstants.WindowSamples]);
using (var warm = (m_Worker.PeekOutput(OutKept) as Tensor).ReadbackAndClone()) { }
ResetLatents();
}
///
/// N(0,1) noise built from chunks along the last axis. Sentis 2.6.1 GPUCompute dispatches RandomNormal as one
/// thread group per 64 elements in a single dimension, so one 16M-element tensor needs 249,838 groups, over the
/// 65,535 limit: the dispatch (or the whole command buffer) is dropped. Chunks stay under 65,535 x 64 elements.
///
static FunctionalTensor ChunkedNormal(int[] shape, int seed)
{
const int maxElements = 65535 * 64;
int last = shape[shape.Length - 1], outer = 1;
for (int i = 0; i < shape.Length - 1; i++) outer *= shape[i];
int chunks = (int)Math.Ceiling((double)outer * last / maxElements);
if (chunks <= 1) return Functional.Normal(0, 1, shape, seed);
var parts = new FunctionalTensor[chunks];
int start = 0;
for (int c = 0; c < chunks; c++)
{
int width = (last - start) / (chunks - c);
var s = (int[])shape.Clone(); s[s.Length - 1] = width;
parts[c] = Functional.Normal(0, 1, s, seed * 7919 + c);
start += width;
}
return Functional.Concat(parts, shape.Length - 1);
}
public void ResetLatents() => SetLatents(new float[2 * 2 * 256]);
/// GRU state [2,2,1,256] for the next window (tests; normally carried automatically).
public void SetLatents(float[] latents)
{
Array.Copy(latents, m_LatentValues, m_LatentValues.Length);
m_Latents.Upload(m_LatentValues);
}
public void SetIdentity(A2FIdentity identity)
{
var v = new float[3]; v[(int)identity] = 1;
m_Identity.Upload(v);
}
/// emotion: [30*10] (per kept frame) or [10] (held for the window); null = neutral.
public void SetEmotion(float[] emotion)
{
var v = new float[A2FConstants.KeptFrames * A2FConstants.EmotionCount];
if (emotion != null && emotion.Length == v.Length) Array.Copy(emotion, v, v.Length);
else if (emotion != null && emotion.Length == A2FConstants.EmotionCount)
for (int f = 0; f < A2FConstants.KeptFrames; f++) Array.Copy(emotion, 0, v, f * A2FConstants.EmotionCount, A2FConstants.EmotionCount);
m_Emotion.Upload(v);
}
public void SetNoise(float[] noise)
{
if (!m_ExternalNoise) throw new InvalidOperationException("noise is drawn in the graph");
m_Noise.Upload(noise);
}
void Schedule(float[] window)
{
m_Window.Upload(window);
m_Worker.SetInput("window", m_Window);
m_Worker.SetInput("identity", m_Identity);
m_Worker.SetInput("emotion", m_Emotion);
m_Worker.SetInput("input_latents", m_Latents);
if (m_ExternalNoise) m_Worker.SetInput("noise", m_Noise);
m_Worker.Schedule();
}
// The GRU state (1,024 floats) is read back and re-uploaded into a tensor this instance owns. Measured on
// Sentis 2.6.1: keeping it with Worker.CopyOutput let another A2FInference's execution change this instance's
// results (CPU backend), so the state never aliases worker-owned memory.
void CarryLatents()
{
using var cpu = (m_Worker.PeekOutput(OutLatents) as Tensor).ReadbackAndClone();
cpu.DownloadToArray().CopyTo(m_LatentValues, 0);
m_Latents.Upload(m_LatentValues);
}
/// Runs one window (16000 samples): [30 * ValuesPerFrame].
public float[] Run(float[] window)
{
Schedule(window);
using var cpu = (m_Worker.PeekOutput(OutKept) as Tensor).ReadbackAndClone();
var values = cpu.DownloadToArray();
CarryLatents();
return Checked(values);
}
public async Awaitable RunAsync(float[] window)
{
Schedule(window);
using var cpu = await (m_Worker.PeekOutput(OutKept) as Tensor).ReadbackAndCloneAsync();
var values = cpu.DownloadToArray();
CarryLatents(); // the graph has finished: this readback does not wait
return Checked(values);
}
/// Number of windows whose result came back all-zero (the backend dropped the work).
public int FailedWindows { get; private set; }
// The diffusion noise makes an exactly-zero result impossible, so all zeros means the backend did not run the
// graph (e.g. a GPU command buffer rejected by the driver). Reported, never silently passed on as "no motion".
float[] Checked(float[] values)
{
for (int i = 0; i < values.Length; i++) if (values[i] != 0) return values;
FailedWindows++;
Debug.LogError($"[Audio2Face] {Backend} returned an all-zero window: the graph did not execute (failure #{FailedWindows}).");
return values;
}
/// Tests: the GRU state after the last run.
public float[] ReadLatents() => (float[])m_LatentValues.Clone();
public void Dispose()
{
m_Worker?.Dispose();
m_Window?.Dispose(); m_Identity?.Dispose(); m_Emotion?.Dispose();
m_Latents?.Dispose(); m_Noise?.Dispose();
}
}
}