using System; using Unity.InferenceEngine; using UnityEngine; namespace Audio2Face { /// /// One network pass per 1 s window. The packaged graph is wrapped (Sentis Functional API) so that /// - the diffusion noise [1,3,60,88831] is drawn on the device (seeded RandomNormal; no 64 MB upload per window), /// - the 30 kept frames are reduced on the device to what the solver needs: either the skin rows of the frontal /// mask ([30, P], ) or their projection onto the active poses ([30, 2K], /// , default: 30 x 78 floats read back instead of 21 MB), /// - the GRU latents stay on the device between windows. /// public sealed class A2FInference : IDisposable { public enum Output { Projected, Rows } static readonly System.Runtime.CompilerServices.ConditionalWeakTable s_Consumed = new System.Runtime.CompilerServices.ConditionalWeakTable(); public const string OutKept = "kept", OutLatents = "latents"; readonly Worker m_Worker; readonly Tensor m_Window, m_Identity, m_Emotion; readonly Tensor m_Latents = new Tensor(new TensorShape(2, 2, 1, 256)); readonly float[] m_LatentValues = new float[2 * 2 * 256]; readonly bool m_ExternalNoise; readonly Tensor m_Noise; public BackendType Backend { get; } public Output Mode { get; } /// Values per kept frame in the read-back buffer (2K projected, P rows). public int ValuesPerFrame { get; } /// Tests only: noise becomes an input (SetNoise) instead of being drawn in the graph. public A2FInference(Model network, A2FIdentityData data, BackendType backend, int seed = 0, Output mode = Output.Projected, bool externalNoise = false) { // Functional.Forward modifies the source model in place (its weights end up shared with this graph), so a // second graph built from the same Model object corrupts the first one's weights. Measured: the first of two // instances sharing a Model intermittently produced wrong or empty output on GPUCompute. if (network == null) throw new ArgumentNullException(nameof(network)); lock (s_Consumed) { if (s_Consumed.TryGetValue(network, out _)) throw new ArgumentException("This Model is already used by another A2FInference (Functional.Forward modifies it). " + "Load a new Model per instance: A2FModel.LoadAsync() or ModelLoader.Load(asset).", nameof(network)); s_Consumed.Add(network, null); } Backend = backend == BackendType.GPUCompute && !SystemInfo.supportsComputeShaders ? BackendType.CPU : backend; Mode = mode; m_ExternalNoise = externalNoise; var graph = new FunctionalGraph(); var window = graph.AddInput(new TensorShape(1, A2FConstants.WindowSamples), "window"); var identity = graph.AddInput(new TensorShape(1, 3), "identity"); var emotion = graph.AddInput(new TensorShape(1, A2FConstants.KeptFrames, A2FConstants.EmotionCount), "emotion"); var latents = graph.AddInput(new TensorShape(2, 2, 1, 256), "input_latents"); var noiseShape = new[] { 1, A2FConstants.DiffusionNoiseSteps, A2FConstants.PredictedFrames, A2FConstants.PredictionSize }; var noise = externalNoise ? graph.AddInput(new TensorShape(noiseShape), "noise") : ChunkedNormal(noiseShape, seed); var outs = Functional.ForwardWithCopy(network, window, identity, emotion, latents, noise); var kept = outs[0].Narrow(1, A2FConstants.FirstKeptFrame, A2FConstants.KeptFrames) // [1,30,88831] .IndexSelect(2, Functional.Constant(data.Rows)); // [1,30,P] if (mode == Output.Projected) { var w = Functional.Constant(new TensorShape(data.RowCount, 2 * data.ActiveCount), data.Projection); kept = Functional.MatMul(kept, w); // [1,30,2K] ValuesPerFrame = 2 * data.ActiveCount; } else ValuesPerFrame = data.RowCount; graph.AddOutput(kept, OutKept); graph.AddOutput(outs[1], OutLatents); m_Worker = new Worker(graph.Compile(), Backend); m_Window = new Tensor(new TensorShape(1, A2FConstants.WindowSamples)); m_Identity = new Tensor(new TensorShape(1, 3)); m_Emotion = new Tensor(new TensorShape(1, A2FConstants.KeptFrames, A2FConstants.EmotionCount)); if (externalNoise) m_Noise = new Tensor(new TensorShape(noiseShape)); SetIdentity(data.Identity); SetEmotion(null); ResetLatents(); // Warm-up: run the graph once now. Measured on Sentis 2.6.1: a worker that has not executed yet returns // all-zero results if another A2FInference graph is compiled before its first run (any backend); once it // has run it is unaffected. The pass also front-loads kernel compilation and allocation. Schedule(new float[A2FConstants.WindowSamples]); using (var warm = (m_Worker.PeekOutput(OutKept) as Tensor).ReadbackAndClone()) { } ResetLatents(); } /// /// N(0,1) noise built from chunks along the last axis. Sentis 2.6.1 GPUCompute dispatches RandomNormal as one /// thread group per 64 elements in a single dimension, so one 16M-element tensor needs 249,838 groups, over the /// 65,535 limit: the dispatch (or the whole command buffer) is dropped. Chunks stay under 65,535 x 64 elements. /// static FunctionalTensor ChunkedNormal(int[] shape, int seed) { const int maxElements = 65535 * 64; int last = shape[shape.Length - 1], outer = 1; for (int i = 0; i < shape.Length - 1; i++) outer *= shape[i]; int chunks = (int)Math.Ceiling((double)outer * last / maxElements); if (chunks <= 1) return Functional.Normal(0, 1, shape, seed); var parts = new FunctionalTensor[chunks]; int start = 0; for (int c = 0; c < chunks; c++) { int width = (last - start) / (chunks - c); var s = (int[])shape.Clone(); s[s.Length - 1] = width; parts[c] = Functional.Normal(0, 1, s, seed * 7919 + c); start += width; } return Functional.Concat(parts, shape.Length - 1); } public void ResetLatents() => SetLatents(new float[2 * 2 * 256]); /// GRU state [2,2,1,256] for the next window (tests; normally carried automatically). public void SetLatents(float[] latents) { Array.Copy(latents, m_LatentValues, m_LatentValues.Length); m_Latents.Upload(m_LatentValues); } public void SetIdentity(A2FIdentity identity) { var v = new float[3]; v[(int)identity] = 1; m_Identity.Upload(v); } /// emotion: [30*10] (per kept frame) or [10] (held for the window); null = neutral. public void SetEmotion(float[] emotion) { var v = new float[A2FConstants.KeptFrames * A2FConstants.EmotionCount]; if (emotion != null && emotion.Length == v.Length) Array.Copy(emotion, v, v.Length); else if (emotion != null && emotion.Length == A2FConstants.EmotionCount) for (int f = 0; f < A2FConstants.KeptFrames; f++) Array.Copy(emotion, 0, v, f * A2FConstants.EmotionCount, A2FConstants.EmotionCount); m_Emotion.Upload(v); } public void SetNoise(float[] noise) { if (!m_ExternalNoise) throw new InvalidOperationException("noise is drawn in the graph"); m_Noise.Upload(noise); } void Schedule(float[] window) { m_Window.Upload(window); m_Worker.SetInput("window", m_Window); m_Worker.SetInput("identity", m_Identity); m_Worker.SetInput("emotion", m_Emotion); m_Worker.SetInput("input_latents", m_Latents); if (m_ExternalNoise) m_Worker.SetInput("noise", m_Noise); m_Worker.Schedule(); } // The GRU state (1,024 floats) is read back and re-uploaded into a tensor this instance owns. Measured on // Sentis 2.6.1: keeping it with Worker.CopyOutput let another A2FInference's execution change this instance's // results (CPU backend), so the state never aliases worker-owned memory. void CarryLatents() { using var cpu = (m_Worker.PeekOutput(OutLatents) as Tensor).ReadbackAndClone(); cpu.DownloadToArray().CopyTo(m_LatentValues, 0); m_Latents.Upload(m_LatentValues); } /// Runs one window (16000 samples): [30 * ValuesPerFrame]. public float[] Run(float[] window) { Schedule(window); using var cpu = (m_Worker.PeekOutput(OutKept) as Tensor).ReadbackAndClone(); var values = cpu.DownloadToArray(); CarryLatents(); return Checked(values); } public async Awaitable RunAsync(float[] window) { Schedule(window); using var cpu = await (m_Worker.PeekOutput(OutKept) as Tensor).ReadbackAndCloneAsync(); var values = cpu.DownloadToArray(); CarryLatents(); // the graph has finished: this readback does not wait return Checked(values); } /// Number of windows whose result came back all-zero (the backend dropped the work). public int FailedWindows { get; private set; } // The diffusion noise makes an exactly-zero result impossible, so all zeros means the backend did not run the // graph (e.g. a GPU command buffer rejected by the driver). Reported, never silently passed on as "no motion". float[] Checked(float[] values) { for (int i = 0; i < values.Length; i++) if (values[i] != 0) return values; FailedWindows++; Debug.LogError($"[Audio2Face] {Backend} returned an all-zero window: the graph did not execute (failure #{FailedWindows})."); return values; } /// Tests: the GRU state after the last run. public float[] ReadLatents() => (float[])m_LatentValues.Clone(); public void Dispose() { m_Worker?.Dispose(); m_Window?.Dispose(); m_Identity?.Dispose(); m_Emotion?.Dispose(); m_Latents?.Dispose(); m_Noise?.Dispose(); } } }