Audio-to-Audio
unity-sentis
ONNX
unity
sentis
inference-engine
on-device
lipsync
facial-animation
arkit
blendshapes
audio2face
Instructions to use Sky-Kim/com.sky.sentis.audio2face with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- unity-sentis
How to use Sky-Kim/com.sky.sentis.audio2face with unity-sentis:
string modelName = "[Your model name here].sentis"; Model model = ModelLoader.Load(Application.streamingAssetsPath + "/" + modelName); IWorker engine = WorkerFactory.CreateWorker(BackendType.GPUCompute, model); // Please see provided C# file for more details
- Notebooks
- Google Colab
- Kaggle
Download Runtime/A2FInference.cs from Sky-Kim/com.sky.sentis.audio2face: direct link, hf CLI and curl.
- Browser
- Download file 11.2 kB
-
https://huggingface.co/Sky-Kim/com.sky.sentis.audio2face/resolve/main/Runtime/A2FInference.cs
- Command line
-
hf download hf://Sky-Kim/com.sky.sentis.audio2face/Runtime/A2FInference.cs
-
curl -L -o A2FInference.cs https://huggingface.co/Sky-Kim/com.sky.sentis.audio2face/resolve/main/Runtime/A2FInference.cs
11.2 kB
| using System; | |
| using Unity.InferenceEngine; | |
| using UnityEngine; | |
| namespace Audio2Face | |
| { | |
| /// <summary> | |
| /// One network pass per 1 s window. The packaged graph is wrapped (Sentis Functional API) so that | |
| /// - the diffusion noise [1,3,60,88831] is drawn on the device (seeded RandomNormal; no 64 MB upload per window), | |
| /// - the 30 kept frames are reduced on the device to what the solver needs: either the skin rows of the frontal | |
| /// mask ([30, P], <see cref="Output.Rows"/>) or their projection onto the active poses ([30, 2K], | |
| /// <see cref="Output.Projected"/>, default: 30 x 78 floats read back instead of 21 MB), | |
| /// - the GRU latents stay on the device between windows. | |
| /// </summary> | |
| public sealed class A2FInference : IDisposable | |
| { | |
| public enum Output { Projected, Rows } | |
| static readonly System.Runtime.CompilerServices.ConditionalWeakTable<Model, object> s_Consumed = | |
| new System.Runtime.CompilerServices.ConditionalWeakTable<Model, object>(); | |
| public const string OutKept = "kept", OutLatents = "latents"; | |
| readonly Worker m_Worker; | |
| readonly Tensor<float> m_Window, m_Identity, m_Emotion; | |
| readonly Tensor<float> m_Latents = new Tensor<float>(new TensorShape(2, 2, 1, 256)); | |
| readonly float[] m_LatentValues = new float[2 * 2 * 256]; | |
| readonly bool m_ExternalNoise; | |
| readonly Tensor<float> m_Noise; | |
| public BackendType Backend { get; } | |
| public Output Mode { get; } | |
| /// <summary>Values per kept frame in the read-back buffer (2K projected, P rows).</summary> | |
| public int ValuesPerFrame { get; } | |
| /// <param name="externalNoise">Tests only: noise becomes an input (SetNoise) instead of being drawn in the graph.</param> | |
| public A2FInference(Model network, A2FIdentityData data, BackendType backend, int seed = 0, | |
| Output mode = Output.Projected, bool externalNoise = false) | |
| { | |
| // Functional.Forward modifies the source model in place (its weights end up shared with this graph), so a | |
| // second graph built from the same Model object corrupts the first one's weights. Measured: the first of two | |
| // instances sharing a Model intermittently produced wrong or empty output on GPUCompute. | |
| if (network == null) throw new ArgumentNullException(nameof(network)); | |
| lock (s_Consumed) | |
| { | |
| if (s_Consumed.TryGetValue(network, out _)) | |
| throw new ArgumentException("This Model is already used by another A2FInference (Functional.Forward modifies it). " + | |
| "Load a new Model per instance: A2FModel.LoadAsync() or ModelLoader.Load(asset).", nameof(network)); | |
| s_Consumed.Add(network, null); | |
| } | |
| Backend = backend == BackendType.GPUCompute && !SystemInfo.supportsComputeShaders ? BackendType.CPU : backend; | |
| Mode = mode; | |
| m_ExternalNoise = externalNoise; | |
| var graph = new FunctionalGraph(); | |
| var window = graph.AddInput<float>(new TensorShape(1, A2FConstants.WindowSamples), "window"); | |
| var identity = graph.AddInput<float>(new TensorShape(1, 3), "identity"); | |
| var emotion = graph.AddInput<float>(new TensorShape(1, A2FConstants.KeptFrames, A2FConstants.EmotionCount), "emotion"); | |
| var latents = graph.AddInput<float>(new TensorShape(2, 2, 1, 256), "input_latents"); | |
| var noiseShape = new[] { 1, A2FConstants.DiffusionNoiseSteps, A2FConstants.PredictedFrames, A2FConstants.PredictionSize }; | |
| var noise = externalNoise | |
| ? graph.AddInput<float>(new TensorShape(noiseShape), "noise") | |
| : ChunkedNormal(noiseShape, seed); | |
| var outs = Functional.ForwardWithCopy(network, window, identity, emotion, latents, noise); | |
| var kept = outs[0].Narrow(1, A2FConstants.FirstKeptFrame, A2FConstants.KeptFrames) // [1,30,88831] | |
| .IndexSelect(2, Functional.Constant(data.Rows)); // [1,30,P] | |
| if (mode == Output.Projected) | |
| { | |
| var w = Functional.Constant(new TensorShape(data.RowCount, 2 * data.ActiveCount), data.Projection); | |
| kept = Functional.MatMul(kept, w); // [1,30,2K] | |
| ValuesPerFrame = 2 * data.ActiveCount; | |
| } | |
| else ValuesPerFrame = data.RowCount; | |
| graph.AddOutput(kept, OutKept); | |
| graph.AddOutput(outs[1], OutLatents); | |
| m_Worker = new Worker(graph.Compile(), Backend); | |
| m_Window = new Tensor<float>(new TensorShape(1, A2FConstants.WindowSamples)); | |
| m_Identity = new Tensor<float>(new TensorShape(1, 3)); | |
| m_Emotion = new Tensor<float>(new TensorShape(1, A2FConstants.KeptFrames, A2FConstants.EmotionCount)); | |
| if (externalNoise) m_Noise = new Tensor<float>(new TensorShape(noiseShape)); | |
| SetIdentity(data.Identity); | |
| SetEmotion(null); | |
| ResetLatents(); | |
| // Warm-up: run the graph once now. Measured on Sentis 2.6.1: a worker that has not executed yet returns | |
| // all-zero results if another A2FInference graph is compiled before its first run (any backend); once it | |
| // has run it is unaffected. The pass also front-loads kernel compilation and allocation. | |
| Schedule(new float[A2FConstants.WindowSamples]); | |
| using (var warm = (m_Worker.PeekOutput(OutKept) as Tensor<float>).ReadbackAndClone()) { } | |
| ResetLatents(); | |
| } | |
| /// <summary> | |
| /// N(0,1) noise built from chunks along the last axis. Sentis 2.6.1 GPUCompute dispatches RandomNormal as one | |
| /// thread group per 64 elements in a single dimension, so one 16M-element tensor needs 249,838 groups, over the | |
| /// 65,535 limit: the dispatch (or the whole command buffer) is dropped. Chunks stay under 65,535 x 64 elements. | |
| /// </summary> | |
| static FunctionalTensor ChunkedNormal(int[] shape, int seed) | |
| { | |
| const int maxElements = 65535 * 64; | |
| int last = shape[shape.Length - 1], outer = 1; | |
| for (int i = 0; i < shape.Length - 1; i++) outer *= shape[i]; | |
| int chunks = (int)Math.Ceiling((double)outer * last / maxElements); | |
| if (chunks <= 1) return Functional.Normal(0, 1, shape, seed); | |
| var parts = new FunctionalTensor[chunks]; | |
| int start = 0; | |
| for (int c = 0; c < chunks; c++) | |
| { | |
| int width = (last - start) / (chunks - c); | |
| var s = (int[])shape.Clone(); s[s.Length - 1] = width; | |
| parts[c] = Functional.Normal(0, 1, s, seed * 7919 + c); | |
| start += width; | |
| } | |
| return Functional.Concat(parts, shape.Length - 1); | |
| } | |
| public void ResetLatents() => SetLatents(new float[2 * 2 * 256]); | |
| /// <summary>GRU state [2,2,1,256] for the next window (tests; normally carried automatically).</summary> | |
| public void SetLatents(float[] latents) | |
| { | |
| Array.Copy(latents, m_LatentValues, m_LatentValues.Length); | |
| m_Latents.Upload(m_LatentValues); | |
| } | |
| public void SetIdentity(A2FIdentity identity) | |
| { | |
| var v = new float[3]; v[(int)identity] = 1; | |
| m_Identity.Upload(v); | |
| } | |
| /// <summary>emotion: [30*10] (per kept frame) or [10] (held for the window); null = neutral.</summary> | |
| public void SetEmotion(float[] emotion) | |
| { | |
| var v = new float[A2FConstants.KeptFrames * A2FConstants.EmotionCount]; | |
| if (emotion != null && emotion.Length == v.Length) Array.Copy(emotion, v, v.Length); | |
| else if (emotion != null && emotion.Length == A2FConstants.EmotionCount) | |
| for (int f = 0; f < A2FConstants.KeptFrames; f++) Array.Copy(emotion, 0, v, f * A2FConstants.EmotionCount, A2FConstants.EmotionCount); | |
| m_Emotion.Upload(v); | |
| } | |
| public void SetNoise(float[] noise) | |
| { | |
| if (!m_ExternalNoise) throw new InvalidOperationException("noise is drawn in the graph"); | |
| m_Noise.Upload(noise); | |
| } | |
| void Schedule(float[] window) | |
| { | |
| m_Window.Upload(window); | |
| m_Worker.SetInput("window", m_Window); | |
| m_Worker.SetInput("identity", m_Identity); | |
| m_Worker.SetInput("emotion", m_Emotion); | |
| m_Worker.SetInput("input_latents", m_Latents); | |
| if (m_ExternalNoise) m_Worker.SetInput("noise", m_Noise); | |
| m_Worker.Schedule(); | |
| } | |
| // The GRU state (1,024 floats) is read back and re-uploaded into a tensor this instance owns. Measured on | |
| // Sentis 2.6.1: keeping it with Worker.CopyOutput let another A2FInference's execution change this instance's | |
| // results (CPU backend), so the state never aliases worker-owned memory. | |
| void CarryLatents() | |
| { | |
| using var cpu = (m_Worker.PeekOutput(OutLatents) as Tensor<float>).ReadbackAndClone(); | |
| cpu.DownloadToArray().CopyTo(m_LatentValues, 0); | |
| m_Latents.Upload(m_LatentValues); | |
| } | |
| /// <summary>Runs one window (16000 samples): [30 * ValuesPerFrame].</summary> | |
| public float[] Run(float[] window) | |
| { | |
| Schedule(window); | |
| using var cpu = (m_Worker.PeekOutput(OutKept) as Tensor<float>).ReadbackAndClone(); | |
| var values = cpu.DownloadToArray(); | |
| CarryLatents(); | |
| return Checked(values); | |
| } | |
| public async Awaitable<float[]> RunAsync(float[] window) | |
| { | |
| Schedule(window); | |
| using var cpu = await (m_Worker.PeekOutput(OutKept) as Tensor<float>).ReadbackAndCloneAsync(); | |
| var values = cpu.DownloadToArray(); | |
| CarryLatents(); // the graph has finished: this readback does not wait | |
| return Checked(values); | |
| } | |
| /// <summary>Number of windows whose result came back all-zero (the backend dropped the work).</summary> | |
| public int FailedWindows { get; private set; } | |
| // The diffusion noise makes an exactly-zero result impossible, so all zeros means the backend did not run the | |
| // graph (e.g. a GPU command buffer rejected by the driver). Reported, never silently passed on as "no motion". | |
| float[] Checked(float[] values) | |
| { | |
| for (int i = 0; i < values.Length; i++) if (values[i] != 0) return values; | |
| FailedWindows++; | |
| Debug.LogError($"[Audio2Face] {Backend} returned an all-zero window: the graph did not execute (failure #{FailedWindows})."); | |
| return values; | |
| } | |
| /// <summary>Tests: the GRU state after the last run.</summary> | |
| public float[] ReadLatents() => (float[])m_LatentValues.Clone(); | |
| public void Dispose() | |
| { | |
| m_Worker?.Dispose(); | |
| m_Window?.Dispose(); m_Identity?.Dispose(); m_Emotion?.Dispose(); | |
| m_Latents?.Dispose(); m_Noise?.Dispose(); | |
| } | |
| } | |
| } | |