Maverick XR Agent 1.0.0-rc.2 (Unity package)
Browse files- Editor/SceneAgentMenu.cs +2 -2
- Runtime/LocalModelServer.cs +1 -1
- Runtime/MiniJson.cs +3 -3
- Runtime/Resources/SceneAgent/system_prompt.txt +16 -0
- Runtime/Resources/SceneAgent/system_prompt.txt.meta +7 -0
- Runtime/SceneAgent.cs +25 -25
- Runtime/SceneAgentScriptRunner.cs +2 -2
- Runtime/SceneAgentSession.cs +1 -1
- Runtime/SceneCatalog.Generated.cs +2 -2
- Runtime/SceneGrammar.Check.cs +4 -5
- Runtime/SceneGrammar.Schema.cs +2 -2
- Runtime/SceneGrammar.Static.cs +1 -1
- Runtime/SceneGrammar.cs +1 -1
- Runtime/SceneMath.cs +35 -0
- Runtime/SceneMath.cs.meta +11 -0
- Runtime/SceneWorld.cs +19 -20
- package.json +25 -26
Editor/SceneAgentMenu.cs
CHANGED
|
@@ -120,8 +120,8 @@ namespace SceneAgent.EditorTools
|
|
| 120 |
var agent = world.GetComponent<SceneAgent>() != null ? world.GetComponent<SceneAgent>() : Object.FindObjectOfType<SceneAgent>();
|
| 121 |
if (agent == null) E("no SceneAgent component");
|
| 122 |
else if (agent.world == null) E("SceneAgent.world is not set");
|
| 123 |
-
if (Resources.Load<TextAsset>("SceneAgent/
|
| 124 |
-
E("the system prompt is missing (package Resources/SceneAgent/
|
| 125 |
var server = agent != null && agent.server != null ? agent.server : Object.FindObjectOfType<LocalModelServer>();
|
| 126 |
if (server == null) W("no LocalModelServer: the agent will call " + (agent != null ? agent.endpoint : "?") + " (start a server yourself)");
|
| 127 |
else sb.AppendLine("model " + ModelFilesStatus(server));
|
|
|
|
| 120 |
var agent = world.GetComponent<SceneAgent>() != null ? world.GetComponent<SceneAgent>() : Object.FindObjectOfType<SceneAgent>();
|
| 121 |
if (agent == null) E("no SceneAgent component");
|
| 122 |
else if (agent.world == null) E("SceneAgent.world is not set");
|
| 123 |
+
if (Resources.Load<TextAsset>("SceneAgent/system_prompt") == null && (agent == null || agent.systemPromptAsset == null))
|
| 124 |
+
E("the system prompt is missing (package Resources/SceneAgent/system_prompt.txt)");
|
| 125 |
var server = agent != null && agent.server != null ? agent.server : Object.FindObjectOfType<LocalModelServer>();
|
| 126 |
if (server == null) W("no LocalModelServer: the agent will call " + (agent != null ? agent.endpoint : "?") + " (start a server yourself)");
|
| 127 |
else sb.AppendLine("model " + ModelFilesStatus(server));
|
Runtime/LocalModelServer.cs
CHANGED
|
@@ -36,7 +36,7 @@ namespace SceneAgent
|
|
| 36 |
public string extraArgs = "--cache-ram 0";
|
| 37 |
public float startTimeoutSeconds = 120f;
|
| 38 |
[Tooltip("While the model runs on the GPU, cap rendering at this frame rate so the renderer does not starve it " +
|
| 39 |
-
"(uncapped rendering
|
| 40 |
"use 0 in XR, where the headset sets the frame rate.")]
|
| 41 |
public int capFrameRate = 30;
|
| 42 |
[Tooltip("Start the server when this component starts. Off: call Launch() yourself.")]
|
|
|
|
| 36 |
public string extraArgs = "--cache-ram 0";
|
| 37 |
public float startTimeoutSeconds = 120f;
|
| 38 |
[Tooltip("While the model runs on the GPU, cap rendering at this frame rate so the renderer does not starve it " +
|
| 39 |
+
"(uncapped rendering can make the model about 5x slower). 0 = leave the frame rate alone; " +
|
| 40 |
"use 0 in XR, where the headset sets the frame rate.")]
|
| 41 |
public int capFrameRate = 30;
|
| 42 |
[Tooltip("Start the server when this component starts. Off: call Launch() yourself.")]
|
Runtime/MiniJson.cs
CHANGED
|
@@ -17,8 +17,8 @@ namespace SceneAgent
|
|
| 17 |
return v;
|
| 18 |
}
|
| 19 |
|
| 20 |
-
/// <summary>Like Parse, but only whitespace may follow the value
|
| 21 |
-
///
|
| 22 |
public static object ParseStrict(string s)
|
| 23 |
{
|
| 24 |
int i = 0;
|
|
@@ -126,7 +126,7 @@ namespace SceneAgent
|
|
| 126 |
}
|
| 127 |
}
|
| 128 |
|
| 129 |
-
//
|
| 130 |
public static string FormatNumber(double d)
|
| 131 |
{
|
| 132 |
string s = d.ToString("R", CultureInfo.InvariantCulture);
|
|
|
|
| 17 |
return v;
|
| 18 |
}
|
| 19 |
|
| 20 |
+
/// <summary>Like Parse, but only whitespace may follow the value, so the output check judges a tool call
|
| 21 |
+
/// exactly as in training.</summary>
|
| 22 |
public static object ParseStrict(string s)
|
| 23 |
{
|
| 24 |
int i = 0;
|
|
|
|
| 126 |
}
|
| 127 |
}
|
| 128 |
|
| 129 |
+
// integral floats print as "2.0", the same shape as the training data
|
| 130 |
public static string FormatNumber(double d)
|
| 131 |
{
|
| 132 |
string s = d.ToString("R", CultureInfo.InvariantCulture);
|
Runtime/Resources/SceneAgent/system_prompt.txt
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
You are an assistant embedded in a 3D Unity scene. Turn the user's request into tool calls that use object ids exactly as listed in the scene; never invent an id or a tool. Only objects in the user's current room are listed. Match objects by meaning: a synonym or a more general word can name a listed object, and details the scene does not record (a colour when no colours are listed, or words like sliced, wooden, empty) do not rule an object out. If exactly one listed object fits, act on it. If several fit equally well, call ask_clarification with their ids. If none fits, use find_objects to look in the other rooms. place moves an object directly, so there is no need to grab it first. If a request is impossible or needs a tool you do not have, call no tool and briefly explain why. Answer questions about the scene in plain text.
|
| 2 |
+
|
| 3 |
+
# Tools
|
| 4 |
+
grab(object_id:str, hand:left|right|both)
|
| 5 |
+
release(hand:left|right|both)
|
| 6 |
+
place(object_id:str, target_id:str, relation?:on|in, orientation?:default|upright|horizontal|vertical|upside_down)
|
| 7 |
+
move_by(object_id:str, direction:left|right|forward|backward|up|down, distance:num, unit:m|cm)
|
| 8 |
+
rotate(object_id:str, degrees:num, axis?:yaw|pitch|roll)
|
| 9 |
+
set_state(object_id:str, state:on|off|open|closed|half_open|locked|unlocked)
|
| 10 |
+
press(object_id:str, hold_seconds?:num)
|
| 11 |
+
highlight(object_ids:list)
|
| 12 |
+
teleport(location_id:str)
|
| 13 |
+
find_objects(label:str, color?:str, room?:str)
|
| 14 |
+
ask_clarification(question:str, candidate_ids:list)
|
| 15 |
+
|
| 16 |
+
Reply with one <tool_call>{"name": ..., "arguments": {...}}</tool_call> block per call, or plain text.
|
Runtime/Resources/SceneAgent/system_prompt.txt.meta
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fileFormatVersion: 2
|
| 2 |
+
guid: 6a38ee93718310f469efc52b22f6a1b9
|
| 3 |
+
TextScriptImporter:
|
| 4 |
+
externalObjects: {}
|
| 5 |
+
userData:
|
| 6 |
+
assetBundleName:
|
| 7 |
+
assetBundleVariant:
|
Runtime/SceneAgent.cs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
-
// Talks to a local OpenAI-compatible server (llama.cpp llama-server with the
|
| 2 |
-
//
|
| 3 |
// and feeds tool results back until the model answers in text, asks the user a clarifying question
|
| 4 |
// (ask_clarification hands the turn back: the answer arrives as the next request), or runs out of turns.
|
| 5 |
//
|
|
@@ -26,8 +26,9 @@ namespace SceneAgent
|
|
| 26 |
public string modelOutput;
|
| 27 |
public List<string> calls = new List<string>();
|
| 28 |
public List<string> results = new List<string>();
|
| 29 |
-
/// <summary>
|
| 30 |
-
|
|
|
|
| 31 |
}
|
| 32 |
|
| 33 |
public class AgentResult
|
|
@@ -44,38 +45,38 @@ namespace SceneAgent
|
|
| 44 |
{
|
| 45 |
public string endpoint = "http://127.0.0.1:8081/v1/chat/completions";
|
| 46 |
public string model = "unity-scene-agent";
|
| 47 |
-
[Tooltip("The model's system prompt.
|
| 48 |
-
"
|
| 49 |
public TextAsset systemPromptAsset;
|
| 50 |
[Tooltip("Optional: the LocalModelServer that runs the model. Run() then waits until it is ready and uses its endpoint.")]
|
| 51 |
public LocalModelServer server;
|
| 52 |
[NonSerialized] public string systemPrompt;
|
| 53 |
public int maxTurns = 4;
|
| 54 |
public SceneWorld world;
|
| 55 |
-
[Tooltip("
|
| 56 |
-
"
|
| 57 |
public bool constrainedDecoding = false;
|
| 58 |
-
[Tooltip("
|
| 59 |
-
"
|
| 60 |
-
"
|
| 61 |
public bool fallbackDecoding = true;
|
| 62 |
[Tooltip("Safety: when the request names an object that is only in another room and the model acts on a different " +
|
| 63 |
-
"object here instead, that call is not executed; the model is told where the named object is.
|
| 64 |
-
"
|
| 65 |
public bool guardSubstitutes = true;
|
| 66 |
|
| 67 |
static readonly HttpClient Http = new HttpClient { Timeout = TimeSpan.FromSeconds(120) };
|
| 68 |
static readonly Regex ToolCallRe = new Regex(@"<tool_call>\s*(.*?)\s*</tool_call>", RegexOptions.Singleline);
|
| 69 |
static readonly Regex ThinkRe = new Regex(@"<think>.*?</think>\s*", RegexOptions.Singleline);
|
| 70 |
-
//
|
| 71 |
static readonly HashSet<string> Handoff = new HashSet<string> { "ask_clarification" };
|
| 72 |
|
| 73 |
static string defaultPrompt;
|
| 74 |
-
static string DefaultPrompt => defaultPrompt ??= Resources.Load<TextAsset>("SceneAgent/
|
| 75 |
// no ?. on systemPromptAsset: in the Editor an unassigned field is a "fake null" Unity object, and ?. would call
|
| 76 |
-
// .text on it (UnassignedReferenceException on the first command in Play mode
|
| 77 |
string Prompt => systemPrompt ?? (systemPromptAsset != null ? systemPromptAsset.text.Replace("\r\n", "\n") : null) ?? DefaultPrompt
|
| 78 |
-
?? throw new InvalidOperationException("no system prompt (the package's Resources/SceneAgent/
|
| 79 |
|
| 80 |
public IEnumerator Run(string utterance, Action<AgentResult> done)
|
| 81 |
{
|
|
@@ -107,12 +108,12 @@ namespace SceneAgent
|
|
| 107 |
var task = Task.Run(() => Post(body)); // worker thread: no Unity API inside
|
| 108 |
while (!task.IsCompleted) yield return null;
|
| 109 |
if (task.IsFaulted) { res.error = task.Exception?.GetBaseException().Message; break; }
|
| 110 |
-
string output = task.Result,
|
| 111 |
if (!constrainedDecoding && fallbackDecoding)
|
| 112 |
{
|
| 113 |
-
|
| 114 |
-
if (
|
| 115 |
-
else if (
|
| 116 |
{
|
| 117 |
req["grammar"] = SceneGrammar.Build(sceneJson, toolResults);
|
| 118 |
string again = MiniJson.Serialize(req);
|
|
@@ -122,7 +123,7 @@ namespace SceneAgent
|
|
| 122 |
output = retry.Result;
|
| 123 |
}
|
| 124 |
}
|
| 125 |
-
var at = new AgentTurn { modelOutput = output,
|
| 126 |
res.turns.Add(at);
|
| 127 |
var blocks = ToolCallRe.Matches(output).Cast<Match>().Select(m => m.Groups[1].Value).ToList();
|
| 128 |
if (blocks.Count == 0)
|
|
@@ -188,8 +189,7 @@ namespace SceneAgent
|
|
| 188 |
var item = id == null ? null : world.Items.FirstOrDefault(x => x.itemId == id);
|
| 189 |
if (item == null) return null;
|
| 190 |
if (string.IsNullOrEmpty(item.label)) return null;
|
| 191 |
-
//
|
| 192 |
-
// made on the sealed and dev sets (13,438 calls): 4 firings, all on wrong actions, no false alarm
|
| 193 |
string text = " " + Regex.Replace(utterance.ToLowerInvariant(), "[^a-z0-9]+", " ") + " ";
|
| 194 |
var words = text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
|
| 195 |
string joined = string.Concat(words);
|
|
@@ -235,7 +235,7 @@ namespace SceneAgent
|
|
| 235 |
return Convert.ToString(msg.TryGetValue("content", out var c) ? c : "") ?? "";
|
| 236 |
}
|
| 237 |
|
| 238 |
-
// Tool results in
|
| 239 |
public static string PyJson(object v)
|
| 240 |
{
|
| 241 |
switch (v)
|
|
|
|
| 1 |
+
// Talks to a local OpenAI-compatible server (llama.cpp llama-server with the Maverick GGUF, Ollama, ...),
|
| 2 |
+
// executes the returned tool calls on the SceneWorld
|
| 3 |
// and feeds tool results back until the model answers in text, asks the user a clarifying question
|
| 4 |
// (ask_clarification hands the turn back: the answer arrives as the next request), or runs out of turns.
|
| 5 |
//
|
|
|
|
| 26 |
public string modelOutput;
|
| 27 |
public List<string> calls = new List<string>();
|
| 28 |
public List<string> results = new List<string>();
|
| 29 |
+
/// <summary>The output check's verdict on this answer (SceneGrammar.Check): ok, invalid_tool, invalid_value,
|
| 30 |
+
/// invalid_id or malformed. Null when the check is off.</summary>
|
| 31 |
+
public string check;
|
| 32 |
}
|
| 33 |
|
| 34 |
public class AgentResult
|
|
|
|
| 45 |
{
|
| 46 |
public string endpoint = "http://127.0.0.1:8081/v1/chat/completions";
|
| 47 |
public string model = "unity-scene-agent";
|
| 48 |
+
[Tooltip("The model's system prompt. Leave empty to use the prompt the model was trained with " +
|
| 49 |
+
"(Resources/SceneAgent/system_prompt). Change it only for a model trained on your prompt.")]
|
| 50 |
public TextAsset systemPromptAsset;
|
| 51 |
[Tooltip("Optional: the LocalModelServer that runs the model. Run() then waits until it is ready and uses its endpoint.")]
|
| 52 |
public LocalModelServer server;
|
| 53 |
[NonSerialized] public string systemPrompt;
|
| 54 |
public int maxTurns = 4;
|
| 55 |
public SceneWorld world;
|
| 56 |
+
[Tooltip("Constrain every answer with a grammar: only the 11 tools and ids that exist. Needs llama-server. " +
|
| 57 |
+
"Not recommended: it can force an action where the model should decline. Use the output check below.")]
|
| 58 |
public bool constrainedDecoding = false;
|
| 59 |
+
[Tooltip("Output check (recommended): the model answers freely, then a call to a tool or value that does not exist " +
|
| 60 |
+
"becomes a refusal, and an unknown object id or a broken call is generated again under a grammar. " +
|
| 61 |
+
"Used when constrainedDecoding is off.")]
|
| 62 |
public bool fallbackDecoding = true;
|
| 63 |
[Tooltip("Safety: when the request names an object that is only in another room and the model acts on a different " +
|
| 64 |
+
"object here instead, that call is not executed; the model is told where the named object is. Example: " +
|
| 65 |
+
"'put the drill on the workbench' with the drill in another room must not move the screwdriver.")]
|
| 66 |
public bool guardSubstitutes = true;
|
| 67 |
|
| 68 |
static readonly HttpClient Http = new HttpClient { Timeout = TimeSpan.FromSeconds(120) };
|
| 69 |
static readonly Regex ToolCallRe = new Regex(@"<tool_call>\s*(.*?)\s*</tool_call>", RegexOptions.Singleline);
|
| 70 |
static readonly Regex ThinkRe = new Regex(@"<think>.*?</think>\s*", RegexOptions.Singleline);
|
| 71 |
+
// These calls end the agent's turn: the user's answer arrives as the next request.
|
| 72 |
static readonly HashSet<string> Handoff = new HashSet<string> { "ask_clarification" };
|
| 73 |
|
| 74 |
static string defaultPrompt;
|
| 75 |
+
static string DefaultPrompt => defaultPrompt ??= Resources.Load<TextAsset>("SceneAgent/system_prompt")?.text.Replace("\r\n", "\n");
|
| 76 |
// no ?. on systemPromptAsset: in the Editor an unassigned field is a "fake null" Unity object, and ?. would call
|
| 77 |
+
// .text on it (UnassignedReferenceException on the first command in Play mode)
|
| 78 |
string Prompt => systemPrompt ?? (systemPromptAsset != null ? systemPromptAsset.text.Replace("\r\n", "\n") : null) ?? DefaultPrompt
|
| 79 |
+
?? throw new InvalidOperationException("no system prompt (the package's Resources/SceneAgent/system_prompt is missing)");
|
| 80 |
|
| 81 |
public IEnumerator Run(string utterance, Action<AgentResult> done)
|
| 82 |
{
|
|
|
|
| 108 |
var task = Task.Run(() => Post(body)); // worker thread: no Unity API inside
|
| 109 |
while (!task.IsCompleted) yield return null;
|
| 110 |
if (task.IsFaulted) { res.error = task.Exception?.GetBaseException().Message; break; }
|
| 111 |
+
string output = task.Result, check = null;
|
| 112 |
if (!constrainedDecoding && fallbackDecoding)
|
| 113 |
{
|
| 114 |
+
check = SceneGrammar.Check(output, sceneJson, toolResults);
|
| 115 |
+
if (check == "invalid_tool" || check == "invalid_value") output = SceneGrammar.Refusal;
|
| 116 |
+
else if (check == "invalid_id" || check == "malformed")
|
| 117 |
{
|
| 118 |
req["grammar"] = SceneGrammar.Build(sceneJson, toolResults);
|
| 119 |
string again = MiniJson.Serialize(req);
|
|
|
|
| 123 |
output = retry.Result;
|
| 124 |
}
|
| 125 |
}
|
| 126 |
+
var at = new AgentTurn { modelOutput = output, check = check };
|
| 127 |
res.turns.Add(at);
|
| 128 |
var blocks = ToolCallRe.Matches(output).Cast<Match>().Select(m => m.Groups[1].Value).ToList();
|
| 129 |
if (blocks.Count == 0)
|
|
|
|
| 189 |
var item = id == null ? null : world.Items.FirstOrDefault(x => x.itemId == id);
|
| 190 |
if (item == null) return null;
|
| 191 |
if (string.IsNullOrEmpty(item.label)) return null;
|
| 192 |
+
// tested on 13,438 model calls: it fired 4 times, each time on a wrong action, with no false alarm
|
|
|
|
| 193 |
string text = " " + Regex.Replace(utterance.ToLowerInvariant(), "[^a-z0-9]+", " ") + " ";
|
| 194 |
var words = text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
|
| 195 |
string joined = string.Concat(words);
|
|
|
|
| 235 |
return Convert.ToString(msg.TryGetValue("content", out var c) ? c : "") ?? "";
|
| 236 |
}
|
| 237 |
|
| 238 |
+
// Tool results in the format the model was trained on (", " and ": " separators).
|
| 239 |
public static string PyJson(object v)
|
| 240 |
{
|
| 241 |
switch (v)
|
Runtime/SceneAgentScriptRunner.cs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
// Plays a list of commands through the agent and writes a report: for
|
| 2 |
// one command per line; "// ..." comments; "#answer <item_id>" answers the pending question; "#look <item_id>|none"
|
| 3 |
// sets the gaze; "#wait <seconds>" pauses (not counted in any step's time).
|
| 4 |
// In the Editor: assign a TextAsset and press Play. In a built game or batch run:
|
|
@@ -85,7 +85,7 @@ namespace SceneAgent
|
|
| 85 |
steps.Add(new Dictionary<string, object> {
|
| 86 |
{ "step", line }, { "calls", r.turns.SelectMany(t => t.calls).Cast<object>().ToList() },
|
| 87 |
{ "results", r.turns.SelectMany(t => t.results).Cast<object>().ToList() },
|
| 88 |
-
{ "
|
| 89 |
{ "clarification", r.clarificationQuestion }, { "error", r.error },
|
| 90 |
{ "seconds", Math.Round(r.seconds, 2) }, { "model_calls", (double)r.turns.Count },
|
| 91 |
{ "room", session.world.playerRoom }, { "signature", session.world.Signature() } });
|
|
|
|
| 1 |
+
// Plays a list of commands through the agent and writes a report: for testing your scene and for demos.
|
| 2 |
// one command per line; "// ..." comments; "#answer <item_id>" answers the pending question; "#look <item_id>|none"
|
| 3 |
// sets the gaze; "#wait <seconds>" pauses (not counted in any step's time).
|
| 4 |
// In the Editor: assign a TextAsset and press Play. In a built game or batch run:
|
|
|
|
| 85 |
steps.Add(new Dictionary<string, object> {
|
| 86 |
{ "step", line }, { "calls", r.turns.SelectMany(t => t.calls).Cast<object>().ToList() },
|
| 87 |
{ "results", r.turns.SelectMany(t => t.results).Cast<object>().ToList() },
|
| 88 |
+
{ "check", r.turns.Select(t => (object)t.check).ToList() }, { "final_text", r.finalText },
|
| 89 |
{ "clarification", r.clarificationQuestion }, { "error", r.error },
|
| 90 |
{ "seconds", Math.Round(r.seconds, 2) }, { "model_calls", (double)r.turns.Count },
|
| 91 |
{ "room", session.world.playerRoom }, { "signature", session.world.Signature() } });
|
Runtime/SceneAgentSession.cs
CHANGED
|
@@ -176,7 +176,7 @@ namespace SceneAgent
|
|
| 176 |
{ "i", (double)index }, { "time", DateTime.Now.ToString("s") }, { "request", request }, { "clarification_answer", answer },
|
| 177 |
{ "scene", scene }, { "calls", res.turns.SelectMany(t => t.calls).Cast<object>().ToList() },
|
| 178 |
{ "results", res.turns.SelectMany(t => t.results).Cast<object>().ToList() },
|
| 179 |
-
{ "
|
| 180 |
{ "final_text", res.finalText }, { "error", res.error }, { "seconds", Math.Round(res.seconds, 2) },
|
| 181 |
{ "model_calls", (double)res.turns.Count } };
|
| 182 |
File.AppendAllText(LogFile, MiniJson.Serialize(rec) + "\n");
|
|
|
|
| 176 |
{ "i", (double)index }, { "time", DateTime.Now.ToString("s") }, { "request", request }, { "clarification_answer", answer },
|
| 177 |
{ "scene", scene }, { "calls", res.turns.SelectMany(t => t.calls).Cast<object>().ToList() },
|
| 178 |
{ "results", res.turns.SelectMany(t => t.results).Cast<object>().ToList() },
|
| 179 |
+
{ "check", res.turns.Select(t => (object)t.check).ToList() }, { "question", res.clarificationQuestion },
|
| 180 |
{ "final_text", res.finalText }, { "error", res.error }, { "seconds", Math.Round(res.seconds, 2) },
|
| 181 |
{ "model_calls", (double)res.turns.Count } };
|
| 182 |
File.AppendAllText(LogFile, MiniJson.Serialize(rec) + "\n");
|
Runtime/SceneCatalog.Generated.cs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
-
//
|
| 2 |
-
//
|
| 3 |
using System.Collections.Generic;
|
| 4 |
|
| 5 |
namespace SceneAgent
|
|
|
|
| 1 |
+
// Generated from the object catalogue the model was trained on. Do not edit.
|
| 2 |
+
// SceneWorld.cs checks tool calls against these, exactly as the training simulator does.
|
| 3 |
using System.Collections.Generic;
|
| 4 |
|
| 5 |
namespace SceneAgent
|
Runtime/SceneGrammar.Check.cs
CHANGED
|
@@ -1,11 +1,10 @@
|
|
| 1 |
-
//
|
| 2 |
-
// scene, with the same verdicts as eval/grammar.py check():
|
| 3 |
// "ok" plain text, or every call names a known tool with known arguments, allowed values and known ids
|
| 4 |
// "invalid_tool" a tool the schema does not have -> the agent refuses (Refusal)
|
| 5 |
// "invalid_value" an argument, allowed value or room the schema/scene does not have -> the agent refuses
|
| 6 |
// "invalid_id" an id outside the scene, the hands, the gaze and earlier tool results -> generate again under the grammar
|
| 7 |
// "malformed" a call that does not parse -> generate again under the grammar
|
| 8 |
-
// The first out-of-schema call decides at once; an unknown id is remembered while the rest is checked
|
| 9 |
using System.Collections.Generic;
|
| 10 |
using System.Linq;
|
| 11 |
using System.Text.RegularExpressions;
|
|
@@ -57,7 +56,7 @@ namespace SceneAgent
|
|
| 57 |
if (!(parsed is Dictionary<string, object> obj) || !obj.TryGetValue("name", out var nameObj)) return "malformed";
|
| 58 |
if (!(nameObj is string name) || !Tools.TryGetValue(name, out var spec)) return "invalid_tool";
|
| 59 |
obj.TryGetValue("arguments", out var argsObj);
|
| 60 |
-
//
|
| 61 |
bool empty = argsObj == null || argsObj is List<object> el && el.Count == 0 || argsObj is string es && es.Length == 0
|
| 62 |
|| argsObj is double dz && dz == 0 || argsObj is bool bf && !bf;
|
| 63 |
if (!empty && !(argsObj is Dictionary<string, object>)) return "malformed";
|
|
@@ -76,7 +75,7 @@ namespace SceneAgent
|
|
| 76 |
return verdict;
|
| 77 |
}
|
| 78 |
|
| 79 |
-
//
|
| 80 |
static bool NamesUnknownId(object v, HashSet<string> ids)
|
| 81 |
{
|
| 82 |
switch (v)
|
|
|
|
| 1 |
+
// The output check: the model answers freely, then its output is checked against the tool schema and the scene:
|
|
|
|
| 2 |
// "ok" plain text, or every call names a known tool with known arguments, allowed values and known ids
|
| 3 |
// "invalid_tool" a tool the schema does not have -> the agent refuses (Refusal)
|
| 4 |
// "invalid_value" an argument, allowed value or room the schema/scene does not have -> the agent refuses
|
| 5 |
// "invalid_id" an id outside the scene, the hands, the gaze and earlier tool results -> generate again under the grammar
|
| 6 |
// "malformed" a call that does not parse -> generate again under the grammar
|
| 7 |
+
// The first out-of-schema call decides at once; an unknown id is remembered while the rest is checked.
|
| 8 |
using System.Collections.Generic;
|
| 9 |
using System.Linq;
|
| 10 |
using System.Text.RegularExpressions;
|
|
|
|
| 56 |
if (!(parsed is Dictionary<string, object> obj) || !obj.TryGetValue("name", out var nameObj)) return "malformed";
|
| 57 |
if (!(nameObj is string name) || !Tools.TryGetValue(name, out var spec)) return "invalid_tool";
|
| 58 |
obj.TryGetValue("arguments", out var argsObj);
|
| 59 |
+
// an empty or false "arguments" value counts as no arguments
|
| 60 |
bool empty = argsObj == null || argsObj is List<object> el && el.Count == 0 || argsObj is string es && es.Length == 0
|
| 61 |
|| argsObj is double dz && dz == 0 || argsObj is bool bf && !bf;
|
| 62 |
if (!empty && !(argsObj is Dictionary<string, object>)) return "malformed";
|
|
|
|
| 75 |
return verdict;
|
| 76 |
}
|
| 77 |
|
| 78 |
+
// a string where a list is expected is checked character by character, as in training
|
| 79 |
static bool NamesUnknownId(object v, HashSet<string> ids)
|
| 80 |
{
|
| 81 |
switch (v)
|
Runtime/SceneGrammar.Schema.cs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
-
//
|
| 2 |
-
// Per tool: its arguments and their allowed values (null = any value of the argument's type). Used by
|
| 3 |
using System.Collections.Generic;
|
| 4 |
|
| 5 |
namespace SceneAgent
|
|
|
|
| 1 |
+
// Generated from the model's tool schema. Do not edit.
|
| 2 |
+
// Per tool: its arguments and their allowed values (null = any value of the argument's type). Used by the output check.
|
| 3 |
using System.Collections.Generic;
|
| 4 |
|
| 5 |
namespace SceneAgent
|
Runtime/SceneGrammar.Static.cs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
//
|
| 2 |
namespace SceneAgent
|
| 3 |
{
|
| 4 |
public static partial class SceneGrammar
|
|
|
|
| 1 |
+
// Generated from the model's tool schema. Do not edit.
|
| 2 |
namespace SceneAgent
|
| 3 |
{
|
| 4 |
public static partial class SceneGrammar
|
Runtime/SceneGrammar.cs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
// Grammar-constrained decoding
|
| 2 |
// The static part (the 11 tools, their arguments in the trained order, the <tool_call> format, plain text that cannot
|
| 3 |
// start a half-written call) is generated into SceneGrammar.Static.cs. This file adds the per-request rules: the ids
|
| 4 |
// the model may name (the scene's objects, the player's hands and gaze, and every id an earlier tool result returned,
|
|
|
|
| 1 |
+
// Grammar-constrained decoding: a GBNF grammar that only allows valid tool calls or plain text.
|
| 2 |
// The static part (the 11 tools, their arguments in the trained order, the <tool_call> format, plain text that cannot
|
| 3 |
// start a half-written call) is generated into SceneGrammar.Static.cs. This file adds the per-request rules: the ids
|
| 4 |
// the model may name (the scene's objects, the player's hands and gaze, and every id an earlier tool result returned,
|
Runtime/SceneMath.cs
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Rounding and number formatting exactly as in the model's training data (Python's round(x, n) and f"{x:.nf}"), so
|
| 2 |
+
// the scene text and the tool effects match it byte for byte. Python rounds the exact binary value of x, ties to even;
|
| 3 |
+
// .NET's Math.Round(x, n) scales by 10^n first, so 0.45 (binary 0.45000000000000001...) became 0.4 where the training
|
| 4 |
+
// scenes say 0.5, and -0.04 became 0.0 where they say -0.0.
|
| 5 |
+
using System;
|
| 6 |
+
using System.Globalization;
|
| 7 |
+
using System.Numerics;
|
| 8 |
+
|
| 9 |
+
namespace SceneAgent
|
| 10 |
+
{
|
| 11 |
+
public static class SceneMath
|
| 12 |
+
{
|
| 13 |
+
/// <summary>Python round(x, n) for n >= 0: exact, ties to even, sign kept (round(-0.04, 1) == -0.0).</summary>
|
| 14 |
+
public static double Round(double x, int n)
|
| 15 |
+
{
|
| 16 |
+
if (double.IsNaN(x) || double.IsInfinity(x)) return x;
|
| 17 |
+
long bits = BitConverter.DoubleToInt64Bits(x);
|
| 18 |
+
bool neg = bits < 0;
|
| 19 |
+
int e = (int)((bits >> 52) & 0x7FF);
|
| 20 |
+
long m = bits & 0xFFFFFFFFFFFFFL;
|
| 21 |
+
if (e == 0) e = 1; else m |= 1L << 52;
|
| 22 |
+
e -= 1075; // |x| = m * 2^e exactly
|
| 23 |
+
BigInteger num = new BigInteger(m) * BigInteger.Pow(10, n), den = BigInteger.One;
|
| 24 |
+
if (e > 0) num <<= e; else den <<= -e;
|
| 25 |
+
BigInteger q = BigInteger.DivRem(num, den, out BigInteger r);
|
| 26 |
+
int c = (r * 2).CompareTo(den);
|
| 27 |
+
if (c > 0 || (c == 0 && !q.IsEven)) q += 1;
|
| 28 |
+
double v = (double)q / Math.Pow(10, n); // q and 10^n are exact doubles: the division is correctly rounded
|
| 29 |
+
return neg ? -v : v;
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
/// <summary>Python f"{x:.{n}f}".</summary>
|
| 33 |
+
public static string Fixed(double x, int n) => Round(x, n).ToString("F" + n, CultureInfo.InvariantCulture);
|
| 34 |
+
}
|
| 35 |
+
}
|
Runtime/SceneMath.cs.meta
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fileFormatVersion: 2
|
| 2 |
+
guid: aa67273f40b6679448917c8a06ebe741
|
| 3 |
+
MonoImporter:
|
| 4 |
+
externalObjects: {}
|
| 5 |
+
serializedVersion: 2
|
| 6 |
+
defaultReferences: []
|
| 7 |
+
executionOrder: 0
|
| 8 |
+
icon: {instanceID: 0}
|
| 9 |
+
userData:
|
| 10 |
+
assetBundleName:
|
| 11 |
+
assetBundleVariant:
|
Runtime/SceneWorld.cs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
-
// Scene state + tool semantics on real GameObjects.
|
| 2 |
-
//
|
| 3 |
using System;
|
| 4 |
using System.Collections.Generic;
|
| 5 |
using System.Globalization;
|
|
@@ -21,19 +21,19 @@ namespace SceneAgent
|
|
| 21 |
public SortedSet<string> clarify = new SortedSet<string>(StringComparer.Ordinal);
|
| 22 |
public readonly SortedDictionary<string, double> pressed = new SortedDictionary<string, double>(StringComparer.Ordinal);
|
| 23 |
public string lastClarifyQuestion;
|
| 24 |
-
/// <summary>Where a room sits in the Unity scene (rooms side by side
|
| 25 |
-
/// room-local positions; null = every room at the origin
|
| 26 |
[NonSerialized] public Func<string, Vector3> roomOrigin;
|
| 27 |
[Tooltip("On: at Start every SceneItem's logical position and rotation are read from where it stands in the scene, " +
|
| 28 |
"and the rooms and the player's room are filled in from the items when empty. Leave off when the scene " +
|
| 29 |
-
"is built from data
|
| 30 |
public bool initFromTransforms;
|
| 31 |
/// <summary>Raised after every tool call Unity accepted: (tool, arguments, result). Hook animations, sounds or the
|
| 32 |
/// camera here, e.g. move the player rig when "teleport" changes playerRoom.</summary>
|
| 33 |
public event Action<string, Dictionary<string, object>, object> ToolExecuted;
|
| 34 |
|
| 35 |
-
// affordances per label:
|
| 36 |
-
//
|
| 37 |
static Dictionary<string, double> Top => SceneCatalog.Top;
|
| 38 |
static HashSet<string> Grasp => SceneCatalog.Grasp;
|
| 39 |
static HashSet<string> Container => SceneCatalog.Container;
|
|
@@ -70,17 +70,17 @@ namespace SceneAgent
|
|
| 70 |
if (string.IsNullOrEmpty(o.itemId) || !o.hasPos || HandSlots.Contains(o.on)) continue;
|
| 71 |
var tr = o.transform;
|
| 72 |
var p = tr.position - (roomOrigin != null ? roomOrigin(o.room) : Vector3.zero);
|
| 73 |
-
o.pos = new[] {
|
| 74 |
var e = tr.rotation.eulerAngles;
|
| 75 |
-
o.yaw =
|
| 76 |
}
|
| 77 |
if (rooms.Count == 0) rooms = Items.Select(o => o.room).Where(r => !string.IsNullOrEmpty(r)).Distinct().ToList();
|
| 78 |
if (string.IsNullOrEmpty(playerRoom) && rooms.Count > 0) playerRoom = rooms[0];
|
| 79 |
}
|
| 80 |
|
| 81 |
// Held items are parented to the hand anchors while they are shown in the hand; when the anchors live outside
|
| 82 |
-
// this hierarchy (
|
| 83 |
-
// item fails with "unknown object" (
|
| 84 |
public IEnumerable<SceneItem> Items
|
| 85 |
{
|
| 86 |
get
|
|
@@ -105,16 +105,15 @@ namespace SceneAgent
|
|
| 105 |
public string BuildSceneJson()
|
| 106 |
{
|
| 107 |
var vis = new List<object>();
|
| 108 |
-
// hierarchy order, not sorted: training scenes list objects in random order
|
| 109 |
-
//
|
| 110 |
-
// hierarchy order lets the Unity test send exactly the offline prompts
|
| 111 |
foreach (var o in Items.Where(x => x.room == playerRoom))
|
| 112 |
{
|
| 113 |
var v = new Dictionary<string, object> { { "id", o.itemId }, { "label", o.label } };
|
| 114 |
if (!string.IsNullOrEmpty(o.color)) v["color"] = o.color;
|
| 115 |
v["on"] = o.on;
|
| 116 |
if (!string.IsNullOrEmpty(o.state)) v["state"] = o.state;
|
| 117 |
-
if (o.hasPos) v["pos"] = o.pos.Select(c => (object)
|
| 118 |
vis.Add(v);
|
| 119 |
}
|
| 120 |
var scene = new Dictionary<string, object> {
|
|
@@ -139,7 +138,7 @@ namespace SceneAgent
|
|
| 139 |
|
| 140 |
void MoveTree(SceneItem o, double[] d)
|
| 141 |
{
|
| 142 |
-
for (int i = 0; i < 3; i++) o.pos[i] =
|
| 143 |
foreach (var c in Items) if (c.on == o.itemId && c.hasPos) MoveTree(c, d);
|
| 144 |
}
|
| 145 |
|
|
@@ -193,7 +192,7 @@ namespace SceneAgent
|
|
| 193 |
FreeHand(o);
|
| 194 |
double top = rel == "on" && TryTop(t, out var th) ? th : 0.0;
|
| 195 |
o.on = t.itemId; o.relation = rel; o.orientation = orient; o.hasPos = true;
|
| 196 |
-
o.pos = new[] { t.pos[0],
|
| 197 |
Sync(o); return ok;
|
| 198 |
}
|
| 199 |
case "move_by":
|
|
@@ -213,7 +212,7 @@ namespace SceneAgent
|
|
| 213 |
var o = Get(S(a, "object_id"));
|
| 214 |
if (HandSlots.Contains(o.on) || IsFixed(o)) throw new ToolError($"'{o.itemId}' cannot be rotated");
|
| 215 |
string axis = S(a, "axis") ?? "yaw";
|
| 216 |
-
double Wrap(double v) =>
|
| 217 |
double deg = D(a, "degrees");
|
| 218 |
if (axis == "yaw") o.yaw = Wrap(o.yaw + deg);
|
| 219 |
else if (axis == "pitch") o.pitch = Wrap(o.pitch + deg);
|
|
@@ -251,7 +250,7 @@ namespace SceneAgent
|
|
| 251 |
case "find_objects":
|
| 252 |
{
|
| 253 |
string label = S(a, "label"), color = S(a, "color"), room = S(a, "room");
|
| 254 |
-
// hierarchy order, like the simulator's world order
|
| 255 |
return Items.Where(o => o.label == label && (color == null || o.color == color) && (room == null || o.room == room))
|
| 256 |
.Select(o =>
|
| 257 |
{
|
|
@@ -297,7 +296,7 @@ namespace SceneAgent
|
|
| 297 |
static string Num(double v, int digits)
|
| 298 |
{
|
| 299 |
if (Math.Abs(v) < 0.5 * Math.Pow(10, -digits)) v = 0.0;
|
| 300 |
-
return
|
| 301 |
}
|
| 302 |
|
| 303 |
public string Signature()
|
|
|
|
| 1 |
+
// Scene state + tool semantics on real GameObjects. The rules match the simulator the model was trained on, so the
|
| 2 |
+
// model's calls behave here exactly as they did in training.
|
| 3 |
using System;
|
| 4 |
using System.Collections.Generic;
|
| 5 |
using System.Globalization;
|
|
|
|
| 21 |
public SortedSet<string> clarify = new SortedSet<string>(StringComparer.Ordinal);
|
| 22 |
public readonly SortedDictionary<string, double> pressed = new SortedDictionary<string, double>(StringComparer.Ordinal);
|
| 23 |
public string lastClarifyQuestion;
|
| 24 |
+
/// <summary>Where a room sits in the Unity scene (for example, rooms side by side). The model only ever sees
|
| 25 |
+
/// room-local positions; null = every room at the origin.</summary>
|
| 26 |
[NonSerialized] public Func<string, Vector3> roomOrigin;
|
| 27 |
[Tooltip("On: at Start every SceneItem's logical position and rotation are read from where it stands in the scene, " +
|
| 28 |
"and the rooms and the player's room are filled in from the items when empty. Leave off when the scene " +
|
| 29 |
+
"is built from data by your own code.")]
|
| 30 |
public bool initFromTransforms;
|
| 31 |
/// <summary>Raised after every tool call Unity accepted: (tool, arguments, result). Hook animations, sounds or the
|
| 32 |
/// camera here, e.g. move the player rig when "teleport" changes playerRoom.</summary>
|
| 33 |
public event Action<string, Dictionary<string, object>, object> ToolExecuted;
|
| 34 |
|
| 35 |
+
// affordances per label (SceneCatalog.Generated.cs): the same catalogue the model was trained on, so Unity
|
| 36 |
+
// accepts exactly the calls the model learned to make
|
| 37 |
static Dictionary<string, double> Top => SceneCatalog.Top;
|
| 38 |
static HashSet<string> Grasp => SceneCatalog.Grasp;
|
| 39 |
static HashSet<string> Container => SceneCatalog.Container;
|
|
|
|
| 70 |
if (string.IsNullOrEmpty(o.itemId) || !o.hasPos || HandSlots.Contains(o.on)) continue;
|
| 71 |
var tr = o.transform;
|
| 72 |
var p = tr.position - (roomOrigin != null ? roomOrigin(o.room) : Vector3.zero);
|
| 73 |
+
o.pos = new[] { SceneMath.Round(p.x, 3), SceneMath.Round(p.y - tr.localScale.y / 2f, 3), SceneMath.Round(p.z, 3) };
|
| 74 |
var e = tr.rotation.eulerAngles;
|
| 75 |
+
o.yaw = SceneMath.Round(e.y, 3); o.pitch = SceneMath.Round(e.x, 3); o.roll = SceneMath.Round(e.z, 3);
|
| 76 |
}
|
| 77 |
if (rooms.Count == 0) rooms = Items.Select(o => o.room).Where(r => !string.IsNullOrEmpty(r)).Distinct().ToList();
|
| 78 |
if (string.IsNullOrEmpty(playerRoom) && rooms.Count > 0) playerRoom = rooms[0];
|
| 79 |
}
|
| 80 |
|
| 81 |
// Held items are parented to the hand anchors while they are shown in the hand; when the anchors live outside
|
| 82 |
+
// this hierarchy (for example, on the camera rig) they must still count as scene items, or every tool call on a
|
| 83 |
+
// held item fails with "unknown object" ("put it on the counter" after a grab).
|
| 84 |
public IEnumerable<SceneItem> Items
|
| 85 |
{
|
| 86 |
get
|
|
|
|
| 105 |
public string BuildSceneJson()
|
| 106 |
{
|
| 107 |
var vis = new List<object>();
|
| 108 |
+
// hierarchy order, not sorted: training scenes list objects in random order, and the model does not depend
|
| 109 |
+
// on the order
|
|
|
|
| 110 |
foreach (var o in Items.Where(x => x.room == playerRoom))
|
| 111 |
{
|
| 112 |
var v = new Dictionary<string, object> { { "id", o.itemId }, { "label", o.label } };
|
| 113 |
if (!string.IsNullOrEmpty(o.color)) v["color"] = o.color;
|
| 114 |
v["on"] = o.on;
|
| 115 |
if (!string.IsNullOrEmpty(o.state)) v["state"] = o.state;
|
| 116 |
+
if (o.hasPos) v["pos"] = o.pos.Select(c => (object)SceneMath.Round(c, 1)).ToList();
|
| 117 |
vis.Add(v);
|
| 118 |
}
|
| 119 |
var scene = new Dictionary<string, object> {
|
|
|
|
| 138 |
|
| 139 |
void MoveTree(SceneItem o, double[] d)
|
| 140 |
{
|
| 141 |
+
for (int i = 0; i < 3; i++) o.pos[i] = SceneMath.Round(o.pos[i] + d[i], 3);
|
| 142 |
foreach (var c in Items) if (c.on == o.itemId && c.hasPos) MoveTree(c, d);
|
| 143 |
}
|
| 144 |
|
|
|
|
| 192 |
FreeHand(o);
|
| 193 |
double top = rel == "on" && TryTop(t, out var th) ? th : 0.0;
|
| 194 |
o.on = t.itemId; o.relation = rel; o.orientation = orient; o.hasPos = true;
|
| 195 |
+
o.pos = new[] { t.pos[0], SceneMath.Round(t.pos[1] + top, 3), t.pos[2] };
|
| 196 |
Sync(o); return ok;
|
| 197 |
}
|
| 198 |
case "move_by":
|
|
|
|
| 212 |
var o = Get(S(a, "object_id"));
|
| 213 |
if (HandSlots.Contains(o.on) || IsFixed(o)) throw new ToolError($"'{o.itemId}' cannot be rotated");
|
| 214 |
string axis = S(a, "axis") ?? "yaw";
|
| 215 |
+
double Wrap(double v) => SceneMath.Round(((v % 360) + 360) % 360, 3);
|
| 216 |
double deg = D(a, "degrees");
|
| 217 |
if (axis == "yaw") o.yaw = Wrap(o.yaw + deg);
|
| 218 |
else if (axis == "pitch") o.pitch = Wrap(o.pitch + deg);
|
|
|
|
| 250 |
case "find_objects":
|
| 251 |
{
|
| 252 |
string label = S(a, "label"), color = S(a, "color"), room = S(a, "room");
|
| 253 |
+
// hierarchy order, like the training simulator's world order
|
| 254 |
return Items.Where(o => o.label == label && (color == null || o.color == color) && (room == null || o.room == room))
|
| 255 |
.Select(o =>
|
| 256 |
{
|
|
|
|
| 296 |
static string Num(double v, int digits)
|
| 297 |
{
|
| 298 |
if (Math.Abs(v) < 0.5 * Math.Pow(10, -digits)) v = 0.0;
|
| 299 |
+
return SceneMath.Fixed(v, digits);
|
| 300 |
}
|
| 301 |
|
| 302 |
public string Signature()
|
package.json
CHANGED
|
@@ -1,26 +1,25 @@
|
|
| 1 |
-
{
|
| 2 |
-
"name": "com.mcbuxrlab.maverick",
|
| 3 |
-
"version": "1.0.0
|
| 4 |
-
"displayName": "Maverick XR Agent",
|
| 5 |
-
"unity": "2022.3",
|
| 6 |
-
"description": "Control a Unity scene
|
| 7 |
-
"keywords": [
|
| 8 |
-
"maverick",
|
| 9 |
-
"llm",
|
| 10 |
-
"
|
| 11 |
-
"
|
| 12 |
-
"
|
| 13 |
-
"
|
| 14 |
-
"
|
| 15 |
-
"
|
| 16 |
-
"
|
| 17 |
-
"
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
"
|
| 21 |
-
|
| 22 |
-
"
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
}
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "com.mcbuxrlab.maverick",
|
| 3 |
+
"version": "1.0.0",
|
| 4 |
+
"displayName": "Maverick XR Agent",
|
| 5 |
+
"unity": "2022.3",
|
| 6 |
+
"description": "Control a Unity scene with typed or spoken English. Maverick-4B-Unity-XR-Agent runs locally through llama.cpp, reads the objects you mark as interactive and answers with tool calls that Unity carries out: grab, place, move, rotate, switch on or off, highlight, teleport, find, and ask which object you mean. Model: https://huggingface.co/ErenAta00/Maverick-4B-Unity-XR-Agent-GGUF",
|
| 7 |
+
"keywords": [
|
| 8 |
+
"maverick",
|
| 9 |
+
"llm",
|
| 10 |
+
"voice control",
|
| 11 |
+
"natural language",
|
| 12 |
+
"xr",
|
| 13 |
+
"vr",
|
| 14 |
+
"offline",
|
| 15 |
+
"llama.cpp",
|
| 16 |
+
"agent",
|
| 17 |
+
"tool calling"
|
| 18 |
+
],
|
| 19 |
+
"license": "MIT",
|
| 20 |
+
"author": {
|
| 21 |
+
"name": "Eren Ata, MCBU XRLab",
|
| 22 |
+
"url": "https://huggingface.co/ErenAta00"
|
| 23 |
+
},
|
| 24 |
+
"documentationUrl": "https://huggingface.co/ErenAta00/Maverick-Unity"
|
| 25 |
+
}
|
|
|