ErenAta00 commited on
Commit
dec5552
·
verified ·
1 Parent(s): 31d56aa

Maverick XR Agent 1.0.0-rc.2 (Unity package)

Browse files
Editor/SceneAgentMenu.cs CHANGED
@@ -120,8 +120,8 @@ namespace SceneAgent.EditorTools
120
  var agent = world.GetComponent<SceneAgent>() != null ? world.GetComponent<SceneAgent>() : Object.FindObjectOfType<SceneAgent>();
121
  if (agent == null) E("no SceneAgent component");
122
  else if (agent.world == null) E("SceneAgent.world is not set");
123
- if (Resources.Load<TextAsset>("SceneAgent/system_prompt_v3") == null && (agent == null || agent.systemPromptAsset == null))
124
- E("the system prompt is missing (package Resources/SceneAgent/system_prompt_v3.txt)");
125
  var server = agent != null && agent.server != null ? agent.server : Object.FindObjectOfType<LocalModelServer>();
126
  if (server == null) W("no LocalModelServer: the agent will call " + (agent != null ? agent.endpoint : "?") + " (start a server yourself)");
127
  else sb.AppendLine("model " + ModelFilesStatus(server));
 
120
  var agent = world.GetComponent<SceneAgent>() != null ? world.GetComponent<SceneAgent>() : Object.FindObjectOfType<SceneAgent>();
121
  if (agent == null) E("no SceneAgent component");
122
  else if (agent.world == null) E("SceneAgent.world is not set");
123
+ if (Resources.Load<TextAsset>("SceneAgent/system_prompt") == null && (agent == null || agent.systemPromptAsset == null))
124
+ E("the system prompt is missing (package Resources/SceneAgent/system_prompt.txt)");
125
  var server = agent != null && agent.server != null ? agent.server : Object.FindObjectOfType<LocalModelServer>();
126
  if (server == null) W("no LocalModelServer: the agent will call " + (agent != null ? agent.endpoint : "?") + " (start a server yourself)");
127
  else sb.AppendLine("model " + ModelFilesStatus(server));
Runtime/LocalModelServer.cs CHANGED
@@ -36,7 +36,7 @@ namespace SceneAgent
36
  public string extraArgs = "--cache-ram 0";
37
  public float startTimeoutSeconds = 120f;
38
  [Tooltip("While the model runs on the GPU, cap rendering at this frame rate so the renderer does not starve it " +
39
- "(uncapped rendering made the model ~5x slower on a 4 GB laptop GPU). 0 = leave the frame rate alone; " +
40
  "use 0 in XR, where the headset sets the frame rate.")]
41
  public int capFrameRate = 30;
42
  [Tooltip("Start the server when this component starts. Off: call Launch() yourself.")]
 
36
  public string extraArgs = "--cache-ram 0";
37
  public float startTimeoutSeconds = 120f;
38
  [Tooltip("While the model runs on the GPU, cap rendering at this frame rate so the renderer does not starve it " +
39
+ "(uncapped rendering can make the model about 5x slower). 0 = leave the frame rate alone; " +
40
  "use 0 in XR, where the headset sets the frame rate.")]
41
  public int capFrameRate = 30;
42
  [Tooltip("Start the server when this component starts. Off: call Launch() yourself.")]
Runtime/MiniJson.cs CHANGED
@@ -17,8 +17,8 @@ namespace SceneAgent
17
  return v;
18
  }
19
 
20
- /// <summary>Like Parse, but only whitespace may follow the value (Python's json.loads rejects extra data; D5b's
21
- /// check must judge a tool call exactly as eval/grammar.py does).</summary>
22
  public static object ParseStrict(string s)
23
  {
24
  int i = 0;
@@ -126,7 +126,7 @@ namespace SceneAgent
126
  }
127
  }
128
 
129
- // Python's json.dumps prints integral floats as "2.0"; keep the same shape as the training data.
130
  public static string FormatNumber(double d)
131
  {
132
  string s = d.ToString("R", CultureInfo.InvariantCulture);
 
17
  return v;
18
  }
19
 
20
+ /// <summary>Like Parse, but only whitespace may follow the value, so the output check judges a tool call
21
+ /// exactly as in training.</summary>
22
  public static object ParseStrict(string s)
23
  {
24
  int i = 0;
 
126
  }
127
  }
128
 
129
+ // integral floats print as "2.0", the same shape as the training data
130
  public static string FormatNumber(double d)
131
  {
132
  string s = d.ToString("R", CultureInfo.InvariantCulture);
Runtime/Resources/SceneAgent/system_prompt.txt ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ You are an assistant embedded in a 3D Unity scene. Turn the user's request into tool calls that use object ids exactly as listed in the scene; never invent an id or a tool. Only objects in the user's current room are listed. Match objects by meaning: a synonym or a more general word can name a listed object, and details the scene does not record (a colour when no colours are listed, or words like sliced, wooden, empty) do not rule an object out. If exactly one listed object fits, act on it. If several fit equally well, call ask_clarification with their ids. If none fits, use find_objects to look in the other rooms. place moves an object directly, so there is no need to grab it first. If a request is impossible or needs a tool you do not have, call no tool and briefly explain why. Answer questions about the scene in plain text.
2
+
3
+ # Tools
4
+ grab(object_id:str, hand:left|right|both)
5
+ release(hand:left|right|both)
6
+ place(object_id:str, target_id:str, relation?:on|in, orientation?:default|upright|horizontal|vertical|upside_down)
7
+ move_by(object_id:str, direction:left|right|forward|backward|up|down, distance:num, unit:m|cm)
8
+ rotate(object_id:str, degrees:num, axis?:yaw|pitch|roll)
9
+ set_state(object_id:str, state:on|off|open|closed|half_open|locked|unlocked)
10
+ press(object_id:str, hold_seconds?:num)
11
+ highlight(object_ids:list)
12
+ teleport(location_id:str)
13
+ find_objects(label:str, color?:str, room?:str)
14
+ ask_clarification(question:str, candidate_ids:list)
15
+
16
+ Reply with one <tool_call>{"name": ..., "arguments": {...}}</tool_call> block per call, or plain text.
Runtime/Resources/SceneAgent/system_prompt.txt.meta ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 6a38ee93718310f469efc52b22f6a1b9
3
+ TextScriptImporter:
4
+ externalObjects: {}
5
+ userData:
6
+ assetBundleName:
7
+ assetBundleVariant:
Runtime/SceneAgent.cs CHANGED
@@ -1,5 +1,5 @@
1
- // Talks to a local OpenAI-compatible server (llama.cpp llama-server with the fine-tuned GGUF,
2
- // Unsloth Studio's /v1 endpoint, Ollama, ...), executes the returned tool calls on the SceneWorld
3
  // and feeds tool results back until the model answers in text, asks the user a clarifying question
4
  // (ask_clarification hands the turn back: the answer arrives as the next request), or runs out of turns.
5
  //
@@ -26,8 +26,9 @@ namespace SceneAgent
26
  public string modelOutput;
27
  public List<string> calls = new List<string>();
28
  public List<string> results = new List<string>();
29
- /// <summary>D5b only: the verdict on the free output (SceneGrammar.Check) and what was done about it.</summary>
30
- public string d5b;
 
31
  }
32
 
33
  public class AgentResult
@@ -44,38 +45,38 @@ namespace SceneAgent
44
  {
45
  public string endpoint = "http://127.0.0.1:8081/v1/chat/completions";
46
  public string model = "unity-scene-agent";
47
- [Tooltip("The model's system prompt. Empty: the one shipped with the package (Resources/SceneAgent/system_prompt_v3), " +
48
- "the exact prompt the v3 model was trained and evaluated with. Change it only with a model trained on yours.")]
49
  public TextAsset systemPromptAsset;
50
  [Tooltip("Optional: the LocalModelServer that runs the model. Run() then waits until it is ready and uses its endpoint.")]
51
  public LocalModelServer server;
52
  [NonSerialized] public string systemPrompt;
53
  public int maxTurns = 4;
54
  public SceneWorld world;
55
- [Tooltip("Strict D5: constrain decoding to the 11 tools, ids that exist and the trained call format (SceneGrammar.cs). " +
56
- "Needs llama-server. The shipped v3 setting is D5b below, not this.")]
57
  public bool constrainedDecoding = false;
58
- [Tooltip("D5b (the shipped v3 setting): the model answers freely; SceneGrammar.Check then turns a call to a tool or " +
59
- "value outside the schema into a refusal and generates an unknown id or a broken call again under the " +
60
- "grammar. Used when constrainedDecoding is off.")]
61
  public bool fallbackDecoding = true;
62
  [Tooltip("Safety: when the request names an object that is only in another room and the model acts on a different " +
63
- "object here instead, that call is not executed; the model is told where the named object is. (Seen in the " +
64
- "user test: 'put the drill on the workbench' with the drill in another room moved the screwdriver.)")]
65
  public bool guardSubstitutes = true;
66
 
67
  static readonly HttpClient Http = new HttpClient { Timeout = TimeSpan.FromSeconds(120) };
68
  static readonly Regex ToolCallRe = new Regex(@"<tool_call>\s*(.*?)\s*</tool_call>", RegexOptions.Singleline);
69
  static readonly Regex ThinkRe = new Regex(@"<think>.*?</think>\s*", RegexOptions.Singleline);
70
- // Same rule as eval/toolcall.py HANDOFF: these calls end the agent turn.
71
  static readonly HashSet<string> Handoff = new HashSet<string> { "ask_clarification" };
72
 
73
  static string defaultPrompt;
74
- static string DefaultPrompt => defaultPrompt ??= Resources.Load<TextAsset>("SceneAgent/system_prompt_v3")?.text.Replace("\r\n", "\n");
75
  // no ?. on systemPromptAsset: in the Editor an unassigned field is a "fake null" Unity object, and ?. would call
76
- // .text on it (UnassignedReferenceException on the first command in Play mode; found by the visible user test)
77
  string Prompt => systemPrompt ?? (systemPromptAsset != null ? systemPromptAsset.text.Replace("\r\n", "\n") : null) ?? DefaultPrompt
78
- ?? throw new InvalidOperationException("no system prompt (the package's Resources/SceneAgent/system_prompt_v3 is missing)");
79
 
80
  public IEnumerator Run(string utterance, Action<AgentResult> done)
81
  {
@@ -107,12 +108,12 @@ namespace SceneAgent
107
  var task = Task.Run(() => Post(body)); // worker thread: no Unity API inside
108
  while (!task.IsCompleted) yield return null;
109
  if (task.IsFaulted) { res.error = task.Exception?.GetBaseException().Message; break; }
110
- string output = task.Result, d5b = null;
111
  if (!constrainedDecoding && fallbackDecoding)
112
  {
113
- d5b = SceneGrammar.Check(output, sceneJson, toolResults);
114
- if (d5b == "invalid_tool" || d5b == "invalid_value") output = SceneGrammar.Refusal;
115
- else if (d5b == "invalid_id" || d5b == "malformed")
116
  {
117
  req["grammar"] = SceneGrammar.Build(sceneJson, toolResults);
118
  string again = MiniJson.Serialize(req);
@@ -122,7 +123,7 @@ namespace SceneAgent
122
  output = retry.Result;
123
  }
124
  }
125
- var at = new AgentTurn { modelOutput = output, d5b = d5b };
126
  res.turns.Add(at);
127
  var blocks = ToolCallRe.Matches(output).Cast<Match>().Select(m => m.Groups[1].Value).ToList();
128
  if (blocks.Count == 0)
@@ -188,8 +189,7 @@ namespace SceneAgent
188
  var item = id == null ? null : world.Items.FirstOrDefault(x => x.itemId == id);
189
  if (item == null) return null;
190
  if (string.IsNullOrEmpty(item.label)) return null;
191
- // the same rules as tests/acceptance/guard_offline.py, which measured them on every call the four v3 models
192
- // made on the sealed and dev sets (13,438 calls): 4 firings, all on wrong actions, no false alarm
193
  string text = " " + Regex.Replace(utterance.ToLowerInvariant(), "[^a-z0-9]+", " ") + " ";
194
  var words = text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
195
  string joined = string.Concat(words);
@@ -235,7 +235,7 @@ namespace SceneAgent
235
  return Convert.ToString(msg.TryGetValue("content", out var c) ? c : "") ?? "";
236
  }
237
 
238
- // Tool results in training were produced by Python's json.dumps (", " and ": " separators).
239
  public static string PyJson(object v)
240
  {
241
  switch (v)
 
1
+ // Talks to a local OpenAI-compatible server (llama.cpp llama-server with the Maverick GGUF, Ollama, ...),
2
+ // executes the returned tool calls on the SceneWorld
3
  // and feeds tool results back until the model answers in text, asks the user a clarifying question
4
  // (ask_clarification hands the turn back: the answer arrives as the next request), or runs out of turns.
5
  //
 
26
  public string modelOutput;
27
  public List<string> calls = new List<string>();
28
  public List<string> results = new List<string>();
29
+ /// <summary>The output check's verdict on this answer (SceneGrammar.Check): ok, invalid_tool, invalid_value,
30
+ /// invalid_id or malformed. Null when the check is off.</summary>
31
+ public string check;
32
  }
33
 
34
  public class AgentResult
 
45
  {
46
  public string endpoint = "http://127.0.0.1:8081/v1/chat/completions";
47
  public string model = "unity-scene-agent";
48
+ [Tooltip("The model's system prompt. Leave empty to use the prompt the model was trained with " +
49
+ "(Resources/SceneAgent/system_prompt). Change it only for a model trained on your prompt.")]
50
  public TextAsset systemPromptAsset;
51
  [Tooltip("Optional: the LocalModelServer that runs the model. Run() then waits until it is ready and uses its endpoint.")]
52
  public LocalModelServer server;
53
  [NonSerialized] public string systemPrompt;
54
  public int maxTurns = 4;
55
  public SceneWorld world;
56
+ [Tooltip("Constrain every answer with a grammar: only the 11 tools and ids that exist. Needs llama-server. " +
57
+ "Not recommended: it can force an action where the model should decline. Use the output check below.")]
58
  public bool constrainedDecoding = false;
59
+ [Tooltip("Output check (recommended): the model answers freely, then a call to a tool or value that does not exist " +
60
+ "becomes a refusal, and an unknown object id or a broken call is generated again under a grammar. " +
61
+ "Used when constrainedDecoding is off.")]
62
  public bool fallbackDecoding = true;
63
  [Tooltip("Safety: when the request names an object that is only in another room and the model acts on a different " +
64
+ "object here instead, that call is not executed; the model is told where the named object is. Example: " +
65
+ "'put the drill on the workbench' with the drill in another room must not move the screwdriver.")]
66
  public bool guardSubstitutes = true;
67
 
68
  static readonly HttpClient Http = new HttpClient { Timeout = TimeSpan.FromSeconds(120) };
69
  static readonly Regex ToolCallRe = new Regex(@"<tool_call>\s*(.*?)\s*</tool_call>", RegexOptions.Singleline);
70
  static readonly Regex ThinkRe = new Regex(@"<think>.*?</think>\s*", RegexOptions.Singleline);
71
+ // These calls end the agent's turn: the user's answer arrives as the next request.
72
  static readonly HashSet<string> Handoff = new HashSet<string> { "ask_clarification" };
73
 
74
  static string defaultPrompt;
75
+ static string DefaultPrompt => defaultPrompt ??= Resources.Load<TextAsset>("SceneAgent/system_prompt")?.text.Replace("\r\n", "\n");
76
  // no ?. on systemPromptAsset: in the Editor an unassigned field is a "fake null" Unity object, and ?. would call
77
+ // .text on it (UnassignedReferenceException on the first command in Play mode)
78
  string Prompt => systemPrompt ?? (systemPromptAsset != null ? systemPromptAsset.text.Replace("\r\n", "\n") : null) ?? DefaultPrompt
79
+ ?? throw new InvalidOperationException("no system prompt (the package's Resources/SceneAgent/system_prompt is missing)");
80
 
81
  public IEnumerator Run(string utterance, Action<AgentResult> done)
82
  {
 
108
  var task = Task.Run(() => Post(body)); // worker thread: no Unity API inside
109
  while (!task.IsCompleted) yield return null;
110
  if (task.IsFaulted) { res.error = task.Exception?.GetBaseException().Message; break; }
111
+ string output = task.Result, check = null;
112
  if (!constrainedDecoding && fallbackDecoding)
113
  {
114
+ check = SceneGrammar.Check(output, sceneJson, toolResults);
115
+ if (check == "invalid_tool" || check == "invalid_value") output = SceneGrammar.Refusal;
116
+ else if (check == "invalid_id" || check == "malformed")
117
  {
118
  req["grammar"] = SceneGrammar.Build(sceneJson, toolResults);
119
  string again = MiniJson.Serialize(req);
 
123
  output = retry.Result;
124
  }
125
  }
126
+ var at = new AgentTurn { modelOutput = output, check = check };
127
  res.turns.Add(at);
128
  var blocks = ToolCallRe.Matches(output).Cast<Match>().Select(m => m.Groups[1].Value).ToList();
129
  if (blocks.Count == 0)
 
189
  var item = id == null ? null : world.Items.FirstOrDefault(x => x.itemId == id);
190
  if (item == null) return null;
191
  if (string.IsNullOrEmpty(item.label)) return null;
192
+ // tested on 13,438 model calls: it fired 4 times, each time on a wrong action, with no false alarm
 
193
  string text = " " + Regex.Replace(utterance.ToLowerInvariant(), "[^a-z0-9]+", " ") + " ";
194
  var words = text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
195
  string joined = string.Concat(words);
 
235
  return Convert.ToString(msg.TryGetValue("content", out var c) ? c : "") ?? "";
236
  }
237
 
238
+ // Tool results in the format the model was trained on (", " and ": " separators).
239
  public static string PyJson(object v)
240
  {
241
  switch (v)
Runtime/SceneAgentScriptRunner.cs CHANGED
@@ -1,4 +1,4 @@
1
- // Plays a list of commands through the agent and writes a report: for QA, regression tests and demos.
2
  // one command per line; "// ..." comments; "#answer <item_id>" answers the pending question; "#look <item_id>|none"
3
  // sets the gaze; "#wait <seconds>" pauses (not counted in any step's time).
4
  // In the Editor: assign a TextAsset and press Play. In a built game or batch run:
@@ -85,7 +85,7 @@ namespace SceneAgent
85
  steps.Add(new Dictionary<string, object> {
86
  { "step", line }, { "calls", r.turns.SelectMany(t => t.calls).Cast<object>().ToList() },
87
  { "results", r.turns.SelectMany(t => t.results).Cast<object>().ToList() },
88
- { "d5b", r.turns.Select(t => (object)t.d5b).ToList() }, { "final_text", r.finalText },
89
  { "clarification", r.clarificationQuestion }, { "error", r.error },
90
  { "seconds", Math.Round(r.seconds, 2) }, { "model_calls", (double)r.turns.Count },
91
  { "room", session.world.playerRoom }, { "signature", session.world.Signature() } });
 
1
+ // Plays a list of commands through the agent and writes a report: for testing your scene and for demos.
2
  // one command per line; "// ..." comments; "#answer <item_id>" answers the pending question; "#look <item_id>|none"
3
  // sets the gaze; "#wait <seconds>" pauses (not counted in any step's time).
4
  // In the Editor: assign a TextAsset and press Play. In a built game or batch run:
 
85
  steps.Add(new Dictionary<string, object> {
86
  { "step", line }, { "calls", r.turns.SelectMany(t => t.calls).Cast<object>().ToList() },
87
  { "results", r.turns.SelectMany(t => t.results).Cast<object>().ToList() },
88
+ { "check", r.turns.Select(t => (object)t.check).ToList() }, { "final_text", r.finalText },
89
  { "clarification", r.clarificationQuestion }, { "error", r.error },
90
  { "seconds", Math.Round(r.seconds, 2) }, { "model_calls", (double)r.turns.Count },
91
  { "room", session.world.playerRoom }, { "signature", session.world.Signature() } });
Runtime/SceneAgentSession.cs CHANGED
@@ -176,7 +176,7 @@ namespace SceneAgent
176
  { "i", (double)index }, { "time", DateTime.Now.ToString("s") }, { "request", request }, { "clarification_answer", answer },
177
  { "scene", scene }, { "calls", res.turns.SelectMany(t => t.calls).Cast<object>().ToList() },
178
  { "results", res.turns.SelectMany(t => t.results).Cast<object>().ToList() },
179
- { "d5b", res.turns.Select(t => (object)t.d5b).ToList() }, { "question", res.clarificationQuestion },
180
  { "final_text", res.finalText }, { "error", res.error }, { "seconds", Math.Round(res.seconds, 2) },
181
  { "model_calls", (double)res.turns.Count } };
182
  File.AppendAllText(LogFile, MiniJson.Serialize(rec) + "\n");
 
176
  { "i", (double)index }, { "time", DateTime.Now.ToString("s") }, { "request", request }, { "clarification_answer", answer },
177
  { "scene", scene }, { "calls", res.turns.SelectMany(t => t.calls).Cast<object>().ToList() },
178
  { "results", res.turns.SelectMany(t => t.results).Cast<object>().ToList() },
179
+ { "check", res.turns.Select(t => (object)t.check).ToList() }, { "question", res.clarificationQuestion },
180
  { "final_text", res.finalText }, { "error", res.error }, { "seconds", Math.Round(res.seconds, 2) },
181
  { "model_calls", (double)res.turns.Count } };
182
  File.AppendAllText(LogFile, MiniJson.Serialize(rec) + "\n");
Runtime/SceneCatalog.Generated.cs CHANGED
@@ -1,5 +1,5 @@
1
- // GENERATED by `py tools/gen_unity_catalog.py` from datagen/sim.py + the v3 catalogue (datagen/v3/core.py).
2
- // Do not edit: regenerate. SceneWorld.cs checks tool calls against these, exactly as the Python simulator does.
3
  using System.Collections.Generic;
4
 
5
  namespace SceneAgent
 
1
+ // Generated from the object catalogue the model was trained on. Do not edit.
2
+ // SceneWorld.cs checks tool calls against these, exactly as the training simulator does.
3
  using System.Collections.Generic;
4
 
5
  namespace SceneAgent
Runtime/SceneGrammar.Check.cs CHANGED
@@ -1,11 +1,10 @@
1
- // D5b (docs/v3_data_plan.md §20.3): the model answers freely, then its output is checked against the schema and the
2
- // scene, with the same verdicts as eval/grammar.py check():
3
  // "ok" plain text, or every call names a known tool with known arguments, allowed values and known ids
4
  // "invalid_tool" a tool the schema does not have -> the agent refuses (Refusal)
5
  // "invalid_value" an argument, allowed value or room the schema/scene does not have -> the agent refuses
6
  // "invalid_id" an id outside the scene, the hands, the gaze and earlier tool results -> generate again under the grammar
7
  // "malformed" a call that does not parse -> generate again under the grammar
8
- // The first out-of-schema call decides at once; an unknown id is remembered while the rest is checked (as in Python).
9
  using System.Collections.Generic;
10
  using System.Linq;
11
  using System.Text.RegularExpressions;
@@ -57,7 +56,7 @@ namespace SceneAgent
57
  if (!(parsed is Dictionary<string, object> obj) || !obj.TryGetValue("name", out var nameObj)) return "malformed";
58
  if (!(nameObj is string name) || !Tools.TryGetValue(name, out var spec)) return "invalid_tool";
59
  obj.TryGetValue("arguments", out var argsObj);
60
- // Python: obj.get("arguments") or {} (an empty or false value counts as no arguments)
61
  bool empty = argsObj == null || argsObj is List<object> el && el.Count == 0 || argsObj is string es && es.Length == 0
62
  || argsObj is double dz && dz == 0 || argsObj is bool bf && !bf;
63
  if (!empty && !(argsObj is Dictionary<string, object>)) return "malformed";
@@ -76,7 +75,7 @@ namespace SceneAgent
76
  return verdict;
77
  }
78
 
79
- // Python: any(x not in ids for x in (v or [])); a string is iterated character by character there
80
  static bool NamesUnknownId(object v, HashSet<string> ids)
81
  {
82
  switch (v)
 
1
+ // The output check: the model answers freely, then its output is checked against the tool schema and the scene:
 
2
  // "ok" plain text, or every call names a known tool with known arguments, allowed values and known ids
3
  // "invalid_tool" a tool the schema does not have -> the agent refuses (Refusal)
4
  // "invalid_value" an argument, allowed value or room the schema/scene does not have -> the agent refuses
5
  // "invalid_id" an id outside the scene, the hands, the gaze and earlier tool results -> generate again under the grammar
6
  // "malformed" a call that does not parse -> generate again under the grammar
7
+ // The first out-of-schema call decides at once; an unknown id is remembered while the rest is checked.
8
  using System.Collections.Generic;
9
  using System.Linq;
10
  using System.Text.RegularExpressions;
 
56
  if (!(parsed is Dictionary<string, object> obj) || !obj.TryGetValue("name", out var nameObj)) return "malformed";
57
  if (!(nameObj is string name) || !Tools.TryGetValue(name, out var spec)) return "invalid_tool";
58
  obj.TryGetValue("arguments", out var argsObj);
59
+ // an empty or false "arguments" value counts as no arguments
60
  bool empty = argsObj == null || argsObj is List<object> el && el.Count == 0 || argsObj is string es && es.Length == 0
61
  || argsObj is double dz && dz == 0 || argsObj is bool bf && !bf;
62
  if (!empty && !(argsObj is Dictionary<string, object>)) return "malformed";
 
75
  return verdict;
76
  }
77
 
78
+ // a string where a list is expected is checked character by character, as in training
79
  static bool NamesUnknownId(object v, HashSet<string> ids)
80
  {
81
  switch (v)
Runtime/SceneGrammar.Schema.cs CHANGED
@@ -1,5 +1,5 @@
1
- // GENERATED by `py tools/gen_unity_schema.py` from spec/tools_v0.json. Do not edit: regenerate.
2
- // Per tool: its arguments and their allowed values (null = any value of the argument's type). Used by D5b.
3
  using System.Collections.Generic;
4
 
5
  namespace SceneAgent
 
1
+ // Generated from the model's tool schema. Do not edit.
2
+ // Per tool: its arguments and their allowed values (null = any value of the argument's type). Used by the output check.
3
  using System.Collections.Generic;
4
 
5
  namespace SceneAgent
Runtime/SceneGrammar.Static.cs CHANGED
@@ -1,4 +1,4 @@
1
- // GENERATED by `py eval/grammar.py --unity` from spec/tools_v0.json. Do not edit: regenerate.
2
  namespace SceneAgent
3
  {
4
  public static partial class SceneGrammar
 
1
+ // Generated from the model's tool schema. Do not edit.
2
  namespace SceneAgent
3
  {
4
  public static partial class SceneGrammar
Runtime/SceneGrammar.cs CHANGED
@@ -1,4 +1,4 @@
1
- // Grammar-constrained decoding (docs/v3_data_plan.md §7.5, decision D5): the same GBNF grammar as eval/grammar.py.
2
  // The static part (the 11 tools, their arguments in the trained order, the <tool_call> format, plain text that cannot
3
  // start a half-written call) is generated into SceneGrammar.Static.cs. This file adds the per-request rules: the ids
4
  // the model may name (the scene's objects, the player's hands and gaze, and every id an earlier tool result returned,
 
1
+ // Grammar-constrained decoding: a GBNF grammar that only allows valid tool calls or plain text.
2
  // The static part (the 11 tools, their arguments in the trained order, the <tool_call> format, plain text that cannot
3
  // start a half-written call) is generated into SceneGrammar.Static.cs. This file adds the per-request rules: the ids
4
  // the model may name (the scene's objects, the player's hands and gaze, and every id an earlier tool result returned,
Runtime/SceneMath.cs ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Rounding and number formatting exactly as in the model's training data (Python's round(x, n) and f"{x:.nf}"), so
2
+ // the scene text and the tool effects match it byte for byte. Python rounds the exact binary value of x, ties to even;
3
+ // .NET's Math.Round(x, n) scales by 10^n first, so 0.45 (binary 0.45000000000000001...) became 0.4 where the training
4
+ // scenes say 0.5, and -0.04 became 0.0 where they say -0.0.
5
+ using System;
6
+ using System.Globalization;
7
+ using System.Numerics;
8
+
9
+ namespace SceneAgent
10
+ {
11
+ public static class SceneMath
12
+ {
13
+ /// <summary>Python round(x, n) for n >= 0: exact, ties to even, sign kept (round(-0.04, 1) == -0.0).</summary>
14
+ public static double Round(double x, int n)
15
+ {
16
+ if (double.IsNaN(x) || double.IsInfinity(x)) return x;
17
+ long bits = BitConverter.DoubleToInt64Bits(x);
18
+ bool neg = bits < 0;
19
+ int e = (int)((bits >> 52) & 0x7FF);
20
+ long m = bits & 0xFFFFFFFFFFFFFL;
21
+ if (e == 0) e = 1; else m |= 1L << 52;
22
+ e -= 1075; // |x| = m * 2^e exactly
23
+ BigInteger num = new BigInteger(m) * BigInteger.Pow(10, n), den = BigInteger.One;
24
+ if (e > 0) num <<= e; else den <<= -e;
25
+ BigInteger q = BigInteger.DivRem(num, den, out BigInteger r);
26
+ int c = (r * 2).CompareTo(den);
27
+ if (c > 0 || (c == 0 && !q.IsEven)) q += 1;
28
+ double v = (double)q / Math.Pow(10, n); // q and 10^n are exact doubles: the division is correctly rounded
29
+ return neg ? -v : v;
30
+ }
31
+
32
+ /// <summary>Python f"{x:.{n}f}".</summary>
33
+ public static string Fixed(double x, int n) => Round(x, n).ToString("F" + n, CultureInfo.InvariantCulture);
34
+ }
35
+ }
Runtime/SceneMath.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: aa67273f40b6679448917c8a06ebe741
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/SceneWorld.cs CHANGED
@@ -1,5 +1,5 @@
1
- // Scene state + tool semantics on real GameObjects. Mirrors datagen/sim.py (and
2
- // eval/unity/SceneToolEval.cs) so a model trained on the generator behaves identically here.
3
  using System;
4
  using System.Collections.Generic;
5
  using System.Globalization;
@@ -21,19 +21,19 @@ namespace SceneAgent
21
  public SortedSet<string> clarify = new SortedSet<string>(StringComparer.Ordinal);
22
  public readonly SortedDictionary<string, double> pressed = new SortedDictionary<string, double>(StringComparer.Ordinal);
23
  public string lastClarifyQuestion;
24
- /// <summary>Where a room sits in the Unity scene (rooms side by side in a demo). The model only ever sees
25
- /// room-local positions; null = every room at the origin (evaluation scenes).</summary>
26
  [NonSerialized] public Func<string, Vector3> roomOrigin;
27
  [Tooltip("On: at Start every SceneItem's logical position and rotation are read from where it stands in the scene, " +
28
  "and the rooms and the player's room are filled in from the items when empty. Leave off when the scene " +
29
- "is built from data (the evaluation harness, the demo builder).")]
30
  public bool initFromTransforms;
31
  /// <summary>Raised after every tool call Unity accepted: (tool, arguments, result). Hook animations, sounds or the
32
  /// camera here, e.g. move the player rig when "teleport" changes playerRoom.</summary>
33
  public event Action<string, Dictionary<string, object>, object> ToolExecuted;
34
 
35
- // affordances per label: generated from the Python simulator + the v3 catalogue (SceneCatalog.Generated.cs,
36
- // `py tools/gen_unity_catalog.py`), so Unity accepts exactly the calls the offline scorer accepts
37
  static Dictionary<string, double> Top => SceneCatalog.Top;
38
  static HashSet<string> Grasp => SceneCatalog.Grasp;
39
  static HashSet<string> Container => SceneCatalog.Container;
@@ -70,17 +70,17 @@ namespace SceneAgent
70
  if (string.IsNullOrEmpty(o.itemId) || !o.hasPos || HandSlots.Contains(o.on)) continue;
71
  var tr = o.transform;
72
  var p = tr.position - (roomOrigin != null ? roomOrigin(o.room) : Vector3.zero);
73
- o.pos = new[] { PyMath.Round(p.x, 3), PyMath.Round(p.y - tr.localScale.y / 2f, 3), PyMath.Round(p.z, 3) };
74
  var e = tr.rotation.eulerAngles;
75
- o.yaw = PyMath.Round(e.y, 3); o.pitch = PyMath.Round(e.x, 3); o.roll = PyMath.Round(e.z, 3);
76
  }
77
  if (rooms.Count == 0) rooms = Items.Select(o => o.room).Where(r => !string.IsNullOrEmpty(r)).Distinct().ToList();
78
  if (string.IsNullOrEmpty(playerRoom) && rooms.Count > 0) playerRoom = rooms[0];
79
  }
80
 
81
  // Held items are parented to the hand anchors while they are shown in the hand; when the anchors live outside
82
- // this hierarchy (the demo's camera rig) they must still count as scene items, or every tool call on a held
83
- // item fails with "unknown object" (found by the v2 demo autotest: "put it on the counter" after a grab).
84
  public IEnumerable<SceneItem> Items
85
  {
86
  get
@@ -105,16 +105,15 @@ namespace SceneAgent
105
  public string BuildSceneJson()
106
  {
107
  var vis = new List<object>();
108
- // hierarchy order, not sorted: training scenes list objects in random order. The model is robust to the
109
- // order (runs/order_probe: id-sorted -1.7, another shuffle -1.0 points on OOD, neither significant), and
110
- // hierarchy order lets the Unity test send exactly the offline prompts
111
  foreach (var o in Items.Where(x => x.room == playerRoom))
112
  {
113
  var v = new Dictionary<string, object> { { "id", o.itemId }, { "label", o.label } };
114
  if (!string.IsNullOrEmpty(o.color)) v["color"] = o.color;
115
  v["on"] = o.on;
116
  if (!string.IsNullOrEmpty(o.state)) v["state"] = o.state;
117
- if (o.hasPos) v["pos"] = o.pos.Select(c => (object)PyMath.Round(c, 1)).ToList();
118
  vis.Add(v);
119
  }
120
  var scene = new Dictionary<string, object> {
@@ -139,7 +138,7 @@ namespace SceneAgent
139
 
140
  void MoveTree(SceneItem o, double[] d)
141
  {
142
- for (int i = 0; i < 3; i++) o.pos[i] = PyMath.Round(o.pos[i] + d[i], 3);
143
  foreach (var c in Items) if (c.on == o.itemId && c.hasPos) MoveTree(c, d);
144
  }
145
 
@@ -193,7 +192,7 @@ namespace SceneAgent
193
  FreeHand(o);
194
  double top = rel == "on" && TryTop(t, out var th) ? th : 0.0;
195
  o.on = t.itemId; o.relation = rel; o.orientation = orient; o.hasPos = true;
196
- o.pos = new[] { t.pos[0], PyMath.Round(t.pos[1] + top, 3), t.pos[2] };
197
  Sync(o); return ok;
198
  }
199
  case "move_by":
@@ -213,7 +212,7 @@ namespace SceneAgent
213
  var o = Get(S(a, "object_id"));
214
  if (HandSlots.Contains(o.on) || IsFixed(o)) throw new ToolError($"'{o.itemId}' cannot be rotated");
215
  string axis = S(a, "axis") ?? "yaw";
216
- double Wrap(double v) => PyMath.Round(((v % 360) + 360) % 360, 3);
217
  double deg = D(a, "degrees");
218
  if (axis == "yaw") o.yaw = Wrap(o.yaw + deg);
219
  else if (axis == "pitch") o.pitch = Wrap(o.pitch + deg);
@@ -251,7 +250,7 @@ namespace SceneAgent
251
  case "find_objects":
252
  {
253
  string label = S(a, "label"), color = S(a, "color"), room = S(a, "room");
254
- // hierarchy order, like the simulator's world order that produced the training tool results
255
  return Items.Where(o => o.label == label && (color == null || o.color == color) && (room == null || o.room == room))
256
  .Select(o =>
257
  {
@@ -297,7 +296,7 @@ namespace SceneAgent
297
  static string Num(double v, int digits)
298
  {
299
  if (Math.Abs(v) < 0.5 * Math.Pow(10, -digits)) v = 0.0;
300
- return PyMath.Fixed(v, digits);
301
  }
302
 
303
  public string Signature()
 
1
+ // Scene state + tool semantics on real GameObjects. The rules match the simulator the model was trained on, so the
2
+ // model's calls behave here exactly as they did in training.
3
  using System;
4
  using System.Collections.Generic;
5
  using System.Globalization;
 
21
  public SortedSet<string> clarify = new SortedSet<string>(StringComparer.Ordinal);
22
  public readonly SortedDictionary<string, double> pressed = new SortedDictionary<string, double>(StringComparer.Ordinal);
23
  public string lastClarifyQuestion;
24
+ /// <summary>Where a room sits in the Unity scene (for example, rooms side by side). The model only ever sees
25
+ /// room-local positions; null = every room at the origin.</summary>
26
  [NonSerialized] public Func<string, Vector3> roomOrigin;
27
  [Tooltip("On: at Start every SceneItem's logical position and rotation are read from where it stands in the scene, " +
28
  "and the rooms and the player's room are filled in from the items when empty. Leave off when the scene " +
29
+ "is built from data by your own code.")]
30
  public bool initFromTransforms;
31
  /// <summary>Raised after every tool call Unity accepted: (tool, arguments, result). Hook animations, sounds or the
32
  /// camera here, e.g. move the player rig when "teleport" changes playerRoom.</summary>
33
  public event Action<string, Dictionary<string, object>, object> ToolExecuted;
34
 
35
+ // affordances per label (SceneCatalog.Generated.cs): the same catalogue the model was trained on, so Unity
36
+ // accepts exactly the calls the model learned to make
37
  static Dictionary<string, double> Top => SceneCatalog.Top;
38
  static HashSet<string> Grasp => SceneCatalog.Grasp;
39
  static HashSet<string> Container => SceneCatalog.Container;
 
70
  if (string.IsNullOrEmpty(o.itemId) || !o.hasPos || HandSlots.Contains(o.on)) continue;
71
  var tr = o.transform;
72
  var p = tr.position - (roomOrigin != null ? roomOrigin(o.room) : Vector3.zero);
73
+ o.pos = new[] { SceneMath.Round(p.x, 3), SceneMath.Round(p.y - tr.localScale.y / 2f, 3), SceneMath.Round(p.z, 3) };
74
  var e = tr.rotation.eulerAngles;
75
+ o.yaw = SceneMath.Round(e.y, 3); o.pitch = SceneMath.Round(e.x, 3); o.roll = SceneMath.Round(e.z, 3);
76
  }
77
  if (rooms.Count == 0) rooms = Items.Select(o => o.room).Where(r => !string.IsNullOrEmpty(r)).Distinct().ToList();
78
  if (string.IsNullOrEmpty(playerRoom) && rooms.Count > 0) playerRoom = rooms[0];
79
  }
80
 
81
  // Held items are parented to the hand anchors while they are shown in the hand; when the anchors live outside
82
+ // this hierarchy (for example, on the camera rig) they must still count as scene items, or every tool call on a
83
+ // held item fails with "unknown object" ("put it on the counter" after a grab).
84
  public IEnumerable<SceneItem> Items
85
  {
86
  get
 
105
  public string BuildSceneJson()
106
  {
107
  var vis = new List<object>();
108
+ // hierarchy order, not sorted: training scenes list objects in random order, and the model does not depend
109
+ // on the order
 
110
  foreach (var o in Items.Where(x => x.room == playerRoom))
111
  {
112
  var v = new Dictionary<string, object> { { "id", o.itemId }, { "label", o.label } };
113
  if (!string.IsNullOrEmpty(o.color)) v["color"] = o.color;
114
  v["on"] = o.on;
115
  if (!string.IsNullOrEmpty(o.state)) v["state"] = o.state;
116
+ if (o.hasPos) v["pos"] = o.pos.Select(c => (object)SceneMath.Round(c, 1)).ToList();
117
  vis.Add(v);
118
  }
119
  var scene = new Dictionary<string, object> {
 
138
 
139
  void MoveTree(SceneItem o, double[] d)
140
  {
141
+ for (int i = 0; i < 3; i++) o.pos[i] = SceneMath.Round(o.pos[i] + d[i], 3);
142
  foreach (var c in Items) if (c.on == o.itemId && c.hasPos) MoveTree(c, d);
143
  }
144
 
 
192
  FreeHand(o);
193
  double top = rel == "on" && TryTop(t, out var th) ? th : 0.0;
194
  o.on = t.itemId; o.relation = rel; o.orientation = orient; o.hasPos = true;
195
+ o.pos = new[] { t.pos[0], SceneMath.Round(t.pos[1] + top, 3), t.pos[2] };
196
  Sync(o); return ok;
197
  }
198
  case "move_by":
 
212
  var o = Get(S(a, "object_id"));
213
  if (HandSlots.Contains(o.on) || IsFixed(o)) throw new ToolError($"'{o.itemId}' cannot be rotated");
214
  string axis = S(a, "axis") ?? "yaw";
215
+ double Wrap(double v) => SceneMath.Round(((v % 360) + 360) % 360, 3);
216
  double deg = D(a, "degrees");
217
  if (axis == "yaw") o.yaw = Wrap(o.yaw + deg);
218
  else if (axis == "pitch") o.pitch = Wrap(o.pitch + deg);
 
250
  case "find_objects":
251
  {
252
  string label = S(a, "label"), color = S(a, "color"), room = S(a, "room");
253
+ // hierarchy order, like the training simulator's world order
254
  return Items.Where(o => o.label == label && (color == null || o.color == color) && (room == null || o.room == room))
255
  .Select(o =>
256
  {
 
296
  static string Num(double v, int digits)
297
  {
298
  if (Math.Abs(v) < 0.5 * Math.Pow(10, -digits)) v = 0.0;
299
+ return SceneMath.Fixed(v, digits);
300
  }
301
 
302
  public string Signature()
package.json CHANGED
@@ -1,26 +1,25 @@
1
- {
2
- "name": "com.mcbuxrlab.maverick",
3
- "version": "1.0.0-rc.2",
4
- "displayName": "Maverick XR Agent",
5
- "unity": "2022.3",
6
- "description": "Control a Unity scene in plain English with Maverick-4B-Unity-XR-Agent, a small language model that runs offline (llama.cpp). The model reads your interactive objects, calls 11 scene tools (grab, place, move, rotate, set state, highlight, teleport, find, ask which one, ...), asks when a request is ambiguous and declines what it cannot do. Includes a local model server, an on-screen chat box, a command-script runner and editor tools to set up a scene. Model: https://huggingface.co/ErenAta00/Maverick-4B-Unity-XR-Agent-GGUF",
7
- "keywords": [
8
- "maverick",
9
- "llm",
10
- "language model",
11
- "voice",
12
- "natural language",
13
- "xr",
14
- "vr",
15
- "offline",
16
- "llama.cpp",
17
- "agent",
18
- "tool calling"
19
- ],
20
- "license": "MIT",
21
- "author": {
22
- "name": "Eren Ata, MCBU XRLab",
23
- "url": "https://huggingface.co/ErenAta00"
24
- },
25
- "documentationUrl": "https://huggingface.co/ErenAta00/Maverick-Unity"
26
- }
 
1
+ {
2
+ "name": "com.mcbuxrlab.maverick",
3
+ "version": "1.0.0",
4
+ "displayName": "Maverick XR Agent",
5
+ "unity": "2022.3",
6
+ "description": "Control a Unity scene with typed or spoken English. Maverick-4B-Unity-XR-Agent runs locally through llama.cpp, reads the objects you mark as interactive and answers with tool calls that Unity carries out: grab, place, move, rotate, switch on or off, highlight, teleport, find, and ask which object you mean. Model: https://huggingface.co/ErenAta00/Maverick-4B-Unity-XR-Agent-GGUF",
7
+ "keywords": [
8
+ "maverick",
9
+ "llm",
10
+ "voice control",
11
+ "natural language",
12
+ "xr",
13
+ "vr",
14
+ "offline",
15
+ "llama.cpp",
16
+ "agent",
17
+ "tool calling"
18
+ ],
19
+ "license": "MIT",
20
+ "author": {
21
+ "name": "Eren Ata, MCBU XRLab",
22
+ "url": "https://huggingface.co/ErenAta00"
23
+ },
24
+ "documentationUrl": "https://huggingface.co/ErenAta00/Maverick-Unity"
25
+ }