ErenAta00 commited on
Commit
34bec90
·
verified ·
1 Parent(s): 1d2d57d

Maverick XR Agent 1.0.0-rc.2 (Unity package)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. CHANGELOG.md +44 -0
  2. CHANGELOG.md.meta +7 -0
  3. Documentation~/SceneAgent.md +98 -0
  4. Editor.meta +8 -0
  5. Editor/AgentE2ETest.cs +94 -0
  6. Editor/AgentE2ETest.cs.meta +11 -0
  7. Editor/SceneAgent.Editor.asmdef +7 -0
  8. Editor/SceneAgent.Editor.asmdef.meta +7 -0
  9. Editor/SceneAgentMenu.cs +260 -0
  10. Editor/SceneAgentMenu.cs.meta +11 -0
  11. LICENSE.md +21 -0
  12. LICENSE.md.meta +7 -0
  13. README.md.meta +7 -0
  14. Runtime.meta +8 -0
  15. Runtime/LocalModelServer.cs +248 -0
  16. Runtime/LocalModelServer.cs.meta +11 -0
  17. Runtime/MiniJson.cs +150 -0
  18. Runtime/MiniJson.cs.meta +11 -0
  19. Runtime/PyMath.cs +35 -0
  20. Runtime/PyMath.cs.meta +11 -0
  21. Runtime/Resources.meta +8 -0
  22. Runtime/Resources/SceneAgent.meta +8 -0
  23. Runtime/Resources/SceneAgent/system_prompt_v3.txt +16 -0
  24. Runtime/Resources/SceneAgent/system_prompt_v3.txt.meta +7 -0
  25. Runtime/SceneAgent.Runtime.asmdef +6 -0
  26. Runtime/SceneAgent.Runtime.asmdef.meta +7 -0
  27. Runtime/SceneAgent.cs +255 -0
  28. Runtime/SceneAgent.cs.meta +11 -0
  29. Runtime/SceneAgentChatBox.cs +99 -0
  30. Runtime/SceneAgentChatBox.cs.meta +11 -0
  31. Runtime/SceneAgentGaze.cs +47 -0
  32. Runtime/SceneAgentGaze.cs.meta +11 -0
  33. Runtime/SceneAgentScriptRunner.cs +114 -0
  34. Runtime/SceneAgentScriptRunner.cs.meta +11 -0
  35. Runtime/SceneAgentSession.cs +187 -0
  36. Runtime/SceneAgentSession.cs.meta +11 -0
  37. Runtime/SceneCatalog.Generated.cs +62 -0
  38. Runtime/SceneCatalog.Generated.cs.meta +11 -0
  39. Runtime/SceneGrammar.Check.cs +91 -0
  40. Runtime/SceneGrammar.Check.cs.meta +11 -0
  41. Runtime/SceneGrammar.Schema.cs +24 -0
  42. Runtime/SceneGrammar.Schema.cs.meta +11 -0
  43. Runtime/SceneGrammar.Static.cs +8 -0
  44. Runtime/SceneGrammar.Static.cs.meta +11 -0
  45. Runtime/SceneGrammar.cs +48 -0
  46. Runtime/SceneGrammar.cs.meta +11 -0
  47. Runtime/SceneItem.cs +36 -0
  48. Runtime/SceneItem.cs.meta +11 -0
  49. Runtime/SceneWorld.cs +319 -0
  50. Runtime/SceneWorld.cs.meta +11 -0
CHANGELOG.md ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Changelog
2
+
3
+ ## [1.0.0-rc.2] - 2026-09-28 (not published)
4
+ ### Changed
5
+ - Renamed to **Maverick XR Agent** (`com.mcbuxrlab.maverick`) for the Maverick 4B model; the C# namespaces and
6
+ component names are unchanged.
7
+ - Install from Git: `https://huggingface.co/ErenAta00/Maverick-Unity.git`.
8
+ - Licence holder: Eren Ata, MCBU XRLab (MIT).
9
+
10
+ ## [1.0.0-rc.1] - 2026-09-27 (not published)
11
+ ### Added
12
+ - `LocalModelServer` in the package (was in the demo): finds llama-server and the `.gguf` in
13
+ `StreamingAssets/SceneAgent`, GPU first with CPU fallback, `capFrameRate` while the model runs on the GPU,
14
+ `ModeChanged` event.
15
+ - `SceneAgentSession`: commands, clarification answers with the trained follow-up, optional JSON-lines session log.
16
+ - `SceneAgentChatBox`: on-screen chat to try the agent without writing UI.
17
+ - `SceneAgentScriptRunner`: plays command scripts and writes a report (Editor, build, or command line).
18
+ - D5b answer checking (`SceneGrammar.Check`, `SceneGrammar.Schema`), on by default, verified against the evaluation's
19
+ Python check on 59,294 cases.
20
+ - Editor menu Tools → Scene Agent: Set Up Scene, Make Selected Interactive, Validate Scene, Open Model Folder.
21
+ - `SceneWorld.initFromTransforms` (positions read from the scene at Start), `SceneWorld.ToolExecuted` event.
22
+ - `SceneItem` affordance overrides for labels outside the catalogue.
23
+ - The v3 system prompt ships in `Resources/SceneAgent`.
24
+ - README, reference documentation, MIT license, third-party notices.
25
+ - `SceneAgentGaze`: "this / that" from the mouse pointer or the camera centre.
26
+ - Typed clarification answers ("the red one") in `SceneAgentSession`.
27
+ - Substitute guard (`SceneAgent.guardSubstitutes`).
28
+ - The model server is tied to the app's lifetime on Windows (a job object: no llama-server left after a crash) and
29
+ waits for the old server to exit on a scene reload.
30
+ - `-sceneAgentScript` works in any scene with a session.
31
+ - Readable tool lines in the chat (`place(hammer_1, toolbox_1, in) ok`); raw JSON with `SceneAgentSession.rawToolLines`.
32
+ - `SceneAgentScriptRunner` gives the gaze back to `SceneAgentGaze` when the script ends.
33
+ ### Fixed
34
+ - Play mode in the Editor: an unassigned `systemPromptAsset` threw `UnassignedReferenceException` on the first command.
35
+ - Make Selected Interactive: duplicated objects (Ctrl+D) get a fresh id; nested objects move directly under the Scene
36
+ Agent object and their supports are inferred from their own renderers; devices get a default state.
37
+ - Validate Scene: a model path pointing nowhere is an error; no dialogs in batch mode.
38
+ - A context overflow gives a readable message instead of the raw HTTP 400.
39
+ ### Changed
40
+ - `SceneAgent` defaults: D5b on, strict D5 off; waits for its `LocalModelServer` when one is set.
41
+
42
+ ## [0.1.0]
43
+ - SceneWorld (tool semantics mirroring the training simulator), SceneAgent (chat loop), strict D5 grammar, the Unity
44
+ E2E harness.
CHANGELOG.md.meta ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 88b114c1c231c3b4d856afc169a4b9d9
3
+ TextScriptImporter:
4
+ externalObjects: {}
5
+ userData:
6
+ assetBundleName:
7
+ assetBundleVariant:
Documentation~/SceneAgent.md ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Scene Agent: reference
2
+
3
+ Start with the package `README.md` (install, quick start). This page lists what each part does and how the model sees
4
+ your scene.
5
+
6
+ ## How a command runs
7
+ 1. `SceneWorld.BuildSceneJson()` lists the player's room: every `SceneItem` there, the room names, the player's hands
8
+ and gaze.
9
+ 2. `SceneAgent.Run` sends the system prompt, the scene and the request to the local model (OpenAI-compatible
10
+ `/v1/chat/completions` on llama.cpp, temperature 0).
11
+ 3. The answer is either text (done) or tool calls. With D5b (default) each answer is checked first (below).
12
+ 4. Each call runs through `SceneWorld.Execute`, which changes the logical state, moves the GameObject
13
+ (`SceneWorld.Sync`) and raises `ToolExecuted`. A call Unity refuses (`ToolError`) goes back to the model as
14
+ `{"ok": false, "error": "…"}`.
15
+ 5. Results go back to the model; up to `maxTurns` (4) rounds. `ask_clarification` ends the turn with a question; the
16
+ user's pick comes back as the next request (`SceneAgentSession.Answer` builds it).
17
+
18
+ ## What the model sees
19
+ ```json
20
+ {"player": {"room": "kitchen", "gaze": null, "left_hand": null, "right_hand": "mug_1"},
21
+ "locations": ["kitchen", "living_room", "office"],
22
+ "visible_objects": [
23
+ {"id": "counter_1", "label": "counter", "on": "floor", "pos": [-1.6, 0.0, 1.2]},
24
+ {"id": "mug_2", "label": "mug", "color": "red", "on": "counter_1", "pos": [-1.6, 0.9, 1.2]},
25
+ {"id": "fridge_1", "label": "fridge", "color": "white", "on": "floor", "state": "closed", "pos": [1.9, 0.0, 2.4]}]}
26
+ ```
27
+ Positions are room-local metres of the object's bottom, rounded to 0.1 m. Only the player's room is listed.
28
+
29
+ ## Components
30
+ | component | fields you may set |
31
+ |---|---|
32
+ | `SceneWorld` | `rooms`, `playerRoom`, `gaze`, `leftHand` / `rightHand`, `leftHandAnchor` / `rightHandAnchor` (where held objects are shown), `initFromTransforms` (read positions from the scene at Start), `roomOrigin` (code: where each room sits in world space) |
33
+ | `SceneItem` | `itemId` (unique, e.g. `mug_1`), `label` (plain noun), `color`, `room`, `on` (support id or `floor`), `state` (e.g. `off`, `closed`), affordance override flags |
34
+ | `SceneAgent` | `world`, `server`, `endpoint`, `systemPromptAsset` (leave empty), `maxTurns`, `fallbackDecoding` (D5b, on), `constrainedDecoding` (strict D5, off) |
35
+ | `LocalModelServer` | `serverExe`, `modelPath`, `port`, `contextSize`, `preferGpu`, `cpuThreads`, `capFrameRate`, `launchOnStart`, `extraArgs` |
36
+ | `SceneAgentSession` | `agent`, `world`, `logDirectory`; `Command(text)`, `Answer(itemId)`, events `Line`, `Finished`, `PendingRequest`, `Candidates` |
37
+ | `SceneAgentChatBox` | `visibleLines`, `toggleKey` (F1), `examples` |
38
+ | `SceneAgentScriptRunner` | `script`, `outputDirectory`, `screenshots`, `quitWhenDone`; command line `-sceneAgentScript`, `-sceneAgentOut`, `-sceneAgentShots` |
39
+
40
+ ## Tools
41
+ | tool | arguments | Unity refuses when |
42
+ |---|---|---|
43
+ | `grab` | object_id, hand (left/right/both) | not graspable, already held, hand not free |
44
+ | `release` | hand | nothing in that hand (the object drops to the floor) |
45
+ | `place` | object_id, target_id, relation (on/in), orientation (default/upright/horizontal/vertical/upside_down) | object not carryable; target not a surface (on) or not a container / closed (in) |
46
+ | `move_by` | object_id, direction (left/right/forward/backward/up/down), distance, unit (m/cm) | held or fixed object |
47
+ | `rotate` | object_id, degrees, axis (yaw/pitch/roll) | held or fixed object |
48
+ | `set_state` | object_id, state (on/off/open/closed/half_open/locked/unlocked) | the object has no such state |
49
+ | `press` | object_id, hold_seconds | not a button |
50
+ | `highlight` | object_ids | an id not in the room |
51
+ | `teleport` | location_id | not one of `rooms` (held objects come along) |
52
+ | `find_objects` | label, color, room | never; returns the matches in every room |
53
+ | `ask_clarification` | question, candidate_ids | never; sets `SceneWorld.clarify` and ends the turn |
54
+
55
+ Every tool acts on the player's room only: an object elsewhere is "not visible" until `teleport`.
56
+
57
+ ## Affordances
58
+ `SceneCatalog` (generated from the training simulator) maps about 200 labels to what they afford: grasp, surface
59
+ height, container, fixed, states, button. `SceneItem.overrideAffordances` replaces the lookup for one item with its
60
+ own flags (`grabbable`, `isSurface` + `surfaceHeight`, `isContainer`, `isFixed`, `isButton`, `states`). The model
61
+ learned the catalogue's common sense; overrides that contradict it (a grabbable "table") will confuse it.
62
+
63
+ ## D5b: checking the model's answer
64
+ With `fallbackDecoding` on, each answer is checked against the tool schema and the scene (`SceneGrammar.Check`,
65
+ byte-for-byte the same verdicts as the evaluation's Python check, 59,294 cases):
66
+ - `ok` → executed as it is.
67
+ - `invalid_tool` (a tool that does not exist) or `invalid_value` (an argument or value outside the schema, or a room
68
+ not in `rooms`) → replaced by the refusal "I can't do that; none of my tools can."
69
+ - `invalid_id` (an id not in the scene, the hands, the gaze or earlier tool results) or `malformed` (a call that does
70
+ not parse) → generated again with a grammar that only allows valid calls.
71
+ Each turn's verdict is in `AgentTurn.d5b`. Strict D5 (`constrainedDecoding`) applies the grammar to every answer; it
72
+ was measured to force actions where the model should refuse, so it is off.
73
+
74
+ ## Substitute guard
75
+ `SceneAgent.guardSubstitutes` (on): before an acting call (grab, place, move_by, rotate, set_state, press) runs, if the
76
+ acted-on object is not named in the request while the request names an object that exists only in other rooms, the
77
+ call is skipped and the model receives `{"ok": false, "error": "not done: 'screwdriver_1' is a screwdriver, but the request
78
+ names a drill; the drill (drill_1) is in the storage, not in this room"}`. Naming uses the label or its last word
79
+ ("desk lamp" is named by "lamp"), with plural endings. It never fires for pronouns ("put it there").
80
+
81
+ ## Events
82
+ - `SceneWorld.ToolExecuted (string tool, Dictionary<string, object> args, object result)`: after every accepted call.
83
+ - `LocalModelServer.ModeChanged (Mode)`: Starting, Gpu, Cpu, External, Failed, Stopped.
84
+ - `SceneAgentSession.Line (string)` and `Finished (AgentResult)`.
85
+
86
+ ## Script format (SceneAgentScriptRunner)
87
+ ```
88
+ // comment
89
+ pick up the red mug a command
90
+ #answer mug_1 answer the pending "which one?" with this id
91
+ #look lamp_2 set the gaze ("#look none" clears it)
92
+ #wait 2.5 pause, in seconds (not counted in any step's time)
93
+ ```
94
+
95
+ ## Changing the model
96
+ Any GGUF of this model family works through `LocalModelServer.modelPath`. The system prompt in
97
+ `Resources/SceneAgent/system_prompt_v3.txt` must be the one the model was trained with; a different model needs its
98
+ own prompt and, for D5b, the same tool schema (`SceneGrammar.Schema.cs`, generated from `spec/tools_v0.json`).
Editor.meta ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 4209fd575d4361a4c85651d5cd0638d2
3
+ folderAsset: yes
4
+ DefaultImporter:
5
+ externalObjects: {}
6
+ userData:
7
+ assetBundleName:
8
+ assetBundleVariant:
Editor/AgentE2ETest.cs ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // End-to-end test: build real GameObjects for each case, let SceneAgent serialise the scene, query the
2
+ // model server, execute the calls on the GameObjects, and compare the final state with gold.
3
+ // Unity -batchmode -nographics -projectPath <proj> -executeMethod AgentE2ETest.Run
4
+ // -e2eInput cases.json -e2eOutput report.json
5
+ using System;
6
+ using System.Collections.Generic;
7
+ using System.IO;
8
+ using System.Linq;
9
+ using UnityEditor;
10
+ using UnityEngine;
11
+ using SceneAgent;
12
+
13
+ [Serializable] public class E2EObj { public string id, label, color, room, on, relation, orientation, state; public bool has_pos; public double[] pos; public double yaw, pitch, roll; }
14
+ [Serializable] public class E2EWorld { public string[] rooms; public E2EObj[] objects; public string room, gaze, left_hand, right_hand; }
15
+ [Serializable] public class E2ECase { public string id; public string category; public string utterance; public E2EWorld world; public string gold_sig; public bool gold_text_end; }
16
+ [Serializable] public class E2ESuite { public string endpoint; public string model; public string prompt_path; public bool constrained; public bool fallback; public E2ECase[] cases; }
17
+ [Serializable] public class E2EResult { public string id, category, error, final_text, clarification, first_output, sig, scene, d5b; public bool ok; public int turns; public double seconds; }
18
+ [Serializable] public class E2EReport { public string unity; public int n; public double success_rate; public double mean_seconds; public List<E2EResult> results = new List<E2EResult>(); }
19
+
20
+ public static class AgentE2ETest
21
+ {
22
+ static Vector3 SizeFor(string label)
23
+ {
24
+ switch (label)
25
+ {
26
+ case "table": case "desk": return new Vector3(1.2f, 0.75f, 0.6f);
27
+ case "counter": return new Vector3(1.4f, 0.9f, 0.6f);
28
+ case "shelf": return new Vector3(1.0f, 1.2f, 0.35f);
29
+ case "door": return new Vector3(0.9f, 2.0f, 0.05f);
30
+ case "window": return new Vector3(1.0f, 1.0f, 0.05f);
31
+ case "chair": case "stool": case "crate": case "chest": return new Vector3(0.5f, 0.5f, 0.5f);
32
+ case "box": case "basket": case "vase": case "drawer": case "tv": case "fan": case "lamp": return new Vector3(0.4f, 0.4f, 0.4f);
33
+ default: return Vector3.one * 0.15f;
34
+ }
35
+ }
36
+
37
+ static string Nn(string s) => string.IsNullOrEmpty(s) ? null : s;
38
+
39
+ public static void Run()
40
+ {
41
+ var args = Environment.GetCommandLineArgs();
42
+ string input = null, output = null;
43
+ for (int i = 0; i < args.Length - 1; i++)
44
+ {
45
+ if (args[i] == "-e2eInput") input = args[i + 1];
46
+ if (args[i] == "-e2eOutput") output = args[i + 1];
47
+ }
48
+ var suite = JsonUtility.FromJson<E2ESuite>(File.ReadAllText(input));
49
+ string prompt = File.ReadAllText(suite.prompt_path).Replace("\r\n", "\n");
50
+ var report = new E2EReport { unity = Application.unityVersion, n = suite.cases.Length };
51
+ foreach (var cs in suite.cases)
52
+ {
53
+ var root = new GameObject("SceneRoot");
54
+ var world = root.AddComponent<SceneWorld>();
55
+ var agent = root.AddComponent<SceneAgent.SceneAgent>();
56
+ agent.endpoint = suite.endpoint; agent.model = suite.model; agent.systemPrompt = prompt; agent.world = world;
57
+ agent.constrainedDecoding = suite.constrained; // D5, off unless the runner asks (--grammar on)
58
+ agent.fallbackDecoding = suite.fallback; // D5b (--grammar d5b)
59
+ world.playerRoom = cs.world.room; world.gaze = Nn(cs.world.gaze);
60
+ world.leftHand = Nn(cs.world.left_hand); world.rightHand = Nn(cs.world.right_hand);
61
+ world.rooms = cs.world.rooms.ToList();
62
+ foreach (var o in cs.world.objects)
63
+ {
64
+ var go = GameObject.CreatePrimitive(PrimitiveType.Cube);
65
+ go.name = o.id;
66
+ go.transform.SetParent(root.transform, false);
67
+ go.transform.localScale = SizeFor(o.label);
68
+ var it = go.AddComponent<SceneItem>();
69
+ it.itemId = o.id; it.label = o.label; it.color = Nn(o.color); it.room = o.room; it.on = o.on;
70
+ it.relation = Nn(o.relation); it.orientation = Nn(o.orientation); it.state = Nn(o.state);
71
+ it.hasPos = o.has_pos; it.pos = (double[])o.pos.Clone(); it.yaw = o.yaw; it.pitch = o.pitch; it.roll = o.roll;
72
+ }
73
+ foreach (var it in world.Items) world.Sync(it);
74
+
75
+ string scene = world.BuildSceneJson(); // what the model is shown; compared with the offline prompt
76
+ var r = agent.RunBlocking(cs.utterance);
77
+ string sig = world.Signature();
78
+ var res = new E2EResult {
79
+ id = cs.id, category = cs.category, error = r.error, final_text = r.finalText, clarification = r.clarificationQuestion, sig = sig, turns = r.turns.Count,
80
+ seconds = r.seconds, first_output = r.turns.Count > 0 ? r.turns[0].modelOutput : null, scene = scene,
81
+ d5b = string.Join(",", r.turns.Select(t => t.d5b ?? "")) };
82
+ res.ok = r.error == null && sig == cs.gold_sig && (!cs.gold_text_end || r.finalText != null);
83
+ report.results.Add(res);
84
+ // one line per command, for the live dashboard (tools/dashboard/v3_view.py reads unity_e2e.log)
85
+ Debug.Log($"[E2E] case {report.results.Count}/{report.n} {res.id} ok={res.ok} turns={res.turns}");
86
+ UnityEngine.Object.DestroyImmediate(root);
87
+ }
88
+ report.success_rate = report.results.Count(x => x.ok) / (double)Math.Max(1, report.n);
89
+ report.mean_seconds = report.results.Average(x => x.seconds);
90
+ File.WriteAllText(output, JsonUtility.ToJson(report, true));
91
+ Debug.Log($"[E2E] n={report.n} success={report.success_rate:F3} mean_s={report.mean_seconds:F2}");
92
+ EditorApplication.Exit(0);
93
+ }
94
+ }
Editor/AgentE2ETest.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 96e9993592c204748af996724da83344
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Editor/SceneAgent.Editor.asmdef ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "SceneAgent.Editor",
3
+ "rootNamespace": "SceneAgent",
4
+ "references": ["SceneAgent.Runtime"],
5
+ "includePlatforms": ["Editor"],
6
+ "autoReferenced": true
7
+ }
Editor/SceneAgent.Editor.asmdef.meta ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 1f3d3340a35b29a4b86a36bb8bf7e845
3
+ AssemblyDefinitionImporter:
4
+ externalObjects: {}
5
+ userData:
6
+ assetBundleName:
7
+ assetBundleVariant:
Editor/SceneAgentMenu.cs ADDED
@@ -0,0 +1,260 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Editor tools for developers: Tools > Scene Agent >
2
+ // Set Up Scene one "Scene Agent" object with SceneWorld, SceneAgent, LocalModelServer, a session and the
3
+ // chat box, wired together (idempotent)
4
+ // Make Selected Interactive SceneItem on every selected object: id, label (and a colour word) from its name, the room,
5
+ // what it stands on (from the renderers' bounds); moved under the Scene Agent object
6
+ // Validate Scene ids, labels the model's catalogue does not know, supports, rooms, model files
7
+ // Open Model Folder StreamingAssets/SceneAgent, where llama-server and the .gguf go
8
+ using System.Collections.Generic;
9
+ using System.IO;
10
+ using System.Linq;
11
+ using System.Text;
12
+ using System.Text.RegularExpressions;
13
+ using UnityEditor;
14
+ using UnityEngine;
15
+
16
+ namespace SceneAgent.EditorTools
17
+ {
18
+ public static class SceneAgentMenu
19
+ {
20
+ const string Root = "Tools/Scene Agent/";
21
+ /// <summary>Automation (tests, build scripts): no modal dialogs, results only in the Console.</summary>
22
+ public static bool Quiet;
23
+ static readonly string[] Colours = { "red", "green", "blue", "yellow", "white", "black", "gray", "grey", "orange",
24
+ "purple", "pink", "brown", "silver", "gold", "beige" };
25
+
26
+ static T Ensure<T>(GameObject g) where T : Component
27
+ {
28
+ var c = g.GetComponent<T>();
29
+ return c != null ? c : Undo.AddComponent<T>(g);
30
+ }
31
+
32
+ [MenuItem(Root + "Set Up Scene", priority = 1)]
33
+ public static void SetUpScene()
34
+ {
35
+ var world = Object.FindObjectOfType<SceneWorld>();
36
+ if (world == null)
37
+ {
38
+ var go = new GameObject("Scene Agent");
39
+ Undo.RegisterCreatedObjectUndo(go, "Set Up Scene Agent");
40
+ world = go.AddComponent<SceneWorld>();
41
+ world.initFromTransforms = true;
42
+ }
43
+ var root = world.gameObject;
44
+ var agent = Ensure<SceneAgent>(root);
45
+ var server = Ensure<LocalModelServer>(root);
46
+ var session = Ensure<SceneAgentSession>(root);
47
+ Ensure<SceneAgentChatBox>(root);
48
+ var gaze = Ensure<SceneAgentGaze>(root);
49
+ Undo.RecordObjects(new Object[] { agent, session, gaze }, "Set Up Scene Agent");
50
+ gaze.world = world;
51
+ gaze.session = session;
52
+ agent.world = world;
53
+ agent.server = server;
54
+ session.agent = agent;
55
+ session.world = world;
56
+ if (string.IsNullOrEmpty(session.logDirectory)) session.logDirectory = "SceneAgentSessions";
57
+ EditorUtility.SetDirty(root);
58
+ Selection.activeGameObject = root;
59
+ Debug.Log("[SceneAgent] Scene set up on \"" + root.name + "\". Next: select the objects the agent may use and run " +
60
+ "Tools > Scene Agent > Make Selected Interactive, then Validate Scene. " + ModelFilesStatus(server));
61
+ }
62
+
63
+ [MenuItem(Root + "Make Selected Interactive", priority = 2)]
64
+ public static void MakeSelectedInteractive()
65
+ {
66
+ var world = Object.FindObjectOfType<SceneWorld>();
67
+ if (world == null) { SetUpScene(); world = Object.FindObjectOfType<SceneWorld>(); }
68
+ string room = !string.IsNullOrEmpty(world.playerRoom) ? world.playerRoom : world.rooms.FirstOrDefault() ?? "room";
69
+ var selected = new HashSet<GameObject>(Selection.gameObjects);
70
+ // ids taken by items that are not being (re)made now; a duplicate (Ctrl+D copies the SceneItem and its id)
71
+ // gets a fresh id below
72
+ var used = new HashSet<string>(world.Items.Where(i => !string.IsNullOrEmpty(i.itemId) && !selected.Contains(i.gameObject)).Select(i => i.itemId));
73
+ var made = new List<SceneItem>();
74
+ foreach (var go in Selection.gameObjects)
75
+ {
76
+ if (go == world.gameObject || go.GetComponent<SceneWorld>() != null) continue;
77
+ var it = Ensure<SceneItem>(go);
78
+ Undo.RecordObject(it, "Make Interactive");
79
+ if (string.IsNullOrEmpty(it.label))
80
+ {
81
+ var (colour, label) = LabelFromName(go.name);
82
+ it.label = label;
83
+ if (string.IsNullOrEmpty(it.color)) it.color = colour;
84
+ }
85
+ if (string.IsNullOrEmpty(it.itemId) || used.Contains(it.itemId)) it.itemId = NextId(it.label, used);
86
+ else used.Add(it.itemId);
87
+ if (string.IsNullOrEmpty(it.state)) it.state = DefaultState(it);
88
+ if (string.IsNullOrEmpty(it.room)) it.room = room;
89
+ // directly under the Scene Agent object: SceneWorld.Sync re-parents items there anyway, and an item nested in
90
+ // another item would make that one's bounds (and so what stands on it) wrong
91
+ if (go.transform.parent != world.transform) Undo.SetTransformParent(go.transform, world.transform, "Make Interactive");
92
+ made.Add(it);
93
+ }
94
+ Undo.RecordObject(world, "Make Interactive");
95
+ foreach (var r in made.Select(i => i.room).Distinct()) if (!world.rooms.Contains(r)) world.rooms.Add(r);
96
+ if (string.IsNullOrEmpty(world.playerRoom)) world.playerRoom = room;
97
+ InferSupports(world);
98
+ var unknown = made.Where(i => !i.overrideAffordances && !KnownLabel(i.label)).Select(i => $"{i.itemId} ('{i.label}')").ToList();
99
+ Debug.Log($"[SceneAgent] {made.Count} interactive: " + string.Join(", ", made.Select(i => $"{i.itemId} on {i.on}")) +
100
+ (unknown.Count > 0 ? $". Not in the model's catalogue (rename to a common word, or tick overrideAffordances): {string.Join(", ", unknown)}" : ""));
101
+ }
102
+
103
+ [MenuItem(Root + "Make Selected Interactive", true)]
104
+ static bool CanMake() => Selection.gameObjects.Length > 0;
105
+
106
+ [MenuItem(Root + "Validate Scene", priority = 3)]
107
+ public static void ValidateScene()
108
+ {
109
+ var sb = new StringBuilder();
110
+ int errors = 0, warnings = 0;
111
+ void E(string m) { errors++; sb.AppendLine("ERROR " + m); }
112
+ void W(string m) { warnings++; sb.AppendLine("warning " + m); }
113
+ var world = Object.FindObjectOfType<SceneWorld>();
114
+ if (world == null)
115
+ {
116
+ Debug.LogWarning("[SceneAgent] Validate Scene: no SceneWorld in the scene: run Tools > Scene Agent > Set Up Scene.");
117
+ if (!Application.isBatchMode && !Quiet) EditorUtility.DisplayDialog("Scene Agent", "No SceneWorld in the scene: run Tools > Scene Agent > Set Up Scene.", "OK");
118
+ return;
119
+ }
120
+ var agent = world.GetComponent<SceneAgent>() != null ? world.GetComponent<SceneAgent>() : Object.FindObjectOfType<SceneAgent>();
121
+ if (agent == null) E("no SceneAgent component");
122
+ else if (agent.world == null) E("SceneAgent.world is not set");
123
+ if (Resources.Load<TextAsset>("SceneAgent/system_prompt_v3") == null && (agent == null || agent.systemPromptAsset == null))
124
+ E("the system prompt is missing (package Resources/SceneAgent/system_prompt_v3.txt)");
125
+ var server = agent != null && agent.server != null ? agent.server : Object.FindObjectOfType<LocalModelServer>();
126
+ if (server == null) W("no LocalModelServer: the agent will call " + (agent != null ? agent.endpoint : "?") + " (start a server yourself)");
127
+ else sb.AppendLine("model " + ModelFilesStatus(server));
128
+ if (server != null)
129
+ {
130
+ string exe = LocalModelServer.Resolve(server.serverExe), model = server.ResolveModel(exe);
131
+ if (exe == null || !File.Exists(exe)) E("llama-server not found: " + exe);
132
+ if (model == null || !File.Exists(model)) E("model file not found: " + (model ?? "no .gguf next to llama-server"));
133
+ }
134
+
135
+ var items = Object.FindObjectsOfType<SceneItem>(true);
136
+ if (items.Length == 0) E("no SceneItem: select objects and run Make Selected Interactive");
137
+ foreach (var g in items.GroupBy(i => i.itemId ?? "").Where(g => g.Count() > 1)) E($"duplicate id '{g.Key}' on {string.Join(", ", g.Select(i => i.name))}");
138
+ var ids = new HashSet<string>(items.Select(i => i.itemId));
139
+ foreach (var it in items)
140
+ {
141
+ string n = $"'{it.name}'";
142
+ if (string.IsNullOrEmpty(it.itemId)) E($"{n} has no id");
143
+ else if (!Regex.IsMatch(it.itemId, "^[a-z0-9_]+$")) W($"{n}: id '{it.itemId}' should look like label_1 (lower case, digits, _)");
144
+ if (string.IsNullOrEmpty(it.label)) E($"{n} has no label");
145
+ else if (!it.overrideAffordances && !KnownLabel(it.label))
146
+ W($"{n}: label '{it.label}' is not in the model's catalogue; Unity will refuse grab/place/set_state on it " +
147
+ "unless you tick overrideAffordances (or use a common word)");
148
+ if (!it.transform.IsChildOf(world.transform)) E($"{n} is not under '{world.name}' (SceneWorld only sees its children)");
149
+ if (string.IsNullOrEmpty(it.state) && DefaultState(it) != null)
150
+ W($"{n}: has states ({string.Join("/", StatesOf(it))}) but none is set; the model will not know whether it is {DefaultState(it)}");
151
+ if (string.IsNullOrEmpty(it.room)) E($"{n} has no room");
152
+ else if (world.rooms.Count > 0 && !world.rooms.Contains(it.room)) E($"{n}: room '{it.room}' is not in SceneWorld.rooms");
153
+ if (!string.IsNullOrEmpty(it.on) && it.on != "floor" && !it.on.EndsWith("_hand") && it.on != "both_hands" && !ids.Contains(it.on))
154
+ E($"{n}: stands on unknown id '{it.on}'");
155
+ }
156
+ foreach (var g in items.GroupBy(i => i.room ?? ""))
157
+ if (g.Count() > 60) W($"room '{g.Key}' has {g.Count()} objects; the model sees one room at a time and 4096 tokens hold about 60 (split the room or raise contextSize)");
158
+ if (!world.initFromTransforms) W("SceneWorld.initFromTransforms is off: positions come from each SceneItem.pos, not from where the objects stand");
159
+
160
+ string head = $"{items.Length} objects, {world.rooms.Count} room(s), {errors} error(s), {warnings} warning(s)";
161
+ Debug.Log("[SceneAgent] Validate Scene: " + head + "\n" + sb);
162
+ if (!Application.isBatchMode && !Quiet)
163
+ EditorUtility.DisplayDialog("Scene Agent · Validate Scene", head + (errors + warnings > 0 ? "\n\nDetails in the Console." : "\n\nReady to press Play."), "OK");
164
+ }
165
+
166
+ [MenuItem(Root + "Open Model Folder", priority = 20)]
167
+ public static void OpenModelFolder()
168
+ {
169
+ string dir = Path.Combine(Application.streamingAssetsPath, "SceneAgent");
170
+ Directory.CreateDirectory(dir);
171
+ AssetDatabase.Refresh();
172
+ EditorUtility.RevealInFinder(dir);
173
+ }
174
+
175
+ // ------------------------------------------------------------------ helpers
176
+ static string ModelFilesStatus(LocalModelServer s)
177
+ {
178
+ string exe = LocalModelServer.Resolve(s.serverExe);
179
+ string model = s.ResolveModel(exe);
180
+ if (exe != null) exe = Path.GetFullPath(exe);
181
+ if (model != null) model = Path.GetFullPath(model);
182
+ bool e = exe != null && File.Exists(exe), m = model != null && File.Exists(model);
183
+ return (e ? $"llama-server: {exe}" : $"llama-server MISSING ({exe})") + " · " +
184
+ (m ? $"model: {Path.GetFileName(model)} ({new FileInfo(model).Length / 1e9:F2} GB)" : "model (.gguf) MISSING") +
185
+ (e && m ? "" : " → Tools > Scene Agent > Open Model Folder, and see the package README 'Model files'.");
186
+ }
187
+
188
+ static string[] StatesOf(SceneItem it) => it.overrideAffordances ? (it.states ?? new string[0])
189
+ : SceneCatalog.States.TryGetValue(it.label ?? "", out var s) ? s : new string[0];
190
+
191
+ /// <summary>The state a device starts in when none is set: off, else closed, else its first state.</summary>
192
+ public static string DefaultState(SceneItem it)
193
+ {
194
+ var s = StatesOf(it);
195
+ return s.Length == 0 ? null : s.Contains("off") ? "off" : s.Contains("closed") ? "closed" : s[0];
196
+ }
197
+
198
+ static readonly HashSet<string> Known = new HashSet<string>(SceneCatalog.Grasp.Concat(SceneCatalog.Top.Keys)
199
+ .Concat(SceneCatalog.Container).Concat(SceneCatalog.Fixed).Concat(SceneCatalog.States.Keys).Concat(SceneCatalog.Button));
200
+
201
+ public static bool KnownLabel(string label) => label != null && Known.Contains(label);
202
+
203
+ /// <summary>"RedMug (2)" → ("red", "mug"); "Floor_Lamp" → ("", "floor lamp").</summary>
204
+ public static (string colour, string label) LabelFromName(string name)
205
+ {
206
+ string s = Regex.Replace(name, @"\s*\(\d+\)\s*$", ""); // "Mug (2)"
207
+ s = Regex.Replace(s, "([a-z])([A-Z])", "$1 $2"); // "RedMug"
208
+ s = Regex.Replace(s, @"[_\-.]+", " ");
209
+ s = Regex.Replace(s, @"\s*\d+$", "").Trim().ToLowerInvariant(); // "mug 2"
210
+ var words = s.Split(new[] { ' ' }, System.StringSplitOptions.RemoveEmptyEntries).ToList();
211
+ string colour = "";
212
+ if (words.Count > 1 && Colours.Contains(words[0])) { colour = words[0] == "grey" ? "gray" : words[0]; words.RemoveAt(0); }
213
+ return (colour, string.Join(" ", words));
214
+ }
215
+
216
+ static string NextId(string label, HashSet<string> used)
217
+ {
218
+ string b = Regex.Replace((label ?? "item").ToLowerInvariant(), "[^a-z0-9]+", "_").Trim('_');
219
+ int n = 1;
220
+ while (used.Contains($"{b}_{n}")) n++;
221
+ used.Add($"{b}_{n}");
222
+ return $"{b}_{n}";
223
+ }
224
+
225
+ /// <summary>What each item stands on: the highest other item whose top is just under its bottom and whose
226
+ /// footprint holds its centre; otherwise the floor.</summary>
227
+ public static void InferSupports(SceneWorld world)
228
+ {
229
+ var items = world.Items.Where(i => i.gameObject.activeInHierarchy).ToList();
230
+ var bounds = items.ToDictionary(i => i, BoundsOf);
231
+ foreach (var a in items)
232
+ {
233
+ if (a.on != null && (a.on.EndsWith("_hand") || a.on == "both_hands")) continue;
234
+ var ba = bounds[a];
235
+ SceneItem best = null;
236
+ foreach (var b in items)
237
+ {
238
+ if (b == a || b.room != a.room) continue;
239
+ var bb = bounds[b];
240
+ bool under = bb.max.y <= ba.min.y + 0.05f && bb.max.y >= ba.min.y - 0.15f;
241
+ bool holds = ba.center.x >= bb.min.x && ba.center.x <= bb.max.x && ba.center.z >= bb.min.z && ba.center.z <= bb.max.z;
242
+ if (under && holds && (best == null || bb.max.y > bounds[best].max.y)) best = b;
243
+ }
244
+ Undo.RecordObject(a, "Infer supports");
245
+ a.on = best != null ? best.itemId : "floor";
246
+ if (best != null && string.IsNullOrEmpty(a.relation)) a.relation = "on";
247
+ }
248
+ }
249
+
250
+ static Bounds BoundsOf(SceneItem i)
251
+ {
252
+ // its own renderers only: not those of other items parented under it
253
+ var rs = i.GetComponentsInChildren<Renderer>().Where(r => r.GetComponentInParent<SceneItem>() == i).ToArray();
254
+ if (rs.Length == 0) return new Bounds(i.transform.position, i.transform.lossyScale);
255
+ var b = rs[0].bounds;
256
+ foreach (var r in rs.Skip(1)) b.Encapsulate(r.bounds);
257
+ return b;
258
+ }
259
+ }
260
+ }
Editor/SceneAgentMenu.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: af0a3b6f63517c24c825d827886fff16
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
LICENSE.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Eren Ata, MCBU XRLab
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
LICENSE.md.meta ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: a89b7561ac2a0b54493ca7132f4097f8
3
+ TextScriptImporter:
4
+ externalObjects: {}
5
+ userData:
6
+ assetBundleName:
7
+ assetBundleVariant:
README.md.meta ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 9731bc90f1cd16b4ca19c31b70160a0c
3
+ TextScriptImporter:
4
+ externalObjects: {}
5
+ userData:
6
+ assetBundleName:
7
+ assetBundleVariant:
Runtime.meta ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 97273a565059476489b4f3f642df153c
3
+ folderAsset: yes
4
+ DefaultImporter:
5
+ externalObjects: {}
6
+ userData:
7
+ assetBundleName:
8
+ assetBundleVariant:
Runtime/LocalModelServer.cs ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Starts the local llama.cpp server with the model when the game starts and stops it when the game ends, so the player
2
+ // installs nothing and nothing leaves the machine (it listens on 127.0.0.1 only). GPU first; if that server does not
3
+ // come up, the same model on the CPU. A server already answering on the port is used as it is.
4
+ //
5
+ // Default layout (relative paths are looked up in StreamingAssets, then next to the project / the built game):
6
+ // StreamingAssets/SceneAgent/llama-server.exe (+ the DLLs of the same llama.cpp release)
7
+ // StreamingAssets/SceneAgent/<model>.gguf (modelPath empty = the first .gguf in that folder)
8
+ using System;
9
+ using System.Collections;
10
+ using System.Diagnostics;
11
+ using System.IO;
12
+ using System.Linq;
13
+ using System.Net.Http;
14
+ using System.Runtime.InteropServices;
15
+ using System.Text;
16
+ using System.Threading.Tasks;
17
+ using UnityEngine;
18
+ using Debug = UnityEngine.Debug;
19
+
20
+ namespace SceneAgent
21
+ {
22
+ public class LocalModelServer : MonoBehaviour
23
+ {
24
+ public enum Mode { Stopped, Starting, Gpu, Cpu, External, Failed }
25
+
26
+ [Tooltip("llama-server executable. Relative: looked up in StreamingAssets, then next to the project / the game.")]
27
+ public string serverExe = "SceneAgent/llama-server.exe";
28
+ [Tooltip("The model (GGUF). Empty: the first .gguf in the server's folder. Relative paths resolve like serverExe.")]
29
+ public string modelPath = "";
30
+ public int port = 8091;
31
+ [Tooltip("Tokens of context. 4096 holds a room of about 60 objects plus a few turns.")]
32
+ public int contextSize = 4096;
33
+ public bool preferGpu = true;
34
+ public int cpuThreads = 8;
35
+ [Tooltip("--cache-ram 0: llama.cpp's host prompt cache (8 GB by default) filled the machine's RAM.")]
36
+ public string extraArgs = "--cache-ram 0";
37
+ public float startTimeoutSeconds = 120f;
38
+ [Tooltip("While the model runs on the GPU, cap rendering at this frame rate so the renderer does not starve it " +
39
+ "(uncapped rendering made the model ~5x slower on a 4 GB laptop GPU). 0 = leave the frame rate alone; " +
40
+ "use 0 in XR, where the headset sets the frame rate.")]
41
+ public int capFrameRate = 30;
42
+ [Tooltip("Start the server when this component starts. Off: call Launch() yourself.")]
43
+ public bool launchOnStart = true;
44
+
45
+ public Mode mode { get; private set; } = Mode.Stopped;
46
+ public string status { get; private set; } = "not started";
47
+ public string Endpoint => $"http://127.0.0.1:{port}/v1/chat/completions";
48
+ public bool Ready => mode == Mode.Gpu || mode == Mode.Cpu || mode == Mode.External;
49
+ public bool Failed => mode == Mode.Failed;
50
+ /// <summary>Raised whenever mode changes (Starting, Gpu, Cpu, External, Failed, Stopped).</summary>
51
+ public event Action<Mode> ModeChanged;
52
+ public string LogPath => Path.Combine(Application.persistentDataPath, "llama-server.log");
53
+
54
+ Process proc;
55
+ StreamWriter log;
56
+ static readonly HttpClient Http = new HttpClient { Timeout = TimeSpan.FromSeconds(2) };
57
+
58
+ IEnumerator Start()
59
+ {
60
+ if (launchOnStart) yield return Launch();
61
+ }
62
+
63
+ void SetMode(Mode m)
64
+ {
65
+ mode = m;
66
+ if (m == Mode.Gpu && capFrameRate > 0)
67
+ {
68
+ QualitySettings.vSyncCount = 0;
69
+ Application.targetFrameRate = capFrameRate;
70
+ Debug.Log($"[SceneAgent] frame rate capped at {capFrameRate} while the model uses the GPU (LocalModelServer.capFrameRate)");
71
+ }
72
+ ModeChanged?.Invoke(m);
73
+ }
74
+
75
+ /// <summary>The first existing file among: the path itself (absolute), StreamingAssets/path, next to the project
76
+ /// (Editor) or the built game.</summary>
77
+ public static string Resolve(string p)
78
+ {
79
+ if (string.IsNullOrEmpty(p)) return null;
80
+ if (Path.IsPathRooted(p)) return p;
81
+ var roots = new[] { Application.streamingAssetsPath, Path.GetFullPath(Path.Combine(Application.dataPath, "..")) };
82
+ return roots.Select(r => Path.Combine(r, p)).FirstOrDefault(File.Exists) ?? Path.Combine(roots[0], p);
83
+ }
84
+
85
+ /// <summary>The model file: modelPath, or the first .gguf next to the server.</summary>
86
+ public string ResolveModel(string exe)
87
+ {
88
+ if (!string.IsNullOrEmpty(modelPath)) return Resolve(modelPath);
89
+ string dir = exe != null ? Path.GetDirectoryName(exe) : null;
90
+ return dir != null && Directory.Exists(dir) ? Directory.GetFiles(dir, "*.gguf").OrderBy(f => f).FirstOrDefault() : null;
91
+ }
92
+
93
+ public IEnumerator Launch()
94
+ {
95
+ SetMode(Mode.Starting);
96
+ // a server already answering on the port (e.g. started by hand) is used as is
97
+ var probe = HealthAsync();
98
+ while (!probe.IsCompleted) yield return null;
99
+ if (probe.Result) { status = $"using the server already running on :{port}"; SetMode(Mode.External); yield break; }
100
+
101
+ string exe = Resolve(serverExe), model = ResolveModel(exe);
102
+ if (exe == null || !File.Exists(exe) || model == null || !File.Exists(model))
103
+ {
104
+ status = exe == null || !File.Exists(exe)
105
+ ? $"llama-server not found: {exe} (see the package README, 'Model files')"
106
+ : $"no model (.gguf) found: {model ?? Path.GetDirectoryName(exe)}";
107
+ Debug.LogError("[SceneAgent] " + status);
108
+ SetMode(Mode.Failed);
109
+ yield break;
110
+ }
111
+ string common = $"-m \"{model}\" --jinja -c {contextSize} --host 127.0.0.1 --port {port} {extraArgs}";
112
+ if (preferGpu)
113
+ {
114
+ status = "loading the model on the GPU…";
115
+ yield return StartAndWait(exe, common + " -ngl 99", cpuOnly: false);
116
+ if (mode == Mode.Gpu) yield break;
117
+ Stop();
118
+ SetMode(Mode.Starting);
119
+ }
120
+ status = "loading the model on the CPU…";
121
+ yield return StartAndWait(exe, common + $" --device none --no-op-offload -ngl 0 -t {cpuThreads}", cpuOnly: true);
122
+ if (!Ready) { status = $"the model server did not start (see {LogPath})"; SetMode(Mode.Failed); }
123
+ }
124
+
125
+ IEnumerator StartAndWait(string exe, string args, bool cpuOnly)
126
+ {
127
+ var psi = new ProcessStartInfo(exe, args)
128
+ {
129
+ UseShellExecute = false, CreateNoWindow = true,
130
+ RedirectStandardOutput = true, RedirectStandardError = true,
131
+ WorkingDirectory = Path.GetDirectoryName(exe),
132
+ };
133
+ if (cpuOnly) psi.EnvironmentVariables["CUDA_VISIBLE_DEVICES"] = "-1";
134
+ try
135
+ {
136
+ log = new StreamWriter(LogPath, append: false, Encoding.UTF8) { AutoFlush = true };
137
+ proc = new Process { StartInfo = psi, EnableRaisingEvents = true };
138
+ // the pipes must be drained, or the server blocks once the buffer is full
139
+ proc.OutputDataReceived += (_, e) => Write(e.Data);
140
+ proc.ErrorDataReceived += (_, e) => Write(e.Data);
141
+ proc.Start();
142
+ ChildProcessGuard.Attach(proc); // dies with this process, even after a crash
143
+ proc.BeginOutputReadLine();
144
+ proc.BeginErrorReadLine();
145
+ Debug.Log($"[SceneAgent] model server started ({(cpuOnly ? "CPU" : "GPU")}), log: {LogPath}");
146
+ }
147
+ catch (Exception e)
148
+ {
149
+ status = "could not start llama-server: " + e.Message;
150
+ Debug.LogError("[SceneAgent] " + status);
151
+ yield break;
152
+ }
153
+ float t0 = Time.realtimeSinceStartup;
154
+ while (Time.realtimeSinceStartup - t0 < startTimeoutSeconds)
155
+ {
156
+ if (proc.HasExited) break;
157
+ var h = HealthAsync();
158
+ while (!h.IsCompleted) yield return null;
159
+ if (h.Result)
160
+ {
161
+ status = $"model ready on the {(cpuOnly ? "CPU" : "GPU")}";
162
+ SetMode(cpuOnly ? Mode.Cpu : Mode.Gpu);
163
+ yield break;
164
+ }
165
+ yield return new WaitForSecondsRealtime(1f);
166
+ }
167
+ }
168
+
169
+ void Write(string line)
170
+ {
171
+ if (line == null) return;
172
+ lock (this) { try { log?.WriteLine(line); } catch (ObjectDisposedException) { } }
173
+ }
174
+
175
+ async Task<bool> HealthAsync()
176
+ {
177
+ try
178
+ {
179
+ var r = await Http.GetAsync($"http://127.0.0.1:{port}/health").ConfigureAwait(false);
180
+ return r.IsSuccessStatusCode;
181
+ }
182
+ catch { return false; }
183
+ }
184
+
185
+ public void Stop()
186
+ {
187
+ // wait for the exit: a scene reload starts the next server at once, and it must find the port free
188
+ try { if (proc != null && !proc.HasExited) { proc.Kill(); proc.WaitForExit(3000); } } catch (Exception) { }
189
+ proc?.Dispose();
190
+ proc = null;
191
+ lock (this) { log?.Dispose(); log = null; }
192
+ if (mode != Mode.External && mode != Mode.Stopped) SetMode(Mode.Stopped);
193
+ }
194
+
195
+ void OnApplicationQuit() => Stop();
196
+ void OnDestroy() => Stop();
197
+ }
198
+
199
+ /// <summary>Puts the model server in a Windows job object that is closed, and its processes killed, when this
200
+ /// process ends in any way (quit, crash, the Editor closing). Elsewhere: nothing (Stop still ends the server).</summary>
201
+ static class ChildProcessGuard
202
+ {
203
+ #if UNITY_STANDALONE_WIN || UNITY_EDITOR_WIN
204
+ [StructLayout(LayoutKind.Sequential)]
205
+ struct BasicLimits
206
+ {
207
+ public long PerProcessUserTimeLimit, PerJobUserTimeLimit;
208
+ public uint LimitFlags;
209
+ public UIntPtr MinimumWorkingSetSize, MaximumWorkingSetSize;
210
+ public uint ActiveProcessLimit;
211
+ public UIntPtr Affinity;
212
+ public uint PriorityClass, SchedulingClass;
213
+ }
214
+ [StructLayout(LayoutKind.Sequential)]
215
+ struct IoCounters { public ulong R, W, O, RB, WB, OB; }
216
+ [StructLayout(LayoutKind.Sequential)]
217
+ struct ExtendedLimits
218
+ {
219
+ public BasicLimits Basic;
220
+ public IoCounters Io;
221
+ public UIntPtr ProcessMemoryLimit, JobMemoryLimit, PeakProcessMemoryUsed, PeakJobMemoryUsed;
222
+ }
223
+ [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] static extern IntPtr CreateJobObject(IntPtr attributes, string name);
224
+ [DllImport("kernel32.dll")] static extern bool SetInformationJobObject(IntPtr job, int infoClass, ref ExtendedLimits info, uint length);
225
+ [DllImport("kernel32.dll", SetLastError = true)] static extern bool AssignProcessToJobObject(IntPtr job, IntPtr process);
226
+ static IntPtr job;
227
+
228
+ public static void Attach(Process p)
229
+ {
230
+ try
231
+ {
232
+ if (job == IntPtr.Zero)
233
+ {
234
+ job = CreateJobObject(IntPtr.Zero, null);
235
+ var info = new ExtendedLimits();
236
+ info.Basic.LimitFlags = 0x2000; // JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
237
+ SetInformationJobObject(job, 9, ref info, (uint)Marshal.SizeOf(typeof(ExtendedLimits))); // 9: extended limits
238
+ }
239
+ if (!AssignProcessToJobObject(job, p.Handle))
240
+ Debug.LogWarning("[SceneAgent] could not tie the model server to this process (error " + Marshal.GetLastWin32Error() + ")");
241
+ }
242
+ catch (Exception e) { Debug.LogWarning("[SceneAgent] could not tie the model server to this process: " + e.Message); }
243
+ }
244
+ #else
245
+ public static void Attach(Process p) { }
246
+ #endif
247
+ }
248
+ }
Runtime/LocalModelServer.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 886cfe192c0362948b23c41866c13b18
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/MiniJson.cs ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Minimal JSON reader/writer (objects -> Dictionary<string, object>, arrays -> List<object>,
2
+ // numbers -> double, plus string/bool/null) so the agent has no package dependencies.
3
+ using System;
4
+ using System.Collections;
5
+ using System.Collections.Generic;
6
+ using System.Globalization;
7
+ using System.Text;
8
+
9
+ namespace SceneAgent
10
+ {
11
+ public static class MiniJson
12
+ {
13
+ public static object Parse(string s)
14
+ {
15
+ int i = 0;
16
+ var v = ReadValue(s, ref i);
17
+ return v;
18
+ }
19
+
20
+ /// <summary>Like Parse, but only whitespace may follow the value (Python's json.loads rejects extra data; D5b's
21
+ /// check must judge a tool call exactly as eval/grammar.py does).</summary>
22
+ public static object ParseStrict(string s)
23
+ {
24
+ int i = 0;
25
+ var v = ReadValue(s, ref i);
26
+ Skip(s, ref i);
27
+ if (i != s.Length) throw new FormatException("extra data after the JSON value");
28
+ return v;
29
+ }
30
+
31
+ static void Skip(string s, ref int i) { while (i < s.Length && char.IsWhiteSpace(s[i])) i++; }
32
+
33
+ static object ReadValue(string s, ref int i)
34
+ {
35
+ Skip(s, ref i);
36
+ if (i >= s.Length) throw new FormatException("unexpected end of JSON");
37
+ char c = s[i];
38
+ if (c == '{')
39
+ {
40
+ var d = new Dictionary<string, object>(); i++;
41
+ Skip(s, ref i);
42
+ if (s[i] == '}') { i++; return d; }
43
+ while (true)
44
+ {
45
+ Skip(s, ref i);
46
+ string key = ReadString(s, ref i);
47
+ Skip(s, ref i);
48
+ if (s[i] != ':') throw new FormatException("expected ':'");
49
+ i++;
50
+ d[key] = ReadValue(s, ref i);
51
+ Skip(s, ref i);
52
+ if (s[i] == ',') { i++; continue; }
53
+ if (s[i] == '}') { i++; return d; }
54
+ throw new FormatException("expected ',' or '}'");
55
+ }
56
+ }
57
+ if (c == '[')
58
+ {
59
+ var l = new List<object>(); i++;
60
+ Skip(s, ref i);
61
+ if (s[i] == ']') { i++; return l; }
62
+ while (true)
63
+ {
64
+ l.Add(ReadValue(s, ref i));
65
+ Skip(s, ref i);
66
+ if (s[i] == ',') { i++; continue; }
67
+ if (s[i] == ']') { i++; return l; }
68
+ throw new FormatException("expected ',' or ']'");
69
+ }
70
+ }
71
+ if (c == '"') return ReadString(s, ref i);
72
+ if (s.Substring(i).StartsWith("true")) { i += 4; return true; }
73
+ if (s.Substring(i).StartsWith("false")) { i += 5; return false; }
74
+ if (s.Substring(i).StartsWith("null")) { i += 4; return null; }
75
+ int start = i;
76
+ while (i < s.Length && "+-0123456789.eE".IndexOf(s[i]) >= 0) i++;
77
+ return double.Parse(s.Substring(start, i - start), CultureInfo.InvariantCulture);
78
+ }
79
+
80
+ static string ReadString(string s, ref int i)
81
+ {
82
+ if (s[i] != '"') throw new FormatException("expected string");
83
+ var sb = new StringBuilder(); i++;
84
+ while (s[i] != '"')
85
+ {
86
+ if (s[i] == '\\')
87
+ {
88
+ i++;
89
+ char e = s[i];
90
+ if (e == 'u') { sb.Append((char)Convert.ToInt32(s.Substring(i + 1, 4), 16)); i += 4; }
91
+ else sb.Append(e == 'n' ? '\n' : e == 't' ? '\t' : e == 'r' ? '\r' : e == 'b' ? '\b' : e == 'f' ? '\f' : e);
92
+ }
93
+ else sb.Append(s[i]);
94
+ i++;
95
+ }
96
+ i++;
97
+ return sb.ToString();
98
+ }
99
+
100
+ public static string Serialize(object v)
101
+ {
102
+ var sb = new StringBuilder();
103
+ Write(sb, v);
104
+ return sb.ToString();
105
+ }
106
+
107
+ static void Write(StringBuilder sb, object v)
108
+ {
109
+ switch (v)
110
+ {
111
+ case null: sb.Append("null"); break;
112
+ case string s: WriteString(sb, s); break;
113
+ case bool b: sb.Append(b ? "true" : "false"); break;
114
+ case double d: sb.Append(FormatNumber(d)); break;
115
+ case float f: sb.Append(FormatNumber(f)); break;
116
+ case int n: sb.Append(n.ToString(CultureInfo.InvariantCulture)); break;
117
+ case IDictionary<string, object> d:
118
+ sb.Append('{'); bool first = true;
119
+ foreach (var kv in d) { if (!first) sb.Append(','); first = false; WriteString(sb, kv.Key); sb.Append(':'); Write(sb, kv.Value); }
120
+ sb.Append('}'); break;
121
+ case IEnumerable e:
122
+ sb.Append('['); bool f1 = true;
123
+ foreach (var x in e) { if (!f1) sb.Append(','); f1 = false; Write(sb, x); }
124
+ sb.Append(']'); break;
125
+ default: WriteString(sb, v.ToString()); break;
126
+ }
127
+ }
128
+
129
+ // Python's json.dumps prints integral floats as "2.0"; keep the same shape as the training data.
130
+ public static string FormatNumber(double d)
131
+ {
132
+ string s = d.ToString("R", CultureInfo.InvariantCulture);
133
+ return s.Contains(".") || s.Contains("E") ? s : s + ".0";
134
+ }
135
+
136
+ static void WriteString(StringBuilder sb, string s)
137
+ {
138
+ sb.Append('"');
139
+ foreach (char c in s)
140
+ {
141
+ if (c == '"') sb.Append("\\\"");
142
+ else if (c == '\\') sb.Append("\\\\");
143
+ else if (c == '\n') sb.Append("\\n");
144
+ else if (c < ' ') sb.Append("\\u").Append(((int)c).ToString("x4"));
145
+ else sb.Append(c);
146
+ }
147
+ sb.Append('"');
148
+ }
149
+ }
150
+ }
Runtime/MiniJson.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 61e217efc401cb446921912c87d7770d
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/PyMath.cs ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Python's round(x, n) and f"{x:.nf}" in C#, so the scene text and the tool effects match the training data byte for
2
+ // byte. Python rounds the exact binary value of x, ties to even; .NET's Math.Round(x, n) scales by 10^n first, so 0.45
3
+ // (binary 0.45000000000000001...) became 0.4 where the training scenes say 0.5, and -0.04 became 0.0 where they say
4
+ // -0.0 (v2 Unity reference on the v3 sample: 81 + 37 of 440 scenes differed from the offline prompt).
5
+ using System;
6
+ using System.Globalization;
7
+ using System.Numerics;
8
+
9
+ namespace SceneAgent
10
+ {
11
+ public static class PyMath
12
+ {
13
+ /// <summary>Python round(x, n) for n >= 0: exact, ties to even, sign kept (round(-0.04, 1) == -0.0).</summary>
14
+ public static double Round(double x, int n)
15
+ {
16
+ if (double.IsNaN(x) || double.IsInfinity(x)) return x;
17
+ long bits = BitConverter.DoubleToInt64Bits(x);
18
+ bool neg = bits < 0;
19
+ int e = (int)((bits >> 52) & 0x7FF);
20
+ long m = bits & 0xFFFFFFFFFFFFFL;
21
+ if (e == 0) e = 1; else m |= 1L << 52;
22
+ e -= 1075; // |x| = m * 2^e exactly
23
+ BigInteger num = new BigInteger(m) * BigInteger.Pow(10, n), den = BigInteger.One;
24
+ if (e > 0) num <<= e; else den <<= -e;
25
+ BigInteger q = BigInteger.DivRem(num, den, out BigInteger r);
26
+ int c = (r * 2).CompareTo(den);
27
+ if (c > 0 || (c == 0 && !q.IsEven)) q += 1;
28
+ double v = (double)q / Math.Pow(10, n); // q and 10^n are exact doubles: the division is correctly rounded
29
+ return neg ? -v : v;
30
+ }
31
+
32
+ /// <summary>Python f"{x:.{n}f}".</summary>
33
+ public static string Fixed(double x, int n) => Round(x, n).ToString("F" + n, CultureInfo.InvariantCulture);
34
+ }
35
+ }
Runtime/PyMath.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: aa67273f40b6679448917c8a06ebe741
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/Resources.meta ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 4e30c3b99c50b4d499baeb3c87541721
3
+ folderAsset: yes
4
+ DefaultImporter:
5
+ externalObjects: {}
6
+ userData:
7
+ assetBundleName:
8
+ assetBundleVariant:
Runtime/Resources/SceneAgent.meta ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 06871ae947abbc942a04f192fb97ac7f
3
+ folderAsset: yes
4
+ DefaultImporter:
5
+ externalObjects: {}
6
+ userData:
7
+ assetBundleName:
8
+ assetBundleVariant:
Runtime/Resources/SceneAgent/system_prompt_v3.txt ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ You are an assistant embedded in a 3D Unity scene. Turn the user's request into tool calls that use object ids exactly as listed in the scene; never invent an id or a tool. Only objects in the user's current room are listed. Match objects by meaning: a synonym or a more general word can name a listed object, and details the scene does not record (a colour when no colours are listed, or words like sliced, wooden, empty) do not rule an object out. If exactly one listed object fits, act on it. If several fit equally well, call ask_clarification with their ids. If none fits, use find_objects to look in the other rooms. place moves an object directly, so there is no need to grab it first. If a request is impossible or needs a tool you do not have, call no tool and briefly explain why. Answer questions about the scene in plain text.
2
+
3
+ # Tools
4
+ grab(object_id:str, hand:left|right|both)
5
+ release(hand:left|right|both)
6
+ place(object_id:str, target_id:str, relation?:on|in, orientation?:default|upright|horizontal|vertical|upside_down)
7
+ move_by(object_id:str, direction:left|right|forward|backward|up|down, distance:num, unit:m|cm)
8
+ rotate(object_id:str, degrees:num, axis?:yaw|pitch|roll)
9
+ set_state(object_id:str, state:on|off|open|closed|half_open|locked|unlocked)
10
+ press(object_id:str, hold_seconds?:num)
11
+ highlight(object_ids:list)
12
+ teleport(location_id:str)
13
+ find_objects(label:str, color?:str, room?:str)
14
+ ask_clarification(question:str, candidate_ids:list)
15
+
16
+ Reply with one <tool_call>{"name": ..., "arguments": {...}}</tool_call> block per call, or plain text.
Runtime/Resources/SceneAgent/system_prompt_v3.txt.meta ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 6a38ee93718310f469efc52b22f6a1b9
3
+ TextScriptImporter:
4
+ externalObjects: {}
5
+ userData:
6
+ assetBundleName:
7
+ assetBundleVariant:
Runtime/SceneAgent.Runtime.asmdef ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "name": "SceneAgent.Runtime",
3
+ "rootNamespace": "SceneAgent",
4
+ "references": [],
5
+ "autoReferenced": true
6
+ }
Runtime/SceneAgent.Runtime.asmdef.meta ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 9fc737831bcf4da4e9b861e32b778d92
3
+ AssemblyDefinitionImporter:
4
+ externalObjects: {}
5
+ userData:
6
+ assetBundleName:
7
+ assetBundleVariant:
Runtime/SceneAgent.cs ADDED
@@ -0,0 +1,255 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Talks to a local OpenAI-compatible server (llama.cpp llama-server with the fine-tuned GGUF,
2
+ // Unsloth Studio's /v1 endpoint, Ollama, ...), executes the returned tool calls on the SceneWorld
3
+ // and feeds tool results back until the model answers in text, asks the user a clarifying question
4
+ // (ask_clarification hands the turn back: the answer arrives as the next request), or runs out of turns.
5
+ //
6
+ // The loop is a coroutine: HTTP runs on a worker thread, tool calls run on the main thread
7
+ // (Unity APIs are main-thread only). Play mode: StartCoroutine(agent.Run(...)).
8
+ // Batch/editor code can drive it synchronously with RunBlocking(...).
9
+ using System;
10
+ using System.Collections;
11
+ using System.Collections.Generic;
12
+ using System.Globalization;
13
+ using System.Linq;
14
+ using System.Net.Http;
15
+ using System.Text;
16
+ using System.Text.RegularExpressions;
17
+ using System.Threading;
18
+ using System.Threading.Tasks;
19
+ using UnityEngine;
20
+
21
+ namespace SceneAgent
22
+ {
23
+ [Serializable]
24
+ public class AgentTurn
25
+ {
26
+ public string modelOutput;
27
+ public List<string> calls = new List<string>();
28
+ public List<string> results = new List<string>();
29
+ /// <summary>D5b only: the verdict on the free output (SceneGrammar.Check) and what was done about it.</summary>
30
+ public string d5b;
31
+ }
32
+
33
+ public class AgentResult
34
+ {
35
+ public List<AgentTurn> turns = new List<AgentTurn>();
36
+ public string finalText;
37
+ /// <summary>Set when the model asked the user a question (ask_clarification); the turn ends there.</summary>
38
+ public string clarificationQuestion;
39
+ public string error;
40
+ public double seconds;
41
+ }
42
+
43
+ public class SceneAgent : MonoBehaviour
44
+ {
45
+ public string endpoint = "http://127.0.0.1:8081/v1/chat/completions";
46
+ public string model = "unity-scene-agent";
47
+ [Tooltip("The model's system prompt. Empty: the one shipped with the package (Resources/SceneAgent/system_prompt_v3), " +
48
+ "the exact prompt the v3 model was trained and evaluated with. Change it only with a model trained on yours.")]
49
+ public TextAsset systemPromptAsset;
50
+ [Tooltip("Optional: the LocalModelServer that runs the model. Run() then waits until it is ready and uses its endpoint.")]
51
+ public LocalModelServer server;
52
+ [NonSerialized] public string systemPrompt;
53
+ public int maxTurns = 4;
54
+ public SceneWorld world;
55
+ [Tooltip("Strict D5: constrain decoding to the 11 tools, ids that exist and the trained call format (SceneGrammar.cs). " +
56
+ "Needs llama-server. The shipped v3 setting is D5b below, not this.")]
57
+ public bool constrainedDecoding = false;
58
+ [Tooltip("D5b (the shipped v3 setting): the model answers freely; SceneGrammar.Check then turns a call to a tool or " +
59
+ "value outside the schema into a refusal and generates an unknown id or a broken call again under the " +
60
+ "grammar. Used when constrainedDecoding is off.")]
61
+ public bool fallbackDecoding = true;
62
+ [Tooltip("Safety: when the request names an object that is only in another room and the model acts on a different " +
63
+ "object here instead, that call is not executed; the model is told where the named object is. (Seen in the " +
64
+ "user test: 'put the drill on the workbench' with the drill in another room moved the screwdriver.)")]
65
+ public bool guardSubstitutes = true;
66
+
67
+ static readonly HttpClient Http = new HttpClient { Timeout = TimeSpan.FromSeconds(120) };
68
+ static readonly Regex ToolCallRe = new Regex(@"<tool_call>\s*(.*?)\s*</tool_call>", RegexOptions.Singleline);
69
+ static readonly Regex ThinkRe = new Regex(@"<think>.*?</think>\s*", RegexOptions.Singleline);
70
+ // Same rule as eval/toolcall.py HANDOFF: these calls end the agent turn.
71
+ static readonly HashSet<string> Handoff = new HashSet<string> { "ask_clarification" };
72
+
73
+ static string defaultPrompt;
74
+ static string DefaultPrompt => defaultPrompt ??= Resources.Load<TextAsset>("SceneAgent/system_prompt_v3")?.text.Replace("\r\n", "\n");
75
+ // no ?. on systemPromptAsset: in the Editor an unassigned field is a "fake null" Unity object, and ?. would call
76
+ // .text on it (UnassignedReferenceException on the first command in Play mode; found by the visible user test)
77
+ string Prompt => systemPrompt ?? (systemPromptAsset != null ? systemPromptAsset.text.Replace("\r\n", "\n") : null) ?? DefaultPrompt
78
+ ?? throw new InvalidOperationException("no system prompt (the package's Resources/SceneAgent/system_prompt_v3 is missing)");
79
+
80
+ public IEnumerator Run(string utterance, Action<AgentResult> done)
81
+ {
82
+ var t0 = DateTime.UtcNow;
83
+ var res = new AgentResult();
84
+ if (server != null)
85
+ {
86
+ while (!server.Ready && !server.Failed) yield return null;
87
+ if (server.Failed)
88
+ {
89
+ res.error = "model server not available: " + server.status;
90
+ res.seconds = (DateTime.UtcNow - t0).TotalSeconds;
91
+ done(res);
92
+ yield break;
93
+ }
94
+ endpoint = server.Endpoint;
95
+ }
96
+ string sceneJson = world.BuildSceneJson();
97
+ var messages = new List<object> {
98
+ Msg("system", Prompt),
99
+ Msg("user", "Scene: " + sceneJson + "\nRequest: " + utterance) };
100
+ var toolResults = new List<string>(); // ids returned here become nameable (grammar)
101
+ for (int turn = 0; turn < maxTurns && res.error == null; turn++)
102
+ {
103
+ var req = new Dictionary<string, object> {
104
+ { "model", model }, { "messages", messages }, { "temperature", 0.0 }, { "max_tokens", 192 } };
105
+ if (constrainedDecoding) req["grammar"] = SceneGrammar.Build(sceneJson, toolResults);
106
+ string body = MiniJson.Serialize(req);
107
+ var task = Task.Run(() => Post(body)); // worker thread: no Unity API inside
108
+ while (!task.IsCompleted) yield return null;
109
+ if (task.IsFaulted) { res.error = task.Exception?.GetBaseException().Message; break; }
110
+ string output = task.Result, d5b = null;
111
+ if (!constrainedDecoding && fallbackDecoding)
112
+ {
113
+ d5b = SceneGrammar.Check(output, sceneJson, toolResults);
114
+ if (d5b == "invalid_tool" || d5b == "invalid_value") output = SceneGrammar.Refusal;
115
+ else if (d5b == "invalid_id" || d5b == "malformed")
116
+ {
117
+ req["grammar"] = SceneGrammar.Build(sceneJson, toolResults);
118
+ string again = MiniJson.Serialize(req);
119
+ var retry = Task.Run(() => Post(again));
120
+ while (!retry.IsCompleted) yield return null;
121
+ if (retry.IsFaulted) { res.error = retry.Exception?.GetBaseException().Message; break; }
122
+ output = retry.Result;
123
+ }
124
+ }
125
+ var at = new AgentTurn { modelOutput = output, d5b = d5b };
126
+ res.turns.Add(at);
127
+ var blocks = ToolCallRe.Matches(output).Cast<Match>().Select(m => m.Groups[1].Value).ToList();
128
+ if (blocks.Count == 0)
129
+ {
130
+ res.finalText = Clean(output);
131
+ break;
132
+ }
133
+ var toolMsgs = new List<object>();
134
+ string question = null;
135
+ foreach (var block in blocks)
136
+ {
137
+ Dictionary<string, object> c;
138
+ try { c = (Dictionary<string, object>)MiniJson.Parse(block); }
139
+ catch (Exception e) { res.error = "unparseable tool call: " + e.Message; break; }
140
+ string name = Convert.ToString(c.TryGetValue("name", out var n) ? n : "");
141
+ var args = c.TryGetValue("arguments", out var a) && a is Dictionary<string, object> d ? d : new Dictionary<string, object>();
142
+ object result;
143
+ string guard = SubstituteGuard(utterance, name, args);
144
+ if (guard != null)
145
+ {
146
+ Debug.Log("[SceneAgent] call not executed: " + guard);
147
+ result = new Dictionary<string, object> { { "ok", false }, { "error", guard } };
148
+ }
149
+ else
150
+ {
151
+ try { result = world.Execute(name, args); }
152
+ catch (ToolError e) { result = new Dictionary<string, object> { { "ok", false }, { "error", e.Message } }; }
153
+ }
154
+ if (Handoff.Contains(name))
155
+ question = Convert.ToString(args.TryGetValue("question", out var q) ? q : "") ?? "";
156
+ string resultJson = PyJson(result);
157
+ at.calls.Add(block);
158
+ at.results.Add(resultJson);
159
+ toolResults.Add(resultJson);
160
+ toolMsgs.Add(new Dictionary<string, object> { { "role", "tool" }, { "name", name }, { "content", resultJson } });
161
+ }
162
+ // history: the model's own call text, then tool results (Qwen3 renders them as <tool_response>)
163
+ messages.Add(Msg("assistant", Clean(output)));
164
+ messages.AddRange(toolMsgs);
165
+ if (question != null) { res.clarificationQuestion = question; break; }
166
+ }
167
+ res.seconds = (DateTime.UtcNow - t0).TotalSeconds;
168
+ done(res);
169
+ }
170
+
171
+ /// <summary>Synchronous driver for editor/batch code (main thread).</summary>
172
+ public AgentResult RunBlocking(string utterance)
173
+ {
174
+ AgentResult result = null;
175
+ var it = Run(utterance, r => result = r);
176
+ while (it.MoveNext()) Thread.Sleep(2);
177
+ return result;
178
+ }
179
+
180
+ static readonly HashSet<string> ActingTools = new HashSet<string> { "grab", "place", "move_by", "rotate", "set_state", "press" };
181
+
182
+ /// <summary>Why this call must not run (see guardSubstitutes), or null. It fires only when the acted-on object is
183
+ /// not named in the request while the request names an object that exists only in other rooms.</summary>
184
+ public string SubstituteGuard(string utterance, string tool, Dictionary<string, object> args)
185
+ {
186
+ if (!guardSubstitutes || world == null || !ActingTools.Contains(tool ?? "")) return null;
187
+ string id = args.TryGetValue("object_id", out var v) && v != null ? Convert.ToString(v, CultureInfo.InvariantCulture) : null;
188
+ var item = id == null ? null : world.Items.FirstOrDefault(x => x.itemId == id);
189
+ if (item == null) return null;
190
+ if (string.IsNullOrEmpty(item.label)) return null;
191
+ // the same rules as tests/acceptance/guard_offline.py, which measured them on every call the four v3 models
192
+ // made on the sealed and dev sets (13,438 calls): 4 firings, all on wrong actions, no false alarm
193
+ string text = " " + Regex.Replace(utterance.ToLowerInvariant(), "[^a-z0-9]+", " ") + " ";
194
+ var words = text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
195
+ string joined = string.Concat(words);
196
+ IEnumerable<string> WordsOf(string label) => label.Split(' ').Where(w => w.Length >= 3);
197
+ bool Phrase(string label) => text.Contains(" " + label + " ") || text.Contains(" " + label + "s ") || text.Contains(" " + label + "es ");
198
+ // the acted-on object: any sign the user meant it ("box" -> toolbox, "note book" -> notebook, "keys" -> key chain)
199
+ bool Loosely(SceneItem o)
200
+ {
201
+ string h = o.label.Split(' ').Last();
202
+ return Phrase(o.label) || WordsOf(o.label).Any(w => words.Contains(w) || words.Contains(w + "s") || words.Contains(w + "es"))
203
+ || words.Any(w => w.Length >= 3 && h.EndsWith(w)) || joined.Contains(o.label.Replace(" ", ""));
204
+ }
205
+ if (Loosely(item)) return null;
206
+ // an object elsewhere counts only when named by its full label, and only if nothing here shares a word with it
207
+ var hereWords = new HashSet<string>(world.Items.Where(y => y.room == world.playerRoom && !string.IsNullOrEmpty(y.label))
208
+ .SelectMany(y => WordsOf(y.label)));
209
+ var elsewhere = world.Items.FirstOrDefault(x => x.room != world.playerRoom && !string.IsNullOrEmpty(x.label)
210
+ && Phrase(x.label) && !WordsOf(x.label).Any(hereWords.Contains));
211
+ if (elsewhere == null) return null;
212
+ return $"not done: '{id}' is a {item.label}, but the request names a {elsewhere.label}; " +
213
+ $"the {elsewhere.label} ({elsewhere.itemId}) is in the {elsewhere.room}, not in this room";
214
+ }
215
+
216
+ static string Clean(string s) => ThinkRe.Replace(s, "").Replace("<|im_end|>", "").Trim();
217
+
218
+ static Dictionary<string, object> Msg(string role, string content) =>
219
+ new Dictionary<string, object> { { "role", role }, { "content", content } };
220
+
221
+ string Post(string body)
222
+ {
223
+ var resp = Http.PostAsync(endpoint, new StringContent(body, Encoding.UTF8, "application/json")).ConfigureAwait(false).GetAwaiter().GetResult();
224
+ string text = resp.Content.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult();
225
+ if (!resp.IsSuccessStatusCode)
226
+ {
227
+ var m = Regex.Match(text, @"request \((\d+) tokens\) exceeds the available context size \((\d+) tokens\)");
228
+ if (m.Success)
229
+ throw new Exception($"the scene and request need {m.Groups[1].Value} tokens but the model's context holds " +
230
+ $"{m.Groups[2].Value}: put fewer objects in this room, or raise LocalModelServer.contextSize");
231
+ throw new Exception($"HTTP {(int)resp.StatusCode}: {text}");
232
+ }
233
+ var root = (Dictionary<string, object>)MiniJson.Parse(text);
234
+ var msg = (Dictionary<string, object>)((Dictionary<string, object>)((List<object>)root["choices"])[0])["message"];
235
+ return Convert.ToString(msg.TryGetValue("content", out var c) ? c : "") ?? "";
236
+ }
237
+
238
+ // Tool results in training were produced by Python's json.dumps (", " and ": " separators).
239
+ public static string PyJson(object v)
240
+ {
241
+ switch (v)
242
+ {
243
+ case null: return "null";
244
+ case bool b: return b ? "true" : "false";
245
+ case string s: return MiniJson.Serialize(s);
246
+ case double d: return MiniJson.FormatNumber(d);
247
+ case IDictionary<string, object> dict:
248
+ return "{" + string.Join(", ", dict.Select(kv => MiniJson.Serialize(kv.Key) + ": " + PyJson(kv.Value))) + "}";
249
+ case IEnumerable e:
250
+ return "[" + string.Join(", ", e.Cast<object>().Select(PyJson)) + "]";
251
+ default: return MiniJson.Serialize(Convert.ToString(v, CultureInfo.InvariantCulture));
252
+ }
253
+ }
254
+ }
255
+ }
Runtime/SceneAgent.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: e77941abfd358644fb748664149b43d0
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/SceneAgentChatBox.cs ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // A minimal on-screen chat for trying the agent without writing any UI (flat screen; for XR, drive SceneAgentSession
2
+ // from your own UI or speech input). Type a command, press Enter; when the model asks which object you mean, click it.
3
+ // F1 hides / shows the box.
4
+ using System.Collections.Generic;
5
+ using System.Linq;
6
+ using UnityEngine;
7
+
8
+ namespace SceneAgent
9
+ {
10
+ [RequireComponent(typeof(SceneAgentSession))]
11
+ public class SceneAgentChatBox : MonoBehaviour
12
+ {
13
+ public int visibleLines = 12;
14
+ public KeyCode toggleKey = KeyCode.F1;
15
+ [Tooltip("Commands offered as one-click examples while the box is empty.")]
16
+ public string[] examples = { "put the red mug on the table", "turn on the lamp", "where is the book?" };
17
+
18
+ SceneAgentSession session;
19
+ LocalModelServer server;
20
+ readonly List<string> lines = new List<string>();
21
+ string input = "";
22
+ bool visible = true;
23
+ Vector2 scroll;
24
+ GUIStyle wrap;
25
+
26
+ void Awake()
27
+ {
28
+ session = GetComponent<SceneAgentSession>();
29
+ session.Line += l => { lines.Add(l); scroll.y = float.MaxValue; };
30
+ }
31
+
32
+ void Start() // after every Awake: the session has found its agent by now
33
+ {
34
+ // no ?? here: Unity objects compare to null through their own operator
35
+ server = session.agent != null ? session.agent.server : null;
36
+ if (server == null) server = GetComponent<LocalModelServer>();
37
+ if (server == null) server = FindObjectOfType<LocalModelServer>();
38
+ }
39
+
40
+ void Send(string text)
41
+ {
42
+ text = text?.Trim();
43
+ if (string.IsNullOrEmpty(text) || session.Busy) return;
44
+ input = "";
45
+ StartCoroutine(session.Command(text));
46
+ }
47
+
48
+ void OnGUI()
49
+ {
50
+ // the toggle through IMGUI events: works with the old Input Manager and with the Input System package
51
+ if (Event.current.type == EventType.KeyDown && Event.current.keyCode == toggleKey) { visible = !visible; Event.current.Use(); }
52
+ if (!visible) return;
53
+ wrap ??= new GUIStyle(GUI.skin.label) { wordWrap = true, richText = false };
54
+ float w = Mathf.Min(Screen.width - 20, 900), h = 60 + visibleLines * 20;
55
+ var area = new Rect(10, Screen.height - h - 10, w, h);
56
+ GUI.Box(area, "");
57
+ GUILayout.BeginArea(new Rect(area.x + 8, area.y + 6, area.width - 16, area.height - 12));
58
+
59
+ string st = server == null ? "no LocalModelServer: using " + session.agent.endpoint
60
+ : server.Ready ? server.status : server.Failed ? "⚠ " + server.status : server.status;
61
+ string gaze = string.IsNullOrEmpty(session.world.gaze) ? "" : $" · pointing at: {session.world.gaze}";
62
+ GUILayout.Label($"Scene Agent · room: {session.world.playerRoom}{gaze} · {st}{(session.Busy ? " · thinking…" : "")}");
63
+
64
+ scroll = GUILayout.BeginScrollView(scroll, GUILayout.Height(visibleLines * 20));
65
+ foreach (var l in lines.Skip(Mathf.Max(0, lines.Count - 200))) GUILayout.Label(l, wrap);
66
+ if (lines.Count == 0)
67
+ {
68
+ GUILayout.Label("Type a command in English, e.g.:");
69
+ GUILayout.BeginHorizontal();
70
+ foreach (var ex in examples) if (GUILayout.Button(ex)) input = ex;
71
+ GUILayout.EndHorizontal();
72
+ }
73
+ GUILayout.EndScrollView();
74
+
75
+ if (session.PendingRequest != null && !session.Busy)
76
+ {
77
+ GUILayout.BeginHorizontal();
78
+ GUILayout.Label("Click one, or type e.g. 'the red one':", GUILayout.Width(240));
79
+ foreach (var id in session.Candidates)
80
+ {
81
+ var it = session.world.Items.FirstOrDefault(x => x.itemId == id);
82
+ if (it != null && GUILayout.Button($"{session.Describe(it)} ({id})")) StartCoroutine(session.Answer(id));
83
+ }
84
+ GUILayout.EndHorizontal();
85
+ }
86
+
87
+ GUILayout.BeginHorizontal();
88
+ GUI.SetNextControlName("SceneAgentInput");
89
+ bool enter = Event.current.type == EventType.KeyDown && Event.current.keyCode == KeyCode.Return
90
+ && GUI.GetNameOfFocusedControl() == "SceneAgentInput";
91
+ input = GUILayout.TextField(input, GUILayout.ExpandWidth(true));
92
+ GUI.enabled = !session.Busy;
93
+ if (GUILayout.Button("Send", GUILayout.Width(70)) || enter) Send(input);
94
+ GUI.enabled = true;
95
+ GUILayout.EndHorizontal();
96
+ GUILayout.EndArea();
97
+ }
98
+ }
99
+ }
Runtime/SceneAgentChatBox.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 21283d1231b900f4fb5a1d5c6462ace9
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/SceneAgentGaze.cs ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Sets SceneWorld.gaze to the SceneItem the user points at, so "this" and "that" resolve: the object under the mouse
2
+ // pointer (flat screen) or at the centre of the camera (head gaze in XR). The objects need colliders. While the agent is
3
+ // answering, the gaze is left as it was, so the command sees what the user pointed at when they asked.
4
+ using System;
5
+ using UnityEngine;
6
+
7
+ namespace SceneAgent
8
+ {
9
+ public class SceneAgentGaze : MonoBehaviour
10
+ {
11
+ public enum Source { MousePointer, CameraCentre }
12
+
13
+ public SceneWorld world;
14
+ public SceneAgentSession session;
15
+ [Tooltip("Empty: Camera.main.")]
16
+ public Camera viewCamera;
17
+ [Tooltip("MousePointer falls back to the camera centre when the old Input Manager is off (Input System only).")]
18
+ public Source source = Source.MousePointer;
19
+ public float maxDistance = 30f;
20
+
21
+ /// <summary>The id the user points at now, or null.</summary>
22
+ public string Current { get; private set; }
23
+
24
+ void Update()
25
+ {
26
+ if (world == null) world = FindObjectOfType<SceneWorld>();
27
+ var cam = viewCamera != null ? viewCamera : Camera.main;
28
+ if (world == null || cam == null || (session != null && session.Busy)) return;
29
+ Ray ray = source == Source.MousePointer && TryMouse(out var p)
30
+ ? cam.ScreenPointToRay(p) : cam.ViewportPointToRay(new Vector3(0.5f, 0.5f, 0f));
31
+ string id = null;
32
+ if (Physics.Raycast(ray, out var hit, maxDistance))
33
+ {
34
+ var it = hit.collider.GetComponentInParent<SceneItem>();
35
+ if (it != null && it.room == world.playerRoom) id = it.itemId;
36
+ }
37
+ Current = id;
38
+ world.gaze = id;
39
+ }
40
+
41
+ static bool TryMouse(out Vector3 p)
42
+ {
43
+ try { p = Input.mousePosition; return true; }
44
+ catch (InvalidOperationException) { p = default; return false; } // Input System only
45
+ }
46
+ }
47
+ }
Runtime/SceneAgentGaze.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 624864f83fffffc46adb473c4cb15787
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/SceneAgentScriptRunner.cs ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Plays a list of commands through the agent and writes a report: for QA, regression tests and demos.
2
+ // one command per line; "// ..." comments; "#answer <item_id>" answers the pending question; "#look <item_id>|none"
3
+ // sets the gaze; "#wait <seconds>" pauses (not counted in any step's time).
4
+ // In the Editor: assign a TextAsset and press Play. In a built game or batch run:
5
+ // Game.exe -sceneAgentScript commands.txt [-sceneAgentOut <dir>] [-sceneAgentShots] (quits when done)
6
+ // Report: <out>/report.json with every step's calls, results, answer, error, seconds and the scene signature.
7
+ using System;
8
+ using System.Collections;
9
+ using System.Collections.Generic;
10
+ using System.Globalization;
11
+ using System.IO;
12
+ using System.Linq;
13
+ using UnityEngine;
14
+
15
+ namespace SceneAgent
16
+ {
17
+ [RequireComponent(typeof(SceneAgentSession))]
18
+ public class SceneAgentScriptRunner : MonoBehaviour
19
+ {
20
+ public TextAsset script;
21
+ [Tooltip("Output folder (relative = under Application.persistentDataPath). Empty: SceneAgentRuns/<time>.")]
22
+ public string outputDirectory = "";
23
+ public bool screenshots;
24
+ public bool quitWhenDone;
25
+ public float serverTimeoutSeconds = 240f;
26
+
27
+ public bool Done { get; private set; }
28
+ public string ReportPath { get; private set; }
29
+
30
+ static string Arg(string name)
31
+ {
32
+ var a = Environment.GetCommandLineArgs();
33
+ int i = Array.IndexOf(a, name);
34
+ return i >= 0 && i + 1 < a.Length ? a[i + 1] : null;
35
+ }
36
+
37
+ IEnumerator Start()
38
+ {
39
+ string[] lines;
40
+ string cli = Arg("-sceneAgentScript");
41
+ if (cli != null) { lines = File.ReadAllLines(cli); quitWhenDone = true; }
42
+ else if (script != null) lines = script.text.Replace("\r\n", "\n").Split('\n');
43
+ else yield break;
44
+ if (Environment.GetCommandLineArgs().Contains("-sceneAgentShots")) screenshots = true;
45
+ string outDir = Arg("-sceneAgentOut") ?? outputDirectory;
46
+ if (string.IsNullOrEmpty(outDir)) outDir = Path.Combine("SceneAgentRuns", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
47
+ outDir = Path.GetFullPath(Path.IsPathRooted(outDir) ? outDir : Path.Combine(Application.persistentDataPath, outDir));
48
+ Directory.CreateDirectory(outDir);
49
+
50
+ var session = GetComponent<SceneAgentSession>();
51
+ // the script sets the gaze (#look); the gaze components come back when it ends, for whoever uses the scene next
52
+ var pausedGaze = FindObjectsOfType<SceneAgentGaze>().Where(g => g.enabled).ToList();
53
+ foreach (var g in pausedGaze) g.enabled = false;
54
+ var server = session.agent != null ? session.agent.server : null;
55
+ var steps = new List<object>();
56
+ float t0 = Time.realtimeSinceStartup;
57
+ if (server != null)
58
+ while (!server.Ready && !server.Failed && Time.realtimeSinceStartup - t0 < serverTimeoutSeconds) yield return null;
59
+ var head = new Dictionary<string, object> {
60
+ { "server_mode", server != null ? server.mode.ToString() : "none (agent.endpoint)" },
61
+ { "server_status", server != null ? server.status : session.agent.endpoint },
62
+ { "server_seconds", Math.Round(Time.realtimeSinceStartup - t0, 1) }, { "unity", Application.unityVersion } };
63
+ int n = 0;
64
+ if (server == null || server.Ready)
65
+ {
66
+ foreach (var raw in lines)
67
+ {
68
+ string line = raw.Trim();
69
+ if (line.Length == 0 || line.StartsWith("//")) continue;
70
+ if (line.StartsWith("#wait "))
71
+ {
72
+ yield return new WaitForSecondsRealtime(float.Parse(line.Substring(6).Trim(), CultureInfo.InvariantCulture));
73
+ continue;
74
+ }
75
+ if (line.StartsWith("#look "))
76
+ {
77
+ string id = line.Substring(6).Trim();
78
+ session.world.gaze = id == "none" ? null : id;
79
+ continue;
80
+ }
81
+ if (line.StartsWith("#answer ")) yield return session.Answer(line.Substring(8).Trim());
82
+ else yield return session.Command(line);
83
+
84
+ var r = session.LastResult;
85
+ steps.Add(new Dictionary<string, object> {
86
+ { "step", line }, { "calls", r.turns.SelectMany(t => t.calls).Cast<object>().ToList() },
87
+ { "results", r.turns.SelectMany(t => t.results).Cast<object>().ToList() },
88
+ { "d5b", r.turns.Select(t => (object)t.d5b).ToList() }, { "final_text", r.finalText },
89
+ { "clarification", r.clarificationQuestion }, { "error", r.error },
90
+ { "seconds", Math.Round(r.seconds, 2) }, { "model_calls", (double)r.turns.Count },
91
+ { "room", session.world.playerRoom }, { "signature", session.world.Signature() } });
92
+ if (screenshots)
93
+ {
94
+ ScreenCapture.CaptureScreenshot(Path.Combine(outDir, $"step_{n:00}.png"));
95
+ yield return new WaitForEndOfFrame();
96
+ yield return null;
97
+ }
98
+ n++;
99
+ }
100
+ }
101
+ var secs = steps.Select(s => (double)((Dictionary<string, object>)s)["seconds"]).OrderBy(x => x).ToList();
102
+ head["steps"] = (double)steps.Count;
103
+ head["errors"] = (double)steps.Count(s => ((Dictionary<string, object>)s)["error"] != null);
104
+ head["seconds_p50"] = secs.Count > 0 ? secs[secs.Count / 2] : 0.0;
105
+ head["results"] = steps;
106
+ ReportPath = Path.Combine(outDir, "report.json");
107
+ File.WriteAllText(ReportPath, MiniJson.Serialize(head));
108
+ Debug.Log($"[SceneAgent] script: {steps.Count} steps, report {ReportPath}");
109
+ foreach (var g in pausedGaze) if (g != null) g.enabled = true;
110
+ Done = true;
111
+ if (quitWhenDone) Application.Quit();
112
+ }
113
+ }
114
+ }
Runtime/SceneAgentScriptRunner.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 6f6265ce4b363674a9792c5129374cb7
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/SceneAgentSession.cs ADDED
@@ -0,0 +1,187 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // One conversation with the agent: runs commands, keeps the pending clarification, turns the user's pick into the
2
+ // follow-up the model was trained on (the original request plus a hint, with the gaze on the picked object), and
3
+ // optionally writes every exchange to a JSON-lines log (what was asked, the scene the model saw, calls, answer, time).
4
+ // SceneAgentChatBox and SceneAgentScriptRunner use it; your own UI can too.
5
+ using System;
6
+ using System.Collections;
7
+ using System.Collections.Generic;
8
+ using System.IO;
9
+ using System.Linq;
10
+ using UnityEngine;
11
+
12
+ namespace SceneAgent
13
+ {
14
+ public class SceneAgentSession : MonoBehaviour
15
+ {
16
+ public SceneAgent agent;
17
+ public SceneWorld world;
18
+ [Tooltip("Folder for a JSON-lines log of every command (relative = under Application.persistentDataPath). Empty: no log.")]
19
+ public string logDirectory = "";
20
+ [Tooltip("Line shows each tool call as raw JSON (for developers) instead of a short readable form.")]
21
+ public bool rawToolLines;
22
+
23
+ public bool Busy { get; private set; }
24
+ public AgentResult LastResult { get; private set; }
25
+ /// <summary>The request waiting for the user's pick after ask_clarification (null when none).</summary>
26
+ public string PendingRequest { get; private set; }
27
+ /// <summary>The ids the model offered in its last question.</summary>
28
+ public List<string> Candidates => world != null ? world.clarify.ToList() : new List<string>();
29
+ /// <summary>Human-readable lines: the request, each call and its result, the answer.</summary>
30
+ public event Action<string> Line;
31
+ public event Action<AgentResult> Finished;
32
+ public string LogFile { get; private set; }
33
+
34
+ int index = -1;
35
+
36
+ void Awake()
37
+ {
38
+ if (agent == null) agent = GetComponent<SceneAgent>();
39
+ if (world == null) world = agent != null && agent.world != null ? agent.world : GetComponent<SceneWorld>();
40
+ // "Game.exe -sceneAgentScript commands.txt" works in any scene with a session, not only where a runner was
41
+ // added (found by the user test: the flag was silently ignored)
42
+ if (Environment.GetCommandLineArgs().Contains("-sceneAgentScript") && GetComponent<SceneAgentScriptRunner>() == null)
43
+ gameObject.AddComponent<SceneAgentScriptRunner>();
44
+ }
45
+
46
+ /// <summary>Runs one user command (typed or recognised speech). While a question is pending, a short reply that
47
+ /// tells the candidates apart ("the blue one", "the one on the shelf") answers it, in the follow-up form the model
48
+ /// was trained on ("pick up the toolbox I mean the blue one"); anything else is a new command.</summary>
49
+ public IEnumerator Command(string text)
50
+ {
51
+ text = text?.Trim();
52
+ if (string.IsNullOrEmpty(text)) yield break;
53
+ if (PendingRequest != null && LooksLikeAnswer(text))
54
+ {
55
+ string reply = text.StartsWith("i mean", StringComparison.OrdinalIgnoreCase) ? text : "I mean " + text;
56
+ string request = $"{PendingRequest} {reply}";
57
+ PendingRequest = null;
58
+ yield return Run(request, null);
59
+ yield break;
60
+ }
61
+ PendingRequest = null;
62
+ yield return Run(text, null);
63
+ }
64
+
65
+ static readonly HashSet<string> CommandVerbs = new HashSet<string> {
66
+ "pick", "put", "place", "grab", "take", "drop", "release", "open", "close", "shut", "turn", "switch", "move",
67
+ "push", "pull", "rotate", "spin", "highlight", "show", "find", "where", "bring", "fetch", "get", "go", "teleport",
68
+ "press", "lift", "set", "make", "throw", "toss", "hand", "give", "lock", "unlock", "what", "how", "can", "could" };
69
+ static readonly HashSet<string> AnswerStarts = new HashSet<string> {
70
+ "the", "that", "this", "one", "it", "it's", "its", "number", "left", "right", "first", "second", "third", "last",
71
+ "top", "bottom", "front", "back", "near", "closest", "furthest", "biggest", "smallest", "both", "neither" };
72
+
73
+ /// <summary>Whether text typed while a question is pending answers it (see Command).</summary>
74
+ public bool LooksLikeAnswer(string text)
75
+ {
76
+ var words = text.ToLowerInvariant().Split(new[] { ' ', ',', '.', '!', '?', ';' }, StringSplitOptions.RemoveEmptyEntries);
77
+ if (words.Length == 0 || words.Length > 8 || CommandVerbs.Contains(words[0])) return false;
78
+ if (words.Length > 1 && words[0] == "i" && words[1] == "mean") return true;
79
+ if (AnswerStarts.Contains(words[0])) return true;
80
+ // a word that tells the offered objects apart: a colour, a label, what one of them stands on
81
+ var cues = new HashSet<string>(Candidates.Select(id => world.Items.FirstOrDefault(x => x.itemId == id)).Where(x => x != null)
82
+ .SelectMany(c => new[] { c.color, c.label, LabelOf(c.on) }).Where(s => !string.IsNullOrEmpty(s)).SelectMany(s => s.Split(' ')));
83
+ return words.Any(cues.Contains);
84
+ }
85
+
86
+ /// <summary>Answers the pending question with the object the user picked.</summary>
87
+ public IEnumerator Answer(string itemId)
88
+ {
89
+ var it = world.Items.FirstOrDefault(x => x.itemId == itemId);
90
+ if (it == null || PendingRequest == null) yield break;
91
+ var cands = world.clarify.Select(id => world.Items.FirstOrDefault(x => x.itemId == id)).Where(x => x != null).ToList();
92
+ string request = $"{PendingRequest} {Hint(it, cands)}";
93
+ PendingRequest = null;
94
+ yield return Run(request, itemId);
95
+ }
96
+
97
+ IEnumerator Run(string request, string gazeItem)
98
+ {
99
+ Busy = true;
100
+ world.highlighted.Clear(); // highlights and questions belong to the previous command
101
+ world.clarify.Clear();
102
+ if (gazeItem != null) world.gaze = gazeItem;
103
+ Line?.Invoke("You: " + request);
104
+ string scene = world.BuildSceneJson();
105
+ AgentResult res = null;
106
+ yield return agent.Run(request, r => res = r);
107
+ foreach (var turn in res.turns)
108
+ for (int i = 0; i < turn.calls.Count; i++)
109
+ {
110
+ string result = i < turn.results.Count ? turn.results[i] : null;
111
+ Line?.Invoke(rawToolLines ? " → " + turn.calls[i] + (result != null ? " " + result : "") : " → " + Pretty(turn.calls[i], result));
112
+ }
113
+ if (res.error != null) Line?.Invoke("Error: " + res.error);
114
+ if (res.clarificationQuestion != null) { PendingRequest = request; Line?.Invoke("Agent: " + res.clarificationQuestion); }
115
+ else if (!string.IsNullOrEmpty(res.finalText)) Line?.Invoke("Agent: " + res.finalText);
116
+ foreach (var it in world.Items) world.Sync(it);
117
+ Record(request, gazeItem, scene, res);
118
+ LastResult = res;
119
+ Busy = false;
120
+ Finished?.Invoke(res);
121
+ }
122
+
123
+ /// <summary>"I mean the red one." / "I mean the one on the shelf." / "I mean the mug I'm looking at."</summary>
124
+ public string Hint(SceneItem it, List<SceneItem> candidates)
125
+ {
126
+ var others = candidates.Where(c => c.itemId != it.itemId).ToList();
127
+ if (!string.IsNullOrEmpty(it.color) && others.All(c => c.color != it.color)) return $"I mean the {it.color} one.";
128
+ if (others.All(c => c.on != it.on))
129
+ return it.on == "floor" ? "I mean the one on the floor." : $"I mean the one on the {LabelOf(it.on)}.";
130
+ return $"I mean the {Describe(it)} I'm looking at.";
131
+ }
132
+
133
+ /// <summary>A tool call for people: place(hammer_1, toolbox_1, in) ok / not done: the reason / found 2.</summary>
134
+ public static string Pretty(string callJson, string resultJson)
135
+ {
136
+ string call;
137
+ try
138
+ {
139
+ var c = (Dictionary<string, object>)MiniJson.Parse(callJson);
140
+ var a = c.TryGetValue("arguments", out var x) ? x as Dictionary<string, object> : null;
141
+ string args = a == null ? "" : string.Join(", ", a.Values.Select(v => v is List<object> l
142
+ ? string.Join(" + ", l.Select(Convert.ToString)) : Convert.ToString(v, System.Globalization.CultureInfo.InvariantCulture)));
143
+ call = $"{c["name"]}({args})";
144
+ }
145
+ catch (Exception) { return callJson; }
146
+ if (resultJson == null) return call;
147
+ try
148
+ {
149
+ var r = MiniJson.Parse(resultJson);
150
+ if (r is Dictionary<string, object> d && d.TryGetValue("ok", out var ok) && ok is bool b)
151
+ return call + (b ? " ok" : " not done: " + (d.TryGetValue("error", out var e) ? Convert.ToString(e) : "").Replace("not done: ", ""));
152
+ if (r is List<object> found) return call + $" found {found.Count}";
153
+ }
154
+ catch (Exception) { }
155
+ return call + " " + resultJson;
156
+ }
157
+
158
+ /// <summary>"red mug" / "lamp": what a button for this object should say.</summary>
159
+ public string Describe(SceneItem it) => string.IsNullOrEmpty(it.color) ? it.label : $"{it.color} {it.label}";
160
+
161
+ string LabelOf(string id) => world.Items.FirstOrDefault(x => x.itemId == id)?.label ?? id;
162
+
163
+ void Record(string request, string answer, string scene, AgentResult res)
164
+ {
165
+ if (string.IsNullOrEmpty(logDirectory)) return;
166
+ try
167
+ {
168
+ if (LogFile == null)
169
+ {
170
+ string dir = Path.IsPathRooted(logDirectory) ? logDirectory : Path.Combine(Application.persistentDataPath, logDirectory);
171
+ Directory.CreateDirectory(dir);
172
+ LogFile = Path.Combine(dir, $"session_{DateTime.Now:yyyyMMdd_HHmmss}.jsonl");
173
+ }
174
+ index++;
175
+ var rec = new Dictionary<string, object> {
176
+ { "i", (double)index }, { "time", DateTime.Now.ToString("s") }, { "request", request }, { "clarification_answer", answer },
177
+ { "scene", scene }, { "calls", res.turns.SelectMany(t => t.calls).Cast<object>().ToList() },
178
+ { "results", res.turns.SelectMany(t => t.results).Cast<object>().ToList() },
179
+ { "d5b", res.turns.Select(t => (object)t.d5b).ToList() }, { "question", res.clarificationQuestion },
180
+ { "final_text", res.finalText }, { "error", res.error }, { "seconds", Math.Round(res.seconds, 2) },
181
+ { "model_calls", (double)res.turns.Count } };
182
+ File.AppendAllText(LogFile, MiniJson.Serialize(rec) + "\n");
183
+ }
184
+ catch (Exception e) { Debug.LogWarning("[SceneAgent] session log: " + e.Message); }
185
+ }
186
+ }
187
+ }
Runtime/SceneAgentSession.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: d6ebd95c1175b72478f4b1d6708c54a5
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/SceneCatalog.Generated.cs ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // GENERATED by `py tools/gen_unity_catalog.py` from datagen/sim.py + the v3 catalogue (datagen/v3/core.py).
2
+ // Do not edit: regenerate. SceneWorld.cs checks tool calls against these, exactly as the Python simulator does.
3
+ using System.Collections.Generic;
4
+
5
+ namespace SceneAgent
6
+ {
7
+ public static class SceneCatalog
8
+ {
9
+ public static readonly Dictionary<string, double> Top = new Dictionary<string, double> { { "armchair", 0.45 }, { "bed", 0.5 }, { "bench", 0.45 }, { "cart", 0.8 }, { "coffee maker", 0.3 }, { "coffee table", 0.45 }, { "counter", 0.9 }, { "cutting board", 0.75 }, { "desk", 0.75 }, { "dining table", 0.75 }, { "dresser", 0.9 }, { "nightstand", 0.6 }, { "ottoman", 0.4 }, { "pallet", 0.15 }, { "shelf", 1.2 }, { "side table", 0.6 }, { "sofa", 0.45 }, { "stove", 0.9 }, { "table", 0.75 }, { "toilet paper hanger", 0.6 }, { "towel holder", 1.0 }, { "tv stand", 0.5 }, { "workbench", 0.9 } };
10
+ public static readonly HashSet<string> Grasp = new HashSet<string> { "alarm clock", "aluminum foil", "apple", "backpack", "ball", "banana", "baseball bat", "basketball", "battery", "blanket", "blender", "book", "boots", "bottle", "bowl", "box", "bread", "bucket", "butter knife", "cable", "calculator", "camera", "candle", "carrot", "cd", "charger", "clipboard", "cloth", "computer mouse", "controller", "credit card", "cube", "cup", "cutting board", "drill", "drone", "dumbbell", "egg", "envelope", "fire extinguisher", "first aid kit", "flashlight", "flower", "folder", "fork", "globe", "gloves", "hair dryer", "hammer", "hand towel", "headphones", "helmet", "jar", "kettle", "key", "key chain", "keyboard", "knife", "ladle", "laptop", "lettuce", "marker", "microphone", "mug", "newspaper", "notebook", "orange", "pan", "paper towel roll", "pen", "pencil", "pepper shaker", "phone", "picture frame", "pillow", "plate", "pliers", "plunger", "pot", "potato", "radio", "remote", "ruler", "salt shaker", "saw", "scissors", "screw box", "screwdriver", "scrub brush", "soap", "soap bottle", "spatula", "sponge", "spoon", "spray bottle", "stapler", "statue", "suitcase", "tablet", "tape", "tape measure", "teddy bear", "tennis racket", "thermos", "tissue box", "toilet paper", "tomato", "toolbox", "toothbrush", "towel", "traffic cone", "trophy", "umbrella", "vr headset", "watch", "water bottle", "watering can", "wine bottle", "wrench" };
11
+ public static readonly HashSet<string> Container = new HashSet<string> { "backpack", "barrel", "basket", "bathtub", "bowl", "box", "bucket", "cabinet", "candle holder", "chest", "cup", "desk organizer", "dishwasher", "drawer", "filing cabinet", "first aid kit", "fridge", "jar", "laundry hamper", "microwave", "oven", "pan", "pot", "safe", "screw box", "sink", "suitcase", "toaster", "toilet", "toolbox", "trash bin", "trash can", "vase", "wardrobe" };
12
+ public static readonly HashSet<string> Fixed = new HashSet<string> { "air conditioner", "bathtub", "blinds", "button", "cabinet", "ceiling light", "clock", "control panel", "dishwasher", "door", "drawer", "faucet", "filing cabinet", "fridge", "garage door", "hologram panel", "lever", "light switch", "mirror", "oven", "painting", "poster", "robot arm", "shower curtain", "shower door", "shower head", "sink", "stove", "stove knob", "terminal", "toilet", "toilet paper hanger", "towel holder", "valve", "wardrobe", "whiteboard", "window" };
13
+ public static readonly HashSet<string> Button = new HashSet<string> { "button", "control panel", "lever" };
14
+ public static readonly Dictionary<string, string[]> States = new Dictionary<string, string[]> {
15
+ { "air conditioner", new[] { "on", "off" } },
16
+ { "blender", new[] { "on", "off" } },
17
+ { "blinds", new[] { "open", "closed" } },
18
+ { "cabinet", new[] { "open", "closed" } },
19
+ { "candle", new[] { "on", "off" } },
20
+ { "ceiling light", new[] { "on", "off" } },
21
+ { "chest", new[] { "open", "closed", "locked", "unlocked" } },
22
+ { "coffee maker", new[] { "on", "off" } },
23
+ { "desk lamp", new[] { "on", "off" } },
24
+ { "dishwasher", new[] { "on", "off", "open", "closed" } },
25
+ { "door", new[] { "open", "closed", "half_open" } },
26
+ { "drawer", new[] { "open", "closed" } },
27
+ { "fan", new[] { "on", "off" } },
28
+ { "faucet", new[] { "on", "off" } },
29
+ { "filing cabinet", new[] { "open", "closed" } },
30
+ { "flashlight", new[] { "on", "off" } },
31
+ { "floor lamp", new[] { "on", "off" } },
32
+ { "fridge", new[] { "open", "closed" } },
33
+ { "garage door", new[] { "open", "closed" } },
34
+ { "hair dryer", new[] { "on", "off" } },
35
+ { "hologram panel", new[] { "on", "off" } },
36
+ { "kettle", new[] { "open", "closed" } },
37
+ { "lamp", new[] { "on", "off" } },
38
+ { "laptop", new[] { "on", "off", "open", "closed" } },
39
+ { "light switch", new[] { "on", "off" } },
40
+ { "microwave", new[] { "on", "off", "open", "closed" } },
41
+ { "monitor", new[] { "on", "off" } },
42
+ { "oven", new[] { "on", "off", "open", "closed" } },
43
+ { "printer", new[] { "on", "off" } },
44
+ { "projector", new[] { "on", "off" } },
45
+ { "radio", new[] { "on", "off" } },
46
+ { "robot arm", new[] { "on", "off" } },
47
+ { "safe", new[] { "open", "closed", "locked", "unlocked" } },
48
+ { "shower curtain", new[] { "open", "closed" } },
49
+ { "shower door", new[] { "open", "closed" } },
50
+ { "shower head", new[] { "on", "off" } },
51
+ { "speaker", new[] { "on", "off" } },
52
+ { "stove", new[] { "on", "off" } },
53
+ { "stove knob", new[] { "on", "off" } },
54
+ { "terminal", new[] { "on", "off" } },
55
+ { "toaster", new[] { "on", "off" } },
56
+ { "toilet", new[] { "open", "closed" } },
57
+ { "tv", new[] { "on", "off" } },
58
+ { "valve", new[] { "open", "closed" } },
59
+ { "wardrobe", new[] { "open", "closed" } },
60
+ { "window", new[] { "open", "closed" } } };
61
+ }
62
+ }
Runtime/SceneCatalog.Generated.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 1dbb1530f90ec5141a9db4c4dad898d8
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/SceneGrammar.Check.cs ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // D5b (docs/v3_data_plan.md §20.3): the model answers freely, then its output is checked against the schema and the
2
+ // scene, with the same verdicts as eval/grammar.py check():
3
+ // "ok" plain text, or every call names a known tool with known arguments, allowed values and known ids
4
+ // "invalid_tool" a tool the schema does not have -> the agent refuses (Refusal)
5
+ // "invalid_value" an argument, allowed value or room the schema/scene does not have -> the agent refuses
6
+ // "invalid_id" an id outside the scene, the hands, the gaze and earlier tool results -> generate again under the grammar
7
+ // "malformed" a call that does not parse -> generate again under the grammar
8
+ // The first out-of-schema call decides at once; an unknown id is remembered while the rest is checked (as in Python).
9
+ using System.Collections.Generic;
10
+ using System.Linq;
11
+ using System.Text.RegularExpressions;
12
+
13
+ namespace SceneAgent
14
+ {
15
+ public static partial class SceneGrammar
16
+ {
17
+ public const string Refusal = "I can't do that; none of my tools can.";
18
+ static readonly HashSet<string> IdKeys = new HashSet<string> { "object_id", "target_id" };
19
+ static readonly HashSet<string> IdListKeys = new HashSet<string> { "object_ids", "candidate_ids" };
20
+ static readonly HashSet<string> RoomKeys = new HashSet<string> { "location_id", "room" };
21
+ static readonly Regex CallRe = new Regex(@"<tool_call>\s*(.*?)\s*</tool_call>", RegexOptions.Singleline);
22
+
23
+ /// <summary>The ids the model may name (scene objects, hands, gaze, ids in earlier tool results) and the rooms.</summary>
24
+ public static void Vocab(string sceneJson, IEnumerable<string> toolResultJsons, out List<string> ids, out List<string> rooms)
25
+ {
26
+ var scene = (Dictionary<string, object>)MiniJson.Parse(sceneJson);
27
+ ids = new List<string>();
28
+ if (scene.TryGetValue("visible_objects", out var vo) && vo is List<object> objs)
29
+ foreach (var o in objs)
30
+ if (o is Dictionary<string, object> d && d.TryGetValue("id", out var id) && id is string s) ids.Add(s);
31
+ if (scene.TryGetValue("player", out var pl) && pl is Dictionary<string, object> player)
32
+ foreach (var k in new[] { "left_hand", "right_hand", "gaze" })
33
+ if (player.TryGetValue(k, out var v) && v is string s && s.Length > 0) ids.Add(s);
34
+ foreach (var r in toolResultJsons ?? Enumerable.Empty<string>())
35
+ {
36
+ try { CollectIds(MiniJson.Parse(r), ids); }
37
+ catch (System.FormatException) { }
38
+ }
39
+ ids = ids.Distinct().ToList();
40
+ rooms = scene.TryGetValue("locations", out var l) && l is List<object> ls
41
+ ? ls.OfType<string>().Distinct().ToList() : new List<string>();
42
+ }
43
+
44
+ public static string Check(string text, string sceneJson, IEnumerable<string> toolResultJsons)
45
+ {
46
+ text = text ?? "";
47
+ var blocks = CallRe.Matches(text).Cast<Match>().Select(m => m.Groups[1].Value).ToList();
48
+ if (blocks.Count == 0) return text.Contains("<tool_call>") ? "malformed" : "ok";
49
+ Vocab(sceneJson, toolResultJsons, out var idList, out var rooms);
50
+ var ids = new HashSet<string>(idList);
51
+ string verdict = "ok";
52
+ foreach (var b in blocks)
53
+ {
54
+ object parsed;
55
+ try { parsed = MiniJson.ParseStrict(b); }
56
+ catch (System.Exception) { return "malformed"; }
57
+ if (!(parsed is Dictionary<string, object> obj) || !obj.TryGetValue("name", out var nameObj)) return "malformed";
58
+ if (!(nameObj is string name) || !Tools.TryGetValue(name, out var spec)) return "invalid_tool";
59
+ obj.TryGetValue("arguments", out var argsObj);
60
+ // Python: obj.get("arguments") or {} (an empty or false value counts as no arguments)
61
+ bool empty = argsObj == null || argsObj is List<object> el && el.Count == 0 || argsObj is string es && es.Length == 0
62
+ || argsObj is double dz && dz == 0 || argsObj is bool bf && !bf;
63
+ if (!empty && !(argsObj is Dictionary<string, object>)) return "malformed";
64
+ var args = empty ? new Dictionary<string, object>() : (Dictionary<string, object>)argsObj;
65
+ foreach (var kv in args)
66
+ {
67
+ string k = kv.Key;
68
+ object v = kv.Value;
69
+ if (!spec.TryGetValue(k, out var allowed)) return "invalid_value";
70
+ if (allowed != null && !(v is string sv && allowed.Contains(sv))) return "invalid_value";
71
+ if (RoomKeys.Contains(k) && rooms.Count > 0 && !(v is string rv && rooms.Contains(rv))) return "invalid_value";
72
+ if (IdKeys.Contains(k) && !(v is string iv && ids.Contains(iv))) verdict = "invalid_id";
73
+ if (IdListKeys.Contains(k) && NamesUnknownId(v, ids)) verdict = "invalid_id";
74
+ }
75
+ }
76
+ return verdict;
77
+ }
78
+
79
+ // Python: any(x not in ids for x in (v or [])); a string is iterated character by character there
80
+ static bool NamesUnknownId(object v, HashSet<string> ids)
81
+ {
82
+ switch (v)
83
+ {
84
+ case null: return false;
85
+ case List<object> l: return l.Any(x => !(x is string s && ids.Contains(s)));
86
+ case string s: return s.Length > 0 && s.Any(ch => !ids.Contains(ch.ToString()));
87
+ default: return true;
88
+ }
89
+ }
90
+ }
91
+ }
Runtime/SceneGrammar.Check.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 6c1709e54f65d5941a3071d744bbf4dd
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/SceneGrammar.Schema.cs ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // GENERATED by `py tools/gen_unity_schema.py` from spec/tools_v0.json. Do not edit: regenerate.
2
+ // Per tool: its arguments and their allowed values (null = any value of the argument's type). Used by D5b.
3
+ using System.Collections.Generic;
4
+
5
+ namespace SceneAgent
6
+ {
7
+ public static partial class SceneGrammar
8
+ {
9
+ public static readonly Dictionary<string, Dictionary<string, string[]>> Tools = new Dictionary<string, Dictionary<string, string[]>>
10
+ {
11
+ { "grab", new Dictionary<string, string[]> { { "object_id", null }, { "hand", new[] { "left", "right", "both" } } } },
12
+ { "release", new Dictionary<string, string[]> { { "hand", new[] { "left", "right", "both" } } } },
13
+ { "place", new Dictionary<string, string[]> { { "object_id", null }, { "target_id", null }, { "relation", new[] { "on", "in" } }, { "orientation", new[] { "default", "upright", "horizontal", "vertical", "upside_down" } } } },
14
+ { "move_by", new Dictionary<string, string[]> { { "object_id", null }, { "direction", new[] { "left", "right", "forward", "backward", "up", "down" } }, { "distance", null }, { "unit", new[] { "m", "cm" } } } },
15
+ { "rotate", new Dictionary<string, string[]> { { "object_id", null }, { "degrees", null }, { "axis", new[] { "yaw", "pitch", "roll" } } } },
16
+ { "set_state", new Dictionary<string, string[]> { { "object_id", null }, { "state", new[] { "on", "off", "open", "closed", "half_open", "locked", "unlocked" } } } },
17
+ { "press", new Dictionary<string, string[]> { { "object_id", null }, { "hold_seconds", null } } },
18
+ { "highlight", new Dictionary<string, string[]> { { "object_ids", null } } },
19
+ { "teleport", new Dictionary<string, string[]> { { "location_id", null } } },
20
+ { "find_objects", new Dictionary<string, string[]> { { "label", null }, { "color", null }, { "room", null } } },
21
+ { "ask_clarification", new Dictionary<string, string[]> { { "question", null }, { "candidate_ids", null } } },
22
+ };
23
+ }
24
+ }
Runtime/SceneGrammar.Schema.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 9692717792d71e247abd473394d876a5
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/SceneGrammar.Static.cs ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // GENERATED by `py eval/grammar.py --unity` from spec/tools_v0.json. Do not edit: regenerate.
2
+ namespace SceneAgent
3
+ {
4
+ public static partial class SceneGrammar
5
+ {
6
+ public const string Static = "root ::= think? ( calls | text )\nthink ::= \"<think>\\n\\n</think>\\n\\n\"\ntext ::= [^<] [^<]*\ncalls ::= call ( \"\\n\" call )*\ncall ::= \"<tool_call>\\n{\\\"name\\\": \" tool \"}\\n</tool_call>\"\ntool ::= t-grab | t-release | t-place | t-move-by | t-rotate | t-set-state | t-press | t-highlight | t-teleport | t-find-objects | t-ask-clarification\nt-grab ::= \"\\\"grab\\\", \\\"arguments\\\": {\" \"\\\"object_id\\\": \" id \", \\\"hand\\\": \" ( \"\\\"left\\\"\" | \"\\\"right\\\"\" | \"\\\"both\\\"\" ) \"}\"\nt-release ::= \"\\\"release\\\", \\\"arguments\\\": {\" \"\\\"hand\\\": \" ( \"\\\"left\\\"\" | \"\\\"right\\\"\" | \"\\\"both\\\"\" ) \"}\"\nt-place ::= \"\\\"place\\\", \\\"arguments\\\": {\" \"\\\"object_id\\\": \" id \", \\\"target_id\\\": \" id ( \", \\\"relation\\\": \" ( \"\\\"on\\\"\" | \"\\\"in\\\"\" ) )? ( \", \\\"orientation\\\": \" ( \"\\\"default\\\"\" | \"\\\"upright\\\"\" | \"\\\"horizontal\\\"\" | \"\\\"vertical\\\"\" | \"\\\"upside_down\\\"\" ) )? \"}\"\nt-move-by ::= \"\\\"move_by\\\", \\\"arguments\\\": {\" \"\\\"object_id\\\": \" id \", \\\"direction\\\": \" ( \"\\\"left\\\"\" | \"\\\"right\\\"\" | \"\\\"forward\\\"\" | \"\\\"backward\\\"\" | \"\\\"up\\\"\" | \"\\\"down\\\"\" ) \", \\\"distance\\\": \" num \", \\\"unit\\\": \" ( \"\\\"m\\\"\" | \"\\\"cm\\\"\" ) \"}\"\nt-rotate ::= \"\\\"rotate\\\", \\\"arguments\\\": {\" \"\\\"object_id\\\": \" id \", \\\"degrees\\\": \" num ( \", \\\"axis\\\": \" ( \"\\\"yaw\\\"\" | \"\\\"pitch\\\"\" | \"\\\"roll\\\"\" ) )? \"}\"\nt-set-state ::= \"\\\"set_state\\\", \\\"arguments\\\": {\" \"\\\"object_id\\\": \" id \", \\\"state\\\": \" ( \"\\\"on\\\"\" | \"\\\"off\\\"\" | \"\\\"open\\\"\" | \"\\\"closed\\\"\" | \"\\\"half_open\\\"\" | \"\\\"locked\\\"\" | \"\\\"unlocked\\\"\" ) \"}\"\nt-press ::= \"\\\"press\\\", \\\"arguments\\\": {\" \"\\\"object_id\\\": \" id ( \", \\\"hold_seconds\\\": \" num )? \"}\"\nt-highlight ::= \"\\\"highlight\\\", \\\"arguments\\\": {\" \"\\\"object_ids\\\": \" idlist \"}\"\nt-teleport ::= \"\\\"teleport\\\", \\\"arguments\\\": {\" \"\\\"location_id\\\": \" room \"}\"\nt-find-objects ::= \"\\\"find_objects\\\", \\\"arguments\\\": {\" \"\\\"label\\\": \" str ( \", \\\"color\\\": \" str )? ( \", \\\"room\\\": \" room )? \"}\"\nt-ask-clarification ::= \"\\\"ask_clarification\\\", \\\"arguments\\\": {\" \"\\\"question\\\": \" str \", \\\"candidate_ids\\\": \" idlist \"}\"\nidlist ::= \"[\" id ( \", \" id )* \"]\"\nnum ::= \"-\"? [0-9]+ ( \".\" [0-9]+ )?\nstr ::= \"\\\"\" [^\"\\\\\\n]* \"\\\"\"\n";
7
+ }
8
+ }
Runtime/SceneGrammar.Static.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: cf9c0452f2a98b24bb83f476cb40105d
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/SceneGrammar.cs ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Grammar-constrained decoding (docs/v3_data_plan.md §7.5, decision D5): the same GBNF grammar as eval/grammar.py.
2
+ // The static part (the 11 tools, their arguments in the trained order, the <tool_call> format, plain text that cannot
3
+ // start a half-written call) is generated into SceneGrammar.Static.cs. This file adds the per-request rules: the ids
4
+ // the model may name (the scene's objects, the player's hands and gaze, and every id an earlier tool result returned,
5
+ // e.g. objects find_objects reported in another room) and the scene's rooms. llama-server applies it as "grammar".
6
+ using System.Collections.Generic;
7
+ using System.Linq;
8
+
9
+ namespace SceneAgent
10
+ {
11
+ public static partial class SceneGrammar
12
+ {
13
+ public static string Build(string sceneJson, IEnumerable<string> toolResultJsons)
14
+ {
15
+ var scene = (Dictionary<string, object>)MiniJson.Parse(sceneJson);
16
+ var ids = new List<string>();
17
+ if (scene.TryGetValue("visible_objects", out var vo) && vo is List<object> objs)
18
+ foreach (var o in objs)
19
+ if (o is Dictionary<string, object> d && d.TryGetValue("id", out var id) && id is string s) ids.Add(s);
20
+ if (scene.TryGetValue("player", out var pl) && pl is Dictionary<string, object> player)
21
+ foreach (var k in new[] { "left_hand", "right_hand", "gaze" })
22
+ if (player.TryGetValue(k, out var v) && v is string s && s.Length > 0) ids.Add(s);
23
+ foreach (var r in toolResultJsons)
24
+ {
25
+ try { CollectIds(MiniJson.Parse(r), ids); }
26
+ catch (System.FormatException) { }
27
+ }
28
+ var rooms = scene.TryGetValue("locations", out var l) && l is List<object> ls
29
+ ? ls.OfType<string>().Distinct().ToList() : new List<string>();
30
+ return Static + "id ::= " + Alts(ids.Distinct()) + "\nroom ::= " + (rooms.Count > 0 ? Alts(rooms) : "str") + "\n";
31
+ }
32
+
33
+ // each value written as a JSON string, as a GBNF literal: cup_1 -> "\"cup_1\""
34
+ static string Alts(IEnumerable<string> values) =>
35
+ string.Join(" | ", values.Select(v => MiniJson.Serialize(MiniJson.Serialize(v))));
36
+
37
+ static void CollectIds(object x, List<string> ids)
38
+ {
39
+ if (x is Dictionary<string, object> d)
40
+ {
41
+ if (d.TryGetValue("id", out var id) && id is string s) ids.Add(s);
42
+ foreach (var v in d.Values) CollectIds(v, ids);
43
+ }
44
+ else if (x is List<object> l)
45
+ foreach (var v in l) CollectIds(v, ids);
46
+ }
47
+ }
48
+ }
Runtime/SceneGrammar.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 7be55edb0d0055b48824c03c6fe283a3
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/SceneItem.cs ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ using UnityEngine;
2
+
3
+ namespace SceneAgent
4
+ {
5
+ /// <summary>Marks a GameObject as addressable by the agent. Ids must be unique and stable.</summary>
6
+ public class SceneItem : MonoBehaviour
7
+ {
8
+ public string itemId;
9
+ public string label; // cube, mug, table, lamp, door, button, ...
10
+ public string color; // optional
11
+ public string room;
12
+ public string on = "floor"; // support id, "floor", "left_hand", "right_hand" or "both_hands"
13
+ public string relation; // "on" / "in" once placed
14
+ public string orientation; // set by place(); null = default
15
+ public string state; // for devices: on/off/open/closed/...
16
+ public bool hasPos = true;
17
+ public double[] pos = new double[3]; // logical position (bottom of the object), metres
18
+ public double yaw, pitch, roll;
19
+
20
+ [Header("Affordances")]
21
+ [Tooltip("Off: what this object can do comes from the model's catalogue of ~200 object types, looked up by its " +
22
+ "label (SceneCatalog). On: the fields below decide instead; use it for labels the catalogue does not know.")]
23
+ public bool overrideAffordances;
24
+ public bool grabbable;
25
+ [Tooltip("Things can be placed on it; surfaceHeight is where they land, above the object's bottom (metres).")]
26
+ public bool isSurface;
27
+ public float surfaceHeight = 0.75f;
28
+ [Tooltip("Things can be placed in it (unless its state is closed or locked).")]
29
+ public bool isContainer;
30
+ [Tooltip("Cannot be moved or rotated (doors, built-in appliances, ...).")]
31
+ public bool isFixed;
32
+ public bool isButton;
33
+ [Tooltip("States set_state may put it in, e.g. on, off / open, closed. Empty: none.")]
34
+ public string[] states = new string[0];
35
+ }
36
+ }
Runtime/SceneItem.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 038ee4e44111e334eaaf9d553ed7af9c
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/SceneWorld.cs ADDED
@@ -0,0 +1,319 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Scene state + tool semantics on real GameObjects. Mirrors datagen/sim.py (and
2
+ // eval/unity/SceneToolEval.cs) so a model trained on the generator behaves identically here.
3
+ using System;
4
+ using System.Collections.Generic;
5
+ using System.Globalization;
6
+ using System.Linq;
7
+ using UnityEngine;
8
+
9
+ namespace SceneAgent
10
+ {
11
+ public class ToolError : Exception { public ToolError(string m) : base(m) { } }
12
+
13
+ public class SceneWorld : MonoBehaviour
14
+ {
15
+ public string playerRoom;
16
+ public string gaze;
17
+ public string leftHand, rightHand;
18
+ public List<string> rooms = new List<string>();
19
+ public Transform leftHandAnchor, rightHandAnchor;
20
+ public readonly SortedSet<string> highlighted = new SortedSet<string>(StringComparer.Ordinal);
21
+ public SortedSet<string> clarify = new SortedSet<string>(StringComparer.Ordinal);
22
+ public readonly SortedDictionary<string, double> pressed = new SortedDictionary<string, double>(StringComparer.Ordinal);
23
+ public string lastClarifyQuestion;
24
+ /// <summary>Where a room sits in the Unity scene (rooms side by side in a demo). The model only ever sees
25
+ /// room-local positions; null = every room at the origin (evaluation scenes).</summary>
26
+ [NonSerialized] public Func<string, Vector3> roomOrigin;
27
+ [Tooltip("On: at Start every SceneItem's logical position and rotation are read from where it stands in the scene, " +
28
+ "and the rooms and the player's room are filled in from the items when empty. Leave off when the scene " +
29
+ "is built from data (the evaluation harness, the demo builder).")]
30
+ public bool initFromTransforms;
31
+ /// <summary>Raised after every tool call Unity accepted: (tool, arguments, result). Hook animations, sounds or the
32
+ /// camera here, e.g. move the player rig when "teleport" changes playerRoom.</summary>
33
+ public event Action<string, Dictionary<string, object>, object> ToolExecuted;
34
+
35
+ // affordances per label: generated from the Python simulator + the v3 catalogue (SceneCatalog.Generated.cs,
36
+ // `py tools/gen_unity_catalog.py`), so Unity accepts exactly the calls the offline scorer accepts
37
+ static Dictionary<string, double> Top => SceneCatalog.Top;
38
+ static HashSet<string> Grasp => SceneCatalog.Grasp;
39
+ static HashSet<string> Container => SceneCatalog.Container;
40
+ static HashSet<string> Fixed => SceneCatalog.Fixed;
41
+ static Dictionary<string, string[]> States => SceneCatalog.States;
42
+ static readonly Dictionary<string, string[]> Required = new Dictionary<string, string[]> {
43
+ { "grab", new[] { "object_id", "hand" } }, { "release", new[] { "hand" } }, { "place", new[] { "object_id", "target_id" } },
44
+ { "move_by", new[] { "object_id", "direction", "distance", "unit" } }, { "rotate", new[] { "object_id", "degrees" } },
45
+ { "set_state", new[] { "object_id", "state" } }, { "press", new[] { "object_id" } }, { "highlight", new[] { "object_ids" } },
46
+ { "teleport", new[] { "location_id" } }, { "find_objects", new[] { "label" } }, { "ask_clarification", new[] { "question", "candidate_ids" } } };
47
+ static readonly Dictionary<string, Vector3> Dir = new Dictionary<string, Vector3> {
48
+ { "left", Vector3.left }, { "right", Vector3.right }, { "forward", Vector3.forward },
49
+ { "backward", Vector3.back }, { "up", Vector3.up }, { "down", Vector3.down } };
50
+ static readonly string[] HandSlots = { "left_hand", "right_hand", "both_hands" };
51
+
52
+ // what an item can do: the catalogue by label, unless the item overrides it (SceneItem.overrideAffordances)
53
+ static bool CanGrasp(SceneItem o) => o.overrideAffordances ? o.grabbable : Grasp.Contains(o.label);
54
+ static bool TryTop(SceneItem t, out double h)
55
+ {
56
+ if (t.overrideAffordances) { h = t.surfaceHeight; return t.isSurface; }
57
+ return Top.TryGetValue(t.label, out h);
58
+ }
59
+ static bool IsContainer(SceneItem o) => o.overrideAffordances ? o.isContainer : Container.Contains(o.label);
60
+ static bool IsFixed(SceneItem o) => o.overrideAffordances ? o.isFixed : Fixed.Contains(o.label);
61
+ static bool IsButton(SceneItem o) => o.overrideAffordances ? o.isButton : SceneCatalog.Button.Contains(o.label);
62
+ static bool CanBe(SceneItem o, string st) => o.overrideAffordances
63
+ ? (o.states ?? new string[0]).Contains(st) : States.ContainsKey(o.label) && States[o.label].Contains(st);
64
+
65
+ void Start()
66
+ {
67
+ if (!initFromTransforms) return;
68
+ foreach (var o in Items)
69
+ {
70
+ if (string.IsNullOrEmpty(o.itemId) || !o.hasPos || HandSlots.Contains(o.on)) continue;
71
+ var tr = o.transform;
72
+ var p = tr.position - (roomOrigin != null ? roomOrigin(o.room) : Vector3.zero);
73
+ o.pos = new[] { PyMath.Round(p.x, 3), PyMath.Round(p.y - tr.localScale.y / 2f, 3), PyMath.Round(p.z, 3) };
74
+ var e = tr.rotation.eulerAngles;
75
+ o.yaw = PyMath.Round(e.y, 3); o.pitch = PyMath.Round(e.x, 3); o.roll = PyMath.Round(e.z, 3);
76
+ }
77
+ if (rooms.Count == 0) rooms = Items.Select(o => o.room).Where(r => !string.IsNullOrEmpty(r)).Distinct().ToList();
78
+ if (string.IsNullOrEmpty(playerRoom) && rooms.Count > 0) playerRoom = rooms[0];
79
+ }
80
+
81
+ // Held items are parented to the hand anchors while they are shown in the hand; when the anchors live outside
82
+ // this hierarchy (the demo's camera rig) they must still count as scene items, or every tool call on a held
83
+ // item fails with "unknown object" (found by the v2 demo autotest: "put it on the counter" after a grab).
84
+ public IEnumerable<SceneItem> Items
85
+ {
86
+ get
87
+ {
88
+ IEnumerable<SceneItem> all = GetComponentsInChildren<SceneItem>(true);
89
+ foreach (var anchor in new[] { leftHandAnchor, rightHandAnchor })
90
+ if (anchor != null && !anchor.IsChildOf(transform))
91
+ all = all.Concat(anchor.GetComponentsInChildren<SceneItem>(true));
92
+ return all.Distinct();
93
+ }
94
+ }
95
+
96
+ SceneItem Get(string id)
97
+ {
98
+ var o = Items.FirstOrDefault(x => x.itemId == id);
99
+ if (o == null) throw new ToolError($"unknown object '{id}'");
100
+ if (o.room != playerRoom) throw new ToolError($"object '{id}' is not visible");
101
+ return o;
102
+ }
103
+
104
+ // ------------------------------------------------------------------ scene JSON for the prompt
105
+ public string BuildSceneJson()
106
+ {
107
+ var vis = new List<object>();
108
+ // hierarchy order, not sorted: training scenes list objects in random order. The model is robust to the
109
+ // order (runs/order_probe: id-sorted -1.7, another shuffle -1.0 points on OOD, neither significant), and
110
+ // hierarchy order lets the Unity test send exactly the offline prompts
111
+ foreach (var o in Items.Where(x => x.room == playerRoom))
112
+ {
113
+ var v = new Dictionary<string, object> { { "id", o.itemId }, { "label", o.label } };
114
+ if (!string.IsNullOrEmpty(o.color)) v["color"] = o.color;
115
+ v["on"] = o.on;
116
+ if (!string.IsNullOrEmpty(o.state)) v["state"] = o.state;
117
+ if (o.hasPos) v["pos"] = o.pos.Select(c => (object)PyMath.Round(c, 1)).ToList();
118
+ vis.Add(v);
119
+ }
120
+ var scene = new Dictionary<string, object> {
121
+ { "player", new Dictionary<string, object> { { "room", playerRoom }, { "gaze", Nz(gaze) }, { "left_hand", Nz(leftHand) }, { "right_hand", Nz(rightHand) } } },
122
+ { "locations", rooms.Cast<object>().ToList() }, { "visible_objects", vis } };
123
+ return MiniJson.Serialize(scene);
124
+ }
125
+
126
+ static object Nz(string s) => string.IsNullOrEmpty(s) ? null : s;
127
+
128
+ // ------------------------------------------------------------------ tools
129
+ static string S(Dictionary<string, object> a, string k) => a.TryGetValue(k, out var v) && v != null ? Convert.ToString(v, CultureInfo.InvariantCulture) : null;
130
+ static double D(Dictionary<string, object> a, string k, double def = 0) => a.TryGetValue(k, out var v) && v != null ? Convert.ToDouble(v, CultureInfo.InvariantCulture) : def;
131
+ static List<string> L(Dictionary<string, object> a, string k) => a.TryGetValue(k, out var v) && v is List<object> l ? l.Select(x => Convert.ToString(x)).ToList() : new List<string>();
132
+
133
+ static string[] HandsFor(string hand) =>
134
+ hand == "left" ? new[] { "left" } : hand == "right" ? new[] { "right" } : hand == "both" ? new[] { "left", "right" } : throw new ToolError($"invalid hand '{hand}'");
135
+
136
+ string HandGet(string h) => h == "left" ? leftHand : rightHand;
137
+ void HandSet(string h, string v) { if (h == "left") leftHand = v; else rightHand = v; }
138
+ void FreeHand(SceneItem o) { if (leftHand == o.itemId) leftHand = null; if (rightHand == o.itemId) rightHand = null; }
139
+
140
+ void MoveTree(SceneItem o, double[] d)
141
+ {
142
+ for (int i = 0; i < 3; i++) o.pos[i] = PyMath.Round(o.pos[i] + d[i], 3);
143
+ foreach (var c in Items) if (c.on == o.itemId && c.hasPos) MoveTree(c, d);
144
+ }
145
+
146
+ /// <summary>Executes one call; returns the tool result object (serialised back to the model). Throws ToolError
147
+ /// when the call is not possible here; ToolExecuted is raised only for accepted calls.</summary>
148
+ public object Execute(string name, Dictionary<string, object> a)
149
+ {
150
+ var result = ExecuteTool(name, a);
151
+ ToolExecuted?.Invoke(name, a, result);
152
+ return result;
153
+ }
154
+
155
+ object ExecuteTool(string name, Dictionary<string, object> a)
156
+ {
157
+ if (!Required.ContainsKey(name ?? "")) throw new ToolError($"unknown tool '{name}'");
158
+ var missing = Required[name].Where(k => !a.ContainsKey(k) || a[k] == null).ToList();
159
+ if (missing.Count > 0) throw new ToolError("missing arguments [" + string.Join(", ", missing.Select(m => $"'{m}'")) + "]");
160
+ var ok = new Dictionary<string, object> { { "ok", true } };
161
+ switch (name)
162
+ {
163
+ case "grab":
164
+ {
165
+ var o = Get(S(a, "object_id"));
166
+ if (!CanGrasp(o)) throw new ToolError($"'{o.itemId}' cannot be picked up");
167
+ if (HandSlots.Contains(o.on)) throw new ToolError($"'{o.itemId}' is already held");
168
+ string hand = S(a, "hand");
169
+ var hs = HandsFor(hand);
170
+ if (hs.Any(h => !string.IsNullOrEmpty(HandGet(h)))) throw new ToolError($"{hand} hand is not free");
171
+ foreach (var h in hs) HandSet(h, o.itemId);
172
+ o.on = hand == "both" ? "both_hands" : hand + "_hand"; o.hasPos = false;
173
+ Sync(o); return ok;
174
+ }
175
+ case "release":
176
+ {
177
+ string hand = S(a, "hand");
178
+ var held = new SortedSet<string>(HandsFor(hand).Select(HandGet).Where(x => !string.IsNullOrEmpty(x)), StringComparer.Ordinal);
179
+ if (held.Count == 0) throw new ToolError($"nothing held in {hand} hand");
180
+ foreach (var id in held) { var o = Get(id); FreeHand(o); o.on = "floor"; o.hasPos = true; o.pos = new[] { 0.0, 0.0, 0.6 }; Sync(o); }
181
+ return ok;
182
+ }
183
+ case "place":
184
+ {
185
+ var o = Get(S(a, "object_id")); var t = Get(S(a, "target_id"));
186
+ string rel = S(a, "relation") ?? "on", orient = S(a, "orientation") ?? "default";
187
+ if (!CanGrasp(o)) throw new ToolError($"'{o.itemId}' cannot be carried");
188
+ if (o.itemId == t.itemId || HandSlots.Contains(t.on)) throw new ToolError("invalid target");
189
+ if (rel == "on" && !TryTop(t, out _)) throw new ToolError($"'{t.itemId}' is not a surface");
190
+ if (rel == "in" && !IsContainer(t)) throw new ToolError($"'{t.itemId}' is not a container");
191
+ if (rel == "in" && (t.state == "closed" || t.state == "locked")) throw new ToolError($"'{t.itemId}' is closed");
192
+ if (!new[] { "default", "upright", "horizontal", "vertical", "upside_down" }.Contains(orient)) throw new ToolError($"invalid orientation '{orient}'");
193
+ FreeHand(o);
194
+ double top = rel == "on" && TryTop(t, out var th) ? th : 0.0;
195
+ o.on = t.itemId; o.relation = rel; o.orientation = orient; o.hasPos = true;
196
+ o.pos = new[] { t.pos[0], PyMath.Round(t.pos[1] + top, 3), t.pos[2] };
197
+ Sync(o); return ok;
198
+ }
199
+ case "move_by":
200
+ {
201
+ var o = Get(S(a, "object_id"));
202
+ if (HandSlots.Contains(o.on) || IsFixed(o)) throw new ToolError($"'{o.itemId}' cannot be moved");
203
+ string dir = S(a, "direction"), unit = S(a, "unit");
204
+ if (dir == null || !Dir.ContainsKey(dir) || (unit != "m" && unit != "cm")) throw new ToolError("invalid direction or unit");
205
+ double dist = D(a, "distance") * (unit == "cm" ? 0.01 : 1.0);
206
+ var v = Dir[dir];
207
+ MoveTree(o, new[] { v.x * dist, v.y * dist, v.z * dist });
208
+ foreach (var x in Items) Sync(x);
209
+ return ok;
210
+ }
211
+ case "rotate":
212
+ {
213
+ var o = Get(S(a, "object_id"));
214
+ if (HandSlots.Contains(o.on) || IsFixed(o)) throw new ToolError($"'{o.itemId}' cannot be rotated");
215
+ string axis = S(a, "axis") ?? "yaw";
216
+ double Wrap(double v) => PyMath.Round(((v % 360) + 360) % 360, 3);
217
+ double deg = D(a, "degrees");
218
+ if (axis == "yaw") o.yaw = Wrap(o.yaw + deg);
219
+ else if (axis == "pitch") o.pitch = Wrap(o.pitch + deg);
220
+ else if (axis == "roll") o.roll = Wrap(o.roll + deg);
221
+ else throw new ToolError($"invalid axis '{axis}'");
222
+ Sync(o); return ok;
223
+ }
224
+ case "set_state":
225
+ {
226
+ var o = Get(S(a, "object_id")); string st = S(a, "state");
227
+ if (!CanBe(o, st)) throw new ToolError($"'{o.itemId}' cannot be {st}");
228
+ o.state = st; Sync(o); return ok;
229
+ }
230
+ case "press":
231
+ {
232
+ var o = Get(S(a, "object_id"));
233
+ if (!IsButton(o)) throw new ToolError($"'{o.itemId}' is not a button");
234
+ pressed[o.itemId] = D(a, "hold_seconds"); return ok;
235
+ }
236
+ case "highlight":
237
+ {
238
+ var ids = L(a, "object_ids");
239
+ if (ids.Count == 0) throw new ToolError("nothing to highlight");
240
+ foreach (var id in ids) Get(id);
241
+ foreach (var id in ids) { highlighted.Add(id); Sync(Get(id)); }
242
+ return ok;
243
+ }
244
+ case "teleport":
245
+ {
246
+ string loc = S(a, "location_id");
247
+ if (!rooms.Contains(loc)) throw new ToolError($"unknown location '{loc}'");
248
+ foreach (var o in Items) if (HandSlots.Contains(o.on)) o.room = loc;
249
+ playerRoom = loc; return ok;
250
+ }
251
+ case "find_objects":
252
+ {
253
+ string label = S(a, "label"), color = S(a, "color"), room = S(a, "room");
254
+ // hierarchy order, like the simulator's world order that produced the training tool results
255
+ return Items.Where(o => o.label == label && (color == null || o.color == color) && (room == null || o.room == room))
256
+ .Select(o =>
257
+ {
258
+ var d = new Dictionary<string, object> { { "id", o.itemId }, { "label", o.label } };
259
+ if (!string.IsNullOrEmpty(o.color)) d["color"] = o.color;
260
+ d["room"] = o.room; d["on"] = o.on;
261
+ return (object)d;
262
+ }).ToList();
263
+ }
264
+ case "ask_clarification":
265
+ clarify = new SortedSet<string>(L(a, "candidate_ids"), StringComparer.Ordinal);
266
+ lastClarifyQuestion = S(a, "question");
267
+ return ok;
268
+ }
269
+ throw new ToolError($"unknown tool '{name}'");
270
+ }
271
+
272
+ // ------------------------------------------------------------------ visuals
273
+ public void Sync(SceneItem o)
274
+ {
275
+ var tr = o.transform;
276
+ if (HandSlots.Contains(o.on))
277
+ {
278
+ var anchor = o.on == "left_hand" ? leftHandAnchor : rightHandAnchor;
279
+ if (anchor != null) { tr.SetParent(anchor, false); tr.localPosition = Vector3.zero; }
280
+ }
281
+ else
282
+ {
283
+ tr.SetParent(transform, true);
284
+ float half = tr.localScale.y / 2f;
285
+ var origin = roomOrigin != null ? roomOrigin(o.room) : Vector3.zero;
286
+ tr.position = origin + new Vector3((float)o.pos[0], (float)o.pos[1] + half, (float)o.pos[2]);
287
+ }
288
+ tr.rotation = Quaternion.Euler((float)o.pitch, (float)o.yaw, (float)o.roll);
289
+ var r = o.GetComponent<Renderer>();
290
+ if (r != null && Application.isPlaying)
291
+ r.material.SetColor("_EmissionColor", highlighted.Contains(o.itemId) ? Color.yellow : Color.black);
292
+ var light = o.GetComponentInChildren<Light>();
293
+ if (light != null) light.enabled = o.state == "on";
294
+ }
295
+
296
+ // ------------------------------------------------------------------ same text form as sim.signature_str
297
+ static string Num(double v, int digits)
298
+ {
299
+ if (Math.Abs(v) < 0.5 * Math.Pow(10, -digits)) v = 0.0;
300
+ return PyMath.Fixed(v, digits);
301
+ }
302
+
303
+ public string Signature()
304
+ {
305
+ var parts = new List<string>();
306
+ foreach (var o in Items.OrderBy(x => x.itemId, StringComparer.Ordinal))
307
+ {
308
+ string p = o.hasPos ? string.Join(",", o.pos.Select(v => Num(v, 2))) : "-";
309
+ parts.Add(string.Join("|", o.itemId, o.room, o.on ?? "", o.relation ?? "", string.IsNullOrEmpty(o.orientation) ? "default" : o.orientation,
310
+ p, $"{Num(o.yaw, 1)},{Num(o.pitch, 1)},{Num(o.roll, 1)}", o.state ?? ""));
311
+ }
312
+ parts.Add($"P:{playerRoom},{leftHand ?? ""},{rightHand ?? ""}");
313
+ parts.Add("H:" + string.Join(",", highlighted));
314
+ parts.Add("C:" + string.Join(",", clarify));
315
+ parts.Add("X:" + string.Join(",", pressed.Select(kv => $"{kv.Key}={Num(kv.Value, 1)}")));
316
+ return string.Join(";", parts);
317
+ }
318
+ }
319
+ }
Runtime/SceneWorld.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 01e763996e739e542a21333d25926802
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant: