// Editor tools for developers: Tools > Scene Agent >
// Set Up Scene one "Scene Agent" object with SceneWorld, SceneAgent, LocalModelServer, a session and the
// chat box, wired together (idempotent)
// Make Selected Interactive SceneItem on every selected object: id, label (and a colour word) from its name, the room,
// what it stands on (from the renderers' bounds); moved under the Scene Agent object
// Validate Scene ids, labels the model's catalogue does not know, supports, rooms, model files
// Open Model Folder StreamingAssets/SceneAgent, where llama-server and the .gguf go
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using UnityEditor;
using UnityEngine;
namespace SceneAgent.EditorTools
{
public static class SceneAgentMenu
{
const string Root = "Tools/Scene Agent/";
/// Automation (tests, build scripts): no modal dialogs, results only in the Console.
public static bool Quiet;
static readonly string[] Colours = { "red", "green", "blue", "yellow", "white", "black", "gray", "grey", "orange",
"purple", "pink", "brown", "silver", "gold", "beige" };
static T Ensure(GameObject g) where T : Component
{
var c = g.GetComponent();
return c != null ? c : Undo.AddComponent(g);
}
[MenuItem(Root + "Set Up Scene", priority = 1)]
public static void SetUpScene()
{
var world = Object.FindObjectOfType();
if (world == null)
{
var go = new GameObject("Scene Agent");
Undo.RegisterCreatedObjectUndo(go, "Set Up Scene Agent");
world = go.AddComponent();
world.initFromTransforms = true;
}
var root = world.gameObject;
var agent = Ensure(root);
var server = Ensure(root);
var session = Ensure(root);
Ensure(root);
var gaze = Ensure(root);
Undo.RecordObjects(new Object[] { agent, session, gaze }, "Set Up Scene Agent");
gaze.world = world;
gaze.session = session;
agent.world = world;
agent.server = server;
session.agent = agent;
session.world = world;
if (string.IsNullOrEmpty(session.logDirectory)) session.logDirectory = "SceneAgentSessions";
EditorUtility.SetDirty(root);
Selection.activeGameObject = root;
Debug.Log("[SceneAgent] Scene set up on \"" + root.name + "\". Next: select the objects the agent may use and run " +
"Tools > Scene Agent > Make Selected Interactive, then Validate Scene. " + ModelFilesStatus(server));
}
[MenuItem(Root + "Make Selected Interactive", priority = 2)]
public static void MakeSelectedInteractive()
{
var world = Object.FindObjectOfType();
if (world == null) { SetUpScene(); world = Object.FindObjectOfType(); }
string room = !string.IsNullOrEmpty(world.playerRoom) ? world.playerRoom : world.rooms.FirstOrDefault() ?? "room";
var selected = new HashSet(Selection.gameObjects);
// ids taken by items that are not being (re)made now; a duplicate (Ctrl+D copies the SceneItem and its id)
// gets a fresh id below
var used = new HashSet(world.Items.Where(i => !string.IsNullOrEmpty(i.itemId) && !selected.Contains(i.gameObject)).Select(i => i.itemId));
var made = new List();
foreach (var go in Selection.gameObjects)
{
if (go == world.gameObject || go.GetComponent() != null) continue;
var it = Ensure(go);
Undo.RecordObject(it, "Make Interactive");
if (string.IsNullOrEmpty(it.label))
{
var (colour, label) = LabelFromName(go.name);
it.label = label;
if (string.IsNullOrEmpty(it.color)) it.color = colour;
}
if (string.IsNullOrEmpty(it.itemId) || used.Contains(it.itemId)) it.itemId = NextId(it.label, used);
else used.Add(it.itemId);
if (string.IsNullOrEmpty(it.state)) it.state = DefaultState(it);
if (string.IsNullOrEmpty(it.room)) it.room = room;
// directly under the Scene Agent object: SceneWorld.Sync re-parents items there anyway, and an item nested in
// another item would make that one's bounds (and so what stands on it) wrong
if (go.transform.parent != world.transform) Undo.SetTransformParent(go.transform, world.transform, "Make Interactive");
made.Add(it);
}
Undo.RecordObject(world, "Make Interactive");
foreach (var r in made.Select(i => i.room).Distinct()) if (!world.rooms.Contains(r)) world.rooms.Add(r);
if (string.IsNullOrEmpty(world.playerRoom)) world.playerRoom = room;
InferSupports(world);
var unknown = made.Where(i => !i.overrideAffordances && !KnownLabel(i.label)).Select(i => $"{i.itemId} ('{i.label}')").ToList();
Debug.Log($"[SceneAgent] {made.Count} interactive: " + string.Join(", ", made.Select(i => $"{i.itemId} on {i.on}")) +
(unknown.Count > 0 ? $". Not in the model's catalogue (rename to a common word, or tick overrideAffordances): {string.Join(", ", unknown)}" : ""));
}
[MenuItem(Root + "Make Selected Interactive", true)]
static bool CanMake() => Selection.gameObjects.Length > 0;
[MenuItem(Root + "Validate Scene", priority = 3)]
public static void ValidateScene()
{
var sb = new StringBuilder();
int errors = 0, warnings = 0;
void E(string m) { errors++; sb.AppendLine("ERROR " + m); }
void W(string m) { warnings++; sb.AppendLine("warning " + m); }
var world = Object.FindObjectOfType();
if (world == null)
{
Debug.LogWarning("[SceneAgent] Validate Scene: no SceneWorld in the scene: run Tools > Scene Agent > Set Up Scene.");
if (!Application.isBatchMode && !Quiet) EditorUtility.DisplayDialog("Scene Agent", "No SceneWorld in the scene: run Tools > Scene Agent > Set Up Scene.", "OK");
return;
}
var agent = world.GetComponent() != null ? world.GetComponent() : Object.FindObjectOfType();
if (agent == null) E("no SceneAgent component");
else if (agent.world == null) E("SceneAgent.world is not set");
if (Resources.Load("SceneAgent/system_prompt") == null && (agent == null || agent.systemPromptAsset == null))
E("the system prompt is missing (package Resources/SceneAgent/system_prompt.txt)");
var server = agent != null && agent.server != null ? agent.server : Object.FindObjectOfType();
if (server == null) W("no LocalModelServer: the agent will call " + (agent != null ? agent.endpoint : "?") + " (start a server yourself)");
else sb.AppendLine("model " + ModelFilesStatus(server));
if (server != null)
{
string exe = LocalModelServer.Resolve(server.serverExe), model = server.ResolveModel(exe);
if (exe == null || !File.Exists(exe)) E("llama-server not found: " + exe);
if (model == null || !File.Exists(model)) E("model file not found: " + (model ?? "no .gguf next to llama-server"));
}
var items = Object.FindObjectsOfType(true);
if (items.Length == 0) E("no SceneItem: select objects and run Make Selected Interactive");
foreach (var g in items.GroupBy(i => i.itemId ?? "").Where(g => g.Count() > 1)) E($"duplicate id '{g.Key}' on {string.Join(", ", g.Select(i => i.name))}");
var ids = new HashSet(items.Select(i => i.itemId));
foreach (var it in items)
{
string n = $"'{it.name}'";
if (string.IsNullOrEmpty(it.itemId)) E($"{n} has no id");
else if (!Regex.IsMatch(it.itemId, "^[a-z0-9_]+$")) W($"{n}: id '{it.itemId}' should look like label_1 (lower case, digits, _)");
if (string.IsNullOrEmpty(it.label)) E($"{n} has no label");
else if (!it.overrideAffordances && !KnownLabel(it.label))
W($"{n}: label '{it.label}' is not in the model's catalogue; Unity will refuse grab/place/set_state on it " +
"unless you tick overrideAffordances (or use a common word)");
if (!it.transform.IsChildOf(world.transform)) E($"{n} is not under '{world.name}' (SceneWorld only sees its children)");
if (string.IsNullOrEmpty(it.state) && DefaultState(it) != null)
W($"{n}: has states ({string.Join("/", StatesOf(it))}) but none is set; the model will not know whether it is {DefaultState(it)}");
if (string.IsNullOrEmpty(it.room)) E($"{n} has no room");
else if (world.rooms.Count > 0 && !world.rooms.Contains(it.room)) E($"{n}: room '{it.room}' is not in SceneWorld.rooms");
if (!string.IsNullOrEmpty(it.on) && it.on != "floor" && !it.on.EndsWith("_hand") && it.on != "both_hands" && !ids.Contains(it.on))
E($"{n}: stands on unknown id '{it.on}'");
}
foreach (var g in items.GroupBy(i => i.room ?? ""))
if (g.Count() > 60) W($"room '{g.Key}' has {g.Count()} objects; the model sees one room at a time and 4096 tokens hold about 60 (split the room or raise contextSize)");
if (!world.initFromTransforms) W("SceneWorld.initFromTransforms is off: positions come from each SceneItem.pos, not from where the objects stand");
string head = $"{items.Length} objects, {world.rooms.Count} room(s), {errors} error(s), {warnings} warning(s)";
Debug.Log("[SceneAgent] Validate Scene: " + head + "\n" + sb);
if (!Application.isBatchMode && !Quiet)
EditorUtility.DisplayDialog("Scene Agent · Validate Scene", head + (errors + warnings > 0 ? "\n\nDetails in the Console." : "\n\nReady to press Play."), "OK");
}
[MenuItem(Root + "Open Model Folder", priority = 20)]
public static void OpenModelFolder()
{
string dir = Path.Combine(Application.streamingAssetsPath, "SceneAgent");
Directory.CreateDirectory(dir);
AssetDatabase.Refresh();
EditorUtility.RevealInFinder(dir);
}
// ------------------------------------------------------------------ helpers
static string ModelFilesStatus(LocalModelServer s)
{
string exe = LocalModelServer.Resolve(s.serverExe);
string model = s.ResolveModel(exe);
if (exe != null) exe = Path.GetFullPath(exe);
if (model != null) model = Path.GetFullPath(model);
bool e = exe != null && File.Exists(exe), m = model != null && File.Exists(model);
return (e ? $"llama-server: {exe}" : $"llama-server MISSING ({exe})") + " · " +
(m ? $"model: {Path.GetFileName(model)} ({new FileInfo(model).Length / 1e9:F2} GB)" : "model (.gguf) MISSING") +
(e && m ? "" : " → Tools > Scene Agent > Open Model Folder, and see the package README 'Model files'.");
}
static string[] StatesOf(SceneItem it) => it.overrideAffordances ? (it.states ?? new string[0])
: SceneCatalog.States.TryGetValue(it.label ?? "", out var s) ? s : new string[0];
/// The state a device starts in when none is set: off, else closed, else its first state.
public static string DefaultState(SceneItem it)
{
var s = StatesOf(it);
return s.Length == 0 ? null : s.Contains("off") ? "off" : s.Contains("closed") ? "closed" : s[0];
}
static readonly HashSet Known = new HashSet(SceneCatalog.Grasp.Concat(SceneCatalog.Top.Keys)
.Concat(SceneCatalog.Container).Concat(SceneCatalog.Fixed).Concat(SceneCatalog.States.Keys).Concat(SceneCatalog.Button));
public static bool KnownLabel(string label) => label != null && Known.Contains(label);
/// "RedMug (2)" → ("red", "mug"); "Floor_Lamp" → ("", "floor lamp").
public static (string colour, string label) LabelFromName(string name)
{
string s = Regex.Replace(name, @"\s*\(\d+\)\s*$", ""); // "Mug (2)"
s = Regex.Replace(s, "([a-z])([A-Z])", "$1 $2"); // "RedMug"
s = Regex.Replace(s, @"[_\-.]+", " ");
s = Regex.Replace(s, @"\s*\d+$", "").Trim().ToLowerInvariant(); // "mug 2"
var words = s.Split(new[] { ' ' }, System.StringSplitOptions.RemoveEmptyEntries).ToList();
string colour = "";
if (words.Count > 1 && Colours.Contains(words[0])) { colour = words[0] == "grey" ? "gray" : words[0]; words.RemoveAt(0); }
return (colour, string.Join(" ", words));
}
static string NextId(string label, HashSet used)
{
string b = Regex.Replace((label ?? "item").ToLowerInvariant(), "[^a-z0-9]+", "_").Trim('_');
int n = 1;
while (used.Contains($"{b}_{n}")) n++;
used.Add($"{b}_{n}");
return $"{b}_{n}";
}
/// What each item stands on: the highest other item whose top is just under its bottom and whose
/// footprint holds its centre; otherwise the floor.
public static void InferSupports(SceneWorld world)
{
var items = world.Items.Where(i => i.gameObject.activeInHierarchy).ToList();
var bounds = items.ToDictionary(i => i, BoundsOf);
foreach (var a in items)
{
if (a.on != null && (a.on.EndsWith("_hand") || a.on == "both_hands")) continue;
var ba = bounds[a];
SceneItem best = null;
foreach (var b in items)
{
if (b == a || b.room != a.room) continue;
var bb = bounds[b];
bool under = bb.max.y <= ba.min.y + 0.05f && bb.max.y >= ba.min.y - 0.15f;
bool holds = ba.center.x >= bb.min.x && ba.center.x <= bb.max.x && ba.center.z >= bb.min.z && ba.center.z <= bb.max.z;
if (under && holds && (best == null || bb.max.y > bounds[best].max.y)) best = b;
}
Undo.RecordObject(a, "Infer supports");
a.on = best != null ? best.itemId : "floor";
if (best != null && string.IsNullOrEmpty(a.relation)) a.relation = "on";
}
}
static Bounds BoundsOf(SceneItem i)
{
// its own renderers only: not those of other items parented under it
var rs = i.GetComponentsInChildren().Where(r => r.GetComponentInParent() == i).ToArray();
if (rs.Length == 0) return new Bounds(i.transform.position, i.transform.lossyScale);
var b = rs[0].bounds;
foreach (var r in rs.Skip(1)) b.Encapsulate(r.bounds);
return b;
}
}
}