File size: 4,780 Bytes
34bec90 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 | // A minimal on-screen chat for trying the agent without writing any UI (flat screen; for XR, drive SceneAgentSession
// from your own UI or speech input). Type a command, press Enter; when the model asks which object you mean, click it.
// F1 hides / shows the box.
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
namespace SceneAgent
{
[RequireComponent(typeof(SceneAgentSession))]
public class SceneAgentChatBox : MonoBehaviour
{
public int visibleLines = 12;
public KeyCode toggleKey = KeyCode.F1;
[Tooltip("Commands offered as one-click examples while the box is empty.")]
public string[] examples = { "put the red mug on the table", "turn on the lamp", "where is the book?" };
SceneAgentSession session;
LocalModelServer server;
readonly List<string> lines = new List<string>();
string input = "";
bool visible = true;
Vector2 scroll;
GUIStyle wrap;
void Awake()
{
session = GetComponent<SceneAgentSession>();
session.Line += l => { lines.Add(l); scroll.y = float.MaxValue; };
}
void Start() // after every Awake: the session has found its agent by now
{
// no ?? here: Unity objects compare to null through their own operator
server = session.agent != null ? session.agent.server : null;
if (server == null) server = GetComponent<LocalModelServer>();
if (server == null) server = FindObjectOfType<LocalModelServer>();
}
void Send(string text)
{
text = text?.Trim();
if (string.IsNullOrEmpty(text) || session.Busy) return;
input = "";
StartCoroutine(session.Command(text));
}
void OnGUI()
{
// the toggle through IMGUI events: works with the old Input Manager and with the Input System package
if (Event.current.type == EventType.KeyDown && Event.current.keyCode == toggleKey) { visible = !visible; Event.current.Use(); }
if (!visible) return;
wrap ??= new GUIStyle(GUI.skin.label) { wordWrap = true, richText = false };
float w = Mathf.Min(Screen.width - 20, 900), h = 60 + visibleLines * 20;
var area = new Rect(10, Screen.height - h - 10, w, h);
GUI.Box(area, "");
GUILayout.BeginArea(new Rect(area.x + 8, area.y + 6, area.width - 16, area.height - 12));
string st = server == null ? "no LocalModelServer: using " + session.agent.endpoint
: server.Ready ? server.status : server.Failed ? "⚠ " + server.status : server.status;
string gaze = string.IsNullOrEmpty(session.world.gaze) ? "" : $" · pointing at: {session.world.gaze}";
GUILayout.Label($"Scene Agent · room: {session.world.playerRoom}{gaze} · {st}{(session.Busy ? " · thinking…" : "")}");
scroll = GUILayout.BeginScrollView(scroll, GUILayout.Height(visibleLines * 20));
foreach (var l in lines.Skip(Mathf.Max(0, lines.Count - 200))) GUILayout.Label(l, wrap);
if (lines.Count == 0)
{
GUILayout.Label("Type a command in English, e.g.:");
GUILayout.BeginHorizontal();
foreach (var ex in examples) if (GUILayout.Button(ex)) input = ex;
GUILayout.EndHorizontal();
}
GUILayout.EndScrollView();
if (session.PendingRequest != null && !session.Busy)
{
GUILayout.BeginHorizontal();
GUILayout.Label("Click one, or type e.g. 'the red one':", GUILayout.Width(240));
foreach (var id in session.Candidates)
{
var it = session.world.Items.FirstOrDefault(x => x.itemId == id);
if (it != null && GUILayout.Button($"{session.Describe(it)} ({id})")) StartCoroutine(session.Answer(id));
}
GUILayout.EndHorizontal();
}
GUILayout.BeginHorizontal();
GUI.SetNextControlName("SceneAgentInput");
bool enter = Event.current.type == EventType.KeyDown && Event.current.keyCode == KeyCode.Return
&& GUI.GetNameOfFocusedControl() == "SceneAgentInput";
input = GUILayout.TextField(input, GUILayout.ExpandWidth(true));
GUI.enabled = !session.Busy;
if (GUILayout.Button("Send", GUILayout.Width(70)) || enter) Send(input);
GUI.enabled = true;
GUILayout.EndHorizontal();
GUILayout.EndArea();
}
}
}
|