// One conversation with the agent: runs commands, keeps the pending clarification, turns the user's pick into the
// follow-up the model was trained on (the original request plus a hint, with the gaze on the picked object), and
// optionally writes every exchange to a JSON-lines log (what was asked, the scene the model saw, calls, answer, time).
// SceneAgentChatBox and SceneAgentScriptRunner use it; your own UI can too.
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using UnityEngine;
namespace SceneAgent
{
public class SceneAgentSession : MonoBehaviour
{
public SceneAgent agent;
public SceneWorld world;
[Tooltip("Folder for a JSON-lines log of every command (relative = under Application.persistentDataPath). Empty: no log.")]
public string logDirectory = "";
[Tooltip("Line shows each tool call as raw JSON (for developers) instead of a short readable form.")]
public bool rawToolLines;
public bool Busy { get; private set; }
public AgentResult LastResult { get; private set; }
/// The request waiting for the user's pick after ask_clarification (null when none).
public string PendingRequest { get; private set; }
/// The ids the model offered in its last question.
public List Candidates => world != null ? world.clarify.ToList() : new List();
/// Human-readable lines: the request, each call and its result, the answer.
public event Action Line;
public event Action Finished;
public string LogFile { get; private set; }
int index = -1;
void Awake()
{
if (agent == null) agent = GetComponent();
if (world == null) world = agent != null && agent.world != null ? agent.world : GetComponent();
// "Game.exe -sceneAgentScript commands.txt" works in any scene with a session, not only where a runner was
// added (found by the user test: the flag was silently ignored)
if (Environment.GetCommandLineArgs().Contains("-sceneAgentScript") && GetComponent() == null)
gameObject.AddComponent();
}
/// Runs one user command (typed or recognised speech). While a question is pending, a short reply that
/// tells the candidates apart ("the blue one", "the one on the shelf") answers it, in the follow-up form the model
/// was trained on ("pick up the toolbox I mean the blue one"); anything else is a new command.
public IEnumerator Command(string text)
{
text = text?.Trim();
if (string.IsNullOrEmpty(text)) yield break;
if (PendingRequest != null && LooksLikeAnswer(text))
{
string reply = text.StartsWith("i mean", StringComparison.OrdinalIgnoreCase) ? text : "I mean " + text;
string request = $"{PendingRequest} {reply}";
PendingRequest = null;
yield return Run(request, null);
yield break;
}
PendingRequest = null;
yield return Run(text, null);
}
static readonly HashSet CommandVerbs = new HashSet {
"pick", "put", "place", "grab", "take", "drop", "release", "open", "close", "shut", "turn", "switch", "move",
"push", "pull", "rotate", "spin", "highlight", "show", "find", "where", "bring", "fetch", "get", "go", "teleport",
"press", "lift", "set", "make", "throw", "toss", "hand", "give", "lock", "unlock", "what", "how", "can", "could" };
static readonly HashSet AnswerStarts = new HashSet {
"the", "that", "this", "one", "it", "it's", "its", "number", "left", "right", "first", "second", "third", "last",
"top", "bottom", "front", "back", "near", "closest", "furthest", "biggest", "smallest", "both", "neither" };
/// Whether text typed while a question is pending answers it (see Command).
public bool LooksLikeAnswer(string text)
{
var words = text.ToLowerInvariant().Split(new[] { ' ', ',', '.', '!', '?', ';' }, StringSplitOptions.RemoveEmptyEntries);
if (words.Length == 0 || words.Length > 8 || CommandVerbs.Contains(words[0])) return false;
if (words.Length > 1 && words[0] == "i" && words[1] == "mean") return true;
if (AnswerStarts.Contains(words[0])) return true;
// a word that tells the offered objects apart: a colour, a label, what one of them stands on
var cues = new HashSet(Candidates.Select(id => world.Items.FirstOrDefault(x => x.itemId == id)).Where(x => x != null)
.SelectMany(c => new[] { c.color, c.label, LabelOf(c.on) }).Where(s => !string.IsNullOrEmpty(s)).SelectMany(s => s.Split(' ')));
return words.Any(cues.Contains);
}
/// Answers the pending question with the object the user picked.
public IEnumerator Answer(string itemId)
{
var it = world.Items.FirstOrDefault(x => x.itemId == itemId);
if (it == null || PendingRequest == null) yield break;
var cands = world.clarify.Select(id => world.Items.FirstOrDefault(x => x.itemId == id)).Where(x => x != null).ToList();
string request = $"{PendingRequest} {Hint(it, cands)}";
PendingRequest = null;
yield return Run(request, itemId);
}
IEnumerator Run(string request, string gazeItem)
{
Busy = true;
world.highlighted.Clear(); // highlights and questions belong to the previous command
world.clarify.Clear();
if (gazeItem != null) world.gaze = gazeItem;
Line?.Invoke("You: " + request);
string scene = world.BuildSceneJson();
AgentResult res = null;
yield return agent.Run(request, r => res = r);
foreach (var turn in res.turns)
for (int i = 0; i < turn.calls.Count; i++)
{
string result = i < turn.results.Count ? turn.results[i] : null;
Line?.Invoke(rawToolLines ? " → " + turn.calls[i] + (result != null ? " " + result : "") : " → " + Pretty(turn.calls[i], result));
}
if (res.error != null) Line?.Invoke("Error: " + res.error);
if (res.clarificationQuestion != null) { PendingRequest = request; Line?.Invoke("Agent: " + res.clarificationQuestion); }
else if (!string.IsNullOrEmpty(res.finalText)) Line?.Invoke("Agent: " + res.finalText);
foreach (var it in world.Items) world.Sync(it);
Record(request, gazeItem, scene, res);
LastResult = res;
Busy = false;
Finished?.Invoke(res);
}
/// "I mean the red one." / "I mean the one on the shelf." / "I mean the mug I'm looking at."
public string Hint(SceneItem it, List candidates)
{
var others = candidates.Where(c => c.itemId != it.itemId).ToList();
if (!string.IsNullOrEmpty(it.color) && others.All(c => c.color != it.color)) return $"I mean the {it.color} one.";
if (others.All(c => c.on != it.on))
return it.on == "floor" ? "I mean the one on the floor." : $"I mean the one on the {LabelOf(it.on)}.";
return $"I mean the {Describe(it)} I'm looking at.";
}
/// A tool call for people: place(hammer_1, toolbox_1, in) ok / not done: the reason / found 2.
public static string Pretty(string callJson, string resultJson)
{
string call;
try
{
var c = (Dictionary)MiniJson.Parse(callJson);
var a = c.TryGetValue("arguments", out var x) ? x as Dictionary : null;
string args = a == null ? "" : string.Join(", ", a.Values.Select(v => v is List