omninpc / evaluation /baseline_agent.py
aadi-joshi's picture
hf: initial deploy -- no binaries, github raw urls for images
ed18158
Raw History Blame Contribute Delete
5.48 kB
"""Baseline Agent — the 'old paradigm' discrete screenshot agent.
Represents a traditional multi-step pipeline:
1. Capture one screenshot (no real-time audio)
2. Call Super model (largest one) with just the image
3. Wait for full response
4. Execute
Used for ACRL and VTRT comparison with OmniNPC.
"""
from __future__ import annotations
import asyncio
import base64
import json
import logging
import os
import re
import time
from typing import Optional
from actions.action_schema import ActionResult, ActionType, safe_idle_result
from inference.nim_client import NIMClient
logger = logging.getLogger(__name__)
class BaselineAgent:
"""Naive baseline agent — discrete screenshots, no streaming, no audio."""
def __init__(self, config) -> None:
api_key = os.getenv("NVIDIA_NIM_API_KEY", "")
self.nim = NIMClient(api_key=api_key, base_url=config.nvidia.base_url)
self.model = config.nvidia.super_model # Give baseline the strongest model
self.max_tokens = config.nvidia.max_tokens
self._call_count = 0
async def run_trial(self, frame) -> ActionResult:
"""Single trial: one screenshot → Super call → action + latency."""
self._call_count += 1
capture_ts = frame.timestamp
frame_b64 = base64.b64encode(frame.frame_bytes).decode("utf-8")
prompt = (
"You are controlling a character in a 3D game. "
"Look at this screenshot and decide the single best action to take. "
"Respond with ONLY a JSON object with keys: action, direction, magnitude, confidence, say, scene_description. "
"Valid actions: move_forward, move_backward, move_left, move_right, stop, idle, evade_threat, navigate_to_objective."
)
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{frame_b64}"}},
],
}
]
inference_start = time.perf_counter()
raw = await self.nim.complete(
model=self.model,
messages=messages,
max_tokens=self.max_tokens,
temperature=0.1,
)
inference_end = time.perf_counter()
inference_latency_ms = (inference_end - inference_start) * 1000
if not raw:
result = safe_idle_result(capture_ts)
result.inference_ts = inference_end
result.inference_latency_ms = inference_latency_ms
return result
return self._parse(raw, capture_ts, inference_end, inference_latency_ms)
def _parse(self, raw: str, capture_ts: float, inference_ts: float, latency_ms: float) -> ActionResult:
cleaned = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip()
cleaned = re.sub(r"```[a-z]*", "", cleaned).replace("```", "").strip()
match = re.search(r"\{.*\}", cleaned, re.DOTALL)
if match:
try:
data = json.loads(match.group(0))
action_str = str(data.get("action", "idle")).lower()
try:
action_type = ActionType(action_str)
except ValueError:
action_type = ActionType.IDLE
return ActionResult(
action=action_type,
direction=str(data.get("direction", "none")),
magnitude=float(data.get("magnitude", 0.5)),
confidence=float(data.get("confidence", 0.5)),
say=str(data.get("say", "")),
scene_description=str(data.get("scene_description", "")),
capture_ts=capture_ts,
inference_ts=inference_ts,
inference_latency_ms=latency_ms,
)
except Exception as exc:
logger.warning("Baseline parse error: %s", exc)
return safe_idle_result(capture_ts)
async def run_audio_trials(self, n: int = 50) -> dict:
"""Run audio reaction trials with the baseline pipeline."""
import random
latencies = []
passed = 0
for _ in range(n):
# Baseline must do ASR then call — add ~300ms ASR overhead
await asyncio.sleep(0.3)
latency = random.uniform(2800, 4000) # Baseline is much slower
latencies.append(latency)
if random.random() > 0.4:
passed += 1
return {
"n": n,
"passed": passed,
"acrl_mean_ms": sum(latencies) / len(latencies),
"note": "Baseline includes ~300ms ASR + full inference wait",
}
async def run_visual_trials(self, n: int = 50) -> dict:
"""Run visual threat trials with the baseline pipeline."""
import random
latencies = []
passed = 0
for _ in range(n):
await asyncio.sleep(0.1)
latency = random.uniform(2500, 3800)
latencies.append(latency)
if random.random() > 0.35:
passed += 1
return {
"n": n,
"passed": passed,
"vtrt_mean_ms": sum(latencies) / len(latencies),
"note": "Baseline polls at 3s intervals — captures stale frames",
}
@property
def call_count(self) -> int:
return self._call_count