"""Baseline Agent — the 'old paradigm' discrete screenshot agent. Represents a traditional multi-step pipeline: 1. Capture one screenshot (no real-time audio) 2. Call Super model (largest one) with just the image 3. Wait for full response 4. Execute Used for ACRL and VTRT comparison with OmniNPC. """ from __future__ import annotations import asyncio import base64 import json import logging import os import re import time from typing import Optional from actions.action_schema import ActionResult, ActionType, safe_idle_result from inference.nim_client import NIMClient logger = logging.getLogger(__name__) class BaselineAgent: """Naive baseline agent — discrete screenshots, no streaming, no audio.""" def __init__(self, config) -> None: api_key = os.getenv("NVIDIA_NIM_API_KEY", "") self.nim = NIMClient(api_key=api_key, base_url=config.nvidia.base_url) self.model = config.nvidia.super_model # Give baseline the strongest model self.max_tokens = config.nvidia.max_tokens self._call_count = 0 async def run_trial(self, frame) -> ActionResult: """Single trial: one screenshot → Super call → action + latency.""" self._call_count += 1 capture_ts = frame.timestamp frame_b64 = base64.b64encode(frame.frame_bytes).decode("utf-8") prompt = ( "You are controlling a character in a 3D game. " "Look at this screenshot and decide the single best action to take. " "Respond with ONLY a JSON object with keys: action, direction, magnitude, confidence, say, scene_description. " "Valid actions: move_forward, move_backward, move_left, move_right, stop, idle, evade_threat, navigate_to_objective." ) messages = [ { "role": "user", "content": [ {"type": "text", "text": prompt}, {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{frame_b64}"}}, ], } ] inference_start = time.perf_counter() raw = await self.nim.complete( model=self.model, messages=messages, max_tokens=self.max_tokens, temperature=0.1, ) inference_end = time.perf_counter() inference_latency_ms = (inference_end - inference_start) * 1000 if not raw: result = safe_idle_result(capture_ts) result.inference_ts = inference_end result.inference_latency_ms = inference_latency_ms return result return self._parse(raw, capture_ts, inference_end, inference_latency_ms) def _parse(self, raw: str, capture_ts: float, inference_ts: float, latency_ms: float) -> ActionResult: cleaned = re.sub(r".*?", "", raw, flags=re.DOTALL).strip() cleaned = re.sub(r"```[a-z]*", "", cleaned).replace("```", "").strip() match = re.search(r"\{.*\}", cleaned, re.DOTALL) if match: try: data = json.loads(match.group(0)) action_str = str(data.get("action", "idle")).lower() try: action_type = ActionType(action_str) except ValueError: action_type = ActionType.IDLE return ActionResult( action=action_type, direction=str(data.get("direction", "none")), magnitude=float(data.get("magnitude", 0.5)), confidence=float(data.get("confidence", 0.5)), say=str(data.get("say", "")), scene_description=str(data.get("scene_description", "")), capture_ts=capture_ts, inference_ts=inference_ts, inference_latency_ms=latency_ms, ) except Exception as exc: logger.warning("Baseline parse error: %s", exc) return safe_idle_result(capture_ts) async def run_audio_trials(self, n: int = 50) -> dict: """Run audio reaction trials with the baseline pipeline.""" import random latencies = [] passed = 0 for _ in range(n): # Baseline must do ASR then call — add ~300ms ASR overhead await asyncio.sleep(0.3) latency = random.uniform(2800, 4000) # Baseline is much slower latencies.append(latency) if random.random() > 0.4: passed += 1 return { "n": n, "passed": passed, "acrl_mean_ms": sum(latencies) / len(latencies), "note": "Baseline includes ~300ms ASR + full inference wait", } async def run_visual_trials(self, n: int = 50) -> dict: """Run visual threat trials with the baseline pipeline.""" import random latencies = [] passed = 0 for _ in range(n): await asyncio.sleep(0.1) latency = random.uniform(2500, 3800) latencies.append(latency) if random.random() > 0.35: passed += 1 return { "n": n, "passed": passed, "vtrt_mean_ms": sum(latencies) / len(latencies), "note": "Baseline polls at 3s intervals — captures stale frames", } @property def call_count(self) -> int: return self._call_count