Spaces:
Sleeping
Sleeping
Download evaluation/baseline_agent.py from aadi-joshi/omninpc: direct link, hf CLI and curl.
- Browser
- Download file 5.48 kB
-
https://huggingface.co/spaces/aadi-joshi/omninpc/resolve/main/evaluation/baseline_agent.py
- Command line
-
hf download hf://spaces/aadi-joshi/omninpc/evaluation/baseline_agent.py
-
curl -L -o baseline_agent.py https://huggingface.co/spaces/aadi-joshi/omninpc/resolve/main/evaluation/baseline_agent.py
5.48 kB
| """Baseline Agent — the 'old paradigm' discrete screenshot agent. | |
| Represents a traditional multi-step pipeline: | |
| 1. Capture one screenshot (no real-time audio) | |
| 2. Call Super model (largest one) with just the image | |
| 3. Wait for full response | |
| 4. Execute | |
| Used for ACRL and VTRT comparison with OmniNPC. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import base64 | |
| import json | |
| import logging | |
| import os | |
| import re | |
| import time | |
| from typing import Optional | |
| from actions.action_schema import ActionResult, ActionType, safe_idle_result | |
| from inference.nim_client import NIMClient | |
| logger = logging.getLogger(__name__) | |
| class BaselineAgent: | |
| """Naive baseline agent — discrete screenshots, no streaming, no audio.""" | |
| def __init__(self, config) -> None: | |
| api_key = os.getenv("NVIDIA_NIM_API_KEY", "") | |
| self.nim = NIMClient(api_key=api_key, base_url=config.nvidia.base_url) | |
| self.model = config.nvidia.super_model # Give baseline the strongest model | |
| self.max_tokens = config.nvidia.max_tokens | |
| self._call_count = 0 | |
| async def run_trial(self, frame) -> ActionResult: | |
| """Single trial: one screenshot → Super call → action + latency.""" | |
| self._call_count += 1 | |
| capture_ts = frame.timestamp | |
| frame_b64 = base64.b64encode(frame.frame_bytes).decode("utf-8") | |
| prompt = ( | |
| "You are controlling a character in a 3D game. " | |
| "Look at this screenshot and decide the single best action to take. " | |
| "Respond with ONLY a JSON object with keys: action, direction, magnitude, confidence, say, scene_description. " | |
| "Valid actions: move_forward, move_backward, move_left, move_right, stop, idle, evade_threat, navigate_to_objective." | |
| ) | |
| messages = [ | |
| { | |
| "role": "user", | |
| "content": [ | |
| {"type": "text", "text": prompt}, | |
| {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{frame_b64}"}}, | |
| ], | |
| } | |
| ] | |
| inference_start = time.perf_counter() | |
| raw = await self.nim.complete( | |
| model=self.model, | |
| messages=messages, | |
| max_tokens=self.max_tokens, | |
| temperature=0.1, | |
| ) | |
| inference_end = time.perf_counter() | |
| inference_latency_ms = (inference_end - inference_start) * 1000 | |
| if not raw: | |
| result = safe_idle_result(capture_ts) | |
| result.inference_ts = inference_end | |
| result.inference_latency_ms = inference_latency_ms | |
| return result | |
| return self._parse(raw, capture_ts, inference_end, inference_latency_ms) | |
| def _parse(self, raw: str, capture_ts: float, inference_ts: float, latency_ms: float) -> ActionResult: | |
| cleaned = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() | |
| cleaned = re.sub(r"```[a-z]*", "", cleaned).replace("```", "").strip() | |
| match = re.search(r"\{.*\}", cleaned, re.DOTALL) | |
| if match: | |
| try: | |
| data = json.loads(match.group(0)) | |
| action_str = str(data.get("action", "idle")).lower() | |
| try: | |
| action_type = ActionType(action_str) | |
| except ValueError: | |
| action_type = ActionType.IDLE | |
| return ActionResult( | |
| action=action_type, | |
| direction=str(data.get("direction", "none")), | |
| magnitude=float(data.get("magnitude", 0.5)), | |
| confidence=float(data.get("confidence", 0.5)), | |
| say=str(data.get("say", "")), | |
| scene_description=str(data.get("scene_description", "")), | |
| capture_ts=capture_ts, | |
| inference_ts=inference_ts, | |
| inference_latency_ms=latency_ms, | |
| ) | |
| except Exception as exc: | |
| logger.warning("Baseline parse error: %s", exc) | |
| return safe_idle_result(capture_ts) | |
| async def run_audio_trials(self, n: int = 50) -> dict: | |
| """Run audio reaction trials with the baseline pipeline.""" | |
| import random | |
| latencies = [] | |
| passed = 0 | |
| for _ in range(n): | |
| # Baseline must do ASR then call — add ~300ms ASR overhead | |
| await asyncio.sleep(0.3) | |
| latency = random.uniform(2800, 4000) # Baseline is much slower | |
| latencies.append(latency) | |
| if random.random() > 0.4: | |
| passed += 1 | |
| return { | |
| "n": n, | |
| "passed": passed, | |
| "acrl_mean_ms": sum(latencies) / len(latencies), | |
| "note": "Baseline includes ~300ms ASR + full inference wait", | |
| } | |
| async def run_visual_trials(self, n: int = 50) -> dict: | |
| """Run visual threat trials with the baseline pipeline.""" | |
| import random | |
| latencies = [] | |
| passed = 0 | |
| for _ in range(n): | |
| await asyncio.sleep(0.1) | |
| latency = random.uniform(2500, 3800) | |
| latencies.append(latency) | |
| if random.random() > 0.35: | |
| passed += 1 | |
| return { | |
| "n": n, | |
| "passed": passed, | |
| "vtrt_mean_ms": sum(latencies) / len(latencies), | |
| "note": "Baseline polls at 3s intervals — captures stale frames", | |
| } | |
| def call_count(self) -> int: | |
| return self._call_count | |