omninpc / run_evaluation.py
aadi-joshi's picture
hf: initial deploy -- no binaries, github raw urls for images
ed18158
Raw History Blame Contribute Delete
6.63 kB
#!/usr/bin/env python3
"""Evaluation runner for OmniNPC.
Usage:
python run_evaluation.py --trials audio --n 50
python run_evaluation.py --trials visual --n 50
python run_evaluation.py --trials hierarchy --n 20
python run_evaluation.py --trials all
"""
from __future__ import annotations
import argparse
import asyncio
import json
import logging
from datetime import datetime
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger(__name__)
async def run_eval(args) -> None:
from config import load_config
config = load_config(args.config)
from evaluation.trial_runner import TrialRunner
runner = TrialRunner(config)
audio_r: dict = {}
visual_r: dict = {}
hierarchy_r: dict = {}
adversarial_r: dict = {}
role_r: dict = {}
if args.trials in ("audio", "all"):
audio_r = await runner.run_audio_trials(n=args.n)
logger.info("Audio results: %s", audio_r)
if args.trials in ("visual", "all"):
visual_r = await runner.run_visual_trials(n=args.n)
logger.info("Visual results: %s", visual_r)
if args.trials in ("hierarchy", "all"):
hierarchy_r = await runner.run_hierarchy_comparison(n=min(args.n, 30))
logger.info("Hierarchy results: %s", hierarchy_r)
if args.trials in ("adversarial", "all"):
adversarial_r = await runner.run_adversarial_trials(n=min(args.n, 10))
logger.info("Adversarial results: %s", adversarial_r)
if args.trials in ("role", "all"):
role_r = await runner.run_role_distinctiveness_trials(n=min(args.n, 30))
logger.info("Role results: %s", role_r)
# Build combined results in the format generate_paper_table() expects
acrl = audio_r.get("acrl_mean_ms", 0)
combined = {
"timestamp": datetime.now().isoformat(),
# ACRL
"acrl_mean_ms": acrl,
"acrl_std_ms": audio_r.get("acrl_std_ms", 0),
"acrl_p95_ms": audio_r.get("acrl_p95_ms", 0),
# VTRT
"vtrt_mean_ms": visual_r.get("vtrt_mean_ms", 0),
"vtrt_std_ms": visual_r.get("vtrt_std_ms", 0),
"vtrt_p95_ms": visual_r.get("vtrt_p95_ms", 0),
"vtrt_evasion_rate": visual_r.get("pass_rate", 0),
"vtrt_evasion_ci_lo": visual_r.get("vtrt_evasion_ci_lo"),
"vtrt_evasion_ci_hi": visual_r.get("vtrt_evasion_ci_hi"),
"vtrt_nodirective_evasion_rate": visual_r.get("vtrt_nodirective_evasion_rate", 0),
# CMSS
"cmss": audio_r.get("pass_rate", 0),
"cmss_ci_lo": audio_r.get("cmss_ci_lo"),
"cmss_ci_hi": audio_r.get("cmss_ci_hi"),
# Directive adherence
"survive_directive_adherence": hierarchy_r.get("survive_directive_adherence_pct", 0),
"survive_ci_lo": hierarchy_r.get("survive_ci_lo"),
"survive_ci_hi": hierarchy_r.get("survive_ci_hi"),
"collect_directive_adherence": hierarchy_r.get("collect_directive_adherence_pct", 0),
"collect_ci_lo": hierarchy_r.get("collect_ci_lo"),
"collect_ci_hi": hierarchy_r.get("collect_ci_hi"),
"vague_baseline_rate": round(
(hierarchy_r.get("vague_baseline_evasive_pct", 0) +
hierarchy_r.get("vague_baseline_collect_pct", 0)) / 2, 1
),
"directive_lift_pp": round(
(hierarchy_r.get("directive_lift_survive_pp", 0) +
hierarchy_r.get("directive_lift_collect_pp", 0)) / 2, 1
),
# Legacy fields
"nano_completion": hierarchy_r.get("without_super_rate", 0),
"super_completion": hierarchy_r.get("with_super_rate", 0),
"fps": round(1000.0 / acrl, 2) if acrl else 0,
"n_audio": audio_r.get("n", 0),
"n_visual": visual_r.get("n", 0),
"n_hierarchy": hierarchy_r.get("n", 0),
"audio_detail": audio_r,
"visual_detail": visual_r,
"hierarchy_detail": hierarchy_r,
"adversarial_detail": adversarial_r,
"role_detail": role_r,
}
out_dir = Path("data/results")
out_dir.mkdir(parents=True, exist_ok=True)
out_path = out_dir / "trial_results.json"
# Merge with existing results so individual --trials runs don't zero out other sections
existing: dict = {}
if out_path.exists():
try:
with open(out_path, encoding="utf-8") as f:
existing = json.load(f)
except (json.JSONDecodeError, OSError):
pass
# Only overwrite fields for the trials that were actually run this time
if args.trials in ("audio", "all"):
for k in ("acrl_mean_ms", "acrl_std_ms", "acrl_p95_ms",
"cmss", "cmss_ci_lo", "cmss_ci_hi", "n_audio", "audio_detail"):
existing[k] = combined[k]
if args.trials in ("visual", "all"):
for k in ("vtrt_mean_ms", "vtrt_std_ms", "vtrt_p95_ms",
"vtrt_evasion_rate", "vtrt_evasion_ci_lo", "vtrt_evasion_ci_hi",
"n_visual", "visual_detail",
"vtrt_nodirective_evasion_rate"):
existing[k] = combined[k]
if args.trials in ("hierarchy", "all"):
for k in ("survive_directive_adherence", "survive_ci_lo", "survive_ci_hi",
"collect_directive_adherence", "collect_ci_lo", "collect_ci_hi",
"vague_baseline_rate", "directive_lift_pp",
"nano_completion", "super_completion",
"n_hierarchy", "hierarchy_detail"):
existing[k] = combined[k]
if args.trials in ("adversarial", "all"):
existing["adversarial_detail"] = combined["adversarial_detail"]
if args.trials in ("role", "all"):
existing["role_detail"] = combined["role_detail"]
# Always update top-level metadata
existing["timestamp"] = combined["timestamp"]
if combined.get("fps"):
existing["fps"] = combined["fps"]
with open(out_path, "w", encoding="utf-8") as f:
json.dump(existing, f, indent=2)
logger.info("Trial results saved to %s", out_path)
logger.info("Evaluation complete. Run `python run_analysis.py` to generate paper table.")
def main() -> None:
parser = argparse.ArgumentParser(description="OmniNPC Evaluation Runner")
parser.add_argument("--trials", choices=["audio", "visual", "hierarchy", "adversarial", "role", "all"], required=True)
parser.add_argument("--n", type=int, default=50)
parser.add_argument("--config", type=str, default="config/config.yaml")
args = parser.parse_args()
asyncio.run(run_eval(args))
if __name__ == "__main__":
main()