#!/usr/bin/env python3 """Evaluation runner for OmniNPC. Usage: python run_evaluation.py --trials audio --n 50 python run_evaluation.py --trials visual --n 50 python run_evaluation.py --trials hierarchy --n 20 python run_evaluation.py --trials all """ from __future__ import annotations import argparse import asyncio import json import logging from datetime import datetime from pathlib import Path from dotenv import load_dotenv load_dotenv() logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger(__name__) async def run_eval(args) -> None: from config import load_config config = load_config(args.config) from evaluation.trial_runner import TrialRunner runner = TrialRunner(config) audio_r: dict = {} visual_r: dict = {} hierarchy_r: dict = {} adversarial_r: dict = {} role_r: dict = {} if args.trials in ("audio", "all"): audio_r = await runner.run_audio_trials(n=args.n) logger.info("Audio results: %s", audio_r) if args.trials in ("visual", "all"): visual_r = await runner.run_visual_trials(n=args.n) logger.info("Visual results: %s", visual_r) if args.trials in ("hierarchy", "all"): hierarchy_r = await runner.run_hierarchy_comparison(n=min(args.n, 30)) logger.info("Hierarchy results: %s", hierarchy_r) if args.trials in ("adversarial", "all"): adversarial_r = await runner.run_adversarial_trials(n=min(args.n, 10)) logger.info("Adversarial results: %s", adversarial_r) if args.trials in ("role", "all"): role_r = await runner.run_role_distinctiveness_trials(n=min(args.n, 30)) logger.info("Role results: %s", role_r) # Build combined results in the format generate_paper_table() expects acrl = audio_r.get("acrl_mean_ms", 0) combined = { "timestamp": datetime.now().isoformat(), # ACRL "acrl_mean_ms": acrl, "acrl_std_ms": audio_r.get("acrl_std_ms", 0), "acrl_p95_ms": audio_r.get("acrl_p95_ms", 0), # VTRT "vtrt_mean_ms": visual_r.get("vtrt_mean_ms", 0), "vtrt_std_ms": visual_r.get("vtrt_std_ms", 0), "vtrt_p95_ms": visual_r.get("vtrt_p95_ms", 0), "vtrt_evasion_rate": visual_r.get("pass_rate", 0), "vtrt_evasion_ci_lo": visual_r.get("vtrt_evasion_ci_lo"), "vtrt_evasion_ci_hi": visual_r.get("vtrt_evasion_ci_hi"), "vtrt_nodirective_evasion_rate": visual_r.get("vtrt_nodirective_evasion_rate", 0), # CMSS "cmss": audio_r.get("pass_rate", 0), "cmss_ci_lo": audio_r.get("cmss_ci_lo"), "cmss_ci_hi": audio_r.get("cmss_ci_hi"), # Directive adherence "survive_directive_adherence": hierarchy_r.get("survive_directive_adherence_pct", 0), "survive_ci_lo": hierarchy_r.get("survive_ci_lo"), "survive_ci_hi": hierarchy_r.get("survive_ci_hi"), "collect_directive_adherence": hierarchy_r.get("collect_directive_adherence_pct", 0), "collect_ci_lo": hierarchy_r.get("collect_ci_lo"), "collect_ci_hi": hierarchy_r.get("collect_ci_hi"), "vague_baseline_rate": round( (hierarchy_r.get("vague_baseline_evasive_pct", 0) + hierarchy_r.get("vague_baseline_collect_pct", 0)) / 2, 1 ), "directive_lift_pp": round( (hierarchy_r.get("directive_lift_survive_pp", 0) + hierarchy_r.get("directive_lift_collect_pp", 0)) / 2, 1 ), # Legacy fields "nano_completion": hierarchy_r.get("without_super_rate", 0), "super_completion": hierarchy_r.get("with_super_rate", 0), "fps": round(1000.0 / acrl, 2) if acrl else 0, "n_audio": audio_r.get("n", 0), "n_visual": visual_r.get("n", 0), "n_hierarchy": hierarchy_r.get("n", 0), "audio_detail": audio_r, "visual_detail": visual_r, "hierarchy_detail": hierarchy_r, "adversarial_detail": adversarial_r, "role_detail": role_r, } out_dir = Path("data/results") out_dir.mkdir(parents=True, exist_ok=True) out_path = out_dir / "trial_results.json" # Merge with existing results so individual --trials runs don't zero out other sections existing: dict = {} if out_path.exists(): try: with open(out_path, encoding="utf-8") as f: existing = json.load(f) except (json.JSONDecodeError, OSError): pass # Only overwrite fields for the trials that were actually run this time if args.trials in ("audio", "all"): for k in ("acrl_mean_ms", "acrl_std_ms", "acrl_p95_ms", "cmss", "cmss_ci_lo", "cmss_ci_hi", "n_audio", "audio_detail"): existing[k] = combined[k] if args.trials in ("visual", "all"): for k in ("vtrt_mean_ms", "vtrt_std_ms", "vtrt_p95_ms", "vtrt_evasion_rate", "vtrt_evasion_ci_lo", "vtrt_evasion_ci_hi", "n_visual", "visual_detail", "vtrt_nodirective_evasion_rate"): existing[k] = combined[k] if args.trials in ("hierarchy", "all"): for k in ("survive_directive_adherence", "survive_ci_lo", "survive_ci_hi", "collect_directive_adherence", "collect_ci_lo", "collect_ci_hi", "vague_baseline_rate", "directive_lift_pp", "nano_completion", "super_completion", "n_hierarchy", "hierarchy_detail"): existing[k] = combined[k] if args.trials in ("adversarial", "all"): existing["adversarial_detail"] = combined["adversarial_detail"] if args.trials in ("role", "all"): existing["role_detail"] = combined["role_detail"] # Always update top-level metadata existing["timestamp"] = combined["timestamp"] if combined.get("fps"): existing["fps"] = combined["fps"] with open(out_path, "w", encoding="utf-8") as f: json.dump(existing, f, indent=2) logger.info("Trial results saved to %s", out_path) logger.info("Evaluation complete. Run `python run_analysis.py` to generate paper table.") def main() -> None: parser = argparse.ArgumentParser(description="OmniNPC Evaluation Runner") parser.add_argument("--trials", choices=["audio", "visual", "hierarchy", "adversarial", "role", "all"], required=True) parser.add_argument("--n", type=int, default=50) parser.add_argument("--config", type=str, default="config/config.yaml") args = parser.parse_args() asyncio.run(run_eval(args)) if __name__ == "__main__": main()