Spaces:
Sleeping
Sleeping
Download run_evaluation.py from aadi-joshi/omninpc: direct link, hf CLI and curl.
- Browser
- Download file 6.63 kB
-
https://huggingface.co/spaces/aadi-joshi/omninpc/resolve/main/run_evaluation.py
- Command line
-
hf download hf://spaces/aadi-joshi/omninpc/run_evaluation.py
-
curl -L -o run_evaluation.py https://huggingface.co/spaces/aadi-joshi/omninpc/resolve/main/run_evaluation.py
6.63 kB
| #!/usr/bin/env python3 | |
| """Evaluation runner for OmniNPC. | |
| Usage: | |
| python run_evaluation.py --trials audio --n 50 | |
| python run_evaluation.py --trials visual --n 50 | |
| python run_evaluation.py --trials hierarchy --n 20 | |
| python run_evaluation.py --trials all | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import asyncio | |
| import json | |
| import logging | |
| from datetime import datetime | |
| from pathlib import Path | |
| from dotenv import load_dotenv | |
| load_dotenv() | |
| logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") | |
| logger = logging.getLogger(__name__) | |
| async def run_eval(args) -> None: | |
| from config import load_config | |
| config = load_config(args.config) | |
| from evaluation.trial_runner import TrialRunner | |
| runner = TrialRunner(config) | |
| audio_r: dict = {} | |
| visual_r: dict = {} | |
| hierarchy_r: dict = {} | |
| adversarial_r: dict = {} | |
| role_r: dict = {} | |
| if args.trials in ("audio", "all"): | |
| audio_r = await runner.run_audio_trials(n=args.n) | |
| logger.info("Audio results: %s", audio_r) | |
| if args.trials in ("visual", "all"): | |
| visual_r = await runner.run_visual_trials(n=args.n) | |
| logger.info("Visual results: %s", visual_r) | |
| if args.trials in ("hierarchy", "all"): | |
| hierarchy_r = await runner.run_hierarchy_comparison(n=min(args.n, 30)) | |
| logger.info("Hierarchy results: %s", hierarchy_r) | |
| if args.trials in ("adversarial", "all"): | |
| adversarial_r = await runner.run_adversarial_trials(n=min(args.n, 10)) | |
| logger.info("Adversarial results: %s", adversarial_r) | |
| if args.trials in ("role", "all"): | |
| role_r = await runner.run_role_distinctiveness_trials(n=min(args.n, 30)) | |
| logger.info("Role results: %s", role_r) | |
| # Build combined results in the format generate_paper_table() expects | |
| acrl = audio_r.get("acrl_mean_ms", 0) | |
| combined = { | |
| "timestamp": datetime.now().isoformat(), | |
| # ACRL | |
| "acrl_mean_ms": acrl, | |
| "acrl_std_ms": audio_r.get("acrl_std_ms", 0), | |
| "acrl_p95_ms": audio_r.get("acrl_p95_ms", 0), | |
| # VTRT | |
| "vtrt_mean_ms": visual_r.get("vtrt_mean_ms", 0), | |
| "vtrt_std_ms": visual_r.get("vtrt_std_ms", 0), | |
| "vtrt_p95_ms": visual_r.get("vtrt_p95_ms", 0), | |
| "vtrt_evasion_rate": visual_r.get("pass_rate", 0), | |
| "vtrt_evasion_ci_lo": visual_r.get("vtrt_evasion_ci_lo"), | |
| "vtrt_evasion_ci_hi": visual_r.get("vtrt_evasion_ci_hi"), | |
| "vtrt_nodirective_evasion_rate": visual_r.get("vtrt_nodirective_evasion_rate", 0), | |
| # CMSS | |
| "cmss": audio_r.get("pass_rate", 0), | |
| "cmss_ci_lo": audio_r.get("cmss_ci_lo"), | |
| "cmss_ci_hi": audio_r.get("cmss_ci_hi"), | |
| # Directive adherence | |
| "survive_directive_adherence": hierarchy_r.get("survive_directive_adherence_pct", 0), | |
| "survive_ci_lo": hierarchy_r.get("survive_ci_lo"), | |
| "survive_ci_hi": hierarchy_r.get("survive_ci_hi"), | |
| "collect_directive_adherence": hierarchy_r.get("collect_directive_adherence_pct", 0), | |
| "collect_ci_lo": hierarchy_r.get("collect_ci_lo"), | |
| "collect_ci_hi": hierarchy_r.get("collect_ci_hi"), | |
| "vague_baseline_rate": round( | |
| (hierarchy_r.get("vague_baseline_evasive_pct", 0) + | |
| hierarchy_r.get("vague_baseline_collect_pct", 0)) / 2, 1 | |
| ), | |
| "directive_lift_pp": round( | |
| (hierarchy_r.get("directive_lift_survive_pp", 0) + | |
| hierarchy_r.get("directive_lift_collect_pp", 0)) / 2, 1 | |
| ), | |
| # Legacy fields | |
| "nano_completion": hierarchy_r.get("without_super_rate", 0), | |
| "super_completion": hierarchy_r.get("with_super_rate", 0), | |
| "fps": round(1000.0 / acrl, 2) if acrl else 0, | |
| "n_audio": audio_r.get("n", 0), | |
| "n_visual": visual_r.get("n", 0), | |
| "n_hierarchy": hierarchy_r.get("n", 0), | |
| "audio_detail": audio_r, | |
| "visual_detail": visual_r, | |
| "hierarchy_detail": hierarchy_r, | |
| "adversarial_detail": adversarial_r, | |
| "role_detail": role_r, | |
| } | |
| out_dir = Path("data/results") | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| out_path = out_dir / "trial_results.json" | |
| # Merge with existing results so individual --trials runs don't zero out other sections | |
| existing: dict = {} | |
| if out_path.exists(): | |
| try: | |
| with open(out_path, encoding="utf-8") as f: | |
| existing = json.load(f) | |
| except (json.JSONDecodeError, OSError): | |
| pass | |
| # Only overwrite fields for the trials that were actually run this time | |
| if args.trials in ("audio", "all"): | |
| for k in ("acrl_mean_ms", "acrl_std_ms", "acrl_p95_ms", | |
| "cmss", "cmss_ci_lo", "cmss_ci_hi", "n_audio", "audio_detail"): | |
| existing[k] = combined[k] | |
| if args.trials in ("visual", "all"): | |
| for k in ("vtrt_mean_ms", "vtrt_std_ms", "vtrt_p95_ms", | |
| "vtrt_evasion_rate", "vtrt_evasion_ci_lo", "vtrt_evasion_ci_hi", | |
| "n_visual", "visual_detail", | |
| "vtrt_nodirective_evasion_rate"): | |
| existing[k] = combined[k] | |
| if args.trials in ("hierarchy", "all"): | |
| for k in ("survive_directive_adherence", "survive_ci_lo", "survive_ci_hi", | |
| "collect_directive_adherence", "collect_ci_lo", "collect_ci_hi", | |
| "vague_baseline_rate", "directive_lift_pp", | |
| "nano_completion", "super_completion", | |
| "n_hierarchy", "hierarchy_detail"): | |
| existing[k] = combined[k] | |
| if args.trials in ("adversarial", "all"): | |
| existing["adversarial_detail"] = combined["adversarial_detail"] | |
| if args.trials in ("role", "all"): | |
| existing["role_detail"] = combined["role_detail"] | |
| # Always update top-level metadata | |
| existing["timestamp"] = combined["timestamp"] | |
| if combined.get("fps"): | |
| existing["fps"] = combined["fps"] | |
| with open(out_path, "w", encoding="utf-8") as f: | |
| json.dump(existing, f, indent=2) | |
| logger.info("Trial results saved to %s", out_path) | |
| logger.info("Evaluation complete. Run `python run_analysis.py` to generate paper table.") | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description="OmniNPC Evaluation Runner") | |
| parser.add_argument("--trials", choices=["audio", "visual", "hierarchy", "adversarial", "role", "all"], required=True) | |
| parser.add_argument("--n", type=int, default=50) | |
| parser.add_argument("--config", type=str, default="config/config.yaml") | |
| args = parser.parse_args() | |
| asyncio.run(run_eval(args)) | |
| if __name__ == "__main__": | |
| main() | |