"""Run the normal fixed 35-case/5-case evaluators with scale-local outputs.""" import argparse import hashlib import importlib.util import json from pathlib import Path import sys ROOT = Path(__file__).resolve().parents[2] SCALE = ROOT / "scale" sys.path.insert(0, str(ROOT)) import start_visualization as normal def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--run-dir", type=Path, required=True) parser.add_argument("--suite", choices=["main", "unseen"], default="main") parser.add_argument("--device", choices=["cpu", "cuda"], default="cpu") parser.add_argument("--check-only", action="store_true") parser.add_argument("--once", action="store_true") args = parser.parse_args() run = args.run_dir.resolve(strict=True) if not run.is_relative_to(SCALE.resolve()): parser.error("Only scale-local runs may be evaluated by this wrapper") normal.PROJECT = SCALE kind, count = ("auto_cases", 35) if args.suite == "main" else ("unseen_video", 5) selection = normal.prepare_selection(run, args.suite, kind, count) source = normal.RESOURCE_PROJECT / "watch_training_cases.py" spec = importlib.util.spec_from_file_location("scale_normal_watcher", source) watcher = importlib.util.module_from_spec(spec) spec.loader.exec_module(watcher) watcher.PREVIEW = SCALE / "protocol" watcher.GPU_STAGE = SCALE / "script/eval_gpu_stage.py" watcher.INFER_PYTHON = sys.executable # PROJECT is kept at the normal resource repo for its unchanged best-checkpoint metric. argv = [str(source), "--run-dir", str(run), "--log", str(run / "training.log"), "--selection", str(selection), "--output-dir", str(SCALE / "evaluations" / kind / run.name), "--dashboard-dir", str(SCALE / "visualization" / run.name / args.suite), "--device", args.device, "--gpu-index", "0", "--gpu-policy", "shared"] if args.suite == "main": argv.append("--save-best-checkpoint") if args.check_only: argv.append("--check-only") if args.once: argv.append("--once") audit = {"watcher": str(source), "watcher_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), "metric": "main_raw_body_jerk_30hz", "suite": args.suite, "case_count": count, "command": argv, "protocol_sha256": {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in watcher.PREVIEW.glob("*.py")}} normal.save_json(SCALE / "evaluations/audits" / run.name / (args.suite + ".json"), audit) sys.argv = argv watcher.main() if __name__ == "__main__": main()