File size: 2,672 Bytes
13cc860 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | """Run the normal fixed 35-case/5-case evaluators with scale-local outputs."""
import argparse
import hashlib
import importlib.util
import json
from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[2]
SCALE = ROOT / "scale"
sys.path.insert(0, str(ROOT))
import start_visualization as normal
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--suite", choices=["main", "unseen"], default="main")
parser.add_argument("--device", choices=["cpu", "cuda"], default="cpu")
parser.add_argument("--check-only", action="store_true")
parser.add_argument("--once", action="store_true")
args = parser.parse_args()
run = args.run_dir.resolve(strict=True)
if not run.is_relative_to(SCALE.resolve()):
parser.error("Only scale-local runs may be evaluated by this wrapper")
normal.PROJECT = SCALE
kind, count = ("auto_cases", 35) if args.suite == "main" else ("unseen_video", 5)
selection = normal.prepare_selection(run, args.suite, kind, count)
source = normal.RESOURCE_PROJECT / "watch_training_cases.py"
spec = importlib.util.spec_from_file_location("scale_normal_watcher", source)
watcher = importlib.util.module_from_spec(spec)
spec.loader.exec_module(watcher)
watcher.PREVIEW = SCALE / "protocol"
watcher.GPU_STAGE = SCALE / "script/eval_gpu_stage.py"
watcher.INFER_PYTHON = sys.executable
# PROJECT is kept at the normal resource repo for its unchanged best-checkpoint metric.
argv = [str(source), "--run-dir", str(run), "--log", str(run / "training.log"),
"--selection", str(selection), "--output-dir", str(SCALE / "evaluations" / kind / run.name),
"--dashboard-dir", str(SCALE / "visualization" / run.name / args.suite),
"--device", args.device, "--gpu-index", "0", "--gpu-policy", "shared"]
if args.suite == "main":
argv.append("--save-best-checkpoint")
if args.check_only:
argv.append("--check-only")
if args.once:
argv.append("--once")
audit = {"watcher": str(source), "watcher_sha256": hashlib.sha256(source.read_bytes()).hexdigest(),
"metric": "main_raw_body_jerk_30hz", "suite": args.suite,
"case_count": count, "command": argv,
"protocol_sha256": {p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in watcher.PREVIEW.glob("*.py")}}
normal.save_json(SCALE / "evaluations/audits" / run.name / (args.suite + ".json"), audit)
sys.argv = argv
watcher.main()
if __name__ == "__main__":
main()
|