File size: 2,672 Bytes
13cc860
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
"""Run the normal fixed 35-case/5-case evaluators with scale-local outputs."""
import argparse
import hashlib
import importlib.util
import json
from pathlib import Path
import sys

ROOT = Path(__file__).resolve().parents[2]
SCALE = ROOT / "scale"
sys.path.insert(0, str(ROOT))
import start_visualization as normal


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--run-dir", type=Path, required=True)
    parser.add_argument("--suite", choices=["main", "unseen"], default="main")
    parser.add_argument("--device", choices=["cpu", "cuda"], default="cpu")
    parser.add_argument("--check-only", action="store_true")
    parser.add_argument("--once", action="store_true")
    args = parser.parse_args()
    run = args.run_dir.resolve(strict=True)
    if not run.is_relative_to(SCALE.resolve()):
        parser.error("Only scale-local runs may be evaluated by this wrapper")
    normal.PROJECT = SCALE
    kind, count = ("auto_cases", 35) if args.suite == "main" else ("unseen_video", 5)
    selection = normal.prepare_selection(run, args.suite, kind, count)
    source = normal.RESOURCE_PROJECT / "watch_training_cases.py"
    spec = importlib.util.spec_from_file_location("scale_normal_watcher", source)
    watcher = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(watcher)
    watcher.PREVIEW = SCALE / "protocol"
    watcher.GPU_STAGE = SCALE / "script/eval_gpu_stage.py"
    watcher.INFER_PYTHON = sys.executable
    # PROJECT is kept at the normal resource repo for its unchanged best-checkpoint metric.
    argv = [str(source), "--run-dir", str(run), "--log", str(run / "training.log"),
            "--selection", str(selection), "--output-dir", str(SCALE / "evaluations" / kind / run.name),
            "--dashboard-dir", str(SCALE / "visualization" / run.name / args.suite),
            "--device", args.device, "--gpu-index", "0", "--gpu-policy", "shared"]
    if args.suite == "main":
        argv.append("--save-best-checkpoint")
    if args.check_only:
        argv.append("--check-only")
    if args.once:
        argv.append("--once")
    audit = {"watcher": str(source), "watcher_sha256": hashlib.sha256(source.read_bytes()).hexdigest(),
             "metric": "main_raw_body_jerk_30hz", "suite": args.suite,
             "case_count": count, "command": argv,
             "protocol_sha256": {p.name: hashlib.sha256(p.read_bytes()).hexdigest()
                                 for p in watcher.PREVIEW.glob("*.py")}}
    normal.save_json(SCALE / "evaluations/audits" / run.name / (args.suite + ".json"), audit)
    sys.argv = argv
    watcher.main()


if __name__ == "__main__":
    main()