File size: 6,583 Bytes
8908233
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
986a92c
 
8908233
 
 
 
 
 
 
 
 
 
 
 
 
986a92c
 
8908233
 
 
 
 
 
 
 
 
986a92c
 
 
8908233
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
"""After a TODO run has finished 20 epochs: run the full test set with the inference script, then upload
the checkpoint + test results to the Hugging Face repo.

    source env.sh
    $PYTHON_BIN scripts/todo_eval_upload.py configs/nano_action_causal_pusht_sensoronly_v1.yaml

Steps
  1. check that $ACTS_RUNS_ROOT/<run_name>/epoch_019.pt exists (= 20 epochs done; refuses otherwise)
  2. eval/run_test_case.py --task <pusht|rope> --case <all test cases> --ckpt <that epoch_019.pt>,
     cases spread over the local GPUs (default 4, one process per GPU; resumable: finished cases are skipped)
  3. check every case has complete.json, write eval/summarize.py --reference output to summary.md
  4. upload to $ACTS_HF_REPO (default WitneyWW/acts-sensoronly-ablation), token from $HF_TOKEN:
       checkpoints/<run_name>/   epoch_019.pt, config.yaml, action_normalizer.json, train.log, metrics.jsonl, ...
       results/<run_name>/test_epoch_019/   every case dir (metrics json, pred latents, PNG strips, mp4s) + summary.md

Options: --ngpu N, --epoch E (default 19), --no-upload (eval only), --upload-only (skip eval).
"""
from __future__ import annotations

import argparse
import json
import os
import subprocess
import sys
from pathlib import Path

from omegaconf import OmegaConf

REPO = Path(__file__).resolve().parents[1]
TREE_TO_TASK = {"mixed_pushT": "pusht", "mixed_rope": "rope", "mixed_mb": "umi_sensoronly_noforce"}
RUN_FILES = ["config.yaml", "action_normalizer.json", "train.log", "metrics.jsonl", "data_manifest.json",
             "trainable_params.json", "load_report.json"]


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("config")
    ap.add_argument("--task", choices=["pusht", "rope", "umi_sensoronly_noforce"], default=None,
                    help="expected task; refuses if the config trains on another task (set by scripts/infer_<task>.sh)")
    ap.add_argument("--epoch", type=int, default=19, help="epoch index to evaluate (19 = the 20th epoch)")
    ap.add_argument("--ngpu", type=int, default=int(os.environ.get("NGPU", 4)))
    ap.add_argument("--repo", default=os.environ.get("ACTS_HF_REPO", "WitneyWW/acts-sensoronly-ablation"))
    ap.add_argument("--no-upload", action="store_true")
    ap.add_argument("--upload-only", action="store_true")
    ap.add_argument("--cases", default=None, help="debug only: comma list of case ids instead of the full test set")
    ap.add_argument("--path-prefix", default="", help="debug only: prefix for the paths in the HF repo")
    a = ap.parse_args()

    cfg = OmegaConf.load(a.config)
    run = cfg.run_name
    tree = Path(str(cfg.data.latent_root)).name.removesuffix("_latents")
    task = TREE_TO_TASK[tree]
    if a.task and a.task != task:
        raise SystemExit(f"{a.config} trains on {tree} (task {task}), not on {a.task}: use scripts/infer_{task}.sh")
    runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs"))
    run_dir = runs / run
    ckpt = run_dir / f"epoch_{a.epoch:03d}.pt"
    if not ckpt.is_file():
        done = sorted(p.name for p in run_dir.glob("epoch_*.pt"))
        raise SystemExit(f"{ckpt} not found -- training is not finished (have: {done[-3:] or 'nothing'}). "
                         f"Re-submit scripts/train.sh with the same config to resume.")
    out_root = run_dir / f"test_epoch_{a.epoch:03d}"
    man = json.loads((REPO / "eval" / "cases" / f"{task}_test_cases.json").read_text())
    # the test cases must come from the test split of the same split file the model was trained with
    if Path(str(cfg.data.split_file)).name != Path(man["split_file"]).name:
        raise SystemExit(f"split mismatch: config {cfg.data.split_file} vs test cases {man['split_file']}")
    if a.cases:
        man["cases"] = [man["cases"][int(i)] for i in a.cases.split(",")]
        ids_all = [int(i) for i in a.cases.split(",")]
    else:
        ids_all = list(range(len(man["cases"])))
    n = len(man["cases"])
    print(f"[todo] run={run} task={task} ckpt={ckpt} cases={n} -> {out_root}", flush=True)

    if not a.upload_only:
        procs = []
        for g in range(a.ngpu):
            ids = ids_all[g::a.ngpu]
            if not ids:
                continue
            env = dict(os.environ, CUDA_VISIBLE_DEVICES=str(g))
            log = open(run_dir / f"test_epoch_{a.epoch:03d}_gpu{g}.log", "a")
            procs.append(subprocess.Popen(
                [sys.executable, "-u", str(REPO / "eval" / "run_test_case.py"), "--task", task,
                 "--case", ",".join(map(str, ids)), "--ckpt", str(ckpt), "--out-root", str(out_root)],
                env=env, cwd=REPO, stdout=log, stderr=subprocess.STDOUT))
            print(f"[todo] gpu{g}: cases {ids}", flush=True)
        rcs = [p.wait() for p in procs]
        if any(rcs):
            raise SystemExit(f"eval failed (return codes {rcs}); see {run_dir}/test_epoch_{a.epoch:03d}_gpu*.log")

    missing = [c["output_subdir"] for c in man["cases"] if not (out_root / c["output_subdir"] / "complete.json").is_file()]
    if missing:
        raise SystemExit(f"{len(missing)}/{n} cases incomplete: {missing[:5]} -- re-run this script to resume")
    summary = subprocess.run([sys.executable, str(REPO / "eval" / "summarize.py"), "--task", task, "--reference",
                              "--root", str(out_root)], cwd=REPO, check=True, capture_output=True, text=True).stdout
    summary = f"<!-- {run} epoch_{a.epoch:03d} on the {task} test cases; reference = the {task} reference run -->\n" + summary
    (out_root / "summary.md").write_text(summary)
    print(summary, flush=True)

    if a.no_upload:
        print("[todo] --no-upload: done", flush=True)
        return
    from huggingface_hub import HfApi
    api = HfApi(token=os.environ.get("HF_TOKEN"))
    files = [ckpt.name] + [f for f in RUN_FILES if (run_dir / f).is_file()]
    api.upload_folder(repo_id=a.repo, folder_path=run_dir, path_in_repo=f"{a.path_prefix}checkpoints/{run}", allow_patterns=files,
                      commit_message=f"add {run} epoch_{a.epoch:03d} checkpoint")
    print(f"[todo] uploaded {a.path_prefix}checkpoints/{run}/ {files}", flush=True)
    api.upload_folder(repo_id=a.repo, folder_path=out_root, path_in_repo=f"{a.path_prefix}results/{run}/{out_root.name}",
                      commit_message=f"add {run} test results ({n} cases, epoch_{a.epoch:03d})")
    print(f"[todo] uploaded {a.path_prefix}results/{run}/{out_root.name}/ -> https://huggingface.co/{a.repo}", flush=True)


if __name__ == "__main__":
    main()