File size: 8,493 Bytes
8908233
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fc95770
 
 
 
 
 
 
 
8908233
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
986a92c
 
8908233
fc95770
 
8908233
 
 
 
 
 
 
 
 
 
 
 
986a92c
 
8908233
 
 
fc95770
 
 
 
8908233
 
fc95770
8908233
fc95770
 
 
 
 
 
 
 
986a92c
fc95770
8908233
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fc95770
8908233
fc95770
8908233
 
 
 
 
fc95770
8908233
 
 
 
fc95770
 
 
 
 
 
 
 
 
 
 
 
 
8908233
 
 
 
 
 
 
 
fc95770
 
 
 
 
 
 
8908233
fc95770
8908233
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
"""After a TODO run has finished 20 epochs: run the full test set with the inference script, then upload
the checkpoint + test results to the Hugging Face repo.

    source env.sh
    $PYTHON_BIN scripts/todo_eval_upload.py configs/nano_action_causal_pusht_sensoronly_v1.yaml

Steps
  1. check that $ACTS_RUNS_ROOT/<run_name>/epoch_019.pt exists (= 20 epochs done; refuses otherwise)
  2. eval/run_test_case.py --task <pusht|rope> --case <all test cases> --ckpt <that epoch_019.pt>,
     cases spread over the local GPUs (default 4, one process per GPU; resumable: finished cases are skipped)
  3. check every case has complete.json, write eval/summarize.py --reference output to summary.md
  4. upload to $ACTS_HF_REPO (default WitneyWW/acts-sensoronly-ablation), token from $HF_TOKEN:
       checkpoints/<run_name>/   epoch_019.pt, config.yaml, action_normalizer.json, train.log, metrics.jsonl, ...
       results/<run_name>/test_epoch_019/   every case dir (metrics json, pred latents, PNG strips, mp4s) + summary.md

Options: --ngpu N, --epoch E (default 19), --no-upload (eval only), --upload-only (skip eval).

VAL split (1116 TODO, pushT / rope only; called by scripts/infer_val.sh for epochs 14 and 15):
    $PYTHON_BIN scripts/todo_eval_upload.py <config> --split val --epoch 14
  runs eval/cases/<task>_val_cases.json on epoch_014.pt -> $ACTS_RUNS_ROOT/<run_name>/val_epoch_014/, summary.md paired
  against the parent run (nano_action_causal_<task>_v1) on the same val cases and epoch if that has already been run,
  uploads checkpoints/<run_name>/epoch_014.pt (TODO runs only; the parent ckpts are already in the repo) and
  results/<run_name>/val_epoch_014/. The parent configs (configs/nano_action_causal_{pusht,rope}_v1.yaml) are accepted:
  their checkpoint is read from $ACTS_CKPT_ROOT/<run_name>/ (no training needed).
"""
from __future__ import annotations

import argparse
import json
import os
import subprocess
import sys
from pathlib import Path

from omegaconf import OmegaConf

REPO = Path(__file__).resolve().parents[1]
TREE_TO_TASK = {"mixed_pushT": "pusht", "mixed_rope": "rope", "mixed_mb": "umi_sensoronly_noforce"}
RUN_FILES = ["config.yaml", "action_normalizer.json", "train.log", "metrics.jsonl", "data_manifest.json",
             "trainable_params.json", "load_report.json"]


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("config")
    ap.add_argument("--task", choices=["pusht", "rope", "umi_sensoronly_noforce"], default=None,
                    help="expected task; refuses if the config trains on another task (set by scripts/infer_<task>.sh)")
    ap.add_argument("--epoch", type=int, default=19, help="epoch index to evaluate (19 = the 20th epoch)")
    ap.add_argument("--split", default="test", choices=["test", "val"],
                    help="test (default, the 20-epoch TODO) or val (1116 TODO, pushT / rope, epochs 14 and 15)")
    ap.add_argument("--ngpu", type=int, default=int(os.environ.get("NGPU", 4)))
    ap.add_argument("--repo", default=os.environ.get("ACTS_HF_REPO", "WitneyWW/acts-sensoronly-ablation"))
    ap.add_argument("--no-upload", action="store_true")
    ap.add_argument("--upload-only", action="store_true")
    ap.add_argument("--cases", default=None, help="debug only: comma list of case ids instead of the full test set")
    ap.add_argument("--path-prefix", default="", help="debug only: prefix for the paths in the HF repo")
    a = ap.parse_args()

    cfg = OmegaConf.load(a.config)
    run = cfg.run_name
    tree = Path(str(cfg.data.latent_root)).name.removesuffix("_latents")
    task = TREE_TO_TASK[tree]
    if a.task and a.task != task:
        raise SystemExit(f"{a.config} trains on {tree} (task {task}), not on {a.task}: use scripts/infer_{task}.sh")
    runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs"))
    run_dir = runs / run
    ckpt = run_dir / f"epoch_{a.epoch:03d}.pt"
    shipped = Path(os.environ.get("ACTS_CKPT_ROOT", REPO / "checkpoints")) / run / ckpt.name
    from_repo = not ckpt.is_file() and shipped.is_file()  # parent run: checkpoint shipped in this repo
    if from_repo:
        ckpt = shipped
    if not ckpt.is_file():
        done = sorted(p.name for p in run_dir.glob("epoch_*.pt"))
        raise SystemExit(f"{ckpt} not found -- training has not reached epoch {a.epoch} yet (have: {done[-3:] or 'nothing'}). "
                         f"Re-submit scripts/train.sh with the same config to resume.")
    run_dir.mkdir(parents=True, exist_ok=True)
    tag = f"{a.split}_epoch_{a.epoch:03d}"
    out_root = run_dir / tag
    case_file = REPO / "eval" / "cases" / f"{task}_{a.split}_cases.json"
    if not case_file.is_file():
        raise SystemExit(f"no {a.split} cases for task {task} ({case_file})")
    man = json.loads(case_file.read_text())
    # the cases must come from the test/val split of the same split file the model was trained with
    if Path(str(cfg.data.split_file)).name != Path(man["split_file"]).name:
        raise SystemExit(f"split mismatch: config {cfg.data.split_file} vs {a.split} cases {man['split_file']}")
    if a.cases:
        man["cases"] = [man["cases"][int(i)] for i in a.cases.split(",")]
        ids_all = [int(i) for i in a.cases.split(",")]
    else:
        ids_all = list(range(len(man["cases"])))
    n = len(man["cases"])
    print(f"[todo] run={run} task={task} ckpt={ckpt} cases={n} -> {out_root}", flush=True)

    if not a.upload_only:
        procs = []
        for g in range(a.ngpu):
            ids = ids_all[g::a.ngpu]
            if not ids:
                continue
            env = dict(os.environ, CUDA_VISIBLE_DEVICES=str(g))
            log = open(run_dir / f"{tag}_gpu{g}.log", "a")
            procs.append(subprocess.Popen(
                [sys.executable, "-u", str(REPO / "eval" / "run_test_case.py"), "--task", task, "--split", a.split,
                 "--case", ",".join(map(str, ids)), "--ckpt", str(ckpt), "--out-root", str(out_root)],
                env=env, cwd=REPO, stdout=log, stderr=subprocess.STDOUT))
            print(f"[todo] gpu{g}: cases {ids}", flush=True)
        rcs = [p.wait() for p in procs]
        if any(rcs):
            raise SystemExit(f"eval failed (return codes {rcs}); see {run_dir}/{tag}_gpu*.log")

    missing = [c["output_subdir"] for c in man["cases"] if not (out_root / c["output_subdir"] / "complete.json").is_file()]
    if missing:
        raise SystemExit(f"{len(missing)}/{n} cases incomplete: {missing[:5]} -- re-run this script to resume")
    cmd = [sys.executable, str(REPO / "eval" / "summarize.py"), "--task", task, "--split", a.split, "--root", str(out_root)]
    if a.split == "test":
        cmd.append("--reference")
        note = f"reference = the {task} reference run"
    else:
        parent = runs / f"nano_action_causal_{task}_v1" / tag
        if run != parent.parent.name and parent.is_dir():
            cmd += ["--ref-root", str(parent)]
            note = f"paired vs {parent.parent.name}/{tag}"
        else:
            note = "no paired reference (parent run on these val cases not found)"
    summary = subprocess.run(cmd, cwd=REPO, check=True, capture_output=True, text=True).stdout
    summary = f"<!-- {run} epoch_{a.epoch:03d} on the {task} {a.split} cases; {note} -->\n" + summary
    (out_root / "summary.md").write_text(summary)
    print(summary, flush=True)

    if a.no_upload:
        print("[todo] --no-upload: done", flush=True)
        return
    from huggingface_hub import HfApi
    api = HfApi(token=os.environ.get("HF_TOKEN"))
    if from_repo:
        print(f"[todo] {ckpt} is already in the repo: checkpoint upload skipped", flush=True)
    else:
        files = [ckpt.name] + [f for f in RUN_FILES if (run_dir / f).is_file()]
        api.upload_folder(repo_id=a.repo, folder_path=run_dir, path_in_repo=f"{a.path_prefix}checkpoints/{run}", allow_patterns=files,
                          commit_message=f"add {run} epoch_{a.epoch:03d} checkpoint")
        print(f"[todo] uploaded {a.path_prefix}checkpoints/{run}/ {files}", flush=True)
    api.upload_folder(repo_id=a.repo, folder_path=out_root, path_in_repo=f"{a.path_prefix}results/{run}/{out_root.name}",
                      commit_message=f"add {run} {a.split} results ({n} cases, epoch_{a.epoch:03d})")
    print(f"[todo] uploaded {a.path_prefix}results/{run}/{out_root.name}/ -> https://huggingface.co/{a.repo}", flush=True)


if __name__ == "__main__":
    main()