Instructions to use WitneyWW/acts-sensoronly-ablation with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Cosmos
How to use WitneyWW/acts-sensoronly-ablation with Cosmos:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
File size: 8,493 Bytes
8908233 fc95770 8908233 986a92c 8908233 fc95770 8908233 986a92c 8908233 fc95770 8908233 fc95770 8908233 fc95770 986a92c fc95770 8908233 fc95770 8908233 fc95770 8908233 fc95770 8908233 fc95770 8908233 fc95770 8908233 fc95770 8908233 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | """After a TODO run has finished 20 epochs: run the full test set with the inference script, then upload
the checkpoint + test results to the Hugging Face repo.
source env.sh
$PYTHON_BIN scripts/todo_eval_upload.py configs/nano_action_causal_pusht_sensoronly_v1.yaml
Steps
1. check that $ACTS_RUNS_ROOT/<run_name>/epoch_019.pt exists (= 20 epochs done; refuses otherwise)
2. eval/run_test_case.py --task <pusht|rope> --case <all test cases> --ckpt <that epoch_019.pt>,
cases spread over the local GPUs (default 4, one process per GPU; resumable: finished cases are skipped)
3. check every case has complete.json, write eval/summarize.py --reference output to summary.md
4. upload to $ACTS_HF_REPO (default WitneyWW/acts-sensoronly-ablation), token from $HF_TOKEN:
checkpoints/<run_name>/ epoch_019.pt, config.yaml, action_normalizer.json, train.log, metrics.jsonl, ...
results/<run_name>/test_epoch_019/ every case dir (metrics json, pred latents, PNG strips, mp4s) + summary.md
Options: --ngpu N, --epoch E (default 19), --no-upload (eval only), --upload-only (skip eval).
VAL split (1116 TODO, pushT / rope only; called by scripts/infer_val.sh for epochs 14 and 15):
$PYTHON_BIN scripts/todo_eval_upload.py <config> --split val --epoch 14
runs eval/cases/<task>_val_cases.json on epoch_014.pt -> $ACTS_RUNS_ROOT/<run_name>/val_epoch_014/, summary.md paired
against the parent run (nano_action_causal_<task>_v1) on the same val cases and epoch if that has already been run,
uploads checkpoints/<run_name>/epoch_014.pt (TODO runs only; the parent ckpts are already in the repo) and
results/<run_name>/val_epoch_014/. The parent configs (configs/nano_action_causal_{pusht,rope}_v1.yaml) are accepted:
their checkpoint is read from $ACTS_CKPT_ROOT/<run_name>/ (no training needed).
"""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
from pathlib import Path
from omegaconf import OmegaConf
REPO = Path(__file__).resolve().parents[1]
TREE_TO_TASK = {"mixed_pushT": "pusht", "mixed_rope": "rope", "mixed_mb": "umi_sensoronly_noforce"}
RUN_FILES = ["config.yaml", "action_normalizer.json", "train.log", "metrics.jsonl", "data_manifest.json",
"trainable_params.json", "load_report.json"]
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("config")
ap.add_argument("--task", choices=["pusht", "rope", "umi_sensoronly_noforce"], default=None,
help="expected task; refuses if the config trains on another task (set by scripts/infer_<task>.sh)")
ap.add_argument("--epoch", type=int, default=19, help="epoch index to evaluate (19 = the 20th epoch)")
ap.add_argument("--split", default="test", choices=["test", "val"],
help="test (default, the 20-epoch TODO) or val (1116 TODO, pushT / rope, epochs 14 and 15)")
ap.add_argument("--ngpu", type=int, default=int(os.environ.get("NGPU", 4)))
ap.add_argument("--repo", default=os.environ.get("ACTS_HF_REPO", "WitneyWW/acts-sensoronly-ablation"))
ap.add_argument("--no-upload", action="store_true")
ap.add_argument("--upload-only", action="store_true")
ap.add_argument("--cases", default=None, help="debug only: comma list of case ids instead of the full test set")
ap.add_argument("--path-prefix", default="", help="debug only: prefix for the paths in the HF repo")
a = ap.parse_args()
cfg = OmegaConf.load(a.config)
run = cfg.run_name
tree = Path(str(cfg.data.latent_root)).name.removesuffix("_latents")
task = TREE_TO_TASK[tree]
if a.task and a.task != task:
raise SystemExit(f"{a.config} trains on {tree} (task {task}), not on {a.task}: use scripts/infer_{task}.sh")
runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs"))
run_dir = runs / run
ckpt = run_dir / f"epoch_{a.epoch:03d}.pt"
shipped = Path(os.environ.get("ACTS_CKPT_ROOT", REPO / "checkpoints")) / run / ckpt.name
from_repo = not ckpt.is_file() and shipped.is_file() # parent run: checkpoint shipped in this repo
if from_repo:
ckpt = shipped
if not ckpt.is_file():
done = sorted(p.name for p in run_dir.glob("epoch_*.pt"))
raise SystemExit(f"{ckpt} not found -- training has not reached epoch {a.epoch} yet (have: {done[-3:] or 'nothing'}). "
f"Re-submit scripts/train.sh with the same config to resume.")
run_dir.mkdir(parents=True, exist_ok=True)
tag = f"{a.split}_epoch_{a.epoch:03d}"
out_root = run_dir / tag
case_file = REPO / "eval" / "cases" / f"{task}_{a.split}_cases.json"
if not case_file.is_file():
raise SystemExit(f"no {a.split} cases for task {task} ({case_file})")
man = json.loads(case_file.read_text())
# the cases must come from the test/val split of the same split file the model was trained with
if Path(str(cfg.data.split_file)).name != Path(man["split_file"]).name:
raise SystemExit(f"split mismatch: config {cfg.data.split_file} vs {a.split} cases {man['split_file']}")
if a.cases:
man["cases"] = [man["cases"][int(i)] for i in a.cases.split(",")]
ids_all = [int(i) for i in a.cases.split(",")]
else:
ids_all = list(range(len(man["cases"])))
n = len(man["cases"])
print(f"[todo] run={run} task={task} ckpt={ckpt} cases={n} -> {out_root}", flush=True)
if not a.upload_only:
procs = []
for g in range(a.ngpu):
ids = ids_all[g::a.ngpu]
if not ids:
continue
env = dict(os.environ, CUDA_VISIBLE_DEVICES=str(g))
log = open(run_dir / f"{tag}_gpu{g}.log", "a")
procs.append(subprocess.Popen(
[sys.executable, "-u", str(REPO / "eval" / "run_test_case.py"), "--task", task, "--split", a.split,
"--case", ",".join(map(str, ids)), "--ckpt", str(ckpt), "--out-root", str(out_root)],
env=env, cwd=REPO, stdout=log, stderr=subprocess.STDOUT))
print(f"[todo] gpu{g}: cases {ids}", flush=True)
rcs = [p.wait() for p in procs]
if any(rcs):
raise SystemExit(f"eval failed (return codes {rcs}); see {run_dir}/{tag}_gpu*.log")
missing = [c["output_subdir"] for c in man["cases"] if not (out_root / c["output_subdir"] / "complete.json").is_file()]
if missing:
raise SystemExit(f"{len(missing)}/{n} cases incomplete: {missing[:5]} -- re-run this script to resume")
cmd = [sys.executable, str(REPO / "eval" / "summarize.py"), "--task", task, "--split", a.split, "--root", str(out_root)]
if a.split == "test":
cmd.append("--reference")
note = f"reference = the {task} reference run"
else:
parent = runs / f"nano_action_causal_{task}_v1" / tag
if run != parent.parent.name and parent.is_dir():
cmd += ["--ref-root", str(parent)]
note = f"paired vs {parent.parent.name}/{tag}"
else:
note = "no paired reference (parent run on these val cases not found)"
summary = subprocess.run(cmd, cwd=REPO, check=True, capture_output=True, text=True).stdout
summary = f"<!-- {run} epoch_{a.epoch:03d} on the {task} {a.split} cases; {note} -->\n" + summary
(out_root / "summary.md").write_text(summary)
print(summary, flush=True)
if a.no_upload:
print("[todo] --no-upload: done", flush=True)
return
from huggingface_hub import HfApi
api = HfApi(token=os.environ.get("HF_TOKEN"))
if from_repo:
print(f"[todo] {ckpt} is already in the repo: checkpoint upload skipped", flush=True)
else:
files = [ckpt.name] + [f for f in RUN_FILES if (run_dir / f).is_file()]
api.upload_folder(repo_id=a.repo, folder_path=run_dir, path_in_repo=f"{a.path_prefix}checkpoints/{run}", allow_patterns=files,
commit_message=f"add {run} epoch_{a.epoch:03d} checkpoint")
print(f"[todo] uploaded {a.path_prefix}checkpoints/{run}/ {files}", flush=True)
api.upload_folder(repo_id=a.repo, folder_path=out_root, path_in_repo=f"{a.path_prefix}results/{run}/{out_root.name}",
commit_message=f"add {run} {a.split} results ({n} cases, epoch_{a.epoch:03d})")
print(f"[todo] uploaded {a.path_prefix}results/{run}/{out_root.name}/ -> https://huggingface.co/{a.repo}", flush=True)
if __name__ == "__main__":
main()
|