Instructions to use WitneyWW/acts-sensoronly-ablation with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Cosmos
How to use WitneyWW/acts-sensoronly-ablation with Cosmos:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download eval/summarize.py from WitneyWW/acts-sensoronly-ablation: direct link, hf CLI and curl.
- Browser
- Download file 4.63 kB
-
https://huggingface.co/WitneyWW/acts-sensoronly-ablation/resolve/main/eval/summarize.py
- Command line
-
hf download hf://WitneyWW/acts-sensoronly-ablation/eval/summarize.py
-
curl -L -o summarize.py https://huggingface.co/WitneyWW/acts-sensoronly-ablation/resolve/main/eval/summarize.py
4.63 kB
| """Aggregate the per-case results of eval/run_test_case.py into a markdown table, optionally | |
| paired against the reference numbers shipped in eval/cases/<task>_reference_metrics.json. | |
| python eval/summarize.py --task rope [--root DIR] [--reference] | |
| python eval/summarize.py --task rope --split val --root runs/<run>/val_epoch_014 \ | |
| --ref-root runs/nano_action_causal_rope_v1/val_epoch_014 # paired vs another run on the same val cases | |
| short = 8 observed -> 8 predicted frames (metrics on the 8 predicted frames); | |
| long = mean over the 15 autoregressive windows (120 generated frames). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| from pathlib import Path | |
| REPO = Path(__file__).resolve().parents[1] | |
| STREAMS = ["video", "wrist_left", "wrist_right", "tactile_left", "tactile_right"] | |
| METRICS = ["psnr", "ssim", "lpips"] | |
| def load(root: Path, man: dict) -> dict: | |
| out = {} | |
| for c in man["cases"]: | |
| d = root / c["output_subdir"] | |
| if not (d / "complete.json").is_file(): | |
| continue | |
| w = json.loads((d / "window_eval.json").read_text())["results"][0] | |
| r = {"category": c["category"], "short": {s: {m: w["image"][s][m] for m in METRICS} for s in STREAMS}} | |
| rr = d / "reference_rollout.json" | |
| if c["rollout_windows"] and rr.is_file(): | |
| wins = json.loads(rr.read_text())[0]["windows"] | |
| r["long_mean"] = {s: {m: sum(x["image"][s][m] for x in wins) / len(wins) for m in METRICS} for s in STREAMS} | |
| out[c["output_subdir"]] = r | |
| return out | |
| def mean(xs): | |
| return sum(xs) / len(xs) if xs else float("nan") | |
| def table(res: dict, hz: str, title: str) -> list[str]: | |
| rows = [r for r in res.values() if hz in r] | |
| lines = [f"### {title} (n={len(rows)})", "", "| stream | PSNR ↑ | SSIM ↑ | LPIPS ↓ |", "|---|---:|---:|---:|"] | |
| for s in STREAMS: | |
| v = [mean([r[hz][s][m] for r in rows]) for m in METRICS] | |
| lines.append(f"| {s} | {v[0]:.2f} | {v[1]:.4f} | {v[2]:.4f} |") | |
| return lines + [""] | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| tasks = sorted(f.name[:-len("_test_cases.json")] for f in (REPO / "eval" / "cases").glob("*_test_cases.json")) | |
| ap.add_argument("--task", required=True, choices=tasks) | |
| ap.add_argument("--root", default=None, help="default: $ACTS_RUNS_ROOT/<run_name>/test_<ckpt stem of the manifest>") | |
| ap.add_argument("--split", default="test", choices=["test", "val"]) | |
| ap.add_argument("--reference", action="store_true", | |
| help="also print paired deltas vs the shipped reference metrics (test split only)") | |
| ap.add_argument("--ref-root", default=None, | |
| help="also print paired deltas vs another result dir on the same cases (e.g. the parent run's " | |
| "val_epoch_014); use this for the val split") | |
| a = ap.parse_args() | |
| man = json.loads((REPO / "eval" / "cases" / f"{a.task}_{a.split}_cases.json").read_text()) | |
| runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs")) | |
| root = Path(a.root or runs / man["run_name"] / f"{a.split}_{Path(man['checkpoint']).stem}") | |
| res = load(root, man) | |
| print(f"# {root.parent.name} {root.name} ({len(res)}/{len(man['cases'])} cases complete) -- {root}\n") | |
| print("\n".join(table(res, "short", "short horizon: 8 -> 8 frames"))) | |
| print("\n".join(table(res, "long_mean", "long horizon: mean over 15 chained windows"))) | |
| refs = [] | |
| if a.reference: | |
| if a.split != "test": | |
| raise SystemExit("--reference is the shipped TEST reference; for val use --ref-root") | |
| ref_file = REPO / "eval" / "cases" / f"{a.task}_reference_metrics.json" | |
| if not ref_file.is_file(): | |
| raise SystemExit(f"no reference metrics for {a.task}: {ref_file}") | |
| refs.append(("reference", {r["output_subdir"]: r for r in json.loads(ref_file.read_text())["cases"]})) | |
| if a.ref_root: | |
| rr = Path(a.ref_root) | |
| refs.append((f"{rr.parent.name}/{rr.name}", load(rr, man))) | |
| for name, ref in refs: | |
| print(f"### paired difference vs {name} (this run − {name}), PSNR dB, n = {len(ref)} reference cases\n") | |
| print("| horizon | stream | n | mean Δ | max |Δ| |\n|---|---|---:|---:|---:|") | |
| for hz in ("short", "long_mean"): | |
| for s in STREAMS: | |
| d = [res[k][hz][s]["psnr"] - ref[k][hz][s]["psnr"] for k in res if hz in res[k] and hz in ref.get(k, {})] | |
| if d: | |
| print(f"| {hz} | {s} | {len(d)} | {mean(d):+.3f} | {max(abs(x) for x in d):.3f} |") | |
| print() | |
| if __name__ == "__main__": | |
| main() | |