Instructions to use MikelRubi/act-up-down-bad with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- LeRobot
How to use MikelRubi/act-up-down-bad with LeRobot:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download tools/robust_eval.py from MikelRubi/act-up-down-bad: direct link, hf CLI and curl.
- Browser
- Download file 9 kB
-
https://huggingface.co/MikelRubi/act-up-down-bad/resolve/main/tools/robust_eval.py
- Command line
-
hf download hf://MikelRubi/act-up-down-bad/tools/robust_eval.py
-
curl -L -o robust_eval.py https://huggingface.co/MikelRubi/act-up-down-bad/resolve/main/tools/robust_eval.py
9 kB
| """Robust held-out metrics for one ACT checkpoint from the .npz written by eval_bad.py, optionally paired with a reference. | |
| python robust_eval.py eval_v3_010000.npz --ref eval_025000.npz --episodes 4,14,24,35,45,55,65,75,85 | |
| Every metric is computed per held-out episode and bootstrapped over episodes (95 % interval), so a difference between | |
| two checkpoints only counts when it is larger than the episode-to-episode noise. "Executed" metrics replay the action | |
| the robot would run when re-planning every --replan frames (run_policy.py uses 25; 24 is the nearest stride multiple). | |
| Release metrics compare where the policy opens the gripper with where the demonstrator did: timing (s) and the five | |
| arm joints at that moment (deg). Human spread = how far each demonstration's release pose is from the median one, | |
| i.e. how precise the demonstrations themselves are. | |
| Acceptance (all must hold on the evaluated episodes): | |
| first-action error: upper 95 % bound below the copy-state first-action error (better than not moving); | |
| executed gripper open F1 >= 0.70; improvement over copy-state >= 50 %; | |
| release pose error <= human release spread and release timing error <= 0.5 s. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import numpy as np | |
| GRIPPER_OPEN, FPS = 30.0, 30 | |
| # action(t) ~= state(t) + GAIN * (state(t) - state(t-3)), least squares per joint on the 129 training episodes | |
| VELOCITY_GAIN = np.array([1.12, 1.16, 1.25, 1.15, 1.23, 1.31]) | |
| def per_episode(data, replan: int, stride: int, blend: int = 0, blend_gripper: bool = True) -> dict[int, dict]: | |
| out = {} | |
| for ep in np.unique(data["episode"]): | |
| idx = np.where(data["episode"] == ep)[0] | |
| idx = idx[np.argsort(data["frame"][idx])] | |
| frames, pred, truth, valid, state = (data[k][idx] for k in ("frame", "pred", "truth", "valid", "state")) | |
| start = frames[0] | |
| executed = [] | |
| for f in frames: | |
| src, off = (f - start) // replan * replan // stride, (f - start) % replan | |
| action = pred[src][off] | |
| if blend: # hand over from the kinematic prior to the policy chunk during the first `blend` frames | |
| velocity = state[src] - state[src - 1] if src > 0 else np.zeros_like(state[src]) | |
| prior = state[src] + VELOCITY_GAIN * velocity * (1 + off / stride) | |
| weight = max(0.0, 1 - off / blend) | |
| blended = weight * prior + (1 - weight) * action | |
| if not blend_gripper: | |
| blended[5] = action[5] | |
| action = blended | |
| executed.append(action) | |
| executed = np.stack(executed) | |
| recorded = truth[:, 0] | |
| true_rel, pred_rel = int(np.argmax(recorded[:, 5])), int(np.argmax(executed[:, 5])) | |
| # release success: the policy opens at least half as much as the demonstrator within +-0.5 s of its release | |
| opening = recorded[true_rel, 5] - recorded[0, 5] | |
| window = np.abs(frames - frames[true_rel]) <= FPS // 2 | |
| released = bool((executed[window, 5] - recorded[0, 5]).max() >= 0.5 * opening) | |
| first = executed[[(f - start) % replan == 0 for f in frames]] | |
| first_rec = recorded[[(f - start) % replan == 0 for f in frames]] | |
| out[int(ep)] = { | |
| "first_err": np.abs(pred[:, 0] - recorded).mean(1) if not blend else np.abs(first - first_rec).mean(1), | |
| "executed_err": np.abs(executed - recorded).mean(1), | |
| "first_copy": np.abs(state - recorded).mean(1), | |
| "chunk_err": (np.abs(pred - truth).mean(2) * valid).sum(1), # per frame, summed over valid steps | |
| "chunk_copy": (np.abs(state[:, None] - truth).mean(2) * valid).sum(1), | |
| "chunk_n": valid.sum(1), | |
| "grip_pred": executed[:, 5] > GRIPPER_OPEN, "grip_true": recorded[:, 5] > GRIPPER_OPEN, | |
| "release_dt": abs(int(frames[pred_rel]) - int(frames[true_rel])) / FPS, | |
| "release_pose_err": float(np.abs(executed[pred_rel, :5] - recorded[true_rel, :5]).mean()), | |
| "released": released, | |
| "true_release_pose": recorded[true_rel, :5], | |
| } | |
| return out | |
| def metrics(eps: list[dict]) -> dict[str, float]: | |
| cat = lambda k: np.concatenate([e[k] for e in eps]) # noqa: E731 | |
| gp, gt = cat("grip_pred"), cat("grip_true") | |
| tp, fp, fn = (gp & gt).sum(), (gp & ~gt).sum(), (~gp & gt).sum() | |
| chunk, copy, n = cat("chunk_err").sum(), cat("chunk_copy").sum(), cat("chunk_n").sum() | |
| return { | |
| "first_action_error": float(cat("first_err").mean()), | |
| "executed_error": float(cat("executed_err").mean()), | |
| "first_action_copy_state": float(cat("first_copy").mean()), | |
| "improvement_vs_copy_state": float(1 - chunk / copy), | |
| "mean_error": float(chunk / n), | |
| "gripper_open_f1": float(2 * tp / max(1, 2 * tp + fp + fn)), | |
| "release_timing_error_s": float(np.mean([e["release_dt"] for e in eps])), | |
| "release_pose_error_deg": float(np.mean([e["release_pose_err"] for e in eps])), | |
| "release_success_rate": float(np.mean([e["released"] for e in eps])), | |
| } | |
| def bootstrap(per_ep: dict, ref_ep: dict | None, n: int, seed: int) -> tuple[dict, dict]: | |
| rng = np.random.default_rng(seed) | |
| keys = sorted(per_ep) | |
| draws, diffs = [], [] | |
| for _ in range(n): | |
| sample = rng.choice(keys, size=len(keys), replace=True) | |
| m = metrics([per_ep[k] for k in sample]) | |
| draws.append(m) | |
| if ref_ep is not None: | |
| r = metrics([ref_ep[k] for k in sample]) | |
| diffs.append({k: m[k] - r[k] for k in m}) | |
| ci = lambda rows: {k: [round(float(np.percentile([r[k] for r in rows], q)), 3) for q in (2.5, 97.5)] for k in rows[0]} # noqa: E731 | |
| return ci(draws), (ci(diffs) if diffs else {}) | |
| def main() -> int: | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("npz") | |
| ap.add_argument("--ref", help="reference checkpoint .npz evaluated on the same frames") | |
| ap.add_argument("--episodes", help="restrict to these held-out episodes, e.g. 4,14,24") | |
| ap.add_argument("--replan", type=int, default=24) | |
| ap.add_argument("--blend", type=int, default=0, help="frames to hand over from the velocity prior to the policy after each re-plan (0 = policy only)") | |
| ap.add_argument("--ref-blend", type=int, default=0, help="--blend used for the reference") | |
| ap.add_argument("--stride", type=int, default=3) | |
| ap.add_argument("--bootstrap", type=int, default=2000) | |
| ap.add_argument("--out") | |
| args = ap.parse_args() | |
| def load(path): | |
| data = dict(np.load(path)) | |
| if args.episodes: | |
| keep = np.isin(data["episode"], [int(x) for x in args.episodes.split(",")]) | |
| data = {k: v[keep] for k, v in data.items()} | |
| return data | |
| data = load(args.npz) | |
| per_ep = per_episode(data, args.replan, args.stride, args.blend) | |
| ref_ep = None | |
| if args.ref: | |
| ref = load(args.ref) | |
| if not (np.array_equal(ref["episode"], data["episode"]) and np.array_equal(ref["frame"], data["frame"])): | |
| raise SystemExit("Reference was evaluated on different frames; re-run eval_bad.py with the same episodes/stride.") | |
| ref_ep = per_episode(ref, args.replan, args.stride, args.ref_blend) | |
| point = {k: round(v, 3) for k, v in metrics(list(per_ep.values())).items()} | |
| ci, diff_ci = bootstrap(per_ep, ref_ep, args.bootstrap, seed=0) | |
| poses = np.stack([e["true_release_pose"] for e in per_ep.values()]) | |
| human_spread = float(np.abs(poses - np.median(poses, 0)).mean()) | |
| checks = { | |
| "first_action_beats_copy_state": ci["first_action_error"][1] < point["first_action_copy_state"], | |
| "gripper_open_f1>=0.70": point["gripper_open_f1"] >= 0.70, | |
| "improvement>=0.50": point["improvement_vs_copy_state"] >= 0.50, | |
| "release_pose<=human_spread": point["release_pose_error_deg"] <= human_spread, | |
| "release_timing<=0.5s": point["release_timing_error_s"] <= 0.5, | |
| "release_success_all_episodes": point["release_success_rate"] == 1.0, | |
| } | |
| report = {"npz": args.npz, "episodes": sorted(per_ep), "replan_frames": args.replan, "blend_frames": args.blend, "point": point, "ci95": ci, | |
| "human_release_spread_deg": round(human_spread, 3), "acceptance": checks, "accepted": all(checks.values()), | |
| "failed_releases": sorted(ep for ep, e in per_ep.items() if not e["released"])} | |
| if ref_ep is not None: | |
| better = {k: (diff_ci[k][1] < 0) if k not in ("improvement_vs_copy_state", "gripper_open_f1") else (diff_ci[k][0] > 0) | |
| for k in diff_ci if k != "first_action_copy_state"} | |
| report.update({"ref": args.ref, "diff_vs_ref_ci95": diff_ci, "significantly_better_than_ref": better}) | |
| print(json.dumps(report, indent=1)) | |
| if args.out: | |
| open(args.out, "w", encoding="utf-8").write(json.dumps(report, indent=1)) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |