Instructions to use MikelRubi/act-up-down-bad with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- LeRobot
How to use MikelRubi/act-up-down-bad with LeRobot:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download tools/eval_bad.py from MikelRubi/act-up-down-bad: direct link, hf CLI and curl.
- Browser
- Download file 6.62 kB
-
https://huggingface.co/MikelRubi/act-up-down-bad/resolve/main/tools/eval_bad.py
- Command line
-
hf download hf://MikelRubi/act-up-down-bad/tools/eval_bad.py
-
curl -L -o eval_bad.py https://huggingface.co/MikelRubi/act-up-down-bad/resolve/main/tools/eval_bad.py
6.62 kB
| """Offline held-out evaluation of an ACT checkpoint: joint error vs baselines and gripper open/close accuracy.""" | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| from lerobot.configs.policies import PreTrainedConfig | |
| from lerobot.datasets.lerobot_dataset import LeRobotDataset | |
| from lerobot.policies.act.modeling_act import ACTPolicy | |
| from lerobot.policies.factory import make_pre_post_processors | |
| GRIPPER_OPEN = 30.0 | |
| HORIZONS = (0, 10, 25, 50, 99) | |
| def executed_action_metrics(frames, chunk, stride): | |
| """Error of the action the robot would execute at each sampled frame, per inference strategy. | |
| Sampled frames of one episode are `stride` apart, so a prediction made `k` frames earlier is available | |
| whenever k is a multiple of `stride`. Open-loop N re-plans every N frames and executes the chunk in order; | |
| temporal ensembling averages every earlier prediction covering this frame (LeRobot weights: oldest first, | |
| exp(-coeff * index)). Observations are the recorded ones, so drift from the policy's own mistakes is absent. | |
| """ | |
| by_episode = {} | |
| for episode, frame, pred, truth in frames: | |
| by_episode.setdefault(episode, {})[frame] = (pred, truth) | |
| def score(choose): | |
| errors, gp, gt = [], [], [] | |
| for preds in by_episode.values(): | |
| start = min(preds) | |
| for frame, (_, truth) in preds.items(): | |
| action = choose(preds, start, frame) | |
| errors.append(np.abs(action - truth)) | |
| gp.append(action[5] > GRIPPER_OPEN); gt.append(truth[5] > GRIPPER_OPEN) | |
| errors, gp, gt = np.array(errors), np.array(gp), np.array(gt) | |
| tp, fp, fn = (gp & gt).sum(), (gp & ~gt).sum(), (~gp & gt).sum() | |
| return {"mae": round(float(errors.mean()), 3), "gripper_open_f1": round(float(2 * tp / max(1, 2 * tp + fp + fn)), 3)} | |
| def open_loop(n): | |
| def choose(preds, start, frame): | |
| source = start + (frame - start) // n * n | |
| return preds[source][0][frame - source] | |
| return choose | |
| def ensemble(coeff): | |
| def choose(preds, start, frame): | |
| offsets = [k for k in reversed(range(0, chunk, stride)) if frame - k >= start and frame - k in preds] | |
| actions = np.stack([preds[frame - k][0][k] for k in offsets]) # oldest first | |
| weights = np.exp(-coeff * np.arange(len(actions))) | |
| return (weights[:, None] * actions).sum(0) / weights.sum() | |
| return choose | |
| out = {f"open_loop_{n}": score(open_loop(n)) for n in (stride, 4 * stride, 8 * stride, 16 * stride, 33 * stride) if n <= chunk} | |
| out.update({f"temporal_ensemble_{coeff}": score(ensemble(coeff)) for coeff in (0.01, 0.1)}) | |
| return out | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--policy", required=True) | |
| ap.add_argument("--dataset-root", required=True) | |
| ap.add_argument("--episodes", required=True) | |
| ap.add_argument("--stride", type=int, default=3) | |
| ap.add_argument("--out", required=True) | |
| ap.add_argument("--device", default="cuda") | |
| args = ap.parse_args() | |
| cfg = PreTrainedConfig.from_pretrained(args.policy) | |
| cfg.device = args.device | |
| policy = ACTPolicy.from_pretrained(args.policy, config=cfg).to(args.device).eval() | |
| pre, post = make_pre_post_processors(policy_cfg=policy.config, pretrained_path=args.policy, | |
| preprocessor_overrides={"device_processor": {"device": args.device}}) | |
| chunk = policy.config.chunk_size | |
| root = Path(args.dataset_root).resolve() | |
| episodes = [int(x) for x in args.episodes.split(",")] | |
| ds = LeRobotDataset(root.name, root=str(root), episodes=episodes, video_backend="pyav", | |
| delta_timestamps={"action": [i / 30 for i in range(chunk)]}) | |
| err = np.zeros((6, chunk)); base = np.zeros((6, chunk)); count = np.zeros(chunk) | |
| grip_pred, grip_true = [], [] | |
| frames = [] # (episode, frame_index, predicted chunk, recorded action at this frame) | |
| raw = {"episode": [], "frame": [], "pred": [], "truth": [], "valid": [], "state": []} | |
| for i in range(0, len(ds), args.stride): | |
| item = ds[i] | |
| valid = ~item["action_is_pad"].numpy() | |
| batch = {k: v.unsqueeze(0) for k, v in item.items() if k.startswith("observation.")} | |
| with torch.no_grad(): | |
| pred = post(policy.predict_action_chunk(pre(batch))).squeeze(0).float().cpu().numpy() | |
| truth, state = item["action"].numpy(), item["observation.state"].numpy() | |
| frames.append((int(item["episode_index"]), int(item["frame_index"]), pred, truth[0])) | |
| for key, value in zip(raw, (int(item["episode_index"]), int(item["frame_index"]), pred, truth, valid, state)): | |
| raw[key].append(value) | |
| err[:, valid] += np.abs(pred - truth)[valid].T | |
| base[:, valid] += np.abs(state[None] - truth)[valid].T | |
| count[valid] += 1 | |
| # gripper decision over the first 25 steps (what the robot executes before re-planning) | |
| grip_pred.extend(pred[:25][valid[:25], 5] > GRIPPER_OPEN) | |
| grip_true.extend(truth[:25][valid[:25], 5] > GRIPPER_OPEN) | |
| gp, gt = np.array(grip_pred), np.array(grip_true) | |
| tp, fp, fn = (gp & gt).sum(), (gp & ~gt).sum(), (~gp & gt).sum() | |
| pol, cp = err.sum() / (6 * count.sum()), base.sum() / (6 * count.sum()) | |
| names = ["pan", "lift", "elbow", "wrist_flex", "wrist_roll", "gripper"] | |
| rep = { | |
| "policy": args.policy, "frames": int(count[0]), | |
| "mae_policy": round(float(pol), 3), "mae_copy_state": round(float(cp), 3), | |
| "improvement_vs_copy_state": round(float(1 - pol / cp), 3), | |
| "mae_first_step": round(float(err[:, 0].mean() / count[0]), 3), | |
| "per_joint_mae": dict(zip(names, (err.sum(1) / count.sum()).round(2).tolist())), | |
| "mae_by_horizon": {h: round(float(err[:, h].mean() / count[h]), 2) for h in HORIZONS if h < chunk and count[h]}, | |
| "gripper_accuracy": round(float((gp == gt).mean()), 3), | |
| "gripper_open_f1": round(float(2 * tp / max(1, 2 * tp + fp + fn)), 3), | |
| "executed_action": executed_action_metrics(frames, chunk, args.stride), | |
| } | |
| print(json.dumps(rep, indent=1)) | |
| Path(args.out).write_text(json.dumps(rep, indent=1), encoding="utf-8") | |
| # raw predictions for robust_eval.py (per-episode bootstrap, release pose, paired comparisons) | |
| np.savez_compressed(Path(args.out).with_suffix(".npz"), **{k: np.array(v) for k, v in raw.items()}) | |
| if __name__ == "__main__": | |
| main() | |