"""Offline held-out evaluation of an ACT checkpoint: joint error vs baselines and gripper open/close accuracy.""" import argparse import json from pathlib import Path import numpy as np import torch from lerobot.configs.policies import PreTrainedConfig from lerobot.datasets.lerobot_dataset import LeRobotDataset from lerobot.policies.act.modeling_act import ACTPolicy from lerobot.policies.factory import make_pre_post_processors GRIPPER_OPEN = 30.0 HORIZONS = (0, 10, 25, 50, 99) def executed_action_metrics(frames, chunk, stride): """Error of the action the robot would execute at each sampled frame, per inference strategy. Sampled frames of one episode are `stride` apart, so a prediction made `k` frames earlier is available whenever k is a multiple of `stride`. Open-loop N re-plans every N frames and executes the chunk in order; temporal ensembling averages every earlier prediction covering this frame (LeRobot weights: oldest first, exp(-coeff * index)). Observations are the recorded ones, so drift from the policy's own mistakes is absent. """ by_episode = {} for episode, frame, pred, truth in frames: by_episode.setdefault(episode, {})[frame] = (pred, truth) def score(choose): errors, gp, gt = [], [], [] for preds in by_episode.values(): start = min(preds) for frame, (_, truth) in preds.items(): action = choose(preds, start, frame) errors.append(np.abs(action - truth)) gp.append(action[5] > GRIPPER_OPEN); gt.append(truth[5] > GRIPPER_OPEN) errors, gp, gt = np.array(errors), np.array(gp), np.array(gt) tp, fp, fn = (gp & gt).sum(), (gp & ~gt).sum(), (~gp & gt).sum() return {"mae": round(float(errors.mean()), 3), "gripper_open_f1": round(float(2 * tp / max(1, 2 * tp + fp + fn)), 3)} def open_loop(n): def choose(preds, start, frame): source = start + (frame - start) // n * n return preds[source][0][frame - source] return choose def ensemble(coeff): def choose(preds, start, frame): offsets = [k for k in reversed(range(0, chunk, stride)) if frame - k >= start and frame - k in preds] actions = np.stack([preds[frame - k][0][k] for k in offsets]) # oldest first weights = np.exp(-coeff * np.arange(len(actions))) return (weights[:, None] * actions).sum(0) / weights.sum() return choose out = {f"open_loop_{n}": score(open_loop(n)) for n in (stride, 4 * stride, 8 * stride, 16 * stride, 33 * stride) if n <= chunk} out.update({f"temporal_ensemble_{coeff}": score(ensemble(coeff)) for coeff in (0.01, 0.1)}) return out def main(): ap = argparse.ArgumentParser() ap.add_argument("--policy", required=True) ap.add_argument("--dataset-root", required=True) ap.add_argument("--episodes", required=True) ap.add_argument("--stride", type=int, default=3) ap.add_argument("--out", required=True) ap.add_argument("--device", default="cuda") args = ap.parse_args() cfg = PreTrainedConfig.from_pretrained(args.policy) cfg.device = args.device policy = ACTPolicy.from_pretrained(args.policy, config=cfg).to(args.device).eval() pre, post = make_pre_post_processors(policy_cfg=policy.config, pretrained_path=args.policy, preprocessor_overrides={"device_processor": {"device": args.device}}) chunk = policy.config.chunk_size root = Path(args.dataset_root).resolve() episodes = [int(x) for x in args.episodes.split(",")] ds = LeRobotDataset(root.name, root=str(root), episodes=episodes, video_backend="pyav", delta_timestamps={"action": [i / 30 for i in range(chunk)]}) err = np.zeros((6, chunk)); base = np.zeros((6, chunk)); count = np.zeros(chunk) grip_pred, grip_true = [], [] frames = [] # (episode, frame_index, predicted chunk, recorded action at this frame) raw = {"episode": [], "frame": [], "pred": [], "truth": [], "valid": [], "state": []} for i in range(0, len(ds), args.stride): item = ds[i] valid = ~item["action_is_pad"].numpy() batch = {k: v.unsqueeze(0) for k, v in item.items() if k.startswith("observation.")} with torch.no_grad(): pred = post(policy.predict_action_chunk(pre(batch))).squeeze(0).float().cpu().numpy() truth, state = item["action"].numpy(), item["observation.state"].numpy() frames.append((int(item["episode_index"]), int(item["frame_index"]), pred, truth[0])) for key, value in zip(raw, (int(item["episode_index"]), int(item["frame_index"]), pred, truth, valid, state)): raw[key].append(value) err[:, valid] += np.abs(pred - truth)[valid].T base[:, valid] += np.abs(state[None] - truth)[valid].T count[valid] += 1 # gripper decision over the first 25 steps (what the robot executes before re-planning) grip_pred.extend(pred[:25][valid[:25], 5] > GRIPPER_OPEN) grip_true.extend(truth[:25][valid[:25], 5] > GRIPPER_OPEN) gp, gt = np.array(grip_pred), np.array(grip_true) tp, fp, fn = (gp & gt).sum(), (gp & ~gt).sum(), (~gp & gt).sum() pol, cp = err.sum() / (6 * count.sum()), base.sum() / (6 * count.sum()) names = ["pan", "lift", "elbow", "wrist_flex", "wrist_roll", "gripper"] rep = { "policy": args.policy, "frames": int(count[0]), "mae_policy": round(float(pol), 3), "mae_copy_state": round(float(cp), 3), "improvement_vs_copy_state": round(float(1 - pol / cp), 3), "mae_first_step": round(float(err[:, 0].mean() / count[0]), 3), "per_joint_mae": dict(zip(names, (err.sum(1) / count.sum()).round(2).tolist())), "mae_by_horizon": {h: round(float(err[:, h].mean() / count[h]), 2) for h in HORIZONS if h < chunk and count[h]}, "gripper_accuracy": round(float((gp == gt).mean()), 3), "gripper_open_f1": round(float(2 * tp / max(1, 2 * tp + fp + fn)), 3), "executed_action": executed_action_metrics(frames, chunk, args.stride), } print(json.dumps(rep, indent=1)) Path(args.out).write_text(json.dumps(rep, indent=1), encoding="utf-8") # raw predictions for robust_eval.py (per-episode bootstrap, release pose, paired comparisons) np.savez_compressed(Path(args.out).with_suffix(".npz"), **{k: np.array(v) for k, v in raw.items()}) if __name__ == "__main__": main()