act-up-down-bad / tools /eval_bad.py
MikelRubi's picture
Tidy repo: Spanish model card, tools/ folder, dataset_features.json per checkpoint, training report
55a7ec1 verified
Raw History Blame Contribute Delete
6.62 kB
"""Offline held-out evaluation of an ACT checkpoint: joint error vs baselines and gripper open/close accuracy."""
import argparse
import json
from pathlib import Path
import numpy as np
import torch
from lerobot.configs.policies import PreTrainedConfig
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.policies.act.modeling_act import ACTPolicy
from lerobot.policies.factory import make_pre_post_processors
GRIPPER_OPEN = 30.0
HORIZONS = (0, 10, 25, 50, 99)
def executed_action_metrics(frames, chunk, stride):
"""Error of the action the robot would execute at each sampled frame, per inference strategy.
Sampled frames of one episode are `stride` apart, so a prediction made `k` frames earlier is available
whenever k is a multiple of `stride`. Open-loop N re-plans every N frames and executes the chunk in order;
temporal ensembling averages every earlier prediction covering this frame (LeRobot weights: oldest first,
exp(-coeff * index)). Observations are the recorded ones, so drift from the policy's own mistakes is absent.
"""
by_episode = {}
for episode, frame, pred, truth in frames:
by_episode.setdefault(episode, {})[frame] = (pred, truth)
def score(choose):
errors, gp, gt = [], [], []
for preds in by_episode.values():
start = min(preds)
for frame, (_, truth) in preds.items():
action = choose(preds, start, frame)
errors.append(np.abs(action - truth))
gp.append(action[5] > GRIPPER_OPEN); gt.append(truth[5] > GRIPPER_OPEN)
errors, gp, gt = np.array(errors), np.array(gp), np.array(gt)
tp, fp, fn = (gp & gt).sum(), (gp & ~gt).sum(), (~gp & gt).sum()
return {"mae": round(float(errors.mean()), 3), "gripper_open_f1": round(float(2 * tp / max(1, 2 * tp + fp + fn)), 3)}
def open_loop(n):
def choose(preds, start, frame):
source = start + (frame - start) // n * n
return preds[source][0][frame - source]
return choose
def ensemble(coeff):
def choose(preds, start, frame):
offsets = [k for k in reversed(range(0, chunk, stride)) if frame - k >= start and frame - k in preds]
actions = np.stack([preds[frame - k][0][k] for k in offsets]) # oldest first
weights = np.exp(-coeff * np.arange(len(actions)))
return (weights[:, None] * actions).sum(0) / weights.sum()
return choose
out = {f"open_loop_{n}": score(open_loop(n)) for n in (stride, 4 * stride, 8 * stride, 16 * stride, 33 * stride) if n <= chunk}
out.update({f"temporal_ensemble_{coeff}": score(ensemble(coeff)) for coeff in (0.01, 0.1)})
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--policy", required=True)
ap.add_argument("--dataset-root", required=True)
ap.add_argument("--episodes", required=True)
ap.add_argument("--stride", type=int, default=3)
ap.add_argument("--out", required=True)
ap.add_argument("--device", default="cuda")
args = ap.parse_args()
cfg = PreTrainedConfig.from_pretrained(args.policy)
cfg.device = args.device
policy = ACTPolicy.from_pretrained(args.policy, config=cfg).to(args.device).eval()
pre, post = make_pre_post_processors(policy_cfg=policy.config, pretrained_path=args.policy,
preprocessor_overrides={"device_processor": {"device": args.device}})
chunk = policy.config.chunk_size
root = Path(args.dataset_root).resolve()
episodes = [int(x) for x in args.episodes.split(",")]
ds = LeRobotDataset(root.name, root=str(root), episodes=episodes, video_backend="pyav",
delta_timestamps={"action": [i / 30 for i in range(chunk)]})
err = np.zeros((6, chunk)); base = np.zeros((6, chunk)); count = np.zeros(chunk)
grip_pred, grip_true = [], []
frames = [] # (episode, frame_index, predicted chunk, recorded action at this frame)
raw = {"episode": [], "frame": [], "pred": [], "truth": [], "valid": [], "state": []}
for i in range(0, len(ds), args.stride):
item = ds[i]
valid = ~item["action_is_pad"].numpy()
batch = {k: v.unsqueeze(0) for k, v in item.items() if k.startswith("observation.")}
with torch.no_grad():
pred = post(policy.predict_action_chunk(pre(batch))).squeeze(0).float().cpu().numpy()
truth, state = item["action"].numpy(), item["observation.state"].numpy()
frames.append((int(item["episode_index"]), int(item["frame_index"]), pred, truth[0]))
for key, value in zip(raw, (int(item["episode_index"]), int(item["frame_index"]), pred, truth, valid, state)):
raw[key].append(value)
err[:, valid] += np.abs(pred - truth)[valid].T
base[:, valid] += np.abs(state[None] - truth)[valid].T
count[valid] += 1
# gripper decision over the first 25 steps (what the robot executes before re-planning)
grip_pred.extend(pred[:25][valid[:25], 5] > GRIPPER_OPEN)
grip_true.extend(truth[:25][valid[:25], 5] > GRIPPER_OPEN)
gp, gt = np.array(grip_pred), np.array(grip_true)
tp, fp, fn = (gp & gt).sum(), (gp & ~gt).sum(), (~gp & gt).sum()
pol, cp = err.sum() / (6 * count.sum()), base.sum() / (6 * count.sum())
names = ["pan", "lift", "elbow", "wrist_flex", "wrist_roll", "gripper"]
rep = {
"policy": args.policy, "frames": int(count[0]),
"mae_policy": round(float(pol), 3), "mae_copy_state": round(float(cp), 3),
"improvement_vs_copy_state": round(float(1 - pol / cp), 3),
"mae_first_step": round(float(err[:, 0].mean() / count[0]), 3),
"per_joint_mae": dict(zip(names, (err.sum(1) / count.sum()).round(2).tolist())),
"mae_by_horizon": {h: round(float(err[:, h].mean() / count[h]), 2) for h in HORIZONS if h < chunk and count[h]},
"gripper_accuracy": round(float((gp == gt).mean()), 3),
"gripper_open_f1": round(float(2 * tp / max(1, 2 * tp + fp + fn)), 3),
"executed_action": executed_action_metrics(frames, chunk, args.stride),
}
print(json.dumps(rep, indent=1))
Path(args.out).write_text(json.dumps(rep, indent=1), encoding="utf-8")
# raw predictions for robust_eval.py (per-episode bootstrap, release pose, paired comparisons)
np.savez_compressed(Path(args.out).with_suffix(".npz"), **{k: np.array(v) for k, v in raw.items()})
if __name__ == "__main__":
main()