"""Robust held-out metrics for one ACT checkpoint from the .npz written by eval_bad.py, optionally paired with a reference. python robust_eval.py eval_v3_010000.npz --ref eval_025000.npz --episodes 4,14,24,35,45,55,65,75,85 Every metric is computed per held-out episode and bootstrapped over episodes (95 % interval), so a difference between two checkpoints only counts when it is larger than the episode-to-episode noise. "Executed" metrics replay the action the robot would run when re-planning every --replan frames (run_policy.py uses 25; 24 is the nearest stride multiple). Release metrics compare where the policy opens the gripper with where the demonstrator did: timing (s) and the five arm joints at that moment (deg). Human spread = how far each demonstration's release pose is from the median one, i.e. how precise the demonstrations themselves are. Acceptance (all must hold on the evaluated episodes): first-action error: upper 95 % bound below the copy-state first-action error (better than not moving); executed gripper open F1 >= 0.70; improvement over copy-state >= 50 %; release pose error <= human release spread and release timing error <= 0.5 s. """ from __future__ import annotations import argparse import json import numpy as np GRIPPER_OPEN, FPS = 30.0, 30 # action(t) ~= state(t) + GAIN * (state(t) - state(t-3)), least squares per joint on the 129 training episodes VELOCITY_GAIN = np.array([1.12, 1.16, 1.25, 1.15, 1.23, 1.31]) def per_episode(data, replan: int, stride: int, blend: int = 0, blend_gripper: bool = True) -> dict[int, dict]: out = {} for ep in np.unique(data["episode"]): idx = np.where(data["episode"] == ep)[0] idx = idx[np.argsort(data["frame"][idx])] frames, pred, truth, valid, state = (data[k][idx] for k in ("frame", "pred", "truth", "valid", "state")) start = frames[0] executed = [] for f in frames: src, off = (f - start) // replan * replan // stride, (f - start) % replan action = pred[src][off] if blend: # hand over from the kinematic prior to the policy chunk during the first `blend` frames velocity = state[src] - state[src - 1] if src > 0 else np.zeros_like(state[src]) prior = state[src] + VELOCITY_GAIN * velocity * (1 + off / stride) weight = max(0.0, 1 - off / blend) blended = weight * prior + (1 - weight) * action if not blend_gripper: blended[5] = action[5] action = blended executed.append(action) executed = np.stack(executed) recorded = truth[:, 0] true_rel, pred_rel = int(np.argmax(recorded[:, 5])), int(np.argmax(executed[:, 5])) # release success: the policy opens at least half as much as the demonstrator within +-0.5 s of its release opening = recorded[true_rel, 5] - recorded[0, 5] window = np.abs(frames - frames[true_rel]) <= FPS // 2 released = bool((executed[window, 5] - recorded[0, 5]).max() >= 0.5 * opening) first = executed[[(f - start) % replan == 0 for f in frames]] first_rec = recorded[[(f - start) % replan == 0 for f in frames]] out[int(ep)] = { "first_err": np.abs(pred[:, 0] - recorded).mean(1) if not blend else np.abs(first - first_rec).mean(1), "executed_err": np.abs(executed - recorded).mean(1), "first_copy": np.abs(state - recorded).mean(1), "chunk_err": (np.abs(pred - truth).mean(2) * valid).sum(1), # per frame, summed over valid steps "chunk_copy": (np.abs(state[:, None] - truth).mean(2) * valid).sum(1), "chunk_n": valid.sum(1), "grip_pred": executed[:, 5] > GRIPPER_OPEN, "grip_true": recorded[:, 5] > GRIPPER_OPEN, "release_dt": abs(int(frames[pred_rel]) - int(frames[true_rel])) / FPS, "release_pose_err": float(np.abs(executed[pred_rel, :5] - recorded[true_rel, :5]).mean()), "released": released, "true_release_pose": recorded[true_rel, :5], } return out def metrics(eps: list[dict]) -> dict[str, float]: cat = lambda k: np.concatenate([e[k] for e in eps]) # noqa: E731 gp, gt = cat("grip_pred"), cat("grip_true") tp, fp, fn = (gp & gt).sum(), (gp & ~gt).sum(), (~gp & gt).sum() chunk, copy, n = cat("chunk_err").sum(), cat("chunk_copy").sum(), cat("chunk_n").sum() return { "first_action_error": float(cat("first_err").mean()), "executed_error": float(cat("executed_err").mean()), "first_action_copy_state": float(cat("first_copy").mean()), "improvement_vs_copy_state": float(1 - chunk / copy), "mean_error": float(chunk / n), "gripper_open_f1": float(2 * tp / max(1, 2 * tp + fp + fn)), "release_timing_error_s": float(np.mean([e["release_dt"] for e in eps])), "release_pose_error_deg": float(np.mean([e["release_pose_err"] for e in eps])), "release_success_rate": float(np.mean([e["released"] for e in eps])), } def bootstrap(per_ep: dict, ref_ep: dict | None, n: int, seed: int) -> tuple[dict, dict]: rng = np.random.default_rng(seed) keys = sorted(per_ep) draws, diffs = [], [] for _ in range(n): sample = rng.choice(keys, size=len(keys), replace=True) m = metrics([per_ep[k] for k in sample]) draws.append(m) if ref_ep is not None: r = metrics([ref_ep[k] for k in sample]) diffs.append({k: m[k] - r[k] for k in m}) ci = lambda rows: {k: [round(float(np.percentile([r[k] for r in rows], q)), 3) for q in (2.5, 97.5)] for k in rows[0]} # noqa: E731 return ci(draws), (ci(diffs) if diffs else {}) def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("npz") ap.add_argument("--ref", help="reference checkpoint .npz evaluated on the same frames") ap.add_argument("--episodes", help="restrict to these held-out episodes, e.g. 4,14,24") ap.add_argument("--replan", type=int, default=24) ap.add_argument("--blend", type=int, default=0, help="frames to hand over from the velocity prior to the policy after each re-plan (0 = policy only)") ap.add_argument("--ref-blend", type=int, default=0, help="--blend used for the reference") ap.add_argument("--stride", type=int, default=3) ap.add_argument("--bootstrap", type=int, default=2000) ap.add_argument("--out") args = ap.parse_args() def load(path): data = dict(np.load(path)) if args.episodes: keep = np.isin(data["episode"], [int(x) for x in args.episodes.split(",")]) data = {k: v[keep] for k, v in data.items()} return data data = load(args.npz) per_ep = per_episode(data, args.replan, args.stride, args.blend) ref_ep = None if args.ref: ref = load(args.ref) if not (np.array_equal(ref["episode"], data["episode"]) and np.array_equal(ref["frame"], data["frame"])): raise SystemExit("Reference was evaluated on different frames; re-run eval_bad.py with the same episodes/stride.") ref_ep = per_episode(ref, args.replan, args.stride, args.ref_blend) point = {k: round(v, 3) for k, v in metrics(list(per_ep.values())).items()} ci, diff_ci = bootstrap(per_ep, ref_ep, args.bootstrap, seed=0) poses = np.stack([e["true_release_pose"] for e in per_ep.values()]) human_spread = float(np.abs(poses - np.median(poses, 0)).mean()) checks = { "first_action_beats_copy_state": ci["first_action_error"][1] < point["first_action_copy_state"], "gripper_open_f1>=0.70": point["gripper_open_f1"] >= 0.70, "improvement>=0.50": point["improvement_vs_copy_state"] >= 0.50, "release_pose<=human_spread": point["release_pose_error_deg"] <= human_spread, "release_timing<=0.5s": point["release_timing_error_s"] <= 0.5, "release_success_all_episodes": point["release_success_rate"] == 1.0, } report = {"npz": args.npz, "episodes": sorted(per_ep), "replan_frames": args.replan, "blend_frames": args.blend, "point": point, "ci95": ci, "human_release_spread_deg": round(human_spread, 3), "acceptance": checks, "accepted": all(checks.values()), "failed_releases": sorted(ep for ep, e in per_ep.items() if not e["released"])} if ref_ep is not None: better = {k: (diff_ci[k][1] < 0) if k not in ("improvement_vs_copy_state", "gripper_open_f1") else (diff_ci[k][0] > 0) for k in diff_ci if k != "first_action_copy_state"} report.update({"ref": args.ref, "diff_vs_ref_ci95": diff_ci, "significantly_better_than_ref": better}) print(json.dumps(report, indent=1)) if args.out: open(args.out, "w", encoding="utf-8").write(json.dumps(report, indent=1)) return 0 if __name__ == "__main__": raise SystemExit(main())