act-up-down-bad / tools /robust_eval.py
MikelRubi's picture
Tidy repo: Spanish model card, tools/ folder, dataset_features.json per checkpoint, training report
55a7ec1 verified
Raw History Blame Contribute Delete
9 kB
"""Robust held-out metrics for one ACT checkpoint from the .npz written by eval_bad.py, optionally paired with a reference.
python robust_eval.py eval_v3_010000.npz --ref eval_025000.npz --episodes 4,14,24,35,45,55,65,75,85
Every metric is computed per held-out episode and bootstrapped over episodes (95 % interval), so a difference between
two checkpoints only counts when it is larger than the episode-to-episode noise. "Executed" metrics replay the action
the robot would run when re-planning every --replan frames (run_policy.py uses 25; 24 is the nearest stride multiple).
Release metrics compare where the policy opens the gripper with where the demonstrator did: timing (s) and the five
arm joints at that moment (deg). Human spread = how far each demonstration's release pose is from the median one,
i.e. how precise the demonstrations themselves are.
Acceptance (all must hold on the evaluated episodes):
first-action error: upper 95 % bound below the copy-state first-action error (better than not moving);
executed gripper open F1 >= 0.70; improvement over copy-state >= 50 %;
release pose error <= human release spread and release timing error <= 0.5 s.
"""
from __future__ import annotations
import argparse
import json
import numpy as np
GRIPPER_OPEN, FPS = 30.0, 30
# action(t) ~= state(t) + GAIN * (state(t) - state(t-3)), least squares per joint on the 129 training episodes
VELOCITY_GAIN = np.array([1.12, 1.16, 1.25, 1.15, 1.23, 1.31])
def per_episode(data, replan: int, stride: int, blend: int = 0, blend_gripper: bool = True) -> dict[int, dict]:
out = {}
for ep in np.unique(data["episode"]):
idx = np.where(data["episode"] == ep)[0]
idx = idx[np.argsort(data["frame"][idx])]
frames, pred, truth, valid, state = (data[k][idx] for k in ("frame", "pred", "truth", "valid", "state"))
start = frames[0]
executed = []
for f in frames:
src, off = (f - start) // replan * replan // stride, (f - start) % replan
action = pred[src][off]
if blend: # hand over from the kinematic prior to the policy chunk during the first `blend` frames
velocity = state[src] - state[src - 1] if src > 0 else np.zeros_like(state[src])
prior = state[src] + VELOCITY_GAIN * velocity * (1 + off / stride)
weight = max(0.0, 1 - off / blend)
blended = weight * prior + (1 - weight) * action
if not blend_gripper:
blended[5] = action[5]
action = blended
executed.append(action)
executed = np.stack(executed)
recorded = truth[:, 0]
true_rel, pred_rel = int(np.argmax(recorded[:, 5])), int(np.argmax(executed[:, 5]))
# release success: the policy opens at least half as much as the demonstrator within +-0.5 s of its release
opening = recorded[true_rel, 5] - recorded[0, 5]
window = np.abs(frames - frames[true_rel]) <= FPS // 2
released = bool((executed[window, 5] - recorded[0, 5]).max() >= 0.5 * opening)
first = executed[[(f - start) % replan == 0 for f in frames]]
first_rec = recorded[[(f - start) % replan == 0 for f in frames]]
out[int(ep)] = {
"first_err": np.abs(pred[:, 0] - recorded).mean(1) if not blend else np.abs(first - first_rec).mean(1),
"executed_err": np.abs(executed - recorded).mean(1),
"first_copy": np.abs(state - recorded).mean(1),
"chunk_err": (np.abs(pred - truth).mean(2) * valid).sum(1), # per frame, summed over valid steps
"chunk_copy": (np.abs(state[:, None] - truth).mean(2) * valid).sum(1),
"chunk_n": valid.sum(1),
"grip_pred": executed[:, 5] > GRIPPER_OPEN, "grip_true": recorded[:, 5] > GRIPPER_OPEN,
"release_dt": abs(int(frames[pred_rel]) - int(frames[true_rel])) / FPS,
"release_pose_err": float(np.abs(executed[pred_rel, :5] - recorded[true_rel, :5]).mean()),
"released": released,
"true_release_pose": recorded[true_rel, :5],
}
return out
def metrics(eps: list[dict]) -> dict[str, float]:
cat = lambda k: np.concatenate([e[k] for e in eps]) # noqa: E731
gp, gt = cat("grip_pred"), cat("grip_true")
tp, fp, fn = (gp & gt).sum(), (gp & ~gt).sum(), (~gp & gt).sum()
chunk, copy, n = cat("chunk_err").sum(), cat("chunk_copy").sum(), cat("chunk_n").sum()
return {
"first_action_error": float(cat("first_err").mean()),
"executed_error": float(cat("executed_err").mean()),
"first_action_copy_state": float(cat("first_copy").mean()),
"improvement_vs_copy_state": float(1 - chunk / copy),
"mean_error": float(chunk / n),
"gripper_open_f1": float(2 * tp / max(1, 2 * tp + fp + fn)),
"release_timing_error_s": float(np.mean([e["release_dt"] for e in eps])),
"release_pose_error_deg": float(np.mean([e["release_pose_err"] for e in eps])),
"release_success_rate": float(np.mean([e["released"] for e in eps])),
}
def bootstrap(per_ep: dict, ref_ep: dict | None, n: int, seed: int) -> tuple[dict, dict]:
rng = np.random.default_rng(seed)
keys = sorted(per_ep)
draws, diffs = [], []
for _ in range(n):
sample = rng.choice(keys, size=len(keys), replace=True)
m = metrics([per_ep[k] for k in sample])
draws.append(m)
if ref_ep is not None:
r = metrics([ref_ep[k] for k in sample])
diffs.append({k: m[k] - r[k] for k in m})
ci = lambda rows: {k: [round(float(np.percentile([r[k] for r in rows], q)), 3) for q in (2.5, 97.5)] for k in rows[0]} # noqa: E731
return ci(draws), (ci(diffs) if diffs else {})
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("npz")
ap.add_argument("--ref", help="reference checkpoint .npz evaluated on the same frames")
ap.add_argument("--episodes", help="restrict to these held-out episodes, e.g. 4,14,24")
ap.add_argument("--replan", type=int, default=24)
ap.add_argument("--blend", type=int, default=0, help="frames to hand over from the velocity prior to the policy after each re-plan (0 = policy only)")
ap.add_argument("--ref-blend", type=int, default=0, help="--blend used for the reference")
ap.add_argument("--stride", type=int, default=3)
ap.add_argument("--bootstrap", type=int, default=2000)
ap.add_argument("--out")
args = ap.parse_args()
def load(path):
data = dict(np.load(path))
if args.episodes:
keep = np.isin(data["episode"], [int(x) for x in args.episodes.split(",")])
data = {k: v[keep] for k, v in data.items()}
return data
data = load(args.npz)
per_ep = per_episode(data, args.replan, args.stride, args.blend)
ref_ep = None
if args.ref:
ref = load(args.ref)
if not (np.array_equal(ref["episode"], data["episode"]) and np.array_equal(ref["frame"], data["frame"])):
raise SystemExit("Reference was evaluated on different frames; re-run eval_bad.py with the same episodes/stride.")
ref_ep = per_episode(ref, args.replan, args.stride, args.ref_blend)
point = {k: round(v, 3) for k, v in metrics(list(per_ep.values())).items()}
ci, diff_ci = bootstrap(per_ep, ref_ep, args.bootstrap, seed=0)
poses = np.stack([e["true_release_pose"] for e in per_ep.values()])
human_spread = float(np.abs(poses - np.median(poses, 0)).mean())
checks = {
"first_action_beats_copy_state": ci["first_action_error"][1] < point["first_action_copy_state"],
"gripper_open_f1>=0.70": point["gripper_open_f1"] >= 0.70,
"improvement>=0.50": point["improvement_vs_copy_state"] >= 0.50,
"release_pose<=human_spread": point["release_pose_error_deg"] <= human_spread,
"release_timing<=0.5s": point["release_timing_error_s"] <= 0.5,
"release_success_all_episodes": point["release_success_rate"] == 1.0,
}
report = {"npz": args.npz, "episodes": sorted(per_ep), "replan_frames": args.replan, "blend_frames": args.blend, "point": point, "ci95": ci,
"human_release_spread_deg": round(human_spread, 3), "acceptance": checks, "accepted": all(checks.values()),
"failed_releases": sorted(ep for ep, e in per_ep.items() if not e["released"])}
if ref_ep is not None:
better = {k: (diff_ci[k][1] < 0) if k not in ("improvement_vs_copy_state", "gripper_open_f1") else (diff_ci[k][0] > 0)
for k in diff_ci if k != "first_action_copy_state"}
report.update({"ref": args.ref, "diff_vs_ref_ci95": diff_ci, "significantly_better_than_ref": better})
print(json.dumps(report, indent=1))
if args.out:
open(args.out, "w", encoding="utf-8").write(json.dumps(report, indent=1))
return 0
if __name__ == "__main__":
raise SystemExit(main())