File size: 3,856 Bytes
dc9f917 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 | """Head-to-head comparison of every planner through one harness.
The cross-architecture numbers already in the repo are not comparable to each
other. ``scripts/eval_controller.py`` runs the plain ``WorldModelPolicy``,
which -- per ``eval_planner.py``'s header -- plans from one repeated frame and
zero past action blocks, because ``stable_worldmodel`` declares ``history_len``
and ``action_history`` and implements neither. ``eval_planner.py`` supplies both
via ``HistoryPolicy``. A controller scored by the first and a planner scored by
the second differ in their observations, not just their architecture.
So every row here goes through ``eval_planner.py``, which can evaluate the
controller, the recursive planner, the cross-attention planner, CEM and the
random floor, with history on for all of them and one shared seed -- meaning
identical held-out start/goal pairs and paired McNemar throughout.
"""
import argparse
import subprocess
import sys
from pathlib import Path
PY = sys.executable
ROOT = Path(__file__).resolve().parents[1]
EVAL = 'lejepa_control_2/scripts/eval_planner.py'
def parse_args():
p = argparse.ArgumentParser()
p.add_argument('--controller', default='data/runs/ah_hold0.5/controller.pt')
p.add_argument('--xa', default='data/runs/planner_xa_D/planner.pt')
p.add_argument('--recursive', default='data/runs/planner_D10_extended/planner.pt')
p.add_argument('--horizon', type=int, default=5)
p.add_argument('--receding-horizon', type=int, nargs='+', default=[1, 5])
p.add_argument('--controller-k', type=int, nargs='+', default=[3, 5])
p.add_argument('--xa-k', type=int, nargs='+', default=[2, 3, 5])
p.add_argument('--seeds', type=int, nargs='+', default=[42, 43, 44])
p.add_argument('--num-eval', type=int, default=50)
p.add_argument('--goal-offset', type=int, default=25)
p.add_argument('--out', default='data/runs/eval_compare')
p.add_argument('--skip-random', action='store_true')
return p.parse_args()
def main():
args = parse_args()
jobs = []
for rh in args.receding_horizon:
for seed in args.seeds:
common = [
'--horizon', str(args.horizon),
'--receding-horizon', str(rh),
'--goal-offset', str(args.goal_offset),
'--num-eval', str(args.num_eval),
'--seed', str(seed),
'--out', args.out,
]
if not args.skip_random and rh == args.receding_horizon[0]:
jobs.append(['--planner', 'random', *common])
for k in args.controller_k:
jobs.append([
'--planner', 'controller',
'--controller', args.controller,
'--refinements', str(k), '--tag', f'ctrl_K{k}', *common,
])
if Path(ROOT / args.recursive).exists():
jobs.append([
'--planner', 'planner', '--checkpoint', args.recursive,
'--tag', 'recursive', *common,
])
if Path(ROOT / args.xa).exists():
for k in args.xa_k:
jobs.append([
'--planner', 'planner', '--checkpoint', args.xa,
'--cycles', str(k), '--tag', f'xa_K{k}', *common,
])
print(f'{len(jobs)} evaluations -> {args.out}')
for i, extra in enumerate(jobs, 1):
print(f'\n[{i}/{len(jobs)}] {" ".join(extra)}', flush=True)
code = subprocess.run([PY, EVAL, *extra], cwd=ROOT).returncode
if code != 0:
print(f'!! exited {code}, continuing', flush=True)
print(f'\nappended to {Path(args.out) / "results.jsonl"}')
if __name__ == '__main__':
main()
|