"""Refinement-count sweep on a controller checkpoint, at several execution lengths. Exists because ``run_ablations.py:67`` hardcodes ``data/runs/controller/controller.pt`` -- the *terminal-only* controller that ``progress.md`` records as superseded. Every K-sweep row in ``data/runs/eval/results.jsonl`` is therefore measured on a checkpoint with a known-broken objective, and the arrival+hold controller that replaced it (``data/runs/ah_hold0.5``, 94% at rh=1) has never been swept at all. The question this answers: does refinement count degrade success for a controller that already does joint whole-plan refinement over a full ``rollout_plan``? If it does, "causal one-step commitment" cannot be the cause of the recursive planner's T-inversion, because this architecture has no causal commitment to begin with. Each row goes through the unmodified ``eval_controller.py``, so it shares the harness, the wrappers and -- at a fixed seed -- the held-out start/goal pairs with every historical row. Results land in their own directory; the historical files are not appended to. """ import argparse import subprocess import sys from pathlib import Path PY = sys.executable ROOT = Path(__file__).resolve().parents[1] def parse_args(): p = argparse.ArgumentParser() p.add_argument( '--controller', default='data/runs/ah_hold0.5/controller.pt' ) p.add_argument('--refinements', type=int, nargs='+', default=[0, 1, 2, 3, 5, 8]) p.add_argument('--receding-horizon', type=int, nargs='+', default=[1, 5]) p.add_argument('--seeds', type=int, nargs='+', default=[42, 43, 44]) p.add_argument('--goal-offset', type=int, nargs='+', default=[25]) p.add_argument('--num-eval', type=int, default=50) p.add_argument('--eval-budget', type=int, default=50) p.add_argument('--out', default='data/runs/eval_refine_sweep') return p.parse_args() def main(): args = parse_args() total = ( len(args.refinements) * len(args.receding_horizon) * len(args.seeds) * len(args.goal_offset) ) done = 0 for q in args.goal_offset: for rh in args.receding_horizon: for seed in args.seeds: for k in args.refinements: done += 1 cmd = [ PY, 'scripts/eval_controller.py', '--controller', args.controller, '--refinements', str(k), '--receding-horizon', str(rh), '--goal-offset', str(q), '--seed', str(seed), '--num-eval', str(args.num_eval), '--eval-budget', str(args.eval_budget), '--out', args.out, ] print( f'\n[{done}/{total}] q={q} rh={rh} seed={seed} K={k}', flush=True, ) # one bad row should not discard the runs queued behind it code = subprocess.run(cmd, cwd=ROOT).returncode if code != 0: print(f'!! exited {code}, continuing', flush=True) print(f'\nappended to {Path(args.out) / "results.jsonl"}') if __name__ == '__main__': main()