File size: 3,397 Bytes
dc9f917 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | """Refinement-count sweep on a controller checkpoint, at several execution lengths.
Exists because ``run_ablations.py:67`` hardcodes ``data/runs/controller/controller.pt``
-- the *terminal-only* controller that ``progress.md`` records as superseded. Every
K-sweep row in ``data/runs/eval/results.jsonl`` is therefore measured on a checkpoint
with a known-broken objective, and the arrival+hold controller that replaced it
(``data/runs/ah_hold0.5``, 94% at rh=1) has never been swept at all.
The question this answers: does refinement count degrade success for a controller
that already does joint whole-plan refinement over a full ``rollout_plan``? If it
does, "causal one-step commitment" cannot be the cause of the recursive planner's
T-inversion, because this architecture has no causal commitment to begin with.
Each row goes through the unmodified ``eval_controller.py``, so it shares the
harness, the wrappers and -- at a fixed seed -- the held-out start/goal pairs with
every historical row. Results land in their own directory; the historical files are
not appended to.
"""
import argparse
import subprocess
import sys
from pathlib import Path
PY = sys.executable
ROOT = Path(__file__).resolve().parents[1]
def parse_args():
p = argparse.ArgumentParser()
p.add_argument(
'--controller', default='data/runs/ah_hold0.5/controller.pt'
)
p.add_argument('--refinements', type=int, nargs='+',
default=[0, 1, 2, 3, 5, 8])
p.add_argument('--receding-horizon', type=int, nargs='+', default=[1, 5])
p.add_argument('--seeds', type=int, nargs='+', default=[42, 43, 44])
p.add_argument('--goal-offset', type=int, nargs='+', default=[25])
p.add_argument('--num-eval', type=int, default=50)
p.add_argument('--eval-budget', type=int, default=50)
p.add_argument('--out', default='data/runs/eval_refine_sweep')
return p.parse_args()
def main():
args = parse_args()
total = (
len(args.refinements) * len(args.receding_horizon)
* len(args.seeds) * len(args.goal_offset)
)
done = 0
for q in args.goal_offset:
for rh in args.receding_horizon:
for seed in args.seeds:
for k in args.refinements:
done += 1
cmd = [
PY, 'scripts/eval_controller.py',
'--controller', args.controller,
'--refinements', str(k),
'--receding-horizon', str(rh),
'--goal-offset', str(q),
'--seed', str(seed),
'--num-eval', str(args.num_eval),
'--eval-budget', str(args.eval_budget),
'--out', args.out,
]
print(
f'\n[{done}/{total}] q={q} rh={rh} seed={seed} K={k}',
flush=True,
)
# one bad row should not discard the runs queued behind it
code = subprocess.run(cmd, cwd=ROOT).returncode
if code != 0:
print(f'!! exited {code}, continuing', flush=True)
print(f'\nappended to {Path(args.out) / "results.jsonl"}')
if __name__ == '__main__':
main()
|