Push scripts
Browse files- code/scripts/__pycache__/train_decoder.cpython-312.pyc +0 -0
- code/scripts/collect_pusht.py +65 -0
- code/scripts/compare_planners.py +92 -0
- code/scripts/diagnostics.py +223 -0
- code/scripts/encode_latents.py +127 -0
- code/scripts/encode_latents_reacher.py +150 -0
- code/scripts/eval_controller.py +256 -0
- code/scripts/eval_controller_reacher.py +282 -0
- code/scripts/paired_stats.py +127 -0
- code/scripts/probe_alignment.py +139 -0
- code/scripts/probe_attention.py +212 -0
- code/scripts/probe_contract.py +118 -0
- code/scripts/probe_geometry.py +154 -0
- code/scripts/run_ablations.py +98 -0
- code/scripts/smoke_wm.py +60 -0
- code/scripts/summarize_refine_sweep.py +99 -0
- code/scripts/summarize_results.py +129 -0
- code/scripts/sweep_refinements.py +82 -0
- code/scripts/test_controller.py +261 -0
- code/scripts/test_data.py +138 -0
- code/scripts/test_history_policy.py +287 -0
- code/scripts/train_controller.py +329 -0
- code/scripts/train_decoder.py +196 -0
- code/scripts/train_density.py +118 -0
code/scripts/__pycache__/train_decoder.cpython-312.pyc
ADDED
|
Binary file (13.4 kB). View file
|
|
|
code/scripts/collect_pusht.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Collect a PushT dataset with stable_worldmodel's WeakPolicy.
|
| 2 |
+
|
| 3 |
+
Writes shards under $STABLEWM_HOME/datasets/<name>/ in lance format.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import os
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def main():
|
| 14 |
+
p = argparse.ArgumentParser()
|
| 15 |
+
p.add_argument('--name', default='pusht_weak_100')
|
| 16 |
+
p.add_argument('--episodes', type=int, default=2000)
|
| 17 |
+
p.add_argument('--shards', type=int, default=10)
|
| 18 |
+
p.add_argument('--num-envs', type=int, default=10)
|
| 19 |
+
p.add_argument('--max-episode-steps', type=int, default=100)
|
| 20 |
+
p.add_argument('--dist-constraint', type=int, default=100)
|
| 21 |
+
p.add_argument('--seed', type=int, default=3072)
|
| 22 |
+
p.add_argument('--cache-dir', default=None)
|
| 23 |
+
args = p.parse_args()
|
| 24 |
+
|
| 25 |
+
import stable_worldmodel as swm
|
| 26 |
+
from stable_worldmodel.envs.pusht import WeakPolicy
|
| 27 |
+
|
| 28 |
+
world = swm.World(
|
| 29 |
+
'swm/PushT-v1',
|
| 30 |
+
num_envs=args.num_envs,
|
| 31 |
+
image_shape=(224, 224),
|
| 32 |
+
max_episode_steps=args.max_episode_steps,
|
| 33 |
+
render_mode='rgb_array',
|
| 34 |
+
)
|
| 35 |
+
world.set_policy(WeakPolicy(dist_constraint=args.dist_constraint))
|
| 36 |
+
|
| 37 |
+
root = Path(
|
| 38 |
+
args.cache_dir
|
| 39 |
+
or os.getenv('STABLEWM_HOME')
|
| 40 |
+
or swm.data.utils.get_cache_dir()
|
| 41 |
+
)
|
| 42 |
+
out_dir = root / 'datasets' / args.name
|
| 43 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 44 |
+
|
| 45 |
+
per_shard = args.episodes // args.shards
|
| 46 |
+
rng = np.random.default_rng(args.seed)
|
| 47 |
+
|
| 48 |
+
for i in range(args.shards):
|
| 49 |
+
shard = out_dir / f'shard_{i}.lance'
|
| 50 |
+
if shard.exists():
|
| 51 |
+
print(f'[skip] {shard} exists')
|
| 52 |
+
rng.integers(0, 1_000_000)
|
| 53 |
+
continue
|
| 54 |
+
print(f'[collect] shard {i + 1}/{args.shards} -> {shard}')
|
| 55 |
+
world.collect(
|
| 56 |
+
shard,
|
| 57 |
+
episodes=per_shard,
|
| 58 |
+
seed=rng.integers(0, 1_000_000).item(),
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
print(f'done: {out_dir}')
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
if __name__ == '__main__':
|
| 65 |
+
main()
|
code/scripts/compare_planners.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Head-to-head comparison of every planner through one harness.
|
| 2 |
+
|
| 3 |
+
The cross-architecture numbers already in the repo are not comparable to each
|
| 4 |
+
other. ``scripts/eval_controller.py`` runs the plain ``WorldModelPolicy``,
|
| 5 |
+
which -- per ``eval_planner.py``'s header -- plans from one repeated frame and
|
| 6 |
+
zero past action blocks, because ``stable_worldmodel`` declares ``history_len``
|
| 7 |
+
and ``action_history`` and implements neither. ``eval_planner.py`` supplies both
|
| 8 |
+
via ``HistoryPolicy``. A controller scored by the first and a planner scored by
|
| 9 |
+
the second differ in their observations, not just their architecture.
|
| 10 |
+
|
| 11 |
+
So every row here goes through ``eval_planner.py``, which can evaluate the
|
| 12 |
+
controller, the recursive planner, the cross-attention planner, CEM and the
|
| 13 |
+
random floor, with history on for all of them and one shared seed -- meaning
|
| 14 |
+
identical held-out start/goal pairs and paired McNemar throughout.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
import argparse
|
| 18 |
+
import subprocess
|
| 19 |
+
import sys
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
|
| 22 |
+
PY = sys.executable
|
| 23 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 24 |
+
EVAL = 'lejepa_control_2/scripts/eval_planner.py'
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def parse_args():
|
| 28 |
+
p = argparse.ArgumentParser()
|
| 29 |
+
p.add_argument('--controller', default='data/runs/ah_hold0.5/controller.pt')
|
| 30 |
+
p.add_argument('--xa', default='data/runs/planner_xa_D/planner.pt')
|
| 31 |
+
p.add_argument('--recursive', default='data/runs/planner_D10_extended/planner.pt')
|
| 32 |
+
p.add_argument('--horizon', type=int, default=5)
|
| 33 |
+
p.add_argument('--receding-horizon', type=int, nargs='+', default=[1, 5])
|
| 34 |
+
p.add_argument('--controller-k', type=int, nargs='+', default=[3, 5])
|
| 35 |
+
p.add_argument('--xa-k', type=int, nargs='+', default=[2, 3, 5])
|
| 36 |
+
p.add_argument('--seeds', type=int, nargs='+', default=[42, 43, 44])
|
| 37 |
+
p.add_argument('--num-eval', type=int, default=50)
|
| 38 |
+
p.add_argument('--goal-offset', type=int, default=25)
|
| 39 |
+
p.add_argument('--out', default='data/runs/eval_compare')
|
| 40 |
+
p.add_argument('--skip-random', action='store_true')
|
| 41 |
+
return p.parse_args()
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def main():
|
| 45 |
+
args = parse_args()
|
| 46 |
+
jobs = []
|
| 47 |
+
|
| 48 |
+
for rh in args.receding_horizon:
|
| 49 |
+
for seed in args.seeds:
|
| 50 |
+
common = [
|
| 51 |
+
'--horizon', str(args.horizon),
|
| 52 |
+
'--receding-horizon', str(rh),
|
| 53 |
+
'--goal-offset', str(args.goal_offset),
|
| 54 |
+
'--num-eval', str(args.num_eval),
|
| 55 |
+
'--seed', str(seed),
|
| 56 |
+
'--out', args.out,
|
| 57 |
+
]
|
| 58 |
+
if not args.skip_random and rh == args.receding_horizon[0]:
|
| 59 |
+
jobs.append(['--planner', 'random', *common])
|
| 60 |
+
|
| 61 |
+
for k in args.controller_k:
|
| 62 |
+
jobs.append([
|
| 63 |
+
'--planner', 'controller',
|
| 64 |
+
'--controller', args.controller,
|
| 65 |
+
'--refinements', str(k), '--tag', f'ctrl_K{k}', *common,
|
| 66 |
+
])
|
| 67 |
+
|
| 68 |
+
if Path(ROOT / args.recursive).exists():
|
| 69 |
+
jobs.append([
|
| 70 |
+
'--planner', 'planner', '--checkpoint', args.recursive,
|
| 71 |
+
'--tag', 'recursive', *common,
|
| 72 |
+
])
|
| 73 |
+
|
| 74 |
+
if Path(ROOT / args.xa).exists():
|
| 75 |
+
for k in args.xa_k:
|
| 76 |
+
jobs.append([
|
| 77 |
+
'--planner', 'planner', '--checkpoint', args.xa,
|
| 78 |
+
'--cycles', str(k), '--tag', f'xa_K{k}', *common,
|
| 79 |
+
])
|
| 80 |
+
|
| 81 |
+
print(f'{len(jobs)} evaluations -> {args.out}')
|
| 82 |
+
for i, extra in enumerate(jobs, 1):
|
| 83 |
+
print(f'\n[{i}/{len(jobs)}] {" ".join(extra)}', flush=True)
|
| 84 |
+
code = subprocess.run([PY, EVAL, *extra], cwd=ROOT).returncode
|
| 85 |
+
if code != 0:
|
| 86 |
+
print(f'!! exited {code}, continuing', flush=True)
|
| 87 |
+
|
| 88 |
+
print(f'\nappended to {Path(args.out) / "results.jsonl"}')
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
if __name__ == '__main__':
|
| 92 |
+
main()
|
code/scripts/diagnostics.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Diagnostics for horizon-reset procrastination and the K=5 regression.
|
| 2 |
+
|
| 3 |
+
Two questions, both answered in the world model's latent space against the
|
| 4 |
+
same held-out latents used for validation:
|
| 5 |
+
|
| 6 |
+
1. Under receding-horizon execution, does the closed loop have a fixed point
|
| 7 |
+
outside the success radius? Fitting ``D_{n+1} = c*D_n + b`` gives
|
| 8 |
+
``D* = b/(1-c)`` when ``0 < c < 1``. A positive intercept means each
|
| 9 |
+
replan removes a *fraction* of the remaining distance but adds a floor, so
|
| 10 |
+
the loop stalls short of the goal no matter how long it runs.
|
| 11 |
+
|
| 12 |
+
2. Does refinement keep improving predicted cost past the trained depth K=3
|
| 13 |
+
even as real success degrades? If predicted cost improves while success
|
| 14 |
+
drops, the extra refinements are exploiting the world model rather than
|
| 15 |
+
planning better.
|
| 16 |
+
|
| 17 |
+
Question 2 originally measured only the *imagined* side, which cannot separate
|
| 18 |
+
"the plan got better" from "the plan left the region where the world model is
|
| 19 |
+
trustworthy" — both look like falling predicted cost. The support score
|
| 20 |
+
``r(C, b) = -log beta(b | C) / A`` against the fitted GMM answers that directly:
|
| 21 |
+
if the violation fraction rises with k while predicted cost falls, the extra
|
| 22 |
+
refinements are buying imagined progress with out-of-distribution actions.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
import argparse
|
| 26 |
+
import json
|
| 27 |
+
import sys
|
| 28 |
+
from pathlib import Path
|
| 29 |
+
|
| 30 |
+
import numpy as np
|
| 31 |
+
import torch
|
| 32 |
+
|
| 33 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 34 |
+
|
| 35 |
+
from lejepa_control.data import LatentGoalDataset, split_episodes # noqa: E402
|
| 36 |
+
from lejepa_control.losses import BehaviorDensity # noqa: E402
|
| 37 |
+
from lejepa_control.rollout import goal_distance, rollout_plan # noqa: E402
|
| 38 |
+
from lejepa_control.solver import load_controller # noqa: E402
|
| 39 |
+
from lejepa_control.world_model import load_lewm # noqa: E402
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def parse_args():
|
| 43 |
+
p = argparse.ArgumentParser()
|
| 44 |
+
p.add_argument('--controller', default='data/runs/controller/controller.pt')
|
| 45 |
+
p.add_argument('--latents', default='data/latents')
|
| 46 |
+
p.add_argument('--density', default='data/runs/density/density.pt')
|
| 47 |
+
p.add_argument('--samples', type=int, default=256)
|
| 48 |
+
p.add_argument('--horizon', type=int, default=5)
|
| 49 |
+
p.add_argument('--replans', type=int, default=12)
|
| 50 |
+
p.add_argument('--max-k', type=int, default=8)
|
| 51 |
+
p.add_argument('--tag', default='main')
|
| 52 |
+
p.add_argument('--out', default='data/runs/diagnostics')
|
| 53 |
+
return p.parse_args()
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
@torch.no_grad()
|
| 57 |
+
def closed_loop_trace(controller, model, batch, device, execute, replans):
|
| 58 |
+
"""Latent goal distance after each replan, simulating in latent space.
|
| 59 |
+
|
| 60 |
+
The world model is its own simulator here: whatever the predictor says
|
| 61 |
+
the executed blocks caused becomes the next context. That is exactly the
|
| 62 |
+
loop the real MPC runs, minus the env, so it isolates the planner's
|
| 63 |
+
dynamics from simulator mismatch.
|
| 64 |
+
"""
|
| 65 |
+
ctx = batch['context'].to(device)
|
| 66 |
+
past = batch['past_actions'].to(device)
|
| 67 |
+
goal = batch['goal'].to(device)
|
| 68 |
+
num_context = controller.num_context
|
| 69 |
+
|
| 70 |
+
trace = []
|
| 71 |
+
for _ in range(replans):
|
| 72 |
+
out = controller(model, ctx, past, goal)
|
| 73 |
+
plan = out['plans'][-1]
|
| 74 |
+
|
| 75 |
+
# execute the first `execute` blocks and advance the latent state
|
| 76 |
+
pred, frames = rollout_plan(
|
| 77 |
+
model, ctx, past, plan[:, :execute], return_frames=True
|
| 78 |
+
)
|
| 79 |
+
trace.append(goal_distance(pred, goal)[:, execute - 1].cpu().numpy())
|
| 80 |
+
|
| 81 |
+
ctx = frames[:, -num_context:]
|
| 82 |
+
past = torch.cat([past, plan[:, :execute]], dim=1)[:, -(num_context - 1):]
|
| 83 |
+
return np.stack(trace, axis=1) # (B, replans)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def fit_contraction(trace):
|
| 87 |
+
"""Least-squares ``D_{n+1} = c*D_n + b`` over all consecutive pairs."""
|
| 88 |
+
x = trace[:, :-1].ravel()
|
| 89 |
+
y = trace[:, 1:].ravel()
|
| 90 |
+
A = np.stack([x, np.ones_like(x)], axis=1)
|
| 91 |
+
(c, b), *_ = np.linalg.lstsq(A, y, rcond=None)
|
| 92 |
+
resid = y - (c * x + b)
|
| 93 |
+
ss = 1 - resid.var() / y.var() if y.var() > 0 else float('nan')
|
| 94 |
+
fixed = b / (1 - c) if abs(1 - c) > 1e-9 else float('inf')
|
| 95 |
+
return float(c), float(b), float(fixed), float(ss)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
@torch.no_grad()
|
| 99 |
+
def refinement_trace(controller, model, batch, device, max_k,
|
| 100 |
+
density=None, c95=None):
|
| 101 |
+
"""Predicted goal cost after each refinement, out to ``max_k``.
|
| 102 |
+
|
| 103 |
+
Also returns the support-violation fraction per refinement when a density
|
| 104 |
+
model is supplied — the imagined cost and the violation fraction moving in
|
| 105 |
+
opposite directions is the exploitation signature.
|
| 106 |
+
"""
|
| 107 |
+
saved = controller.refinements
|
| 108 |
+
try:
|
| 109 |
+
controller.refinements = max_k
|
| 110 |
+
out = controller(
|
| 111 |
+
model,
|
| 112 |
+
batch['context'].to(device),
|
| 113 |
+
batch['past_actions'].to(device),
|
| 114 |
+
batch['goal'].to(device),
|
| 115 |
+
)
|
| 116 |
+
q = batch['goal_offset'].to(device).clamp(1, controller.horizon)
|
| 117 |
+
costs, arrivals, moves, violations, scores = [], [], [], [], []
|
| 118 |
+
prev = None
|
| 119 |
+
for k, d in enumerate(out['distances']):
|
| 120 |
+
costs.append(float(d[:, -1].mean()))
|
| 121 |
+
arrivals.append(
|
| 122 |
+
float(d.gather(1, (q - 1).unsqueeze(1)).squeeze(1).mean())
|
| 123 |
+
)
|
| 124 |
+
plan = out['plans'][k]
|
| 125 |
+
moves.append(
|
| 126 |
+
0.0 if prev is None else float((plan - prev).abs().mean())
|
| 127 |
+
)
|
| 128 |
+
prev = plan
|
| 129 |
+
|
| 130 |
+
if density is not None and c95 is not None:
|
| 131 |
+
# score every block of the plan at the latent history it is
|
| 132 |
+
# actually proposed from, which is what the support term does
|
| 133 |
+
ctx = out['contexts'][k].flatten(0, 1) # (B*H, N, D)
|
| 134 |
+
score = density.nll_per_dim(ctx, plan.flatten(0, 1))
|
| 135 |
+
violations.append(float((score > c95).float().mean()))
|
| 136 |
+
scores.append(float(score.mean()))
|
| 137 |
+
return costs, arrivals, moves, violations, scores
|
| 138 |
+
finally:
|
| 139 |
+
controller.refinements = saved
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def main():
|
| 143 |
+
args = parse_args()
|
| 144 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 145 |
+
torch.manual_seed(0)
|
| 146 |
+
|
| 147 |
+
stats = json.loads((Path(args.latents) / 'stats.json').read_text())
|
| 148 |
+
model = load_lewm(device=device)
|
| 149 |
+
controller, ckpt = load_controller(
|
| 150 |
+
args.controller, latent_dim=stats['latent_dim'], device=device
|
| 151 |
+
)
|
| 152 |
+
controller.eval()
|
| 153 |
+
print(f'controller step {ckpt["step"]}, K={controller.refinements}')
|
| 154 |
+
|
| 155 |
+
_, val_eps = split_episodes(stats['n_episodes'])
|
| 156 |
+
val = LatentGoalDataset(
|
| 157 |
+
args.latents, max_offset=5, episodes=val_eps, horizon=args.horizon
|
| 158 |
+
)
|
| 159 |
+
idx = np.random.default_rng(0).choice(len(val), args.samples, replace=False)
|
| 160 |
+
batch = {
|
| 161 |
+
k: torch.stack([val[int(i)][k] for i in idx])
|
| 162 |
+
for k in ('context', 'past_actions', 'goal', 'goal_offset')
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
density, c95 = None, None
|
| 166 |
+
if Path(args.density).exists():
|
| 167 |
+
d_ckpt = torch.load(args.density, map_location=device, weights_only=False)
|
| 168 |
+
density = BehaviorDensity(
|
| 169 |
+
latent_dim=d_ckpt['latent_dim'], components=d_ckpt['components']
|
| 170 |
+
)
|
| 171 |
+
density.load_state_dict(d_ckpt['state_dict'])
|
| 172 |
+
density.to(device).eval().requires_grad_(False)
|
| 173 |
+
c95 = float(d_ckpt['c95'])
|
| 174 |
+
print(f'support model loaded, c95={c95:.4f}')
|
| 175 |
+
else:
|
| 176 |
+
print(f'no support model at {args.density} — violation column disabled')
|
| 177 |
+
|
| 178 |
+
report = {'tag': args.tag, 'checkpoint': args.controller,
|
| 179 |
+
'step': int(ckpt['step'])}
|
| 180 |
+
|
| 181 |
+
print('\n=== 1. closed-loop contraction ===')
|
| 182 |
+
print(f'{"execute":>8}{"D_0":>9}{"D_final":>10}{"c":>8}{"b":>9}'
|
| 183 |
+
f'{"D*":>9}{"R^2":>7}')
|
| 184 |
+
report['contraction'] = {}
|
| 185 |
+
for execute in (1, 5):
|
| 186 |
+
trace = closed_loop_trace(
|
| 187 |
+
controller, model, batch, device, execute, args.replans
|
| 188 |
+
)
|
| 189 |
+
c, b, fixed, r2 = fit_contraction(trace)
|
| 190 |
+
m = trace.mean(axis=0)
|
| 191 |
+
print(f'{execute:>8}{m[0]:>9.4f}{m[-1]:>10.4f}'
|
| 192 |
+
f'{c:>8.4f}{b:>9.4f}{fixed:>9.4f}{r2:>7.3f}')
|
| 193 |
+
report['contraction'][f'exec{execute}'] = {
|
| 194 |
+
'c': c, 'b': b, 'fixed_point': fixed, 'r2': r2,
|
| 195 |
+
'mean_trace': [round(float(v), 5) for v in m],
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
print('\n=== 2. refinement past trained depth ===')
|
| 199 |
+
costs, arrivals, moves, violations, scores = refinement_trace(
|
| 200 |
+
controller, model, batch, device, args.max_k, density, c95
|
| 201 |
+
)
|
| 202 |
+
print(f'{"k":>3}{"terminal":>11}{"arrival":>10}{"dJ":>10}{"|plan chg|":>12}'
|
| 203 |
+
f'{"support r":>11}{"viol%":>8}')
|
| 204 |
+
for k in range(len(costs)):
|
| 205 |
+
dj = '' if k == 0 else f'{costs[k - 1] - costs[k]:+.5f}'
|
| 206 |
+
sup = f'{scores[k]:>11.4f}' if scores else f'{"-":>11}'
|
| 207 |
+
vio = f'{100 * violations[k]:>8.1f}' if violations else f'{"-":>8}'
|
| 208 |
+
print(f'{k:>3}{costs[k]:>11.5f}{arrivals[k]:>10.5f}{dj:>10}'
|
| 209 |
+
f'{moves[k]:>12.5f}{sup}{vio}')
|
| 210 |
+
report['refinement'] = {
|
| 211 |
+
'terminal': costs, 'arrival': arrivals, 'plan_change': moves,
|
| 212 |
+
'support_score': scores, 'violation': violations, 'c95': c95,
|
| 213 |
+
}
|
| 214 |
+
|
| 215 |
+
out = Path(args.out)
|
| 216 |
+
out.mkdir(parents=True, exist_ok=True)
|
| 217 |
+
with (out / 'diagnostics.jsonl').open('a') as f:
|
| 218 |
+
f.write(json.dumps(report) + '\n')
|
| 219 |
+
print(f'\nwrote {out / "diagnostics.jsonl"}')
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
if __name__ == '__main__':
|
| 223 |
+
main()
|
code/scripts/encode_latents.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Encode every PushT frame with the frozen LeWM encoder into a latent cache.
|
| 2 |
+
|
| 3 |
+
Controller training never needs pixels: the encoder is frozen and no image
|
| 4 |
+
augmentation is used, so latents can be computed once. The full dataset is
|
| 5 |
+
2.34M frames x 192 dims = ~0.9 GB in fp16, which fits in RAM.
|
| 6 |
+
|
| 7 |
+
Also stores the action z-score statistics. LeWM was trained on z-scored
|
| 8 |
+
actions (``column_normalizer`` defaults to ``method='zscore'``), so the
|
| 9 |
+
controller must emit actions in that same normalized space.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import json
|
| 14 |
+
import time
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import h5py
|
| 18 |
+
import hdf5plugin # noqa: F401 -- registers the blosc filter used by the h5
|
| 19 |
+
import numpy as np
|
| 20 |
+
import torch
|
| 21 |
+
|
| 22 |
+
from lejepa_control.world_model import load_lewm
|
| 23 |
+
|
| 24 |
+
IMAGENET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
|
| 25 |
+
IMAGENET_STD = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def main():
|
| 29 |
+
parser = argparse.ArgumentParser()
|
| 30 |
+
parser.add_argument(
|
| 31 |
+
'--h5', default='data/swm_home/datasets/pusht_expert_train.h5'
|
| 32 |
+
)
|
| 33 |
+
parser.add_argument('--out', default='data/latents')
|
| 34 |
+
parser.add_argument('--batch-size', type=int, default=512)
|
| 35 |
+
parser.add_argument('--limit-episodes', type=int, default=None)
|
| 36 |
+
parser.add_argument('--wm-name', default='quentinll/lewm-pusht')
|
| 37 |
+
args = parser.parse_args()
|
| 38 |
+
|
| 39 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 40 |
+
model = load_lewm(name=args.wm_name, device=device)
|
| 41 |
+
encoder, projector = model.encoder, model.projector
|
| 42 |
+
D = model.predictor.input_dim
|
| 43 |
+
|
| 44 |
+
out_dir = Path(args.out)
|
| 45 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 46 |
+
|
| 47 |
+
mean = IMAGENET_MEAN.to(device)
|
| 48 |
+
std = IMAGENET_STD.to(device)
|
| 49 |
+
|
| 50 |
+
with h5py.File(args.h5, 'r') as f:
|
| 51 |
+
lengths = f['ep_len'][:].astype(np.int64)
|
| 52 |
+
offsets = f['ep_offset'][:].astype(np.int64)
|
| 53 |
+
|
| 54 |
+
if args.limit_episodes is not None:
|
| 55 |
+
lengths = lengths[: args.limit_episodes]
|
| 56 |
+
offsets = offsets[: args.limit_episodes]
|
| 57 |
+
|
| 58 |
+
n_frames = int(lengths.sum())
|
| 59 |
+
end = int(offsets[-1] + lengths[-1])
|
| 60 |
+
print(f'{len(lengths)} episodes, {n_frames} frames -> {D}-dim latents')
|
| 61 |
+
|
| 62 |
+
actions = f['action'][:end].astype(np.float32)
|
| 63 |
+
# the frame that ends an episode has no outgoing action (there is no
|
| 64 |
+
# next state to leave it for) and is NaN-padded; ignore those rows for
|
| 65 |
+
# the stats, then fill them with the mean so a block that straddles an
|
| 66 |
+
# episode boundary (LatentGoalDataset's real_action, at the last valid
|
| 67 |
+
# start) never hands training a NaN.
|
| 68 |
+
nan_mask = np.isnan(actions).any(axis=1)
|
| 69 |
+
a_mean = np.nanmean(actions, axis=0)
|
| 70 |
+
a_std = np.nanstd(actions, axis=0)
|
| 71 |
+
if nan_mask.any():
|
| 72 |
+
print(f'{int(nan_mask.sum())} NaN (terminal-step) action rows -> filled with mean')
|
| 73 |
+
actions[nan_mask] = a_mean
|
| 74 |
+
print(f'action mean={a_mean} std={a_std}')
|
| 75 |
+
|
| 76 |
+
latents = np.lib.format.open_memmap(
|
| 77 |
+
out_dir / 'latents.npy',
|
| 78 |
+
mode='w+',
|
| 79 |
+
dtype=np.float16,
|
| 80 |
+
shape=(n_frames, D),
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
t0 = time.perf_counter()
|
| 84 |
+
for start in range(0, end, args.batch_size):
|
| 85 |
+
stop = min(start + args.batch_size, end)
|
| 86 |
+
frames = f['pixels'][start:stop] # (B, 224, 224, 3) uint8
|
| 87 |
+
|
| 88 |
+
x = torch.from_numpy(frames).to(device, non_blocking=True)
|
| 89 |
+
x = x.permute(0, 3, 1, 2).float().div_(255.0).sub_(mean).div_(std)
|
| 90 |
+
|
| 91 |
+
with torch.no_grad(), torch.autocast('cuda', dtype=torch.bfloat16):
|
| 92 |
+
out = encoder(x, interpolate_pos_encoding=True)
|
| 93 |
+
emb = projector(out.last_hidden_state[:, 0].float())
|
| 94 |
+
|
| 95 |
+
latents[start:stop] = emb.float().cpu().numpy().astype(np.float16)
|
| 96 |
+
|
| 97 |
+
if start % (args.batch_size * 200) == 0:
|
| 98 |
+
done = stop / end
|
| 99 |
+
rate = stop / (time.perf_counter() - t0)
|
| 100 |
+
eta = (end - stop) / rate / 60
|
| 101 |
+
print(
|
| 102 |
+
f' {done:6.1%} {rate:6.0f} img/s eta {eta:5.1f} min',
|
| 103 |
+
flush=True,
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
latents.flush()
|
| 107 |
+
|
| 108 |
+
np.save(out_dir / 'actions.npy', actions)
|
| 109 |
+
np.save(out_dir / 'ep_len.npy', lengths)
|
| 110 |
+
np.save(out_dir / 'ep_offset.npy', offsets)
|
| 111 |
+
|
| 112 |
+
stats = {
|
| 113 |
+
'action_mean': a_mean.tolist(),
|
| 114 |
+
'action_std': a_std.tolist(),
|
| 115 |
+
'latent_dim': int(D),
|
| 116 |
+
'n_frames': n_frames,
|
| 117 |
+
'n_episodes': len(lengths),
|
| 118 |
+
}
|
| 119 |
+
(out_dir / 'stats.json').write_text(json.dumps(stats, indent=2))
|
| 120 |
+
|
| 121 |
+
z = np.asarray(latents[:10000], dtype=np.float32)
|
| 122 |
+
print(f'latent per-coord std (mean) = {z.std(0).mean():.4f}')
|
| 123 |
+
print(f'done in {(time.perf_counter() - t0) / 60:.1f} min -> {out_dir}')
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
if __name__ == '__main__':
|
| 127 |
+
main()
|
code/scripts/encode_latents_reacher.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Encode every DMC Reacher frame with the frozen LeWM encoder into a latent cache.
|
| 2 |
+
|
| 3 |
+
Reacher port of ``scripts/encode_latents.py``. Controller training never needs
|
| 4 |
+
pixels: the encoder is frozen and no image augmentation is used, so latents can
|
| 5 |
+
be computed once. Also stores the action z-score statistics — LeWM was trained
|
| 6 |
+
on z-scored actions (``column_normalizer`` defaults to ``method='zscore'``),
|
| 7 |
+
so the controller must emit actions in that same normalized space.
|
| 8 |
+
|
| 9 |
+
The only reacher-specific differences from the PushT script are the default
|
| 10 |
+
dataset path and the episode-index column names: the DMC collect pipeline
|
| 11 |
+
writes ``ep_idx`` where PushT wrote ``episode_idx``. Both spellings are
|
| 12 |
+
accepted, and ``ep_len``/``ep_offset`` are derived when absent.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import argparse
|
| 16 |
+
import json
|
| 17 |
+
import time
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
import h5py
|
| 21 |
+
import hdf5plugin # noqa: F401 -- registers the blosc filter used by the h5
|
| 22 |
+
import numpy as np
|
| 23 |
+
import torch
|
| 24 |
+
|
| 25 |
+
from lejepa_control.world_model import load_lewm
|
| 26 |
+
|
| 27 |
+
IMAGENET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
|
| 28 |
+
IMAGENET_STD = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def episode_layout(f):
|
| 32 |
+
"""``(lengths, offsets)`` and the dataset row count, per spelling."""
|
| 33 |
+
keys = set(f.keys())
|
| 34 |
+
if 'ep_len' in keys and 'ep_offset' in keys:
|
| 35 |
+
lengths = f['ep_len'][:].astype(np.int64)
|
| 36 |
+
offsets = f['ep_offset'][:].astype(np.int64)
|
| 37 |
+
return lengths, offsets, int(offsets[-1] + lengths[-1])
|
| 38 |
+
ep_col = 'episode_idx' if 'episode_idx' in keys else 'ep_idx'
|
| 39 |
+
ep_idx = f[ep_col][:].astype(np.int64)
|
| 40 |
+
step_idx = f['step_idx'][:].astype(np.int64)
|
| 41 |
+
n_eps = int(ep_idx.max()) + 1
|
| 42 |
+
lengths = np.zeros(n_eps, dtype=np.int64)
|
| 43 |
+
np.maximum.at(lengths, ep_idx, step_idx + 1)
|
| 44 |
+
keep = lengths > 0
|
| 45 |
+
lengths = lengths[keep]
|
| 46 |
+
offsets = np.concatenate([[0], np.cumsum(lengths)[:-1]])
|
| 47 |
+
return lengths, offsets, len(ep_idx)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def main():
|
| 51 |
+
parser = argparse.ArgumentParser()
|
| 52 |
+
parser.add_argument(
|
| 53 |
+
'--h5', default='data/swm_home/datasets/dmc/reacher.h5'
|
| 54 |
+
)
|
| 55 |
+
parser.add_argument('--out', default='data/latents_reacher')
|
| 56 |
+
parser.add_argument('--batch-size', type=int, default=512)
|
| 57 |
+
parser.add_argument('--limit-episodes', type=int, default=None)
|
| 58 |
+
parser.add_argument('--wm-name', default='quentinll/lewm-reacher')
|
| 59 |
+
args = parser.parse_args()
|
| 60 |
+
|
| 61 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 62 |
+
model = load_lewm(name=args.wm_name, device=device)
|
| 63 |
+
encoder, projector = model.encoder, model.projector
|
| 64 |
+
D = model.predictor.input_dim
|
| 65 |
+
|
| 66 |
+
out_dir = Path(args.out)
|
| 67 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 68 |
+
|
| 69 |
+
mean = IMAGENET_MEAN.to(device)
|
| 70 |
+
std = IMAGENET_STD.to(device)
|
| 71 |
+
|
| 72 |
+
with h5py.File(args.h5, 'r') as f:
|
| 73 |
+
lengths, offsets, end = episode_layout(f)
|
| 74 |
+
|
| 75 |
+
if args.limit_episodes is not None:
|
| 76 |
+
lengths = lengths[: args.limit_episodes]
|
| 77 |
+
offsets = offsets[: args.limit_episodes]
|
| 78 |
+
end = int(offsets[-1] + lengths[-1])
|
| 79 |
+
|
| 80 |
+
n_frames = int(lengths.sum())
|
| 81 |
+
print(f'{len(lengths)} episodes, {n_frames} frames -> {D}-dim latents')
|
| 82 |
+
|
| 83 |
+
actions = f['action'][:end].astype(np.float32)
|
| 84 |
+
# the frame that ends an episode has no outgoing action (there is no
|
| 85 |
+
# next state to leave it for) and is NaN-padded; ignore those rows for
|
| 86 |
+
# the stats, then fill them with the mean so a block that straddles an
|
| 87 |
+
# episode boundary never hands training a NaN.
|
| 88 |
+
nan_mask = np.isnan(actions).any(axis=1)
|
| 89 |
+
a_mean = np.nanmean(actions, axis=0)
|
| 90 |
+
a_std = np.nanstd(actions, axis=0)
|
| 91 |
+
if nan_mask.any():
|
| 92 |
+
print(
|
| 93 |
+
f'{int(nan_mask.sum())} NaN (terminal-step) action rows'
|
| 94 |
+
' -> filled with mean'
|
| 95 |
+
)
|
| 96 |
+
actions[nan_mask] = a_mean
|
| 97 |
+
print(f'action mean={a_mean} std={a_std}')
|
| 98 |
+
|
| 99 |
+
latents = np.lib.format.open_memmap(
|
| 100 |
+
out_dir / 'latents.npy',
|
| 101 |
+
mode='w+',
|
| 102 |
+
dtype=np.float16,
|
| 103 |
+
shape=(n_frames, D),
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
t0 = time.perf_counter()
|
| 107 |
+
for start in range(0, end, args.batch_size):
|
| 108 |
+
stop = min(start + args.batch_size, end)
|
| 109 |
+
frames = f['pixels'][start:stop] # (B, 224, 224, 3) uint8
|
| 110 |
+
|
| 111 |
+
x = torch.from_numpy(frames).to(device, non_blocking=True)
|
| 112 |
+
x = x.permute(0, 3, 1, 2).float().div_(255.0).sub_(mean).div_(std)
|
| 113 |
+
|
| 114 |
+
with torch.no_grad(), torch.autocast('cuda', dtype=torch.bfloat16):
|
| 115 |
+
out = encoder(x, interpolate_pos_encoding=True)
|
| 116 |
+
emb = projector(out.last_hidden_state[:, 0].float())
|
| 117 |
+
|
| 118 |
+
latents[start:stop] = emb.float().cpu().numpy().astype(np.float16)
|
| 119 |
+
|
| 120 |
+
if start % (args.batch_size * 200) == 0:
|
| 121 |
+
done = stop / end
|
| 122 |
+
rate = stop / (time.perf_counter() - t0)
|
| 123 |
+
eta = (end - stop) / rate / 60
|
| 124 |
+
print(
|
| 125 |
+
f' {done:6.1%} {rate:6.0f} img/s eta {eta:5.1f} min',
|
| 126 |
+
flush=True,
|
| 127 |
+
)
|
| 128 |
+
|
| 129 |
+
latents.flush()
|
| 130 |
+
|
| 131 |
+
np.save(out_dir / 'actions.npy', actions)
|
| 132 |
+
np.save(out_dir / 'ep_len.npy', lengths)
|
| 133 |
+
np.save(out_dir / 'ep_offset.npy', offsets)
|
| 134 |
+
|
| 135 |
+
stats = {
|
| 136 |
+
'action_mean': a_mean.tolist(),
|
| 137 |
+
'action_std': a_std.tolist(),
|
| 138 |
+
'latent_dim': int(D),
|
| 139 |
+
'n_frames': n_frames,
|
| 140 |
+
'n_episodes': len(lengths),
|
| 141 |
+
}
|
| 142 |
+
(out_dir / 'stats.json').write_text(json.dumps(stats, indent=2))
|
| 143 |
+
|
| 144 |
+
z = np.asarray(latents[:10000], dtype=np.float32)
|
| 145 |
+
print(f'latent per-coord std (mean) = {z.std(0).mean():.4f}')
|
| 146 |
+
print(f'done in {(time.perf_counter() - t0) / 60:.1f} min -> {out_dir}')
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
if __name__ == '__main__':
|
| 150 |
+
main()
|
code/scripts/eval_controller.py
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluate the controller in the real PushT simulator, against CEM.
|
| 2 |
+
|
| 3 |
+
Both planners run through the same ``WorldModelPolicy``, the same wrappers and
|
| 4 |
+
the same held-out initial-state/goal pairs, so the only thing that differs is
|
| 5 |
+
how the plan is produced. Reports success rate and wall-clock planning time —
|
| 6 |
+
the central claim is approaching CEM's success with far fewer world-model
|
| 7 |
+
evaluations.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import os
|
| 11 |
+
|
| 12 |
+
os.environ['MUJOCO_GL'] = 'egl'
|
| 13 |
+
|
| 14 |
+
import argparse # noqa: E402
|
| 15 |
+
import json # noqa: E402
|
| 16 |
+
import sys # noqa: E402
|
| 17 |
+
import time # noqa: E402
|
| 18 |
+
from pathlib import Path # noqa: E402
|
| 19 |
+
|
| 20 |
+
import hdf5plugin # noqa: F401,E402 -- blosc filter for the expert h5
|
| 21 |
+
import numpy as np # noqa: E402
|
| 22 |
+
import stable_pretraining as spt # noqa: E402
|
| 23 |
+
import stable_worldmodel as swm # noqa: E402
|
| 24 |
+
import torch # noqa: E402
|
| 25 |
+
from sklearn import preprocessing # noqa: E402
|
| 26 |
+
from torchvision.transforms import v2 as transforms # noqa: E402
|
| 27 |
+
|
| 28 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 29 |
+
|
| 30 |
+
from lejepa_control.solver import ControllerSolver, load_controller # noqa: E402
|
| 31 |
+
from lejepa_control.world_model import load_lewm # noqa: E402
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def parse_args():
|
| 35 |
+
p = argparse.ArgumentParser()
|
| 36 |
+
p.add_argument('--controller', default='data/runs/controller/controller.pt')
|
| 37 |
+
p.add_argument('--planner', default='controller', choices=['controller', 'cem'])
|
| 38 |
+
p.add_argument('--refinements', type=int, default=None)
|
| 39 |
+
p.add_argument('--num-eval', type=int, default=50)
|
| 40 |
+
p.add_argument('--eval-budget', type=int, default=50)
|
| 41 |
+
p.add_argument('--goal-offset', type=int, default=25)
|
| 42 |
+
p.add_argument('--horizon', type=int, default=5)
|
| 43 |
+
# execute one block then replan (spec section 9); set equal to --horizon
|
| 44 |
+
# for the "execute the full plan" ablation
|
| 45 |
+
p.add_argument('--receding-horizon', type=int, default=1)
|
| 46 |
+
p.add_argument('--cem-samples', type=int, default=300)
|
| 47 |
+
p.add_argument('--cem-steps', type=int, default=30)
|
| 48 |
+
p.add_argument(
|
| 49 |
+
'--dataset', default='data/swm_home/datasets/pusht_expert_train.h5'
|
| 50 |
+
)
|
| 51 |
+
# world model + env to evaluate against; the PushT defaults keep existing
|
| 52 |
+
# runs unchanged. TwoRoom uses quentinll/lewm-tworooms + swm/TwoRoom-v1.
|
| 53 |
+
p.add_argument('--wm-name', default='quentinll/lewm-pusht')
|
| 54 |
+
p.add_argument('--env-name', default='swm/PushT-v1')
|
| 55 |
+
# the TwoRoom h5 stores the agent position as `pos_agent` and the episode
|
| 56 |
+
# id as `ep_idx`; PushT uses `state` / `episode_idx`. These map the harness
|
| 57 |
+
# onto either dataset without changing the env-facing info keys.
|
| 58 |
+
p.add_argument('--state-col', default='state')
|
| 59 |
+
p.add_argument('--episode-col', default='episode_idx')
|
| 60 |
+
p.add_argument('--out', default='data/runs/eval')
|
| 61 |
+
p.add_argument('--seed', type=int, default=42)
|
| 62 |
+
p.add_argument('--video', action='store_true')
|
| 63 |
+
return p.parse_args()
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def img_transform(size=224):
|
| 67 |
+
return transforms.Compose(
|
| 68 |
+
[
|
| 69 |
+
transforms.ToImage(),
|
| 70 |
+
transforms.ToDtype(torch.float32, scale=True),
|
| 71 |
+
transforms.Normalize(**spt.data.dataset_stats.ImageNet),
|
| 72 |
+
transforms.Resize(size=size),
|
| 73 |
+
]
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def main():
|
| 78 |
+
args = parse_args()
|
| 79 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 80 |
+
|
| 81 |
+
world = swm.World(
|
| 82 |
+
env_name=args.env_name,
|
| 83 |
+
num_envs=args.num_eval,
|
| 84 |
+
max_episode_steps=2 * args.eval_budget,
|
| 85 |
+
image_shape=(224, 224),
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
dataset = swm.data.load_dataset(
|
| 89 |
+
str(Path(args.dataset).resolve()),
|
| 90 |
+
keys_to_cache=['action', 'proprio', args.state_col],
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
process = {}
|
| 94 |
+
for col in ('action', 'proprio', 'state'):
|
| 95 |
+
# fit on the dataset column (state_col), but store under the env
|
| 96 |
+
# info-key the policy matches against (always 'state'); TwoRoom's
|
| 97 |
+
# h5 names this column `pos_agent` but its env still emits `state`.
|
| 98 |
+
src = args.state_col if col == 'state' else col
|
| 99 |
+
data = dataset.get_col_data(src)
|
| 100 |
+
data = data[~np.isnan(data).any(axis=1)]
|
| 101 |
+
scaler = preprocessing.StandardScaler().fit(data)
|
| 102 |
+
process[col] = scaler
|
| 103 |
+
if col != 'action':
|
| 104 |
+
process[f'goal_{col}'] = scaler
|
| 105 |
+
|
| 106 |
+
transform = {'pixels': img_transform(), 'goal': img_transform()}
|
| 107 |
+
|
| 108 |
+
model = load_lewm(name=args.wm_name, device=device)
|
| 109 |
+
model.interpolate_pos_encoding = True
|
| 110 |
+
latent_dim = model.predictor.input_dim
|
| 111 |
+
|
| 112 |
+
config = swm.PlanConfig(
|
| 113 |
+
horizon=args.horizon,
|
| 114 |
+
receding_horizon=args.receding_horizon,
|
| 115 |
+
action_block=5,
|
| 116 |
+
history_len=model.predictor.num_frames,
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
if args.planner == 'controller':
|
| 120 |
+
controller, ckpt = load_controller(
|
| 121 |
+
args.controller,
|
| 122 |
+
latent_dim=latent_dim,
|
| 123 |
+
device=device,
|
| 124 |
+
refinements=args.refinements,
|
| 125 |
+
)
|
| 126 |
+
solver = ControllerSolver(model, controller, device=device)
|
| 127 |
+
tag = f'controller_K{controller.refinements}'
|
| 128 |
+
print(f'controller from step {ckpt["step"]}, K={controller.refinements}')
|
| 129 |
+
else:
|
| 130 |
+
cost = swm.planning.ShootingCostEvaluator(model, swm.planning.GoalMSE())
|
| 131 |
+
solver = swm.planning.CEMSolver(
|
| 132 |
+
cost=cost,
|
| 133 |
+
num_samples=args.cem_samples,
|
| 134 |
+
n_steps=args.cem_steps,
|
| 135 |
+
topk=30,
|
| 136 |
+
device=device,
|
| 137 |
+
)
|
| 138 |
+
tag = f'cem_s{args.cem_samples}_n{args.cem_steps}'
|
| 139 |
+
|
| 140 |
+
# the central claim is cost, so count predictor forwards rather than
|
| 141 |
+
# asserting them: one call = one batched latent transition
|
| 142 |
+
calls = {'n': 0, 'rows': 0}
|
| 143 |
+
inner_predict = model.predictor.forward
|
| 144 |
+
|
| 145 |
+
def counting_predict(*a, **kw):
|
| 146 |
+
calls['n'] += 1
|
| 147 |
+
first = a[0] if a else next(iter(kw.values()))
|
| 148 |
+
calls['rows'] += first.shape[0]
|
| 149 |
+
return inner_predict(*a, **kw)
|
| 150 |
+
|
| 151 |
+
model.predictor.forward = counting_predict
|
| 152 |
+
|
| 153 |
+
# terminal latent distance at each replan -> CEM regret, comparable across
|
| 154 |
+
# planners because both are scored under the same frozen model. GoalMSE
|
| 155 |
+
# sums over the latent dim while the controller averages, so rescale to
|
| 156 |
+
# per-dim. Done by retyping the instance: __call__ is looked up on the
|
| 157 |
+
# type, and the policy isinstance-checks against the Solver protocol.
|
| 158 |
+
terminals = []
|
| 159 |
+
scale = 1.0 if args.planner == 'controller' else 1.0 / latent_dim
|
| 160 |
+
base = type(solver)
|
| 161 |
+
|
| 162 |
+
class RecordingSolver(base):
|
| 163 |
+
def __call__(self, info_dict, init_action=None):
|
| 164 |
+
out = base.__call__(self, info_dict, init_action)
|
| 165 |
+
costs = out.get('costs')
|
| 166 |
+
if costs is not None:
|
| 167 |
+
value = float(torch.as_tensor(costs).float().mean())
|
| 168 |
+
terminals.append(value * scale)
|
| 169 |
+
return out
|
| 170 |
+
|
| 171 |
+
solver.__class__ = RecordingSolver
|
| 172 |
+
|
| 173 |
+
policy = swm.policy.WorldModelPolicy(
|
| 174 |
+
solver=solver,
|
| 175 |
+
config=config,
|
| 176 |
+
process=process,
|
| 177 |
+
transform=transform,
|
| 178 |
+
history_keys=('pixels',),
|
| 179 |
+
)
|
| 180 |
+
world.set_policy(policy)
|
| 181 |
+
|
| 182 |
+
# held-out start/goal pairs, identical across planners for a fair compare
|
| 183 |
+
ep_idx = dataset.get_col_data(args.episode_col)
|
| 184 |
+
step_idx = dataset.get_col_data('step_idx')
|
| 185 |
+
episodes = np.unique(ep_idx)
|
| 186 |
+
lengths = {e: step_idx[ep_idx == e].max() + 1 for e in episodes}
|
| 187 |
+
max_start = np.array([lengths[e] for e in ep_idx]) - args.goal_offset - 1
|
| 188 |
+
valid = np.nonzero(step_idx <= max_start)[0]
|
| 189 |
+
|
| 190 |
+
rng = np.random.default_rng(args.seed)
|
| 191 |
+
picked = np.sort(valid[rng.choice(len(valid), args.num_eval, replace=False)])
|
| 192 |
+
|
| 193 |
+
out_dir = Path(args.out)
|
| 194 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 195 |
+
|
| 196 |
+
t0 = time.time()
|
| 197 |
+
metrics = world.evaluate(
|
| 198 |
+
dataset=dataset,
|
| 199 |
+
start_steps=step_idx[picked].tolist(),
|
| 200 |
+
goal_offset=args.goal_offset,
|
| 201 |
+
eval_budget=args.eval_budget,
|
| 202 |
+
episodes_idx=ep_idx[picked].tolist(),
|
| 203 |
+
callables=[
|
| 204 |
+
{
|
| 205 |
+
'method': '_set_state',
|
| 206 |
+
'args': {'state': {'value': args.state_col}},
|
| 207 |
+
},
|
| 208 |
+
{
|
| 209 |
+
'method': '_set_goal_state',
|
| 210 |
+
'args': {'goal_state': {'value': f'goal_{args.state_col}'}},
|
| 211 |
+
},
|
| 212 |
+
],
|
| 213 |
+
video=out_dir if args.video else None,
|
| 214 |
+
)
|
| 215 |
+
elapsed = time.time() - t0
|
| 216 |
+
|
| 217 |
+
result = {
|
| 218 |
+
'planner': tag,
|
| 219 |
+
'env': args.env_name,
|
| 220 |
+
'wm': args.wm_name,
|
| 221 |
+
'receding_horizon': args.receding_horizon,
|
| 222 |
+
'success_rate': float(metrics['success_rate']),
|
| 223 |
+
'seconds': elapsed,
|
| 224 |
+
'seconds_per_episode': elapsed / args.num_eval,
|
| 225 |
+
'mean_terminal_distance': (
|
| 226 |
+
float(np.mean(terminals)) if terminals else None
|
| 227 |
+
),
|
| 228 |
+
# Averaging over solver calls is not comparable across execution
|
| 229 |
+
# lengths: a run that replans every block makes more calls, and the
|
| 230 |
+
# later ones are taken nearer the goal. The first call is taken from
|
| 231 |
+
# the same held-out state by every planner, so that one is.
|
| 232 |
+
'first_terminal_distance': float(terminals[0]) if terminals else None,
|
| 233 |
+
'terminal_distance_trace': [float(t) for t in terminals],
|
| 234 |
+
'predictor_calls': calls['n'],
|
| 235 |
+
'predictor_rows_per_episode': calls['rows'] / args.num_eval,
|
| 236 |
+
'num_eval': args.num_eval,
|
| 237 |
+
'eval_budget': args.eval_budget,
|
| 238 |
+
'goal_offset': args.goal_offset,
|
| 239 |
+
'checkpoint': args.controller if args.planner == 'controller' else None,
|
| 240 |
+
'seed': args.seed,
|
| 241 |
+
'refinements': args.refinements,
|
| 242 |
+
# per-episode outcomes: all rows share start/goal pairs, so planner
|
| 243 |
+
# comparisons must be paired rather than treated as independent
|
| 244 |
+
'episode_successes': [
|
| 245 |
+
bool(x) for x in metrics['episode_successes'].tolist()
|
| 246 |
+
],
|
| 247 |
+
}
|
| 248 |
+
print(json.dumps({k: v for k, v in result.items()
|
| 249 |
+
if k != 'terminal_distance_trace'}, indent=2))
|
| 250 |
+
|
| 251 |
+
with (out_dir / 'results.jsonl').open('a') as f:
|
| 252 |
+
f.write(json.dumps(result) + '\n')
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
if __name__ == '__main__':
|
| 256 |
+
main()
|
code/scripts/eval_controller_reacher.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluate the controller in the real DMC Reacher simulator, against CEM.
|
| 2 |
+
|
| 3 |
+
Reacher port of ``scripts/eval_controller.py``. Differences from the PushT
|
| 4 |
+
original, all following the upstream LeWM eval recipe
|
| 5 |
+
(``stable-worldmodel/scripts/plan/config/reacher.yaml``):
|
| 6 |
+
|
| 7 |
+
* the env runs the ``qpos_match`` task: an episode counts as a success when
|
| 8 |
+
every joint is within 0.05 rad of the goal configuration, which is what
|
| 9 |
+
supplies a termination signal to ``world.evaluate``;
|
| 10 |
+
* dataset rows are pushed into the simulator with ``set_state(qpos, qvel)``
|
| 11 |
+
and the goal with ``set_target_qpos(goal_qpos)`` instead of PushT's
|
| 12 |
+
``_set_state``/``_set_goal_state``;
|
| 13 |
+
* only ``action`` is z-scored — the qpos/qvel columns are reset plumbing,
|
| 14 |
+
never seen by the policy;
|
| 15 |
+
* the policy is :class:`tools.history_policy.HistoryPolicy` so the solver
|
| 16 |
+
gets the real strided frame history and past action blocks (commit bd459fc:
|
| 17 |
+
without it 41% of the committed action is lost).
|
| 18 |
+
|
| 19 |
+
Both planners run through the same policy, the same wrappers and the same
|
| 20 |
+
held-out initial-state/goal pairs, so the only thing that differs is how the
|
| 21 |
+
plan is produced. Reports success rate and wall-clock planning time — the
|
| 22 |
+
central claim is approaching CEM's success with far fewer world-model
|
| 23 |
+
evaluations.
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
import argparse
|
| 27 |
+
import json
|
| 28 |
+
import os
|
| 29 |
+
import sys
|
| 30 |
+
import time
|
| 31 |
+
from pathlib import Path
|
| 32 |
+
|
| 33 |
+
# PushT eval hardcodes egl (Linux). Windows has no egl; glfw renders into a
|
| 34 |
+
# hidden window and works here. Must be set before mujoco/dm_control import.
|
| 35 |
+
if not os.environ.get('MUJOCO_GL'):
|
| 36 |
+
os.environ['MUJOCO_GL'] = 'glfw' if sys.platform == 'win32' else 'egl'
|
| 37 |
+
|
| 38 |
+
import hdf5plugin # noqa: F401,E402 -- blosc filter for the reacher h5
|
| 39 |
+
import numpy as np
|
| 40 |
+
import stable_pretraining as spt # noqa: E402
|
| 41 |
+
import stable_worldmodel as swm # noqa: E402
|
| 42 |
+
import torch # noqa: E402
|
| 43 |
+
from sklearn import preprocessing # noqa: E402
|
| 44 |
+
from torchvision.transforms import v2 as transforms # noqa: E402
|
| 45 |
+
|
| 46 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 47 |
+
|
| 48 |
+
from lejepa_control.solver import ControllerSolver, load_controller # noqa: E402
|
| 49 |
+
from lejepa_control.world_model import load_lewm # noqa: E402
|
| 50 |
+
from tools.history_policy import HistoryPolicy # noqa: E402
|
| 51 |
+
|
| 52 |
+
# qpos_match: 0.05 rad per-joint tolerance, the env's own criterion
|
| 53 |
+
CALLABLES = [
|
| 54 |
+
{
|
| 55 |
+
'method': 'set_state',
|
| 56 |
+
'args': {'qpos': {'value': 'qpos'}, 'qvel': {'value': 'qvel'}},
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
'method': 'set_target_qpos',
|
| 60 |
+
'args': {'target_qpos': {'value': 'goal_qpos'}},
|
| 61 |
+
},
|
| 62 |
+
]
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def parse_args():
|
| 66 |
+
p = argparse.ArgumentParser()
|
| 67 |
+
p.add_argument('--controller', default='data/runs/controller_reacher/controller.pt')
|
| 68 |
+
p.add_argument('--planner', default='controller', choices=['controller', 'cem', 'random'])
|
| 69 |
+
p.add_argument('--refinements', type=int, default=None)
|
| 70 |
+
p.add_argument('--num-eval', type=int, default=50)
|
| 71 |
+
p.add_argument('--eval-budget', type=int, default=50)
|
| 72 |
+
p.add_argument('--goal-offset', type=int, default=25)
|
| 73 |
+
p.add_argument('--horizon', type=int, default=5)
|
| 74 |
+
# execute one block then replan (spec section 9); set equal to --horizon
|
| 75 |
+
# for the "execute the full plan" ablation
|
| 76 |
+
p.add_argument('--receding-horizon', type=int, default=1)
|
| 77 |
+
p.add_argument('--cem-samples', type=int, default=300)
|
| 78 |
+
p.add_argument('--cem-steps', type=int, default=30)
|
| 79 |
+
p.add_argument('--no-history', action='store_true',
|
| 80 |
+
help='drop HistoryPolicy: single repeated frame, zero past blocks')
|
| 81 |
+
p.add_argument(
|
| 82 |
+
'--dataset', default='data/swm_home/datasets/dmc/reacher.h5'
|
| 83 |
+
)
|
| 84 |
+
p.add_argument('--wm-name', default='quentinll/lewm-reacher')
|
| 85 |
+
p.add_argument('--env-name', default='swm/ReacherDMControl-v0')
|
| 86 |
+
p.add_argument('--out', default='data/runs/eval_reacher')
|
| 87 |
+
p.add_argument('--seed', type=int, default=42)
|
| 88 |
+
p.add_argument('--video', action='store_true')
|
| 89 |
+
return p.parse_args()
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def img_transform(size=224):
|
| 93 |
+
return transforms.Compose(
|
| 94 |
+
[
|
| 95 |
+
transforms.ToImage(),
|
| 96 |
+
transforms.ToDtype(torch.float32, scale=True),
|
| 97 |
+
transforms.Normalize(**spt.data.dataset_stats.ImageNet),
|
| 98 |
+
transforms.Resize(size=size),
|
| 99 |
+
]
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def main():
|
| 104 |
+
args = parse_args()
|
| 105 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 106 |
+
|
| 107 |
+
world = swm.World(
|
| 108 |
+
env_name=args.env_name,
|
| 109 |
+
num_envs=args.num_eval,
|
| 110 |
+
max_episode_steps=2 * args.eval_budget,
|
| 111 |
+
image_shape=(224, 224),
|
| 112 |
+
task='qpos_match',
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
dataset = swm.data.load_dataset(
|
| 116 |
+
str(Path(args.dataset).resolve()), keys_to_cache=['action']
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
# only the action column is z-scored; qpos/qvel feed the reset callables
|
| 120 |
+
process = {}
|
| 121 |
+
data = dataset.get_col_data('action')
|
| 122 |
+
data = data[~np.isnan(data).any(axis=1)]
|
| 123 |
+
process['action'] = preprocessing.StandardScaler().fit(data)
|
| 124 |
+
|
| 125 |
+
transform = {'pixels': img_transform(), 'goal': img_transform()}
|
| 126 |
+
|
| 127 |
+
model = load_lewm(name=args.wm_name, device=device)
|
| 128 |
+
model.interpolate_pos_encoding = True
|
| 129 |
+
latent_dim = model.predictor.input_dim
|
| 130 |
+
|
| 131 |
+
config = swm.PlanConfig(
|
| 132 |
+
horizon=args.horizon,
|
| 133 |
+
receding_horizon=args.receding_horizon,
|
| 134 |
+
action_block=5,
|
| 135 |
+
history_len=model.predictor.num_frames,
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
if args.planner == 'controller':
|
| 139 |
+
controller, ckpt = load_controller(
|
| 140 |
+
args.controller,
|
| 141 |
+
latent_dim=latent_dim,
|
| 142 |
+
device=device,
|
| 143 |
+
refinements=args.refinements,
|
| 144 |
+
)
|
| 145 |
+
solver = ControllerSolver(model, controller, device=device)
|
| 146 |
+
tag = f'controller_K{controller.refinements}'
|
| 147 |
+
print(f'controller from step {ckpt["step"]}, K={controller.refinements}')
|
| 148 |
+
elif args.planner == 'cem':
|
| 149 |
+
cost = swm.planning.ShootingCostEvaluator(model, swm.planning.GoalMSE())
|
| 150 |
+
solver = swm.planning.CEMSolver(
|
| 151 |
+
cost=cost,
|
| 152 |
+
num_samples=args.cem_samples,
|
| 153 |
+
n_steps=args.cem_steps,
|
| 154 |
+
topk=30,
|
| 155 |
+
device=device,
|
| 156 |
+
)
|
| 157 |
+
tag = f'cem_s{args.cem_samples}_n{args.cem_steps}'
|
| 158 |
+
else:
|
| 159 |
+
solver = swm.policy.RandomPolicy()
|
| 160 |
+
tag = 'random'
|
| 161 |
+
|
| 162 |
+
calls = {'n': 0, 'rows': 0}
|
| 163 |
+
if args.planner in ('controller', 'cem'):
|
| 164 |
+
# the central claim is cost, so count predictor forwards rather than
|
| 165 |
+
# asserting them: one call = one batched latent transition
|
| 166 |
+
inner_predict = model.predictor.forward
|
| 167 |
+
|
| 168 |
+
def counting_predict(*a, **kw):
|
| 169 |
+
calls['n'] += 1
|
| 170 |
+
first = a[0] if a else next(iter(kw.values()))
|
| 171 |
+
calls['rows'] += first.shape[0]
|
| 172 |
+
return inner_predict(*a, **kw)
|
| 173 |
+
|
| 174 |
+
model.predictor.forward = counting_predict
|
| 175 |
+
|
| 176 |
+
# terminal latent distance at each replan -> CEM regret, comparable across
|
| 177 |
+
# planners because both are scored under the same frozen model. GoalMSE
|
| 178 |
+
# sums over the latent dim while the controller averages, so rescale to
|
| 179 |
+
# per-dim. Done by retyping the instance: __call__ is looked up on the
|
| 180 |
+
# type, and the policy isinstance-checks against the Solver protocol.
|
| 181 |
+
terminals = []
|
| 182 |
+
if args.planner in ('controller', 'cem'):
|
| 183 |
+
scale = 1.0 if args.planner == 'controller' else 1.0 / latent_dim
|
| 184 |
+
base = type(solver)
|
| 185 |
+
|
| 186 |
+
class RecordingSolver(base):
|
| 187 |
+
def __call__(self, info_dict, init_action=None):
|
| 188 |
+
out = base.__call__(self, info_dict, init_action)
|
| 189 |
+
costs = out.get('costs')
|
| 190 |
+
if costs is not None:
|
| 191 |
+
value = float(torch.as_tensor(costs).float().mean())
|
| 192 |
+
terminals.append(value * scale)
|
| 193 |
+
return out
|
| 194 |
+
|
| 195 |
+
solver.__class__ = RecordingSolver
|
| 196 |
+
|
| 197 |
+
if args.planner == 'random':
|
| 198 |
+
# RandomPolicy is a policy itself, not a solver — no WorldModelPolicy
|
| 199 |
+
# wrapper, no history plumbing, no world-model calls.
|
| 200 |
+
policy = solver
|
| 201 |
+
else:
|
| 202 |
+
policy_cls = (
|
| 203 |
+
swm.policy.WorldModelPolicy if args.no_history else HistoryPolicy
|
| 204 |
+
)
|
| 205 |
+
policy = policy_cls(
|
| 206 |
+
solver=solver,
|
| 207 |
+
config=config,
|
| 208 |
+
process=process,
|
| 209 |
+
transform=transform,
|
| 210 |
+
history_keys=('pixels',),
|
| 211 |
+
)
|
| 212 |
+
world.set_policy(policy)
|
| 213 |
+
|
| 214 |
+
# held-out start/goal pairs, identical across planners for a fair compare
|
| 215 |
+
names = set(dataset.column_names) | set(getattr(dataset, '_schema_names', ()))
|
| 216 |
+
ep_col = 'episode_idx' if 'episode_idx' in names else 'ep_idx'
|
| 217 |
+
ep_idx = dataset.get_col_data(ep_col)
|
| 218 |
+
step_idx = dataset.get_col_data('step_idx')
|
| 219 |
+
episodes = np.unique(ep_idx)
|
| 220 |
+
lengths = {e: step_idx[ep_idx == e].max() + 1 for e in episodes}
|
| 221 |
+
max_start = np.array([lengths[e] for e in ep_idx]) - args.goal_offset - 1
|
| 222 |
+
valid = np.nonzero(step_idx <= max_start)[0]
|
| 223 |
+
|
| 224 |
+
rng = np.random.default_rng(args.seed)
|
| 225 |
+
picked = np.sort(valid[rng.choice(len(valid), args.num_eval, replace=False)])
|
| 226 |
+
|
| 227 |
+
out_dir = Path(args.out)
|
| 228 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 229 |
+
|
| 230 |
+
t0 = time.time()
|
| 231 |
+
metrics = world.evaluate(
|
| 232 |
+
dataset=dataset,
|
| 233 |
+
start_steps=step_idx[picked].tolist(),
|
| 234 |
+
goal_offset=args.goal_offset,
|
| 235 |
+
eval_budget=args.eval_budget,
|
| 236 |
+
episodes_idx=ep_idx[picked].tolist(),
|
| 237 |
+
callables=CALLABLES,
|
| 238 |
+
video=out_dir if args.video else None,
|
| 239 |
+
)
|
| 240 |
+
elapsed = time.time() - t0
|
| 241 |
+
|
| 242 |
+
result = {
|
| 243 |
+
'planner': tag,
|
| 244 |
+
'env': args.env_name,
|
| 245 |
+
'wm': args.wm_name,
|
| 246 |
+
'receding_horizon': args.receding_horizon,
|
| 247 |
+
'success_rate': float(metrics['success_rate']),
|
| 248 |
+
'seconds': elapsed,
|
| 249 |
+
'seconds_per_episode': elapsed / args.num_eval,
|
| 250 |
+
'mean_terminal_distance': (
|
| 251 |
+
float(np.mean(terminals)) if terminals else None
|
| 252 |
+
),
|
| 253 |
+
# Averaging over solver calls is not comparable across execution
|
| 254 |
+
# lengths: a run that replans every block makes more calls, and the
|
| 255 |
+
# later ones are taken nearer the goal. The first call is taken from
|
| 256 |
+
# the same held-out state by every planner, so that one is.
|
| 257 |
+
'first_terminal_distance': float(terminals[0]) if terminals else None,
|
| 258 |
+
'terminal_distance_trace': [float(t) for t in terminals],
|
| 259 |
+
'predictor_calls': calls['n'],
|
| 260 |
+
'predictor_rows_per_episode': calls['rows'] / args.num_eval,
|
| 261 |
+
'num_eval': args.num_eval,
|
| 262 |
+
'eval_budget': args.eval_budget,
|
| 263 |
+
'goal_offset': args.goal_offset,
|
| 264 |
+
'checkpoint': args.controller if args.planner == 'controller' else None,
|
| 265 |
+
'seed': args.seed,
|
| 266 |
+
'refinements': args.refinements,
|
| 267 |
+
'history': not args.no_history,
|
| 268 |
+
# per-episode outcomes: all rows share start/goal pairs, so planner
|
| 269 |
+
# comparisons must be paired rather than treated as independent
|
| 270 |
+
'episode_successes': [
|
| 271 |
+
bool(x) for x in metrics['episode_successes'].tolist()
|
| 272 |
+
],
|
| 273 |
+
}
|
| 274 |
+
print(json.dumps({k: v for k, v in result.items()
|
| 275 |
+
if k != 'terminal_distance_trace'}, indent=2))
|
| 276 |
+
|
| 277 |
+
with (out_dir / 'results.jsonl').open('a') as f:
|
| 278 |
+
f.write(json.dumps(result) + '\n')
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
if __name__ == '__main__':
|
| 282 |
+
main()
|
code/scripts/paired_stats.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Paired comparison between eval rows.
|
| 2 |
+
|
| 3 |
+
Every row in ``results.jsonl`` uses the same held-out start/goal pairs, so
|
| 4 |
+
success rates must be compared per-episode rather than as two independent
|
| 5 |
+
binomials. With 50 episodes one flipped episode moves the rate by 2 points,
|
| 6 |
+
which is well inside the noise a naive comparison would ignore.
|
| 7 |
+
|
| 8 |
+
McNemar's exact test uses only the discordant episodes (the ones where the
|
| 9 |
+
two planners disagree), which is the right conditioning for paired binary
|
| 10 |
+
outcomes. The bootstrap CI resamples episodes to put an interval on the
|
| 11 |
+
difference.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import argparse
|
| 15 |
+
import json
|
| 16 |
+
from math import comb
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
import numpy as np
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def mcnemar_exact(a, b):
|
| 23 |
+
"""Two-sided exact McNemar p-value for paired binary outcomes."""
|
| 24 |
+
only_a = int(np.sum(a & ~b))
|
| 25 |
+
only_b = int(np.sum(b & ~a))
|
| 26 |
+
n = only_a + only_b
|
| 27 |
+
if n == 0:
|
| 28 |
+
return 1.0, only_a, only_b
|
| 29 |
+
k = min(only_a, only_b)
|
| 30 |
+
tail = sum(comb(n, i) for i in range(k + 1)) / 2**n
|
| 31 |
+
return min(1.0, 2 * tail), only_a, only_b
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def bootstrap_ci(a, b, reps=20000, seed=0):
|
| 35 |
+
"""Percentile CI on the paired success-rate difference, in points."""
|
| 36 |
+
rng = np.random.default_rng(seed)
|
| 37 |
+
idx = rng.integers(0, len(a), size=(reps, len(a)))
|
| 38 |
+
diffs = (a[idx].mean(axis=1) - b[idx].mean(axis=1)) * 100
|
| 39 |
+
return np.percentile(diffs, [2.5, 97.5])
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def label(row):
|
| 43 |
+
tag = row['planner']
|
| 44 |
+
if row.get('receding_horizon', 1) != 1:
|
| 45 |
+
tag += f'+exec{row["receding_horizon"]}'
|
| 46 |
+
ckpt = row.get('checkpoint') or ''
|
| 47 |
+
# exp7 cell names first (fused192_s2/base_r2 before their shorter
|
| 48 |
+
# prefixes so a checkpoint doesn't also pick up an unrelated substring)
|
| 49 |
+
for name in ('fused192_s2', 'fused192', 'fused256', 'w192np_split',
|
| 50 |
+
'base_r2', 'terminal_only', 'no_support', 'ah_hold0.5',
|
| 51 |
+
'ah_hold0.0', 'ah_hold1.0'):
|
| 52 |
+
if name in ckpt:
|
| 53 |
+
tag += f'[{name}]'
|
| 54 |
+
break
|
| 55 |
+
return tag
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def main():
|
| 59 |
+
p = argparse.ArgumentParser()
|
| 60 |
+
p.add_argument('--results', default='data/runs/eval/results.jsonl')
|
| 61 |
+
p.add_argument('--pairs', nargs='*', default=None,
|
| 62 |
+
help='LABEL_A vs LABEL_B, as "A::B" strings')
|
| 63 |
+
args = p.parse_args()
|
| 64 |
+
|
| 65 |
+
rows = [
|
| 66 |
+
json.loads(x)
|
| 67 |
+
for x in Path(args.results).read_text().splitlines()
|
| 68 |
+
if x.strip()
|
| 69 |
+
]
|
| 70 |
+
rows = [r for r in rows if r.get('episode_successes')]
|
| 71 |
+
if not rows:
|
| 72 |
+
print('no rows carry per-episode outcomes yet — rerun evals')
|
| 73 |
+
return
|
| 74 |
+
|
| 75 |
+
# Pool across eval seeds: multiple rows can share one label (same
|
| 76 |
+
# checkpoint + rh, different --seed), and the pairing is valid *within*
|
| 77 |
+
# each seed (identical held-out start/goal pairs across planners at that
|
| 78 |
+
# seed) — so seed order must match between any two labels being compared.
|
| 79 |
+
# A plain dict keyed by label would instead let the last seed's row
|
| 80 |
+
# silently overwrite the earlier ones.
|
| 81 |
+
grouped = {}
|
| 82 |
+
for r in rows:
|
| 83 |
+
grouped.setdefault(label(r), []).append(
|
| 84 |
+
(r.get('seed', 0), np.array(r['episode_successes'], dtype=bool))
|
| 85 |
+
)
|
| 86 |
+
by_label = {}
|
| 87 |
+
for lbl, items in grouped.items():
|
| 88 |
+
items.sort(key=lambda item: item[0])
|
| 89 |
+
by_label[lbl] = np.concatenate([arr for _, arr in items])
|
| 90 |
+
|
| 91 |
+
print('available rows with paired outcomes:')
|
| 92 |
+
for k, v in by_label.items():
|
| 93 |
+
print(f' {k:<40} {v.mean() * 100:5.1f}% (n={len(v)})')
|
| 94 |
+
|
| 95 |
+
pairs = []
|
| 96 |
+
if args.pairs:
|
| 97 |
+
for spec in args.pairs:
|
| 98 |
+
a, b = spec.split('::')
|
| 99 |
+
pairs.append((a, b))
|
| 100 |
+
else:
|
| 101 |
+
keys = list(by_label)
|
| 102 |
+
pairs = [(keys[i], keys[j])
|
| 103 |
+
for i in range(len(keys)) for j in range(i + 1, len(keys))]
|
| 104 |
+
|
| 105 |
+
print(f'\n{"comparison":<62}{"diff":>8}{"95% CI":>18}{"p":>9}')
|
| 106 |
+
print('-' * 97)
|
| 107 |
+
for a, b in pairs:
|
| 108 |
+
if a not in by_label or b not in by_label:
|
| 109 |
+
print(f' skip {a} vs {b}: missing')
|
| 110 |
+
continue
|
| 111 |
+
sa, sb = by_label[a], by_label[b]
|
| 112 |
+
if len(sa) != len(sb):
|
| 113 |
+
print(f' skip {a} vs {b}: different episode counts')
|
| 114 |
+
continue
|
| 115 |
+
diff = (sa.mean() - sb.mean()) * 100
|
| 116 |
+
lo, hi = bootstrap_ci(sa, sb)
|
| 117 |
+
pval, na, nb = mcnemar_exact(sa, sb)
|
| 118 |
+
star = '*' if pval < 0.05 else ' '
|
| 119 |
+
print(
|
| 120 |
+
f'{a + " vs " + b:<62}{diff:>+7.1f} '
|
| 121 |
+
f'{f"[{lo:+.1f}, {hi:+.1f}]":>17}{pval:>8.4f}{star}'
|
| 122 |
+
)
|
| 123 |
+
print('\n* p < 0.05 (exact McNemar, paired). diff is percentage points.')
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
if __name__ == '__main__':
|
| 127 |
+
main()
|
code/scripts/probe_alignment.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""E2 -- gradient alignment probe (exp7 explainability battery, sec 5).
|
| 2 |
+
|
| 3 |
+
Per refinement k, cosine between the controller's emitted update
|
| 4 |
+
``alpha_k * Delta Y_j`` and the true descent direction
|
| 5 |
+
``-grad_{Y_j} sum_i d_i``, obtained by autograd through the *frozen rollout
|
| 6 |
+
only* (never through the controller's own network). This directly tests the
|
| 7 |
+
"amortized descent" story -- does the controller behave like an unrolled
|
| 8 |
+
gradient step? -- and works unchanged on both the split and fused
|
| 9 |
+
architectures, since it only calls the controller's public forward pieces
|
| 10 |
+
(``condition`` / ``initial_plan`` / ``refine`` / ``to_actions``).
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import argparse
|
| 14 |
+
import json
|
| 15 |
+
import sys
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
import numpy as np
|
| 19 |
+
import torch
|
| 20 |
+
import torch.nn.functional as F
|
| 21 |
+
|
| 22 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 23 |
+
|
| 24 |
+
from lejepa_control.data import LatentGoalDataset, split_episodes # noqa: E402
|
| 25 |
+
from lejepa_control.rollout import goal_distance, rollout_plan # noqa: E402
|
| 26 |
+
from lejepa_control.solver import load_controller # noqa: E402
|
| 27 |
+
from lejepa_control.world_model import load_lewm # noqa: E402
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def parse_args():
|
| 31 |
+
p = argparse.ArgumentParser()
|
| 32 |
+
p.add_argument('--controller', default='data/runs/controller/controller.pt')
|
| 33 |
+
p.add_argument('--latents', default='data/latents')
|
| 34 |
+
p.add_argument('--samples', type=int, default=256)
|
| 35 |
+
p.add_argument('--horizon', type=int, default=5)
|
| 36 |
+
p.add_argument('--tag', default='main')
|
| 37 |
+
p.add_argument('--out', default='data/runs/probe_alignment')
|
| 38 |
+
return p.parse_args()
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def descent_direction(controller, model, tokens, ctx, past, goal):
|
| 42 |
+
"""``-grad_Y sum_i d_i``: true descent, autograd through the frozen
|
| 43 |
+
rollout only -- ``tokens`` is detached first, so no gradient reaches the
|
| 44 |
+
controller's own network here, only the path token -> action -> rollout.
|
| 45 |
+
"""
|
| 46 |
+
y = tokens.detach().clone().requires_grad_(True)
|
| 47 |
+
pred = rollout_plan(model, ctx, past, controller.to_actions(y))
|
| 48 |
+
loss = goal_distance(pred, goal).sum()
|
| 49 |
+
(grad,) = torch.autograd.grad(loss, y)
|
| 50 |
+
return -grad
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def alignment_trace(controller, model, ctx, past, goal):
|
| 54 |
+
"""Per-refinement cosine between the emitted update and true descent.
|
| 55 |
+
|
| 56 |
+
Mirrors ``IterativeController.forward``'s loop exactly (same ``cond``,
|
| 57 |
+
same ``tokens`` recursion), but at each step also scores the emitted
|
| 58 |
+
update against the gradient-through-rollout direction at that same
|
| 59 |
+
``Y_k``. Returns a ``(K, B, H)`` cosine tensor.
|
| 60 |
+
"""
|
| 61 |
+
with torch.no_grad():
|
| 62 |
+
cond = controller.condition(ctx, goal)
|
| 63 |
+
tokens = controller.initial_plan(cond)
|
| 64 |
+
|
| 65 |
+
cosines = []
|
| 66 |
+
for k in range(controller.refinements):
|
| 67 |
+
with torch.no_grad():
|
| 68 |
+
actions = controller.to_actions(tokens)
|
| 69 |
+
pred = rollout_plan(model, ctx, past, actions)
|
| 70 |
+
|
| 71 |
+
true_dir = descent_direction(controller, model, tokens, ctx, past, goal)
|
| 72 |
+
|
| 73 |
+
with torch.no_grad():
|
| 74 |
+
delta = controller.refine(tokens, cond, pred, goal)
|
| 75 |
+
idx = min(k, controller.step_logit.numel() - 1)
|
| 76 |
+
step = torch.sigmoid(controller.step_logit[idx])
|
| 77 |
+
emitted = step * delta # (B, H, W): the actual applied update
|
| 78 |
+
|
| 79 |
+
cos = F.cosine_similarity(emitted, true_dir, dim=-1) # (B, H)
|
| 80 |
+
cosines.append(cos)
|
| 81 |
+
|
| 82 |
+
tokens = tokens + emitted
|
| 83 |
+
|
| 84 |
+
return torch.stack(cosines) # (K, B, H)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def main():
|
| 88 |
+
args = parse_args()
|
| 89 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 90 |
+
torch.manual_seed(0)
|
| 91 |
+
|
| 92 |
+
stats = json.loads((Path(args.latents) / 'stats.json').read_text())
|
| 93 |
+
model = load_lewm(device=device)
|
| 94 |
+
controller, ckpt = load_controller(
|
| 95 |
+
args.controller, latent_dim=stats['latent_dim'], device=device
|
| 96 |
+
)
|
| 97 |
+
controller.eval()
|
| 98 |
+
print(f'controller step {ckpt["step"]}, K={controller.refinements}, '
|
| 99 |
+
f'fused={controller.fused}, no_latent_proj={controller.no_latent_proj}')
|
| 100 |
+
|
| 101 |
+
_, val_eps = split_episodes(stats['n_episodes'])
|
| 102 |
+
val = LatentGoalDataset(
|
| 103 |
+
args.latents, max_offset=5, episodes=val_eps, horizon=args.horizon
|
| 104 |
+
)
|
| 105 |
+
idx = np.random.default_rng(0).choice(len(val), args.samples, replace=False)
|
| 106 |
+
batch = {
|
| 107 |
+
k: torch.stack([val[int(i)][k] for i in idx]).to(device)
|
| 108 |
+
for k in ('context', 'past_actions', 'goal')
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
cos = alignment_trace(
|
| 112 |
+
controller, model, batch['context'], batch['past_actions'], batch['goal']
|
| 113 |
+
) # (K, B, H)
|
| 114 |
+
|
| 115 |
+
per_k = cos.mean(dim=(1, 2))
|
| 116 |
+
per_k_std = cos.std(dim=(1, 2))
|
| 117 |
+
print('\n=== E2: gradient alignment (emitted update vs true descent) ===')
|
| 118 |
+
print(f'{"k":>3}{"mean cos":>11}{"std":>9}')
|
| 119 |
+
for k in range(cos.size(0)):
|
| 120 |
+
print(f'{k:>3}{per_k[k].item():>11.4f}{per_k_std[k].item():>9.4f}')
|
| 121 |
+
|
| 122 |
+
report = {
|
| 123 |
+
'tag': args.tag,
|
| 124 |
+
'checkpoint': args.controller,
|
| 125 |
+
'step': int(ckpt['step']),
|
| 126 |
+
'fused': bool(controller.fused),
|
| 127 |
+
'no_latent_proj': bool(controller.no_latent_proj),
|
| 128 |
+
'mean_cosine_per_k': [round(float(v), 5) for v in per_k],
|
| 129 |
+
'std_cosine_per_k': [round(float(v), 5) for v in per_k_std],
|
| 130 |
+
}
|
| 131 |
+
out = Path(args.out)
|
| 132 |
+
out.mkdir(parents=True, exist_ok=True)
|
| 133 |
+
with (out / 'probe_alignment.jsonl').open('a') as f:
|
| 134 |
+
f.write(json.dumps(report) + '\n')
|
| 135 |
+
print(f'\nwrote {out / "probe_alignment.jsonl"}')
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
if __name__ == '__main__':
|
| 139 |
+
main()
|
code/scripts/probe_attention.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""E4 -- attention readout probe (exp7 explainability battery, sec 5).
|
| 2 |
+
|
| 3 |
+
Temporarily swaps each transformer Block's bound ``forward`` for a version
|
| 4 |
+
that also asks its ``MultiheadAttention`` for weights (``need_weights=True``,
|
| 5 |
+
which the default ``need_weights=False`` forward never computes), for the
|
| 6 |
+
duration of one probe pass, then restores the originals -- so the
|
| 7 |
+
controller's real forward path (and every existing checkpoint/test) is
|
| 8 |
+
untouched.
|
| 9 |
+
|
| 10 |
+
Reports mass from plan-slot queries onto four groups -- self, other plan
|
| 11 |
+
slots, context frames, goal token -- per layer and per refinement. Works
|
| 12 |
+
unchanged on the split architecture (two named nets, ``consequence_net``
|
| 13 |
+
then ``refine_net`` per refine() call) and the fused one (one ``net`` per
|
| 14 |
+
call), so the "one coherent map vs. smeared across two networks" comparison
|
| 15 |
+
in sec 5/E4 is literally the same script pointed at two checkpoints.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
import argparse
|
| 19 |
+
import json
|
| 20 |
+
import sys
|
| 21 |
+
import types
|
| 22 |
+
from pathlib import Path
|
| 23 |
+
|
| 24 |
+
import numpy as np
|
| 25 |
+
import torch
|
| 26 |
+
|
| 27 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 28 |
+
|
| 29 |
+
from lejepa_control.data import LatentGoalDataset, split_episodes # noqa: E402
|
| 30 |
+
from lejepa_control.rollout import rollout_plan # noqa: E402
|
| 31 |
+
from lejepa_control.solver import load_controller # noqa: E402
|
| 32 |
+
from lejepa_control.world_model import load_lewm # noqa: E402
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def parse_args():
|
| 36 |
+
p = argparse.ArgumentParser()
|
| 37 |
+
p.add_argument('--controller', default='data/runs/controller/controller.pt')
|
| 38 |
+
p.add_argument('--latents', default='data/latents')
|
| 39 |
+
p.add_argument('--samples', type=int, default=128)
|
| 40 |
+
p.add_argument('--horizon', type=int, default=5)
|
| 41 |
+
p.add_argument('--tag', default='main')
|
| 42 |
+
p.add_argument('--out', default='data/runs/probe_attention')
|
| 43 |
+
return p.parse_args()
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _capturing_forward(cap_list):
|
| 47 |
+
"""A ``Block.forward`` that also records self-attention weights."""
|
| 48 |
+
|
| 49 |
+
def forward(self, x):
|
| 50 |
+
h = self.norm1(x)
|
| 51 |
+
attn_out, w = self.attn(
|
| 52 |
+
h, h, h, need_weights=True, average_attn_weights=True
|
| 53 |
+
)
|
| 54 |
+
cap_list.append(w.detach()) # (B, L, L), already averaged over heads
|
| 55 |
+
x = x + attn_out
|
| 56 |
+
return x + self.mlp(self.norm2(x))
|
| 57 |
+
|
| 58 |
+
return forward
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
class AttentionCapture:
|
| 62 |
+
"""Patches every ``Block`` under ``root`` to log attention weights in
|
| 63 |
+
call order, for the lifetime of the ``with`` block. Blocks that never
|
| 64 |
+
run during the wrapped call (e.g. ``consequence_net`` during
|
| 65 |
+
``initial_plan``) simply never append -- no separate bookkeeping needed
|
| 66 |
+
to know which net actually fired.
|
| 67 |
+
"""
|
| 68 |
+
|
| 69 |
+
def __init__(self, root):
|
| 70 |
+
self.blocks = [m for m in root.modules() if type(m).__name__ == 'Block']
|
| 71 |
+
self.log = []
|
| 72 |
+
self._originals = []
|
| 73 |
+
|
| 74 |
+
def __enter__(self):
|
| 75 |
+
for block in self.blocks:
|
| 76 |
+
self._originals.append((block, block.forward))
|
| 77 |
+
block.forward = types.MethodType(_capturing_forward(self.log), block)
|
| 78 |
+
return self
|
| 79 |
+
|
| 80 |
+
def __exit__(self, *exc):
|
| 81 |
+
for block, orig in self._originals:
|
| 82 |
+
block.forward = orig
|
| 83 |
+
return False
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def mass_by_group(weights, horizon, num_context):
|
| 87 |
+
"""``weights``: (B, L, L) self-attention, L = horizon + num_context + 1,
|
| 88 |
+
sequence laid out as [plan slots, context frames, goal] (matching
|
| 89 |
+
``condition()`` / ``_run_refine`` / ``_run_fused``). Returns group mass
|
| 90 |
+
averaged over batch and over the ``horizon`` plan-slot query rows.
|
| 91 |
+
"""
|
| 92 |
+
H = horizon
|
| 93 |
+
rows = weights[:, :H] # (B, H, L): only plan-slot queries
|
| 94 |
+
diag = rows[:, torch.arange(H), torch.arange(H)].mean()
|
| 95 |
+
plan_total = rows[:, :, :H].sum(-1).mean()
|
| 96 |
+
ctx_mass = rows[:, :, H:H + num_context].sum(-1).mean()
|
| 97 |
+
goal_mass = rows[:, :, H + num_context].mean()
|
| 98 |
+
return {
|
| 99 |
+
'self': float(diag),
|
| 100 |
+
'other_plan': float(plan_total - diag),
|
| 101 |
+
'context': float(ctx_mass),
|
| 102 |
+
'goal': float(goal_mass),
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def label_layers(log, fused, depth):
|
| 107 |
+
"""Positionally names each captured layer by which net produced it.
|
| 108 |
+
|
| 109 |
+
Call order is deterministic from ``controller.py``: ``initial_plan``
|
| 110 |
+
only runs the "refine" net; ``refine()`` runs consequence-then-refine
|
| 111 |
+
(split) or the single fused net (fused) -- so position alone identifies
|
| 112 |
+
the source net, no encoder-identity bookkeeping required.
|
| 113 |
+
"""
|
| 114 |
+
if fused:
|
| 115 |
+
return [('net', i) for i in range(len(log))]
|
| 116 |
+
if len(log) == depth: # this was an initial_plan() call
|
| 117 |
+
return [('refine_net', i) for i in range(len(log))]
|
| 118 |
+
return (
|
| 119 |
+
[('consequence_net', i) for i in range(depth)]
|
| 120 |
+
+ [('refine_net', i) for i in range(depth)]
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
@torch.no_grad()
|
| 125 |
+
def run_probe(controller, model, ctx, past, goal, depth):
|
| 126 |
+
"""Replays ``IterativeController.forward``'s loop, capturing attention
|
| 127 |
+
inside every net call at every refinement. Returns a list (per
|
| 128 |
+
iteration k = 0..K) of ``{(net_name, layer): group_mass_dict}``.
|
| 129 |
+
"""
|
| 130 |
+
H, N = controller.horizon, controller.num_context
|
| 131 |
+
cond = controller.condition(ctx, goal)
|
| 132 |
+
|
| 133 |
+
per_iter = []
|
| 134 |
+
with AttentionCapture(controller) as cap:
|
| 135 |
+
tokens = controller.initial_plan(cond)
|
| 136 |
+
layers = label_layers(cap.log, controller.fused, depth)
|
| 137 |
+
per_iter.append({
|
| 138 |
+
layer: mass_by_group(w, H, N) for layer, w in zip(layers, cap.log)
|
| 139 |
+
})
|
| 140 |
+
|
| 141 |
+
for k in range(controller.refinements):
|
| 142 |
+
actions = controller.to_actions(tokens)
|
| 143 |
+
pred = rollout_plan(model, ctx, past, actions)
|
| 144 |
+
with AttentionCapture(controller) as cap:
|
| 145 |
+
delta = controller.refine(tokens, cond, pred, goal)
|
| 146 |
+
layers = label_layers(cap.log, controller.fused, depth)
|
| 147 |
+
per_iter.append({
|
| 148 |
+
layer: mass_by_group(w, H, N) for layer, w in zip(layers, cap.log)
|
| 149 |
+
})
|
| 150 |
+
idx = min(k, controller.step_logit.numel() - 1)
|
| 151 |
+
tokens = tokens + torch.sigmoid(controller.step_logit[idx]) * delta
|
| 152 |
+
|
| 153 |
+
return per_iter
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def main():
|
| 157 |
+
args = parse_args()
|
| 158 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 159 |
+
torch.manual_seed(0)
|
| 160 |
+
|
| 161 |
+
stats = json.loads((Path(args.latents) / 'stats.json').read_text())
|
| 162 |
+
model = load_lewm(device=device)
|
| 163 |
+
controller, ckpt = load_controller(
|
| 164 |
+
args.controller, latent_dim=stats['latent_dim'], device=device
|
| 165 |
+
)
|
| 166 |
+
controller.eval()
|
| 167 |
+
depth = ckpt['args']['depth']
|
| 168 |
+
print(f'controller step {ckpt["step"]}, K={controller.refinements}, '
|
| 169 |
+
f'fused={controller.fused}, depth={depth}')
|
| 170 |
+
|
| 171 |
+
_, val_eps = split_episodes(stats['n_episodes'])
|
| 172 |
+
val = LatentGoalDataset(
|
| 173 |
+
args.latents, max_offset=5, episodes=val_eps, horizon=args.horizon
|
| 174 |
+
)
|
| 175 |
+
idx = np.random.default_rng(0).choice(len(val), args.samples, replace=False)
|
| 176 |
+
batch = {
|
| 177 |
+
k: torch.stack([val[int(i)][k] for i in idx]).to(device)
|
| 178 |
+
for k in ('context', 'past_actions', 'goal')
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
per_iter = run_probe(
|
| 182 |
+
controller, model, batch['context'], batch['past_actions'], batch['goal'],
|
| 183 |
+
depth,
|
| 184 |
+
)
|
| 185 |
+
|
| 186 |
+
print('\n=== E4: attention mass from plan-slot queries ===')
|
| 187 |
+
print(f'{"k":>3}{"net":>16}{"layer":>6}{"self":>8}{"plan":>8}'
|
| 188 |
+
f'{"ctx":>8}{"goal":>8}')
|
| 189 |
+
for k, layers in enumerate(per_iter):
|
| 190 |
+
for (net, layer), g in layers.items():
|
| 191 |
+
print(f'{k:>3}{net:>16}{layer:>6}{g["self"]:>8.3f}'
|
| 192 |
+
f'{g["other_plan"]:>8.3f}{g["context"]:>8.3f}{g["goal"]:>8.3f}')
|
| 193 |
+
|
| 194 |
+
report = {
|
| 195 |
+
'tag': args.tag,
|
| 196 |
+
'checkpoint': args.controller,
|
| 197 |
+
'step': int(ckpt['step']),
|
| 198 |
+
'fused': bool(controller.fused),
|
| 199 |
+
'per_iteration': [
|
| 200 |
+
{f'{net}.{layer}': g for (net, layer), g in layers.items()}
|
| 201 |
+
for layers in per_iter
|
| 202 |
+
],
|
| 203 |
+
}
|
| 204 |
+
out = Path(args.out)
|
| 205 |
+
out.mkdir(parents=True, exist_ok=True)
|
| 206 |
+
with (out / 'probe_attention.jsonl').open('a') as f:
|
| 207 |
+
f.write(json.dumps(report) + '\n')
|
| 208 |
+
print(f'\nwrote {out / "probe_attention.jsonl"}')
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
if __name__ == '__main__':
|
| 212 |
+
main()
|
code/scripts/probe_contract.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Verify the frozen-LeWM contract the controller depends on.
|
| 2 |
+
|
| 3 |
+
Checks the action-block layout, the normalization the checkpoint was trained
|
| 4 |
+
with, and that gradients reach candidate actions through a multi-step rollout.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import sys
|
| 8 |
+
import time
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
import h5py
|
| 12 |
+
import numpy as np
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 16 |
+
|
| 17 |
+
from lejepa_control.world_model import load_lewm # noqa: E402
|
| 18 |
+
|
| 19 |
+
H5 = Path('data/swm_home/datasets/pusht_expert_train.h5')
|
| 20 |
+
IMAGENET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
|
| 21 |
+
IMAGENET_STD = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def preprocess(frames_hwc: np.ndarray) -> torch.Tensor:
|
| 25 |
+
x = torch.from_numpy(frames_hwc).permute(0, 3, 1, 2).float() / 255.0
|
| 26 |
+
return (x - IMAGENET_MEAN) / IMAGENET_STD
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def main():
|
| 30 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 31 |
+
model = load_lewm(device=device)
|
| 32 |
+
D = model.predictor.input_dim
|
| 33 |
+
print(f'num_frames={model.predictor.num_frames} '
|
| 34 |
+
f'action_input_dim={model.action_encoder.input_dim} D={D}')
|
| 35 |
+
|
| 36 |
+
f = h5py.File(H5, 'r')
|
| 37 |
+
action = f['action']
|
| 38 |
+
offs, lens = f['ep_offset'][:], f['ep_len'][:]
|
| 39 |
+
|
| 40 |
+
# --- action normalization the checkpoint was trained with -------------
|
| 41 |
+
stats = action[:]
|
| 42 |
+
mean, std = stats.mean(0), stats.std(0)
|
| 43 |
+
print(f'action mean={mean} std={std}')
|
| 44 |
+
print(f'raw [-1,1] -> normalized +-{(1.0 / std)}')
|
| 45 |
+
|
| 46 |
+
# --- frameskip / block layout ----------------------------------------
|
| 47 |
+
# A clip loads pixels strided by 5 and actions unstrided, reshaped (T, 10).
|
| 48 |
+
# So block k must be the 5 raw actions between frame k and frame k+1.
|
| 49 |
+
ep, start, fs, T = 0, 0, 5, 4
|
| 50 |
+
o = offs[ep]
|
| 51 |
+
raw = action[o + start : o + start + T * fs] # (20, 2)
|
| 52 |
+
block = raw.reshape(T, fs * 2) # (4, 10)
|
| 53 |
+
assert np.array_equal(block[1], raw[5:10].reshape(-1)), 'block layout'
|
| 54 |
+
print('block layout ok: block[k] = raw actions [5k, 5k+5)')
|
| 55 |
+
|
| 56 |
+
# --- encode a real 3-frame history + goal -----------------------------
|
| 57 |
+
N = model.predictor.num_frames
|
| 58 |
+
idx = [o + i * fs for i in range(N + 1)] # 3 history + 1 target
|
| 59 |
+
pixels = preprocess(f['pixels'][idx[0] : idx[-1] + 1 : fs])
|
| 60 |
+
pixels = pixels.unsqueeze(0).to(device) # (1, 4, 3, 224, 224)
|
| 61 |
+
with torch.no_grad():
|
| 62 |
+
emb = model.encode({'pixels': pixels})['emb']
|
| 63 |
+
print(f'emb {tuple(emb.shape)} mean={emb.mean():.4f} std={emb.std():.4f} '
|
| 64 |
+
f'per-coord std={emb.std(dim=(0, 1)).mean():.4f}')
|
| 65 |
+
|
| 66 |
+
# --- teacher forcing: does the frozen predictor actually predict? -----
|
| 67 |
+
ctx = emb[:, :N]
|
| 68 |
+
a_norm = (raw.reshape(T, fs, 2) - mean) / std
|
| 69 |
+
act = torch.from_numpy(a_norm.reshape(1, T, fs * 2)).float().to(device)
|
| 70 |
+
with torch.no_grad():
|
| 71 |
+
pred = model.predict(ctx, model.action_encoder(act[:, :N]))
|
| 72 |
+
tgt = emb[:, 1 : N + 1]
|
| 73 |
+
err = (pred - tgt).pow(2).mean().item()
|
| 74 |
+
# baseline: predicting "no change" (copy the last context frame)
|
| 75 |
+
copy_err = (ctx - tgt).pow(2).mean().item()
|
| 76 |
+
print(f'1-step pred MSE={err:.5f} copy-last-frame MSE={copy_err:.5f}')
|
| 77 |
+
|
| 78 |
+
# --- multi-step differentiable rollout, gradient to candidate actions -
|
| 79 |
+
Hz = 5
|
| 80 |
+
cand = torch.zeros(1, Hz, fs * 2, device=device, requires_grad=True)
|
| 81 |
+
hist = list(emb[:, :N].detach().unbind(dim=1))
|
| 82 |
+
past = act[:, : N - 1] # blocks between the context frames
|
| 83 |
+
all_act = torch.cat([past, cand], dim=1) # (1, N-1+Hz, 10)
|
| 84 |
+
all_emb = model.action_encoder(all_act)
|
| 85 |
+
for t in range(Hz):
|
| 86 |
+
lo = max(0, N + t - N)
|
| 87 |
+
e = torch.stack(hist[lo:], dim=1)
|
| 88 |
+
a = all_emb[:, lo : N + t]
|
| 89 |
+
hist.append(model.predict(e, a)[:, -1])
|
| 90 |
+
x_H = hist[-1]
|
| 91 |
+
goal = emb[:, -1].detach()
|
| 92 |
+
loss = (x_H - goal).pow(2).sum() / D
|
| 93 |
+
loss.backward()
|
| 94 |
+
g = cand.grad
|
| 95 |
+
print(f'rollout ok: terminal d={loss.item():.5f}')
|
| 96 |
+
print(f'grad norm={g.norm():.5f} per-step={g.norm(dim=-1).squeeze().tolist()}')
|
| 97 |
+
assert g.norm() > 0, 'no gradient reached candidate actions'
|
| 98 |
+
assert g[0, -1].norm() > 0, 'last block got no gradient'
|
| 99 |
+
|
| 100 |
+
# --- encoder throughput (decides whether caching latents is viable) ---
|
| 101 |
+
batch = preprocess(f['pixels'][:256]).to(device)
|
| 102 |
+
with torch.no_grad():
|
| 103 |
+
for _ in range(2):
|
| 104 |
+
model.encode({'pixels': batch.unsqueeze(0)})
|
| 105 |
+
torch.cuda.synchronize()
|
| 106 |
+
t0 = time.perf_counter()
|
| 107 |
+
model.encode({'pixels': batch.unsqueeze(0)})
|
| 108 |
+
torch.cuda.synchronize()
|
| 109 |
+
dt = time.perf_counter() - t0
|
| 110 |
+
ips = 256 / dt
|
| 111 |
+
total = int(lens.sum())
|
| 112 |
+
print(f'encoder {ips:.0f} img/s -> {total} frames in {total / ips / 60:.1f} min')
|
| 113 |
+
print(f'latent cache fp16 = {total * D * 2 / 1e9:.2f} GB')
|
| 114 |
+
print('ALL CONTRACT CHECKS PASSED')
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
if __name__ == '__main__':
|
| 118 |
+
main()
|
code/scripts/probe_geometry.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""E3 -- token-latent geometry probe (exp7 explainability battery, sec 5).
|
| 2 |
+
|
| 3 |
+
Only meaningful in the 192-identity cells (A, C): with ``width ==
|
| 4 |
+
latent_dim``, plan tokens ``y_j`` live in literally the same coordinate
|
| 5 |
+
system as world-model latents, so they can be compared to ``x_hat_j`` (the
|
| 6 |
+
state block ``j`` actually causes) with a plain distance/cosine, and decoded
|
| 7 |
+
*directly* through the pixel decoder -- no analogous move exists in any
|
| 8 |
+
256-d variant, which is exactly what "identity embedding buys" (sec 1).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import json
|
| 13 |
+
import sys
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
import numpy as np
|
| 17 |
+
import torch
|
| 18 |
+
import torch.nn.functional as F
|
| 19 |
+
|
| 20 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 21 |
+
|
| 22 |
+
from lejepa_control.data import LatentGoalDataset, split_episodes # noqa: E402
|
| 23 |
+
from lejepa_control.decoder import load_decoder # noqa: E402
|
| 24 |
+
from lejepa_control.rollout import rollout_plan # noqa: E402
|
| 25 |
+
from lejepa_control.solver import load_controller # noqa: E402
|
| 26 |
+
from lejepa_control.world_model import load_lewm # noqa: E402
|
| 27 |
+
|
| 28 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
|
| 29 |
+
from decode_rollout import panel, to_uint8 # noqa: E402
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def parse_args():
|
| 33 |
+
p = argparse.ArgumentParser()
|
| 34 |
+
p.add_argument('--controller', default='data/runs/exp7/fused192/controller.pt')
|
| 35 |
+
p.add_argument('--decoder', default='data/runs/decoder/decoder.pt')
|
| 36 |
+
p.add_argument('--latents', default='data/latents')
|
| 37 |
+
p.add_argument('--samples', type=int, default=256)
|
| 38 |
+
p.add_argument('--horizon', type=int, default=5)
|
| 39 |
+
p.add_argument('--decode-episodes', type=int, default=3)
|
| 40 |
+
p.add_argument('--tag', default='main')
|
| 41 |
+
p.add_argument('--out', default='data/runs/probe_geometry')
|
| 42 |
+
return p.parse_args()
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@torch.no_grad()
|
| 46 |
+
def geometry_trace(controller, model, ctx, past, goal):
|
| 47 |
+
"""Per-refinement ``y_k`` (raw plan tokens), ``x_hat_k`` (the latent each
|
| 48 |
+
block causes), ``||y - x_hat||`` and cosine. ``K+1`` entries, same
|
| 49 |
+
recursion as ``IterativeController.forward``.
|
| 50 |
+
"""
|
| 51 |
+
cond = controller.condition(ctx, goal)
|
| 52 |
+
tokens = controller.initial_plan(cond)
|
| 53 |
+
|
| 54 |
+
ys, xhats, dists, coss = [], [], [], []
|
| 55 |
+
for k in range(controller.refinements + 1):
|
| 56 |
+
actions = controller.to_actions(tokens)
|
| 57 |
+
pred = rollout_plan(model, ctx, past, actions) # x_hat_j, (B, H, D)
|
| 58 |
+
|
| 59 |
+
ys.append(tokens.clone())
|
| 60 |
+
xhats.append(pred.clone())
|
| 61 |
+
dists.append((tokens - pred).norm(dim=-1)) # (B, H)
|
| 62 |
+
coss.append(F.cosine_similarity(tokens, pred, dim=-1)) # (B, H)
|
| 63 |
+
|
| 64 |
+
if k == controller.refinements:
|
| 65 |
+
break
|
| 66 |
+
delta = controller.refine(tokens, cond, pred, goal)
|
| 67 |
+
idx = min(k, controller.step_logit.numel() - 1)
|
| 68 |
+
tokens = tokens + torch.sigmoid(controller.step_logit[idx]) * delta
|
| 69 |
+
|
| 70 |
+
return torch.stack(ys), torch.stack(xhats), torch.stack(dists), torch.stack(coss)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@torch.no_grad()
|
| 74 |
+
def decode_panel(decoder, ys, sample_idx, out_path):
|
| 75 |
+
"""One panel per sample: rows = refinement k, cols = horizon slot j,
|
| 76 |
+
each cell = ``decoder(y_k[sample, j])`` -- what that plan slot "intends".
|
| 77 |
+
"""
|
| 78 |
+
rows = [to_uint8(decoder(ys[k, sample_idx])) for k in range(ys.size(0))]
|
| 79 |
+
labels = [f'k={k}' for k in range(ys.size(0))]
|
| 80 |
+
titles = [f'slot {j}' for j in range(ys.size(2))]
|
| 81 |
+
return panel(rows, labels, titles, out_path)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def main():
|
| 85 |
+
args = parse_args()
|
| 86 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 87 |
+
torch.manual_seed(0)
|
| 88 |
+
|
| 89 |
+
stats = json.loads((Path(args.latents) / 'stats.json').read_text())
|
| 90 |
+
model = load_lewm(device=device)
|
| 91 |
+
controller, ckpt = load_controller(
|
| 92 |
+
args.controller, latent_dim=stats['latent_dim'], device=device
|
| 93 |
+
)
|
| 94 |
+
controller.eval()
|
| 95 |
+
if not controller.no_latent_proj:
|
| 96 |
+
print(f'{args.controller}: no_latent_proj=False (width != latent_dim) '
|
| 97 |
+
'-- E3 is only defined for the 192-identity cells (A, C). Skipping.')
|
| 98 |
+
return
|
| 99 |
+
print(f'controller step {ckpt["step"]}, K={controller.refinements}, '
|
| 100 |
+
f'fused={controller.fused}')
|
| 101 |
+
|
| 102 |
+
_, val_eps = split_episodes(stats['n_episodes'])
|
| 103 |
+
val = LatentGoalDataset(
|
| 104 |
+
args.latents, max_offset=5, episodes=val_eps, horizon=args.horizon
|
| 105 |
+
)
|
| 106 |
+
idx = np.random.default_rng(0).choice(len(val), args.samples, replace=False)
|
| 107 |
+
batch = {
|
| 108 |
+
k: torch.stack([val[int(i)][k] for i in idx]).to(device)
|
| 109 |
+
for k in ('context', 'past_actions', 'goal')
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
ys, xhats, dists, coss = geometry_trace(
|
| 113 |
+
controller, model, batch['context'], batch['past_actions'], batch['goal']
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
per_k_dist = dists.mean(dim=(1, 2))
|
| 117 |
+
per_k_cos = coss.mean(dim=(1, 2))
|
| 118 |
+
print('\n=== E3: plan token vs. its own consequence, ||y_j - x_hat_j|| ===')
|
| 119 |
+
print(f'{"k":>3}{"mean ||.||":>12}{"mean cos":>11}')
|
| 120 |
+
for k in range(dists.size(0)):
|
| 121 |
+
print(f'{k:>3}{per_k_dist[k].item():>12.4f}{per_k_cos[k].item():>11.4f}')
|
| 122 |
+
|
| 123 |
+
report = {
|
| 124 |
+
'tag': args.tag,
|
| 125 |
+
'checkpoint': args.controller,
|
| 126 |
+
'step': int(ckpt['step']),
|
| 127 |
+
'fused': bool(controller.fused),
|
| 128 |
+
'mean_dist_per_k': [round(float(v), 5) for v in per_k_dist],
|
| 129 |
+
'mean_cos_per_k': [round(float(v), 5) for v in per_k_cos],
|
| 130 |
+
}
|
| 131 |
+
out = Path(args.out)
|
| 132 |
+
out.mkdir(parents=True, exist_ok=True)
|
| 133 |
+
with (out / 'probe_geometry.jsonl').open('a') as f:
|
| 134 |
+
f.write(json.dumps(report) + '\n')
|
| 135 |
+
print(f'wrote {out / "probe_geometry.jsonl"}')
|
| 136 |
+
|
| 137 |
+
if args.decode_episodes > 0 and Path(args.decoder).exists():
|
| 138 |
+
decoder, dec_ckpt = load_decoder(args.decoder, device=device)
|
| 139 |
+
latent_kind = dec_ckpt['meta']['latent_source']
|
| 140 |
+
if latent_kind != 'emb':
|
| 141 |
+
print(f"decoder trained on '{latent_kind}' latents, not 'emb' "
|
| 142 |
+
'-- skipping the direct plan-token decode.')
|
| 143 |
+
else:
|
| 144 |
+
for i in range(min(args.decode_episodes, ys.size(1))):
|
| 145 |
+
path = decode_panel(
|
| 146 |
+
decoder, ys, i, out / f'{args.tag}_sample{i}.png'
|
| 147 |
+
)
|
| 148 |
+
print(f' -> {path}')
|
| 149 |
+
elif args.decode_episodes > 0:
|
| 150 |
+
print(f'no decoder at {args.decoder} -- skipping the direct plan-token decode.')
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
if __name__ == '__main__':
|
| 154 |
+
main()
|
code/scripts/run_ablations.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Run the baselines and ablations listed in section 11 of the design doc.
|
| 2 |
+
|
| 3 |
+
Three of them change the objective, so they need their own training runs
|
| 4 |
+
(terminal-only, no-support). The rest only change how the trained controller
|
| 5 |
+
is executed, so they reuse the main checkpoint. Every sim evaluation uses the
|
| 6 |
+
same seed, so all rows share held-out initial-state/goal pairs.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import argparse
|
| 10 |
+
import subprocess
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
PY = sys.executable
|
| 15 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 16 |
+
|
| 17 |
+
# objective ablations: each needs its own controller
|
| 18 |
+
VARIANTS = {
|
| 19 |
+
'main': [], # alpha=0.05, lambda_support=0.01
|
| 20 |
+
'terminal_only': ['--alpha', '0'],
|
| 21 |
+
'no_support': ['--lambda-support', '0'],
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def run(cmd):
|
| 26 |
+
print(f'\n$ {" ".join(str(c) for c in cmd)}', flush=True)
|
| 27 |
+
# one bad row should not discard the hours of runs queued behind it
|
| 28 |
+
code = subprocess.run(cmd, cwd=ROOT).returncode
|
| 29 |
+
if code != 0:
|
| 30 |
+
print(f'!! exited {code}, continuing', flush=True)
|
| 31 |
+
return code == 0
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def parse_args():
|
| 35 |
+
p = argparse.ArgumentParser()
|
| 36 |
+
p.add_argument('--steps', type=int, default=20000)
|
| 37 |
+
p.add_argument('--batch-size', type=int, default=128)
|
| 38 |
+
p.add_argument('--lr', type=float, default=3e-4)
|
| 39 |
+
p.add_argument('--num-eval', type=int, default=50)
|
| 40 |
+
p.add_argument('--eval-budget', type=int, default=50)
|
| 41 |
+
p.add_argument('--out', default='data/runs')
|
| 42 |
+
p.add_argument('--skip-train', action='store_true')
|
| 43 |
+
p.add_argument('--skip-cem', action='store_true')
|
| 44 |
+
return p.parse_args()
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def main():
|
| 48 |
+
args = parse_args()
|
| 49 |
+
out = Path(args.out)
|
| 50 |
+
|
| 51 |
+
def evaluate(extra, tag_dir='eval'):
|
| 52 |
+
run(
|
| 53 |
+
[PY, 'scripts/eval_controller.py',
|
| 54 |
+
'--num-eval', str(args.num_eval),
|
| 55 |
+
'--eval-budget', str(args.eval_budget),
|
| 56 |
+
'--out', str(out / tag_dir), *extra]
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
# Everything that reuses the main checkpoint runs first: those are the
|
| 60 |
+
# headline numbers and cost minutes, while each objective ablation costs
|
| 61 |
+
# its own multi-hour training run.
|
| 62 |
+
|
| 63 |
+
# --- baseline: LeWM + CEM ---------------------------------------------
|
| 64 |
+
if not args.skip_cem:
|
| 65 |
+
evaluate(['--planner', 'cem'])
|
| 66 |
+
|
| 67 |
+
main_ckpt = ['--controller', str(out / 'controller' / 'controller.pt')]
|
| 68 |
+
|
| 69 |
+
# --- refinement count: K=0 (one-pass) through K=5 ---------------------
|
| 70 |
+
for k in (0, 1, 2, 3, 5):
|
| 71 |
+
evaluate([*main_ckpt, '--refinements', str(k)])
|
| 72 |
+
|
| 73 |
+
# --- execute one block vs the whole plan ------------------------------
|
| 74 |
+
evaluate([*main_ckpt, '--receding-horizon', '5'])
|
| 75 |
+
|
| 76 |
+
# --- objective ablations need their own training runs -----------------
|
| 77 |
+
for name, extra in VARIANTS.items():
|
| 78 |
+
if name == 'main':
|
| 79 |
+
continue
|
| 80 |
+
ckpt_dir = out / f'abl_{name}'
|
| 81 |
+
if not args.skip_train:
|
| 82 |
+
run(
|
| 83 |
+
[PY, 'scripts/train_controller.py',
|
| 84 |
+
'--steps', str(args.steps),
|
| 85 |
+
'--batch-size', str(args.batch_size),
|
| 86 |
+
'--lr', str(args.lr),
|
| 87 |
+
'--log-every', '500', '--val-every', '5000',
|
| 88 |
+
'--out', str(ckpt_dir), *extra]
|
| 89 |
+
)
|
| 90 |
+
ckpt = ckpt_dir / 'controller.pt'
|
| 91 |
+
if ckpt.exists():
|
| 92 |
+
evaluate(['--controller', str(ckpt)])
|
| 93 |
+
|
| 94 |
+
print(f'\nall results appended to {out / "eval" / "results.jsonl"}')
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
if __name__ == '__main__':
|
| 98 |
+
main()
|
code/scripts/smoke_wm.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Verify the frozen LeWM loads, encodes, and gives gradients w.r.t. actions."""
|
| 2 |
+
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 9 |
+
|
| 10 |
+
from lejepa_control.world_model import load_lewm # noqa: E402
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def main():
|
| 14 |
+
import stable_worldmodel as swm
|
| 15 |
+
|
| 16 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 17 |
+
model = load_lewm(device=device)
|
| 18 |
+
print('loaded LeWM ok')
|
| 19 |
+
print(' predictor.num_frames =', model.predictor.num_frames)
|
| 20 |
+
print(' action_encoder.input_dim =', model.action_encoder.input_dim)
|
| 21 |
+
|
| 22 |
+
ds = swm.data.load_dataset('pusht_smoke')
|
| 23 |
+
print(' dataset columns =', ds.column_names)
|
| 24 |
+
print(' episodes =', len(ds.lengths))
|
| 25 |
+
|
| 26 |
+
ep = ds.load_episode(0)
|
| 27 |
+
for k, v in ep.items():
|
| 28 |
+
if hasattr(v, 'shape'):
|
| 29 |
+
print(f' ep[{k}] {tuple(v.shape)} {v.dtype}')
|
| 30 |
+
|
| 31 |
+
pixels_key = 'pixels' if 'pixels' in ep else 'obs.pixels'
|
| 32 |
+
frames = ep[pixels_key]
|
| 33 |
+
if not torch.is_tensor(frames):
|
| 34 |
+
frames = torch.as_tensor(frames)
|
| 35 |
+
if frames.shape[-1] in (1, 3): # NHWC -> NCHW
|
| 36 |
+
frames = frames.permute(0, 3, 1, 2)
|
| 37 |
+
frames = frames.float() / 255.0 if frames.dtype == torch.uint8 else frames
|
| 38 |
+
|
| 39 |
+
T = model.predictor.num_frames
|
| 40 |
+
pixels = frames[:T].unsqueeze(0).to(device) # (1, T, C, H, W)
|
| 41 |
+
with torch.no_grad():
|
| 42 |
+
info = model.encode({'pixels': pixels})
|
| 43 |
+
emb = info['emb']
|
| 44 |
+
print(' emb', tuple(emb.shape))
|
| 45 |
+
|
| 46 |
+
# gradient check: does d(pred)/d(action) flow?
|
| 47 |
+
action_dim = model.action_encoder.input_dim
|
| 48 |
+
action = torch.zeros(1, T, action_dim, device=device, requires_grad=True)
|
| 49 |
+
act_emb = model.action_encoder(action)
|
| 50 |
+
pred = model.predict(emb, act_emb)
|
| 51 |
+
print(' pred', tuple(pred.shape))
|
| 52 |
+
pred.sum().backward()
|
| 53 |
+
g = action.grad
|
| 54 |
+
print(' action.grad norm =', float(g.norm()))
|
| 55 |
+
assert g.norm() > 0, 'no gradient reached the action'
|
| 56 |
+
print('OK: gradients flow through the frozen predictor to actions')
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
if __name__ == '__main__':
|
| 60 |
+
main()
|
code/scripts/summarize_refine_sweep.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Summarize the refinement-count sweep, with paired tests.
|
| 2 |
+
|
| 3 |
+
Success rates at n=50 have a binomial standard error near 7 points, so an
|
| 4 |
+
8-point gap between two K values is about one SE and means nothing on its own.
|
| 5 |
+
Every row in a sweep shares its held-out start/goal pairs at a fixed seed, so
|
| 6 |
+
the comparisons are paired and McNemar's exact test applies -- which is the
|
| 7 |
+
standard ``run_report.md`` already holds the rest of the project to.
|
| 8 |
+
|
| 9 |
+
Reports success vs K per receding horizon, pooled across seeds, and tests each
|
| 10 |
+
K against the best-performing K.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import argparse
|
| 14 |
+
import json
|
| 15 |
+
from collections import defaultdict
|
| 16 |
+
from math import comb
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def mcnemar_exact(a, b):
|
| 21 |
+
"""Two-sided exact McNemar on paired boolean outcome lists.
|
| 22 |
+
|
| 23 |
+
Returns ``(delta_points, n_discordant, p)``. ``delta`` is ``b`` minus ``a``
|
| 24 |
+
in percentage points.
|
| 25 |
+
"""
|
| 26 |
+
assert len(a) == len(b), 'unpaired inputs'
|
| 27 |
+
b_only = sum(1 for x, y in zip(a, b) if not x and y)
|
| 28 |
+
a_only = sum(1 for x, y in zip(a, b) if x and not y)
|
| 29 |
+
n = b_only + a_only
|
| 30 |
+
delta = 100.0 * (b_only - a_only) / len(a)
|
| 31 |
+
if n == 0:
|
| 32 |
+
return delta, 0, 1.0
|
| 33 |
+
k = min(b_only, a_only)
|
| 34 |
+
tail = sum(comb(n, i) for i in range(k + 1)) / (2 ** n)
|
| 35 |
+
return delta, n, min(1.0, 2 * tail)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def main():
|
| 39 |
+
p = argparse.ArgumentParser()
|
| 40 |
+
p.add_argument('--results', default='data/runs/eval_refine_sweep/results.jsonl')
|
| 41 |
+
args = p.parse_args()
|
| 42 |
+
|
| 43 |
+
rows = [json.loads(line) for line in Path(args.results).read_text().splitlines()]
|
| 44 |
+
rows = [r for r in rows if r.get('refinements') is not None]
|
| 45 |
+
|
| 46 |
+
# (rh, K) -> {seed: episode_successes}
|
| 47 |
+
grouped = defaultdict(dict)
|
| 48 |
+
for r in rows:
|
| 49 |
+
grouped[(r['receding_horizon'], r['refinements'])][r['seed']] = (
|
| 50 |
+
r['episode_successes']
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
horizons = sorted({rh for rh, _ in grouped})
|
| 54 |
+
for rh in horizons:
|
| 55 |
+
ks = sorted({k for h, k in grouped if h == rh})
|
| 56 |
+
seeds = sorted(set.intersection(
|
| 57 |
+
*[set(grouped[(rh, k)]) for k in ks]
|
| 58 |
+
)) if ks else []
|
| 59 |
+
if not seeds:
|
| 60 |
+
continue
|
| 61 |
+
|
| 62 |
+
def pooled(k):
|
| 63 |
+
out = []
|
| 64 |
+
for s in seeds:
|
| 65 |
+
out.extend(grouped[(rh, k)][s])
|
| 66 |
+
return out
|
| 67 |
+
|
| 68 |
+
series = {k: pooled(k) for k in ks}
|
| 69 |
+
rates = {k: 100 * sum(v) / len(v) for k, v in series.items()}
|
| 70 |
+
best = max(rates, key=rates.get)
|
| 71 |
+
|
| 72 |
+
n = len(series[best])
|
| 73 |
+
print(f'\nreceding_horizon={rh} seeds={seeds} n={n} episodes pooled')
|
| 74 |
+
print(f'{"K":>4}{"success%":>10}{"per-seed":>16}'
|
| 75 |
+
f'{"vs best (K=%d)" % best:>28}')
|
| 76 |
+
for k in ks:
|
| 77 |
+
per_seed = ' '.join(
|
| 78 |
+
f'{100 * sum(grouped[(rh, k)][s]) / len(grouped[(rh, k)][s]):.0f}'
|
| 79 |
+
for s in seeds
|
| 80 |
+
)
|
| 81 |
+
if k == best:
|
| 82 |
+
verdict = '(best)'
|
| 83 |
+
else:
|
| 84 |
+
delta, nd, pv = mcnemar_exact(series[k], series[best])
|
| 85 |
+
verdict = f'{delta:+.1f}pt n_d={nd} p={pv:.3f}'
|
| 86 |
+
print(f'{k:>4}{rates[k]:>10.1f}{per_seed:>16}{verdict:>28}')
|
| 87 |
+
|
| 88 |
+
# the question the sweep exists to answer
|
| 89 |
+
tail = [rates[k] for k in ks if k >= best]
|
| 90 |
+
monotone = all(
|
| 91 |
+
rates[ks[i]] <= rates[ks[i + 1]] + 4 for i in range(len(ks) - 1)
|
| 92 |
+
)
|
| 93 |
+
print(f' peak at K={best}; '
|
| 94 |
+
f'{"non-decreasing in K (within 4pt noise)" if monotone else "NOT monotone in K"}'
|
| 95 |
+
f'; drop from peak to K={ks[-1]}: {rates[best] - rates[ks[-1]]:+.1f}pt')
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
if __name__ == '__main__':
|
| 99 |
+
main()
|
code/scripts/summarize_results.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Summarize eval runs into the tables the design doc asks for.
|
| 2 |
+
|
| 3 |
+
Reads the jsonl written by ``scripts/eval_controller.py``. The central claim
|
| 4 |
+
is matching CEM's success at a fraction of the predictor evaluations, so cost
|
| 5 |
+
is reported as predictor rows per episode alongside wall-clock: CEM batches
|
| 6 |
+
its samples, so the two ratios differ by an order of magnitude and only
|
| 7 |
+
stating both is honest.
|
| 8 |
+
|
| 9 |
+
Rows are compared against the CEM baseline *at the same execution length*.
|
| 10 |
+
Comparing against CEM at receding-horizon 1 flatters the controller, because
|
| 11 |
+
the fixed-terminal objective makes every planner procrastinate under short
|
| 12 |
+
execution.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import argparse
|
| 16 |
+
import json
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def refinements(row):
|
| 21 |
+
k = row.get('refinements')
|
| 22 |
+
if k is not None:
|
| 23 |
+
return k
|
| 24 |
+
if row['planner'].startswith('controller'):
|
| 25 |
+
return int(row['planner'].split('_K')[-1])
|
| 26 |
+
return None
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def variant(row):
|
| 30 |
+
"""Which trained checkpoint a row came from."""
|
| 31 |
+
ckpt = (row.get('checkpoint') or '').replace('\\', '/')
|
| 32 |
+
for name in ('terminal_only', 'no_support',
|
| 33 |
+
'ah_hold0.5', 'ah_hold0.0', 'ah_hold1.0'):
|
| 34 |
+
if name in ckpt:
|
| 35 |
+
return name
|
| 36 |
+
return 'main'
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def label(row):
|
| 40 |
+
tag = row['planner']
|
| 41 |
+
if row.get('receding_horizon', 1) != 1:
|
| 42 |
+
tag += f'+exec{row["receding_horizon"]}'
|
| 43 |
+
v = variant(row)
|
| 44 |
+
if v != 'main':
|
| 45 |
+
tag += f'[{v}]'
|
| 46 |
+
return tag
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def main():
|
| 50 |
+
p = argparse.ArgumentParser()
|
| 51 |
+
p.add_argument('--results', default='data/runs/eval/results.jsonl')
|
| 52 |
+
args = p.parse_args()
|
| 53 |
+
|
| 54 |
+
rows = [
|
| 55 |
+
json.loads(x)
|
| 56 |
+
for x in Path(args.results).read_text().splitlines()
|
| 57 |
+
if x.strip()
|
| 58 |
+
]
|
| 59 |
+
if not rows:
|
| 60 |
+
print('no results yet')
|
| 61 |
+
return
|
| 62 |
+
|
| 63 |
+
# baseline per execution length, so cost ratios are never computed across
|
| 64 |
+
# schedules that make different numbers of solver calls
|
| 65 |
+
cem_by_exec = {
|
| 66 |
+
r.get('receding_horizon', 1): r
|
| 67 |
+
for r in rows if r['planner'].startswith('cem')
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
head = (
|
| 71 |
+
f'{"planner":<34}{"exec":>5}{"success%":>10}{"first_d":>9}'
|
| 72 |
+
f'{"rows/ep":>10}{"sec/ep":>8}'
|
| 73 |
+
)
|
| 74 |
+
print(head)
|
| 75 |
+
print('-' * len(head))
|
| 76 |
+
for r in rows:
|
| 77 |
+
d = r.get('first_terminal_distance')
|
| 78 |
+
print(
|
| 79 |
+
f'{label(r):<34}{r.get("receding_horizon", 1):>5}'
|
| 80 |
+
f'{r["success_rate"]:>10.1f}'
|
| 81 |
+
f'{(f"{d:.4f}" if d is not None else "-"):>9}'
|
| 82 |
+
f'{r["predictor_rows_per_episode"]:>10.0f}'
|
| 83 |
+
f'{r["seconds_per_episode"]:>8.2f}'
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
# --- K x execution grid, per trained variant ---------------------------
|
| 87 |
+
grids = {}
|
| 88 |
+
for r in rows:
|
| 89 |
+
if not r['planner'].startswith('controller'):
|
| 90 |
+
continue
|
| 91 |
+
grids.setdefault(variant(r), {})[
|
| 92 |
+
(refinements(r), r.get('receding_horizon', 1))
|
| 93 |
+
] = r['success_rate']
|
| 94 |
+
|
| 95 |
+
for name, grid in grids.items():
|
| 96 |
+
ks = sorted({k for k, _ in grid})
|
| 97 |
+
ms = sorted({m for _, m in grid})
|
| 98 |
+
print(f'\nsuccess% [{name}] — refinements K (rows) x execution (cols)')
|
| 99 |
+
print(f'{"K":>3}' + ''.join(f'{m:>8}' for m in ms))
|
| 100 |
+
for k in ks:
|
| 101 |
+
cells = ''.join(
|
| 102 |
+
f'{grid[(k, m)]:>8.1f}' if (k, m) in grid else f'{"-":>8}'
|
| 103 |
+
for m in ms
|
| 104 |
+
)
|
| 105 |
+
print(f'{k:>3}{cells}')
|
| 106 |
+
|
| 107 |
+
# --- matched-execution comparison --------------------------------------
|
| 108 |
+
print('\nvs CEM at the same execution length:')
|
| 109 |
+
for r in rows:
|
| 110 |
+
if not r['planner'].startswith('controller'):
|
| 111 |
+
continue
|
| 112 |
+
base = cem_by_exec.get(r.get('receding_horizon', 1))
|
| 113 |
+
if base is None:
|
| 114 |
+
continue
|
| 115 |
+
rows_x = base['predictor_rows_per_episode'] / max(
|
| 116 |
+
r['predictor_rows_per_episode'], 1e-9
|
| 117 |
+
)
|
| 118 |
+
time_x = base['seconds_per_episode'] / max(
|
| 119 |
+
r['seconds_per_episode'], 1e-9
|
| 120 |
+
)
|
| 121 |
+
print(
|
| 122 |
+
f' {label(r):<36}{r["success_rate"]:>6.1f}% vs '
|
| 123 |
+
f'{base["success_rate"]:>5.1f}% '
|
| 124 |
+
f'{rows_x:>7.0f}x fewer rows {time_x:>5.1f}x faster'
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
if __name__ == '__main__':
|
| 129 |
+
main()
|
code/scripts/sweep_refinements.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Refinement-count sweep on a controller checkpoint, at several execution lengths.
|
| 2 |
+
|
| 3 |
+
Exists because ``run_ablations.py:67`` hardcodes ``data/runs/controller/controller.pt``
|
| 4 |
+
-- the *terminal-only* controller that ``progress.md`` records as superseded. Every
|
| 5 |
+
K-sweep row in ``data/runs/eval/results.jsonl`` is therefore measured on a checkpoint
|
| 6 |
+
with a known-broken objective, and the arrival+hold controller that replaced it
|
| 7 |
+
(``data/runs/ah_hold0.5``, 94% at rh=1) has never been swept at all.
|
| 8 |
+
|
| 9 |
+
The question this answers: does refinement count degrade success for a controller
|
| 10 |
+
that already does joint whole-plan refinement over a full ``rollout_plan``? If it
|
| 11 |
+
does, "causal one-step commitment" cannot be the cause of the recursive planner's
|
| 12 |
+
T-inversion, because this architecture has no causal commitment to begin with.
|
| 13 |
+
|
| 14 |
+
Each row goes through the unmodified ``eval_controller.py``, so it shares the
|
| 15 |
+
harness, the wrappers and -- at a fixed seed -- the held-out start/goal pairs with
|
| 16 |
+
every historical row. Results land in their own directory; the historical files are
|
| 17 |
+
not appended to.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
import argparse
|
| 21 |
+
import subprocess
|
| 22 |
+
import sys
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
|
| 25 |
+
PY = sys.executable
|
| 26 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def parse_args():
|
| 30 |
+
p = argparse.ArgumentParser()
|
| 31 |
+
p.add_argument(
|
| 32 |
+
'--controller', default='data/runs/ah_hold0.5/controller.pt'
|
| 33 |
+
)
|
| 34 |
+
p.add_argument('--refinements', type=int, nargs='+',
|
| 35 |
+
default=[0, 1, 2, 3, 5, 8])
|
| 36 |
+
p.add_argument('--receding-horizon', type=int, nargs='+', default=[1, 5])
|
| 37 |
+
p.add_argument('--seeds', type=int, nargs='+', default=[42, 43, 44])
|
| 38 |
+
p.add_argument('--goal-offset', type=int, nargs='+', default=[25])
|
| 39 |
+
p.add_argument('--num-eval', type=int, default=50)
|
| 40 |
+
p.add_argument('--eval-budget', type=int, default=50)
|
| 41 |
+
p.add_argument('--out', default='data/runs/eval_refine_sweep')
|
| 42 |
+
return p.parse_args()
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def main():
|
| 46 |
+
args = parse_args()
|
| 47 |
+
total = (
|
| 48 |
+
len(args.refinements) * len(args.receding_horizon)
|
| 49 |
+
* len(args.seeds) * len(args.goal_offset)
|
| 50 |
+
)
|
| 51 |
+
done = 0
|
| 52 |
+
|
| 53 |
+
for q in args.goal_offset:
|
| 54 |
+
for rh in args.receding_horizon:
|
| 55 |
+
for seed in args.seeds:
|
| 56 |
+
for k in args.refinements:
|
| 57 |
+
done += 1
|
| 58 |
+
cmd = [
|
| 59 |
+
PY, 'scripts/eval_controller.py',
|
| 60 |
+
'--controller', args.controller,
|
| 61 |
+
'--refinements', str(k),
|
| 62 |
+
'--receding-horizon', str(rh),
|
| 63 |
+
'--goal-offset', str(q),
|
| 64 |
+
'--seed', str(seed),
|
| 65 |
+
'--num-eval', str(args.num_eval),
|
| 66 |
+
'--eval-budget', str(args.eval_budget),
|
| 67 |
+
'--out', args.out,
|
| 68 |
+
]
|
| 69 |
+
print(
|
| 70 |
+
f'\n[{done}/{total}] q={q} rh={rh} seed={seed} K={k}',
|
| 71 |
+
flush=True,
|
| 72 |
+
)
|
| 73 |
+
# one bad row should not discard the runs queued behind it
|
| 74 |
+
code = subprocess.run(cmd, cwd=ROOT).returncode
|
| 75 |
+
if code != 0:
|
| 76 |
+
print(f'!! exited {code}, continuing', flush=True)
|
| 77 |
+
|
| 78 |
+
print(f'\nappended to {Path(args.out) / "results.jsonl"}')
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
if __name__ == '__main__':
|
| 82 |
+
main()
|
code/scripts/test_controller.py
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""End-to-end checks on the controller stack, with no data dependency.
|
| 2 |
+
|
| 3 |
+
Verifies the shape contract, that gradients reach the controller but never the
|
| 4 |
+
frozen world model, that the tanh bound maps exactly onto raw PushT limits,
|
| 5 |
+
and that refinement actually changes the plan.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 14 |
+
|
| 15 |
+
from lejepa_control.controller import IterativeController # noqa: E402
|
| 16 |
+
from lejepa_control.losses import ( # noqa: E402
|
| 17 |
+
BehaviorDensity,
|
| 18 |
+
refinement_loss,
|
| 19 |
+
support_loss,
|
| 20 |
+
)
|
| 21 |
+
from lejepa_control.rollout import rollout_contexts, rollout_plan # noqa: E402
|
| 22 |
+
from lejepa_control.solver import load_controller # noqa: E402
|
| 23 |
+
from lejepa_control.world_model import load_lewm # noqa: E402
|
| 24 |
+
|
| 25 |
+
# z-score stats measured on the full expert dataset
|
| 26 |
+
A_MEAN = torch.tensor([-0.00781276, 0.00686056])
|
| 27 |
+
A_STD = torch.tensor([0.20824118, 0.20649128])
|
| 28 |
+
|
| 29 |
+
# PushT checkpoint dirs under data/runs/ that predate exp7 (reacher/tworoom
|
| 30 |
+
# use different action_dim/frameskip and are out of scope for this regression)
|
| 31 |
+
LEGACY_PUSHT_CHECKPOINTS = [
|
| 32 |
+
'ah_hold0.0', 'ah_hold0.5', 'ah_hold1.0',
|
| 33 |
+
'abl_no_support', 'abl_terminal_only', 'controller', 'dryrun',
|
| 34 |
+
]
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def test_checkpoint_rebuild(device):
|
| 38 |
+
"""exp7: adding no_latent_proj/fused to load_controller must not disturb
|
| 39 |
+
any pre-existing checkpoint (``saved.get(..., False)`` keeps them at the
|
| 40 |
+
split/projected architecture they were actually trained with)."""
|
| 41 |
+
root = Path(__file__).resolve().parents[1] / 'data' / 'runs'
|
| 42 |
+
checked = 0
|
| 43 |
+
for name in LEGACY_PUSHT_CHECKPOINTS:
|
| 44 |
+
path = root / name / 'controller.pt'
|
| 45 |
+
if not path.exists():
|
| 46 |
+
continue
|
| 47 |
+
controller, _ = load_controller(str(path), device=device)
|
| 48 |
+
assert controller.fused is False, f'{name}: fused should default False'
|
| 49 |
+
assert controller.no_latent_proj is False, f'{name}: no_latent_proj should default False'
|
| 50 |
+
checked += 1
|
| 51 |
+
assert checked > 0, 'no legacy PushT checkpoints found to regression-test'
|
| 52 |
+
print(f'checkpoint rebuild ok ({checked} checkpoints, strict load, flags default False)')
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def test_no_latent_proj_guard():
|
| 56 |
+
"""--no-latent-proj with width != latent_dim must raise, clearly."""
|
| 57 |
+
try:
|
| 58 |
+
IterativeController(latent_dim=192, width=256, no_latent_proj=True)
|
| 59 |
+
except AssertionError as e:
|
| 60 |
+
assert 'no-latent-proj' in str(e)
|
| 61 |
+
else:
|
| 62 |
+
raise AssertionError('no_latent_proj with width != latent_dim did not raise')
|
| 63 |
+
print('no_latent_proj width guard ok')
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def test_param_counts():
|
| 67 |
+
"""Param counts for all four SPACE x OPERATOR cells (exp7 sec 1/7).
|
| 68 |
+
|
| 69 |
+
Only the two cells the spec gives explicit numeric bands for (today's
|
| 70 |
+
baseline, and the fused192 candidate) are hard-asserted; A/B are printed
|
| 71 |
+
for the record.
|
| 72 |
+
"""
|
| 73 |
+
def count(**kw):
|
| 74 |
+
c = IterativeController(latent_dim=192, **kw)
|
| 75 |
+
return sum(p.numel() for p in c.parameters())
|
| 76 |
+
|
| 77 |
+
baseline = count(width=256) # today
|
| 78 |
+
cell_a = count(width=192, no_latent_proj=True) # 192id, split
|
| 79 |
+
cell_b = count(width=256, fused=True) # 256+proj, fused
|
| 80 |
+
cell_c = count(width=192, no_latent_proj=True, fused=True) # candidate
|
| 81 |
+
|
| 82 |
+
print(
|
| 83 |
+
f'params baseline(256+proj/split)={baseline / 1e6:.2f}M '
|
| 84 |
+
f'A(192id/split)={cell_a / 1e6:.2f}M '
|
| 85 |
+
f'B(256+proj/fused)={cell_b / 1e6:.2f}M '
|
| 86 |
+
f'C(192id/fused)={cell_c / 1e6:.2f}M'
|
| 87 |
+
)
|
| 88 |
+
assert 6.5e6 <= baseline <= 7.1e6, f'baseline param count out of band: {baseline}'
|
| 89 |
+
# spec sec 1 estimates "~4-4.5M"; verified by hand (net=8 layers@width192
|
| 90 |
+
# is 3.56M of the 3.75M total, both numbers pinned explicitly by the spec
|
| 91 |
+
# text) the true figure is ~3.75M — the spec's figure was a pre-impl
|
| 92 |
+
# approximation that ran ~7-17% high, not an implementation bug. Band
|
| 93 |
+
# widened to the verified value; still catches gross regressions (e.g. a
|
| 94 |
+
# missing/duplicated encoder would be off by ~2x).
|
| 95 |
+
assert 3.5e6 <= cell_c <= 4.0e6, f'fused192 param count out of verified band: {cell_c}'
|
| 96 |
+
print('param counts in expected band (fused192: see comment on the spec\'s ~4-4.5M estimate)')
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def test_fused_variant(model, device):
|
| 100 |
+
"""Fused-operator forward: shapes, k=0 bypasses delta_head, rows differ."""
|
| 101 |
+
D = model.predictor.input_dim
|
| 102 |
+
N, H, K, B = 3, 5, 3, 4
|
| 103 |
+
|
| 104 |
+
controller = IterativeController(
|
| 105 |
+
latent_dim=D,
|
| 106 |
+
horizon=H,
|
| 107 |
+
refinements=K,
|
| 108 |
+
width=D,
|
| 109 |
+
no_latent_proj=True,
|
| 110 |
+
fused=True,
|
| 111 |
+
action_center=(-A_MEAN / A_STD),
|
| 112 |
+
action_scale=(1.0 / A_STD),
|
| 113 |
+
).to(device).eval()
|
| 114 |
+
|
| 115 |
+
ctx = torch.randn(B, N, D, device=device)
|
| 116 |
+
past = torch.randn(B, N - 1, 10, device=device) * 0.5
|
| 117 |
+
goal = torch.randn(B, D, device=device)
|
| 118 |
+
|
| 119 |
+
out = controller(model, ctx, past, goal)
|
| 120 |
+
assert len(out['plans']) == K + 1
|
| 121 |
+
assert out['plans'][0].shape == (B, H, 10), out['plans'][0].shape
|
| 122 |
+
assert out['rollouts'][0].shape == (B, H, D)
|
| 123 |
+
assert out['distances'][0].shape == (B, H)
|
| 124 |
+
assert out['contexts'][0].shape == (B, H, N, D)
|
| 125 |
+
print(f'fused shapes ok: plan {tuple(out["plans"][0].shape)} '
|
| 126 |
+
f'rollout {tuple(out["rollouts"][0].shape)}')
|
| 127 |
+
|
| 128 |
+
# --- k=0 output does not pass through delta_head -----------------------
|
| 129 |
+
cond = controller.condition(ctx, goal)
|
| 130 |
+
y0_before = controller.initial_plan(cond)
|
| 131 |
+
with torch.no_grad():
|
| 132 |
+
saved_w = controller.delta_head.weight.clone()
|
| 133 |
+
controller.delta_head.weight.fill_(float('nan'))
|
| 134 |
+
y0_after = controller.initial_plan(cond)
|
| 135 |
+
assert torch.equal(y0_before, y0_after), 'k=0 output changed when delta_head was corrupted'
|
| 136 |
+
assert not torch.isnan(y0_after).any(), 'k=0 output leaked delta_head NaNs'
|
| 137 |
+
with torch.no_grad():
|
| 138 |
+
controller.delta_head.weight.copy_(saved_w)
|
| 139 |
+
print('k=0 bypasses delta_head ok')
|
| 140 |
+
|
| 141 |
+
# --- per-slot ΔY rows actually differ across slots ----------------------
|
| 142 |
+
pred = torch.randn(B, H, D, device=device)
|
| 143 |
+
delta = controller.refine(y0_after, cond, pred, goal)
|
| 144 |
+
row_diffs = [
|
| 145 |
+
(delta[:, i] - delta[:, j]).abs().mean().item()
|
| 146 |
+
for i in range(H)
|
| 147 |
+
for j in range(i + 1, H)
|
| 148 |
+
]
|
| 149 |
+
assert min(row_diffs) > 1e-6, f'fused refine rows too similar: {row_diffs}'
|
| 150 |
+
print(f'fused per-slot rows differ ok (min pairwise |diff| {min(row_diffs):.5f})')
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def main():
|
| 154 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 155 |
+
torch.manual_seed(0)
|
| 156 |
+
|
| 157 |
+
model = load_lewm(device=device)
|
| 158 |
+
D = model.predictor.input_dim
|
| 159 |
+
N, H, K, B = 3, 5, 3, 4
|
| 160 |
+
|
| 161 |
+
controller = IterativeController(
|
| 162 |
+
latent_dim=D,
|
| 163 |
+
horizon=H,
|
| 164 |
+
refinements=K,
|
| 165 |
+
action_center=(-A_MEAN / A_STD),
|
| 166 |
+
action_scale=(1.0 / A_STD),
|
| 167 |
+
).to(device)
|
| 168 |
+
print(f'params {sum(p.numel() for p in controller.parameters()) / 1e6:.2f}M')
|
| 169 |
+
|
| 170 |
+
ctx = torch.randn(B, N, D, device=device)
|
| 171 |
+
past = torch.randn(B, N - 1, 10, device=device) * 0.5
|
| 172 |
+
goal = torch.randn(B, D, device=device)
|
| 173 |
+
|
| 174 |
+
out = controller(model, ctx, past, goal)
|
| 175 |
+
|
| 176 |
+
assert len(out['plans']) == K + 1, 'one plan per refinement, plus initial'
|
| 177 |
+
assert out['plans'][0].shape == (B, H, 10), out['plans'][0].shape
|
| 178 |
+
assert out['rollouts'][0].shape == (B, H, D)
|
| 179 |
+
assert out['distances'][0].shape == (B, H)
|
| 180 |
+
assert out['contexts'][0].shape == (B, H, N, D)
|
| 181 |
+
print(f'shapes ok: plan {tuple(out["plans"][0].shape)} '
|
| 182 |
+
f'rollout {tuple(out["rollouts"][0].shape)} '
|
| 183 |
+
f'contexts {tuple(out["contexts"][0].shape)}')
|
| 184 |
+
|
| 185 |
+
# --- tanh bound maps exactly onto raw actions in [-1, 1] --------------
|
| 186 |
+
plan = out['plans'][-1].reshape(B, H, 5, 2).cpu()
|
| 187 |
+
raw = plan * A_STD + A_MEAN
|
| 188 |
+
assert raw.abs().max() <= 1.0 + 1e-5, f'raw action out of bounds {raw.abs().max()}'
|
| 189 |
+
lo = (controller.action_center - controller.action_scale).cpu()
|
| 190 |
+
hi = (controller.action_center + controller.action_scale).cpu()
|
| 191 |
+
print(f'raw action range [{raw.min():.3f}, {raw.max():.3f}] within [-1, 1]')
|
| 192 |
+
print(f' normalized bounds {(lo * A_STD + A_MEAN).tolist()} '
|
| 193 |
+
f'{(hi * A_STD + A_MEAN).tolist()}')
|
| 194 |
+
|
| 195 |
+
# --- refinement changes the plan --------------------------------------
|
| 196 |
+
deltas = [
|
| 197 |
+
(out['plans'][k + 1] - out['plans'][k]).abs().mean().item()
|
| 198 |
+
for k in range(K)
|
| 199 |
+
]
|
| 200 |
+
print(f'plan change per refinement {[round(d, 5) for d in deltas]}')
|
| 201 |
+
assert max(deltas) > 0, 'refinement never changed the plan'
|
| 202 |
+
|
| 203 |
+
# --- context windows line up with the rollout -------------------------
|
| 204 |
+
pred, frames = rollout_plan(
|
| 205 |
+
model, ctx, past, out['plans'][-1], return_frames=True
|
| 206 |
+
)
|
| 207 |
+
cw = rollout_contexts(frames, N)
|
| 208 |
+
assert torch.equal(cw[:, 0], ctx), 'first window must be the real context'
|
| 209 |
+
assert torch.allclose(cw[:, 1, -1], pred[:, 0]), 'window must advance by 1'
|
| 210 |
+
print('rollout context windows aligned')
|
| 211 |
+
|
| 212 |
+
# --- loss and gradient flow -------------------------------------------
|
| 213 |
+
density = BehaviorDensity(latent_dim=D).to(device)
|
| 214 |
+
loss = refinement_loss(out['distances'], alpha=0.05)
|
| 215 |
+
s_loss, viol = support_loss(
|
| 216 |
+
density,
|
| 217 |
+
torch.cat(out['contexts'][1:], 0).flatten(0, 1),
|
| 218 |
+
torch.cat(out['plans'][1:], 0).flatten(0, 1),
|
| 219 |
+
threshold=0.0,
|
| 220 |
+
)
|
| 221 |
+
total = loss + 0.01 * s_loss
|
| 222 |
+
total.backward()
|
| 223 |
+
|
| 224 |
+
trained = [
|
| 225 |
+
n for n, p in controller.named_parameters() if p.grad is not None
|
| 226 |
+
]
|
| 227 |
+
untrained = [
|
| 228 |
+
n
|
| 229 |
+
for n, p in controller.named_parameters()
|
| 230 |
+
if p.grad is None or p.grad.abs().sum() == 0
|
| 231 |
+
]
|
| 232 |
+
frozen_grads = [n for n, p in model.named_parameters() if p.grad is not None]
|
| 233 |
+
|
| 234 |
+
print(f'refine loss {loss.item():.4f} support {s_loss.item():.4f} '
|
| 235 |
+
f'violation rate {viol.item():.2f}')
|
| 236 |
+
print(f'controller params with grad: {len(trained)}, '
|
| 237 |
+
f'without: {len(untrained)}')
|
| 238 |
+
assert not frozen_grads, f'world model got gradients: {frozen_grads[:3]}'
|
| 239 |
+
assert not untrained, f'no gradient reached: {untrained}'
|
| 240 |
+
|
| 241 |
+
# --- refinement loss weighting is 2^k ----------------------------------
|
| 242 |
+
fake = [
|
| 243 |
+
torch.full((1, H), float(v), device=device) for v in (1.0, 1.0, 1.0, 0.0)
|
| 244 |
+
]
|
| 245 |
+
got = refinement_loss(fake, alpha=0.0).item()
|
| 246 |
+
expect = (1 * 1 + 2 * 1 + 4 * 1 + 8 * 0) / 15
|
| 247 |
+
assert abs(got - expect) < 1e-6, (got, expect)
|
| 248 |
+
print(f'refinement weighting ok ({got:.4f} == {expect:.4f})')
|
| 249 |
+
|
| 250 |
+
# --- exp7: fused192 architecture ---------------------------------------
|
| 251 |
+
print()
|
| 252 |
+
test_checkpoint_rebuild(device)
|
| 253 |
+
test_no_latent_proj_guard()
|
| 254 |
+
test_param_counts()
|
| 255 |
+
test_fused_variant(model, device)
|
| 256 |
+
|
| 257 |
+
print('ALL CONTROLLER TESTS PASSED')
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
if __name__ == '__main__':
|
| 261 |
+
main()
|
code/scripts/test_data.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Validate the latent dataset against the raw HDF5 and the world model.
|
| 2 |
+
|
| 3 |
+
The indexing here is easy to get subtly wrong: pixels are strided by frameskip
|
| 4 |
+
while actions are not, and the block leaving frame k must be the actions that
|
| 5 |
+
actually produced frame k+1. This checks that alignment against ground truth,
|
| 6 |
+
then confirms the frozen predictor can reach the sampled goals — if teacher
|
| 7 |
+
forcing on real actions doesn't reduce the goal distance, the goals aren't
|
| 8 |
+
reachable and the controller has no learnable signal.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
import h5py
|
| 15 |
+
import hdf5plugin # noqa: F401 -- blosc filter
|
| 16 |
+
import numpy as np
|
| 17 |
+
import torch
|
| 18 |
+
|
| 19 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 20 |
+
|
| 21 |
+
from lejepa_control.data import ( # noqa: E402
|
| 22 |
+
FRAMESKIP,
|
| 23 |
+
NUM_CONTEXT,
|
| 24 |
+
LatentGoalDataset,
|
| 25 |
+
split_episodes,
|
| 26 |
+
)
|
| 27 |
+
from lejepa_control.rollout import goal_distance, rollout_plan # noqa: E402
|
| 28 |
+
from lejepa_control.world_model import load_lewm # noqa: E402
|
| 29 |
+
|
| 30 |
+
H5 = 'data/swm_home/datasets/pusht_expert_train.h5'
|
| 31 |
+
IMAGENET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
|
| 32 |
+
IMAGENET_STD = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def main():
|
| 36 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 37 |
+
torch.manual_seed(0)
|
| 38 |
+
|
| 39 |
+
ds = LatentGoalDataset('data/latents', max_offset=5)
|
| 40 |
+
print(f'{len(ds)} clips over {len(ds.episodes)} episodes')
|
| 41 |
+
|
| 42 |
+
f = h5py.File(H5, 'r')
|
| 43 |
+
model = load_lewm(device=device)
|
| 44 |
+
|
| 45 |
+
# --- cached latents match a fresh encode of the same raw frames -------
|
| 46 |
+
sample_idx = [0, 12345, 999999]
|
| 47 |
+
for idx in sample_idx:
|
| 48 |
+
ep = int(ds.clip_episode[idx])
|
| 49 |
+
start = int(ds.clip_start[idx])
|
| 50 |
+
base = int(ds.offsets[ep])
|
| 51 |
+
rows = [base + start + i * FRAMESKIP for i in range(NUM_CONTEXT)]
|
| 52 |
+
|
| 53 |
+
px = torch.from_numpy(f['pixels'][rows[0] : rows[-1] + 1 : FRAMESKIP])
|
| 54 |
+
px = px.permute(0, 3, 1, 2).float() / 255.0
|
| 55 |
+
px = ((px - IMAGENET_MEAN) / IMAGENET_STD).unsqueeze(0).to(device)
|
| 56 |
+
with torch.no_grad():
|
| 57 |
+
fresh = model.encode({'pixels': px})['emb'][0].cpu()
|
| 58 |
+
|
| 59 |
+
cached = ds[idx]['context']
|
| 60 |
+
rel = (cached - fresh).norm() / fresh.norm()
|
| 61 |
+
assert rel < 0.02, f'latent cache mismatch at {idx}: {rel}'
|
| 62 |
+
print(f'latent cache matches fresh encode (rel err < 2%) at {sample_idx}')
|
| 63 |
+
|
| 64 |
+
# --- action blocks are the raw actions between the context frames -----
|
| 65 |
+
idx = 12345
|
| 66 |
+
ep = int(ds.clip_episode[idx])
|
| 67 |
+
start = int(ds.clip_start[idx])
|
| 68 |
+
base = int(ds.offsets[ep])
|
| 69 |
+
item = ds[idx]
|
| 70 |
+
for i in range(NUM_CONTEXT - 1):
|
| 71 |
+
lo = base + start + i * FRAMESKIP
|
| 72 |
+
raw = f['action'][lo : lo + FRAMESKIP] # (5, 2)
|
| 73 |
+
want = ((raw - ds.action_mean) / ds.action_std).reshape(-1)
|
| 74 |
+
got = item['past_actions'][i].numpy()
|
| 75 |
+
assert np.allclose(got, want, atol=1e-5), f'block {i} misaligned'
|
| 76 |
+
print('past action blocks align with raw actions between context frames')
|
| 77 |
+
|
| 78 |
+
# --- goal offsets respect the curriculum ------------------------------
|
| 79 |
+
for max_offset in (2, 3, 5):
|
| 80 |
+
ds.set_max_offset(max_offset)
|
| 81 |
+
offsets = [int(ds[i]['goal_offset']) for i in range(200)]
|
| 82 |
+
assert min(offsets) >= 1 and max(offsets) <= max_offset, offsets
|
| 83 |
+
print(f' max_offset={max_offset}: sampled {sorted(set(offsets))}')
|
| 84 |
+
|
| 85 |
+
# --- reachability: real actions must beat a do-nothing plan -----------
|
| 86 |
+
ds.set_max_offset(5)
|
| 87 |
+
loader = torch.utils.data.DataLoader(ds, batch_size=64, shuffle=True)
|
| 88 |
+
batch = next(iter(loader))
|
| 89 |
+
|
| 90 |
+
ctx = batch['context'].to(device)
|
| 91 |
+
past = batch['past_actions'].to(device)
|
| 92 |
+
goal = batch['goal'].to(device)
|
| 93 |
+
real = batch['real_action'].to(device)
|
| 94 |
+
|
| 95 |
+
# replay the real first block, then hold still
|
| 96 |
+
H = 5
|
| 97 |
+
plan = torch.zeros(ctx.size(0), H, 10, device=device)
|
| 98 |
+
plan[:, 0] = real
|
| 99 |
+
with torch.no_grad():
|
| 100 |
+
pred = rollout_plan(model, ctx, past, plan)
|
| 101 |
+
d_real = goal_distance(pred, goal)
|
| 102 |
+
|
| 103 |
+
zero = torch.zeros_like(plan)
|
| 104 |
+
d_zero = goal_distance(rollout_plan(model, ctx, past, zero), goal)
|
| 105 |
+
|
| 106 |
+
# distance from the current frame to the goal, before acting at all
|
| 107 |
+
d_start = (ctx[:, -1] - goal).pow(2).mean(-1)
|
| 108 |
+
|
| 109 |
+
print(f'start distance to goal {d_start.mean():.4f}')
|
| 110 |
+
print(f'after real 1st block (d_1) {d_real[:, 0].mean():.4f}')
|
| 111 |
+
print(f'after zero 1st block (d_1) {d_zero[:, 0].mean():.4f}')
|
| 112 |
+
assert d_real[:, 0].mean() < d_start.mean(), (
|
| 113 |
+
'real actions do not move toward the goal — indexing is likely wrong'
|
| 114 |
+
)
|
| 115 |
+
assert d_real[:, 0].mean() < d_zero[:, 0].mean(), (
|
| 116 |
+
'real actions are no better than doing nothing'
|
| 117 |
+
)
|
| 118 |
+
print('reachability ok: real actions reduce latent goal distance')
|
| 119 |
+
|
| 120 |
+
# --- one-step goals: teacher forcing should nearly reach --------------
|
| 121 |
+
ds.set_max_offset(1)
|
| 122 |
+
batch = next(iter(torch.utils.data.DataLoader(ds, batch_size=64, shuffle=True)))
|
| 123 |
+
ctx = batch['context'].to(device)
|
| 124 |
+
past = batch['past_actions'].to(device)
|
| 125 |
+
goal = batch['goal'].to(device)
|
| 126 |
+
plan = batch['real_action'].to(device).unsqueeze(1)
|
| 127 |
+
with torch.no_grad():
|
| 128 |
+
d1 = goal_distance(rollout_plan(model, ctx, past, plan), goal)[:, 0]
|
| 129 |
+
d0 = (ctx[:, -1] - goal).pow(2).mean(-1)
|
| 130 |
+
print(f'1-step goal: before {d0.mean():.4f} -> after {d1.mean():.4f} '
|
| 131 |
+
f'({100 * (1 - d1.mean() / d0.mean()):.0f}% closed)')
|
| 132 |
+
assert d1.mean() < 0.5 * d0.mean(), 'one-step goals not being reached'
|
| 133 |
+
|
| 134 |
+
print('ALL DATA TESTS PASSED')
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
if __name__ == '__main__':
|
| 138 |
+
main()
|
code/scripts/test_history_policy.py
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for tools/history_policy.HistoryPolicy.
|
| 2 |
+
|
| 3 |
+
Runs the real PushT env pool behind a spy solver that records exactly what the
|
| 4 |
+
solver was handed at each replan, so the assertions are on the contract the
|
| 5 |
+
planner actually sees rather than on the buffer internals.
|
| 6 |
+
|
| 7 |
+
python scripts/test_history_policy.py
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import os
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
os.environ.setdefault('MUJOCO_GL', 'egl')
|
| 15 |
+
os.environ.setdefault('PYTHONUTF8', '1')
|
| 16 |
+
|
| 17 |
+
import numpy as np # noqa: E402
|
| 18 |
+
import torch # noqa: E402
|
| 19 |
+
|
| 20 |
+
REPO = Path(__file__).resolve().parents[1]
|
| 21 |
+
os.environ.setdefault('STABLEWM_HOME', str(REPO / 'data' / 'swm_home'))
|
| 22 |
+
sys.path.insert(0, str(REPO))
|
| 23 |
+
|
| 24 |
+
import stable_worldmodel as swm # noqa: E402
|
| 25 |
+
from sklearn.preprocessing import StandardScaler # noqa: E402
|
| 26 |
+
|
| 27 |
+
from tools.history_policy import HistoryPolicy # noqa: E402
|
| 28 |
+
|
| 29 |
+
N_ENVS, BLOCK, HORIZON, N_FRAMES = 2, 5, 5, 3
|
| 30 |
+
FAILURES = []
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def check(name, ok, detail=''):
|
| 34 |
+
print(f' {"PASS" if ok else "FAIL"} {name}{" -- " + detail if detail else ""}')
|
| 35 |
+
if not ok:
|
| 36 |
+
FAILURES.append(name)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class SpySolver:
|
| 40 |
+
"""Records every solve, returns a deterministic non-zero plan."""
|
| 41 |
+
|
| 42 |
+
def __init__(self):
|
| 43 |
+
self.seen = []
|
| 44 |
+
self._n_envs = N_ENVS
|
| 45 |
+
self._horizon = HORIZON
|
| 46 |
+
self._action_block = BLOCK
|
| 47 |
+
self._action_dim = 2
|
| 48 |
+
|
| 49 |
+
def configure(self, *, action_space, n_envs, config):
|
| 50 |
+
self._n_envs = n_envs
|
| 51 |
+
self._horizon = config.horizon
|
| 52 |
+
self._action_block = config.action_block
|
| 53 |
+
self._action_dim = int(action_space.shape[-1])
|
| 54 |
+
|
| 55 |
+
@property
|
| 56 |
+
def action_dim(self):
|
| 57 |
+
return self._action_dim * self._action_block
|
| 58 |
+
|
| 59 |
+
@property
|
| 60 |
+
def n_envs(self):
|
| 61 |
+
return self._n_envs
|
| 62 |
+
|
| 63 |
+
@property
|
| 64 |
+
def horizon(self):
|
| 65 |
+
return self._horizon
|
| 66 |
+
|
| 67 |
+
def solve(self, info_dict, init_action=None):
|
| 68 |
+
px = info_dict['pixels']
|
| 69 |
+
hist = info_dict.get('action_history')
|
| 70 |
+
b = px.shape[0]
|
| 71 |
+
self.seen.append({
|
| 72 |
+
'pixels_shape': tuple(px.shape),
|
| 73 |
+
'n_frames': int(px.shape[1]) if px.ndim == 5 else 1,
|
| 74 |
+
'frames_identical': bool(
|
| 75 |
+
px.ndim == 5 and px.shape[1] > 1
|
| 76 |
+
and torch.allclose(px[:, 0], px[:, -1])
|
| 77 |
+
),
|
| 78 |
+
'has_action_history': hist is not None,
|
| 79 |
+
'action_history': None if hist is None else hist.clone(),
|
| 80 |
+
})
|
| 81 |
+
# a distinctive, non-constant plan so the recorded blocks are checkable
|
| 82 |
+
step = len(self.seen)
|
| 83 |
+
plan = torch.full((b, self._horizon, self.action_dim), 0.1 * step)
|
| 84 |
+
return {'actions': plan, 'costs': torch.zeros(b)}
|
| 85 |
+
|
| 86 |
+
__call__ = solve
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def build(use_frames, use_actions, scaler):
|
| 90 |
+
world = swm.World(
|
| 91 |
+
env_name='swm/PushT-v1', num_envs=N_ENVS, image_shape=(224, 224),
|
| 92 |
+
max_episode_steps=500,
|
| 93 |
+
)
|
| 94 |
+
config = swm.PlanConfig(
|
| 95 |
+
horizon=HORIZON, receding_horizon=1, action_block=BLOCK,
|
| 96 |
+
history_len=N_FRAMES,
|
| 97 |
+
)
|
| 98 |
+
spy = SpySolver()
|
| 99 |
+
policy = HistoryPolicy(
|
| 100 |
+
solver=spy, config=config, process={'action': scaler}, transform={},
|
| 101 |
+
use_frames=use_frames, use_actions=use_actions,
|
| 102 |
+
)
|
| 103 |
+
world.set_policy(policy)
|
| 104 |
+
world.reset(seed=0)
|
| 105 |
+
return world, spy, policy
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def run(world, steps):
|
| 109 |
+
for _ in range(steps):
|
| 110 |
+
world.envs.step(world._get_actions())
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def main():
|
| 114 |
+
scaler = StandardScaler().fit(np.random.RandomState(0).randn(500, 2) * 0.4)
|
| 115 |
+
|
| 116 |
+
print('\n=== history ON (the fix) ===')
|
| 117 |
+
world, spy, policy = build(True, True, scaler)
|
| 118 |
+
run(world, 16)
|
| 119 |
+
|
| 120 |
+
n_frames = [s['n_frames'] for s in spy.seen[:3]]
|
| 121 |
+
check('frame history grows 1 -> 2 -> 3 over the first three replans',
|
| 122 |
+
n_frames == [1, 2, 3], f'got {n_frames}')
|
| 123 |
+
check('saturates at history_len and never exceeds it',
|
| 124 |
+
all(s['n_frames'] <= N_FRAMES for s in spy.seen),
|
| 125 |
+
f'max {max(s["n_frames"] for s in spy.seen)}')
|
| 126 |
+
check('the three frames are genuinely different, not one repeated',
|
| 127 |
+
not spy.seen[2]['frames_identical'])
|
| 128 |
+
check('pixels keep the (B, T, H, W, C) layout the solvers expect',
|
| 129 |
+
len(spy.seen[2]['pixels_shape']) == 5,
|
| 130 |
+
str(spy.seen[2]['pixels_shape']))
|
| 131 |
+
|
| 132 |
+
check('no action_history at the first replan (nothing executed yet)',
|
| 133 |
+
not spy.seen[0]['has_action_history'])
|
| 134 |
+
check('action_history present from the second replan',
|
| 135 |
+
spy.seen[1]['has_action_history'])
|
| 136 |
+
hist = spy.seen[2]['action_history']
|
| 137 |
+
check('action_history is (B, N-1, block*action_dim)',
|
| 138 |
+
tuple(hist.shape) == (N_ENVS, N_FRAMES - 1, BLOCK * 2),
|
| 139 |
+
str(tuple(hist.shape)))
|
| 140 |
+
|
| 141 |
+
# Solvers emit NORMALIZED actions; WorldModelPolicy inverse_transforms them
|
| 142 |
+
# to raw env units before stepping. _record must undo exactly that, so the
|
| 143 |
+
# stored block equals what the solver emitted (0.1 from solve #1). Two
|
| 144 |
+
# assertions: the round trip is identity, and it is not vacuous -- the raw
|
| 145 |
+
# value it passed through is genuinely different.
|
| 146 |
+
got = hist[0, 0].view(BLOCK, 2)
|
| 147 |
+
raw = torch.tensor(
|
| 148 |
+
scaler.inverse_transform(np.full((1, 2), 0.1, np.float32))[0]
|
| 149 |
+
).float()
|
| 150 |
+
check('recorded blocks round-trip to the normalized units the solver emitted',
|
| 151 |
+
torch.allclose(got, torch.full((BLOCK, 2), 0.1), atol=1e-5),
|
| 152 |
+
f'got {got[0].tolist()} want [0.1, 0.1]')
|
| 153 |
+
check('and are NOT the raw env units (the transform is not a no-op)',
|
| 154 |
+
not torch.allclose(got[0], raw, atol=1e-3),
|
| 155 |
+
f'normalized {got[0].tolist()} vs raw {raw.tolist()}')
|
| 156 |
+
check('the two past blocks differ (they came from different plans)',
|
| 157 |
+
not torch.allclose(hist[0, 0], hist[0, 1]))
|
| 158 |
+
check('no NaN reaches the solver', bool(torch.isfinite(hist).all()))
|
| 159 |
+
|
| 160 |
+
check('world.infos was not mutated -- pixels still (B, 1, ...) for video/goal',
|
| 161 |
+
world.infos['pixels'].shape[1] == 1,
|
| 162 |
+
str(tuple(world.infos['pixels'].shape)))
|
| 163 |
+
check('frames sampled on the action_block stride, not the replan cadence',
|
| 164 |
+
policy._t % BLOCK == 16 % BLOCK and len(policy._frames[0]) == N_FRAMES)
|
| 165 |
+
|
| 166 |
+
print('\n=== history OFF (--legacy-history) ===')
|
| 167 |
+
world2, spy2, _ = build(False, False, scaler)
|
| 168 |
+
run(world2, 16)
|
| 169 |
+
check('legacy path still hands the solver a single frame',
|
| 170 |
+
all(s['n_frames'] == 1 for s in spy2.seen),
|
| 171 |
+
f'{[s["n_frames"] for s in spy2.seen]}')
|
| 172 |
+
check('legacy path never supplies action_history',
|
| 173 |
+
not any(s['has_action_history'] for s in spy2.seen))
|
| 174 |
+
|
| 175 |
+
print('\n=== rh=5: replans are 25 steps apart, frames must still be 5 apart ===')
|
| 176 |
+
world3 = swm.World(env_name='swm/PushT-v1', num_envs=N_ENVS,
|
| 177 |
+
image_shape=(224, 224), max_episode_steps=500)
|
| 178 |
+
cfg3 = swm.PlanConfig(horizon=HORIZON, receding_horizon=5,
|
| 179 |
+
action_block=BLOCK, history_len=N_FRAMES)
|
| 180 |
+
spy3 = SpySolver()
|
| 181 |
+
p3 = HistoryPolicy(solver=spy3, config=cfg3, process={'action': scaler},
|
| 182 |
+
transform={}, use_frames=True, use_actions=True)
|
| 183 |
+
world3.set_policy(p3)
|
| 184 |
+
world3.reset(seed=0)
|
| 185 |
+
run(world3, 30)
|
| 186 |
+
check('only 2 solves in 30 steps at rh=5', len(spy3.seen) == 2,
|
| 187 |
+
f'{len(spy3.seen)} solves')
|
| 188 |
+
check('second solve still gets 3 distinct frames (stride survived)',
|
| 189 |
+
spy3.seen[1]['n_frames'] == N_FRAMES
|
| 190 |
+
and not spy3.seen[1]['frames_identical'],
|
| 191 |
+
f'n_frames={spy3.seen[1]["n_frames"]}')
|
| 192 |
+
|
| 193 |
+
integration(scaler)
|
| 194 |
+
|
| 195 |
+
print()
|
| 196 |
+
if FAILURES:
|
| 197 |
+
print(f'{len(FAILURES)} FAILED: {FAILURES}')
|
| 198 |
+
return 1
|
| 199 |
+
print('all checks passed')
|
| 200 |
+
return 0
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def integration(scaler):
|
| 204 |
+
"""The real PlannerSolver on the real LeWM, driven by HistoryPolicy.
|
| 205 |
+
|
| 206 |
+
The unit checks above prove the buffers are correct. This proves the
|
| 207 |
+
contract holds where it matters: that PlannerSolver stops hitting its
|
| 208 |
+
padding branch and actually encodes three distinct frames plus real blocks.
|
| 209 |
+
"""
|
| 210 |
+
print('\n=== integration: real planner, real world model ===')
|
| 211 |
+
ckpt_path = REPO / 'data/runs/planner_D10/planner.pt'
|
| 212 |
+
if not ckpt_path.exists():
|
| 213 |
+
print(f' SKIP no checkpoint at {ckpt_path}')
|
| 214 |
+
return
|
| 215 |
+
|
| 216 |
+
from lejepa_control.world_model import load_lewm
|
| 217 |
+
from lejepa_control_2.solver import PlannerSolver, load_planner
|
| 218 |
+
|
| 219 |
+
model = load_lewm(device='cpu')
|
| 220 |
+
model.interpolate_pos_encoding = True
|
| 221 |
+
planner, _ = load_planner(str(ckpt_path), device='cpu', horizon=HORIZON)
|
| 222 |
+
|
| 223 |
+
seen = []
|
| 224 |
+
|
| 225 |
+
class RecordingPlannerSolver(PlannerSolver):
|
| 226 |
+
# PlannerSolver binds `__call__ = solve` at class level, so overriding
|
| 227 |
+
# `solve` alone would never be reached. Same reason eval_planner.py's
|
| 228 |
+
# RecordingSolver overrides __call__.
|
| 229 |
+
def __call__(self, info_dict, init_action=None):
|
| 230 |
+
px = info_dict['pixels']
|
| 231 |
+
ctx = self._encode(px if px.ndim == 5 else px.unsqueeze(1))
|
| 232 |
+
past = info_dict.get('action_history')
|
| 233 |
+
seen.append({
|
| 234 |
+
'n_frames': int(px.shape[1]) if px.ndim == 5 else 1,
|
| 235 |
+
# distance between the oldest and newest encoded frame: zero
|
| 236 |
+
# means the padding branch fired
|
| 237 |
+
'ctx_spread': float((ctx[:, 0] - ctx[:, -1]).pow(2).mean()),
|
| 238 |
+
'past_absmean': None if past is None else float(past.abs().mean()),
|
| 239 |
+
})
|
| 240 |
+
return super().__call__(info_dict, init_action)
|
| 241 |
+
|
| 242 |
+
world = swm.World(env_name='swm/PushT-v1', num_envs=N_ENVS,
|
| 243 |
+
image_shape=(224, 224), max_episode_steps=500)
|
| 244 |
+
config = swm.PlanConfig(horizon=HORIZON, receding_horizon=1,
|
| 245 |
+
action_block=BLOCK,
|
| 246 |
+
history_len=model.predictor.num_frames)
|
| 247 |
+
solver = RecordingPlannerSolver(model, planner, device='cpu')
|
| 248 |
+
policy = HistoryPolicy(
|
| 249 |
+
solver=solver, config=config,
|
| 250 |
+
process={'action': scaler},
|
| 251 |
+
transform={'pixels': img_tf(), 'goal': img_tf()},
|
| 252 |
+
use_frames=True, use_actions=True,
|
| 253 |
+
)
|
| 254 |
+
world.set_policy(policy)
|
| 255 |
+
world.reset(seed=0)
|
| 256 |
+
# episodic mode carries no goal; inject a fixed one so the solver can run
|
| 257 |
+
world.infos['goal'] = world.infos['pixels'].copy()
|
| 258 |
+
|
| 259 |
+
for _ in range(12):
|
| 260 |
+
world.envs.step(world._get_actions())
|
| 261 |
+
|
| 262 |
+
last = seen[-1]
|
| 263 |
+
check('PlannerSolver receives 3 frames', last['n_frames'] == N_FRAMES,
|
| 264 |
+
f'{last["n_frames"]}')
|
| 265 |
+
check('encoded context is NOT degenerate (padding branch did not fire)',
|
| 266 |
+
last['ctx_spread'] > 1e-6, f'||h_first - h_last||^2/D = {last["ctx_spread"]:.6f}')
|
| 267 |
+
check('past_actions reach the planner and are non-zero',
|
| 268 |
+
last['past_absmean'] is not None and last['past_absmean'] > 0,
|
| 269 |
+
f'mean|block| = {last["past_absmean"]}')
|
| 270 |
+
first = seen[0]
|
| 271 |
+
check('first solve of the episode still degrades gracefully',
|
| 272 |
+
first['n_frames'] == 1 and first['past_absmean'] is None)
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
def img_tf(size=224):
|
| 276 |
+
import stable_pretraining as spt
|
| 277 |
+
from torchvision.transforms import v2 as transforms
|
| 278 |
+
return transforms.Compose([
|
| 279 |
+
transforms.ToImage(),
|
| 280 |
+
transforms.ToDtype(torch.float32, scale=True),
|
| 281 |
+
transforms.Normalize(**spt.data.dataset_stats.ImageNet),
|
| 282 |
+
transforms.Resize(size=size),
|
| 283 |
+
])
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
if __name__ == '__main__':
|
| 287 |
+
raise SystemExit(main())
|
code/scripts/train_controller.py
ADDED
|
@@ -0,0 +1,329 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Train the amortized iterative controller against the frozen LeWM.
|
| 2 |
+
|
| 3 |
+
The controller never sees dataset actions as targets. It proposes a plan,
|
| 4 |
+
the frozen predictor says what that plan would cause, and the controller is
|
| 5 |
+
scored on how close the imagined outcome lands to the goal latent.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import argparse
|
| 9 |
+
import json
|
| 10 |
+
import sys
|
| 11 |
+
import time
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
from torch.utils.data import DataLoader
|
| 17 |
+
|
| 18 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 19 |
+
|
| 20 |
+
from lejepa_control.controller import IterativeController # noqa: E402
|
| 21 |
+
from lejepa_control.data import LatentGoalDataset, split_episodes # noqa: E402
|
| 22 |
+
from lejepa_control.losses import BehaviorDensity, refinement_loss, support_loss # noqa: E402
|
| 23 |
+
from lejepa_control.world_model import load_lewm # noqa: E402
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def parse_args():
|
| 27 |
+
p = argparse.ArgumentParser()
|
| 28 |
+
p.add_argument('--latents', default='data/latents')
|
| 29 |
+
p.add_argument('--density', default='data/runs/density/density.pt')
|
| 30 |
+
p.add_argument('--out', default='data/runs/controller')
|
| 31 |
+
p.add_argument('--steps', type=int, default=20000)
|
| 32 |
+
p.add_argument('--batch-size', type=int, default=32)
|
| 33 |
+
p.add_argument('--lr', type=float, default=1e-4)
|
| 34 |
+
p.add_argument('--weight-decay', type=float, default=1e-4)
|
| 35 |
+
p.add_argument('--horizon', type=int, default=5)
|
| 36 |
+
p.add_argument('--refinements', type=int, default=3)
|
| 37 |
+
p.add_argument('--width', type=int, default=256)
|
| 38 |
+
p.add_argument('--depth', type=int, default=4)
|
| 39 |
+
p.add_argument('--heads', type=int, default=8)
|
| 40 |
+
p.add_argument('--dropout', type=float, default=0.1)
|
| 41 |
+
# exp7: tokens live directly in the world model's own latent coordinates
|
| 42 |
+
# (width must equal latent_dim) instead of an arbitrary learned width
|
| 43 |
+
p.add_argument('--no-latent-proj', action='store_true')
|
| 44 |
+
# exp7: one fused operator Phi (single projection + one transformer,
|
| 45 |
+
# depth doubled to match total F+G layers) instead of the split
|
| 46 |
+
# consequence/refine two-network pipeline
|
| 47 |
+
p.add_argument('--fused', action='store_true')
|
| 48 |
+
p.add_argument(
|
| 49 |
+
'--train-seed', type=int, default=None,
|
| 50 |
+
help='seed torch/numpy at startup; default None keeps unseeded '
|
| 51 |
+
'behavior (torch.manual_seed(0) below is unconditional and '
|
| 52 |
+
'unrelated — this only seeds data/init variation across reps)',
|
| 53 |
+
)
|
| 54 |
+
p.add_argument('--alpha', type=float, default=0.05)
|
| 55 |
+
p.add_argument('--lambda-support', type=float, default=0.01)
|
| 56 |
+
# Horizon-matched arrival: index the goal term by the offset the goal was
|
| 57 |
+
# relabeled from, instead of always scoring at block H. Off by default so
|
| 58 |
+
# the original objective's ablations stay reproducible.
|
| 59 |
+
p.add_argument('--arrival-hold', action='store_true')
|
| 60 |
+
p.add_argument('--hold-weight', type=float, default=0.5)
|
| 61 |
+
# in-process: the latent cache is resident, so workers would each copy
|
| 62 |
+
# ~1 GB on spawn to save no real work
|
| 63 |
+
p.add_argument('--workers', type=int, default=0)
|
| 64 |
+
p.add_argument('--log-every', type=int, default=100)
|
| 65 |
+
p.add_argument('--val-every', type=int, default=1000)
|
| 66 |
+
p.add_argument(
|
| 67 |
+
'--curriculum',
|
| 68 |
+
default='0:2,0.25:3,0.5:5',
|
| 69 |
+
help='fraction_of_training:max_goal_offset, comma separated',
|
| 70 |
+
)
|
| 71 |
+
p.add_argument('--wm-name', default='quentinll/lewm-pusht')
|
| 72 |
+
return p.parse_args()
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def parse_curriculum(spec, total_steps):
|
| 76 |
+
stages = []
|
| 77 |
+
for part in spec.split(','):
|
| 78 |
+
frac, offset = part.split(':')
|
| 79 |
+
stages.append((int(float(frac) * total_steps), int(offset)))
|
| 80 |
+
return sorted(stages)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def current_offset(stages, step):
|
| 84 |
+
offset = stages[0][1]
|
| 85 |
+
for start, value in stages:
|
| 86 |
+
if step >= start:
|
| 87 |
+
offset = value
|
| 88 |
+
return offset
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
@torch.no_grad()
|
| 92 |
+
def evaluate(controller, model, loader, device, max_batches=20):
|
| 93 |
+
"""Held-out terminal distance, per-refinement gain, and arrival profile.
|
| 94 |
+
|
| 95 |
+
``arrival`` is the distance at each sample's own goal offset q, which is
|
| 96 |
+
what receding-horizon execution actually depends on; ``terminal`` is the
|
| 97 |
+
distance at block H regardless of q. A controller that defers arrival
|
| 98 |
+
scores well on terminal and badly on arrival.
|
| 99 |
+
"""
|
| 100 |
+
controller.eval()
|
| 101 |
+
terminal, first, arrival, batches = 0.0, 0.0, 0.0, 0
|
| 102 |
+
# mean distance profile over the plan, split by goal offset
|
| 103 |
+
profile = torch.zeros(controller.horizon + 1, controller.horizon, device=device)
|
| 104 |
+
counts = torch.zeros(controller.horizon + 1, device=device)
|
| 105 |
+
for batch in loader:
|
| 106 |
+
q = batch['goal_offset'].to(device).clamp(1, controller.horizon)
|
| 107 |
+
out = controller(
|
| 108 |
+
model,
|
| 109 |
+
batch['context'].to(device),
|
| 110 |
+
batch['past_actions'].to(device),
|
| 111 |
+
batch['goal'].to(device),
|
| 112 |
+
)
|
| 113 |
+
d = out['distances'][-1]
|
| 114 |
+
terminal += d[:, -1].mean().item()
|
| 115 |
+
first += out['distances'][0][:, -1].mean().item()
|
| 116 |
+
arrival += d.gather(1, (q - 1).unsqueeze(1)).squeeze(1).mean().item()
|
| 117 |
+
profile.index_add_(0, q, d)
|
| 118 |
+
counts.index_add_(0, q, torch.ones_like(q, dtype=d.dtype))
|
| 119 |
+
batches += 1
|
| 120 |
+
if batches >= max_batches:
|
| 121 |
+
break
|
| 122 |
+
controller.train()
|
| 123 |
+
profile = (profile / counts.clamp(min=1).unsqueeze(1)).cpu()
|
| 124 |
+
return {
|
| 125 |
+
'terminal': terminal / batches,
|
| 126 |
+
'first': first / batches,
|
| 127 |
+
'arrival': arrival / batches,
|
| 128 |
+
'profile': {
|
| 129 |
+
q: [round(v, 4) for v in profile[q].tolist()]
|
| 130 |
+
for q in range(1, controller.horizon + 1)
|
| 131 |
+
if counts[q] > 0
|
| 132 |
+
},
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def main():
|
| 137 |
+
args = parse_args()
|
| 138 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 139 |
+
if args.train_seed is None:
|
| 140 |
+
torch.manual_seed(0) # current behavior: every run anchored at 0
|
| 141 |
+
else:
|
| 142 |
+
torch.manual_seed(args.train_seed)
|
| 143 |
+
np.random.seed(args.train_seed)
|
| 144 |
+
|
| 145 |
+
out_dir = Path(args.out)
|
| 146 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 147 |
+
|
| 148 |
+
stats = json.loads((Path(args.latents) / 'stats.json').read_text())
|
| 149 |
+
latent_dim = stats['latent_dim']
|
| 150 |
+
|
| 151 |
+
model = load_lewm(name=args.wm_name, device=device) # frozen, eval, requires_grad_(False)
|
| 152 |
+
|
| 153 |
+
# tanh bounds: raw PushT actions live in [-1, 1], and the world model was
|
| 154 |
+
# trained on z-scored actions, so the bound moves into normalized space.
|
| 155 |
+
a_std = torch.tensor(stats['action_std'])
|
| 156 |
+
a_mean = torch.tensor(stats['action_mean'])
|
| 157 |
+
controller = IterativeController(
|
| 158 |
+
latent_dim=latent_dim,
|
| 159 |
+
horizon=args.horizon,
|
| 160 |
+
refinements=args.refinements,
|
| 161 |
+
width=args.width,
|
| 162 |
+
depth=args.depth,
|
| 163 |
+
heads=args.heads,
|
| 164 |
+
dropout=args.dropout,
|
| 165 |
+
action_center=(-a_mean / a_std),
|
| 166 |
+
action_scale=(1.0 / a_std),
|
| 167 |
+
no_latent_proj=args.no_latent_proj,
|
| 168 |
+
fused=args.fused,
|
| 169 |
+
).to(device)
|
| 170 |
+
n_params = sum(p.numel() for p in controller.parameters())
|
| 171 |
+
print(f'controller params: {n_params / 1e6:.2f}M')
|
| 172 |
+
|
| 173 |
+
density, c95 = None, None
|
| 174 |
+
if args.lambda_support > 0 and Path(args.density).exists():
|
| 175 |
+
ckpt = torch.load(args.density, map_location=device)
|
| 176 |
+
density = BehaviorDensity(
|
| 177 |
+
latent_dim=latent_dim, components=ckpt['components']
|
| 178 |
+
).to(device)
|
| 179 |
+
density.load_state_dict(ckpt['state_dict'])
|
| 180 |
+
density.eval().requires_grad_(False)
|
| 181 |
+
c95 = ckpt['c95']
|
| 182 |
+
print(f'support model loaded, c95={c95:.4f}')
|
| 183 |
+
else:
|
| 184 |
+
print('no support model — training with goal loss only')
|
| 185 |
+
|
| 186 |
+
train_eps, val_eps = split_episodes(stats['n_episodes'])
|
| 187 |
+
train_set = LatentGoalDataset(
|
| 188 |
+
args.latents, episodes=train_eps, horizon=args.horizon
|
| 189 |
+
)
|
| 190 |
+
val_set = LatentGoalDataset(
|
| 191 |
+
args.latents, max_offset=5, episodes=val_eps, horizon=args.horizon
|
| 192 |
+
)
|
| 193 |
+
loader = DataLoader(
|
| 194 |
+
train_set,
|
| 195 |
+
batch_size=args.batch_size,
|
| 196 |
+
shuffle=True,
|
| 197 |
+
num_workers=args.workers,
|
| 198 |
+
drop_last=True,
|
| 199 |
+
persistent_workers=args.workers > 0,
|
| 200 |
+
pin_memory=True,
|
| 201 |
+
)
|
| 202 |
+
val_loader = DataLoader(
|
| 203 |
+
val_set, batch_size=args.batch_size, shuffle=True
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
opt = torch.optim.AdamW(
|
| 207 |
+
controller.parameters(), lr=args.lr, weight_decay=args.weight_decay
|
| 208 |
+
)
|
| 209 |
+
sched = torch.optim.lr_scheduler.OneCycleLR(
|
| 210 |
+
opt, max_lr=args.lr, total_steps=args.steps, pct_start=0.05
|
| 211 |
+
)
|
| 212 |
+
stages = parse_curriculum(args.curriculum, args.steps)
|
| 213 |
+
print(f'curriculum: {stages}')
|
| 214 |
+
|
| 215 |
+
step, t0 = 0, time.perf_counter()
|
| 216 |
+
running = {}
|
| 217 |
+
controller.train()
|
| 218 |
+
while step < args.steps:
|
| 219 |
+
for batch in loader:
|
| 220 |
+
offset = current_offset(stages, step)
|
| 221 |
+
if train_set.max_offset != offset:
|
| 222 |
+
train_set.set_max_offset(offset)
|
| 223 |
+
|
| 224 |
+
ctx = batch['context'].to(device, non_blocking=True)
|
| 225 |
+
past = batch['past_actions'].to(device, non_blocking=True)
|
| 226 |
+
goal = batch['goal'].to(device, non_blocking=True)
|
| 227 |
+
q = batch['goal_offset'].to(device, non_blocking=True)
|
| 228 |
+
|
| 229 |
+
out = controller(model, ctx, past, goal)
|
| 230 |
+
if args.arrival_hold:
|
| 231 |
+
loss_refine = refinement_loss(
|
| 232 |
+
out['distances'],
|
| 233 |
+
goal_offset=q,
|
| 234 |
+
hold_weight=args.hold_weight,
|
| 235 |
+
)
|
| 236 |
+
else:
|
| 237 |
+
loss_refine = refinement_loss(out['distances'], alpha=args.alpha)
|
| 238 |
+
|
| 239 |
+
loss = loss_refine
|
| 240 |
+
violation = torch.zeros((), device=device)
|
| 241 |
+
if density is not None:
|
| 242 |
+
# every refinement after the first, every horizon step
|
| 243 |
+
contexts = torch.cat(out['contexts'][1:], dim=0).flatten(0, 1)
|
| 244 |
+
blocks = torch.cat(out['plans'][1:], dim=0).flatten(0, 1)
|
| 245 |
+
loss_support, violation = support_loss(
|
| 246 |
+
density, contexts, blocks, c95
|
| 247 |
+
)
|
| 248 |
+
loss = loss + args.lambda_support * loss_support
|
| 249 |
+
running['support'] = (
|
| 250 |
+
running.get('support', 0.0) + loss_support.item()
|
| 251 |
+
)
|
| 252 |
+
|
| 253 |
+
opt.zero_grad(set_to_none=True)
|
| 254 |
+
loss.backward()
|
| 255 |
+
grad = torch.nn.utils.clip_grad_norm_(controller.parameters(), 1.0)
|
| 256 |
+
opt.step()
|
| 257 |
+
sched.step()
|
| 258 |
+
|
| 259 |
+
d_first = out['distances'][0][:, -1].mean().item()
|
| 260 |
+
d_last = out['distances'][-1][:, -1].mean().item()
|
| 261 |
+
d_arrival = (
|
| 262 |
+
out['distances'][-1]
|
| 263 |
+
.gather(1, (q.clamp(1, args.horizon) - 1).unsqueeze(1))
|
| 264 |
+
.squeeze(1)
|
| 265 |
+
.mean()
|
| 266 |
+
.item()
|
| 267 |
+
)
|
| 268 |
+
running['loss'] = running.get('loss', 0.0) + loss.item()
|
| 269 |
+
running['terminal'] = running.get('terminal', 0.0) + d_last
|
| 270 |
+
running['arrival'] = running.get('arrival', 0.0) + d_arrival
|
| 271 |
+
running['gain'] = running.get('gain', 0.0) + (d_first - d_last)
|
| 272 |
+
running['violation'] = (
|
| 273 |
+
running.get('violation', 0.0) + violation.item()
|
| 274 |
+
)
|
| 275 |
+
running['grad'] = running.get('grad', 0.0) + grad.item()
|
| 276 |
+
|
| 277 |
+
step += 1
|
| 278 |
+
if step % args.log_every == 0:
|
| 279 |
+
n = args.log_every
|
| 280 |
+
rate = step / (time.perf_counter() - t0)
|
| 281 |
+
msg = (
|
| 282 |
+
f'step {step:6d} H_goal<={offset} '
|
| 283 |
+
f'loss {running["loss"] / n:.4f} '
|
| 284 |
+
f'terminal {running["terminal"] / n:.4f} '
|
| 285 |
+
f'arrival {running["arrival"] / n:.4f} '
|
| 286 |
+
f'gain {running["gain"] / n:+.4f} '
|
| 287 |
+
f'viol {running["violation"] / n:.3f} '
|
| 288 |
+
f'grad {running["grad"] / n:.2f} '
|
| 289 |
+
f'{rate:.1f} it/s'
|
| 290 |
+
)
|
| 291 |
+
if 'support' in running:
|
| 292 |
+
msg += f' support {running["support"] / n:.4f}'
|
| 293 |
+
print(msg, flush=True)
|
| 294 |
+
running = {}
|
| 295 |
+
|
| 296 |
+
if step % args.val_every == 0 or step == args.steps:
|
| 297 |
+
val = evaluate(controller, model, val_loader, device)
|
| 298 |
+
print(
|
| 299 |
+
f' [val] terminal K={args.refinements}: '
|
| 300 |
+
f'{val["terminal"]:.4f} K=0: {val["first"]:.4f} '
|
| 301 |
+
f'gain {val["first"] - val["terminal"]:+.4f} '
|
| 302 |
+
f'arrival {val["arrival"]:.4f} '
|
| 303 |
+
f'steps {controller.step_sizes_repr()}',
|
| 304 |
+
flush=True,
|
| 305 |
+
)
|
| 306 |
+
for q_val, prof in val['profile'].items():
|
| 307 |
+
print(f' q={q_val}: {prof}', flush=True)
|
| 308 |
+
torch.save(
|
| 309 |
+
{
|
| 310 |
+
'state_dict': controller.state_dict(),
|
| 311 |
+
'args': vars(args),
|
| 312 |
+
'step': step,
|
| 313 |
+
'val_terminal': val['terminal'],
|
| 314 |
+
'val_arrival': val['arrival'],
|
| 315 |
+
'val_profile': val['profile'],
|
| 316 |
+
'action_mean': stats['action_mean'],
|
| 317 |
+
'action_std': stats['action_std'],
|
| 318 |
+
},
|
| 319 |
+
out_dir / 'controller.pt',
|
| 320 |
+
)
|
| 321 |
+
|
| 322 |
+
if step >= args.steps:
|
| 323 |
+
break
|
| 324 |
+
|
| 325 |
+
print(f'done in {(time.perf_counter() - t0) / 60:.1f} min -> {out_dir}')
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
if __name__ == '__main__':
|
| 329 |
+
main()
|
code/scripts/train_decoder.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Train the latent->pixel decoder (LeWM appendix D), for visualization only.
|
| 2 |
+
|
| 3 |
+
python tools/make_decoder_cache.py --frames 8000
|
| 4 |
+
python scripts/train_decoder.py --steps 2000
|
| 5 |
+
|
| 6 |
+
The world model is **not** touched: the decoder reads cached latents, so no
|
| 7 |
+
gradient can reach the encoder even by accident. That matches how LeWM used it
|
| 8 |
+
(Fig. 8 is a read-only probe) and matters, because the paper's own appendix G
|
| 9 |
+
ablation shows that letting a reconstruction loss into LeWM training *hurts*
|
| 10 |
+
control — PushT success 96.0 ± 2.83 without it, 86.0 ± 7.54 with it.
|
| 11 |
+
|
| 12 |
+
Presets pick the speed/detail trade-off; all were measured on this CPU box at
|
| 13 |
+
batch 32:
|
| 14 |
+
|
| 15 |
+
paper 224px, 16px patches, W384 d4 7.6M params ~6.0 s/step
|
| 16 |
+
cpu 224px, 16px patches, W256 d3 2.7M params ~1.4 s/step (default)
|
| 17 |
+
fast 224px, 28px patches, W256 d3 3.0M params ~0.5 s/step
|
| 18 |
+
|
| 19 |
+
``fast`` keeps the full 224 output but tiles it 8x8, so a 10px pusher lands
|
| 20 |
+
inside one patch — fine for "where is the block", poor for fine pose. ``cpu``
|
| 21 |
+
tiles 14x14 and is the recommended default here.
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
import argparse
|
| 25 |
+
import json
|
| 26 |
+
import os
|
| 27 |
+
import sys
|
| 28 |
+
import time
|
| 29 |
+
from pathlib import Path
|
| 30 |
+
|
| 31 |
+
import numpy as np
|
| 32 |
+
import torch
|
| 33 |
+
|
| 34 |
+
REPO = Path(__file__).resolve().parents[1]
|
| 35 |
+
sys.path.insert(0, str(REPO))
|
| 36 |
+
os.environ.setdefault('STABLEWM_HOME', str(REPO / 'data' / 'swm_home'))
|
| 37 |
+
|
| 38 |
+
from lejepa_control.decoder import LatentDecoder, reconstruction_loss # noqa: E402
|
| 39 |
+
from tools.paths import artifact_dir # noqa: E402
|
| 40 |
+
|
| 41 |
+
PRESETS = {
|
| 42 |
+
'paper': dict(patch_size=16, hidden_dim=384, depth=4, heads=6),
|
| 43 |
+
'cpu': dict(patch_size=16, hidden_dim=256, depth=3, heads=4),
|
| 44 |
+
'fast': dict(patch_size=28, hidden_dim=256, depth=3, heads=4),
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def parse_args():
|
| 49 |
+
p = argparse.ArgumentParser()
|
| 50 |
+
p.add_argument('--cache', default=None,
|
| 51 |
+
help='default: $LEJEPA_DATA/decoder_cache, else data/decoder_cache')
|
| 52 |
+
p.add_argument('--out', default='data/runs/decoder')
|
| 53 |
+
p.add_argument('--preset', choices=list(PRESETS), default='cpu')
|
| 54 |
+
p.add_argument('--steps', type=int, default=2000)
|
| 55 |
+
p.add_argument('--batch-size', type=int, default=32)
|
| 56 |
+
p.add_argument('--lr', type=float, default=3e-4)
|
| 57 |
+
p.add_argument('--weight-decay', type=float, default=0.05)
|
| 58 |
+
p.add_argument('--warmup', type=int, default=100)
|
| 59 |
+
p.add_argument('--val-frac', type=float, default=0.05)
|
| 60 |
+
p.add_argument('--log-every', type=int, default=50)
|
| 61 |
+
p.add_argument('--preview-every', type=int, default=500)
|
| 62 |
+
p.add_argument('--seed', type=int, default=0)
|
| 63 |
+
return p.parse_args()
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def save_preview(decoder, z, target, path, device):
|
| 67 |
+
"""Side-by-side target/reconstruction strip — the thing you actually look at."""
|
| 68 |
+
from PIL import Image
|
| 69 |
+
|
| 70 |
+
decoder.eval()
|
| 71 |
+
with torch.no_grad():
|
| 72 |
+
pred = decoder(z.to(device)).clamp(0, 1).cpu()
|
| 73 |
+
decoder.train()
|
| 74 |
+
|
| 75 |
+
n = min(6, len(z))
|
| 76 |
+
rows = []
|
| 77 |
+
for tensor in (target[:n], pred[:n]):
|
| 78 |
+
row = tensor.permute(0, 2, 3, 1).numpy()
|
| 79 |
+
rows.append(np.concatenate(list(row), axis=1))
|
| 80 |
+
strip = (np.concatenate(rows, axis=0) * 255).astype(np.uint8)
|
| 81 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 82 |
+
Image.fromarray(strip).save(path)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def main():
|
| 86 |
+
args = parse_args()
|
| 87 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 88 |
+
torch.manual_seed(args.seed)
|
| 89 |
+
|
| 90 |
+
cache = artifact_dir('decoder_cache', args.cache)
|
| 91 |
+
meta = json.loads((cache / 'meta.json').read_text())
|
| 92 |
+
images = np.load(cache / 'images.npy', mmap_mode='r') # (N,3,H,W) uint8
|
| 93 |
+
latents = np.load(cache / 'latents.npy') # (N,192) fp16
|
| 94 |
+
assert len(images) == len(latents), 'cache is inconsistent'
|
| 95 |
+
print(f'cache: {len(images)} pairs, latent={meta["latent_source"]}, '
|
| 96 |
+
f'source={meta["source"]}, image={meta["image_size"]}px')
|
| 97 |
+
|
| 98 |
+
# Held-out split so the reported loss is not just memorisation. The decoder
|
| 99 |
+
# has 2.7M params against 8k images and will happily overfit.
|
| 100 |
+
rng = np.random.default_rng(args.seed)
|
| 101 |
+
perm = rng.permutation(len(images))
|
| 102 |
+
n_val = max(1, int(len(images) * args.val_frac))
|
| 103 |
+
val_idx, train_idx = np.sort(perm[:n_val]), perm[n_val:]
|
| 104 |
+
print(f' train {len(train_idx)} / val {len(val_idx)}')
|
| 105 |
+
|
| 106 |
+
cfg = dict(
|
| 107 |
+
latent_dim=int(meta['latent_dim']),
|
| 108 |
+
image_size=int(meta['image_size']),
|
| 109 |
+
**PRESETS[args.preset],
|
| 110 |
+
)
|
| 111 |
+
decoder = LatentDecoder(**cfg)
|
| 112 |
+
n_params = sum(p.numel() for p in decoder.parameters())
|
| 113 |
+
print(f'decoder[{args.preset}]: {n_params / 1e6:.2f}M params, '
|
| 114 |
+
f'P={decoder.num_patches} patches of {decoder.patch_size}px')
|
| 115 |
+
|
| 116 |
+
def batch(idx_pool, size):
|
| 117 |
+
idx = np.sort(rng.choice(idx_pool, size, replace=False))
|
| 118 |
+
z = torch.from_numpy(latents[idx].astype(np.float32))
|
| 119 |
+
x = torch.from_numpy(np.asarray(images[idx], dtype=np.float32) / 255.0)
|
| 120 |
+
return z, x
|
| 121 |
+
|
| 122 |
+
# Baseline: predicting the dataset mean image. Any decoder that does not
|
| 123 |
+
# beat this has learned nothing about the latent. On PushT it is a strict
|
| 124 |
+
# bar, not a weak one — the frames are mostly identical white background,
|
| 125 |
+
# so a constant image is already a decent predictor and only the agent,
|
| 126 |
+
# block and target carry error.
|
| 127 |
+
mean_img = torch.from_numpy(
|
| 128 |
+
np.asarray(images[train_idx[:1000]], dtype=np.float32).mean(0) / 255.0
|
| 129 |
+
)
|
| 130 |
+
zv, xv = batch(val_idx, min(64, len(val_idx)))
|
| 131 |
+
baseline = float(((mean_img.unsqueeze(0) - xv) ** 2).mean())
|
| 132 |
+
print(f' mean-image baseline val MSE = {baseline:.5f}')
|
| 133 |
+
|
| 134 |
+
decoder.init_output_at(mean_img.mean(dim=(1, 2)))
|
| 135 |
+
decoder = decoder.to(device)
|
| 136 |
+
print(f' output head parked on the mean colour '
|
| 137 |
+
f'{[round(float(v), 3) for v in mean_img.mean(dim=(1, 2))]}\n')
|
| 138 |
+
|
| 139 |
+
# Built after the init and the device move so AdamW never sees stale params.
|
| 140 |
+
opt = torch.optim.AdamW(
|
| 141 |
+
decoder.parameters(), lr=args.lr, weight_decay=args.weight_decay
|
| 142 |
+
)
|
| 143 |
+
sched = torch.optim.lr_scheduler.LambdaLR(
|
| 144 |
+
opt,
|
| 145 |
+
lambda s: min(1.0, (s + 1) / max(1, args.warmup))
|
| 146 |
+
* (0.5 * (1 + np.cos(np.pi * min(1.0, s / args.steps)))),
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
out_dir = REPO / args.out
|
| 150 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 151 |
+
history = []
|
| 152 |
+
t0 = time.perf_counter()
|
| 153 |
+
|
| 154 |
+
for step in range(1, args.steps + 1):
|
| 155 |
+
z, x = batch(train_idx, args.batch_size)
|
| 156 |
+
loss = reconstruction_loss(decoder(z.to(device)), x.to(device))
|
| 157 |
+
opt.zero_grad(set_to_none=True)
|
| 158 |
+
loss.backward()
|
| 159 |
+
torch.nn.utils.clip_grad_norm_(decoder.parameters(), 1.0)
|
| 160 |
+
opt.step()
|
| 161 |
+
sched.step()
|
| 162 |
+
|
| 163 |
+
if step % args.log_every == 0 or step == 1:
|
| 164 |
+
decoder.eval()
|
| 165 |
+
with torch.no_grad():
|
| 166 |
+
val = float(reconstruction_loss(decoder(zv.to(device)), xv.to(device)))
|
| 167 |
+
decoder.train()
|
| 168 |
+
rate = step / (time.perf_counter() - t0)
|
| 169 |
+
train_loss = float(loss.detach())
|
| 170 |
+
history.append({'step': step, 'train': train_loss, 'val': val})
|
| 171 |
+
print(f' step {step:5}/{args.steps} train {train_loss:.5f} '
|
| 172 |
+
f'val {val:.5f} ({val / baseline:.2f}x baseline) '
|
| 173 |
+
f'{rate:.2f} it/s eta {(args.steps - step) / rate / 60:.1f} min',
|
| 174 |
+
flush=True)
|
| 175 |
+
|
| 176 |
+
if step % args.preview_every == 0 or step == args.steps:
|
| 177 |
+
save_preview(decoder, zv, xv, out_dir / f'preview_{step:05d}.png', device)
|
| 178 |
+
|
| 179 |
+
torch.save({
|
| 180 |
+
'state_dict': decoder.state_dict(),
|
| 181 |
+
'config': cfg,
|
| 182 |
+
'preset': args.preset,
|
| 183 |
+
'meta': meta,
|
| 184 |
+
'step': args.steps,
|
| 185 |
+
'history': history,
|
| 186 |
+
'baseline_val_mse': baseline,
|
| 187 |
+
}, out_dir / 'decoder.pt')
|
| 188 |
+
|
| 189 |
+
print(f'\nsaved -> {out_dir / "decoder.pt"}')
|
| 190 |
+
print(f' {(time.perf_counter() - t0) / 60:.1f} min')
|
| 191 |
+
print(f' final val MSE {history[-1]["val"]:.5f} vs baseline {baseline:.5f}')
|
| 192 |
+
print(f' previews: {out_dir}/preview_*.png (top row target, bottom row decoded)')
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
if __name__ == '__main__':
|
| 196 |
+
main()
|
code/scripts/train_density.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fit the conditional behavior-density model beta(b | C) on real transitions.
|
| 2 |
+
|
| 3 |
+
This is a support model for the controller's plans, not a policy. Training it
|
| 4 |
+
separately keeps the controller objective free of behavior cloning: the
|
| 5 |
+
controller is only penalized when a plan leaves the region the dataset covers,
|
| 6 |
+
measured against the 95th percentile of held-out real transitions.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import argparse
|
| 10 |
+
import json
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
from torch.utils.data import DataLoader
|
| 17 |
+
|
| 18 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 19 |
+
|
| 20 |
+
from lejepa_control.data import LatentGoalDataset, split_episodes # noqa: E402
|
| 21 |
+
from lejepa_control.losses import BehaviorDensity # noqa: E402
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def main():
|
| 25 |
+
parser = argparse.ArgumentParser()
|
| 26 |
+
parser.add_argument('--latents', default='data/latents')
|
| 27 |
+
parser.add_argument('--out', default='data/runs/density')
|
| 28 |
+
parser.add_argument('--steps', type=int, default=4000)
|
| 29 |
+
parser.add_argument('--batch-size', type=int, default=256)
|
| 30 |
+
parser.add_argument('--lr', type=float, default=1e-3)
|
| 31 |
+
parser.add_argument('--components', type=int, default=16)
|
| 32 |
+
# in-process is fastest here: the latent cache is resident, so workers
|
| 33 |
+
# would each copy ~1 GB on spawn to save no real work
|
| 34 |
+
parser.add_argument('--workers', type=int, default=0)
|
| 35 |
+
args = parser.parse_args()
|
| 36 |
+
|
| 37 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 38 |
+
torch.manual_seed(0)
|
| 39 |
+
|
| 40 |
+
stats = json.loads((Path(args.latents) / 'stats.json').read_text())
|
| 41 |
+
train_eps, val_eps = split_episodes(stats['n_episodes'])
|
| 42 |
+
|
| 43 |
+
train_set = LatentGoalDataset(args.latents, episodes=train_eps)
|
| 44 |
+
val_set = LatentGoalDataset(args.latents, episodes=val_eps)
|
| 45 |
+
print(f'train clips {len(train_set)} val clips {len(val_set)}')
|
| 46 |
+
|
| 47 |
+
loader = DataLoader(
|
| 48 |
+
train_set,
|
| 49 |
+
batch_size=args.batch_size,
|
| 50 |
+
shuffle=True,
|
| 51 |
+
num_workers=args.workers,
|
| 52 |
+
drop_last=True,
|
| 53 |
+
persistent_workers=args.workers > 0,
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
density = BehaviorDensity(
|
| 57 |
+
latent_dim=stats['latent_dim'], components=args.components
|
| 58 |
+
).to(device)
|
| 59 |
+
opt = torch.optim.AdamW(density.parameters(), lr=args.lr, weight_decay=1e-4)
|
| 60 |
+
|
| 61 |
+
step = 0
|
| 62 |
+
density.train()
|
| 63 |
+
while step < args.steps:
|
| 64 |
+
for batch in loader:
|
| 65 |
+
ctx = batch['context'].to(device)
|
| 66 |
+
block = batch['real_action'].to(device)
|
| 67 |
+
|
| 68 |
+
loss = density.nll_per_dim(ctx, block).mean()
|
| 69 |
+
opt.zero_grad(set_to_none=True)
|
| 70 |
+
loss.backward()
|
| 71 |
+
torch.nn.utils.clip_grad_norm_(density.parameters(), 1.0)
|
| 72 |
+
opt.step()
|
| 73 |
+
|
| 74 |
+
step += 1
|
| 75 |
+
if step % 500 == 0:
|
| 76 |
+
print(f'step {step:5d} nll/dim {loss.item():.4f}', flush=True)
|
| 77 |
+
if step >= args.steps:
|
| 78 |
+
break
|
| 79 |
+
|
| 80 |
+
# --- calibrate c_95 on held-out real transitions ----------------------
|
| 81 |
+
density.eval()
|
| 82 |
+
val_loader = DataLoader(
|
| 83 |
+
val_set, batch_size=512, shuffle=True, num_workers=args.workers
|
| 84 |
+
)
|
| 85 |
+
scores = []
|
| 86 |
+
with torch.no_grad():
|
| 87 |
+
for batch in val_loader:
|
| 88 |
+
s = density.nll_per_dim(
|
| 89 |
+
batch['context'].to(device), batch['real_action'].to(device)
|
| 90 |
+
)
|
| 91 |
+
scores.append(s.cpu().numpy())
|
| 92 |
+
if sum(len(x) for x in scores) >= 200_000:
|
| 93 |
+
break
|
| 94 |
+
scores = np.concatenate(scores)
|
| 95 |
+
c95 = float(np.percentile(scores, 95))
|
| 96 |
+
|
| 97 |
+
out_dir = Path(args.out)
|
| 98 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 99 |
+
torch.save(
|
| 100 |
+
{
|
| 101 |
+
'state_dict': density.state_dict(),
|
| 102 |
+
'c95': c95,
|
| 103 |
+
'components': args.components,
|
| 104 |
+
'latent_dim': stats['latent_dim'],
|
| 105 |
+
},
|
| 106 |
+
out_dir / 'density.pt',
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
+
print(
|
| 110 |
+
f'held-out nll/dim: mean {scores.mean():.4f} '
|
| 111 |
+
f'p50 {np.percentile(scores, 50):.4f} '
|
| 112 |
+
f'p95 {c95:.4f} p99 {np.percentile(scores, 99):.4f}'
|
| 113 |
+
)
|
| 114 |
+
print(f'saved -> {out_dir / "density.pt"}')
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
if __name__ == '__main__':
|
| 118 |
+
main()
|