nottygian commited on
Commit
dc9f917
·
verified ·
1 Parent(s): 872cf4d

Push scripts

Browse files
code/scripts/__pycache__/train_decoder.cpython-312.pyc ADDED
Binary file (13.4 kB). View file
 
code/scripts/collect_pusht.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Collect a PushT dataset with stable_worldmodel's WeakPolicy.
2
+
3
+ Writes shards under $STABLEWM_HOME/datasets/<name>/ in lance format.
4
+ """
5
+
6
+ import argparse
7
+ import os
8
+ from pathlib import Path
9
+
10
+ import numpy as np
11
+
12
+
13
+ def main():
14
+ p = argparse.ArgumentParser()
15
+ p.add_argument('--name', default='pusht_weak_100')
16
+ p.add_argument('--episodes', type=int, default=2000)
17
+ p.add_argument('--shards', type=int, default=10)
18
+ p.add_argument('--num-envs', type=int, default=10)
19
+ p.add_argument('--max-episode-steps', type=int, default=100)
20
+ p.add_argument('--dist-constraint', type=int, default=100)
21
+ p.add_argument('--seed', type=int, default=3072)
22
+ p.add_argument('--cache-dir', default=None)
23
+ args = p.parse_args()
24
+
25
+ import stable_worldmodel as swm
26
+ from stable_worldmodel.envs.pusht import WeakPolicy
27
+
28
+ world = swm.World(
29
+ 'swm/PushT-v1',
30
+ num_envs=args.num_envs,
31
+ image_shape=(224, 224),
32
+ max_episode_steps=args.max_episode_steps,
33
+ render_mode='rgb_array',
34
+ )
35
+ world.set_policy(WeakPolicy(dist_constraint=args.dist_constraint))
36
+
37
+ root = Path(
38
+ args.cache_dir
39
+ or os.getenv('STABLEWM_HOME')
40
+ or swm.data.utils.get_cache_dir()
41
+ )
42
+ out_dir = root / 'datasets' / args.name
43
+ out_dir.mkdir(parents=True, exist_ok=True)
44
+
45
+ per_shard = args.episodes // args.shards
46
+ rng = np.random.default_rng(args.seed)
47
+
48
+ for i in range(args.shards):
49
+ shard = out_dir / f'shard_{i}.lance'
50
+ if shard.exists():
51
+ print(f'[skip] {shard} exists')
52
+ rng.integers(0, 1_000_000)
53
+ continue
54
+ print(f'[collect] shard {i + 1}/{args.shards} -> {shard}')
55
+ world.collect(
56
+ shard,
57
+ episodes=per_shard,
58
+ seed=rng.integers(0, 1_000_000).item(),
59
+ )
60
+
61
+ print(f'done: {out_dir}')
62
+
63
+
64
+ if __name__ == '__main__':
65
+ main()
code/scripts/compare_planners.py ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Head-to-head comparison of every planner through one harness.
2
+
3
+ The cross-architecture numbers already in the repo are not comparable to each
4
+ other. ``scripts/eval_controller.py`` runs the plain ``WorldModelPolicy``,
5
+ which -- per ``eval_planner.py``'s header -- plans from one repeated frame and
6
+ zero past action blocks, because ``stable_worldmodel`` declares ``history_len``
7
+ and ``action_history`` and implements neither. ``eval_planner.py`` supplies both
8
+ via ``HistoryPolicy``. A controller scored by the first and a planner scored by
9
+ the second differ in their observations, not just their architecture.
10
+
11
+ So every row here goes through ``eval_planner.py``, which can evaluate the
12
+ controller, the recursive planner, the cross-attention planner, CEM and the
13
+ random floor, with history on for all of them and one shared seed -- meaning
14
+ identical held-out start/goal pairs and paired McNemar throughout.
15
+ """
16
+
17
+ import argparse
18
+ import subprocess
19
+ import sys
20
+ from pathlib import Path
21
+
22
+ PY = sys.executable
23
+ ROOT = Path(__file__).resolve().parents[1]
24
+ EVAL = 'lejepa_control_2/scripts/eval_planner.py'
25
+
26
+
27
+ def parse_args():
28
+ p = argparse.ArgumentParser()
29
+ p.add_argument('--controller', default='data/runs/ah_hold0.5/controller.pt')
30
+ p.add_argument('--xa', default='data/runs/planner_xa_D/planner.pt')
31
+ p.add_argument('--recursive', default='data/runs/planner_D10_extended/planner.pt')
32
+ p.add_argument('--horizon', type=int, default=5)
33
+ p.add_argument('--receding-horizon', type=int, nargs='+', default=[1, 5])
34
+ p.add_argument('--controller-k', type=int, nargs='+', default=[3, 5])
35
+ p.add_argument('--xa-k', type=int, nargs='+', default=[2, 3, 5])
36
+ p.add_argument('--seeds', type=int, nargs='+', default=[42, 43, 44])
37
+ p.add_argument('--num-eval', type=int, default=50)
38
+ p.add_argument('--goal-offset', type=int, default=25)
39
+ p.add_argument('--out', default='data/runs/eval_compare')
40
+ p.add_argument('--skip-random', action='store_true')
41
+ return p.parse_args()
42
+
43
+
44
+ def main():
45
+ args = parse_args()
46
+ jobs = []
47
+
48
+ for rh in args.receding_horizon:
49
+ for seed in args.seeds:
50
+ common = [
51
+ '--horizon', str(args.horizon),
52
+ '--receding-horizon', str(rh),
53
+ '--goal-offset', str(args.goal_offset),
54
+ '--num-eval', str(args.num_eval),
55
+ '--seed', str(seed),
56
+ '--out', args.out,
57
+ ]
58
+ if not args.skip_random and rh == args.receding_horizon[0]:
59
+ jobs.append(['--planner', 'random', *common])
60
+
61
+ for k in args.controller_k:
62
+ jobs.append([
63
+ '--planner', 'controller',
64
+ '--controller', args.controller,
65
+ '--refinements', str(k), '--tag', f'ctrl_K{k}', *common,
66
+ ])
67
+
68
+ if Path(ROOT / args.recursive).exists():
69
+ jobs.append([
70
+ '--planner', 'planner', '--checkpoint', args.recursive,
71
+ '--tag', 'recursive', *common,
72
+ ])
73
+
74
+ if Path(ROOT / args.xa).exists():
75
+ for k in args.xa_k:
76
+ jobs.append([
77
+ '--planner', 'planner', '--checkpoint', args.xa,
78
+ '--cycles', str(k), '--tag', f'xa_K{k}', *common,
79
+ ])
80
+
81
+ print(f'{len(jobs)} evaluations -> {args.out}')
82
+ for i, extra in enumerate(jobs, 1):
83
+ print(f'\n[{i}/{len(jobs)}] {" ".join(extra)}', flush=True)
84
+ code = subprocess.run([PY, EVAL, *extra], cwd=ROOT).returncode
85
+ if code != 0:
86
+ print(f'!! exited {code}, continuing', flush=True)
87
+
88
+ print(f'\nappended to {Path(args.out) / "results.jsonl"}')
89
+
90
+
91
+ if __name__ == '__main__':
92
+ main()
code/scripts/diagnostics.py ADDED
@@ -0,0 +1,223 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Diagnostics for horizon-reset procrastination and the K=5 regression.
2
+
3
+ Two questions, both answered in the world model's latent space against the
4
+ same held-out latents used for validation:
5
+
6
+ 1. Under receding-horizon execution, does the closed loop have a fixed point
7
+ outside the success radius? Fitting ``D_{n+1} = c*D_n + b`` gives
8
+ ``D* = b/(1-c)`` when ``0 < c < 1``. A positive intercept means each
9
+ replan removes a *fraction* of the remaining distance but adds a floor, so
10
+ the loop stalls short of the goal no matter how long it runs.
11
+
12
+ 2. Does refinement keep improving predicted cost past the trained depth K=3
13
+ even as real success degrades? If predicted cost improves while success
14
+ drops, the extra refinements are exploiting the world model rather than
15
+ planning better.
16
+
17
+ Question 2 originally measured only the *imagined* side, which cannot separate
18
+ "the plan got better" from "the plan left the region where the world model is
19
+ trustworthy" — both look like falling predicted cost. The support score
20
+ ``r(C, b) = -log beta(b | C) / A`` against the fitted GMM answers that directly:
21
+ if the violation fraction rises with k while predicted cost falls, the extra
22
+ refinements are buying imagined progress with out-of-distribution actions.
23
+ """
24
+
25
+ import argparse
26
+ import json
27
+ import sys
28
+ from pathlib import Path
29
+
30
+ import numpy as np
31
+ import torch
32
+
33
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
34
+
35
+ from lejepa_control.data import LatentGoalDataset, split_episodes # noqa: E402
36
+ from lejepa_control.losses import BehaviorDensity # noqa: E402
37
+ from lejepa_control.rollout import goal_distance, rollout_plan # noqa: E402
38
+ from lejepa_control.solver import load_controller # noqa: E402
39
+ from lejepa_control.world_model import load_lewm # noqa: E402
40
+
41
+
42
+ def parse_args():
43
+ p = argparse.ArgumentParser()
44
+ p.add_argument('--controller', default='data/runs/controller/controller.pt')
45
+ p.add_argument('--latents', default='data/latents')
46
+ p.add_argument('--density', default='data/runs/density/density.pt')
47
+ p.add_argument('--samples', type=int, default=256)
48
+ p.add_argument('--horizon', type=int, default=5)
49
+ p.add_argument('--replans', type=int, default=12)
50
+ p.add_argument('--max-k', type=int, default=8)
51
+ p.add_argument('--tag', default='main')
52
+ p.add_argument('--out', default='data/runs/diagnostics')
53
+ return p.parse_args()
54
+
55
+
56
+ @torch.no_grad()
57
+ def closed_loop_trace(controller, model, batch, device, execute, replans):
58
+ """Latent goal distance after each replan, simulating in latent space.
59
+
60
+ The world model is its own simulator here: whatever the predictor says
61
+ the executed blocks caused becomes the next context. That is exactly the
62
+ loop the real MPC runs, minus the env, so it isolates the planner's
63
+ dynamics from simulator mismatch.
64
+ """
65
+ ctx = batch['context'].to(device)
66
+ past = batch['past_actions'].to(device)
67
+ goal = batch['goal'].to(device)
68
+ num_context = controller.num_context
69
+
70
+ trace = []
71
+ for _ in range(replans):
72
+ out = controller(model, ctx, past, goal)
73
+ plan = out['plans'][-1]
74
+
75
+ # execute the first `execute` blocks and advance the latent state
76
+ pred, frames = rollout_plan(
77
+ model, ctx, past, plan[:, :execute], return_frames=True
78
+ )
79
+ trace.append(goal_distance(pred, goal)[:, execute - 1].cpu().numpy())
80
+
81
+ ctx = frames[:, -num_context:]
82
+ past = torch.cat([past, plan[:, :execute]], dim=1)[:, -(num_context - 1):]
83
+ return np.stack(trace, axis=1) # (B, replans)
84
+
85
+
86
+ def fit_contraction(trace):
87
+ """Least-squares ``D_{n+1} = c*D_n + b`` over all consecutive pairs."""
88
+ x = trace[:, :-1].ravel()
89
+ y = trace[:, 1:].ravel()
90
+ A = np.stack([x, np.ones_like(x)], axis=1)
91
+ (c, b), *_ = np.linalg.lstsq(A, y, rcond=None)
92
+ resid = y - (c * x + b)
93
+ ss = 1 - resid.var() / y.var() if y.var() > 0 else float('nan')
94
+ fixed = b / (1 - c) if abs(1 - c) > 1e-9 else float('inf')
95
+ return float(c), float(b), float(fixed), float(ss)
96
+
97
+
98
+ @torch.no_grad()
99
+ def refinement_trace(controller, model, batch, device, max_k,
100
+ density=None, c95=None):
101
+ """Predicted goal cost after each refinement, out to ``max_k``.
102
+
103
+ Also returns the support-violation fraction per refinement when a density
104
+ model is supplied — the imagined cost and the violation fraction moving in
105
+ opposite directions is the exploitation signature.
106
+ """
107
+ saved = controller.refinements
108
+ try:
109
+ controller.refinements = max_k
110
+ out = controller(
111
+ model,
112
+ batch['context'].to(device),
113
+ batch['past_actions'].to(device),
114
+ batch['goal'].to(device),
115
+ )
116
+ q = batch['goal_offset'].to(device).clamp(1, controller.horizon)
117
+ costs, arrivals, moves, violations, scores = [], [], [], [], []
118
+ prev = None
119
+ for k, d in enumerate(out['distances']):
120
+ costs.append(float(d[:, -1].mean()))
121
+ arrivals.append(
122
+ float(d.gather(1, (q - 1).unsqueeze(1)).squeeze(1).mean())
123
+ )
124
+ plan = out['plans'][k]
125
+ moves.append(
126
+ 0.0 if prev is None else float((plan - prev).abs().mean())
127
+ )
128
+ prev = plan
129
+
130
+ if density is not None and c95 is not None:
131
+ # score every block of the plan at the latent history it is
132
+ # actually proposed from, which is what the support term does
133
+ ctx = out['contexts'][k].flatten(0, 1) # (B*H, N, D)
134
+ score = density.nll_per_dim(ctx, plan.flatten(0, 1))
135
+ violations.append(float((score > c95).float().mean()))
136
+ scores.append(float(score.mean()))
137
+ return costs, arrivals, moves, violations, scores
138
+ finally:
139
+ controller.refinements = saved
140
+
141
+
142
+ def main():
143
+ args = parse_args()
144
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
145
+ torch.manual_seed(0)
146
+
147
+ stats = json.loads((Path(args.latents) / 'stats.json').read_text())
148
+ model = load_lewm(device=device)
149
+ controller, ckpt = load_controller(
150
+ args.controller, latent_dim=stats['latent_dim'], device=device
151
+ )
152
+ controller.eval()
153
+ print(f'controller step {ckpt["step"]}, K={controller.refinements}')
154
+
155
+ _, val_eps = split_episodes(stats['n_episodes'])
156
+ val = LatentGoalDataset(
157
+ args.latents, max_offset=5, episodes=val_eps, horizon=args.horizon
158
+ )
159
+ idx = np.random.default_rng(0).choice(len(val), args.samples, replace=False)
160
+ batch = {
161
+ k: torch.stack([val[int(i)][k] for i in idx])
162
+ for k in ('context', 'past_actions', 'goal', 'goal_offset')
163
+ }
164
+
165
+ density, c95 = None, None
166
+ if Path(args.density).exists():
167
+ d_ckpt = torch.load(args.density, map_location=device, weights_only=False)
168
+ density = BehaviorDensity(
169
+ latent_dim=d_ckpt['latent_dim'], components=d_ckpt['components']
170
+ )
171
+ density.load_state_dict(d_ckpt['state_dict'])
172
+ density.to(device).eval().requires_grad_(False)
173
+ c95 = float(d_ckpt['c95'])
174
+ print(f'support model loaded, c95={c95:.4f}')
175
+ else:
176
+ print(f'no support model at {args.density} — violation column disabled')
177
+
178
+ report = {'tag': args.tag, 'checkpoint': args.controller,
179
+ 'step': int(ckpt['step'])}
180
+
181
+ print('\n=== 1. closed-loop contraction ===')
182
+ print(f'{"execute":>8}{"D_0":>9}{"D_final":>10}{"c":>8}{"b":>9}'
183
+ f'{"D*":>9}{"R^2":>7}')
184
+ report['contraction'] = {}
185
+ for execute in (1, 5):
186
+ trace = closed_loop_trace(
187
+ controller, model, batch, device, execute, args.replans
188
+ )
189
+ c, b, fixed, r2 = fit_contraction(trace)
190
+ m = trace.mean(axis=0)
191
+ print(f'{execute:>8}{m[0]:>9.4f}{m[-1]:>10.4f}'
192
+ f'{c:>8.4f}{b:>9.4f}{fixed:>9.4f}{r2:>7.3f}')
193
+ report['contraction'][f'exec{execute}'] = {
194
+ 'c': c, 'b': b, 'fixed_point': fixed, 'r2': r2,
195
+ 'mean_trace': [round(float(v), 5) for v in m],
196
+ }
197
+
198
+ print('\n=== 2. refinement past trained depth ===')
199
+ costs, arrivals, moves, violations, scores = refinement_trace(
200
+ controller, model, batch, device, args.max_k, density, c95
201
+ )
202
+ print(f'{"k":>3}{"terminal":>11}{"arrival":>10}{"dJ":>10}{"|plan chg|":>12}'
203
+ f'{"support r":>11}{"viol%":>8}')
204
+ for k in range(len(costs)):
205
+ dj = '' if k == 0 else f'{costs[k - 1] - costs[k]:+.5f}'
206
+ sup = f'{scores[k]:>11.4f}' if scores else f'{"-":>11}'
207
+ vio = f'{100 * violations[k]:>8.1f}' if violations else f'{"-":>8}'
208
+ print(f'{k:>3}{costs[k]:>11.5f}{arrivals[k]:>10.5f}{dj:>10}'
209
+ f'{moves[k]:>12.5f}{sup}{vio}')
210
+ report['refinement'] = {
211
+ 'terminal': costs, 'arrival': arrivals, 'plan_change': moves,
212
+ 'support_score': scores, 'violation': violations, 'c95': c95,
213
+ }
214
+
215
+ out = Path(args.out)
216
+ out.mkdir(parents=True, exist_ok=True)
217
+ with (out / 'diagnostics.jsonl').open('a') as f:
218
+ f.write(json.dumps(report) + '\n')
219
+ print(f'\nwrote {out / "diagnostics.jsonl"}')
220
+
221
+
222
+ if __name__ == '__main__':
223
+ main()
code/scripts/encode_latents.py ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Encode every PushT frame with the frozen LeWM encoder into a latent cache.
2
+
3
+ Controller training never needs pixels: the encoder is frozen and no image
4
+ augmentation is used, so latents can be computed once. The full dataset is
5
+ 2.34M frames x 192 dims = ~0.9 GB in fp16, which fits in RAM.
6
+
7
+ Also stores the action z-score statistics. LeWM was trained on z-scored
8
+ actions (``column_normalizer`` defaults to ``method='zscore'``), so the
9
+ controller must emit actions in that same normalized space.
10
+ """
11
+
12
+ import argparse
13
+ import json
14
+ import time
15
+ from pathlib import Path
16
+
17
+ import h5py
18
+ import hdf5plugin # noqa: F401 -- registers the blosc filter used by the h5
19
+ import numpy as np
20
+ import torch
21
+
22
+ from lejepa_control.world_model import load_lewm
23
+
24
+ IMAGENET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
25
+ IMAGENET_STD = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
26
+
27
+
28
+ def main():
29
+ parser = argparse.ArgumentParser()
30
+ parser.add_argument(
31
+ '--h5', default='data/swm_home/datasets/pusht_expert_train.h5'
32
+ )
33
+ parser.add_argument('--out', default='data/latents')
34
+ parser.add_argument('--batch-size', type=int, default=512)
35
+ parser.add_argument('--limit-episodes', type=int, default=None)
36
+ parser.add_argument('--wm-name', default='quentinll/lewm-pusht')
37
+ args = parser.parse_args()
38
+
39
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
40
+ model = load_lewm(name=args.wm_name, device=device)
41
+ encoder, projector = model.encoder, model.projector
42
+ D = model.predictor.input_dim
43
+
44
+ out_dir = Path(args.out)
45
+ out_dir.mkdir(parents=True, exist_ok=True)
46
+
47
+ mean = IMAGENET_MEAN.to(device)
48
+ std = IMAGENET_STD.to(device)
49
+
50
+ with h5py.File(args.h5, 'r') as f:
51
+ lengths = f['ep_len'][:].astype(np.int64)
52
+ offsets = f['ep_offset'][:].astype(np.int64)
53
+
54
+ if args.limit_episodes is not None:
55
+ lengths = lengths[: args.limit_episodes]
56
+ offsets = offsets[: args.limit_episodes]
57
+
58
+ n_frames = int(lengths.sum())
59
+ end = int(offsets[-1] + lengths[-1])
60
+ print(f'{len(lengths)} episodes, {n_frames} frames -> {D}-dim latents')
61
+
62
+ actions = f['action'][:end].astype(np.float32)
63
+ # the frame that ends an episode has no outgoing action (there is no
64
+ # next state to leave it for) and is NaN-padded; ignore those rows for
65
+ # the stats, then fill them with the mean so a block that straddles an
66
+ # episode boundary (LatentGoalDataset's real_action, at the last valid
67
+ # start) never hands training a NaN.
68
+ nan_mask = np.isnan(actions).any(axis=1)
69
+ a_mean = np.nanmean(actions, axis=0)
70
+ a_std = np.nanstd(actions, axis=0)
71
+ if nan_mask.any():
72
+ print(f'{int(nan_mask.sum())} NaN (terminal-step) action rows -> filled with mean')
73
+ actions[nan_mask] = a_mean
74
+ print(f'action mean={a_mean} std={a_std}')
75
+
76
+ latents = np.lib.format.open_memmap(
77
+ out_dir / 'latents.npy',
78
+ mode='w+',
79
+ dtype=np.float16,
80
+ shape=(n_frames, D),
81
+ )
82
+
83
+ t0 = time.perf_counter()
84
+ for start in range(0, end, args.batch_size):
85
+ stop = min(start + args.batch_size, end)
86
+ frames = f['pixels'][start:stop] # (B, 224, 224, 3) uint8
87
+
88
+ x = torch.from_numpy(frames).to(device, non_blocking=True)
89
+ x = x.permute(0, 3, 1, 2).float().div_(255.0).sub_(mean).div_(std)
90
+
91
+ with torch.no_grad(), torch.autocast('cuda', dtype=torch.bfloat16):
92
+ out = encoder(x, interpolate_pos_encoding=True)
93
+ emb = projector(out.last_hidden_state[:, 0].float())
94
+
95
+ latents[start:stop] = emb.float().cpu().numpy().astype(np.float16)
96
+
97
+ if start % (args.batch_size * 200) == 0:
98
+ done = stop / end
99
+ rate = stop / (time.perf_counter() - t0)
100
+ eta = (end - stop) / rate / 60
101
+ print(
102
+ f' {done:6.1%} {rate:6.0f} img/s eta {eta:5.1f} min',
103
+ flush=True,
104
+ )
105
+
106
+ latents.flush()
107
+
108
+ np.save(out_dir / 'actions.npy', actions)
109
+ np.save(out_dir / 'ep_len.npy', lengths)
110
+ np.save(out_dir / 'ep_offset.npy', offsets)
111
+
112
+ stats = {
113
+ 'action_mean': a_mean.tolist(),
114
+ 'action_std': a_std.tolist(),
115
+ 'latent_dim': int(D),
116
+ 'n_frames': n_frames,
117
+ 'n_episodes': len(lengths),
118
+ }
119
+ (out_dir / 'stats.json').write_text(json.dumps(stats, indent=2))
120
+
121
+ z = np.asarray(latents[:10000], dtype=np.float32)
122
+ print(f'latent per-coord std (mean) = {z.std(0).mean():.4f}')
123
+ print(f'done in {(time.perf_counter() - t0) / 60:.1f} min -> {out_dir}')
124
+
125
+
126
+ if __name__ == '__main__':
127
+ main()
code/scripts/encode_latents_reacher.py ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Encode every DMC Reacher frame with the frozen LeWM encoder into a latent cache.
2
+
3
+ Reacher port of ``scripts/encode_latents.py``. Controller training never needs
4
+ pixels: the encoder is frozen and no image augmentation is used, so latents can
5
+ be computed once. Also stores the action z-score statistics — LeWM was trained
6
+ on z-scored actions (``column_normalizer`` defaults to ``method='zscore'``),
7
+ so the controller must emit actions in that same normalized space.
8
+
9
+ The only reacher-specific differences from the PushT script are the default
10
+ dataset path and the episode-index column names: the DMC collect pipeline
11
+ writes ``ep_idx`` where PushT wrote ``episode_idx``. Both spellings are
12
+ accepted, and ``ep_len``/``ep_offset`` are derived when absent.
13
+ """
14
+
15
+ import argparse
16
+ import json
17
+ import time
18
+ from pathlib import Path
19
+
20
+ import h5py
21
+ import hdf5plugin # noqa: F401 -- registers the blosc filter used by the h5
22
+ import numpy as np
23
+ import torch
24
+
25
+ from lejepa_control.world_model import load_lewm
26
+
27
+ IMAGENET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
28
+ IMAGENET_STD = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
29
+
30
+
31
+ def episode_layout(f):
32
+ """``(lengths, offsets)`` and the dataset row count, per spelling."""
33
+ keys = set(f.keys())
34
+ if 'ep_len' in keys and 'ep_offset' in keys:
35
+ lengths = f['ep_len'][:].astype(np.int64)
36
+ offsets = f['ep_offset'][:].astype(np.int64)
37
+ return lengths, offsets, int(offsets[-1] + lengths[-1])
38
+ ep_col = 'episode_idx' if 'episode_idx' in keys else 'ep_idx'
39
+ ep_idx = f[ep_col][:].astype(np.int64)
40
+ step_idx = f['step_idx'][:].astype(np.int64)
41
+ n_eps = int(ep_idx.max()) + 1
42
+ lengths = np.zeros(n_eps, dtype=np.int64)
43
+ np.maximum.at(lengths, ep_idx, step_idx + 1)
44
+ keep = lengths > 0
45
+ lengths = lengths[keep]
46
+ offsets = np.concatenate([[0], np.cumsum(lengths)[:-1]])
47
+ return lengths, offsets, len(ep_idx)
48
+
49
+
50
+ def main():
51
+ parser = argparse.ArgumentParser()
52
+ parser.add_argument(
53
+ '--h5', default='data/swm_home/datasets/dmc/reacher.h5'
54
+ )
55
+ parser.add_argument('--out', default='data/latents_reacher')
56
+ parser.add_argument('--batch-size', type=int, default=512)
57
+ parser.add_argument('--limit-episodes', type=int, default=None)
58
+ parser.add_argument('--wm-name', default='quentinll/lewm-reacher')
59
+ args = parser.parse_args()
60
+
61
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
62
+ model = load_lewm(name=args.wm_name, device=device)
63
+ encoder, projector = model.encoder, model.projector
64
+ D = model.predictor.input_dim
65
+
66
+ out_dir = Path(args.out)
67
+ out_dir.mkdir(parents=True, exist_ok=True)
68
+
69
+ mean = IMAGENET_MEAN.to(device)
70
+ std = IMAGENET_STD.to(device)
71
+
72
+ with h5py.File(args.h5, 'r') as f:
73
+ lengths, offsets, end = episode_layout(f)
74
+
75
+ if args.limit_episodes is not None:
76
+ lengths = lengths[: args.limit_episodes]
77
+ offsets = offsets[: args.limit_episodes]
78
+ end = int(offsets[-1] + lengths[-1])
79
+
80
+ n_frames = int(lengths.sum())
81
+ print(f'{len(lengths)} episodes, {n_frames} frames -> {D}-dim latents')
82
+
83
+ actions = f['action'][:end].astype(np.float32)
84
+ # the frame that ends an episode has no outgoing action (there is no
85
+ # next state to leave it for) and is NaN-padded; ignore those rows for
86
+ # the stats, then fill them with the mean so a block that straddles an
87
+ # episode boundary never hands training a NaN.
88
+ nan_mask = np.isnan(actions).any(axis=1)
89
+ a_mean = np.nanmean(actions, axis=0)
90
+ a_std = np.nanstd(actions, axis=0)
91
+ if nan_mask.any():
92
+ print(
93
+ f'{int(nan_mask.sum())} NaN (terminal-step) action rows'
94
+ ' -> filled with mean'
95
+ )
96
+ actions[nan_mask] = a_mean
97
+ print(f'action mean={a_mean} std={a_std}')
98
+
99
+ latents = np.lib.format.open_memmap(
100
+ out_dir / 'latents.npy',
101
+ mode='w+',
102
+ dtype=np.float16,
103
+ shape=(n_frames, D),
104
+ )
105
+
106
+ t0 = time.perf_counter()
107
+ for start in range(0, end, args.batch_size):
108
+ stop = min(start + args.batch_size, end)
109
+ frames = f['pixels'][start:stop] # (B, 224, 224, 3) uint8
110
+
111
+ x = torch.from_numpy(frames).to(device, non_blocking=True)
112
+ x = x.permute(0, 3, 1, 2).float().div_(255.0).sub_(mean).div_(std)
113
+
114
+ with torch.no_grad(), torch.autocast('cuda', dtype=torch.bfloat16):
115
+ out = encoder(x, interpolate_pos_encoding=True)
116
+ emb = projector(out.last_hidden_state[:, 0].float())
117
+
118
+ latents[start:stop] = emb.float().cpu().numpy().astype(np.float16)
119
+
120
+ if start % (args.batch_size * 200) == 0:
121
+ done = stop / end
122
+ rate = stop / (time.perf_counter() - t0)
123
+ eta = (end - stop) / rate / 60
124
+ print(
125
+ f' {done:6.1%} {rate:6.0f} img/s eta {eta:5.1f} min',
126
+ flush=True,
127
+ )
128
+
129
+ latents.flush()
130
+
131
+ np.save(out_dir / 'actions.npy', actions)
132
+ np.save(out_dir / 'ep_len.npy', lengths)
133
+ np.save(out_dir / 'ep_offset.npy', offsets)
134
+
135
+ stats = {
136
+ 'action_mean': a_mean.tolist(),
137
+ 'action_std': a_std.tolist(),
138
+ 'latent_dim': int(D),
139
+ 'n_frames': n_frames,
140
+ 'n_episodes': len(lengths),
141
+ }
142
+ (out_dir / 'stats.json').write_text(json.dumps(stats, indent=2))
143
+
144
+ z = np.asarray(latents[:10000], dtype=np.float32)
145
+ print(f'latent per-coord std (mean) = {z.std(0).mean():.4f}')
146
+ print(f'done in {(time.perf_counter() - t0) / 60:.1f} min -> {out_dir}')
147
+
148
+
149
+ if __name__ == '__main__':
150
+ main()
code/scripts/eval_controller.py ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Evaluate the controller in the real PushT simulator, against CEM.
2
+
3
+ Both planners run through the same ``WorldModelPolicy``, the same wrappers and
4
+ the same held-out initial-state/goal pairs, so the only thing that differs is
5
+ how the plan is produced. Reports success rate and wall-clock planning time —
6
+ the central claim is approaching CEM's success with far fewer world-model
7
+ evaluations.
8
+ """
9
+
10
+ import os
11
+
12
+ os.environ['MUJOCO_GL'] = 'egl'
13
+
14
+ import argparse # noqa: E402
15
+ import json # noqa: E402
16
+ import sys # noqa: E402
17
+ import time # noqa: E402
18
+ from pathlib import Path # noqa: E402
19
+
20
+ import hdf5plugin # noqa: F401,E402 -- blosc filter for the expert h5
21
+ import numpy as np # noqa: E402
22
+ import stable_pretraining as spt # noqa: E402
23
+ import stable_worldmodel as swm # noqa: E402
24
+ import torch # noqa: E402
25
+ from sklearn import preprocessing # noqa: E402
26
+ from torchvision.transforms import v2 as transforms # noqa: E402
27
+
28
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
29
+
30
+ from lejepa_control.solver import ControllerSolver, load_controller # noqa: E402
31
+ from lejepa_control.world_model import load_lewm # noqa: E402
32
+
33
+
34
+ def parse_args():
35
+ p = argparse.ArgumentParser()
36
+ p.add_argument('--controller', default='data/runs/controller/controller.pt')
37
+ p.add_argument('--planner', default='controller', choices=['controller', 'cem'])
38
+ p.add_argument('--refinements', type=int, default=None)
39
+ p.add_argument('--num-eval', type=int, default=50)
40
+ p.add_argument('--eval-budget', type=int, default=50)
41
+ p.add_argument('--goal-offset', type=int, default=25)
42
+ p.add_argument('--horizon', type=int, default=5)
43
+ # execute one block then replan (spec section 9); set equal to --horizon
44
+ # for the "execute the full plan" ablation
45
+ p.add_argument('--receding-horizon', type=int, default=1)
46
+ p.add_argument('--cem-samples', type=int, default=300)
47
+ p.add_argument('--cem-steps', type=int, default=30)
48
+ p.add_argument(
49
+ '--dataset', default='data/swm_home/datasets/pusht_expert_train.h5'
50
+ )
51
+ # world model + env to evaluate against; the PushT defaults keep existing
52
+ # runs unchanged. TwoRoom uses quentinll/lewm-tworooms + swm/TwoRoom-v1.
53
+ p.add_argument('--wm-name', default='quentinll/lewm-pusht')
54
+ p.add_argument('--env-name', default='swm/PushT-v1')
55
+ # the TwoRoom h5 stores the agent position as `pos_agent` and the episode
56
+ # id as `ep_idx`; PushT uses `state` / `episode_idx`. These map the harness
57
+ # onto either dataset without changing the env-facing info keys.
58
+ p.add_argument('--state-col', default='state')
59
+ p.add_argument('--episode-col', default='episode_idx')
60
+ p.add_argument('--out', default='data/runs/eval')
61
+ p.add_argument('--seed', type=int, default=42)
62
+ p.add_argument('--video', action='store_true')
63
+ return p.parse_args()
64
+
65
+
66
+ def img_transform(size=224):
67
+ return transforms.Compose(
68
+ [
69
+ transforms.ToImage(),
70
+ transforms.ToDtype(torch.float32, scale=True),
71
+ transforms.Normalize(**spt.data.dataset_stats.ImageNet),
72
+ transforms.Resize(size=size),
73
+ ]
74
+ )
75
+
76
+
77
+ def main():
78
+ args = parse_args()
79
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
80
+
81
+ world = swm.World(
82
+ env_name=args.env_name,
83
+ num_envs=args.num_eval,
84
+ max_episode_steps=2 * args.eval_budget,
85
+ image_shape=(224, 224),
86
+ )
87
+
88
+ dataset = swm.data.load_dataset(
89
+ str(Path(args.dataset).resolve()),
90
+ keys_to_cache=['action', 'proprio', args.state_col],
91
+ )
92
+
93
+ process = {}
94
+ for col in ('action', 'proprio', 'state'):
95
+ # fit on the dataset column (state_col), but store under the env
96
+ # info-key the policy matches against (always 'state'); TwoRoom's
97
+ # h5 names this column `pos_agent` but its env still emits `state`.
98
+ src = args.state_col if col == 'state' else col
99
+ data = dataset.get_col_data(src)
100
+ data = data[~np.isnan(data).any(axis=1)]
101
+ scaler = preprocessing.StandardScaler().fit(data)
102
+ process[col] = scaler
103
+ if col != 'action':
104
+ process[f'goal_{col}'] = scaler
105
+
106
+ transform = {'pixels': img_transform(), 'goal': img_transform()}
107
+
108
+ model = load_lewm(name=args.wm_name, device=device)
109
+ model.interpolate_pos_encoding = True
110
+ latent_dim = model.predictor.input_dim
111
+
112
+ config = swm.PlanConfig(
113
+ horizon=args.horizon,
114
+ receding_horizon=args.receding_horizon,
115
+ action_block=5,
116
+ history_len=model.predictor.num_frames,
117
+ )
118
+
119
+ if args.planner == 'controller':
120
+ controller, ckpt = load_controller(
121
+ args.controller,
122
+ latent_dim=latent_dim,
123
+ device=device,
124
+ refinements=args.refinements,
125
+ )
126
+ solver = ControllerSolver(model, controller, device=device)
127
+ tag = f'controller_K{controller.refinements}'
128
+ print(f'controller from step {ckpt["step"]}, K={controller.refinements}')
129
+ else:
130
+ cost = swm.planning.ShootingCostEvaluator(model, swm.planning.GoalMSE())
131
+ solver = swm.planning.CEMSolver(
132
+ cost=cost,
133
+ num_samples=args.cem_samples,
134
+ n_steps=args.cem_steps,
135
+ topk=30,
136
+ device=device,
137
+ )
138
+ tag = f'cem_s{args.cem_samples}_n{args.cem_steps}'
139
+
140
+ # the central claim is cost, so count predictor forwards rather than
141
+ # asserting them: one call = one batched latent transition
142
+ calls = {'n': 0, 'rows': 0}
143
+ inner_predict = model.predictor.forward
144
+
145
+ def counting_predict(*a, **kw):
146
+ calls['n'] += 1
147
+ first = a[0] if a else next(iter(kw.values()))
148
+ calls['rows'] += first.shape[0]
149
+ return inner_predict(*a, **kw)
150
+
151
+ model.predictor.forward = counting_predict
152
+
153
+ # terminal latent distance at each replan -> CEM regret, comparable across
154
+ # planners because both are scored under the same frozen model. GoalMSE
155
+ # sums over the latent dim while the controller averages, so rescale to
156
+ # per-dim. Done by retyping the instance: __call__ is looked up on the
157
+ # type, and the policy isinstance-checks against the Solver protocol.
158
+ terminals = []
159
+ scale = 1.0 if args.planner == 'controller' else 1.0 / latent_dim
160
+ base = type(solver)
161
+
162
+ class RecordingSolver(base):
163
+ def __call__(self, info_dict, init_action=None):
164
+ out = base.__call__(self, info_dict, init_action)
165
+ costs = out.get('costs')
166
+ if costs is not None:
167
+ value = float(torch.as_tensor(costs).float().mean())
168
+ terminals.append(value * scale)
169
+ return out
170
+
171
+ solver.__class__ = RecordingSolver
172
+
173
+ policy = swm.policy.WorldModelPolicy(
174
+ solver=solver,
175
+ config=config,
176
+ process=process,
177
+ transform=transform,
178
+ history_keys=('pixels',),
179
+ )
180
+ world.set_policy(policy)
181
+
182
+ # held-out start/goal pairs, identical across planners for a fair compare
183
+ ep_idx = dataset.get_col_data(args.episode_col)
184
+ step_idx = dataset.get_col_data('step_idx')
185
+ episodes = np.unique(ep_idx)
186
+ lengths = {e: step_idx[ep_idx == e].max() + 1 for e in episodes}
187
+ max_start = np.array([lengths[e] for e in ep_idx]) - args.goal_offset - 1
188
+ valid = np.nonzero(step_idx <= max_start)[0]
189
+
190
+ rng = np.random.default_rng(args.seed)
191
+ picked = np.sort(valid[rng.choice(len(valid), args.num_eval, replace=False)])
192
+
193
+ out_dir = Path(args.out)
194
+ out_dir.mkdir(parents=True, exist_ok=True)
195
+
196
+ t0 = time.time()
197
+ metrics = world.evaluate(
198
+ dataset=dataset,
199
+ start_steps=step_idx[picked].tolist(),
200
+ goal_offset=args.goal_offset,
201
+ eval_budget=args.eval_budget,
202
+ episodes_idx=ep_idx[picked].tolist(),
203
+ callables=[
204
+ {
205
+ 'method': '_set_state',
206
+ 'args': {'state': {'value': args.state_col}},
207
+ },
208
+ {
209
+ 'method': '_set_goal_state',
210
+ 'args': {'goal_state': {'value': f'goal_{args.state_col}'}},
211
+ },
212
+ ],
213
+ video=out_dir if args.video else None,
214
+ )
215
+ elapsed = time.time() - t0
216
+
217
+ result = {
218
+ 'planner': tag,
219
+ 'env': args.env_name,
220
+ 'wm': args.wm_name,
221
+ 'receding_horizon': args.receding_horizon,
222
+ 'success_rate': float(metrics['success_rate']),
223
+ 'seconds': elapsed,
224
+ 'seconds_per_episode': elapsed / args.num_eval,
225
+ 'mean_terminal_distance': (
226
+ float(np.mean(terminals)) if terminals else None
227
+ ),
228
+ # Averaging over solver calls is not comparable across execution
229
+ # lengths: a run that replans every block makes more calls, and the
230
+ # later ones are taken nearer the goal. The first call is taken from
231
+ # the same held-out state by every planner, so that one is.
232
+ 'first_terminal_distance': float(terminals[0]) if terminals else None,
233
+ 'terminal_distance_trace': [float(t) for t in terminals],
234
+ 'predictor_calls': calls['n'],
235
+ 'predictor_rows_per_episode': calls['rows'] / args.num_eval,
236
+ 'num_eval': args.num_eval,
237
+ 'eval_budget': args.eval_budget,
238
+ 'goal_offset': args.goal_offset,
239
+ 'checkpoint': args.controller if args.planner == 'controller' else None,
240
+ 'seed': args.seed,
241
+ 'refinements': args.refinements,
242
+ # per-episode outcomes: all rows share start/goal pairs, so planner
243
+ # comparisons must be paired rather than treated as independent
244
+ 'episode_successes': [
245
+ bool(x) for x in metrics['episode_successes'].tolist()
246
+ ],
247
+ }
248
+ print(json.dumps({k: v for k, v in result.items()
249
+ if k != 'terminal_distance_trace'}, indent=2))
250
+
251
+ with (out_dir / 'results.jsonl').open('a') as f:
252
+ f.write(json.dumps(result) + '\n')
253
+
254
+
255
+ if __name__ == '__main__':
256
+ main()
code/scripts/eval_controller_reacher.py ADDED
@@ -0,0 +1,282 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Evaluate the controller in the real DMC Reacher simulator, against CEM.
2
+
3
+ Reacher port of ``scripts/eval_controller.py``. Differences from the PushT
4
+ original, all following the upstream LeWM eval recipe
5
+ (``stable-worldmodel/scripts/plan/config/reacher.yaml``):
6
+
7
+ * the env runs the ``qpos_match`` task: an episode counts as a success when
8
+ every joint is within 0.05 rad of the goal configuration, which is what
9
+ supplies a termination signal to ``world.evaluate``;
10
+ * dataset rows are pushed into the simulator with ``set_state(qpos, qvel)``
11
+ and the goal with ``set_target_qpos(goal_qpos)`` instead of PushT's
12
+ ``_set_state``/``_set_goal_state``;
13
+ * only ``action`` is z-scored — the qpos/qvel columns are reset plumbing,
14
+ never seen by the policy;
15
+ * the policy is :class:`tools.history_policy.HistoryPolicy` so the solver
16
+ gets the real strided frame history and past action blocks (commit bd459fc:
17
+ without it 41% of the committed action is lost).
18
+
19
+ Both planners run through the same policy, the same wrappers and the same
20
+ held-out initial-state/goal pairs, so the only thing that differs is how the
21
+ plan is produced. Reports success rate and wall-clock planning time — the
22
+ central claim is approaching CEM's success with far fewer world-model
23
+ evaluations.
24
+ """
25
+
26
+ import argparse
27
+ import json
28
+ import os
29
+ import sys
30
+ import time
31
+ from pathlib import Path
32
+
33
+ # PushT eval hardcodes egl (Linux). Windows has no egl; glfw renders into a
34
+ # hidden window and works here. Must be set before mujoco/dm_control import.
35
+ if not os.environ.get('MUJOCO_GL'):
36
+ os.environ['MUJOCO_GL'] = 'glfw' if sys.platform == 'win32' else 'egl'
37
+
38
+ import hdf5plugin # noqa: F401,E402 -- blosc filter for the reacher h5
39
+ import numpy as np
40
+ import stable_pretraining as spt # noqa: E402
41
+ import stable_worldmodel as swm # noqa: E402
42
+ import torch # noqa: E402
43
+ from sklearn import preprocessing # noqa: E402
44
+ from torchvision.transforms import v2 as transforms # noqa: E402
45
+
46
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
47
+
48
+ from lejepa_control.solver import ControllerSolver, load_controller # noqa: E402
49
+ from lejepa_control.world_model import load_lewm # noqa: E402
50
+ from tools.history_policy import HistoryPolicy # noqa: E402
51
+
52
+ # qpos_match: 0.05 rad per-joint tolerance, the env's own criterion
53
+ CALLABLES = [
54
+ {
55
+ 'method': 'set_state',
56
+ 'args': {'qpos': {'value': 'qpos'}, 'qvel': {'value': 'qvel'}},
57
+ },
58
+ {
59
+ 'method': 'set_target_qpos',
60
+ 'args': {'target_qpos': {'value': 'goal_qpos'}},
61
+ },
62
+ ]
63
+
64
+
65
+ def parse_args():
66
+ p = argparse.ArgumentParser()
67
+ p.add_argument('--controller', default='data/runs/controller_reacher/controller.pt')
68
+ p.add_argument('--planner', default='controller', choices=['controller', 'cem', 'random'])
69
+ p.add_argument('--refinements', type=int, default=None)
70
+ p.add_argument('--num-eval', type=int, default=50)
71
+ p.add_argument('--eval-budget', type=int, default=50)
72
+ p.add_argument('--goal-offset', type=int, default=25)
73
+ p.add_argument('--horizon', type=int, default=5)
74
+ # execute one block then replan (spec section 9); set equal to --horizon
75
+ # for the "execute the full plan" ablation
76
+ p.add_argument('--receding-horizon', type=int, default=1)
77
+ p.add_argument('--cem-samples', type=int, default=300)
78
+ p.add_argument('--cem-steps', type=int, default=30)
79
+ p.add_argument('--no-history', action='store_true',
80
+ help='drop HistoryPolicy: single repeated frame, zero past blocks')
81
+ p.add_argument(
82
+ '--dataset', default='data/swm_home/datasets/dmc/reacher.h5'
83
+ )
84
+ p.add_argument('--wm-name', default='quentinll/lewm-reacher')
85
+ p.add_argument('--env-name', default='swm/ReacherDMControl-v0')
86
+ p.add_argument('--out', default='data/runs/eval_reacher')
87
+ p.add_argument('--seed', type=int, default=42)
88
+ p.add_argument('--video', action='store_true')
89
+ return p.parse_args()
90
+
91
+
92
+ def img_transform(size=224):
93
+ return transforms.Compose(
94
+ [
95
+ transforms.ToImage(),
96
+ transforms.ToDtype(torch.float32, scale=True),
97
+ transforms.Normalize(**spt.data.dataset_stats.ImageNet),
98
+ transforms.Resize(size=size),
99
+ ]
100
+ )
101
+
102
+
103
+ def main():
104
+ args = parse_args()
105
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
106
+
107
+ world = swm.World(
108
+ env_name=args.env_name,
109
+ num_envs=args.num_eval,
110
+ max_episode_steps=2 * args.eval_budget,
111
+ image_shape=(224, 224),
112
+ task='qpos_match',
113
+ )
114
+
115
+ dataset = swm.data.load_dataset(
116
+ str(Path(args.dataset).resolve()), keys_to_cache=['action']
117
+ )
118
+
119
+ # only the action column is z-scored; qpos/qvel feed the reset callables
120
+ process = {}
121
+ data = dataset.get_col_data('action')
122
+ data = data[~np.isnan(data).any(axis=1)]
123
+ process['action'] = preprocessing.StandardScaler().fit(data)
124
+
125
+ transform = {'pixels': img_transform(), 'goal': img_transform()}
126
+
127
+ model = load_lewm(name=args.wm_name, device=device)
128
+ model.interpolate_pos_encoding = True
129
+ latent_dim = model.predictor.input_dim
130
+
131
+ config = swm.PlanConfig(
132
+ horizon=args.horizon,
133
+ receding_horizon=args.receding_horizon,
134
+ action_block=5,
135
+ history_len=model.predictor.num_frames,
136
+ )
137
+
138
+ if args.planner == 'controller':
139
+ controller, ckpt = load_controller(
140
+ args.controller,
141
+ latent_dim=latent_dim,
142
+ device=device,
143
+ refinements=args.refinements,
144
+ )
145
+ solver = ControllerSolver(model, controller, device=device)
146
+ tag = f'controller_K{controller.refinements}'
147
+ print(f'controller from step {ckpt["step"]}, K={controller.refinements}')
148
+ elif args.planner == 'cem':
149
+ cost = swm.planning.ShootingCostEvaluator(model, swm.planning.GoalMSE())
150
+ solver = swm.planning.CEMSolver(
151
+ cost=cost,
152
+ num_samples=args.cem_samples,
153
+ n_steps=args.cem_steps,
154
+ topk=30,
155
+ device=device,
156
+ )
157
+ tag = f'cem_s{args.cem_samples}_n{args.cem_steps}'
158
+ else:
159
+ solver = swm.policy.RandomPolicy()
160
+ tag = 'random'
161
+
162
+ calls = {'n': 0, 'rows': 0}
163
+ if args.planner in ('controller', 'cem'):
164
+ # the central claim is cost, so count predictor forwards rather than
165
+ # asserting them: one call = one batched latent transition
166
+ inner_predict = model.predictor.forward
167
+
168
+ def counting_predict(*a, **kw):
169
+ calls['n'] += 1
170
+ first = a[0] if a else next(iter(kw.values()))
171
+ calls['rows'] += first.shape[0]
172
+ return inner_predict(*a, **kw)
173
+
174
+ model.predictor.forward = counting_predict
175
+
176
+ # terminal latent distance at each replan -> CEM regret, comparable across
177
+ # planners because both are scored under the same frozen model. GoalMSE
178
+ # sums over the latent dim while the controller averages, so rescale to
179
+ # per-dim. Done by retyping the instance: __call__ is looked up on the
180
+ # type, and the policy isinstance-checks against the Solver protocol.
181
+ terminals = []
182
+ if args.planner in ('controller', 'cem'):
183
+ scale = 1.0 if args.planner == 'controller' else 1.0 / latent_dim
184
+ base = type(solver)
185
+
186
+ class RecordingSolver(base):
187
+ def __call__(self, info_dict, init_action=None):
188
+ out = base.__call__(self, info_dict, init_action)
189
+ costs = out.get('costs')
190
+ if costs is not None:
191
+ value = float(torch.as_tensor(costs).float().mean())
192
+ terminals.append(value * scale)
193
+ return out
194
+
195
+ solver.__class__ = RecordingSolver
196
+
197
+ if args.planner == 'random':
198
+ # RandomPolicy is a policy itself, not a solver — no WorldModelPolicy
199
+ # wrapper, no history plumbing, no world-model calls.
200
+ policy = solver
201
+ else:
202
+ policy_cls = (
203
+ swm.policy.WorldModelPolicy if args.no_history else HistoryPolicy
204
+ )
205
+ policy = policy_cls(
206
+ solver=solver,
207
+ config=config,
208
+ process=process,
209
+ transform=transform,
210
+ history_keys=('pixels',),
211
+ )
212
+ world.set_policy(policy)
213
+
214
+ # held-out start/goal pairs, identical across planners for a fair compare
215
+ names = set(dataset.column_names) | set(getattr(dataset, '_schema_names', ()))
216
+ ep_col = 'episode_idx' if 'episode_idx' in names else 'ep_idx'
217
+ ep_idx = dataset.get_col_data(ep_col)
218
+ step_idx = dataset.get_col_data('step_idx')
219
+ episodes = np.unique(ep_idx)
220
+ lengths = {e: step_idx[ep_idx == e].max() + 1 for e in episodes}
221
+ max_start = np.array([lengths[e] for e in ep_idx]) - args.goal_offset - 1
222
+ valid = np.nonzero(step_idx <= max_start)[0]
223
+
224
+ rng = np.random.default_rng(args.seed)
225
+ picked = np.sort(valid[rng.choice(len(valid), args.num_eval, replace=False)])
226
+
227
+ out_dir = Path(args.out)
228
+ out_dir.mkdir(parents=True, exist_ok=True)
229
+
230
+ t0 = time.time()
231
+ metrics = world.evaluate(
232
+ dataset=dataset,
233
+ start_steps=step_idx[picked].tolist(),
234
+ goal_offset=args.goal_offset,
235
+ eval_budget=args.eval_budget,
236
+ episodes_idx=ep_idx[picked].tolist(),
237
+ callables=CALLABLES,
238
+ video=out_dir if args.video else None,
239
+ )
240
+ elapsed = time.time() - t0
241
+
242
+ result = {
243
+ 'planner': tag,
244
+ 'env': args.env_name,
245
+ 'wm': args.wm_name,
246
+ 'receding_horizon': args.receding_horizon,
247
+ 'success_rate': float(metrics['success_rate']),
248
+ 'seconds': elapsed,
249
+ 'seconds_per_episode': elapsed / args.num_eval,
250
+ 'mean_terminal_distance': (
251
+ float(np.mean(terminals)) if terminals else None
252
+ ),
253
+ # Averaging over solver calls is not comparable across execution
254
+ # lengths: a run that replans every block makes more calls, and the
255
+ # later ones are taken nearer the goal. The first call is taken from
256
+ # the same held-out state by every planner, so that one is.
257
+ 'first_terminal_distance': float(terminals[0]) if terminals else None,
258
+ 'terminal_distance_trace': [float(t) for t in terminals],
259
+ 'predictor_calls': calls['n'],
260
+ 'predictor_rows_per_episode': calls['rows'] / args.num_eval,
261
+ 'num_eval': args.num_eval,
262
+ 'eval_budget': args.eval_budget,
263
+ 'goal_offset': args.goal_offset,
264
+ 'checkpoint': args.controller if args.planner == 'controller' else None,
265
+ 'seed': args.seed,
266
+ 'refinements': args.refinements,
267
+ 'history': not args.no_history,
268
+ # per-episode outcomes: all rows share start/goal pairs, so planner
269
+ # comparisons must be paired rather than treated as independent
270
+ 'episode_successes': [
271
+ bool(x) for x in metrics['episode_successes'].tolist()
272
+ ],
273
+ }
274
+ print(json.dumps({k: v for k, v in result.items()
275
+ if k != 'terminal_distance_trace'}, indent=2))
276
+
277
+ with (out_dir / 'results.jsonl').open('a') as f:
278
+ f.write(json.dumps(result) + '\n')
279
+
280
+
281
+ if __name__ == '__main__':
282
+ main()
code/scripts/paired_stats.py ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Paired comparison between eval rows.
2
+
3
+ Every row in ``results.jsonl`` uses the same held-out start/goal pairs, so
4
+ success rates must be compared per-episode rather than as two independent
5
+ binomials. With 50 episodes one flipped episode moves the rate by 2 points,
6
+ which is well inside the noise a naive comparison would ignore.
7
+
8
+ McNemar's exact test uses only the discordant episodes (the ones where the
9
+ two planners disagree), which is the right conditioning for paired binary
10
+ outcomes. The bootstrap CI resamples episodes to put an interval on the
11
+ difference.
12
+ """
13
+
14
+ import argparse
15
+ import json
16
+ from math import comb
17
+ from pathlib import Path
18
+
19
+ import numpy as np
20
+
21
+
22
+ def mcnemar_exact(a, b):
23
+ """Two-sided exact McNemar p-value for paired binary outcomes."""
24
+ only_a = int(np.sum(a & ~b))
25
+ only_b = int(np.sum(b & ~a))
26
+ n = only_a + only_b
27
+ if n == 0:
28
+ return 1.0, only_a, only_b
29
+ k = min(only_a, only_b)
30
+ tail = sum(comb(n, i) for i in range(k + 1)) / 2**n
31
+ return min(1.0, 2 * tail), only_a, only_b
32
+
33
+
34
+ def bootstrap_ci(a, b, reps=20000, seed=0):
35
+ """Percentile CI on the paired success-rate difference, in points."""
36
+ rng = np.random.default_rng(seed)
37
+ idx = rng.integers(0, len(a), size=(reps, len(a)))
38
+ diffs = (a[idx].mean(axis=1) - b[idx].mean(axis=1)) * 100
39
+ return np.percentile(diffs, [2.5, 97.5])
40
+
41
+
42
+ def label(row):
43
+ tag = row['planner']
44
+ if row.get('receding_horizon', 1) != 1:
45
+ tag += f'+exec{row["receding_horizon"]}'
46
+ ckpt = row.get('checkpoint') or ''
47
+ # exp7 cell names first (fused192_s2/base_r2 before their shorter
48
+ # prefixes so a checkpoint doesn't also pick up an unrelated substring)
49
+ for name in ('fused192_s2', 'fused192', 'fused256', 'w192np_split',
50
+ 'base_r2', 'terminal_only', 'no_support', 'ah_hold0.5',
51
+ 'ah_hold0.0', 'ah_hold1.0'):
52
+ if name in ckpt:
53
+ tag += f'[{name}]'
54
+ break
55
+ return tag
56
+
57
+
58
+ def main():
59
+ p = argparse.ArgumentParser()
60
+ p.add_argument('--results', default='data/runs/eval/results.jsonl')
61
+ p.add_argument('--pairs', nargs='*', default=None,
62
+ help='LABEL_A vs LABEL_B, as "A::B" strings')
63
+ args = p.parse_args()
64
+
65
+ rows = [
66
+ json.loads(x)
67
+ for x in Path(args.results).read_text().splitlines()
68
+ if x.strip()
69
+ ]
70
+ rows = [r for r in rows if r.get('episode_successes')]
71
+ if not rows:
72
+ print('no rows carry per-episode outcomes yet — rerun evals')
73
+ return
74
+
75
+ # Pool across eval seeds: multiple rows can share one label (same
76
+ # checkpoint + rh, different --seed), and the pairing is valid *within*
77
+ # each seed (identical held-out start/goal pairs across planners at that
78
+ # seed) — so seed order must match between any two labels being compared.
79
+ # A plain dict keyed by label would instead let the last seed's row
80
+ # silently overwrite the earlier ones.
81
+ grouped = {}
82
+ for r in rows:
83
+ grouped.setdefault(label(r), []).append(
84
+ (r.get('seed', 0), np.array(r['episode_successes'], dtype=bool))
85
+ )
86
+ by_label = {}
87
+ for lbl, items in grouped.items():
88
+ items.sort(key=lambda item: item[0])
89
+ by_label[lbl] = np.concatenate([arr for _, arr in items])
90
+
91
+ print('available rows with paired outcomes:')
92
+ for k, v in by_label.items():
93
+ print(f' {k:<40} {v.mean() * 100:5.1f}% (n={len(v)})')
94
+
95
+ pairs = []
96
+ if args.pairs:
97
+ for spec in args.pairs:
98
+ a, b = spec.split('::')
99
+ pairs.append((a, b))
100
+ else:
101
+ keys = list(by_label)
102
+ pairs = [(keys[i], keys[j])
103
+ for i in range(len(keys)) for j in range(i + 1, len(keys))]
104
+
105
+ print(f'\n{"comparison":<62}{"diff":>8}{"95% CI":>18}{"p":>9}')
106
+ print('-' * 97)
107
+ for a, b in pairs:
108
+ if a not in by_label or b not in by_label:
109
+ print(f' skip {a} vs {b}: missing')
110
+ continue
111
+ sa, sb = by_label[a], by_label[b]
112
+ if len(sa) != len(sb):
113
+ print(f' skip {a} vs {b}: different episode counts')
114
+ continue
115
+ diff = (sa.mean() - sb.mean()) * 100
116
+ lo, hi = bootstrap_ci(sa, sb)
117
+ pval, na, nb = mcnemar_exact(sa, sb)
118
+ star = '*' if pval < 0.05 else ' '
119
+ print(
120
+ f'{a + " vs " + b:<62}{diff:>+7.1f} '
121
+ f'{f"[{lo:+.1f}, {hi:+.1f}]":>17}{pval:>8.4f}{star}'
122
+ )
123
+ print('\n* p < 0.05 (exact McNemar, paired). diff is percentage points.')
124
+
125
+
126
+ if __name__ == '__main__':
127
+ main()
code/scripts/probe_alignment.py ADDED
@@ -0,0 +1,139 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """E2 -- gradient alignment probe (exp7 explainability battery, sec 5).
2
+
3
+ Per refinement k, cosine between the controller's emitted update
4
+ ``alpha_k * Delta Y_j`` and the true descent direction
5
+ ``-grad_{Y_j} sum_i d_i``, obtained by autograd through the *frozen rollout
6
+ only* (never through the controller's own network). This directly tests the
7
+ "amortized descent" story -- does the controller behave like an unrolled
8
+ gradient step? -- and works unchanged on both the split and fused
9
+ architectures, since it only calls the controller's public forward pieces
10
+ (``condition`` / ``initial_plan`` / ``refine`` / ``to_actions``).
11
+ """
12
+
13
+ import argparse
14
+ import json
15
+ import sys
16
+ from pathlib import Path
17
+
18
+ import numpy as np
19
+ import torch
20
+ import torch.nn.functional as F
21
+
22
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
23
+
24
+ from lejepa_control.data import LatentGoalDataset, split_episodes # noqa: E402
25
+ from lejepa_control.rollout import goal_distance, rollout_plan # noqa: E402
26
+ from lejepa_control.solver import load_controller # noqa: E402
27
+ from lejepa_control.world_model import load_lewm # noqa: E402
28
+
29
+
30
+ def parse_args():
31
+ p = argparse.ArgumentParser()
32
+ p.add_argument('--controller', default='data/runs/controller/controller.pt')
33
+ p.add_argument('--latents', default='data/latents')
34
+ p.add_argument('--samples', type=int, default=256)
35
+ p.add_argument('--horizon', type=int, default=5)
36
+ p.add_argument('--tag', default='main')
37
+ p.add_argument('--out', default='data/runs/probe_alignment')
38
+ return p.parse_args()
39
+
40
+
41
+ def descent_direction(controller, model, tokens, ctx, past, goal):
42
+ """``-grad_Y sum_i d_i``: true descent, autograd through the frozen
43
+ rollout only -- ``tokens`` is detached first, so no gradient reaches the
44
+ controller's own network here, only the path token -> action -> rollout.
45
+ """
46
+ y = tokens.detach().clone().requires_grad_(True)
47
+ pred = rollout_plan(model, ctx, past, controller.to_actions(y))
48
+ loss = goal_distance(pred, goal).sum()
49
+ (grad,) = torch.autograd.grad(loss, y)
50
+ return -grad
51
+
52
+
53
+ def alignment_trace(controller, model, ctx, past, goal):
54
+ """Per-refinement cosine between the emitted update and true descent.
55
+
56
+ Mirrors ``IterativeController.forward``'s loop exactly (same ``cond``,
57
+ same ``tokens`` recursion), but at each step also scores the emitted
58
+ update against the gradient-through-rollout direction at that same
59
+ ``Y_k``. Returns a ``(K, B, H)`` cosine tensor.
60
+ """
61
+ with torch.no_grad():
62
+ cond = controller.condition(ctx, goal)
63
+ tokens = controller.initial_plan(cond)
64
+
65
+ cosines = []
66
+ for k in range(controller.refinements):
67
+ with torch.no_grad():
68
+ actions = controller.to_actions(tokens)
69
+ pred = rollout_plan(model, ctx, past, actions)
70
+
71
+ true_dir = descent_direction(controller, model, tokens, ctx, past, goal)
72
+
73
+ with torch.no_grad():
74
+ delta = controller.refine(tokens, cond, pred, goal)
75
+ idx = min(k, controller.step_logit.numel() - 1)
76
+ step = torch.sigmoid(controller.step_logit[idx])
77
+ emitted = step * delta # (B, H, W): the actual applied update
78
+
79
+ cos = F.cosine_similarity(emitted, true_dir, dim=-1) # (B, H)
80
+ cosines.append(cos)
81
+
82
+ tokens = tokens + emitted
83
+
84
+ return torch.stack(cosines) # (K, B, H)
85
+
86
+
87
+ def main():
88
+ args = parse_args()
89
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
90
+ torch.manual_seed(0)
91
+
92
+ stats = json.loads((Path(args.latents) / 'stats.json').read_text())
93
+ model = load_lewm(device=device)
94
+ controller, ckpt = load_controller(
95
+ args.controller, latent_dim=stats['latent_dim'], device=device
96
+ )
97
+ controller.eval()
98
+ print(f'controller step {ckpt["step"]}, K={controller.refinements}, '
99
+ f'fused={controller.fused}, no_latent_proj={controller.no_latent_proj}')
100
+
101
+ _, val_eps = split_episodes(stats['n_episodes'])
102
+ val = LatentGoalDataset(
103
+ args.latents, max_offset=5, episodes=val_eps, horizon=args.horizon
104
+ )
105
+ idx = np.random.default_rng(0).choice(len(val), args.samples, replace=False)
106
+ batch = {
107
+ k: torch.stack([val[int(i)][k] for i in idx]).to(device)
108
+ for k in ('context', 'past_actions', 'goal')
109
+ }
110
+
111
+ cos = alignment_trace(
112
+ controller, model, batch['context'], batch['past_actions'], batch['goal']
113
+ ) # (K, B, H)
114
+
115
+ per_k = cos.mean(dim=(1, 2))
116
+ per_k_std = cos.std(dim=(1, 2))
117
+ print('\n=== E2: gradient alignment (emitted update vs true descent) ===')
118
+ print(f'{"k":>3}{"mean cos":>11}{"std":>9}')
119
+ for k in range(cos.size(0)):
120
+ print(f'{k:>3}{per_k[k].item():>11.4f}{per_k_std[k].item():>9.4f}')
121
+
122
+ report = {
123
+ 'tag': args.tag,
124
+ 'checkpoint': args.controller,
125
+ 'step': int(ckpt['step']),
126
+ 'fused': bool(controller.fused),
127
+ 'no_latent_proj': bool(controller.no_latent_proj),
128
+ 'mean_cosine_per_k': [round(float(v), 5) for v in per_k],
129
+ 'std_cosine_per_k': [round(float(v), 5) for v in per_k_std],
130
+ }
131
+ out = Path(args.out)
132
+ out.mkdir(parents=True, exist_ok=True)
133
+ with (out / 'probe_alignment.jsonl').open('a') as f:
134
+ f.write(json.dumps(report) + '\n')
135
+ print(f'\nwrote {out / "probe_alignment.jsonl"}')
136
+
137
+
138
+ if __name__ == '__main__':
139
+ main()
code/scripts/probe_attention.py ADDED
@@ -0,0 +1,212 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """E4 -- attention readout probe (exp7 explainability battery, sec 5).
2
+
3
+ Temporarily swaps each transformer Block's bound ``forward`` for a version
4
+ that also asks its ``MultiheadAttention`` for weights (``need_weights=True``,
5
+ which the default ``need_weights=False`` forward never computes), for the
6
+ duration of one probe pass, then restores the originals -- so the
7
+ controller's real forward path (and every existing checkpoint/test) is
8
+ untouched.
9
+
10
+ Reports mass from plan-slot queries onto four groups -- self, other plan
11
+ slots, context frames, goal token -- per layer and per refinement. Works
12
+ unchanged on the split architecture (two named nets, ``consequence_net``
13
+ then ``refine_net`` per refine() call) and the fused one (one ``net`` per
14
+ call), so the "one coherent map vs. smeared across two networks" comparison
15
+ in sec 5/E4 is literally the same script pointed at two checkpoints.
16
+ """
17
+
18
+ import argparse
19
+ import json
20
+ import sys
21
+ import types
22
+ from pathlib import Path
23
+
24
+ import numpy as np
25
+ import torch
26
+
27
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
28
+
29
+ from lejepa_control.data import LatentGoalDataset, split_episodes # noqa: E402
30
+ from lejepa_control.rollout import rollout_plan # noqa: E402
31
+ from lejepa_control.solver import load_controller # noqa: E402
32
+ from lejepa_control.world_model import load_lewm # noqa: E402
33
+
34
+
35
+ def parse_args():
36
+ p = argparse.ArgumentParser()
37
+ p.add_argument('--controller', default='data/runs/controller/controller.pt')
38
+ p.add_argument('--latents', default='data/latents')
39
+ p.add_argument('--samples', type=int, default=128)
40
+ p.add_argument('--horizon', type=int, default=5)
41
+ p.add_argument('--tag', default='main')
42
+ p.add_argument('--out', default='data/runs/probe_attention')
43
+ return p.parse_args()
44
+
45
+
46
+ def _capturing_forward(cap_list):
47
+ """A ``Block.forward`` that also records self-attention weights."""
48
+
49
+ def forward(self, x):
50
+ h = self.norm1(x)
51
+ attn_out, w = self.attn(
52
+ h, h, h, need_weights=True, average_attn_weights=True
53
+ )
54
+ cap_list.append(w.detach()) # (B, L, L), already averaged over heads
55
+ x = x + attn_out
56
+ return x + self.mlp(self.norm2(x))
57
+
58
+ return forward
59
+
60
+
61
+ class AttentionCapture:
62
+ """Patches every ``Block`` under ``root`` to log attention weights in
63
+ call order, for the lifetime of the ``with`` block. Blocks that never
64
+ run during the wrapped call (e.g. ``consequence_net`` during
65
+ ``initial_plan``) simply never append -- no separate bookkeeping needed
66
+ to know which net actually fired.
67
+ """
68
+
69
+ def __init__(self, root):
70
+ self.blocks = [m for m in root.modules() if type(m).__name__ == 'Block']
71
+ self.log = []
72
+ self._originals = []
73
+
74
+ def __enter__(self):
75
+ for block in self.blocks:
76
+ self._originals.append((block, block.forward))
77
+ block.forward = types.MethodType(_capturing_forward(self.log), block)
78
+ return self
79
+
80
+ def __exit__(self, *exc):
81
+ for block, orig in self._originals:
82
+ block.forward = orig
83
+ return False
84
+
85
+
86
+ def mass_by_group(weights, horizon, num_context):
87
+ """``weights``: (B, L, L) self-attention, L = horizon + num_context + 1,
88
+ sequence laid out as [plan slots, context frames, goal] (matching
89
+ ``condition()`` / ``_run_refine`` / ``_run_fused``). Returns group mass
90
+ averaged over batch and over the ``horizon`` plan-slot query rows.
91
+ """
92
+ H = horizon
93
+ rows = weights[:, :H] # (B, H, L): only plan-slot queries
94
+ diag = rows[:, torch.arange(H), torch.arange(H)].mean()
95
+ plan_total = rows[:, :, :H].sum(-1).mean()
96
+ ctx_mass = rows[:, :, H:H + num_context].sum(-1).mean()
97
+ goal_mass = rows[:, :, H + num_context].mean()
98
+ return {
99
+ 'self': float(diag),
100
+ 'other_plan': float(plan_total - diag),
101
+ 'context': float(ctx_mass),
102
+ 'goal': float(goal_mass),
103
+ }
104
+
105
+
106
+ def label_layers(log, fused, depth):
107
+ """Positionally names each captured layer by which net produced it.
108
+
109
+ Call order is deterministic from ``controller.py``: ``initial_plan``
110
+ only runs the "refine" net; ``refine()`` runs consequence-then-refine
111
+ (split) or the single fused net (fused) -- so position alone identifies
112
+ the source net, no encoder-identity bookkeeping required.
113
+ """
114
+ if fused:
115
+ return [('net', i) for i in range(len(log))]
116
+ if len(log) == depth: # this was an initial_plan() call
117
+ return [('refine_net', i) for i in range(len(log))]
118
+ return (
119
+ [('consequence_net', i) for i in range(depth)]
120
+ + [('refine_net', i) for i in range(depth)]
121
+ )
122
+
123
+
124
+ @torch.no_grad()
125
+ def run_probe(controller, model, ctx, past, goal, depth):
126
+ """Replays ``IterativeController.forward``'s loop, capturing attention
127
+ inside every net call at every refinement. Returns a list (per
128
+ iteration k = 0..K) of ``{(net_name, layer): group_mass_dict}``.
129
+ """
130
+ H, N = controller.horizon, controller.num_context
131
+ cond = controller.condition(ctx, goal)
132
+
133
+ per_iter = []
134
+ with AttentionCapture(controller) as cap:
135
+ tokens = controller.initial_plan(cond)
136
+ layers = label_layers(cap.log, controller.fused, depth)
137
+ per_iter.append({
138
+ layer: mass_by_group(w, H, N) for layer, w in zip(layers, cap.log)
139
+ })
140
+
141
+ for k in range(controller.refinements):
142
+ actions = controller.to_actions(tokens)
143
+ pred = rollout_plan(model, ctx, past, actions)
144
+ with AttentionCapture(controller) as cap:
145
+ delta = controller.refine(tokens, cond, pred, goal)
146
+ layers = label_layers(cap.log, controller.fused, depth)
147
+ per_iter.append({
148
+ layer: mass_by_group(w, H, N) for layer, w in zip(layers, cap.log)
149
+ })
150
+ idx = min(k, controller.step_logit.numel() - 1)
151
+ tokens = tokens + torch.sigmoid(controller.step_logit[idx]) * delta
152
+
153
+ return per_iter
154
+
155
+
156
+ def main():
157
+ args = parse_args()
158
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
159
+ torch.manual_seed(0)
160
+
161
+ stats = json.loads((Path(args.latents) / 'stats.json').read_text())
162
+ model = load_lewm(device=device)
163
+ controller, ckpt = load_controller(
164
+ args.controller, latent_dim=stats['latent_dim'], device=device
165
+ )
166
+ controller.eval()
167
+ depth = ckpt['args']['depth']
168
+ print(f'controller step {ckpt["step"]}, K={controller.refinements}, '
169
+ f'fused={controller.fused}, depth={depth}')
170
+
171
+ _, val_eps = split_episodes(stats['n_episodes'])
172
+ val = LatentGoalDataset(
173
+ args.latents, max_offset=5, episodes=val_eps, horizon=args.horizon
174
+ )
175
+ idx = np.random.default_rng(0).choice(len(val), args.samples, replace=False)
176
+ batch = {
177
+ k: torch.stack([val[int(i)][k] for i in idx]).to(device)
178
+ for k in ('context', 'past_actions', 'goal')
179
+ }
180
+
181
+ per_iter = run_probe(
182
+ controller, model, batch['context'], batch['past_actions'], batch['goal'],
183
+ depth,
184
+ )
185
+
186
+ print('\n=== E4: attention mass from plan-slot queries ===')
187
+ print(f'{"k":>3}{"net":>16}{"layer":>6}{"self":>8}{"plan":>8}'
188
+ f'{"ctx":>8}{"goal":>8}')
189
+ for k, layers in enumerate(per_iter):
190
+ for (net, layer), g in layers.items():
191
+ print(f'{k:>3}{net:>16}{layer:>6}{g["self"]:>8.3f}'
192
+ f'{g["other_plan"]:>8.3f}{g["context"]:>8.3f}{g["goal"]:>8.3f}')
193
+
194
+ report = {
195
+ 'tag': args.tag,
196
+ 'checkpoint': args.controller,
197
+ 'step': int(ckpt['step']),
198
+ 'fused': bool(controller.fused),
199
+ 'per_iteration': [
200
+ {f'{net}.{layer}': g for (net, layer), g in layers.items()}
201
+ for layers in per_iter
202
+ ],
203
+ }
204
+ out = Path(args.out)
205
+ out.mkdir(parents=True, exist_ok=True)
206
+ with (out / 'probe_attention.jsonl').open('a') as f:
207
+ f.write(json.dumps(report) + '\n')
208
+ print(f'\nwrote {out / "probe_attention.jsonl"}')
209
+
210
+
211
+ if __name__ == '__main__':
212
+ main()
code/scripts/probe_contract.py ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Verify the frozen-LeWM contract the controller depends on.
2
+
3
+ Checks the action-block layout, the normalization the checkpoint was trained
4
+ with, and that gradients reach candidate actions through a multi-step rollout.
5
+ """
6
+
7
+ import sys
8
+ import time
9
+ from pathlib import Path
10
+
11
+ import h5py
12
+ import numpy as np
13
+ import torch
14
+
15
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
16
+
17
+ from lejepa_control.world_model import load_lewm # noqa: E402
18
+
19
+ H5 = Path('data/swm_home/datasets/pusht_expert_train.h5')
20
+ IMAGENET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
21
+ IMAGENET_STD = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
22
+
23
+
24
+ def preprocess(frames_hwc: np.ndarray) -> torch.Tensor:
25
+ x = torch.from_numpy(frames_hwc).permute(0, 3, 1, 2).float() / 255.0
26
+ return (x - IMAGENET_MEAN) / IMAGENET_STD
27
+
28
+
29
+ def main():
30
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
31
+ model = load_lewm(device=device)
32
+ D = model.predictor.input_dim
33
+ print(f'num_frames={model.predictor.num_frames} '
34
+ f'action_input_dim={model.action_encoder.input_dim} D={D}')
35
+
36
+ f = h5py.File(H5, 'r')
37
+ action = f['action']
38
+ offs, lens = f['ep_offset'][:], f['ep_len'][:]
39
+
40
+ # --- action normalization the checkpoint was trained with -------------
41
+ stats = action[:]
42
+ mean, std = stats.mean(0), stats.std(0)
43
+ print(f'action mean={mean} std={std}')
44
+ print(f'raw [-1,1] -> normalized +-{(1.0 / std)}')
45
+
46
+ # --- frameskip / block layout ----------------------------------------
47
+ # A clip loads pixels strided by 5 and actions unstrided, reshaped (T, 10).
48
+ # So block k must be the 5 raw actions between frame k and frame k+1.
49
+ ep, start, fs, T = 0, 0, 5, 4
50
+ o = offs[ep]
51
+ raw = action[o + start : o + start + T * fs] # (20, 2)
52
+ block = raw.reshape(T, fs * 2) # (4, 10)
53
+ assert np.array_equal(block[1], raw[5:10].reshape(-1)), 'block layout'
54
+ print('block layout ok: block[k] = raw actions [5k, 5k+5)')
55
+
56
+ # --- encode a real 3-frame history + goal -----------------------------
57
+ N = model.predictor.num_frames
58
+ idx = [o + i * fs for i in range(N + 1)] # 3 history + 1 target
59
+ pixels = preprocess(f['pixels'][idx[0] : idx[-1] + 1 : fs])
60
+ pixels = pixels.unsqueeze(0).to(device) # (1, 4, 3, 224, 224)
61
+ with torch.no_grad():
62
+ emb = model.encode({'pixels': pixels})['emb']
63
+ print(f'emb {tuple(emb.shape)} mean={emb.mean():.4f} std={emb.std():.4f} '
64
+ f'per-coord std={emb.std(dim=(0, 1)).mean():.4f}')
65
+
66
+ # --- teacher forcing: does the frozen predictor actually predict? -----
67
+ ctx = emb[:, :N]
68
+ a_norm = (raw.reshape(T, fs, 2) - mean) / std
69
+ act = torch.from_numpy(a_norm.reshape(1, T, fs * 2)).float().to(device)
70
+ with torch.no_grad():
71
+ pred = model.predict(ctx, model.action_encoder(act[:, :N]))
72
+ tgt = emb[:, 1 : N + 1]
73
+ err = (pred - tgt).pow(2).mean().item()
74
+ # baseline: predicting "no change" (copy the last context frame)
75
+ copy_err = (ctx - tgt).pow(2).mean().item()
76
+ print(f'1-step pred MSE={err:.5f} copy-last-frame MSE={copy_err:.5f}')
77
+
78
+ # --- multi-step differentiable rollout, gradient to candidate actions -
79
+ Hz = 5
80
+ cand = torch.zeros(1, Hz, fs * 2, device=device, requires_grad=True)
81
+ hist = list(emb[:, :N].detach().unbind(dim=1))
82
+ past = act[:, : N - 1] # blocks between the context frames
83
+ all_act = torch.cat([past, cand], dim=1) # (1, N-1+Hz, 10)
84
+ all_emb = model.action_encoder(all_act)
85
+ for t in range(Hz):
86
+ lo = max(0, N + t - N)
87
+ e = torch.stack(hist[lo:], dim=1)
88
+ a = all_emb[:, lo : N + t]
89
+ hist.append(model.predict(e, a)[:, -1])
90
+ x_H = hist[-1]
91
+ goal = emb[:, -1].detach()
92
+ loss = (x_H - goal).pow(2).sum() / D
93
+ loss.backward()
94
+ g = cand.grad
95
+ print(f'rollout ok: terminal d={loss.item():.5f}')
96
+ print(f'grad norm={g.norm():.5f} per-step={g.norm(dim=-1).squeeze().tolist()}')
97
+ assert g.norm() > 0, 'no gradient reached candidate actions'
98
+ assert g[0, -1].norm() > 0, 'last block got no gradient'
99
+
100
+ # --- encoder throughput (decides whether caching latents is viable) ---
101
+ batch = preprocess(f['pixels'][:256]).to(device)
102
+ with torch.no_grad():
103
+ for _ in range(2):
104
+ model.encode({'pixels': batch.unsqueeze(0)})
105
+ torch.cuda.synchronize()
106
+ t0 = time.perf_counter()
107
+ model.encode({'pixels': batch.unsqueeze(0)})
108
+ torch.cuda.synchronize()
109
+ dt = time.perf_counter() - t0
110
+ ips = 256 / dt
111
+ total = int(lens.sum())
112
+ print(f'encoder {ips:.0f} img/s -> {total} frames in {total / ips / 60:.1f} min')
113
+ print(f'latent cache fp16 = {total * D * 2 / 1e9:.2f} GB')
114
+ print('ALL CONTRACT CHECKS PASSED')
115
+
116
+
117
+ if __name__ == '__main__':
118
+ main()
code/scripts/probe_geometry.py ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """E3 -- token-latent geometry probe (exp7 explainability battery, sec 5).
2
+
3
+ Only meaningful in the 192-identity cells (A, C): with ``width ==
4
+ latent_dim``, plan tokens ``y_j`` live in literally the same coordinate
5
+ system as world-model latents, so they can be compared to ``x_hat_j`` (the
6
+ state block ``j`` actually causes) with a plain distance/cosine, and decoded
7
+ *directly* through the pixel decoder -- no analogous move exists in any
8
+ 256-d variant, which is exactly what "identity embedding buys" (sec 1).
9
+ """
10
+
11
+ import argparse
12
+ import json
13
+ import sys
14
+ from pathlib import Path
15
+
16
+ import numpy as np
17
+ import torch
18
+ import torch.nn.functional as F
19
+
20
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
21
+
22
+ from lejepa_control.data import LatentGoalDataset, split_episodes # noqa: E402
23
+ from lejepa_control.decoder import load_decoder # noqa: E402
24
+ from lejepa_control.rollout import rollout_plan # noqa: E402
25
+ from lejepa_control.solver import load_controller # noqa: E402
26
+ from lejepa_control.world_model import load_lewm # noqa: E402
27
+
28
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
29
+ from decode_rollout import panel, to_uint8 # noqa: E402
30
+
31
+
32
+ def parse_args():
33
+ p = argparse.ArgumentParser()
34
+ p.add_argument('--controller', default='data/runs/exp7/fused192/controller.pt')
35
+ p.add_argument('--decoder', default='data/runs/decoder/decoder.pt')
36
+ p.add_argument('--latents', default='data/latents')
37
+ p.add_argument('--samples', type=int, default=256)
38
+ p.add_argument('--horizon', type=int, default=5)
39
+ p.add_argument('--decode-episodes', type=int, default=3)
40
+ p.add_argument('--tag', default='main')
41
+ p.add_argument('--out', default='data/runs/probe_geometry')
42
+ return p.parse_args()
43
+
44
+
45
+ @torch.no_grad()
46
+ def geometry_trace(controller, model, ctx, past, goal):
47
+ """Per-refinement ``y_k`` (raw plan tokens), ``x_hat_k`` (the latent each
48
+ block causes), ``||y - x_hat||`` and cosine. ``K+1`` entries, same
49
+ recursion as ``IterativeController.forward``.
50
+ """
51
+ cond = controller.condition(ctx, goal)
52
+ tokens = controller.initial_plan(cond)
53
+
54
+ ys, xhats, dists, coss = [], [], [], []
55
+ for k in range(controller.refinements + 1):
56
+ actions = controller.to_actions(tokens)
57
+ pred = rollout_plan(model, ctx, past, actions) # x_hat_j, (B, H, D)
58
+
59
+ ys.append(tokens.clone())
60
+ xhats.append(pred.clone())
61
+ dists.append((tokens - pred).norm(dim=-1)) # (B, H)
62
+ coss.append(F.cosine_similarity(tokens, pred, dim=-1)) # (B, H)
63
+
64
+ if k == controller.refinements:
65
+ break
66
+ delta = controller.refine(tokens, cond, pred, goal)
67
+ idx = min(k, controller.step_logit.numel() - 1)
68
+ tokens = tokens + torch.sigmoid(controller.step_logit[idx]) * delta
69
+
70
+ return torch.stack(ys), torch.stack(xhats), torch.stack(dists), torch.stack(coss)
71
+
72
+
73
+ @torch.no_grad()
74
+ def decode_panel(decoder, ys, sample_idx, out_path):
75
+ """One panel per sample: rows = refinement k, cols = horizon slot j,
76
+ each cell = ``decoder(y_k[sample, j])`` -- what that plan slot "intends".
77
+ """
78
+ rows = [to_uint8(decoder(ys[k, sample_idx])) for k in range(ys.size(0))]
79
+ labels = [f'k={k}' for k in range(ys.size(0))]
80
+ titles = [f'slot {j}' for j in range(ys.size(2))]
81
+ return panel(rows, labels, titles, out_path)
82
+
83
+
84
+ def main():
85
+ args = parse_args()
86
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
87
+ torch.manual_seed(0)
88
+
89
+ stats = json.loads((Path(args.latents) / 'stats.json').read_text())
90
+ model = load_lewm(device=device)
91
+ controller, ckpt = load_controller(
92
+ args.controller, latent_dim=stats['latent_dim'], device=device
93
+ )
94
+ controller.eval()
95
+ if not controller.no_latent_proj:
96
+ print(f'{args.controller}: no_latent_proj=False (width != latent_dim) '
97
+ '-- E3 is only defined for the 192-identity cells (A, C). Skipping.')
98
+ return
99
+ print(f'controller step {ckpt["step"]}, K={controller.refinements}, '
100
+ f'fused={controller.fused}')
101
+
102
+ _, val_eps = split_episodes(stats['n_episodes'])
103
+ val = LatentGoalDataset(
104
+ args.latents, max_offset=5, episodes=val_eps, horizon=args.horizon
105
+ )
106
+ idx = np.random.default_rng(0).choice(len(val), args.samples, replace=False)
107
+ batch = {
108
+ k: torch.stack([val[int(i)][k] for i in idx]).to(device)
109
+ for k in ('context', 'past_actions', 'goal')
110
+ }
111
+
112
+ ys, xhats, dists, coss = geometry_trace(
113
+ controller, model, batch['context'], batch['past_actions'], batch['goal']
114
+ )
115
+
116
+ per_k_dist = dists.mean(dim=(1, 2))
117
+ per_k_cos = coss.mean(dim=(1, 2))
118
+ print('\n=== E3: plan token vs. its own consequence, ||y_j - x_hat_j|| ===')
119
+ print(f'{"k":>3}{"mean ||.||":>12}{"mean cos":>11}')
120
+ for k in range(dists.size(0)):
121
+ print(f'{k:>3}{per_k_dist[k].item():>12.4f}{per_k_cos[k].item():>11.4f}')
122
+
123
+ report = {
124
+ 'tag': args.tag,
125
+ 'checkpoint': args.controller,
126
+ 'step': int(ckpt['step']),
127
+ 'fused': bool(controller.fused),
128
+ 'mean_dist_per_k': [round(float(v), 5) for v in per_k_dist],
129
+ 'mean_cos_per_k': [round(float(v), 5) for v in per_k_cos],
130
+ }
131
+ out = Path(args.out)
132
+ out.mkdir(parents=True, exist_ok=True)
133
+ with (out / 'probe_geometry.jsonl').open('a') as f:
134
+ f.write(json.dumps(report) + '\n')
135
+ print(f'wrote {out / "probe_geometry.jsonl"}')
136
+
137
+ if args.decode_episodes > 0 and Path(args.decoder).exists():
138
+ decoder, dec_ckpt = load_decoder(args.decoder, device=device)
139
+ latent_kind = dec_ckpt['meta']['latent_source']
140
+ if latent_kind != 'emb':
141
+ print(f"decoder trained on '{latent_kind}' latents, not 'emb' "
142
+ '-- skipping the direct plan-token decode.')
143
+ else:
144
+ for i in range(min(args.decode_episodes, ys.size(1))):
145
+ path = decode_panel(
146
+ decoder, ys, i, out / f'{args.tag}_sample{i}.png'
147
+ )
148
+ print(f' -> {path}')
149
+ elif args.decode_episodes > 0:
150
+ print(f'no decoder at {args.decoder} -- skipping the direct plan-token decode.')
151
+
152
+
153
+ if __name__ == '__main__':
154
+ main()
code/scripts/run_ablations.py ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Run the baselines and ablations listed in section 11 of the design doc.
2
+
3
+ Three of them change the objective, so they need their own training runs
4
+ (terminal-only, no-support). The rest only change how the trained controller
5
+ is executed, so they reuse the main checkpoint. Every sim evaluation uses the
6
+ same seed, so all rows share held-out initial-state/goal pairs.
7
+ """
8
+
9
+ import argparse
10
+ import subprocess
11
+ import sys
12
+ from pathlib import Path
13
+
14
+ PY = sys.executable
15
+ ROOT = Path(__file__).resolve().parents[1]
16
+
17
+ # objective ablations: each needs its own controller
18
+ VARIANTS = {
19
+ 'main': [], # alpha=0.05, lambda_support=0.01
20
+ 'terminal_only': ['--alpha', '0'],
21
+ 'no_support': ['--lambda-support', '0'],
22
+ }
23
+
24
+
25
+ def run(cmd):
26
+ print(f'\n$ {" ".join(str(c) for c in cmd)}', flush=True)
27
+ # one bad row should not discard the hours of runs queued behind it
28
+ code = subprocess.run(cmd, cwd=ROOT).returncode
29
+ if code != 0:
30
+ print(f'!! exited {code}, continuing', flush=True)
31
+ return code == 0
32
+
33
+
34
+ def parse_args():
35
+ p = argparse.ArgumentParser()
36
+ p.add_argument('--steps', type=int, default=20000)
37
+ p.add_argument('--batch-size', type=int, default=128)
38
+ p.add_argument('--lr', type=float, default=3e-4)
39
+ p.add_argument('--num-eval', type=int, default=50)
40
+ p.add_argument('--eval-budget', type=int, default=50)
41
+ p.add_argument('--out', default='data/runs')
42
+ p.add_argument('--skip-train', action='store_true')
43
+ p.add_argument('--skip-cem', action='store_true')
44
+ return p.parse_args()
45
+
46
+
47
+ def main():
48
+ args = parse_args()
49
+ out = Path(args.out)
50
+
51
+ def evaluate(extra, tag_dir='eval'):
52
+ run(
53
+ [PY, 'scripts/eval_controller.py',
54
+ '--num-eval', str(args.num_eval),
55
+ '--eval-budget', str(args.eval_budget),
56
+ '--out', str(out / tag_dir), *extra]
57
+ )
58
+
59
+ # Everything that reuses the main checkpoint runs first: those are the
60
+ # headline numbers and cost minutes, while each objective ablation costs
61
+ # its own multi-hour training run.
62
+
63
+ # --- baseline: LeWM + CEM ---------------------------------------------
64
+ if not args.skip_cem:
65
+ evaluate(['--planner', 'cem'])
66
+
67
+ main_ckpt = ['--controller', str(out / 'controller' / 'controller.pt')]
68
+
69
+ # --- refinement count: K=0 (one-pass) through K=5 ---------------------
70
+ for k in (0, 1, 2, 3, 5):
71
+ evaluate([*main_ckpt, '--refinements', str(k)])
72
+
73
+ # --- execute one block vs the whole plan ------------------------------
74
+ evaluate([*main_ckpt, '--receding-horizon', '5'])
75
+
76
+ # --- objective ablations need their own training runs -----------------
77
+ for name, extra in VARIANTS.items():
78
+ if name == 'main':
79
+ continue
80
+ ckpt_dir = out / f'abl_{name}'
81
+ if not args.skip_train:
82
+ run(
83
+ [PY, 'scripts/train_controller.py',
84
+ '--steps', str(args.steps),
85
+ '--batch-size', str(args.batch_size),
86
+ '--lr', str(args.lr),
87
+ '--log-every', '500', '--val-every', '5000',
88
+ '--out', str(ckpt_dir), *extra]
89
+ )
90
+ ckpt = ckpt_dir / 'controller.pt'
91
+ if ckpt.exists():
92
+ evaluate(['--controller', str(ckpt)])
93
+
94
+ print(f'\nall results appended to {out / "eval" / "results.jsonl"}')
95
+
96
+
97
+ if __name__ == '__main__':
98
+ main()
code/scripts/smoke_wm.py ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Verify the frozen LeWM loads, encodes, and gives gradients w.r.t. actions."""
2
+
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ import torch
7
+
8
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
9
+
10
+ from lejepa_control.world_model import load_lewm # noqa: E402
11
+
12
+
13
+ def main():
14
+ import stable_worldmodel as swm
15
+
16
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
17
+ model = load_lewm(device=device)
18
+ print('loaded LeWM ok')
19
+ print(' predictor.num_frames =', model.predictor.num_frames)
20
+ print(' action_encoder.input_dim =', model.action_encoder.input_dim)
21
+
22
+ ds = swm.data.load_dataset('pusht_smoke')
23
+ print(' dataset columns =', ds.column_names)
24
+ print(' episodes =', len(ds.lengths))
25
+
26
+ ep = ds.load_episode(0)
27
+ for k, v in ep.items():
28
+ if hasattr(v, 'shape'):
29
+ print(f' ep[{k}] {tuple(v.shape)} {v.dtype}')
30
+
31
+ pixels_key = 'pixels' if 'pixels' in ep else 'obs.pixels'
32
+ frames = ep[pixels_key]
33
+ if not torch.is_tensor(frames):
34
+ frames = torch.as_tensor(frames)
35
+ if frames.shape[-1] in (1, 3): # NHWC -> NCHW
36
+ frames = frames.permute(0, 3, 1, 2)
37
+ frames = frames.float() / 255.0 if frames.dtype == torch.uint8 else frames
38
+
39
+ T = model.predictor.num_frames
40
+ pixels = frames[:T].unsqueeze(0).to(device) # (1, T, C, H, W)
41
+ with torch.no_grad():
42
+ info = model.encode({'pixels': pixels})
43
+ emb = info['emb']
44
+ print(' emb', tuple(emb.shape))
45
+
46
+ # gradient check: does d(pred)/d(action) flow?
47
+ action_dim = model.action_encoder.input_dim
48
+ action = torch.zeros(1, T, action_dim, device=device, requires_grad=True)
49
+ act_emb = model.action_encoder(action)
50
+ pred = model.predict(emb, act_emb)
51
+ print(' pred', tuple(pred.shape))
52
+ pred.sum().backward()
53
+ g = action.grad
54
+ print(' action.grad norm =', float(g.norm()))
55
+ assert g.norm() > 0, 'no gradient reached the action'
56
+ print('OK: gradients flow through the frozen predictor to actions')
57
+
58
+
59
+ if __name__ == '__main__':
60
+ main()
code/scripts/summarize_refine_sweep.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Summarize the refinement-count sweep, with paired tests.
2
+
3
+ Success rates at n=50 have a binomial standard error near 7 points, so an
4
+ 8-point gap between two K values is about one SE and means nothing on its own.
5
+ Every row in a sweep shares its held-out start/goal pairs at a fixed seed, so
6
+ the comparisons are paired and McNemar's exact test applies -- which is the
7
+ standard ``run_report.md`` already holds the rest of the project to.
8
+
9
+ Reports success vs K per receding horizon, pooled across seeds, and tests each
10
+ K against the best-performing K.
11
+ """
12
+
13
+ import argparse
14
+ import json
15
+ from collections import defaultdict
16
+ from math import comb
17
+ from pathlib import Path
18
+
19
+
20
+ def mcnemar_exact(a, b):
21
+ """Two-sided exact McNemar on paired boolean outcome lists.
22
+
23
+ Returns ``(delta_points, n_discordant, p)``. ``delta`` is ``b`` minus ``a``
24
+ in percentage points.
25
+ """
26
+ assert len(a) == len(b), 'unpaired inputs'
27
+ b_only = sum(1 for x, y in zip(a, b) if not x and y)
28
+ a_only = sum(1 for x, y in zip(a, b) if x and not y)
29
+ n = b_only + a_only
30
+ delta = 100.0 * (b_only - a_only) / len(a)
31
+ if n == 0:
32
+ return delta, 0, 1.0
33
+ k = min(b_only, a_only)
34
+ tail = sum(comb(n, i) for i in range(k + 1)) / (2 ** n)
35
+ return delta, n, min(1.0, 2 * tail)
36
+
37
+
38
+ def main():
39
+ p = argparse.ArgumentParser()
40
+ p.add_argument('--results', default='data/runs/eval_refine_sweep/results.jsonl')
41
+ args = p.parse_args()
42
+
43
+ rows = [json.loads(line) for line in Path(args.results).read_text().splitlines()]
44
+ rows = [r for r in rows if r.get('refinements') is not None]
45
+
46
+ # (rh, K) -> {seed: episode_successes}
47
+ grouped = defaultdict(dict)
48
+ for r in rows:
49
+ grouped[(r['receding_horizon'], r['refinements'])][r['seed']] = (
50
+ r['episode_successes']
51
+ )
52
+
53
+ horizons = sorted({rh for rh, _ in grouped})
54
+ for rh in horizons:
55
+ ks = sorted({k for h, k in grouped if h == rh})
56
+ seeds = sorted(set.intersection(
57
+ *[set(grouped[(rh, k)]) for k in ks]
58
+ )) if ks else []
59
+ if not seeds:
60
+ continue
61
+
62
+ def pooled(k):
63
+ out = []
64
+ for s in seeds:
65
+ out.extend(grouped[(rh, k)][s])
66
+ return out
67
+
68
+ series = {k: pooled(k) for k in ks}
69
+ rates = {k: 100 * sum(v) / len(v) for k, v in series.items()}
70
+ best = max(rates, key=rates.get)
71
+
72
+ n = len(series[best])
73
+ print(f'\nreceding_horizon={rh} seeds={seeds} n={n} episodes pooled')
74
+ print(f'{"K":>4}{"success%":>10}{"per-seed":>16}'
75
+ f'{"vs best (K=%d)" % best:>28}')
76
+ for k in ks:
77
+ per_seed = ' '.join(
78
+ f'{100 * sum(grouped[(rh, k)][s]) / len(grouped[(rh, k)][s]):.0f}'
79
+ for s in seeds
80
+ )
81
+ if k == best:
82
+ verdict = '(best)'
83
+ else:
84
+ delta, nd, pv = mcnemar_exact(series[k], series[best])
85
+ verdict = f'{delta:+.1f}pt n_d={nd} p={pv:.3f}'
86
+ print(f'{k:>4}{rates[k]:>10.1f}{per_seed:>16}{verdict:>28}')
87
+
88
+ # the question the sweep exists to answer
89
+ tail = [rates[k] for k in ks if k >= best]
90
+ monotone = all(
91
+ rates[ks[i]] <= rates[ks[i + 1]] + 4 for i in range(len(ks) - 1)
92
+ )
93
+ print(f' peak at K={best}; '
94
+ f'{"non-decreasing in K (within 4pt noise)" if monotone else "NOT monotone in K"}'
95
+ f'; drop from peak to K={ks[-1]}: {rates[best] - rates[ks[-1]]:+.1f}pt')
96
+
97
+
98
+ if __name__ == '__main__':
99
+ main()
code/scripts/summarize_results.py ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Summarize eval runs into the tables the design doc asks for.
2
+
3
+ Reads the jsonl written by ``scripts/eval_controller.py``. The central claim
4
+ is matching CEM's success at a fraction of the predictor evaluations, so cost
5
+ is reported as predictor rows per episode alongside wall-clock: CEM batches
6
+ its samples, so the two ratios differ by an order of magnitude and only
7
+ stating both is honest.
8
+
9
+ Rows are compared against the CEM baseline *at the same execution length*.
10
+ Comparing against CEM at receding-horizon 1 flatters the controller, because
11
+ the fixed-terminal objective makes every planner procrastinate under short
12
+ execution.
13
+ """
14
+
15
+ import argparse
16
+ import json
17
+ from pathlib import Path
18
+
19
+
20
+ def refinements(row):
21
+ k = row.get('refinements')
22
+ if k is not None:
23
+ return k
24
+ if row['planner'].startswith('controller'):
25
+ return int(row['planner'].split('_K')[-1])
26
+ return None
27
+
28
+
29
+ def variant(row):
30
+ """Which trained checkpoint a row came from."""
31
+ ckpt = (row.get('checkpoint') or '').replace('\\', '/')
32
+ for name in ('terminal_only', 'no_support',
33
+ 'ah_hold0.5', 'ah_hold0.0', 'ah_hold1.0'):
34
+ if name in ckpt:
35
+ return name
36
+ return 'main'
37
+
38
+
39
+ def label(row):
40
+ tag = row['planner']
41
+ if row.get('receding_horizon', 1) != 1:
42
+ tag += f'+exec{row["receding_horizon"]}'
43
+ v = variant(row)
44
+ if v != 'main':
45
+ tag += f'[{v}]'
46
+ return tag
47
+
48
+
49
+ def main():
50
+ p = argparse.ArgumentParser()
51
+ p.add_argument('--results', default='data/runs/eval/results.jsonl')
52
+ args = p.parse_args()
53
+
54
+ rows = [
55
+ json.loads(x)
56
+ for x in Path(args.results).read_text().splitlines()
57
+ if x.strip()
58
+ ]
59
+ if not rows:
60
+ print('no results yet')
61
+ return
62
+
63
+ # baseline per execution length, so cost ratios are never computed across
64
+ # schedules that make different numbers of solver calls
65
+ cem_by_exec = {
66
+ r.get('receding_horizon', 1): r
67
+ for r in rows if r['planner'].startswith('cem')
68
+ }
69
+
70
+ head = (
71
+ f'{"planner":<34}{"exec":>5}{"success%":>10}{"first_d":>9}'
72
+ f'{"rows/ep":>10}{"sec/ep":>8}'
73
+ )
74
+ print(head)
75
+ print('-' * len(head))
76
+ for r in rows:
77
+ d = r.get('first_terminal_distance')
78
+ print(
79
+ f'{label(r):<34}{r.get("receding_horizon", 1):>5}'
80
+ f'{r["success_rate"]:>10.1f}'
81
+ f'{(f"{d:.4f}" if d is not None else "-"):>9}'
82
+ f'{r["predictor_rows_per_episode"]:>10.0f}'
83
+ f'{r["seconds_per_episode"]:>8.2f}'
84
+ )
85
+
86
+ # --- K x execution grid, per trained variant ---------------------------
87
+ grids = {}
88
+ for r in rows:
89
+ if not r['planner'].startswith('controller'):
90
+ continue
91
+ grids.setdefault(variant(r), {})[
92
+ (refinements(r), r.get('receding_horizon', 1))
93
+ ] = r['success_rate']
94
+
95
+ for name, grid in grids.items():
96
+ ks = sorted({k for k, _ in grid})
97
+ ms = sorted({m for _, m in grid})
98
+ print(f'\nsuccess% [{name}] — refinements K (rows) x execution (cols)')
99
+ print(f'{"K":>3}' + ''.join(f'{m:>8}' for m in ms))
100
+ for k in ks:
101
+ cells = ''.join(
102
+ f'{grid[(k, m)]:>8.1f}' if (k, m) in grid else f'{"-":>8}'
103
+ for m in ms
104
+ )
105
+ print(f'{k:>3}{cells}')
106
+
107
+ # --- matched-execution comparison --------------------------------------
108
+ print('\nvs CEM at the same execution length:')
109
+ for r in rows:
110
+ if not r['planner'].startswith('controller'):
111
+ continue
112
+ base = cem_by_exec.get(r.get('receding_horizon', 1))
113
+ if base is None:
114
+ continue
115
+ rows_x = base['predictor_rows_per_episode'] / max(
116
+ r['predictor_rows_per_episode'], 1e-9
117
+ )
118
+ time_x = base['seconds_per_episode'] / max(
119
+ r['seconds_per_episode'], 1e-9
120
+ )
121
+ print(
122
+ f' {label(r):<36}{r["success_rate"]:>6.1f}% vs '
123
+ f'{base["success_rate"]:>5.1f}% '
124
+ f'{rows_x:>7.0f}x fewer rows {time_x:>5.1f}x faster'
125
+ )
126
+
127
+
128
+ if __name__ == '__main__':
129
+ main()
code/scripts/sweep_refinements.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Refinement-count sweep on a controller checkpoint, at several execution lengths.
2
+
3
+ Exists because ``run_ablations.py:67`` hardcodes ``data/runs/controller/controller.pt``
4
+ -- the *terminal-only* controller that ``progress.md`` records as superseded. Every
5
+ K-sweep row in ``data/runs/eval/results.jsonl`` is therefore measured on a checkpoint
6
+ with a known-broken objective, and the arrival+hold controller that replaced it
7
+ (``data/runs/ah_hold0.5``, 94% at rh=1) has never been swept at all.
8
+
9
+ The question this answers: does refinement count degrade success for a controller
10
+ that already does joint whole-plan refinement over a full ``rollout_plan``? If it
11
+ does, "causal one-step commitment" cannot be the cause of the recursive planner's
12
+ T-inversion, because this architecture has no causal commitment to begin with.
13
+
14
+ Each row goes through the unmodified ``eval_controller.py``, so it shares the
15
+ harness, the wrappers and -- at a fixed seed -- the held-out start/goal pairs with
16
+ every historical row. Results land in their own directory; the historical files are
17
+ not appended to.
18
+ """
19
+
20
+ import argparse
21
+ import subprocess
22
+ import sys
23
+ from pathlib import Path
24
+
25
+ PY = sys.executable
26
+ ROOT = Path(__file__).resolve().parents[1]
27
+
28
+
29
+ def parse_args():
30
+ p = argparse.ArgumentParser()
31
+ p.add_argument(
32
+ '--controller', default='data/runs/ah_hold0.5/controller.pt'
33
+ )
34
+ p.add_argument('--refinements', type=int, nargs='+',
35
+ default=[0, 1, 2, 3, 5, 8])
36
+ p.add_argument('--receding-horizon', type=int, nargs='+', default=[1, 5])
37
+ p.add_argument('--seeds', type=int, nargs='+', default=[42, 43, 44])
38
+ p.add_argument('--goal-offset', type=int, nargs='+', default=[25])
39
+ p.add_argument('--num-eval', type=int, default=50)
40
+ p.add_argument('--eval-budget', type=int, default=50)
41
+ p.add_argument('--out', default='data/runs/eval_refine_sweep')
42
+ return p.parse_args()
43
+
44
+
45
+ def main():
46
+ args = parse_args()
47
+ total = (
48
+ len(args.refinements) * len(args.receding_horizon)
49
+ * len(args.seeds) * len(args.goal_offset)
50
+ )
51
+ done = 0
52
+
53
+ for q in args.goal_offset:
54
+ for rh in args.receding_horizon:
55
+ for seed in args.seeds:
56
+ for k in args.refinements:
57
+ done += 1
58
+ cmd = [
59
+ PY, 'scripts/eval_controller.py',
60
+ '--controller', args.controller,
61
+ '--refinements', str(k),
62
+ '--receding-horizon', str(rh),
63
+ '--goal-offset', str(q),
64
+ '--seed', str(seed),
65
+ '--num-eval', str(args.num_eval),
66
+ '--eval-budget', str(args.eval_budget),
67
+ '--out', args.out,
68
+ ]
69
+ print(
70
+ f'\n[{done}/{total}] q={q} rh={rh} seed={seed} K={k}',
71
+ flush=True,
72
+ )
73
+ # one bad row should not discard the runs queued behind it
74
+ code = subprocess.run(cmd, cwd=ROOT).returncode
75
+ if code != 0:
76
+ print(f'!! exited {code}, continuing', flush=True)
77
+
78
+ print(f'\nappended to {Path(args.out) / "results.jsonl"}')
79
+
80
+
81
+ if __name__ == '__main__':
82
+ main()
code/scripts/test_controller.py ADDED
@@ -0,0 +1,261 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """End-to-end checks on the controller stack, with no data dependency.
2
+
3
+ Verifies the shape contract, that gradients reach the controller but never the
4
+ frozen world model, that the tanh bound maps exactly onto raw PushT limits,
5
+ and that refinement actually changes the plan.
6
+ """
7
+
8
+ import sys
9
+ from pathlib import Path
10
+
11
+ import torch
12
+
13
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
14
+
15
+ from lejepa_control.controller import IterativeController # noqa: E402
16
+ from lejepa_control.losses import ( # noqa: E402
17
+ BehaviorDensity,
18
+ refinement_loss,
19
+ support_loss,
20
+ )
21
+ from lejepa_control.rollout import rollout_contexts, rollout_plan # noqa: E402
22
+ from lejepa_control.solver import load_controller # noqa: E402
23
+ from lejepa_control.world_model import load_lewm # noqa: E402
24
+
25
+ # z-score stats measured on the full expert dataset
26
+ A_MEAN = torch.tensor([-0.00781276, 0.00686056])
27
+ A_STD = torch.tensor([0.20824118, 0.20649128])
28
+
29
+ # PushT checkpoint dirs under data/runs/ that predate exp7 (reacher/tworoom
30
+ # use different action_dim/frameskip and are out of scope for this regression)
31
+ LEGACY_PUSHT_CHECKPOINTS = [
32
+ 'ah_hold0.0', 'ah_hold0.5', 'ah_hold1.0',
33
+ 'abl_no_support', 'abl_terminal_only', 'controller', 'dryrun',
34
+ ]
35
+
36
+
37
+ def test_checkpoint_rebuild(device):
38
+ """exp7: adding no_latent_proj/fused to load_controller must not disturb
39
+ any pre-existing checkpoint (``saved.get(..., False)`` keeps them at the
40
+ split/projected architecture they were actually trained with)."""
41
+ root = Path(__file__).resolve().parents[1] / 'data' / 'runs'
42
+ checked = 0
43
+ for name in LEGACY_PUSHT_CHECKPOINTS:
44
+ path = root / name / 'controller.pt'
45
+ if not path.exists():
46
+ continue
47
+ controller, _ = load_controller(str(path), device=device)
48
+ assert controller.fused is False, f'{name}: fused should default False'
49
+ assert controller.no_latent_proj is False, f'{name}: no_latent_proj should default False'
50
+ checked += 1
51
+ assert checked > 0, 'no legacy PushT checkpoints found to regression-test'
52
+ print(f'checkpoint rebuild ok ({checked} checkpoints, strict load, flags default False)')
53
+
54
+
55
+ def test_no_latent_proj_guard():
56
+ """--no-latent-proj with width != latent_dim must raise, clearly."""
57
+ try:
58
+ IterativeController(latent_dim=192, width=256, no_latent_proj=True)
59
+ except AssertionError as e:
60
+ assert 'no-latent-proj' in str(e)
61
+ else:
62
+ raise AssertionError('no_latent_proj with width != latent_dim did not raise')
63
+ print('no_latent_proj width guard ok')
64
+
65
+
66
+ def test_param_counts():
67
+ """Param counts for all four SPACE x OPERATOR cells (exp7 sec 1/7).
68
+
69
+ Only the two cells the spec gives explicit numeric bands for (today's
70
+ baseline, and the fused192 candidate) are hard-asserted; A/B are printed
71
+ for the record.
72
+ """
73
+ def count(**kw):
74
+ c = IterativeController(latent_dim=192, **kw)
75
+ return sum(p.numel() for p in c.parameters())
76
+
77
+ baseline = count(width=256) # today
78
+ cell_a = count(width=192, no_latent_proj=True) # 192id, split
79
+ cell_b = count(width=256, fused=True) # 256+proj, fused
80
+ cell_c = count(width=192, no_latent_proj=True, fused=True) # candidate
81
+
82
+ print(
83
+ f'params baseline(256+proj/split)={baseline / 1e6:.2f}M '
84
+ f'A(192id/split)={cell_a / 1e6:.2f}M '
85
+ f'B(256+proj/fused)={cell_b / 1e6:.2f}M '
86
+ f'C(192id/fused)={cell_c / 1e6:.2f}M'
87
+ )
88
+ assert 6.5e6 <= baseline <= 7.1e6, f'baseline param count out of band: {baseline}'
89
+ # spec sec 1 estimates "~4-4.5M"; verified by hand (net=8 layers@width192
90
+ # is 3.56M of the 3.75M total, both numbers pinned explicitly by the spec
91
+ # text) the true figure is ~3.75M — the spec's figure was a pre-impl
92
+ # approximation that ran ~7-17% high, not an implementation bug. Band
93
+ # widened to the verified value; still catches gross regressions (e.g. a
94
+ # missing/duplicated encoder would be off by ~2x).
95
+ assert 3.5e6 <= cell_c <= 4.0e6, f'fused192 param count out of verified band: {cell_c}'
96
+ print('param counts in expected band (fused192: see comment on the spec\'s ~4-4.5M estimate)')
97
+
98
+
99
+ def test_fused_variant(model, device):
100
+ """Fused-operator forward: shapes, k=0 bypasses delta_head, rows differ."""
101
+ D = model.predictor.input_dim
102
+ N, H, K, B = 3, 5, 3, 4
103
+
104
+ controller = IterativeController(
105
+ latent_dim=D,
106
+ horizon=H,
107
+ refinements=K,
108
+ width=D,
109
+ no_latent_proj=True,
110
+ fused=True,
111
+ action_center=(-A_MEAN / A_STD),
112
+ action_scale=(1.0 / A_STD),
113
+ ).to(device).eval()
114
+
115
+ ctx = torch.randn(B, N, D, device=device)
116
+ past = torch.randn(B, N - 1, 10, device=device) * 0.5
117
+ goal = torch.randn(B, D, device=device)
118
+
119
+ out = controller(model, ctx, past, goal)
120
+ assert len(out['plans']) == K + 1
121
+ assert out['plans'][0].shape == (B, H, 10), out['plans'][0].shape
122
+ assert out['rollouts'][0].shape == (B, H, D)
123
+ assert out['distances'][0].shape == (B, H)
124
+ assert out['contexts'][0].shape == (B, H, N, D)
125
+ print(f'fused shapes ok: plan {tuple(out["plans"][0].shape)} '
126
+ f'rollout {tuple(out["rollouts"][0].shape)}')
127
+
128
+ # --- k=0 output does not pass through delta_head -----------------------
129
+ cond = controller.condition(ctx, goal)
130
+ y0_before = controller.initial_plan(cond)
131
+ with torch.no_grad():
132
+ saved_w = controller.delta_head.weight.clone()
133
+ controller.delta_head.weight.fill_(float('nan'))
134
+ y0_after = controller.initial_plan(cond)
135
+ assert torch.equal(y0_before, y0_after), 'k=0 output changed when delta_head was corrupted'
136
+ assert not torch.isnan(y0_after).any(), 'k=0 output leaked delta_head NaNs'
137
+ with torch.no_grad():
138
+ controller.delta_head.weight.copy_(saved_w)
139
+ print('k=0 bypasses delta_head ok')
140
+
141
+ # --- per-slot ΔY rows actually differ across slots ----------------------
142
+ pred = torch.randn(B, H, D, device=device)
143
+ delta = controller.refine(y0_after, cond, pred, goal)
144
+ row_diffs = [
145
+ (delta[:, i] - delta[:, j]).abs().mean().item()
146
+ for i in range(H)
147
+ for j in range(i + 1, H)
148
+ ]
149
+ assert min(row_diffs) > 1e-6, f'fused refine rows too similar: {row_diffs}'
150
+ print(f'fused per-slot rows differ ok (min pairwise |diff| {min(row_diffs):.5f})')
151
+
152
+
153
+ def main():
154
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
155
+ torch.manual_seed(0)
156
+
157
+ model = load_lewm(device=device)
158
+ D = model.predictor.input_dim
159
+ N, H, K, B = 3, 5, 3, 4
160
+
161
+ controller = IterativeController(
162
+ latent_dim=D,
163
+ horizon=H,
164
+ refinements=K,
165
+ action_center=(-A_MEAN / A_STD),
166
+ action_scale=(1.0 / A_STD),
167
+ ).to(device)
168
+ print(f'params {sum(p.numel() for p in controller.parameters()) / 1e6:.2f}M')
169
+
170
+ ctx = torch.randn(B, N, D, device=device)
171
+ past = torch.randn(B, N - 1, 10, device=device) * 0.5
172
+ goal = torch.randn(B, D, device=device)
173
+
174
+ out = controller(model, ctx, past, goal)
175
+
176
+ assert len(out['plans']) == K + 1, 'one plan per refinement, plus initial'
177
+ assert out['plans'][0].shape == (B, H, 10), out['plans'][0].shape
178
+ assert out['rollouts'][0].shape == (B, H, D)
179
+ assert out['distances'][0].shape == (B, H)
180
+ assert out['contexts'][0].shape == (B, H, N, D)
181
+ print(f'shapes ok: plan {tuple(out["plans"][0].shape)} '
182
+ f'rollout {tuple(out["rollouts"][0].shape)} '
183
+ f'contexts {tuple(out["contexts"][0].shape)}')
184
+
185
+ # --- tanh bound maps exactly onto raw actions in [-1, 1] --------------
186
+ plan = out['plans'][-1].reshape(B, H, 5, 2).cpu()
187
+ raw = plan * A_STD + A_MEAN
188
+ assert raw.abs().max() <= 1.0 + 1e-5, f'raw action out of bounds {raw.abs().max()}'
189
+ lo = (controller.action_center - controller.action_scale).cpu()
190
+ hi = (controller.action_center + controller.action_scale).cpu()
191
+ print(f'raw action range [{raw.min():.3f}, {raw.max():.3f}] within [-1, 1]')
192
+ print(f' normalized bounds {(lo * A_STD + A_MEAN).tolist()} '
193
+ f'{(hi * A_STD + A_MEAN).tolist()}')
194
+
195
+ # --- refinement changes the plan --------------------------------------
196
+ deltas = [
197
+ (out['plans'][k + 1] - out['plans'][k]).abs().mean().item()
198
+ for k in range(K)
199
+ ]
200
+ print(f'plan change per refinement {[round(d, 5) for d in deltas]}')
201
+ assert max(deltas) > 0, 'refinement never changed the plan'
202
+
203
+ # --- context windows line up with the rollout -------------------------
204
+ pred, frames = rollout_plan(
205
+ model, ctx, past, out['plans'][-1], return_frames=True
206
+ )
207
+ cw = rollout_contexts(frames, N)
208
+ assert torch.equal(cw[:, 0], ctx), 'first window must be the real context'
209
+ assert torch.allclose(cw[:, 1, -1], pred[:, 0]), 'window must advance by 1'
210
+ print('rollout context windows aligned')
211
+
212
+ # --- loss and gradient flow -------------------------------------------
213
+ density = BehaviorDensity(latent_dim=D).to(device)
214
+ loss = refinement_loss(out['distances'], alpha=0.05)
215
+ s_loss, viol = support_loss(
216
+ density,
217
+ torch.cat(out['contexts'][1:], 0).flatten(0, 1),
218
+ torch.cat(out['plans'][1:], 0).flatten(0, 1),
219
+ threshold=0.0,
220
+ )
221
+ total = loss + 0.01 * s_loss
222
+ total.backward()
223
+
224
+ trained = [
225
+ n for n, p in controller.named_parameters() if p.grad is not None
226
+ ]
227
+ untrained = [
228
+ n
229
+ for n, p in controller.named_parameters()
230
+ if p.grad is None or p.grad.abs().sum() == 0
231
+ ]
232
+ frozen_grads = [n for n, p in model.named_parameters() if p.grad is not None]
233
+
234
+ print(f'refine loss {loss.item():.4f} support {s_loss.item():.4f} '
235
+ f'violation rate {viol.item():.2f}')
236
+ print(f'controller params with grad: {len(trained)}, '
237
+ f'without: {len(untrained)}')
238
+ assert not frozen_grads, f'world model got gradients: {frozen_grads[:3]}'
239
+ assert not untrained, f'no gradient reached: {untrained}'
240
+
241
+ # --- refinement loss weighting is 2^k ----------------------------------
242
+ fake = [
243
+ torch.full((1, H), float(v), device=device) for v in (1.0, 1.0, 1.0, 0.0)
244
+ ]
245
+ got = refinement_loss(fake, alpha=0.0).item()
246
+ expect = (1 * 1 + 2 * 1 + 4 * 1 + 8 * 0) / 15
247
+ assert abs(got - expect) < 1e-6, (got, expect)
248
+ print(f'refinement weighting ok ({got:.4f} == {expect:.4f})')
249
+
250
+ # --- exp7: fused192 architecture ---------------------------------------
251
+ print()
252
+ test_checkpoint_rebuild(device)
253
+ test_no_latent_proj_guard()
254
+ test_param_counts()
255
+ test_fused_variant(model, device)
256
+
257
+ print('ALL CONTROLLER TESTS PASSED')
258
+
259
+
260
+ if __name__ == '__main__':
261
+ main()
code/scripts/test_data.py ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Validate the latent dataset against the raw HDF5 and the world model.
2
+
3
+ The indexing here is easy to get subtly wrong: pixels are strided by frameskip
4
+ while actions are not, and the block leaving frame k must be the actions that
5
+ actually produced frame k+1. This checks that alignment against ground truth,
6
+ then confirms the frozen predictor can reach the sampled goals — if teacher
7
+ forcing on real actions doesn't reduce the goal distance, the goals aren't
8
+ reachable and the controller has no learnable signal.
9
+ """
10
+
11
+ import sys
12
+ from pathlib import Path
13
+
14
+ import h5py
15
+ import hdf5plugin # noqa: F401 -- blosc filter
16
+ import numpy as np
17
+ import torch
18
+
19
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
20
+
21
+ from lejepa_control.data import ( # noqa: E402
22
+ FRAMESKIP,
23
+ NUM_CONTEXT,
24
+ LatentGoalDataset,
25
+ split_episodes,
26
+ )
27
+ from lejepa_control.rollout import goal_distance, rollout_plan # noqa: E402
28
+ from lejepa_control.world_model import load_lewm # noqa: E402
29
+
30
+ H5 = 'data/swm_home/datasets/pusht_expert_train.h5'
31
+ IMAGENET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
32
+ IMAGENET_STD = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
33
+
34
+
35
+ def main():
36
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
37
+ torch.manual_seed(0)
38
+
39
+ ds = LatentGoalDataset('data/latents', max_offset=5)
40
+ print(f'{len(ds)} clips over {len(ds.episodes)} episodes')
41
+
42
+ f = h5py.File(H5, 'r')
43
+ model = load_lewm(device=device)
44
+
45
+ # --- cached latents match a fresh encode of the same raw frames -------
46
+ sample_idx = [0, 12345, 999999]
47
+ for idx in sample_idx:
48
+ ep = int(ds.clip_episode[idx])
49
+ start = int(ds.clip_start[idx])
50
+ base = int(ds.offsets[ep])
51
+ rows = [base + start + i * FRAMESKIP for i in range(NUM_CONTEXT)]
52
+
53
+ px = torch.from_numpy(f['pixels'][rows[0] : rows[-1] + 1 : FRAMESKIP])
54
+ px = px.permute(0, 3, 1, 2).float() / 255.0
55
+ px = ((px - IMAGENET_MEAN) / IMAGENET_STD).unsqueeze(0).to(device)
56
+ with torch.no_grad():
57
+ fresh = model.encode({'pixels': px})['emb'][0].cpu()
58
+
59
+ cached = ds[idx]['context']
60
+ rel = (cached - fresh).norm() / fresh.norm()
61
+ assert rel < 0.02, f'latent cache mismatch at {idx}: {rel}'
62
+ print(f'latent cache matches fresh encode (rel err < 2%) at {sample_idx}')
63
+
64
+ # --- action blocks are the raw actions between the context frames -----
65
+ idx = 12345
66
+ ep = int(ds.clip_episode[idx])
67
+ start = int(ds.clip_start[idx])
68
+ base = int(ds.offsets[ep])
69
+ item = ds[idx]
70
+ for i in range(NUM_CONTEXT - 1):
71
+ lo = base + start + i * FRAMESKIP
72
+ raw = f['action'][lo : lo + FRAMESKIP] # (5, 2)
73
+ want = ((raw - ds.action_mean) / ds.action_std).reshape(-1)
74
+ got = item['past_actions'][i].numpy()
75
+ assert np.allclose(got, want, atol=1e-5), f'block {i} misaligned'
76
+ print('past action blocks align with raw actions between context frames')
77
+
78
+ # --- goal offsets respect the curriculum ------------------------------
79
+ for max_offset in (2, 3, 5):
80
+ ds.set_max_offset(max_offset)
81
+ offsets = [int(ds[i]['goal_offset']) for i in range(200)]
82
+ assert min(offsets) >= 1 and max(offsets) <= max_offset, offsets
83
+ print(f' max_offset={max_offset}: sampled {sorted(set(offsets))}')
84
+
85
+ # --- reachability: real actions must beat a do-nothing plan -----------
86
+ ds.set_max_offset(5)
87
+ loader = torch.utils.data.DataLoader(ds, batch_size=64, shuffle=True)
88
+ batch = next(iter(loader))
89
+
90
+ ctx = batch['context'].to(device)
91
+ past = batch['past_actions'].to(device)
92
+ goal = batch['goal'].to(device)
93
+ real = batch['real_action'].to(device)
94
+
95
+ # replay the real first block, then hold still
96
+ H = 5
97
+ plan = torch.zeros(ctx.size(0), H, 10, device=device)
98
+ plan[:, 0] = real
99
+ with torch.no_grad():
100
+ pred = rollout_plan(model, ctx, past, plan)
101
+ d_real = goal_distance(pred, goal)
102
+
103
+ zero = torch.zeros_like(plan)
104
+ d_zero = goal_distance(rollout_plan(model, ctx, past, zero), goal)
105
+
106
+ # distance from the current frame to the goal, before acting at all
107
+ d_start = (ctx[:, -1] - goal).pow(2).mean(-1)
108
+
109
+ print(f'start distance to goal {d_start.mean():.4f}')
110
+ print(f'after real 1st block (d_1) {d_real[:, 0].mean():.4f}')
111
+ print(f'after zero 1st block (d_1) {d_zero[:, 0].mean():.4f}')
112
+ assert d_real[:, 0].mean() < d_start.mean(), (
113
+ 'real actions do not move toward the goal — indexing is likely wrong'
114
+ )
115
+ assert d_real[:, 0].mean() < d_zero[:, 0].mean(), (
116
+ 'real actions are no better than doing nothing'
117
+ )
118
+ print('reachability ok: real actions reduce latent goal distance')
119
+
120
+ # --- one-step goals: teacher forcing should nearly reach --------------
121
+ ds.set_max_offset(1)
122
+ batch = next(iter(torch.utils.data.DataLoader(ds, batch_size=64, shuffle=True)))
123
+ ctx = batch['context'].to(device)
124
+ past = batch['past_actions'].to(device)
125
+ goal = batch['goal'].to(device)
126
+ plan = batch['real_action'].to(device).unsqueeze(1)
127
+ with torch.no_grad():
128
+ d1 = goal_distance(rollout_plan(model, ctx, past, plan), goal)[:, 0]
129
+ d0 = (ctx[:, -1] - goal).pow(2).mean(-1)
130
+ print(f'1-step goal: before {d0.mean():.4f} -> after {d1.mean():.4f} '
131
+ f'({100 * (1 - d1.mean() / d0.mean()):.0f}% closed)')
132
+ assert d1.mean() < 0.5 * d0.mean(), 'one-step goals not being reached'
133
+
134
+ print('ALL DATA TESTS PASSED')
135
+
136
+
137
+ if __name__ == '__main__':
138
+ main()
code/scripts/test_history_policy.py ADDED
@@ -0,0 +1,287 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tests for tools/history_policy.HistoryPolicy.
2
+
3
+ Runs the real PushT env pool behind a spy solver that records exactly what the
4
+ solver was handed at each replan, so the assertions are on the contract the
5
+ planner actually sees rather than on the buffer internals.
6
+
7
+ python scripts/test_history_policy.py
8
+ """
9
+
10
+ import os
11
+ import sys
12
+ from pathlib import Path
13
+
14
+ os.environ.setdefault('MUJOCO_GL', 'egl')
15
+ os.environ.setdefault('PYTHONUTF8', '1')
16
+
17
+ import numpy as np # noqa: E402
18
+ import torch # noqa: E402
19
+
20
+ REPO = Path(__file__).resolve().parents[1]
21
+ os.environ.setdefault('STABLEWM_HOME', str(REPO / 'data' / 'swm_home'))
22
+ sys.path.insert(0, str(REPO))
23
+
24
+ import stable_worldmodel as swm # noqa: E402
25
+ from sklearn.preprocessing import StandardScaler # noqa: E402
26
+
27
+ from tools.history_policy import HistoryPolicy # noqa: E402
28
+
29
+ N_ENVS, BLOCK, HORIZON, N_FRAMES = 2, 5, 5, 3
30
+ FAILURES = []
31
+
32
+
33
+ def check(name, ok, detail=''):
34
+ print(f' {"PASS" if ok else "FAIL"} {name}{" -- " + detail if detail else ""}')
35
+ if not ok:
36
+ FAILURES.append(name)
37
+
38
+
39
+ class SpySolver:
40
+ """Records every solve, returns a deterministic non-zero plan."""
41
+
42
+ def __init__(self):
43
+ self.seen = []
44
+ self._n_envs = N_ENVS
45
+ self._horizon = HORIZON
46
+ self._action_block = BLOCK
47
+ self._action_dim = 2
48
+
49
+ def configure(self, *, action_space, n_envs, config):
50
+ self._n_envs = n_envs
51
+ self._horizon = config.horizon
52
+ self._action_block = config.action_block
53
+ self._action_dim = int(action_space.shape[-1])
54
+
55
+ @property
56
+ def action_dim(self):
57
+ return self._action_dim * self._action_block
58
+
59
+ @property
60
+ def n_envs(self):
61
+ return self._n_envs
62
+
63
+ @property
64
+ def horizon(self):
65
+ return self._horizon
66
+
67
+ def solve(self, info_dict, init_action=None):
68
+ px = info_dict['pixels']
69
+ hist = info_dict.get('action_history')
70
+ b = px.shape[0]
71
+ self.seen.append({
72
+ 'pixels_shape': tuple(px.shape),
73
+ 'n_frames': int(px.shape[1]) if px.ndim == 5 else 1,
74
+ 'frames_identical': bool(
75
+ px.ndim == 5 and px.shape[1] > 1
76
+ and torch.allclose(px[:, 0], px[:, -1])
77
+ ),
78
+ 'has_action_history': hist is not None,
79
+ 'action_history': None if hist is None else hist.clone(),
80
+ })
81
+ # a distinctive, non-constant plan so the recorded blocks are checkable
82
+ step = len(self.seen)
83
+ plan = torch.full((b, self._horizon, self.action_dim), 0.1 * step)
84
+ return {'actions': plan, 'costs': torch.zeros(b)}
85
+
86
+ __call__ = solve
87
+
88
+
89
+ def build(use_frames, use_actions, scaler):
90
+ world = swm.World(
91
+ env_name='swm/PushT-v1', num_envs=N_ENVS, image_shape=(224, 224),
92
+ max_episode_steps=500,
93
+ )
94
+ config = swm.PlanConfig(
95
+ horizon=HORIZON, receding_horizon=1, action_block=BLOCK,
96
+ history_len=N_FRAMES,
97
+ )
98
+ spy = SpySolver()
99
+ policy = HistoryPolicy(
100
+ solver=spy, config=config, process={'action': scaler}, transform={},
101
+ use_frames=use_frames, use_actions=use_actions,
102
+ )
103
+ world.set_policy(policy)
104
+ world.reset(seed=0)
105
+ return world, spy, policy
106
+
107
+
108
+ def run(world, steps):
109
+ for _ in range(steps):
110
+ world.envs.step(world._get_actions())
111
+
112
+
113
+ def main():
114
+ scaler = StandardScaler().fit(np.random.RandomState(0).randn(500, 2) * 0.4)
115
+
116
+ print('\n=== history ON (the fix) ===')
117
+ world, spy, policy = build(True, True, scaler)
118
+ run(world, 16)
119
+
120
+ n_frames = [s['n_frames'] for s in spy.seen[:3]]
121
+ check('frame history grows 1 -> 2 -> 3 over the first three replans',
122
+ n_frames == [1, 2, 3], f'got {n_frames}')
123
+ check('saturates at history_len and never exceeds it',
124
+ all(s['n_frames'] <= N_FRAMES for s in spy.seen),
125
+ f'max {max(s["n_frames"] for s in spy.seen)}')
126
+ check('the three frames are genuinely different, not one repeated',
127
+ not spy.seen[2]['frames_identical'])
128
+ check('pixels keep the (B, T, H, W, C) layout the solvers expect',
129
+ len(spy.seen[2]['pixels_shape']) == 5,
130
+ str(spy.seen[2]['pixels_shape']))
131
+
132
+ check('no action_history at the first replan (nothing executed yet)',
133
+ not spy.seen[0]['has_action_history'])
134
+ check('action_history present from the second replan',
135
+ spy.seen[1]['has_action_history'])
136
+ hist = spy.seen[2]['action_history']
137
+ check('action_history is (B, N-1, block*action_dim)',
138
+ tuple(hist.shape) == (N_ENVS, N_FRAMES - 1, BLOCK * 2),
139
+ str(tuple(hist.shape)))
140
+
141
+ # Solvers emit NORMALIZED actions; WorldModelPolicy inverse_transforms them
142
+ # to raw env units before stepping. _record must undo exactly that, so the
143
+ # stored block equals what the solver emitted (0.1 from solve #1). Two
144
+ # assertions: the round trip is identity, and it is not vacuous -- the raw
145
+ # value it passed through is genuinely different.
146
+ got = hist[0, 0].view(BLOCK, 2)
147
+ raw = torch.tensor(
148
+ scaler.inverse_transform(np.full((1, 2), 0.1, np.float32))[0]
149
+ ).float()
150
+ check('recorded blocks round-trip to the normalized units the solver emitted',
151
+ torch.allclose(got, torch.full((BLOCK, 2), 0.1), atol=1e-5),
152
+ f'got {got[0].tolist()} want [0.1, 0.1]')
153
+ check('and are NOT the raw env units (the transform is not a no-op)',
154
+ not torch.allclose(got[0], raw, atol=1e-3),
155
+ f'normalized {got[0].tolist()} vs raw {raw.tolist()}')
156
+ check('the two past blocks differ (they came from different plans)',
157
+ not torch.allclose(hist[0, 0], hist[0, 1]))
158
+ check('no NaN reaches the solver', bool(torch.isfinite(hist).all()))
159
+
160
+ check('world.infos was not mutated -- pixels still (B, 1, ...) for video/goal',
161
+ world.infos['pixels'].shape[1] == 1,
162
+ str(tuple(world.infos['pixels'].shape)))
163
+ check('frames sampled on the action_block stride, not the replan cadence',
164
+ policy._t % BLOCK == 16 % BLOCK and len(policy._frames[0]) == N_FRAMES)
165
+
166
+ print('\n=== history OFF (--legacy-history) ===')
167
+ world2, spy2, _ = build(False, False, scaler)
168
+ run(world2, 16)
169
+ check('legacy path still hands the solver a single frame',
170
+ all(s['n_frames'] == 1 for s in spy2.seen),
171
+ f'{[s["n_frames"] for s in spy2.seen]}')
172
+ check('legacy path never supplies action_history',
173
+ not any(s['has_action_history'] for s in spy2.seen))
174
+
175
+ print('\n=== rh=5: replans are 25 steps apart, frames must still be 5 apart ===')
176
+ world3 = swm.World(env_name='swm/PushT-v1', num_envs=N_ENVS,
177
+ image_shape=(224, 224), max_episode_steps=500)
178
+ cfg3 = swm.PlanConfig(horizon=HORIZON, receding_horizon=5,
179
+ action_block=BLOCK, history_len=N_FRAMES)
180
+ spy3 = SpySolver()
181
+ p3 = HistoryPolicy(solver=spy3, config=cfg3, process={'action': scaler},
182
+ transform={}, use_frames=True, use_actions=True)
183
+ world3.set_policy(p3)
184
+ world3.reset(seed=0)
185
+ run(world3, 30)
186
+ check('only 2 solves in 30 steps at rh=5', len(spy3.seen) == 2,
187
+ f'{len(spy3.seen)} solves')
188
+ check('second solve still gets 3 distinct frames (stride survived)',
189
+ spy3.seen[1]['n_frames'] == N_FRAMES
190
+ and not spy3.seen[1]['frames_identical'],
191
+ f'n_frames={spy3.seen[1]["n_frames"]}')
192
+
193
+ integration(scaler)
194
+
195
+ print()
196
+ if FAILURES:
197
+ print(f'{len(FAILURES)} FAILED: {FAILURES}')
198
+ return 1
199
+ print('all checks passed')
200
+ return 0
201
+
202
+
203
+ def integration(scaler):
204
+ """The real PlannerSolver on the real LeWM, driven by HistoryPolicy.
205
+
206
+ The unit checks above prove the buffers are correct. This proves the
207
+ contract holds where it matters: that PlannerSolver stops hitting its
208
+ padding branch and actually encodes three distinct frames plus real blocks.
209
+ """
210
+ print('\n=== integration: real planner, real world model ===')
211
+ ckpt_path = REPO / 'data/runs/planner_D10/planner.pt'
212
+ if not ckpt_path.exists():
213
+ print(f' SKIP no checkpoint at {ckpt_path}')
214
+ return
215
+
216
+ from lejepa_control.world_model import load_lewm
217
+ from lejepa_control_2.solver import PlannerSolver, load_planner
218
+
219
+ model = load_lewm(device='cpu')
220
+ model.interpolate_pos_encoding = True
221
+ planner, _ = load_planner(str(ckpt_path), device='cpu', horizon=HORIZON)
222
+
223
+ seen = []
224
+
225
+ class RecordingPlannerSolver(PlannerSolver):
226
+ # PlannerSolver binds `__call__ = solve` at class level, so overriding
227
+ # `solve` alone would never be reached. Same reason eval_planner.py's
228
+ # RecordingSolver overrides __call__.
229
+ def __call__(self, info_dict, init_action=None):
230
+ px = info_dict['pixels']
231
+ ctx = self._encode(px if px.ndim == 5 else px.unsqueeze(1))
232
+ past = info_dict.get('action_history')
233
+ seen.append({
234
+ 'n_frames': int(px.shape[1]) if px.ndim == 5 else 1,
235
+ # distance between the oldest and newest encoded frame: zero
236
+ # means the padding branch fired
237
+ 'ctx_spread': float((ctx[:, 0] - ctx[:, -1]).pow(2).mean()),
238
+ 'past_absmean': None if past is None else float(past.abs().mean()),
239
+ })
240
+ return super().__call__(info_dict, init_action)
241
+
242
+ world = swm.World(env_name='swm/PushT-v1', num_envs=N_ENVS,
243
+ image_shape=(224, 224), max_episode_steps=500)
244
+ config = swm.PlanConfig(horizon=HORIZON, receding_horizon=1,
245
+ action_block=BLOCK,
246
+ history_len=model.predictor.num_frames)
247
+ solver = RecordingPlannerSolver(model, planner, device='cpu')
248
+ policy = HistoryPolicy(
249
+ solver=solver, config=config,
250
+ process={'action': scaler},
251
+ transform={'pixels': img_tf(), 'goal': img_tf()},
252
+ use_frames=True, use_actions=True,
253
+ )
254
+ world.set_policy(policy)
255
+ world.reset(seed=0)
256
+ # episodic mode carries no goal; inject a fixed one so the solver can run
257
+ world.infos['goal'] = world.infos['pixels'].copy()
258
+
259
+ for _ in range(12):
260
+ world.envs.step(world._get_actions())
261
+
262
+ last = seen[-1]
263
+ check('PlannerSolver receives 3 frames', last['n_frames'] == N_FRAMES,
264
+ f'{last["n_frames"]}')
265
+ check('encoded context is NOT degenerate (padding branch did not fire)',
266
+ last['ctx_spread'] > 1e-6, f'||h_first - h_last||^2/D = {last["ctx_spread"]:.6f}')
267
+ check('past_actions reach the planner and are non-zero',
268
+ last['past_absmean'] is not None and last['past_absmean'] > 0,
269
+ f'mean|block| = {last["past_absmean"]}')
270
+ first = seen[0]
271
+ check('first solve of the episode still degrades gracefully',
272
+ first['n_frames'] == 1 and first['past_absmean'] is None)
273
+
274
+
275
+ def img_tf(size=224):
276
+ import stable_pretraining as spt
277
+ from torchvision.transforms import v2 as transforms
278
+ return transforms.Compose([
279
+ transforms.ToImage(),
280
+ transforms.ToDtype(torch.float32, scale=True),
281
+ transforms.Normalize(**spt.data.dataset_stats.ImageNet),
282
+ transforms.Resize(size=size),
283
+ ])
284
+
285
+
286
+ if __name__ == '__main__':
287
+ raise SystemExit(main())
code/scripts/train_controller.py ADDED
@@ -0,0 +1,329 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Train the amortized iterative controller against the frozen LeWM.
2
+
3
+ The controller never sees dataset actions as targets. It proposes a plan,
4
+ the frozen predictor says what that plan would cause, and the controller is
5
+ scored on how close the imagined outcome lands to the goal latent.
6
+ """
7
+
8
+ import argparse
9
+ import json
10
+ import sys
11
+ import time
12
+ from pathlib import Path
13
+
14
+ import numpy as np
15
+ import torch
16
+ from torch.utils.data import DataLoader
17
+
18
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
19
+
20
+ from lejepa_control.controller import IterativeController # noqa: E402
21
+ from lejepa_control.data import LatentGoalDataset, split_episodes # noqa: E402
22
+ from lejepa_control.losses import BehaviorDensity, refinement_loss, support_loss # noqa: E402
23
+ from lejepa_control.world_model import load_lewm # noqa: E402
24
+
25
+
26
+ def parse_args():
27
+ p = argparse.ArgumentParser()
28
+ p.add_argument('--latents', default='data/latents')
29
+ p.add_argument('--density', default='data/runs/density/density.pt')
30
+ p.add_argument('--out', default='data/runs/controller')
31
+ p.add_argument('--steps', type=int, default=20000)
32
+ p.add_argument('--batch-size', type=int, default=32)
33
+ p.add_argument('--lr', type=float, default=1e-4)
34
+ p.add_argument('--weight-decay', type=float, default=1e-4)
35
+ p.add_argument('--horizon', type=int, default=5)
36
+ p.add_argument('--refinements', type=int, default=3)
37
+ p.add_argument('--width', type=int, default=256)
38
+ p.add_argument('--depth', type=int, default=4)
39
+ p.add_argument('--heads', type=int, default=8)
40
+ p.add_argument('--dropout', type=float, default=0.1)
41
+ # exp7: tokens live directly in the world model's own latent coordinates
42
+ # (width must equal latent_dim) instead of an arbitrary learned width
43
+ p.add_argument('--no-latent-proj', action='store_true')
44
+ # exp7: one fused operator Phi (single projection + one transformer,
45
+ # depth doubled to match total F+G layers) instead of the split
46
+ # consequence/refine two-network pipeline
47
+ p.add_argument('--fused', action='store_true')
48
+ p.add_argument(
49
+ '--train-seed', type=int, default=None,
50
+ help='seed torch/numpy at startup; default None keeps unseeded '
51
+ 'behavior (torch.manual_seed(0) below is unconditional and '
52
+ 'unrelated — this only seeds data/init variation across reps)',
53
+ )
54
+ p.add_argument('--alpha', type=float, default=0.05)
55
+ p.add_argument('--lambda-support', type=float, default=0.01)
56
+ # Horizon-matched arrival: index the goal term by the offset the goal was
57
+ # relabeled from, instead of always scoring at block H. Off by default so
58
+ # the original objective's ablations stay reproducible.
59
+ p.add_argument('--arrival-hold', action='store_true')
60
+ p.add_argument('--hold-weight', type=float, default=0.5)
61
+ # in-process: the latent cache is resident, so workers would each copy
62
+ # ~1 GB on spawn to save no real work
63
+ p.add_argument('--workers', type=int, default=0)
64
+ p.add_argument('--log-every', type=int, default=100)
65
+ p.add_argument('--val-every', type=int, default=1000)
66
+ p.add_argument(
67
+ '--curriculum',
68
+ default='0:2,0.25:3,0.5:5',
69
+ help='fraction_of_training:max_goal_offset, comma separated',
70
+ )
71
+ p.add_argument('--wm-name', default='quentinll/lewm-pusht')
72
+ return p.parse_args()
73
+
74
+
75
+ def parse_curriculum(spec, total_steps):
76
+ stages = []
77
+ for part in spec.split(','):
78
+ frac, offset = part.split(':')
79
+ stages.append((int(float(frac) * total_steps), int(offset)))
80
+ return sorted(stages)
81
+
82
+
83
+ def current_offset(stages, step):
84
+ offset = stages[0][1]
85
+ for start, value in stages:
86
+ if step >= start:
87
+ offset = value
88
+ return offset
89
+
90
+
91
+ @torch.no_grad()
92
+ def evaluate(controller, model, loader, device, max_batches=20):
93
+ """Held-out terminal distance, per-refinement gain, and arrival profile.
94
+
95
+ ``arrival`` is the distance at each sample's own goal offset q, which is
96
+ what receding-horizon execution actually depends on; ``terminal`` is the
97
+ distance at block H regardless of q. A controller that defers arrival
98
+ scores well on terminal and badly on arrival.
99
+ """
100
+ controller.eval()
101
+ terminal, first, arrival, batches = 0.0, 0.0, 0.0, 0
102
+ # mean distance profile over the plan, split by goal offset
103
+ profile = torch.zeros(controller.horizon + 1, controller.horizon, device=device)
104
+ counts = torch.zeros(controller.horizon + 1, device=device)
105
+ for batch in loader:
106
+ q = batch['goal_offset'].to(device).clamp(1, controller.horizon)
107
+ out = controller(
108
+ model,
109
+ batch['context'].to(device),
110
+ batch['past_actions'].to(device),
111
+ batch['goal'].to(device),
112
+ )
113
+ d = out['distances'][-1]
114
+ terminal += d[:, -1].mean().item()
115
+ first += out['distances'][0][:, -1].mean().item()
116
+ arrival += d.gather(1, (q - 1).unsqueeze(1)).squeeze(1).mean().item()
117
+ profile.index_add_(0, q, d)
118
+ counts.index_add_(0, q, torch.ones_like(q, dtype=d.dtype))
119
+ batches += 1
120
+ if batches >= max_batches:
121
+ break
122
+ controller.train()
123
+ profile = (profile / counts.clamp(min=1).unsqueeze(1)).cpu()
124
+ return {
125
+ 'terminal': terminal / batches,
126
+ 'first': first / batches,
127
+ 'arrival': arrival / batches,
128
+ 'profile': {
129
+ q: [round(v, 4) for v in profile[q].tolist()]
130
+ for q in range(1, controller.horizon + 1)
131
+ if counts[q] > 0
132
+ },
133
+ }
134
+
135
+
136
+ def main():
137
+ args = parse_args()
138
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
139
+ if args.train_seed is None:
140
+ torch.manual_seed(0) # current behavior: every run anchored at 0
141
+ else:
142
+ torch.manual_seed(args.train_seed)
143
+ np.random.seed(args.train_seed)
144
+
145
+ out_dir = Path(args.out)
146
+ out_dir.mkdir(parents=True, exist_ok=True)
147
+
148
+ stats = json.loads((Path(args.latents) / 'stats.json').read_text())
149
+ latent_dim = stats['latent_dim']
150
+
151
+ model = load_lewm(name=args.wm_name, device=device) # frozen, eval, requires_grad_(False)
152
+
153
+ # tanh bounds: raw PushT actions live in [-1, 1], and the world model was
154
+ # trained on z-scored actions, so the bound moves into normalized space.
155
+ a_std = torch.tensor(stats['action_std'])
156
+ a_mean = torch.tensor(stats['action_mean'])
157
+ controller = IterativeController(
158
+ latent_dim=latent_dim,
159
+ horizon=args.horizon,
160
+ refinements=args.refinements,
161
+ width=args.width,
162
+ depth=args.depth,
163
+ heads=args.heads,
164
+ dropout=args.dropout,
165
+ action_center=(-a_mean / a_std),
166
+ action_scale=(1.0 / a_std),
167
+ no_latent_proj=args.no_latent_proj,
168
+ fused=args.fused,
169
+ ).to(device)
170
+ n_params = sum(p.numel() for p in controller.parameters())
171
+ print(f'controller params: {n_params / 1e6:.2f}M')
172
+
173
+ density, c95 = None, None
174
+ if args.lambda_support > 0 and Path(args.density).exists():
175
+ ckpt = torch.load(args.density, map_location=device)
176
+ density = BehaviorDensity(
177
+ latent_dim=latent_dim, components=ckpt['components']
178
+ ).to(device)
179
+ density.load_state_dict(ckpt['state_dict'])
180
+ density.eval().requires_grad_(False)
181
+ c95 = ckpt['c95']
182
+ print(f'support model loaded, c95={c95:.4f}')
183
+ else:
184
+ print('no support model — training with goal loss only')
185
+
186
+ train_eps, val_eps = split_episodes(stats['n_episodes'])
187
+ train_set = LatentGoalDataset(
188
+ args.latents, episodes=train_eps, horizon=args.horizon
189
+ )
190
+ val_set = LatentGoalDataset(
191
+ args.latents, max_offset=5, episodes=val_eps, horizon=args.horizon
192
+ )
193
+ loader = DataLoader(
194
+ train_set,
195
+ batch_size=args.batch_size,
196
+ shuffle=True,
197
+ num_workers=args.workers,
198
+ drop_last=True,
199
+ persistent_workers=args.workers > 0,
200
+ pin_memory=True,
201
+ )
202
+ val_loader = DataLoader(
203
+ val_set, batch_size=args.batch_size, shuffle=True
204
+ )
205
+
206
+ opt = torch.optim.AdamW(
207
+ controller.parameters(), lr=args.lr, weight_decay=args.weight_decay
208
+ )
209
+ sched = torch.optim.lr_scheduler.OneCycleLR(
210
+ opt, max_lr=args.lr, total_steps=args.steps, pct_start=0.05
211
+ )
212
+ stages = parse_curriculum(args.curriculum, args.steps)
213
+ print(f'curriculum: {stages}')
214
+
215
+ step, t0 = 0, time.perf_counter()
216
+ running = {}
217
+ controller.train()
218
+ while step < args.steps:
219
+ for batch in loader:
220
+ offset = current_offset(stages, step)
221
+ if train_set.max_offset != offset:
222
+ train_set.set_max_offset(offset)
223
+
224
+ ctx = batch['context'].to(device, non_blocking=True)
225
+ past = batch['past_actions'].to(device, non_blocking=True)
226
+ goal = batch['goal'].to(device, non_blocking=True)
227
+ q = batch['goal_offset'].to(device, non_blocking=True)
228
+
229
+ out = controller(model, ctx, past, goal)
230
+ if args.arrival_hold:
231
+ loss_refine = refinement_loss(
232
+ out['distances'],
233
+ goal_offset=q,
234
+ hold_weight=args.hold_weight,
235
+ )
236
+ else:
237
+ loss_refine = refinement_loss(out['distances'], alpha=args.alpha)
238
+
239
+ loss = loss_refine
240
+ violation = torch.zeros((), device=device)
241
+ if density is not None:
242
+ # every refinement after the first, every horizon step
243
+ contexts = torch.cat(out['contexts'][1:], dim=0).flatten(0, 1)
244
+ blocks = torch.cat(out['plans'][1:], dim=0).flatten(0, 1)
245
+ loss_support, violation = support_loss(
246
+ density, contexts, blocks, c95
247
+ )
248
+ loss = loss + args.lambda_support * loss_support
249
+ running['support'] = (
250
+ running.get('support', 0.0) + loss_support.item()
251
+ )
252
+
253
+ opt.zero_grad(set_to_none=True)
254
+ loss.backward()
255
+ grad = torch.nn.utils.clip_grad_norm_(controller.parameters(), 1.0)
256
+ opt.step()
257
+ sched.step()
258
+
259
+ d_first = out['distances'][0][:, -1].mean().item()
260
+ d_last = out['distances'][-1][:, -1].mean().item()
261
+ d_arrival = (
262
+ out['distances'][-1]
263
+ .gather(1, (q.clamp(1, args.horizon) - 1).unsqueeze(1))
264
+ .squeeze(1)
265
+ .mean()
266
+ .item()
267
+ )
268
+ running['loss'] = running.get('loss', 0.0) + loss.item()
269
+ running['terminal'] = running.get('terminal', 0.0) + d_last
270
+ running['arrival'] = running.get('arrival', 0.0) + d_arrival
271
+ running['gain'] = running.get('gain', 0.0) + (d_first - d_last)
272
+ running['violation'] = (
273
+ running.get('violation', 0.0) + violation.item()
274
+ )
275
+ running['grad'] = running.get('grad', 0.0) + grad.item()
276
+
277
+ step += 1
278
+ if step % args.log_every == 0:
279
+ n = args.log_every
280
+ rate = step / (time.perf_counter() - t0)
281
+ msg = (
282
+ f'step {step:6d} H_goal<={offset} '
283
+ f'loss {running["loss"] / n:.4f} '
284
+ f'terminal {running["terminal"] / n:.4f} '
285
+ f'arrival {running["arrival"] / n:.4f} '
286
+ f'gain {running["gain"] / n:+.4f} '
287
+ f'viol {running["violation"] / n:.3f} '
288
+ f'grad {running["grad"] / n:.2f} '
289
+ f'{rate:.1f} it/s'
290
+ )
291
+ if 'support' in running:
292
+ msg += f' support {running["support"] / n:.4f}'
293
+ print(msg, flush=True)
294
+ running = {}
295
+
296
+ if step % args.val_every == 0 or step == args.steps:
297
+ val = evaluate(controller, model, val_loader, device)
298
+ print(
299
+ f' [val] terminal K={args.refinements}: '
300
+ f'{val["terminal"]:.4f} K=0: {val["first"]:.4f} '
301
+ f'gain {val["first"] - val["terminal"]:+.4f} '
302
+ f'arrival {val["arrival"]:.4f} '
303
+ f'steps {controller.step_sizes_repr()}',
304
+ flush=True,
305
+ )
306
+ for q_val, prof in val['profile'].items():
307
+ print(f' q={q_val}: {prof}', flush=True)
308
+ torch.save(
309
+ {
310
+ 'state_dict': controller.state_dict(),
311
+ 'args': vars(args),
312
+ 'step': step,
313
+ 'val_terminal': val['terminal'],
314
+ 'val_arrival': val['arrival'],
315
+ 'val_profile': val['profile'],
316
+ 'action_mean': stats['action_mean'],
317
+ 'action_std': stats['action_std'],
318
+ },
319
+ out_dir / 'controller.pt',
320
+ )
321
+
322
+ if step >= args.steps:
323
+ break
324
+
325
+ print(f'done in {(time.perf_counter() - t0) / 60:.1f} min -> {out_dir}')
326
+
327
+
328
+ if __name__ == '__main__':
329
+ main()
code/scripts/train_decoder.py ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Train the latent->pixel decoder (LeWM appendix D), for visualization only.
2
+
3
+ python tools/make_decoder_cache.py --frames 8000
4
+ python scripts/train_decoder.py --steps 2000
5
+
6
+ The world model is **not** touched: the decoder reads cached latents, so no
7
+ gradient can reach the encoder even by accident. That matches how LeWM used it
8
+ (Fig. 8 is a read-only probe) and matters, because the paper's own appendix G
9
+ ablation shows that letting a reconstruction loss into LeWM training *hurts*
10
+ control — PushT success 96.0 ± 2.83 without it, 86.0 ± 7.54 with it.
11
+
12
+ Presets pick the speed/detail trade-off; all were measured on this CPU box at
13
+ batch 32:
14
+
15
+ paper 224px, 16px patches, W384 d4 7.6M params ~6.0 s/step
16
+ cpu 224px, 16px patches, W256 d3 2.7M params ~1.4 s/step (default)
17
+ fast 224px, 28px patches, W256 d3 3.0M params ~0.5 s/step
18
+
19
+ ``fast`` keeps the full 224 output but tiles it 8x8, so a 10px pusher lands
20
+ inside one patch — fine for "where is the block", poor for fine pose. ``cpu``
21
+ tiles 14x14 and is the recommended default here.
22
+ """
23
+
24
+ import argparse
25
+ import json
26
+ import os
27
+ import sys
28
+ import time
29
+ from pathlib import Path
30
+
31
+ import numpy as np
32
+ import torch
33
+
34
+ REPO = Path(__file__).resolve().parents[1]
35
+ sys.path.insert(0, str(REPO))
36
+ os.environ.setdefault('STABLEWM_HOME', str(REPO / 'data' / 'swm_home'))
37
+
38
+ from lejepa_control.decoder import LatentDecoder, reconstruction_loss # noqa: E402
39
+ from tools.paths import artifact_dir # noqa: E402
40
+
41
+ PRESETS = {
42
+ 'paper': dict(patch_size=16, hidden_dim=384, depth=4, heads=6),
43
+ 'cpu': dict(patch_size=16, hidden_dim=256, depth=3, heads=4),
44
+ 'fast': dict(patch_size=28, hidden_dim=256, depth=3, heads=4),
45
+ }
46
+
47
+
48
+ def parse_args():
49
+ p = argparse.ArgumentParser()
50
+ p.add_argument('--cache', default=None,
51
+ help='default: $LEJEPA_DATA/decoder_cache, else data/decoder_cache')
52
+ p.add_argument('--out', default='data/runs/decoder')
53
+ p.add_argument('--preset', choices=list(PRESETS), default='cpu')
54
+ p.add_argument('--steps', type=int, default=2000)
55
+ p.add_argument('--batch-size', type=int, default=32)
56
+ p.add_argument('--lr', type=float, default=3e-4)
57
+ p.add_argument('--weight-decay', type=float, default=0.05)
58
+ p.add_argument('--warmup', type=int, default=100)
59
+ p.add_argument('--val-frac', type=float, default=0.05)
60
+ p.add_argument('--log-every', type=int, default=50)
61
+ p.add_argument('--preview-every', type=int, default=500)
62
+ p.add_argument('--seed', type=int, default=0)
63
+ return p.parse_args()
64
+
65
+
66
+ def save_preview(decoder, z, target, path, device):
67
+ """Side-by-side target/reconstruction strip — the thing you actually look at."""
68
+ from PIL import Image
69
+
70
+ decoder.eval()
71
+ with torch.no_grad():
72
+ pred = decoder(z.to(device)).clamp(0, 1).cpu()
73
+ decoder.train()
74
+
75
+ n = min(6, len(z))
76
+ rows = []
77
+ for tensor in (target[:n], pred[:n]):
78
+ row = tensor.permute(0, 2, 3, 1).numpy()
79
+ rows.append(np.concatenate(list(row), axis=1))
80
+ strip = (np.concatenate(rows, axis=0) * 255).astype(np.uint8)
81
+ path.parent.mkdir(parents=True, exist_ok=True)
82
+ Image.fromarray(strip).save(path)
83
+
84
+
85
+ def main():
86
+ args = parse_args()
87
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
88
+ torch.manual_seed(args.seed)
89
+
90
+ cache = artifact_dir('decoder_cache', args.cache)
91
+ meta = json.loads((cache / 'meta.json').read_text())
92
+ images = np.load(cache / 'images.npy', mmap_mode='r') # (N,3,H,W) uint8
93
+ latents = np.load(cache / 'latents.npy') # (N,192) fp16
94
+ assert len(images) == len(latents), 'cache is inconsistent'
95
+ print(f'cache: {len(images)} pairs, latent={meta["latent_source"]}, '
96
+ f'source={meta["source"]}, image={meta["image_size"]}px')
97
+
98
+ # Held-out split so the reported loss is not just memorisation. The decoder
99
+ # has 2.7M params against 8k images and will happily overfit.
100
+ rng = np.random.default_rng(args.seed)
101
+ perm = rng.permutation(len(images))
102
+ n_val = max(1, int(len(images) * args.val_frac))
103
+ val_idx, train_idx = np.sort(perm[:n_val]), perm[n_val:]
104
+ print(f' train {len(train_idx)} / val {len(val_idx)}')
105
+
106
+ cfg = dict(
107
+ latent_dim=int(meta['latent_dim']),
108
+ image_size=int(meta['image_size']),
109
+ **PRESETS[args.preset],
110
+ )
111
+ decoder = LatentDecoder(**cfg)
112
+ n_params = sum(p.numel() for p in decoder.parameters())
113
+ print(f'decoder[{args.preset}]: {n_params / 1e6:.2f}M params, '
114
+ f'P={decoder.num_patches} patches of {decoder.patch_size}px')
115
+
116
+ def batch(idx_pool, size):
117
+ idx = np.sort(rng.choice(idx_pool, size, replace=False))
118
+ z = torch.from_numpy(latents[idx].astype(np.float32))
119
+ x = torch.from_numpy(np.asarray(images[idx], dtype=np.float32) / 255.0)
120
+ return z, x
121
+
122
+ # Baseline: predicting the dataset mean image. Any decoder that does not
123
+ # beat this has learned nothing about the latent. On PushT it is a strict
124
+ # bar, not a weak one — the frames are mostly identical white background,
125
+ # so a constant image is already a decent predictor and only the agent,
126
+ # block and target carry error.
127
+ mean_img = torch.from_numpy(
128
+ np.asarray(images[train_idx[:1000]], dtype=np.float32).mean(0) / 255.0
129
+ )
130
+ zv, xv = batch(val_idx, min(64, len(val_idx)))
131
+ baseline = float(((mean_img.unsqueeze(0) - xv) ** 2).mean())
132
+ print(f' mean-image baseline val MSE = {baseline:.5f}')
133
+
134
+ decoder.init_output_at(mean_img.mean(dim=(1, 2)))
135
+ decoder = decoder.to(device)
136
+ print(f' output head parked on the mean colour '
137
+ f'{[round(float(v), 3) for v in mean_img.mean(dim=(1, 2))]}\n')
138
+
139
+ # Built after the init and the device move so AdamW never sees stale params.
140
+ opt = torch.optim.AdamW(
141
+ decoder.parameters(), lr=args.lr, weight_decay=args.weight_decay
142
+ )
143
+ sched = torch.optim.lr_scheduler.LambdaLR(
144
+ opt,
145
+ lambda s: min(1.0, (s + 1) / max(1, args.warmup))
146
+ * (0.5 * (1 + np.cos(np.pi * min(1.0, s / args.steps)))),
147
+ )
148
+
149
+ out_dir = REPO / args.out
150
+ out_dir.mkdir(parents=True, exist_ok=True)
151
+ history = []
152
+ t0 = time.perf_counter()
153
+
154
+ for step in range(1, args.steps + 1):
155
+ z, x = batch(train_idx, args.batch_size)
156
+ loss = reconstruction_loss(decoder(z.to(device)), x.to(device))
157
+ opt.zero_grad(set_to_none=True)
158
+ loss.backward()
159
+ torch.nn.utils.clip_grad_norm_(decoder.parameters(), 1.0)
160
+ opt.step()
161
+ sched.step()
162
+
163
+ if step % args.log_every == 0 or step == 1:
164
+ decoder.eval()
165
+ with torch.no_grad():
166
+ val = float(reconstruction_loss(decoder(zv.to(device)), xv.to(device)))
167
+ decoder.train()
168
+ rate = step / (time.perf_counter() - t0)
169
+ train_loss = float(loss.detach())
170
+ history.append({'step': step, 'train': train_loss, 'val': val})
171
+ print(f' step {step:5}/{args.steps} train {train_loss:.5f} '
172
+ f'val {val:.5f} ({val / baseline:.2f}x baseline) '
173
+ f'{rate:.2f} it/s eta {(args.steps - step) / rate / 60:.1f} min',
174
+ flush=True)
175
+
176
+ if step % args.preview_every == 0 or step == args.steps:
177
+ save_preview(decoder, zv, xv, out_dir / f'preview_{step:05d}.png', device)
178
+
179
+ torch.save({
180
+ 'state_dict': decoder.state_dict(),
181
+ 'config': cfg,
182
+ 'preset': args.preset,
183
+ 'meta': meta,
184
+ 'step': args.steps,
185
+ 'history': history,
186
+ 'baseline_val_mse': baseline,
187
+ }, out_dir / 'decoder.pt')
188
+
189
+ print(f'\nsaved -> {out_dir / "decoder.pt"}')
190
+ print(f' {(time.perf_counter() - t0) / 60:.1f} min')
191
+ print(f' final val MSE {history[-1]["val"]:.5f} vs baseline {baseline:.5f}')
192
+ print(f' previews: {out_dir}/preview_*.png (top row target, bottom row decoded)')
193
+
194
+
195
+ if __name__ == '__main__':
196
+ main()
code/scripts/train_density.py ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Fit the conditional behavior-density model beta(b | C) on real transitions.
2
+
3
+ This is a support model for the controller's plans, not a policy. Training it
4
+ separately keeps the controller objective free of behavior cloning: the
5
+ controller is only penalized when a plan leaves the region the dataset covers,
6
+ measured against the 95th percentile of held-out real transitions.
7
+ """
8
+
9
+ import argparse
10
+ import json
11
+ import sys
12
+ from pathlib import Path
13
+
14
+ import numpy as np
15
+ import torch
16
+ from torch.utils.data import DataLoader
17
+
18
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
19
+
20
+ from lejepa_control.data import LatentGoalDataset, split_episodes # noqa: E402
21
+ from lejepa_control.losses import BehaviorDensity # noqa: E402
22
+
23
+
24
+ def main():
25
+ parser = argparse.ArgumentParser()
26
+ parser.add_argument('--latents', default='data/latents')
27
+ parser.add_argument('--out', default='data/runs/density')
28
+ parser.add_argument('--steps', type=int, default=4000)
29
+ parser.add_argument('--batch-size', type=int, default=256)
30
+ parser.add_argument('--lr', type=float, default=1e-3)
31
+ parser.add_argument('--components', type=int, default=16)
32
+ # in-process is fastest here: the latent cache is resident, so workers
33
+ # would each copy ~1 GB on spawn to save no real work
34
+ parser.add_argument('--workers', type=int, default=0)
35
+ args = parser.parse_args()
36
+
37
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
38
+ torch.manual_seed(0)
39
+
40
+ stats = json.loads((Path(args.latents) / 'stats.json').read_text())
41
+ train_eps, val_eps = split_episodes(stats['n_episodes'])
42
+
43
+ train_set = LatentGoalDataset(args.latents, episodes=train_eps)
44
+ val_set = LatentGoalDataset(args.latents, episodes=val_eps)
45
+ print(f'train clips {len(train_set)} val clips {len(val_set)}')
46
+
47
+ loader = DataLoader(
48
+ train_set,
49
+ batch_size=args.batch_size,
50
+ shuffle=True,
51
+ num_workers=args.workers,
52
+ drop_last=True,
53
+ persistent_workers=args.workers > 0,
54
+ )
55
+
56
+ density = BehaviorDensity(
57
+ latent_dim=stats['latent_dim'], components=args.components
58
+ ).to(device)
59
+ opt = torch.optim.AdamW(density.parameters(), lr=args.lr, weight_decay=1e-4)
60
+
61
+ step = 0
62
+ density.train()
63
+ while step < args.steps:
64
+ for batch in loader:
65
+ ctx = batch['context'].to(device)
66
+ block = batch['real_action'].to(device)
67
+
68
+ loss = density.nll_per_dim(ctx, block).mean()
69
+ opt.zero_grad(set_to_none=True)
70
+ loss.backward()
71
+ torch.nn.utils.clip_grad_norm_(density.parameters(), 1.0)
72
+ opt.step()
73
+
74
+ step += 1
75
+ if step % 500 == 0:
76
+ print(f'step {step:5d} nll/dim {loss.item():.4f}', flush=True)
77
+ if step >= args.steps:
78
+ break
79
+
80
+ # --- calibrate c_95 on held-out real transitions ----------------------
81
+ density.eval()
82
+ val_loader = DataLoader(
83
+ val_set, batch_size=512, shuffle=True, num_workers=args.workers
84
+ )
85
+ scores = []
86
+ with torch.no_grad():
87
+ for batch in val_loader:
88
+ s = density.nll_per_dim(
89
+ batch['context'].to(device), batch['real_action'].to(device)
90
+ )
91
+ scores.append(s.cpu().numpy())
92
+ if sum(len(x) for x in scores) >= 200_000:
93
+ break
94
+ scores = np.concatenate(scores)
95
+ c95 = float(np.percentile(scores, 95))
96
+
97
+ out_dir = Path(args.out)
98
+ out_dir.mkdir(parents=True, exist_ok=True)
99
+ torch.save(
100
+ {
101
+ 'state_dict': density.state_dict(),
102
+ 'c95': c95,
103
+ 'components': args.components,
104
+ 'latent_dim': stats['latent_dim'],
105
+ },
106
+ out_dir / 'density.pt',
107
+ )
108
+
109
+ print(
110
+ f'held-out nll/dim: mean {scores.mean():.4f} '
111
+ f'p50 {np.percentile(scores, 50):.4f} '
112
+ f'p95 {c95:.4f} p99 {np.percentile(scores, 99):.4f}'
113
+ )
114
+ print(f'saved -> {out_dir / "density.pt"}')
115
+
116
+
117
+ if __name__ == '__main__':
118
+ main()