Hugging Face's logo

mepi31415
/
WorldToken_Checkpoints

Robotics
imitation-learning
robocasa
Model card Files
xet
 Community
mepi31415 commited on
Commit
2ea5c3c
·
verified ·
1 Parent(s): 43e55d7

Publish WorldToken checkpoints: release files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. 04_robocasa_scaling/README.md +26 -0
  2. 04_robocasa_scaling/bc_transformer_d300_seed123/config.json +462 -0
  3. 04_robocasa_scaling/bc_transformer_d300_seed123/run_info.json +11 -0
  4. 04_robocasa_scaling/scaling_n1_d1000_seed0/checkpoint_step_00100000.pt +3 -0
  5. 04_robocasa_scaling/scaling_n1_d1000_seed0/config.json +329 -0
  6. 04_robocasa_scaling/scaling_n1_d1000_seed0/run_info.json +13 -0
  7. 04_robocasa_scaling/scaling_n1_d100_seed0/checkpoint_step_00010000.pt +3 -0
  8. 04_robocasa_scaling/scaling_n1_d100_seed0/config.json +329 -0
  9. 04_robocasa_scaling/scaling_n1_d100_seed0/run_info.json +13 -0
  10. 04_robocasa_scaling/scaling_n1_d2900_seed0/checkpoint_step_00280000.pt +3 -0
  11. 04_robocasa_scaling/scaling_n1_d2900_seed0/config.json +329 -0
  12. 04_robocasa_scaling/scaling_n1_d2900_seed0/run_info.json +13 -0
  13. 04_robocasa_scaling/scaling_n1_d300_seed0/checkpoint_step_00030000.pt +3 -0
  14. 04_robocasa_scaling/scaling_n1_d300_seed0/config.json +331 -0
  15. 04_robocasa_scaling/scaling_n1_d300_seed0/run_info.json +13 -0
  16. 04_robocasa_scaling/scaling_n1_d50_seed0/checkpoint_step_00005000.pt +3 -0
  17. 04_robocasa_scaling/scaling_n1_d50_seed0/config.json +329 -0
  18. 04_robocasa_scaling/scaling_n1_d50_seed0/run_info.json +13 -0
  19. 04_robocasa_scaling/scaling_n2_d1000_seed0/checkpoint_step_00100000.pt +3 -0
  20. 04_robocasa_scaling/scaling_n2_d1000_seed0/config.json +331 -0
  21. 04_robocasa_scaling/scaling_n2_d1000_seed0/run_info.json +13 -0
  22. 04_robocasa_scaling/scaling_n2_d100_seed0/checkpoint_step_00010000.pt +3 -0
  23. 04_robocasa_scaling/scaling_n2_d100_seed0/config.json +331 -0
  24. 04_robocasa_scaling/scaling_n2_d100_seed0/run_info.json +13 -0
  25. 04_robocasa_scaling/scaling_n2_d2900_seed0/checkpoint_step_00280000.pt +3 -0
  26. 04_robocasa_scaling/scaling_n2_d2900_seed0/config.json +331 -0
  27. 04_robocasa_scaling/scaling_n2_d2900_seed0/run_info.json +13 -0
  28. 04_robocasa_scaling/scaling_n2_d300_seed0/checkpoint_step_00030000.pt +3 -0
  29. 04_robocasa_scaling/scaling_n2_d300_seed0/config.json +331 -0
  30. 04_robocasa_scaling/scaling_n2_d300_seed0/run_info.json +13 -0
  31. 04_robocasa_scaling/scaling_n2_d50_seed0/checkpoint_step_00005000.pt +3 -0
  32. 04_robocasa_scaling/scaling_n2_d50_seed0/config.json +331 -0
  33. 04_robocasa_scaling/scaling_n2_d50_seed0/run_info.json +13 -0
  34. 04_robocasa_scaling/scaling_n3_d1000_seed0/config.json +327 -0
  35. 04_robocasa_scaling/scaling_n3_d1000_seed0/run_info.json +13 -0
  36. 04_robocasa_scaling/scaling_n3_d100_seed0/config.json +327 -0
  37. 04_robocasa_scaling/scaling_n3_d100_seed0/run_info.json +13 -0
  38. 04_robocasa_scaling/scaling_n3_d2900_seed0/config.json +327 -0
  39. 04_robocasa_scaling/scaling_n3_d2900_seed0/run_info.json +13 -0
  40. 04_robocasa_scaling/scaling_n3_d300_seed0/config.json +328 -0
  41. 04_robocasa_scaling/scaling_n3_d300_seed0/run_info.json +13 -0
  42. 04_robocasa_scaling/scaling_n3_d50_seed0/config.json +327 -0
  43. 04_robocasa_scaling/scaling_n3_d50_seed0/run_info.json +13 -0
  44. 05_token_interface/README.md +20 -0
  45. 05_token_interface/multitoken_n2_k4_d1000_seed0/config.json +337 -0
  46. 05_token_interface/multitoken_n2_k4_d1000_seed0/run_info.json +13 -0
  47. 05_token_interface/multitoken_n2_k4_d100_seed0/config.json +333 -0
  48. 05_token_interface/multitoken_n2_k4_d100_seed0/run_info.json +13 -0
  49. 05_token_interface/multitoken_n2_k4_d2900_seed0/config.json +333 -0
  50. 05_token_interface/multitoken_n2_k4_d2900_seed0/run_info.json +13 -0
04_robocasa_scaling/README.md ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Section 4 — Multitask control and scaling
2
+
3
+ Paper: **Can WorldToken achieve effective multitask control and scale systematically?**
4
+
5
+ Fifteen final scaling-grid checkpoints (N1–N3, seed 0, K=1, C_train=10) and the D300 / seed-123 BC-Transformer baseline.
6
+
7
+ | Run | Training steps | Checkpoint file |
8
+ |---|---:|---|
9
+ | [scaling_n1_d50_seed0](scaling_n1_d50_seed0/) | 5,000 | `checkpoint_step_00005000.pt` |
10
+ | [scaling_n2_d50_seed0](scaling_n2_d50_seed0/) | 5,000 | `checkpoint_step_00005000.pt` |
11
+ | [scaling_n3_d50_seed0](scaling_n3_d50_seed0/) | 5,000 | `checkpoint_step_00005000.pt` |
12
+ | [scaling_n1_d100_seed0](scaling_n1_d100_seed0/) | 10,000 | `checkpoint_step_00010000.pt` |
13
+ | [scaling_n2_d100_seed0](scaling_n2_d100_seed0/) | 10,000 | `checkpoint_step_00010000.pt` |
14
+ | [scaling_n3_d100_seed0](scaling_n3_d100_seed0/) | 10,000 | `checkpoint_step_00010000.pt` |
15
+ | [bc_transformer_d300_seed123](bc_transformer_d300_seed123/) | 500,000 | `models/model_epoch_1000.pth` |
16
+ | [scaling_n1_d300_seed0](scaling_n1_d300_seed0/) | 30,000 | `checkpoint_step_00030000.pt` |
17
+ | [scaling_n2_d300_seed0](scaling_n2_d300_seed0/) | 30,000 | `checkpoint_step_00030000.pt` |
18
+ | [scaling_n3_d300_seed0](scaling_n3_d300_seed0/) | 30,000 | `checkpoint_step_00030000.pt` |
19
+ | [scaling_n1_d1000_seed0](scaling_n1_d1000_seed0/) | 100,000 | `checkpoint_step_00100000.pt` |
20
+ | [scaling_n2_d1000_seed0](scaling_n2_d1000_seed0/) | 100,000 | `checkpoint_step_00100000.pt` |
21
+ | [scaling_n3_d1000_seed0](scaling_n3_d1000_seed0/) | 100,000 | `checkpoint_step_00100000.pt` |
22
+ | [scaling_n1_d2900_seed0](scaling_n1_d2900_seed0/) | 280,000 | `checkpoint_step_00280000.pt` |
23
+ | [scaling_n2_d2900_seed0](scaling_n2_d2900_seed0/) | 280,000 | `checkpoint_step_00280000.pt` |
24
+ | [scaling_n3_d2900_seed0](scaling_n3_d2900_seed0/) | 280,000 | `checkpoint_step_00280000.pt` |
25
+
26
+ BC-Transformer ends at epoch 1,000, corresponding to 500,000 training steps.
04_robocasa_scaling/bc_transformer_d300_seed123/config.json ADDED
@@ -0,0 +1,462 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "algo_name": "bc",
3
+ "experiment": {
4
+ "name": "e0_bc_xfmr_mg23_d300_seed123_official500k_cuda4",
5
+ "validate": false,
6
+ "logging": {
7
+ "terminal_output_to_txt": true,
8
+ "log_tb": true,
9
+ "log_wandb": false,
10
+ "wandb_proj_name": "robocasa_official_baselines_mg23_d300"
11
+ },
12
+ "mse": {
13
+ "enabled": false,
14
+ "every_n_epochs": 50,
15
+ "on_save_ckpt": true,
16
+ "num_samples": 20,
17
+ "visualize": true
18
+ },
19
+ "save": {
20
+ "enabled": true,
21
+ "every_n_seconds": null,
22
+ "every_n_epochs": 100,
23
+ "epochs": [],
24
+ "on_best_validation": false,
25
+ "on_best_rollout_return": false,
26
+ "on_best_rollout_success_rate": false
27
+ },
28
+ "epoch_every_n_steps": 500,
29
+ "validation_epoch_every_n_steps": 10,
30
+ "env": null,
31
+ "additional_envs": null,
32
+ "render": false,
33
+ "render_video": false,
34
+ "keep_all_videos": false,
35
+ "video_skip": 5,
36
+ "rollout": {
37
+ "enabled": false,
38
+ "n": 50,
39
+ "horizon": 500,
40
+ "rate": 100,
41
+ "warmstart": 0,
42
+ "terminate_on_success": true,
43
+ "batched": false,
44
+ "num_batch_envs": 5
45
+ },
46
+ "env_meta_update_dict": {
47
+ "env_kwargs": {
48
+ "generative_textures": null,
49
+ "scene_split": null,
50
+ "style_ids": null,
51
+ "layout_ids": null,
52
+ "layout_and_style_ids": [
53
+ [
54
+ 1,
55
+ 1
56
+ ],
57
+ [
58
+ 2,
59
+ 2
60
+ ],
61
+ [
62
+ 4,
63
+ 4
64
+ ],
65
+ [
66
+ 6,
67
+ 9
68
+ ],
69
+ [
70
+ 7,
71
+ 10
72
+ ]
73
+ ],
74
+ "randomize_cameras": false,
75
+ "obj_instance_split": "B"
76
+ }
77
+ },
78
+ "ckpt_path": null
79
+ },
80
+ "train": {
81
+ "data": [
82
+ {
83
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
84
+ "horizon": 700,
85
+ "do_eval": false,
86
+ "filter_key": "300_demos"
87
+ },
88
+ {
89
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
90
+ "horizon": 500,
91
+ "do_eval": false,
92
+ "filter_key": "300_demos"
93
+ },
94
+ {
95
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
96
+ "horizon": 500,
97
+ "do_eval": false,
98
+ "filter_key": "300_demos"
99
+ },
100
+ {
101
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
102
+ "horizon": 300,
103
+ "do_eval": false,
104
+ "filter_key": "300_demos"
105
+ },
106
+ {
107
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
108
+ "horizon": 600,
109
+ "do_eval": false,
110
+ "filter_key": "300_demos"
111
+ },
112
+ {
113
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
114
+ "horizon": 600,
115
+ "do_eval": false,
116
+ "filter_key": "300_demos"
117
+ },
118
+ {
119
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
120
+ "horizon": 500,
121
+ "do_eval": false,
122
+ "filter_key": "300_demos"
123
+ },
124
+ {
125
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
126
+ "horizon": 500,
127
+ "do_eval": false,
128
+ "filter_key": "300_demos"
129
+ },
130
+ {
131
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
132
+ "horizon": 500,
133
+ "do_eval": false,
134
+ "filter_key": "300_demos"
135
+ },
136
+ {
137
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
138
+ "horizon": 500,
139
+ "do_eval": true,
140
+ "filter_key": "300_demos"
141
+ },
142
+ {
143
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
144
+ "horizon": 600,
145
+ "do_eval": false,
146
+ "filter_key": "300_demos"
147
+ },
148
+ {
149
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
150
+ "horizon": 700,
151
+ "do_eval": true,
152
+ "filter_key": "300_demos"
153
+ },
154
+ {
155
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
156
+ "horizon": 500,
157
+ "do_eval": false,
158
+ "filter_key": "300_demos"
159
+ },
160
+ {
161
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
162
+ "horizon": 500,
163
+ "do_eval": false,
164
+ "filter_key": "300_demos"
165
+ },
166
+ {
167
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
168
+ "horizon": 500,
169
+ "do_eval": false,
170
+ "filter_key": "300_demos"
171
+ },
172
+ {
173
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
174
+ "horizon": 500,
175
+ "do_eval": false,
176
+ "filter_key": "300_demos"
177
+ },
178
+ {
179
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
180
+ "horizon": 500,
181
+ "do_eval": false,
182
+ "filter_key": "300_demos"
183
+ },
184
+ {
185
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
186
+ "horizon": 500,
187
+ "do_eval": false,
188
+ "filter_key": "300_demos"
189
+ },
190
+ {
191
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
192
+ "horizon": 500,
193
+ "do_eval": false,
194
+ "filter_key": "300_demos"
195
+ },
196
+ {
197
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
198
+ "horizon": 500,
199
+ "do_eval": false,
200
+ "filter_key": "300_demos"
201
+ },
202
+ {
203
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
204
+ "horizon": 500,
205
+ "do_eval": false,
206
+ "filter_key": "300_demos"
207
+ },
208
+ {
209
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
210
+ "horizon": 500,
211
+ "do_eval": false,
212
+ "filter_key": "300_demos"
213
+ },
214
+ {
215
+ "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
216
+ "horizon": 500,
217
+ "do_eval": false,
218
+ "filter_key": "300_demos"
219
+ }
220
+ ],
221
+ "output_dir": "${EXPERIMENT_ROOT}/E0/runs",
222
+ "num_data_workers": 5,
223
+ "hdf5_cache_mode": null,
224
+ "hdf5_use_swmr": true,
225
+ "hdf5_load_next_obs": false,
226
+ "hdf5_normalize_obs": false,
227
+ "hdf5_filter_key": null,
228
+ "hdf5_validation_filter_key": null,
229
+ "seq_length": 10,
230
+ "pad_seq_length": true,
231
+ "frame_stack": 10,
232
+ "pad_frame_stack": true,
233
+ "dataset_keys": [
234
+ "actions"
235
+ ],
236
+ "action_keys": [
237
+ "actions"
238
+ ],
239
+ "action_config": {
240
+ "actions": {
241
+ "normalization": null
242
+ },
243
+ "actions_abs": {
244
+ "normalization": "min_max"
245
+ },
246
+ "action_dict/abs_pos": {
247
+ "normalization": "min_max"
248
+ },
249
+ "action_dict/abs_rot_axis_angle": {
250
+ "normalization": "min_max",
251
+ "format": "rot_axis_angle"
252
+ },
253
+ "action_dict/abs_rot_6d": {
254
+ "normalization": null,
255
+ "format": "rot_6d"
256
+ },
257
+ "action_dict/rel_pos": {
258
+ "normalization": null
259
+ },
260
+ "action_dict/rel_rot_axis_angle": {
261
+ "normalization": null,
262
+ "format": "rot_axis_angle"
263
+ },
264
+ "action_dict/rel_rot_6d": {
265
+ "normalization": null,
266
+ "format": "rot_6d"
267
+ },
268
+ "action_dict/gripper": {
269
+ "normalization": null
270
+ },
271
+ "action_dict/base_mode": {
272
+ "normalization": null
273
+ }
274
+ },
275
+ "goal_mode": null,
276
+ "cuda": true,
277
+ "batch_size": 16,
278
+ "num_epochs": 1000,
279
+ "seed": 123,
280
+ "max_grad_norm": 100.0,
281
+ "data_format": "robomimic",
282
+ "shuffled_obs_key_groups": null
283
+ },
284
+ "algo": {
285
+ "optim_params": {
286
+ "policy": {
287
+ "optimizer_type": "adamw",
288
+ "learning_rate": {
289
+ "initial": 0.0001,
290
+ "decay_factor": 1.0,
291
+ "epoch_schedule": [
292
+ 100
293
+ ],
294
+ "scheduler_type": "constant_with_warmup"
295
+ },
296
+ "regularization": {
297
+ "L2": 0.01
298
+ }
299
+ }
300
+ },
301
+ "loss": {
302
+ "l2_weight": 1.0,
303
+ "l1_weight": 0.0,
304
+ "cos_weight": 0.0
305
+ },
306
+ "actor_layer_dims": [],
307
+ "gaussian": {
308
+ "enabled": false,
309
+ "fixed_std": false,
310
+ "init_std": 0.1,
311
+ "min_std": 0.01,
312
+ "std_activation": "softplus",
313
+ "low_noise_eval": true
314
+ },
315
+ "gmm": {
316
+ "enabled": true,
317
+ "num_modes": 5,
318
+ "min_std": 0.005,
319
+ "std_activation": "softplus",
320
+ "low_noise_eval": true
321
+ },
322
+ "vae": {
323
+ "enabled": false,
324
+ "latent_dim": 14,
325
+ "latent_clip": null,
326
+ "kl_weight": 1.0,
327
+ "decoder": {
328
+ "is_conditioned": true,
329
+ "reconstruction_sum_across_elements": false
330
+ },
331
+ "prior": {
332
+ "learn": false,
333
+ "is_conditioned": false,
334
+ "use_gmm": false,
335
+ "gmm_num_modes": 10,
336
+ "gmm_learn_weights": false,
337
+ "use_categorical": false,
338
+ "categorical_dim": 10,
339
+ "categorical_gumbel_softmax_hard": false,
340
+ "categorical_init_temp": 1.0,
341
+ "categorical_temp_anneal_step": 0.001,
342
+ "categorical_min_temp": 0.3
343
+ },
344
+ "encoder_layer_dims": [
345
+ 300,
346
+ 400
347
+ ],
348
+ "decoder_layer_dims": [
349
+ 300,
350
+ 400
351
+ ],
352
+ "prior_layer_dims": [
353
+ 300,
354
+ 400
355
+ ]
356
+ },
357
+ "rnn": {
358
+ "enabled": false,
359
+ "horizon": 10,
360
+ "hidden_dim": 400,
361
+ "rnn_type": "LSTM",
362
+ "num_layers": 2,
363
+ "open_loop": false,
364
+ "kwargs": {
365
+ "bidirectional": false
366
+ }
367
+ },
368
+ "transformer": {
369
+ "enabled": true,
370
+ "context_length": 10,
371
+ "embed_dim": 512,
372
+ "num_layers": 6,
373
+ "num_heads": 8,
374
+ "emb_dropout": 0.1,
375
+ "attn_dropout": 0.1,
376
+ "block_output_dropout": 0.1,
377
+ "sinusoidal_embedding": false,
378
+ "activation": "gelu",
379
+ "supervise_all_steps": true,
380
+ "nn_parameter_for_timesteps": true,
381
+ "pred_future_acs": true,
382
+ "causal": false
383
+ },
384
+ "language_conditioned": true
385
+ },
386
+ "observation": {
387
+ "modalities": {
388
+ "obs": {
389
+ "low_dim": [
390
+ "robot0_base_to_eef_pos",
391
+ "robot0_base_to_eef_quat",
392
+ "robot0_base_pos",
393
+ "robot0_base_quat",
394
+ "robot0_gripper_qpos"
395
+ ],
396
+ "rgb": [
397
+ "robot0_agentview_left_image",
398
+ "robot0_agentview_right_image",
399
+ "robot0_eye_in_hand_image"
400
+ ],
401
+ "depth": [],
402
+ "scan": []
403
+ },
404
+ "goal": {
405
+ "low_dim": [],
406
+ "rgb": [],
407
+ "depth": [],
408
+ "scan": []
409
+ }
410
+ },
411
+ "encoder": {
412
+ "low_dim": {
413
+ "core_class": null,
414
+ "core_kwargs": {},
415
+ "obs_randomizer_class": null,
416
+ "obs_randomizer_kwargs": {}
417
+ },
418
+ "rgb": {
419
+ "core_class": "VisualCoreLanguageConditioned",
420
+ "core_kwargs": {
421
+ "feature_dimension": 64,
422
+ "backbone_class": "ResNet18ConvFiLM",
423
+ "backbone_kwargs": {
424
+ "pretrained": false,
425
+ "input_coord_conv": false
426
+ },
427
+ "pool_class": "SpatialSoftmax",
428
+ "pool_kwargs": {
429
+ "num_kp": 32,
430
+ "learnable_temperature": false,
431
+ "temperature": 1.0,
432
+ "noise_std": 0.0
433
+ }
434
+ },
435
+ "obs_randomizer_class": "CropRandomizer",
436
+ "obs_randomizer_kwargs": {
437
+ "crop_height": 116,
438
+ "crop_width": 116,
439
+ "num_crops": 1,
440
+ "pos_enc": false
441
+ }
442
+ },
443
+ "depth": {
444
+ "core_class": "VisualCore",
445
+ "core_kwargs": {},
446
+ "obs_randomizer_class": null,
447
+ "obs_randomizer_kwargs": {}
448
+ },
449
+ "scan": {
450
+ "core_class": "ScanCore",
451
+ "core_kwargs": {},
452
+ "obs_randomizer_class": null,
453
+ "obs_randomizer_kwargs": {}
454
+ }
455
+ }
456
+ },
457
+ "meta": {
458
+ "hp_base_config_file": null,
459
+ "hp_keys": [],
460
+ "hp_values": []
461
+ }
462
+ }
04_robocasa_scaling/bc_transformer_d300_seed123/run_info.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "bc_transformer_d300_seed123",
3
+ "original_run_name": "e0_bc_xfmr_mg23_d300_seed123_official500k_cuda4",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 300,
7
+ "C_train": 10,
8
+ "seed": 123
9
+ },
10
+ "final_training_step": 500000
11
+ }
04_robocasa_scaling/scaling_n1_d1000_seed0/checkpoint_step_00100000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:adf58c8a3cf22a7733968c3c2d42e407fad079d4d7b747d186ae80098e742693
3
+ size 531280901
04_robocasa_scaling/scaling_n1_d1000_seed0/config.json ADDED
@@ -0,0 +1,329 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "d_model": 128,
7
+ "denoising_steps": 20,
8
+ "dropout": 0.0,
9
+ "h_adaln_bottleneck": false,
10
+ "n_heads": 4,
11
+ "n_layers": 1,
12
+ "use_h_cross_attn": false,
13
+ "use_obs_cross_attn": false
14
+ },
15
+ "type": "diffusion_dit"
16
+ },
17
+ "action_head_lr": 0.0003,
18
+ "action_spec": {
19
+ "dim": 12,
20
+ "discrete_dims": [
21
+ 6,
22
+ 11
23
+ ]
24
+ },
25
+ "action_weight": 1.0,
26
+ "adamw_beta1": 0.9,
27
+ "adamw_beta2": 0.95,
28
+ "adamw_eps": 1e-08,
29
+ "allow_holdout_overlap": false,
30
+ "attn_backend": "auto",
31
+ "attn_impl": "eager",
32
+ "backbone_lr": null,
33
+ "backbone_type": "qwen2",
34
+ "batch_size": 48,
35
+ "cnn_depth": 48,
36
+ "cnn_kernel": 5,
37
+ "cnn_mults": [
38
+ 2,
39
+ 3,
40
+ 4,
41
+ 4
42
+ ],
43
+ "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d1000_n1_world2/phase1grid_d1000_n1_nodyn_seed0_100k.yaml",
44
+ "crops_per_demo": 8,
45
+ "d_model": 2048,
46
+ "dataset": [
47
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
48
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
70
+ ],
71
+ "ddp_timeout_minutes": 180.0,
72
+ "denoising_steps": 20,
73
+ "device": "cuda",
74
+ "diffusion_mlp_dims": [
75
+ 1024,
76
+ 1024,
77
+ 1024
78
+ ],
79
+ "diffusion_residual_style": true,
80
+ "diffusion_time_dim": 16,
81
+ "dropout": 0.1,
82
+ "dry_run_data": false,
83
+ "dynamics": {
84
+ "enabled": false,
85
+ "params": {},
86
+ "type": "none"
87
+ },
88
+ "dynamics_decoder_type": "film",
89
+ "enable_pred_next": false,
90
+ "encoder": {
91
+ "params": {
92
+ "cnn_depth": 48,
93
+ "cnn_kernel": 5,
94
+ "cnn_mults": [
95
+ 2,
96
+ 3,
97
+ 4,
98
+ 8,
99
+ 16
100
+ ],
101
+ "d_model": 512,
102
+ "dropout": 0.0,
103
+ "mlp_ratio": 4,
104
+ "n_fusion_layers": 1,
105
+ "n_heads": 4,
106
+ "readout_depth": 1,
107
+ "readout_mlp_ratio": 2,
108
+ "readout_queries": 4,
109
+ "rope_theta": 10000.0,
110
+ "use_proprio": true
111
+ },
112
+ "type": "attn_fusion_latent_token"
113
+ },
114
+ "encoder_lr": 0.0006,
115
+ "eval_at_start": false,
116
+ "eval_batch_size": 64,
117
+ "eval_crops_per_demo": 8,
118
+ "eval_every": 5000,
119
+ "eval_num_workers": 24,
120
+ "eval_seed": 0,
121
+ "ffn_hidden_size": 1024,
122
+ "filter_key": "wl23_mg1000_seed0",
123
+ "grad_accum_steps": 2,
124
+ "grad_clip": 1.0,
125
+ "hdf5_paths": [
126
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
127
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
149
+ ],
150
+ "holdout_action_mean_samples": 1,
151
+ "holdout_crops_per_demo": 8,
152
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
153
+ "holdout_per_task_metrics": true,
154
+ "holdout_rmse_samplers": [
155
+ "deterministic",
156
+ "stochastic"
157
+ ],
158
+ "holdout_sample_deterministic": null,
159
+ "holdout_sample_prefix_horizon": 4,
160
+ "holdout_sample_rmse": true,
161
+ "holdout_size": 2300,
162
+ "image_emb_dim": 512,
163
+ "init_device": "meta",
164
+ "input_norm": true,
165
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
166
+ "lang_emb_mode": "clip",
167
+ "lang_encoder_device": "auto",
168
+ "lang_obs_emb_dim": 256,
169
+ "log_every": 25,
170
+ "lora": false,
171
+ "lora_alpha": 32,
172
+ "lora_dropout": 0.0,
173
+ "lora_rank": 16,
174
+ "lora_target": [
175
+ "q_proj",
176
+ "k_proj",
177
+ "v_proj",
178
+ "o_proj",
179
+ "gate_proj",
180
+ "up_proj",
181
+ "down_proj"
182
+ ],
183
+ "lr": 0.0003,
184
+ "max_context_len": 1024,
185
+ "max_steps": 100000,
186
+ "min_lr_ratio": 0.1,
187
+ "model": {
188
+ "action_chunk_len": 10,
189
+ "latent_dim": 512
190
+ },
191
+ "model_dtype": "float32",
192
+ "n_heads": 16,
193
+ "n_kv_heads": null,
194
+ "n_layers": 2,
195
+ "norm_bias_weight_decay": null,
196
+ "norm_fit_batches": 0,
197
+ "num_workers": 24,
198
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
199
+ "obs_spec": {
200
+ "image_channels": 3,
201
+ "image_hw": [
202
+ 128,
203
+ 128
204
+ ],
205
+ "image_keys": [
206
+ "robot0_agentview_left_image",
207
+ "robot0_agentview_right_image",
208
+ "robot0_eye_in_hand_image"
209
+ ],
210
+ "lang_dim": 768,
211
+ "layout": "NHWC",
212
+ "low_dim_dims": [
213
+ 3,
214
+ 4,
215
+ 3,
216
+ 4,
217
+ 2
218
+ ],
219
+ "low_dim_keys": [
220
+ "robot0_base_to_eef_pos",
221
+ "robot0_base_to_eef_quat",
222
+ "robot0_base_pos",
223
+ "robot0_base_quat",
224
+ "robot0_gripper_qpos"
225
+ ]
226
+ },
227
+ "obs_stride": 4,
228
+ "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n1_d1000_seed0",
229
+ "persistent_workers": true,
230
+ "precision": "bf16",
231
+ "pred_next_image_key_weights": {},
232
+ "pred_next_image_weight": 1.0,
233
+ "pred_next_lang_weight": 0.1,
234
+ "pred_next_mode": "terminal",
235
+ "pred_next_obs_offset": 1,
236
+ "pred_next_proprio_weight": 0.15,
237
+ "pred_next_steps": 4,
238
+ "pred_warmup_steps": 0,
239
+ "predictor_lr": 0.0006,
240
+ "prefetch_factor": 4,
241
+ "prepare_lang_cache_only": false,
242
+ "proprio_emb_dim": 128,
243
+ "recon": {
244
+ "enabled": false,
245
+ "params": {
246
+ "depth": 48,
247
+ "kernel_size": 5,
248
+ "mults": [
249
+ 2,
250
+ 3,
251
+ 4,
252
+ 4
253
+ ]
254
+ }
255
+ },
256
+ "recon_image_weight": 1.0,
257
+ "recon_lang_weight": 0.1,
258
+ "recon_obs": false,
259
+ "recon_proprio_weight": 1.0,
260
+ "resume": null,
261
+ "resume_final_lr": null,
262
+ "resume_lr_schedule": "checkpoint",
263
+ "robomimic_src": null,
264
+ "save_every": 10000,
265
+ "save_holdout_traces": true,
266
+ "save_latest_checkpoint": true,
267
+ "scaling_action_head_d_model": 128,
268
+ "scaling_action_head_layers": 1,
269
+ "scaling_action_head_lr": 0.0003,
270
+ "scaling_backbone_d_model": 512,
271
+ "scaling_backbone_layers": 2,
272
+ "scaling_batch_policy": "phase1_grid_h100_v20260711",
273
+ "scaling_batch_policy_d50_microbatch": 48,
274
+ "scaling_encoder_d_model": 512,
275
+ "scaling_encoder_layers": 1,
276
+ "scaling_encoder_lr": 0.0006,
277
+ "scaling_expected_world_size": 2,
278
+ "scaling_global_batch_size": 192,
279
+ "scaling_group_b_lr_role": null,
280
+ "scaling_latent_dim": 512,
281
+ "scaling_lr_action_head": 0.0003,
282
+ "scaling_lr_base": null,
283
+ "scaling_lr_base_frozen": false,
284
+ "scaling_lr_encoder": 0.0006,
285
+ "scaling_lr_formula_size_axis": "discrete_family_lookup",
286
+ "scaling_lr_lookup_calibration": false,
287
+ "scaling_lr_lookup_model": "n1",
288
+ "scaling_lr_lookup_status": "confirmed",
289
+ "scaling_lr_predictor": 0.0006,
290
+ "scaling_lr_reference": null,
291
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
292
+ "scaling_lr_table_version": "n_lookup_v20260711_n5",
293
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
294
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
295
+ "scaling_per_rank_microbatch": 48,
296
+ "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0",
297
+ "scaling_phase1_topology": "phase1_5x5_seed0_v20260711",
298
+ "scaling_predictor_lr": 0.0006,
299
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
300
+ "scaling_recipe": "nodyn",
301
+ "scaling_run_name": "phase1grid_d1000_n1_nodyn_seed0_100k",
302
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04",
303
+ "scaling_suite": "phase1_grid",
304
+ "seed": 0,
305
+ "seq_len": 10,
306
+ "sequence_model": {
307
+ "backbone_type": "qwen2",
308
+ "hidden_dim": 512,
309
+ "params": {
310
+ "attn_impl": "sdpa",
311
+ "dropout": 0.1,
312
+ "ffn_hidden_size": 1408,
313
+ "input_norm": false,
314
+ "max_context_len": 1024,
315
+ "model_dtype": "float32",
316
+ "n_heads": 4,
317
+ "n_kv_heads": null,
318
+ "n_layers": 2
319
+ },
320
+ "type": "continuous_transformer"
321
+ },
322
+ "use_proprio": true,
323
+ "warmup_steps": 5000,
324
+ "weight_decay": 0.0,
325
+ "z_bottleneck": {
326
+ "dim": null
327
+ },
328
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz"
329
+ }
04_robocasa_scaling/scaling_n1_d1000_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n1_d1000_seed0",
3
+ "original_run_name": "phase1grid_d1000_n1_nodyn_seed0_100k_cuda4-5",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 1000,
7
+ "N": 1,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 100000
13
+ }
04_robocasa_scaling/scaling_n1_d100_seed0/checkpoint_step_00010000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:22c14fc8c7dac158b75f7d3c2c8b3f545461f4486b6979a6c2644abdb79a7b14
3
+ size 531280901
04_robocasa_scaling/scaling_n1_d100_seed0/config.json ADDED
@@ -0,0 +1,329 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "d_model": 128,
7
+ "denoising_steps": 20,
8
+ "dropout": 0.0,
9
+ "h_adaln_bottleneck": false,
10
+ "n_heads": 4,
11
+ "n_layers": 1,
12
+ "use_h_cross_attn": false,
13
+ "use_obs_cross_attn": false
14
+ },
15
+ "type": "diffusion_dit"
16
+ },
17
+ "action_head_lr": 0.0003,
18
+ "action_spec": {
19
+ "dim": 12,
20
+ "discrete_dims": [
21
+ 6,
22
+ 11
23
+ ]
24
+ },
25
+ "action_weight": 1.0,
26
+ "adamw_beta1": 0.9,
27
+ "adamw_beta2": 0.95,
28
+ "adamw_eps": 1e-08,
29
+ "allow_holdout_overlap": false,
30
+ "attn_backend": "auto",
31
+ "attn_impl": "eager",
32
+ "backbone_lr": null,
33
+ "backbone_type": "qwen2",
34
+ "batch_size": 64,
35
+ "cnn_depth": 48,
36
+ "cnn_kernel": 5,
37
+ "cnn_mults": [
38
+ 2,
39
+ 3,
40
+ 4,
41
+ 4
42
+ ],
43
+ "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d100_n1_world1/phase1grid_d100_n1_nodyn_seed0_10k.yaml",
44
+ "crops_per_demo": 8,
45
+ "d_model": 2048,
46
+ "dataset": [
47
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
48
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
70
+ ],
71
+ "ddp_timeout_minutes": 180.0,
72
+ "denoising_steps": 20,
73
+ "device": "cuda",
74
+ "diffusion_mlp_dims": [
75
+ 1024,
76
+ 1024,
77
+ 1024
78
+ ],
79
+ "diffusion_residual_style": true,
80
+ "diffusion_time_dim": 16,
81
+ "dropout": 0.1,
82
+ "dry_run_data": false,
83
+ "dynamics": {
84
+ "enabled": false,
85
+ "params": {},
86
+ "type": "none"
87
+ },
88
+ "dynamics_decoder_type": "film",
89
+ "enable_pred_next": false,
90
+ "encoder": {
91
+ "params": {
92
+ "cnn_depth": 48,
93
+ "cnn_kernel": 5,
94
+ "cnn_mults": [
95
+ 2,
96
+ 3,
97
+ 4,
98
+ 8,
99
+ 16
100
+ ],
101
+ "d_model": 512,
102
+ "dropout": 0.0,
103
+ "mlp_ratio": 4,
104
+ "n_fusion_layers": 1,
105
+ "n_heads": 4,
106
+ "readout_depth": 1,
107
+ "readout_mlp_ratio": 2,
108
+ "readout_queries": 4,
109
+ "rope_theta": 10000.0,
110
+ "use_proprio": true
111
+ },
112
+ "type": "attn_fusion_latent_token"
113
+ },
114
+ "encoder_lr": 0.0006,
115
+ "eval_at_start": false,
116
+ "eval_batch_size": 64,
117
+ "eval_crops_per_demo": 8,
118
+ "eval_every": 500,
119
+ "eval_num_workers": 24,
120
+ "eval_seed": 0,
121
+ "ffn_hidden_size": 1024,
122
+ "filter_key": "wl23_mg100_seed0",
123
+ "grad_accum_steps": 3,
124
+ "grad_clip": 1.0,
125
+ "hdf5_paths": [
126
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
127
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
149
+ ],
150
+ "holdout_action_mean_samples": 1,
151
+ "holdout_crops_per_demo": 8,
152
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
153
+ "holdout_per_task_metrics": true,
154
+ "holdout_rmse_samplers": [
155
+ "deterministic",
156
+ "stochastic"
157
+ ],
158
+ "holdout_sample_deterministic": null,
159
+ "holdout_sample_prefix_horizon": 4,
160
+ "holdout_sample_rmse": true,
161
+ "holdout_size": 2300,
162
+ "image_emb_dim": 512,
163
+ "init_device": "meta",
164
+ "input_norm": true,
165
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
166
+ "lang_emb_mode": "clip",
167
+ "lang_encoder_device": "auto",
168
+ "lang_obs_emb_dim": 256,
169
+ "log_every": 25,
170
+ "lora": false,
171
+ "lora_alpha": 32,
172
+ "lora_dropout": 0.0,
173
+ "lora_rank": 16,
174
+ "lora_target": [
175
+ "q_proj",
176
+ "k_proj",
177
+ "v_proj",
178
+ "o_proj",
179
+ "gate_proj",
180
+ "up_proj",
181
+ "down_proj"
182
+ ],
183
+ "lr": 0.0003,
184
+ "max_context_len": 1024,
185
+ "max_steps": 10000,
186
+ "min_lr_ratio": 0.1,
187
+ "model": {
188
+ "action_chunk_len": 10,
189
+ "latent_dim": 512
190
+ },
191
+ "model_dtype": "float32",
192
+ "n_heads": 16,
193
+ "n_kv_heads": null,
194
+ "n_layers": 2,
195
+ "norm_bias_weight_decay": null,
196
+ "norm_fit_batches": 0,
197
+ "num_workers": 24,
198
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
199
+ "obs_spec": {
200
+ "image_channels": 3,
201
+ "image_hw": [
202
+ 128,
203
+ 128
204
+ ],
205
+ "image_keys": [
206
+ "robot0_agentview_left_image",
207
+ "robot0_agentview_right_image",
208
+ "robot0_eye_in_hand_image"
209
+ ],
210
+ "lang_dim": 768,
211
+ "layout": "NHWC",
212
+ "low_dim_dims": [
213
+ 3,
214
+ 4,
215
+ 3,
216
+ 4,
217
+ 2
218
+ ],
219
+ "low_dim_keys": [
220
+ "robot0_base_to_eef_pos",
221
+ "robot0_base_to_eef_quat",
222
+ "robot0_base_pos",
223
+ "robot0_base_quat",
224
+ "robot0_gripper_qpos"
225
+ ]
226
+ },
227
+ "obs_stride": 4,
228
+ "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n1_d100_seed0",
229
+ "persistent_workers": true,
230
+ "precision": "bf16",
231
+ "pred_next_image_key_weights": {},
232
+ "pred_next_image_weight": 1.0,
233
+ "pred_next_lang_weight": 0.1,
234
+ "pred_next_mode": "terminal",
235
+ "pred_next_obs_offset": 1,
236
+ "pred_next_proprio_weight": 0.15,
237
+ "pred_next_steps": 4,
238
+ "pred_warmup_steps": 0,
239
+ "predictor_lr": 0.0006,
240
+ "prefetch_factor": 4,
241
+ "prepare_lang_cache_only": false,
242
+ "proprio_emb_dim": 128,
243
+ "recon": {
244
+ "enabled": false,
245
+ "params": {
246
+ "depth": 48,
247
+ "kernel_size": 5,
248
+ "mults": [
249
+ 2,
250
+ 3,
251
+ 4,
252
+ 4
253
+ ]
254
+ }
255
+ },
256
+ "recon_image_weight": 1.0,
257
+ "recon_lang_weight": 0.1,
258
+ "recon_obs": false,
259
+ "recon_proprio_weight": 1.0,
260
+ "resume": null,
261
+ "resume_final_lr": null,
262
+ "resume_lr_schedule": "checkpoint",
263
+ "robomimic_src": null,
264
+ "save_every": 1000,
265
+ "save_holdout_traces": true,
266
+ "save_latest_checkpoint": true,
267
+ "scaling_action_head_d_model": 128,
268
+ "scaling_action_head_layers": 1,
269
+ "scaling_action_head_lr": 0.0003,
270
+ "scaling_backbone_d_model": 512,
271
+ "scaling_backbone_layers": 2,
272
+ "scaling_batch_policy": "phase1_grid_h100_v20260711",
273
+ "scaling_batch_policy_d50_microbatch": 48,
274
+ "scaling_encoder_d_model": 512,
275
+ "scaling_encoder_layers": 1,
276
+ "scaling_encoder_lr": 0.0006,
277
+ "scaling_expected_world_size": 1,
278
+ "scaling_global_batch_size": 192,
279
+ "scaling_group_b_lr_role": null,
280
+ "scaling_latent_dim": 512,
281
+ "scaling_lr_action_head": 0.0003,
282
+ "scaling_lr_base": null,
283
+ "scaling_lr_base_frozen": false,
284
+ "scaling_lr_encoder": 0.0006,
285
+ "scaling_lr_formula_size_axis": "discrete_family_lookup",
286
+ "scaling_lr_lookup_calibration": false,
287
+ "scaling_lr_lookup_model": "n1",
288
+ "scaling_lr_lookup_status": "confirmed",
289
+ "scaling_lr_predictor": 0.0006,
290
+ "scaling_lr_reference": null,
291
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
292
+ "scaling_lr_table_version": "n_lookup_v20260711_n5",
293
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
294
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
295
+ "scaling_per_rank_microbatch": 64,
296
+ "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0",
297
+ "scaling_phase1_topology": "phase1_5x5_seed0_v20260711",
298
+ "scaling_predictor_lr": 0.0006,
299
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
300
+ "scaling_recipe": "nodyn",
301
+ "scaling_run_name": "phase1grid_d100_n1_nodyn_seed0_10k",
302
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04",
303
+ "scaling_suite": "phase1_grid",
304
+ "seed": 0,
305
+ "seq_len": 10,
306
+ "sequence_model": {
307
+ "backbone_type": "qwen2",
308
+ "hidden_dim": 512,
309
+ "params": {
310
+ "attn_impl": "sdpa",
311
+ "dropout": 0.1,
312
+ "ffn_hidden_size": 1408,
313
+ "input_norm": false,
314
+ "max_context_len": 1024,
315
+ "model_dtype": "float32",
316
+ "n_heads": 4,
317
+ "n_kv_heads": null,
318
+ "n_layers": 2
319
+ },
320
+ "type": "continuous_transformer"
321
+ },
322
+ "use_proprio": true,
323
+ "warmup_steps": 500,
324
+ "weight_decay": 0.0,
325
+ "z_bottleneck": {
326
+ "dim": null
327
+ },
328
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz"
329
+ }
04_robocasa_scaling/scaling_n1_d100_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n1_d100_seed0",
3
+ "original_run_name": "phase1grid_d100_n1_nodyn_seed0_10k_cuda6",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 100,
7
+ "N": 1,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 10000
13
+ }
04_robocasa_scaling/scaling_n1_d2900_seed0/checkpoint_step_00280000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8e8d7446b9b077368928f26d82dbe3b41da3b878ce97f17d6ce819380eff432
3
+ size 531280901
04_robocasa_scaling/scaling_n1_d2900_seed0/config.json ADDED
@@ -0,0 +1,329 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "d_model": 128,
7
+ "denoising_steps": 20,
8
+ "dropout": 0.0,
9
+ "h_adaln_bottleneck": false,
10
+ "n_heads": 4,
11
+ "n_layers": 1,
12
+ "use_h_cross_attn": false,
13
+ "use_obs_cross_attn": false
14
+ },
15
+ "type": "diffusion_dit"
16
+ },
17
+ "action_head_lr": 0.0003,
18
+ "action_spec": {
19
+ "dim": 12,
20
+ "discrete_dims": [
21
+ 6,
22
+ 11
23
+ ]
24
+ },
25
+ "action_weight": 1.0,
26
+ "adamw_beta1": 0.9,
27
+ "adamw_beta2": 0.95,
28
+ "adamw_eps": 1e-08,
29
+ "allow_holdout_overlap": false,
30
+ "attn_backend": "auto",
31
+ "attn_impl": "eager",
32
+ "backbone_lr": null,
33
+ "backbone_type": "qwen2",
34
+ "batch_size": 48,
35
+ "cnn_depth": 48,
36
+ "cnn_kernel": 5,
37
+ "cnn_mults": [
38
+ 2,
39
+ 3,
40
+ 4,
41
+ 4
42
+ ],
43
+ "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d2900_n1_world4/phase1grid_d2900_n1_nodyn_seed0_280k.yaml",
44
+ "crops_per_demo": 8,
45
+ "d_model": 2048,
46
+ "dataset": [
47
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
48
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
70
+ ],
71
+ "ddp_timeout_minutes": 180.0,
72
+ "denoising_steps": 20,
73
+ "device": "cuda",
74
+ "diffusion_mlp_dims": [
75
+ 1024,
76
+ 1024,
77
+ 1024
78
+ ],
79
+ "diffusion_residual_style": true,
80
+ "diffusion_time_dim": 16,
81
+ "dropout": 0.1,
82
+ "dry_run_data": false,
83
+ "dynamics": {
84
+ "enabled": false,
85
+ "params": {},
86
+ "type": "none"
87
+ },
88
+ "dynamics_decoder_type": "film",
89
+ "enable_pred_next": false,
90
+ "encoder": {
91
+ "params": {
92
+ "cnn_depth": 48,
93
+ "cnn_kernel": 5,
94
+ "cnn_mults": [
95
+ 2,
96
+ 3,
97
+ 4,
98
+ 8,
99
+ 16
100
+ ],
101
+ "d_model": 512,
102
+ "dropout": 0.0,
103
+ "mlp_ratio": 4,
104
+ "n_fusion_layers": 1,
105
+ "n_heads": 4,
106
+ "readout_depth": 1,
107
+ "readout_mlp_ratio": 2,
108
+ "readout_queries": 4,
109
+ "rope_theta": 10000.0,
110
+ "use_proprio": true
111
+ },
112
+ "type": "attn_fusion_latent_token"
113
+ },
114
+ "encoder_lr": 0.0006,
115
+ "eval_at_start": false,
116
+ "eval_batch_size": 64,
117
+ "eval_crops_per_demo": 8,
118
+ "eval_every": 14000,
119
+ "eval_num_workers": 24,
120
+ "eval_seed": 0,
121
+ "ffn_hidden_size": 1024,
122
+ "filter_key": "wl23_mg2900_seed0",
123
+ "grad_accum_steps": 1,
124
+ "grad_clip": 1.0,
125
+ "hdf5_paths": [
126
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
127
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
149
+ ],
150
+ "holdout_action_mean_samples": 1,
151
+ "holdout_crops_per_demo": 8,
152
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
153
+ "holdout_per_task_metrics": true,
154
+ "holdout_rmse_samplers": [
155
+ "deterministic",
156
+ "stochastic"
157
+ ],
158
+ "holdout_sample_deterministic": null,
159
+ "holdout_sample_prefix_horizon": 4,
160
+ "holdout_sample_rmse": true,
161
+ "holdout_size": 2300,
162
+ "image_emb_dim": 512,
163
+ "init_device": "meta",
164
+ "input_norm": true,
165
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
166
+ "lang_emb_mode": "clip",
167
+ "lang_encoder_device": "auto",
168
+ "lang_obs_emb_dim": 256,
169
+ "log_every": 25,
170
+ "lora": false,
171
+ "lora_alpha": 32,
172
+ "lora_dropout": 0.0,
173
+ "lora_rank": 16,
174
+ "lora_target": [
175
+ "q_proj",
176
+ "k_proj",
177
+ "v_proj",
178
+ "o_proj",
179
+ "gate_proj",
180
+ "up_proj",
181
+ "down_proj"
182
+ ],
183
+ "lr": 0.0003,
184
+ "max_context_len": 1024,
185
+ "max_steps": 280000,
186
+ "min_lr_ratio": 0.1,
187
+ "model": {
188
+ "action_chunk_len": 10,
189
+ "latent_dim": 512
190
+ },
191
+ "model_dtype": "float32",
192
+ "n_heads": 16,
193
+ "n_kv_heads": null,
194
+ "n_layers": 2,
195
+ "norm_bias_weight_decay": null,
196
+ "norm_fit_batches": 0,
197
+ "num_workers": 24,
198
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
199
+ "obs_spec": {
200
+ "image_channels": 3,
201
+ "image_hw": [
202
+ 128,
203
+ 128
204
+ ],
205
+ "image_keys": [
206
+ "robot0_agentview_left_image",
207
+ "robot0_agentview_right_image",
208
+ "robot0_eye_in_hand_image"
209
+ ],
210
+ "lang_dim": 768,
211
+ "layout": "NHWC",
212
+ "low_dim_dims": [
213
+ 3,
214
+ 4,
215
+ 3,
216
+ 4,
217
+ 2
218
+ ],
219
+ "low_dim_keys": [
220
+ "robot0_base_to_eef_pos",
221
+ "robot0_base_to_eef_quat",
222
+ "robot0_base_pos",
223
+ "robot0_base_quat",
224
+ "robot0_gripper_qpos"
225
+ ]
226
+ },
227
+ "obs_stride": 4,
228
+ "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n1_d2900_seed0",
229
+ "persistent_workers": true,
230
+ "precision": "bf16",
231
+ "pred_next_image_key_weights": {},
232
+ "pred_next_image_weight": 1.0,
233
+ "pred_next_lang_weight": 0.1,
234
+ "pred_next_mode": "terminal",
235
+ "pred_next_obs_offset": 1,
236
+ "pred_next_proprio_weight": 0.15,
237
+ "pred_next_steps": 4,
238
+ "pred_warmup_steps": 0,
239
+ "predictor_lr": 0.0006,
240
+ "prefetch_factor": 4,
241
+ "prepare_lang_cache_only": false,
242
+ "proprio_emb_dim": 128,
243
+ "recon": {
244
+ "enabled": false,
245
+ "params": {
246
+ "depth": 48,
247
+ "kernel_size": 5,
248
+ "mults": [
249
+ 2,
250
+ 3,
251
+ 4,
252
+ 4
253
+ ]
254
+ }
255
+ },
256
+ "recon_image_weight": 1.0,
257
+ "recon_lang_weight": 0.1,
258
+ "recon_obs": false,
259
+ "recon_proprio_weight": 1.0,
260
+ "resume": null,
261
+ "resume_final_lr": null,
262
+ "resume_lr_schedule": "checkpoint",
263
+ "robomimic_src": null,
264
+ "save_every": 28000,
265
+ "save_holdout_traces": true,
266
+ "save_latest_checkpoint": true,
267
+ "scaling_action_head_d_model": 128,
268
+ "scaling_action_head_layers": 1,
269
+ "scaling_action_head_lr": 0.0003,
270
+ "scaling_backbone_d_model": 512,
271
+ "scaling_backbone_layers": 2,
272
+ "scaling_batch_policy": "phase1_grid_h100_v20260711",
273
+ "scaling_batch_policy_d50_microbatch": 48,
274
+ "scaling_encoder_d_model": 512,
275
+ "scaling_encoder_layers": 1,
276
+ "scaling_encoder_lr": 0.0006,
277
+ "scaling_expected_world_size": 4,
278
+ "scaling_global_batch_size": 192,
279
+ "scaling_group_b_lr_role": null,
280
+ "scaling_latent_dim": 512,
281
+ "scaling_lr_action_head": 0.0003,
282
+ "scaling_lr_base": null,
283
+ "scaling_lr_base_frozen": false,
284
+ "scaling_lr_encoder": 0.0006,
285
+ "scaling_lr_formula_size_axis": "discrete_family_lookup",
286
+ "scaling_lr_lookup_calibration": false,
287
+ "scaling_lr_lookup_model": "n1",
288
+ "scaling_lr_lookup_status": "confirmed",
289
+ "scaling_lr_predictor": 0.0006,
290
+ "scaling_lr_reference": null,
291
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
292
+ "scaling_lr_table_version": "n_lookup_v20260711_n5",
293
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
294
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
295
+ "scaling_per_rank_microbatch": 48,
296
+ "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0",
297
+ "scaling_phase1_topology": "phase1_5x5_seed0_v20260711",
298
+ "scaling_predictor_lr": 0.0006,
299
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
300
+ "scaling_recipe": "nodyn",
301
+ "scaling_run_name": "phase1grid_d2900_n1_nodyn_seed0_280k",
302
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04",
303
+ "scaling_suite": "phase1_grid",
304
+ "seed": 0,
305
+ "seq_len": 10,
306
+ "sequence_model": {
307
+ "backbone_type": "qwen2",
308
+ "hidden_dim": 512,
309
+ "params": {
310
+ "attn_impl": "sdpa",
311
+ "dropout": 0.1,
312
+ "ffn_hidden_size": 1408,
313
+ "input_norm": false,
314
+ "max_context_len": 1024,
315
+ "model_dtype": "float32",
316
+ "n_heads": 4,
317
+ "n_kv_heads": null,
318
+ "n_layers": 2
319
+ },
320
+ "type": "continuous_transformer"
321
+ },
322
+ "use_proprio": true,
323
+ "warmup_steps": 14000,
324
+ "weight_decay": 0.0,
325
+ "z_bottleneck": {
326
+ "dim": null
327
+ },
328
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz"
329
+ }
04_robocasa_scaling/scaling_n1_d2900_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n1_d2900_seed0",
3
+ "original_run_name": "phase1grid_d2900_n1_nodyn_seed0_280k_cuda0-1-2-3",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 2900,
7
+ "N": 1,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 280000
13
+ }
04_robocasa_scaling/scaling_n1_d300_seed0/checkpoint_step_00030000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3c340b937db28c148aeb9f4a161a2f039b86c8916fd95214b7bfaf72052fb7d
3
+ size 531280965
04_robocasa_scaling/scaling_n1_d300_seed0/config.json ADDED
@@ -0,0 +1,331 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 128,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 512,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 4,
13
+ "n_layers": 1,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d300_n1_world2/phase1grid_d300_n1_nodyn_seed0_30k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_data": false,
85
+ "dynamics": {
86
+ "enabled": false,
87
+ "params": {},
88
+ "type": "none"
89
+ },
90
+ "dynamics_decoder_type": "film",
91
+ "enable_pred_next": false,
92
+ "encoder": {
93
+ "params": {
94
+ "cnn_depth": 48,
95
+ "cnn_kernel": 5,
96
+ "cnn_mults": [
97
+ 2,
98
+ 3,
99
+ 4,
100
+ 8,
101
+ 16
102
+ ],
103
+ "d_model": 512,
104
+ "dropout": 0.0,
105
+ "mlp_ratio": 4,
106
+ "n_fusion_layers": 1,
107
+ "n_heads": 4,
108
+ "readout_depth": 1,
109
+ "readout_mlp_ratio": 2,
110
+ "readout_queries": 4,
111
+ "rope_theta": 10000.0,
112
+ "use_proprio": true
113
+ },
114
+ "type": "attn_fusion_latent_token"
115
+ },
116
+ "encoder_lr": 0.0006,
117
+ "eval_at_start": false,
118
+ "eval_batch_size": 64,
119
+ "eval_crops_per_demo": 8,
120
+ "eval_every": 1500,
121
+ "eval_num_workers": 24,
122
+ "eval_seed": 0,
123
+ "ffn_hidden_size": 1024,
124
+ "filter_key": "300_demos",
125
+ "grad_accum_steps": 2,
126
+ "grad_clip": 1.0,
127
+ "hdf5_paths": [
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
151
+ ],
152
+ "holdout_action_mean_samples": 1,
153
+ "holdout_crops_per_demo": 8,
154
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
155
+ "holdout_per_task_metrics": true,
156
+ "holdout_rmse_samplers": [
157
+ "deterministic",
158
+ "stochastic"
159
+ ],
160
+ "holdout_sample_deterministic": null,
161
+ "holdout_sample_prefix_horizon": 4,
162
+ "holdout_sample_rmse": true,
163
+ "holdout_size": 2300,
164
+ "image_emb_dim": 512,
165
+ "init_device": "meta",
166
+ "input_norm": true,
167
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
168
+ "lang_emb_mode": "clip",
169
+ "lang_encoder_device": "auto",
170
+ "lang_obs_emb_dim": 256,
171
+ "log_every": 25,
172
+ "lora": false,
173
+ "lora_alpha": 32,
174
+ "lora_dropout": 0.0,
175
+ "lora_rank": 16,
176
+ "lora_target": [
177
+ "q_proj",
178
+ "k_proj",
179
+ "v_proj",
180
+ "o_proj",
181
+ "gate_proj",
182
+ "up_proj",
183
+ "down_proj"
184
+ ],
185
+ "lr": 0.0003,
186
+ "max_context_len": 1024,
187
+ "max_steps": 30000,
188
+ "min_lr_ratio": 0.1,
189
+ "model": {
190
+ "action_chunk_len": 10,
191
+ "latent_dim": 512
192
+ },
193
+ "model_dtype": "float32",
194
+ "n_heads": 16,
195
+ "n_kv_heads": null,
196
+ "n_layers": 2,
197
+ "norm_bias_weight_decay": null,
198
+ "norm_fit_batches": 0,
199
+ "num_workers": 24,
200
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
201
+ "obs_spec": {
202
+ "image_channels": 3,
203
+ "image_hw": [
204
+ 128,
205
+ 128
206
+ ],
207
+ "image_keys": [
208
+ "robot0_agentview_left_image",
209
+ "robot0_agentview_right_image",
210
+ "robot0_eye_in_hand_image"
211
+ ],
212
+ "lang_dim": 768,
213
+ "layout": "NHWC",
214
+ "low_dim_dims": [
215
+ 3,
216
+ 4,
217
+ 3,
218
+ 4,
219
+ 2
220
+ ],
221
+ "low_dim_keys": [
222
+ "robot0_base_to_eef_pos",
223
+ "robot0_base_to_eef_quat",
224
+ "robot0_base_pos",
225
+ "robot0_base_quat",
226
+ "robot0_gripper_qpos"
227
+ ]
228
+ },
229
+ "obs_stride": 4,
230
+ "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n1_d300_seed0",
231
+ "persistent_workers": true,
232
+ "precision": "bf16",
233
+ "pred_next_image_key_weights": {},
234
+ "pred_next_image_weight": 1.0,
235
+ "pred_next_lang_weight": 0.1,
236
+ "pred_next_mode": "terminal",
237
+ "pred_next_obs_offset": 1,
238
+ "pred_next_proprio_weight": 0.15,
239
+ "pred_next_steps": 4,
240
+ "pred_warmup_steps": 0,
241
+ "predictor_lr": 0.0006,
242
+ "prefetch_factor": 4,
243
+ "prepare_lang_cache_only": false,
244
+ "proprio_emb_dim": 128,
245
+ "recon": {
246
+ "enabled": false,
247
+ "params": {
248
+ "depth": 48,
249
+ "kernel_size": 5,
250
+ "mults": [
251
+ 2,
252
+ 3,
253
+ 4,
254
+ 4
255
+ ]
256
+ }
257
+ },
258
+ "recon_image_weight": 1.0,
259
+ "recon_lang_weight": 0.1,
260
+ "recon_obs": false,
261
+ "recon_proprio_weight": 1.0,
262
+ "resume": null,
263
+ "resume_final_lr": null,
264
+ "resume_lr_schedule": "checkpoint",
265
+ "robomimic_src": null,
266
+ "save_every": 3000,
267
+ "save_holdout_traces": true,
268
+ "save_latest_checkpoint": true,
269
+ "scaling_action_head_d_model": 128,
270
+ "scaling_action_head_layers": 1,
271
+ "scaling_action_head_lr": 0.0003,
272
+ "scaling_backbone_d_model": 512,
273
+ "scaling_backbone_layers": 2,
274
+ "scaling_batch_policy": "phase1_grid_h100_v20260711",
275
+ "scaling_batch_policy_d50_microbatch": 48,
276
+ "scaling_encoder_d_model": 512,
277
+ "scaling_encoder_layers": 1,
278
+ "scaling_encoder_lr": 0.0006,
279
+ "scaling_expected_world_size": 2,
280
+ "scaling_global_batch_size": 192,
281
+ "scaling_group_b_lr_role": null,
282
+ "scaling_latent_dim": 512,
283
+ "scaling_lr_action_head": 0.0003,
284
+ "scaling_lr_base": null,
285
+ "scaling_lr_base_frozen": false,
286
+ "scaling_lr_encoder": 0.0006,
287
+ "scaling_lr_formula_size_axis": "discrete_family_lookup",
288
+ "scaling_lr_lookup_calibration": false,
289
+ "scaling_lr_lookup_model": "n1",
290
+ "scaling_lr_lookup_status": "confirmed",
291
+ "scaling_lr_predictor": 0.0006,
292
+ "scaling_lr_reference": null,
293
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
294
+ "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter",
295
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
296
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
297
+ "scaling_per_rank_microbatch": 48,
298
+ "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0",
299
+ "scaling_phase1_topology": "phase1_5x5_seed0_v20260711",
300
+ "scaling_predictor_lr": 0.0006,
301
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
302
+ "scaling_recipe": "nodyn",
303
+ "scaling_run_name": "phase1grid_d300_n1_nodyn_seed0_30k",
304
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04",
305
+ "scaling_suite": "phase1_grid",
306
+ "seed": 0,
307
+ "seq_len": 10,
308
+ "sequence_model": {
309
+ "backbone_type": "qwen2",
310
+ "hidden_dim": 512,
311
+ "params": {
312
+ "attn_impl": "sdpa",
313
+ "dropout": 0.1,
314
+ "ffn_hidden_size": 1408,
315
+ "input_norm": false,
316
+ "max_context_len": 1024,
317
+ "model_dtype": "float32",
318
+ "n_heads": 4,
319
+ "n_kv_heads": null,
320
+ "n_layers": 2
321
+ },
322
+ "type": "continuous_transformer"
323
+ },
324
+ "use_proprio": true,
325
+ "warmup_steps": 1500,
326
+ "weight_decay": 0.0,
327
+ "z_bottleneck": {
328
+ "dim": null
329
+ },
330
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz"
331
+ }
04_robocasa_scaling/scaling_n1_d300_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n1_d300_seed0",
3
+ "original_run_name": "phase1grid_d300_n1_nodyn_seed0_30k_cuda2-4",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 300,
7
+ "N": 1,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 30000
13
+ }
04_robocasa_scaling/scaling_n1_d50_seed0/checkpoint_step_00005000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:682682b296ebde324402a0edfaa296f9c13b1a912efd5ad130c48ac3a24ed9b9
3
+ size 531280837
04_robocasa_scaling/scaling_n1_d50_seed0/config.json ADDED
@@ -0,0 +1,329 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "d_model": 128,
7
+ "denoising_steps": 20,
8
+ "dropout": 0.0,
9
+ "h_adaln_bottleneck": false,
10
+ "n_heads": 4,
11
+ "n_layers": 1,
12
+ "use_h_cross_attn": false,
13
+ "use_obs_cross_attn": false
14
+ },
15
+ "type": "diffusion_dit"
16
+ },
17
+ "action_head_lr": 0.0003,
18
+ "action_spec": {
19
+ "dim": 12,
20
+ "discrete_dims": [
21
+ 6,
22
+ 11
23
+ ]
24
+ },
25
+ "action_weight": 1.0,
26
+ "adamw_beta1": 0.9,
27
+ "adamw_beta2": 0.95,
28
+ "adamw_eps": 1e-08,
29
+ "allow_holdout_overlap": false,
30
+ "attn_backend": "auto",
31
+ "attn_impl": "eager",
32
+ "backbone_lr": null,
33
+ "backbone_type": "qwen2",
34
+ "batch_size": 48,
35
+ "cnn_depth": 48,
36
+ "cnn_kernel": 5,
37
+ "cnn_mults": [
38
+ 2,
39
+ 3,
40
+ 4,
41
+ 4
42
+ ],
43
+ "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d50_n1_world1/phase1grid_d50_n1_nodyn_seed0_5k.yaml",
44
+ "crops_per_demo": 8,
45
+ "d_model": 2048,
46
+ "dataset": [
47
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
48
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
70
+ ],
71
+ "ddp_timeout_minutes": 180.0,
72
+ "denoising_steps": 20,
73
+ "device": "cuda",
74
+ "diffusion_mlp_dims": [
75
+ 1024,
76
+ 1024,
77
+ 1024
78
+ ],
79
+ "diffusion_residual_style": true,
80
+ "diffusion_time_dim": 16,
81
+ "dropout": 0.1,
82
+ "dry_run_data": false,
83
+ "dynamics": {
84
+ "enabled": false,
85
+ "params": {},
86
+ "type": "none"
87
+ },
88
+ "dynamics_decoder_type": "film",
89
+ "enable_pred_next": false,
90
+ "encoder": {
91
+ "params": {
92
+ "cnn_depth": 48,
93
+ "cnn_kernel": 5,
94
+ "cnn_mults": [
95
+ 2,
96
+ 3,
97
+ 4,
98
+ 8,
99
+ 16
100
+ ],
101
+ "d_model": 512,
102
+ "dropout": 0.0,
103
+ "mlp_ratio": 4,
104
+ "n_fusion_layers": 1,
105
+ "n_heads": 4,
106
+ "readout_depth": 1,
107
+ "readout_mlp_ratio": 2,
108
+ "readout_queries": 4,
109
+ "rope_theta": 10000.0,
110
+ "use_proprio": true
111
+ },
112
+ "type": "attn_fusion_latent_token"
113
+ },
114
+ "encoder_lr": 0.0006,
115
+ "eval_at_start": false,
116
+ "eval_batch_size": 64,
117
+ "eval_crops_per_demo": 8,
118
+ "eval_every": 250,
119
+ "eval_num_workers": 24,
120
+ "eval_seed": 0,
121
+ "ffn_hidden_size": 1024,
122
+ "filter_key": "wl23_mg50_seed0",
123
+ "grad_accum_steps": 4,
124
+ "grad_clip": 1.0,
125
+ "hdf5_paths": [
126
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
127
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
149
+ ],
150
+ "holdout_action_mean_samples": 1,
151
+ "holdout_crops_per_demo": 8,
152
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
153
+ "holdout_per_task_metrics": true,
154
+ "holdout_rmse_samplers": [
155
+ "deterministic",
156
+ "stochastic"
157
+ ],
158
+ "holdout_sample_deterministic": null,
159
+ "holdout_sample_prefix_horizon": 4,
160
+ "holdout_sample_rmse": true,
161
+ "holdout_size": 2300,
162
+ "image_emb_dim": 512,
163
+ "init_device": "meta",
164
+ "input_norm": true,
165
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
166
+ "lang_emb_mode": "clip",
167
+ "lang_encoder_device": "auto",
168
+ "lang_obs_emb_dim": 256,
169
+ "log_every": 25,
170
+ "lora": false,
171
+ "lora_alpha": 32,
172
+ "lora_dropout": 0.0,
173
+ "lora_rank": 16,
174
+ "lora_target": [
175
+ "q_proj",
176
+ "k_proj",
177
+ "v_proj",
178
+ "o_proj",
179
+ "gate_proj",
180
+ "up_proj",
181
+ "down_proj"
182
+ ],
183
+ "lr": 0.0003,
184
+ "max_context_len": 1024,
185
+ "max_steps": 5000,
186
+ "min_lr_ratio": 0.1,
187
+ "model": {
188
+ "action_chunk_len": 10,
189
+ "latent_dim": 512
190
+ },
191
+ "model_dtype": "float32",
192
+ "n_heads": 16,
193
+ "n_kv_heads": null,
194
+ "n_layers": 2,
195
+ "norm_bias_weight_decay": null,
196
+ "norm_fit_batches": 0,
197
+ "num_workers": 24,
198
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
199
+ "obs_spec": {
200
+ "image_channels": 3,
201
+ "image_hw": [
202
+ 128,
203
+ 128
204
+ ],
205
+ "image_keys": [
206
+ "robot0_agentview_left_image",
207
+ "robot0_agentview_right_image",
208
+ "robot0_eye_in_hand_image"
209
+ ],
210
+ "lang_dim": 768,
211
+ "layout": "NHWC",
212
+ "low_dim_dims": [
213
+ 3,
214
+ 4,
215
+ 3,
216
+ 4,
217
+ 2
218
+ ],
219
+ "low_dim_keys": [
220
+ "robot0_base_to_eef_pos",
221
+ "robot0_base_to_eef_quat",
222
+ "robot0_base_pos",
223
+ "robot0_base_quat",
224
+ "robot0_gripper_qpos"
225
+ ]
226
+ },
227
+ "obs_stride": 4,
228
+ "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n1_d50_seed0",
229
+ "persistent_workers": true,
230
+ "precision": "bf16",
231
+ "pred_next_image_key_weights": {},
232
+ "pred_next_image_weight": 1.0,
233
+ "pred_next_lang_weight": 0.1,
234
+ "pred_next_mode": "terminal",
235
+ "pred_next_obs_offset": 1,
236
+ "pred_next_proprio_weight": 0.15,
237
+ "pred_next_steps": 4,
238
+ "pred_warmup_steps": 0,
239
+ "predictor_lr": 0.0006,
240
+ "prefetch_factor": 4,
241
+ "prepare_lang_cache_only": false,
242
+ "proprio_emb_dim": 128,
243
+ "recon": {
244
+ "enabled": false,
245
+ "params": {
246
+ "depth": 48,
247
+ "kernel_size": 5,
248
+ "mults": [
249
+ 2,
250
+ 3,
251
+ 4,
252
+ 4
253
+ ]
254
+ }
255
+ },
256
+ "recon_image_weight": 1.0,
257
+ "recon_lang_weight": 0.1,
258
+ "recon_obs": false,
259
+ "recon_proprio_weight": 1.0,
260
+ "resume": null,
261
+ "resume_final_lr": null,
262
+ "resume_lr_schedule": "checkpoint",
263
+ "robomimic_src": null,
264
+ "save_every": 500,
265
+ "save_holdout_traces": true,
266
+ "save_latest_checkpoint": true,
267
+ "scaling_action_head_d_model": 128,
268
+ "scaling_action_head_layers": 1,
269
+ "scaling_action_head_lr": 0.0003,
270
+ "scaling_backbone_d_model": 512,
271
+ "scaling_backbone_layers": 2,
272
+ "scaling_batch_policy": "phase1_grid_h100_v20260711",
273
+ "scaling_batch_policy_d50_microbatch": 48,
274
+ "scaling_encoder_d_model": 512,
275
+ "scaling_encoder_layers": 1,
276
+ "scaling_encoder_lr": 0.0006,
277
+ "scaling_expected_world_size": 1,
278
+ "scaling_global_batch_size": 192,
279
+ "scaling_group_b_lr_role": null,
280
+ "scaling_latent_dim": 512,
281
+ "scaling_lr_action_head": 0.0003,
282
+ "scaling_lr_base": null,
283
+ "scaling_lr_base_frozen": false,
284
+ "scaling_lr_encoder": 0.0006,
285
+ "scaling_lr_formula_size_axis": "discrete_family_lookup",
286
+ "scaling_lr_lookup_calibration": false,
287
+ "scaling_lr_lookup_model": "n1",
288
+ "scaling_lr_lookup_status": "confirmed",
289
+ "scaling_lr_predictor": 0.0006,
290
+ "scaling_lr_reference": null,
291
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
292
+ "scaling_lr_table_version": "n_lookup_v20260711_n5",
293
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
294
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
295
+ "scaling_per_rank_microbatch": 48,
296
+ "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0",
297
+ "scaling_phase1_topology": "phase1_5x5_seed0_v20260711",
298
+ "scaling_predictor_lr": 0.0006,
299
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
300
+ "scaling_recipe": "nodyn",
301
+ "scaling_run_name": "phase1grid_d50_n1_nodyn_seed0_5k",
302
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04",
303
+ "scaling_suite": "phase1_grid",
304
+ "seed": 0,
305
+ "seq_len": 10,
306
+ "sequence_model": {
307
+ "backbone_type": "qwen2",
308
+ "hidden_dim": 512,
309
+ "params": {
310
+ "attn_impl": "sdpa",
311
+ "dropout": 0.1,
312
+ "ffn_hidden_size": 1408,
313
+ "input_norm": false,
314
+ "max_context_len": 1024,
315
+ "model_dtype": "float32",
316
+ "n_heads": 4,
317
+ "n_kv_heads": null,
318
+ "n_layers": 2
319
+ },
320
+ "type": "continuous_transformer"
321
+ },
322
+ "use_proprio": true,
323
+ "warmup_steps": 250,
324
+ "weight_decay": 0.0,
325
+ "z_bottleneck": {
326
+ "dim": null
327
+ },
328
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz"
329
+ }
04_robocasa_scaling/scaling_n1_d50_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n1_d50_seed0",
3
+ "original_run_name": "phase1grid_d50_n1_nodyn_seed0_5k_cuda6",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 50,
7
+ "N": 1,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 5000
13
+ }
04_robocasa_scaling/scaling_n2_d1000_seed0/checkpoint_step_00100000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5371a75babaaf64880677ac3ed6db74f29dca9f2483e58998ecc6db7b1e6a158
3
+ size 1024355757
04_robocasa_scaling/scaling_n2_d1000_seed0/config.json ADDED
@@ -0,0 +1,331 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 192,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 768,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 6,
13
+ "n_layers": 2,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d1000_n2_world2/phase1grid_d1000_n2_nodyn_seed0_100k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_data": false,
85
+ "dynamics": {
86
+ "enabled": false,
87
+ "params": {},
88
+ "type": "none"
89
+ },
90
+ "dynamics_decoder_type": "film",
91
+ "enable_pred_next": false,
92
+ "encoder": {
93
+ "params": {
94
+ "cnn_depth": 48,
95
+ "cnn_kernel": 5,
96
+ "cnn_mults": [
97
+ 2,
98
+ 3,
99
+ 4,
100
+ 8,
101
+ 16
102
+ ],
103
+ "d_model": 768,
104
+ "dropout": 0.0,
105
+ "mlp_ratio": 4,
106
+ "n_fusion_layers": 2,
107
+ "n_heads": 6,
108
+ "readout_depth": 1,
109
+ "readout_mlp_ratio": 2,
110
+ "readout_queries": 4,
111
+ "rope_theta": 10000.0,
112
+ "use_proprio": true
113
+ },
114
+ "type": "attn_fusion_latent_token"
115
+ },
116
+ "encoder_lr": 0.000425,
117
+ "eval_at_start": false,
118
+ "eval_batch_size": 64,
119
+ "eval_crops_per_demo": 8,
120
+ "eval_every": 5000,
121
+ "eval_num_workers": 24,
122
+ "eval_seed": 0,
123
+ "ffn_hidden_size": 1024,
124
+ "filter_key": "wl23_mg1000_seed0",
125
+ "grad_accum_steps": 2,
126
+ "grad_clip": 1.0,
127
+ "hdf5_paths": [
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
151
+ ],
152
+ "holdout_action_mean_samples": 1,
153
+ "holdout_crops_per_demo": 8,
154
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
155
+ "holdout_per_task_metrics": true,
156
+ "holdout_rmse_samplers": [
157
+ "deterministic",
158
+ "stochastic"
159
+ ],
160
+ "holdout_sample_deterministic": null,
161
+ "holdout_sample_prefix_horizon": 4,
162
+ "holdout_sample_rmse": true,
163
+ "holdout_size": 2300,
164
+ "image_emb_dim": 512,
165
+ "init_device": "meta",
166
+ "input_norm": true,
167
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
168
+ "lang_emb_mode": "clip",
169
+ "lang_encoder_device": "auto",
170
+ "lang_obs_emb_dim": 256,
171
+ "log_every": 25,
172
+ "lora": false,
173
+ "lora_alpha": 32,
174
+ "lora_dropout": 0.0,
175
+ "lora_rank": 16,
176
+ "lora_target": [
177
+ "q_proj",
178
+ "k_proj",
179
+ "v_proj",
180
+ "o_proj",
181
+ "gate_proj",
182
+ "up_proj",
183
+ "down_proj"
184
+ ],
185
+ "lr": 0.0003,
186
+ "max_context_len": 1024,
187
+ "max_steps": 100000,
188
+ "min_lr_ratio": 0.1,
189
+ "model": {
190
+ "action_chunk_len": 10,
191
+ "latent_dim": 768
192
+ },
193
+ "model_dtype": "float32",
194
+ "n_heads": 16,
195
+ "n_kv_heads": null,
196
+ "n_layers": 2,
197
+ "norm_bias_weight_decay": null,
198
+ "norm_fit_batches": 0,
199
+ "num_workers": 24,
200
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
201
+ "obs_spec": {
202
+ "image_channels": 3,
203
+ "image_hw": [
204
+ 128,
205
+ 128
206
+ ],
207
+ "image_keys": [
208
+ "robot0_agentview_left_image",
209
+ "robot0_agentview_right_image",
210
+ "robot0_eye_in_hand_image"
211
+ ],
212
+ "lang_dim": 768,
213
+ "layout": "NHWC",
214
+ "low_dim_dims": [
215
+ 3,
216
+ 4,
217
+ 3,
218
+ 4,
219
+ 2
220
+ ],
221
+ "low_dim_keys": [
222
+ "robot0_base_to_eef_pos",
223
+ "robot0_base_to_eef_quat",
224
+ "robot0_base_pos",
225
+ "robot0_base_quat",
226
+ "robot0_gripper_qpos"
227
+ ]
228
+ },
229
+ "obs_stride": 4,
230
+ "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n2_d1000_seed0",
231
+ "persistent_workers": true,
232
+ "precision": "bf16",
233
+ "pred_next_image_key_weights": {},
234
+ "pred_next_image_weight": 1.0,
235
+ "pred_next_lang_weight": 0.1,
236
+ "pred_next_mode": "terminal",
237
+ "pred_next_obs_offset": 1,
238
+ "pred_next_proprio_weight": 0.15,
239
+ "pred_next_steps": 4,
240
+ "pred_warmup_steps": 0,
241
+ "predictor_lr": 0.000425,
242
+ "prefetch_factor": 4,
243
+ "prepare_lang_cache_only": false,
244
+ "proprio_emb_dim": 128,
245
+ "recon": {
246
+ "enabled": false,
247
+ "params": {
248
+ "depth": 48,
249
+ "kernel_size": 5,
250
+ "mults": [
251
+ 2,
252
+ 3,
253
+ 4,
254
+ 4
255
+ ]
256
+ }
257
+ },
258
+ "recon_image_weight": 1.0,
259
+ "recon_lang_weight": 0.1,
260
+ "recon_obs": false,
261
+ "recon_proprio_weight": 1.0,
262
+ "resume": null,
263
+ "resume_final_lr": null,
264
+ "resume_lr_schedule": "checkpoint",
265
+ "robomimic_src": null,
266
+ "save_every": 10000,
267
+ "save_holdout_traces": true,
268
+ "save_latest_checkpoint": true,
269
+ "scaling_action_head_d_model": 192,
270
+ "scaling_action_head_layers": 2,
271
+ "scaling_action_head_lr": 0.0003,
272
+ "scaling_backbone_d_model": 768,
273
+ "scaling_backbone_layers": 4,
274
+ "scaling_batch_policy": "phase1_grid_h100_v20260711",
275
+ "scaling_batch_policy_d50_microbatch": 48,
276
+ "scaling_encoder_d_model": 768,
277
+ "scaling_encoder_layers": 2,
278
+ "scaling_encoder_lr": 0.000425,
279
+ "scaling_expected_world_size": 2,
280
+ "scaling_global_batch_size": 192,
281
+ "scaling_group_b_lr_role": null,
282
+ "scaling_latent_dim": 768,
283
+ "scaling_lr_action_head": 0.0003,
284
+ "scaling_lr_base": null,
285
+ "scaling_lr_base_frozen": false,
286
+ "scaling_lr_encoder": 0.000425,
287
+ "scaling_lr_formula_size_axis": "discrete_family_lookup",
288
+ "scaling_lr_lookup_calibration": false,
289
+ "scaling_lr_lookup_model": "n2",
290
+ "scaling_lr_lookup_status": "confirmed",
291
+ "scaling_lr_predictor": 0.000425,
292
+ "scaling_lr_reference": null,
293
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
294
+ "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter",
295
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
296
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
297
+ "scaling_per_rank_microbatch": 48,
298
+ "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0",
299
+ "scaling_phase1_topology": "phase1_5x5_seed0_v20260711",
300
+ "scaling_predictor_lr": 0.000425,
301
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
302
+ "scaling_recipe": "nodyn",
303
+ "scaling_run_name": "phase1grid_d1000_n2_nodyn_seed0_100k",
304
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04",
305
+ "scaling_suite": "phase1_grid",
306
+ "seed": 0,
307
+ "seq_len": 10,
308
+ "sequence_model": {
309
+ "backbone_type": "qwen2",
310
+ "hidden_dim": 768,
311
+ "params": {
312
+ "attn_impl": "sdpa",
313
+ "dropout": 0.1,
314
+ "ffn_hidden_size": 2048,
315
+ "input_norm": false,
316
+ "max_context_len": 1024,
317
+ "model_dtype": "float32",
318
+ "n_heads": 6,
319
+ "n_kv_heads": null,
320
+ "n_layers": 4
321
+ },
322
+ "type": "continuous_transformer"
323
+ },
324
+ "use_proprio": true,
325
+ "warmup_steps": 5000,
326
+ "weight_decay": 0.0,
327
+ "z_bottleneck": {
328
+ "dim": null
329
+ },
330
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz"
331
+ }
04_robocasa_scaling/scaling_n2_d1000_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n2_d1000_seed0",
3
+ "original_run_name": "phase1grid_d1000_n2_nodyn_seed0_100k_cuda4-5",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 1000,
7
+ "N": 2,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 100000
13
+ }
04_robocasa_scaling/scaling_n2_d100_seed0/checkpoint_step_00010000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f03b059f1c4b471497a3b3b4678cfd8b5df585be82f21eefc21b34ab5a5b65c1
3
+ size 1024357869
04_robocasa_scaling/scaling_n2_d100_seed0/config.json ADDED
@@ -0,0 +1,331 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 192,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 768,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 6,
13
+ "n_layers": 2,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_resume_n2_d100/phase1grid_d100_n2_nodyn_seed0_10k_resume_mb48_accum4.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_data": false,
85
+ "dynamics": {
86
+ "enabled": false,
87
+ "params": {},
88
+ "type": "none"
89
+ },
90
+ "dynamics_decoder_type": "film",
91
+ "enable_pred_next": false,
92
+ "encoder": {
93
+ "params": {
94
+ "cnn_depth": 48,
95
+ "cnn_kernel": 5,
96
+ "cnn_mults": [
97
+ 2,
98
+ 3,
99
+ 4,
100
+ 8,
101
+ 16
102
+ ],
103
+ "d_model": 768,
104
+ "dropout": 0.0,
105
+ "mlp_ratio": 4,
106
+ "n_fusion_layers": 2,
107
+ "n_heads": 6,
108
+ "readout_depth": 1,
109
+ "readout_mlp_ratio": 2,
110
+ "readout_queries": 4,
111
+ "rope_theta": 10000.0,
112
+ "use_proprio": true
113
+ },
114
+ "type": "attn_fusion_latent_token"
115
+ },
116
+ "encoder_lr": 0.000425,
117
+ "eval_at_start": false,
118
+ "eval_batch_size": 64,
119
+ "eval_crops_per_demo": 8,
120
+ "eval_every": 500,
121
+ "eval_num_workers": 24,
122
+ "eval_seed": 0,
123
+ "ffn_hidden_size": 1024,
124
+ "filter_key": "wl23_mg100_seed0",
125
+ "grad_accum_steps": 4,
126
+ "grad_clip": 1.0,
127
+ "hdf5_paths": [
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
151
+ ],
152
+ "holdout_action_mean_samples": 1,
153
+ "holdout_crops_per_demo": 8,
154
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
155
+ "holdout_per_task_metrics": true,
156
+ "holdout_rmse_samplers": [
157
+ "deterministic",
158
+ "stochastic"
159
+ ],
160
+ "holdout_sample_deterministic": null,
161
+ "holdout_sample_prefix_horizon": 4,
162
+ "holdout_sample_rmse": true,
163
+ "holdout_size": 2300,
164
+ "image_emb_dim": 512,
165
+ "init_device": "meta",
166
+ "input_norm": true,
167
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
168
+ "lang_emb_mode": "clip",
169
+ "lang_encoder_device": "auto",
170
+ "lang_obs_emb_dim": 256,
171
+ "log_every": 25,
172
+ "lora": false,
173
+ "lora_alpha": 32,
174
+ "lora_dropout": 0.0,
175
+ "lora_rank": 16,
176
+ "lora_target": [
177
+ "q_proj",
178
+ "k_proj",
179
+ "v_proj",
180
+ "o_proj",
181
+ "gate_proj",
182
+ "up_proj",
183
+ "down_proj"
184
+ ],
185
+ "lr": 0.0003,
186
+ "max_context_len": 1024,
187
+ "max_steps": 10000,
188
+ "min_lr_ratio": 0.1,
189
+ "model": {
190
+ "action_chunk_len": 10,
191
+ "latent_dim": 768
192
+ },
193
+ "model_dtype": "float32",
194
+ "n_heads": 16,
195
+ "n_kv_heads": null,
196
+ "n_layers": 2,
197
+ "norm_bias_weight_decay": null,
198
+ "norm_fit_batches": 0,
199
+ "num_workers": 24,
200
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
201
+ "obs_spec": {
202
+ "image_channels": 3,
203
+ "image_hw": [
204
+ 128,
205
+ 128
206
+ ],
207
+ "image_keys": [
208
+ "robot0_agentview_left_image",
209
+ "robot0_agentview_right_image",
210
+ "robot0_eye_in_hand_image"
211
+ ],
212
+ "lang_dim": 768,
213
+ "layout": "NHWC",
214
+ "low_dim_dims": [
215
+ 3,
216
+ 4,
217
+ 3,
218
+ 4,
219
+ 2
220
+ ],
221
+ "low_dim_keys": [
222
+ "robot0_base_to_eef_pos",
223
+ "robot0_base_to_eef_quat",
224
+ "robot0_base_pos",
225
+ "robot0_base_quat",
226
+ "robot0_gripper_qpos"
227
+ ]
228
+ },
229
+ "obs_stride": 4,
230
+ "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n2_d100_seed0",
231
+ "persistent_workers": true,
232
+ "precision": "bf16",
233
+ "pred_next_image_key_weights": {},
234
+ "pred_next_image_weight": 1.0,
235
+ "pred_next_lang_weight": 0.1,
236
+ "pred_next_mode": "terminal",
237
+ "pred_next_obs_offset": 1,
238
+ "pred_next_proprio_weight": 0.15,
239
+ "pred_next_steps": 4,
240
+ "pred_warmup_steps": 0,
241
+ "predictor_lr": 0.000425,
242
+ "prefetch_factor": 4,
243
+ "prepare_lang_cache_only": false,
244
+ "proprio_emb_dim": 128,
245
+ "recon": {
246
+ "enabled": false,
247
+ "params": {
248
+ "depth": 48,
249
+ "kernel_size": 5,
250
+ "mults": [
251
+ 2,
252
+ 3,
253
+ 4,
254
+ 4
255
+ ]
256
+ }
257
+ },
258
+ "recon_image_weight": 1.0,
259
+ "recon_lang_weight": 0.1,
260
+ "recon_obs": false,
261
+ "recon_proprio_weight": 1.0,
262
+ "resume": "${CHECKPOINT_ROOT}/checkpoint_step_00008000.pt",
263
+ "resume_final_lr": null,
264
+ "resume_lr_schedule": "checkpoint",
265
+ "robomimic_src": null,
266
+ "save_every": 1000,
267
+ "save_holdout_traces": true,
268
+ "save_latest_checkpoint": true,
269
+ "scaling_action_head_d_model": 192,
270
+ "scaling_action_head_layers": 2,
271
+ "scaling_action_head_lr": 0.0003,
272
+ "scaling_backbone_d_model": 768,
273
+ "scaling_backbone_layers": 4,
274
+ "scaling_batch_policy": "phase1_grid_h100_v20260711",
275
+ "scaling_batch_policy_d50_microbatch": 48,
276
+ "scaling_encoder_d_model": 768,
277
+ "scaling_encoder_layers": 2,
278
+ "scaling_encoder_lr": 0.000425,
279
+ "scaling_expected_world_size": 1,
280
+ "scaling_global_batch_size": 192,
281
+ "scaling_group_b_lr_role": null,
282
+ "scaling_latent_dim": 768,
283
+ "scaling_lr_action_head": 0.0003,
284
+ "scaling_lr_base": null,
285
+ "scaling_lr_base_frozen": false,
286
+ "scaling_lr_encoder": 0.000425,
287
+ "scaling_lr_formula_size_axis": "discrete_family_lookup",
288
+ "scaling_lr_lookup_calibration": false,
289
+ "scaling_lr_lookup_model": "n2",
290
+ "scaling_lr_lookup_status": "confirmed",
291
+ "scaling_lr_predictor": 0.000425,
292
+ "scaling_lr_reference": null,
293
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
294
+ "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter",
295
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
296
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
297
+ "scaling_per_rank_microbatch": 48,
298
+ "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0",
299
+ "scaling_phase1_topology": "phase1_5x5_seed0_v20260711",
300
+ "scaling_predictor_lr": 0.000425,
301
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
302
+ "scaling_recipe": "nodyn",
303
+ "scaling_run_name": "phase1grid_d100_n2_nodyn_seed0_10k",
304
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04",
305
+ "scaling_suite": "phase1_grid",
306
+ "seed": 0,
307
+ "seq_len": 10,
308
+ "sequence_model": {
309
+ "backbone_type": "qwen2",
310
+ "hidden_dim": 768,
311
+ "params": {
312
+ "attn_impl": "sdpa",
313
+ "dropout": 0.1,
314
+ "ffn_hidden_size": 2048,
315
+ "input_norm": false,
316
+ "max_context_len": 1024,
317
+ "model_dtype": "float32",
318
+ "n_heads": 6,
319
+ "n_kv_heads": null,
320
+ "n_layers": 4
321
+ },
322
+ "type": "continuous_transformer"
323
+ },
324
+ "use_proprio": true,
325
+ "warmup_steps": 500,
326
+ "weight_decay": 0.0,
327
+ "z_bottleneck": {
328
+ "dim": null
329
+ },
330
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz"
331
+ }
04_robocasa_scaling/scaling_n2_d100_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n2_d100_seed0",
3
+ "original_run_name": "phase1grid_d100_n2_nodyn_seed0_10k_cuda5",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 100,
7
+ "N": 2,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 10000
13
+ }
04_robocasa_scaling/scaling_n2_d2900_seed0/checkpoint_step_00280000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d321278276a2f3919cc1c3b5472675ee3b7fffe32a44d2559dc1dddabae3b100
3
+ size 1024355757
04_robocasa_scaling/scaling_n2_d2900_seed0/config.json ADDED
@@ -0,0 +1,331 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 192,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 768,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 6,
13
+ "n_layers": 2,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d2900_n2_world4/phase1grid_d2900_n2_nodyn_seed0_280k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_data": false,
85
+ "dynamics": {
86
+ "enabled": false,
87
+ "params": {},
88
+ "type": "none"
89
+ },
90
+ "dynamics_decoder_type": "film",
91
+ "enable_pred_next": false,
92
+ "encoder": {
93
+ "params": {
94
+ "cnn_depth": 48,
95
+ "cnn_kernel": 5,
96
+ "cnn_mults": [
97
+ 2,
98
+ 3,
99
+ 4,
100
+ 8,
101
+ 16
102
+ ],
103
+ "d_model": 768,
104
+ "dropout": 0.0,
105
+ "mlp_ratio": 4,
106
+ "n_fusion_layers": 2,
107
+ "n_heads": 6,
108
+ "readout_depth": 1,
109
+ "readout_mlp_ratio": 2,
110
+ "readout_queries": 4,
111
+ "rope_theta": 10000.0,
112
+ "use_proprio": true
113
+ },
114
+ "type": "attn_fusion_latent_token"
115
+ },
116
+ "encoder_lr": 0.000425,
117
+ "eval_at_start": false,
118
+ "eval_batch_size": 64,
119
+ "eval_crops_per_demo": 8,
120
+ "eval_every": 14000,
121
+ "eval_num_workers": 24,
122
+ "eval_seed": 0,
123
+ "ffn_hidden_size": 1024,
124
+ "filter_key": "wl23_mg2900_seed0",
125
+ "grad_accum_steps": 1,
126
+ "grad_clip": 1.0,
127
+ "hdf5_paths": [
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
151
+ ],
152
+ "holdout_action_mean_samples": 1,
153
+ "holdout_crops_per_demo": 8,
154
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
155
+ "holdout_per_task_metrics": true,
156
+ "holdout_rmse_samplers": [
157
+ "deterministic",
158
+ "stochastic"
159
+ ],
160
+ "holdout_sample_deterministic": null,
161
+ "holdout_sample_prefix_horizon": 4,
162
+ "holdout_sample_rmse": true,
163
+ "holdout_size": 2300,
164
+ "image_emb_dim": 512,
165
+ "init_device": "meta",
166
+ "input_norm": true,
167
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
168
+ "lang_emb_mode": "clip",
169
+ "lang_encoder_device": "auto",
170
+ "lang_obs_emb_dim": 256,
171
+ "log_every": 25,
172
+ "lora": false,
173
+ "lora_alpha": 32,
174
+ "lora_dropout": 0.0,
175
+ "lora_rank": 16,
176
+ "lora_target": [
177
+ "q_proj",
178
+ "k_proj",
179
+ "v_proj",
180
+ "o_proj",
181
+ "gate_proj",
182
+ "up_proj",
183
+ "down_proj"
184
+ ],
185
+ "lr": 0.0003,
186
+ "max_context_len": 1024,
187
+ "max_steps": 280000,
188
+ "min_lr_ratio": 0.1,
189
+ "model": {
190
+ "action_chunk_len": 10,
191
+ "latent_dim": 768
192
+ },
193
+ "model_dtype": "float32",
194
+ "n_heads": 16,
195
+ "n_kv_heads": null,
196
+ "n_layers": 2,
197
+ "norm_bias_weight_decay": null,
198
+ "norm_fit_batches": 0,
199
+ "num_workers": 24,
200
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
201
+ "obs_spec": {
202
+ "image_channels": 3,
203
+ "image_hw": [
204
+ 128,
205
+ 128
206
+ ],
207
+ "image_keys": [
208
+ "robot0_agentview_left_image",
209
+ "robot0_agentview_right_image",
210
+ "robot0_eye_in_hand_image"
211
+ ],
212
+ "lang_dim": 768,
213
+ "layout": "NHWC",
214
+ "low_dim_dims": [
215
+ 3,
216
+ 4,
217
+ 3,
218
+ 4,
219
+ 2
220
+ ],
221
+ "low_dim_keys": [
222
+ "robot0_base_to_eef_pos",
223
+ "robot0_base_to_eef_quat",
224
+ "robot0_base_pos",
225
+ "robot0_base_quat",
226
+ "robot0_gripper_qpos"
227
+ ]
228
+ },
229
+ "obs_stride": 4,
230
+ "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n2_d2900_seed0",
231
+ "persistent_workers": true,
232
+ "precision": "bf16",
233
+ "pred_next_image_key_weights": {},
234
+ "pred_next_image_weight": 1.0,
235
+ "pred_next_lang_weight": 0.1,
236
+ "pred_next_mode": "terminal",
237
+ "pred_next_obs_offset": 1,
238
+ "pred_next_proprio_weight": 0.15,
239
+ "pred_next_steps": 4,
240
+ "pred_warmup_steps": 0,
241
+ "predictor_lr": 0.000425,
242
+ "prefetch_factor": 4,
243
+ "prepare_lang_cache_only": false,
244
+ "proprio_emb_dim": 128,
245
+ "recon": {
246
+ "enabled": false,
247
+ "params": {
248
+ "depth": 48,
249
+ "kernel_size": 5,
250
+ "mults": [
251
+ 2,
252
+ 3,
253
+ 4,
254
+ 4
255
+ ]
256
+ }
257
+ },
258
+ "recon_image_weight": 1.0,
259
+ "recon_lang_weight": 0.1,
260
+ "recon_obs": false,
261
+ "recon_proprio_weight": 1.0,
262
+ "resume": null,
263
+ "resume_final_lr": null,
264
+ "resume_lr_schedule": "checkpoint",
265
+ "robomimic_src": null,
266
+ "save_every": 28000,
267
+ "save_holdout_traces": true,
268
+ "save_latest_checkpoint": true,
269
+ "scaling_action_head_d_model": 192,
270
+ "scaling_action_head_layers": 2,
271
+ "scaling_action_head_lr": 0.0003,
272
+ "scaling_backbone_d_model": 768,
273
+ "scaling_backbone_layers": 4,
274
+ "scaling_batch_policy": "phase1_grid_h100_v20260711",
275
+ "scaling_batch_policy_d50_microbatch": 48,
276
+ "scaling_encoder_d_model": 768,
277
+ "scaling_encoder_layers": 2,
278
+ "scaling_encoder_lr": 0.000425,
279
+ "scaling_expected_world_size": 4,
280
+ "scaling_global_batch_size": 192,
281
+ "scaling_group_b_lr_role": null,
282
+ "scaling_latent_dim": 768,
283
+ "scaling_lr_action_head": 0.0003,
284
+ "scaling_lr_base": null,
285
+ "scaling_lr_base_frozen": false,
286
+ "scaling_lr_encoder": 0.000425,
287
+ "scaling_lr_formula_size_axis": "discrete_family_lookup",
288
+ "scaling_lr_lookup_calibration": false,
289
+ "scaling_lr_lookup_model": "n2",
290
+ "scaling_lr_lookup_status": "confirmed",
291
+ "scaling_lr_predictor": 0.000425,
292
+ "scaling_lr_reference": null,
293
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
294
+ "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter",
295
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
296
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
297
+ "scaling_per_rank_microbatch": 48,
298
+ "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0",
299
+ "scaling_phase1_topology": "phase1_5x5_seed0_v20260711",
300
+ "scaling_predictor_lr": 0.000425,
301
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
302
+ "scaling_recipe": "nodyn",
303
+ "scaling_run_name": "phase1grid_d2900_n2_nodyn_seed0_280k",
304
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04",
305
+ "scaling_suite": "phase1_grid",
306
+ "seed": 0,
307
+ "seq_len": 10,
308
+ "sequence_model": {
309
+ "backbone_type": "qwen2",
310
+ "hidden_dim": 768,
311
+ "params": {
312
+ "attn_impl": "sdpa",
313
+ "dropout": 0.1,
314
+ "ffn_hidden_size": 2048,
315
+ "input_norm": false,
316
+ "max_context_len": 1024,
317
+ "model_dtype": "float32",
318
+ "n_heads": 6,
319
+ "n_kv_heads": null,
320
+ "n_layers": 4
321
+ },
322
+ "type": "continuous_transformer"
323
+ },
324
+ "use_proprio": true,
325
+ "warmup_steps": 14000,
326
+ "weight_decay": 0.0,
327
+ "z_bottleneck": {
328
+ "dim": null
329
+ },
330
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz"
331
+ }
04_robocasa_scaling/scaling_n2_d2900_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n2_d2900_seed0",
3
+ "original_run_name": "phase1grid_d2900_n2_nodyn_seed0_280k_cuda1-2-4-5",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 2900,
7
+ "N": 2,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 280000
13
+ }
04_robocasa_scaling/scaling_n2_d300_seed0/checkpoint_step_00030000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c209e131e5438c4430d7274f764e1867023101d6a58b71bae7c233405920d8d6
3
+ size 1024355757
04_robocasa_scaling/scaling_n2_d300_seed0/config.json ADDED
@@ -0,0 +1,331 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 192,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 768,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 6,
13
+ "n_layers": 2,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d300_n2_world2/phase1grid_d300_n2_nodyn_seed0_30k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_data": false,
85
+ "dynamics": {
86
+ "enabled": false,
87
+ "params": {},
88
+ "type": "none"
89
+ },
90
+ "dynamics_decoder_type": "film",
91
+ "enable_pred_next": false,
92
+ "encoder": {
93
+ "params": {
94
+ "cnn_depth": 48,
95
+ "cnn_kernel": 5,
96
+ "cnn_mults": [
97
+ 2,
98
+ 3,
99
+ 4,
100
+ 8,
101
+ 16
102
+ ],
103
+ "d_model": 768,
104
+ "dropout": 0.0,
105
+ "mlp_ratio": 4,
106
+ "n_fusion_layers": 2,
107
+ "n_heads": 6,
108
+ "readout_depth": 1,
109
+ "readout_mlp_ratio": 2,
110
+ "readout_queries": 4,
111
+ "rope_theta": 10000.0,
112
+ "use_proprio": true
113
+ },
114
+ "type": "attn_fusion_latent_token"
115
+ },
116
+ "encoder_lr": 0.000425,
117
+ "eval_at_start": false,
118
+ "eval_batch_size": 64,
119
+ "eval_crops_per_demo": 8,
120
+ "eval_every": 1500,
121
+ "eval_num_workers": 24,
122
+ "eval_seed": 0,
123
+ "ffn_hidden_size": 1024,
124
+ "filter_key": "300_demos",
125
+ "grad_accum_steps": 2,
126
+ "grad_clip": 1.0,
127
+ "hdf5_paths": [
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
151
+ ],
152
+ "holdout_action_mean_samples": 1,
153
+ "holdout_crops_per_demo": 8,
154
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
155
+ "holdout_per_task_metrics": true,
156
+ "holdout_rmse_samplers": [
157
+ "deterministic",
158
+ "stochastic"
159
+ ],
160
+ "holdout_sample_deterministic": null,
161
+ "holdout_sample_prefix_horizon": 4,
162
+ "holdout_sample_rmse": true,
163
+ "holdout_size": 2300,
164
+ "image_emb_dim": 512,
165
+ "init_device": "meta",
166
+ "input_norm": true,
167
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
168
+ "lang_emb_mode": "clip",
169
+ "lang_encoder_device": "auto",
170
+ "lang_obs_emb_dim": 256,
171
+ "log_every": 25,
172
+ "lora": false,
173
+ "lora_alpha": 32,
174
+ "lora_dropout": 0.0,
175
+ "lora_rank": 16,
176
+ "lora_target": [
177
+ "q_proj",
178
+ "k_proj",
179
+ "v_proj",
180
+ "o_proj",
181
+ "gate_proj",
182
+ "up_proj",
183
+ "down_proj"
184
+ ],
185
+ "lr": 0.0003,
186
+ "max_context_len": 1024,
187
+ "max_steps": 30000,
188
+ "min_lr_ratio": 0.1,
189
+ "model": {
190
+ "action_chunk_len": 10,
191
+ "latent_dim": 768
192
+ },
193
+ "model_dtype": "float32",
194
+ "n_heads": 16,
195
+ "n_kv_heads": null,
196
+ "n_layers": 2,
197
+ "norm_bias_weight_decay": null,
198
+ "norm_fit_batches": 0,
199
+ "num_workers": 24,
200
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
201
+ "obs_spec": {
202
+ "image_channels": 3,
203
+ "image_hw": [
204
+ 128,
205
+ 128
206
+ ],
207
+ "image_keys": [
208
+ "robot0_agentview_left_image",
209
+ "robot0_agentview_right_image",
210
+ "robot0_eye_in_hand_image"
211
+ ],
212
+ "lang_dim": 768,
213
+ "layout": "NHWC",
214
+ "low_dim_dims": [
215
+ 3,
216
+ 4,
217
+ 3,
218
+ 4,
219
+ 2
220
+ ],
221
+ "low_dim_keys": [
222
+ "robot0_base_to_eef_pos",
223
+ "robot0_base_to_eef_quat",
224
+ "robot0_base_pos",
225
+ "robot0_base_quat",
226
+ "robot0_gripper_qpos"
227
+ ]
228
+ },
229
+ "obs_stride": 4,
230
+ "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n2_d300_seed0",
231
+ "persistent_workers": true,
232
+ "precision": "bf16",
233
+ "pred_next_image_key_weights": {},
234
+ "pred_next_image_weight": 1.0,
235
+ "pred_next_lang_weight": 0.1,
236
+ "pred_next_mode": "terminal",
237
+ "pred_next_obs_offset": 1,
238
+ "pred_next_proprio_weight": 0.15,
239
+ "pred_next_steps": 4,
240
+ "pred_warmup_steps": 0,
241
+ "predictor_lr": 0.000425,
242
+ "prefetch_factor": 4,
243
+ "prepare_lang_cache_only": false,
244
+ "proprio_emb_dim": 128,
245
+ "recon": {
246
+ "enabled": false,
247
+ "params": {
248
+ "depth": 48,
249
+ "kernel_size": 5,
250
+ "mults": [
251
+ 2,
252
+ 3,
253
+ 4,
254
+ 4
255
+ ]
256
+ }
257
+ },
258
+ "recon_image_weight": 1.0,
259
+ "recon_lang_weight": 0.1,
260
+ "recon_obs": false,
261
+ "recon_proprio_weight": 1.0,
262
+ "resume": null,
263
+ "resume_final_lr": null,
264
+ "resume_lr_schedule": "checkpoint",
265
+ "robomimic_src": null,
266
+ "save_every": 3000,
267
+ "save_holdout_traces": true,
268
+ "save_latest_checkpoint": true,
269
+ "scaling_action_head_d_model": 192,
270
+ "scaling_action_head_layers": 2,
271
+ "scaling_action_head_lr": 0.0003,
272
+ "scaling_backbone_d_model": 768,
273
+ "scaling_backbone_layers": 4,
274
+ "scaling_batch_policy": "phase1_grid_h100_v20260711",
275
+ "scaling_batch_policy_d50_microbatch": 48,
276
+ "scaling_encoder_d_model": 768,
277
+ "scaling_encoder_layers": 2,
278
+ "scaling_encoder_lr": 0.000425,
279
+ "scaling_expected_world_size": 2,
280
+ "scaling_global_batch_size": 192,
281
+ "scaling_group_b_lr_role": null,
282
+ "scaling_latent_dim": 768,
283
+ "scaling_lr_action_head": 0.0003,
284
+ "scaling_lr_base": null,
285
+ "scaling_lr_base_frozen": false,
286
+ "scaling_lr_encoder": 0.000425,
287
+ "scaling_lr_formula_size_axis": "discrete_family_lookup",
288
+ "scaling_lr_lookup_calibration": false,
289
+ "scaling_lr_lookup_model": "n2",
290
+ "scaling_lr_lookup_status": "confirmed",
291
+ "scaling_lr_predictor": 0.000425,
292
+ "scaling_lr_reference": null,
293
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
294
+ "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter",
295
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
296
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
297
+ "scaling_per_rank_microbatch": 48,
298
+ "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0",
299
+ "scaling_phase1_topology": "phase1_5x5_seed0_v20260711",
300
+ "scaling_predictor_lr": 0.000425,
301
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
302
+ "scaling_recipe": "nodyn",
303
+ "scaling_run_name": "phase1grid_d300_n2_nodyn_seed0_30k",
304
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04",
305
+ "scaling_suite": "phase1_grid",
306
+ "seed": 0,
307
+ "seq_len": 10,
308
+ "sequence_model": {
309
+ "backbone_type": "qwen2",
310
+ "hidden_dim": 768,
311
+ "params": {
312
+ "attn_impl": "sdpa",
313
+ "dropout": 0.1,
314
+ "ffn_hidden_size": 2048,
315
+ "input_norm": false,
316
+ "max_context_len": 1024,
317
+ "model_dtype": "float32",
318
+ "n_heads": 6,
319
+ "n_kv_heads": null,
320
+ "n_layers": 4
321
+ },
322
+ "type": "continuous_transformer"
323
+ },
324
+ "use_proprio": true,
325
+ "warmup_steps": 1500,
326
+ "weight_decay": 0.0,
327
+ "z_bottleneck": {
328
+ "dim": null
329
+ },
330
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz"
331
+ }
04_robocasa_scaling/scaling_n2_d300_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n2_d300_seed0",
3
+ "original_run_name": "phase1grid_d300_n2_nodyn_seed0_30k_cuda0-1",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 300,
7
+ "N": 2,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 30000
13
+ }
04_robocasa_scaling/scaling_n2_d50_seed0/checkpoint_step_00005000.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68a50cdaab591898585aac9470797ee73fc08a0fec036ac0bb038d839fbaa9fe
3
+ size 1024355757
04_robocasa_scaling/scaling_n2_d50_seed0/config.json ADDED
@@ -0,0 +1,331 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 192,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 768,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 6,
13
+ "n_layers": 2,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d50_d100_n2_world1/phase1grid_d50_n2_nodyn_seed0_5k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_data": false,
85
+ "dynamics": {
86
+ "enabled": false,
87
+ "params": {},
88
+ "type": "none"
89
+ },
90
+ "dynamics_decoder_type": "film",
91
+ "enable_pred_next": false,
92
+ "encoder": {
93
+ "params": {
94
+ "cnn_depth": 48,
95
+ "cnn_kernel": 5,
96
+ "cnn_mults": [
97
+ 2,
98
+ 3,
99
+ 4,
100
+ 8,
101
+ 16
102
+ ],
103
+ "d_model": 768,
104
+ "dropout": 0.0,
105
+ "mlp_ratio": 4,
106
+ "n_fusion_layers": 2,
107
+ "n_heads": 6,
108
+ "readout_depth": 1,
109
+ "readout_mlp_ratio": 2,
110
+ "readout_queries": 4,
111
+ "rope_theta": 10000.0,
112
+ "use_proprio": true
113
+ },
114
+ "type": "attn_fusion_latent_token"
115
+ },
116
+ "encoder_lr": 0.000425,
117
+ "eval_at_start": false,
118
+ "eval_batch_size": 64,
119
+ "eval_crops_per_demo": 8,
120
+ "eval_every": 250,
121
+ "eval_num_workers": 24,
122
+ "eval_seed": 0,
123
+ "ffn_hidden_size": 1024,
124
+ "filter_key": "wl23_mg50_seed0",
125
+ "grad_accum_steps": 4,
126
+ "grad_clip": 1.0,
127
+ "hdf5_paths": [
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
151
+ ],
152
+ "holdout_action_mean_samples": 1,
153
+ "holdout_crops_per_demo": 8,
154
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
155
+ "holdout_per_task_metrics": true,
156
+ "holdout_rmse_samplers": [
157
+ "deterministic",
158
+ "stochastic"
159
+ ],
160
+ "holdout_sample_deterministic": null,
161
+ "holdout_sample_prefix_horizon": 4,
162
+ "holdout_sample_rmse": true,
163
+ "holdout_size": 2300,
164
+ "image_emb_dim": 512,
165
+ "init_device": "meta",
166
+ "input_norm": true,
167
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
168
+ "lang_emb_mode": "clip",
169
+ "lang_encoder_device": "auto",
170
+ "lang_obs_emb_dim": 256,
171
+ "log_every": 25,
172
+ "lora": false,
173
+ "lora_alpha": 32,
174
+ "lora_dropout": 0.0,
175
+ "lora_rank": 16,
176
+ "lora_target": [
177
+ "q_proj",
178
+ "k_proj",
179
+ "v_proj",
180
+ "o_proj",
181
+ "gate_proj",
182
+ "up_proj",
183
+ "down_proj"
184
+ ],
185
+ "lr": 0.0003,
186
+ "max_context_len": 1024,
187
+ "max_steps": 5000,
188
+ "min_lr_ratio": 0.1,
189
+ "model": {
190
+ "action_chunk_len": 10,
191
+ "latent_dim": 768
192
+ },
193
+ "model_dtype": "float32",
194
+ "n_heads": 16,
195
+ "n_kv_heads": null,
196
+ "n_layers": 2,
197
+ "norm_bias_weight_decay": null,
198
+ "norm_fit_batches": 0,
199
+ "num_workers": 24,
200
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
201
+ "obs_spec": {
202
+ "image_channels": 3,
203
+ "image_hw": [
204
+ 128,
205
+ 128
206
+ ],
207
+ "image_keys": [
208
+ "robot0_agentview_left_image",
209
+ "robot0_agentview_right_image",
210
+ "robot0_eye_in_hand_image"
211
+ ],
212
+ "lang_dim": 768,
213
+ "layout": "NHWC",
214
+ "low_dim_dims": [
215
+ 3,
216
+ 4,
217
+ 3,
218
+ 4,
219
+ 2
220
+ ],
221
+ "low_dim_keys": [
222
+ "robot0_base_to_eef_pos",
223
+ "robot0_base_to_eef_quat",
224
+ "robot0_base_pos",
225
+ "robot0_base_quat",
226
+ "robot0_gripper_qpos"
227
+ ]
228
+ },
229
+ "obs_stride": 4,
230
+ "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n2_d50_seed0",
231
+ "persistent_workers": true,
232
+ "precision": "bf16",
233
+ "pred_next_image_key_weights": {},
234
+ "pred_next_image_weight": 1.0,
235
+ "pred_next_lang_weight": 0.1,
236
+ "pred_next_mode": "terminal",
237
+ "pred_next_obs_offset": 1,
238
+ "pred_next_proprio_weight": 0.15,
239
+ "pred_next_steps": 4,
240
+ "pred_warmup_steps": 0,
241
+ "predictor_lr": 0.000425,
242
+ "prefetch_factor": 4,
243
+ "prepare_lang_cache_only": false,
244
+ "proprio_emb_dim": 128,
245
+ "recon": {
246
+ "enabled": false,
247
+ "params": {
248
+ "depth": 48,
249
+ "kernel_size": 5,
250
+ "mults": [
251
+ 2,
252
+ 3,
253
+ 4,
254
+ 4
255
+ ]
256
+ }
257
+ },
258
+ "recon_image_weight": 1.0,
259
+ "recon_lang_weight": 0.1,
260
+ "recon_obs": false,
261
+ "recon_proprio_weight": 1.0,
262
+ "resume": null,
263
+ "resume_final_lr": null,
264
+ "resume_lr_schedule": "checkpoint",
265
+ "robomimic_src": null,
266
+ "save_every": 500,
267
+ "save_holdout_traces": true,
268
+ "save_latest_checkpoint": true,
269
+ "scaling_action_head_d_model": 192,
270
+ "scaling_action_head_layers": 2,
271
+ "scaling_action_head_lr": 0.0003,
272
+ "scaling_backbone_d_model": 768,
273
+ "scaling_backbone_layers": 4,
274
+ "scaling_batch_policy": "phase1_grid_h100_v20260711",
275
+ "scaling_batch_policy_d50_microbatch": 48,
276
+ "scaling_encoder_d_model": 768,
277
+ "scaling_encoder_layers": 2,
278
+ "scaling_encoder_lr": 0.000425,
279
+ "scaling_expected_world_size": 1,
280
+ "scaling_global_batch_size": 192,
281
+ "scaling_group_b_lr_role": null,
282
+ "scaling_latent_dim": 768,
283
+ "scaling_lr_action_head": 0.0003,
284
+ "scaling_lr_base": null,
285
+ "scaling_lr_base_frozen": false,
286
+ "scaling_lr_encoder": 0.000425,
287
+ "scaling_lr_formula_size_axis": "discrete_family_lookup",
288
+ "scaling_lr_lookup_calibration": false,
289
+ "scaling_lr_lookup_model": "n2",
290
+ "scaling_lr_lookup_status": "confirmed",
291
+ "scaling_lr_predictor": 0.000425,
292
+ "scaling_lr_reference": null,
293
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
294
+ "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter",
295
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
296
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
297
+ "scaling_per_rank_microbatch": 48,
298
+ "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0",
299
+ "scaling_phase1_topology": "phase1_5x5_seed0_v20260711",
300
+ "scaling_predictor_lr": 0.000425,
301
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
302
+ "scaling_recipe": "nodyn",
303
+ "scaling_run_name": "phase1grid_d50_n2_nodyn_seed0_5k",
304
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04",
305
+ "scaling_suite": "phase1_grid",
306
+ "seed": 0,
307
+ "seq_len": 10,
308
+ "sequence_model": {
309
+ "backbone_type": "qwen2",
310
+ "hidden_dim": 768,
311
+ "params": {
312
+ "attn_impl": "sdpa",
313
+ "dropout": 0.1,
314
+ "ffn_hidden_size": 2048,
315
+ "input_norm": false,
316
+ "max_context_len": 1024,
317
+ "model_dtype": "float32",
318
+ "n_heads": 6,
319
+ "n_kv_heads": null,
320
+ "n_layers": 4
321
+ },
322
+ "type": "continuous_transformer"
323
+ },
324
+ "use_proprio": true,
325
+ "warmup_steps": 250,
326
+ "weight_decay": 0.0,
327
+ "z_bottleneck": {
328
+ "dim": null
329
+ },
330
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz"
331
+ }
04_robocasa_scaling/scaling_n2_d50_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n2_d50_seed0",
3
+ "original_run_name": "phase1grid_d50_n2_nodyn_seed0_5k_cuda4",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 50,
7
+ "N": 2,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 5000
13
+ }
04_robocasa_scaling/scaling_n3_d1000_seed0/config.json ADDED
@@ -0,0 +1,327 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 256,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 1024,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 8,
13
+ "n_layers": 4,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/E1_scaling_grid/configs_world4/e1_grid_d1000_n3_nodyn_seed0_100k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_data": false,
85
+ "dynamics": {
86
+ "enabled": false,
87
+ "params": {},
88
+ "type": "none"
89
+ },
90
+ "dynamics_decoder_type": "film",
91
+ "enable_pred_next": false,
92
+ "encoder": {
93
+ "params": {
94
+ "cnn_depth": 48,
95
+ "cnn_kernel": 5,
96
+ "cnn_mults": [
97
+ 2,
98
+ 3,
99
+ 4,
100
+ 8,
101
+ 16
102
+ ],
103
+ "d_model": 1024,
104
+ "dropout": 0.0,
105
+ "mlp_ratio": 4,
106
+ "n_fusion_layers": 4,
107
+ "n_heads": 8,
108
+ "readout_depth": 1,
109
+ "readout_mlp_ratio": 2,
110
+ "readout_queries": 4,
111
+ "rope_theta": 10000.0,
112
+ "use_proprio": true
113
+ },
114
+ "type": "attn_fusion_latent_token"
115
+ },
116
+ "encoder_lr": 0.0003,
117
+ "eval_at_start": false,
118
+ "eval_batch_size": 64,
119
+ "eval_crops_per_demo": 8,
120
+ "eval_every": 5000,
121
+ "eval_num_workers": 24,
122
+ "eval_seed": 0,
123
+ "ffn_hidden_size": 1024,
124
+ "filter_key": "wl23_mg1000_seed0",
125
+ "grad_accum_steps": 1,
126
+ "grad_clip": 1.0,
127
+ "hdf5_paths": [
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
151
+ ],
152
+ "holdout_action_mean_samples": 1,
153
+ "holdout_crops_per_demo": 8,
154
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
155
+ "holdout_per_task_metrics": true,
156
+ "holdout_rmse_samplers": [
157
+ "deterministic",
158
+ "stochastic"
159
+ ],
160
+ "holdout_sample_deterministic": null,
161
+ "holdout_sample_prefix_horizon": 4,
162
+ "holdout_sample_rmse": true,
163
+ "holdout_size": 2300,
164
+ "image_emb_dim": 512,
165
+ "init_device": "meta",
166
+ "input_norm": true,
167
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
168
+ "lang_emb_mode": "clip",
169
+ "lang_encoder_device": "auto",
170
+ "lang_obs_emb_dim": 256,
171
+ "log_every": 25,
172
+ "lora": false,
173
+ "lora_alpha": 32,
174
+ "lora_dropout": 0.0,
175
+ "lora_rank": 16,
176
+ "lora_target": [
177
+ "q_proj",
178
+ "k_proj",
179
+ "v_proj",
180
+ "o_proj",
181
+ "gate_proj",
182
+ "up_proj",
183
+ "down_proj"
184
+ ],
185
+ "lr": 0.0003,
186
+ "max_context_len": 1024,
187
+ "max_steps": 100000,
188
+ "min_lr_ratio": 0.1,
189
+ "model": {
190
+ "action_chunk_len": 10,
191
+ "latent_dim": 1024
192
+ },
193
+ "model_dtype": "float32",
194
+ "n_heads": 16,
195
+ "n_kv_heads": null,
196
+ "n_layers": 2,
197
+ "norm_bias_weight_decay": null,
198
+ "norm_fit_batches": 0,
199
+ "num_workers": 24,
200
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
201
+ "obs_spec": {
202
+ "image_channels": 3,
203
+ "image_hw": [
204
+ 128,
205
+ 128
206
+ ],
207
+ "image_keys": [
208
+ "robot0_agentview_left_image",
209
+ "robot0_agentview_right_image",
210
+ "robot0_eye_in_hand_image"
211
+ ],
212
+ "lang_dim": 768,
213
+ "layout": "NHWC",
214
+ "low_dim_dims": [
215
+ 3,
216
+ 4,
217
+ 3,
218
+ 4,
219
+ 2
220
+ ],
221
+ "low_dim_keys": [
222
+ "robot0_base_to_eef_pos",
223
+ "robot0_base_to_eef_quat",
224
+ "robot0_base_pos",
225
+ "robot0_base_quat",
226
+ "robot0_gripper_qpos"
227
+ ]
228
+ },
229
+ "obs_stride": 4,
230
+ "output_dir": "${EXPERIMENT_ROOT}/E1_scaling_grid/runs/e1_grid_d1000_n3_nodyn_seed0_100k_cuda0-4-5-7",
231
+ "persistent_workers": true,
232
+ "precision": "bf16",
233
+ "pred_next_image_key_weights": {},
234
+ "pred_next_image_weight": 1.0,
235
+ "pred_next_lang_weight": 0.1,
236
+ "pred_next_mode": "terminal",
237
+ "pred_next_obs_offset": 1,
238
+ "pred_next_proprio_weight": 0.15,
239
+ "pred_next_steps": 4,
240
+ "pred_warmup_steps": 0,
241
+ "predictor_lr": 0.0003,
242
+ "prefetch_factor": 4,
243
+ "prepare_lang_cache_only": false,
244
+ "proprio_emb_dim": 128,
245
+ "recon": {
246
+ "enabled": false,
247
+ "params": {
248
+ "depth": 48,
249
+ "kernel_size": 5,
250
+ "mults": [
251
+ 2,
252
+ 3,
253
+ 4,
254
+ 4
255
+ ]
256
+ }
257
+ },
258
+ "recon_image_weight": 1.0,
259
+ "recon_lang_weight": 0.1,
260
+ "recon_obs": false,
261
+ "recon_proprio_weight": 1.0,
262
+ "resume": null,
263
+ "resume_final_lr": null,
264
+ "resume_lr_schedule": "checkpoint",
265
+ "robomimic_src": null,
266
+ "save_every": 10000,
267
+ "save_holdout_traces": true,
268
+ "save_latest_checkpoint": true,
269
+ "scaling_action_head_layers": 4,
270
+ "scaling_action_head_width": 256,
271
+ "scaling_analysis_suites": [
272
+ "e1_grid"
273
+ ],
274
+ "scaling_backbone_layers": 8,
275
+ "scaling_encoder_layers": 4,
276
+ "scaling_expected_world_size": 4,
277
+ "scaling_family_width": 1024,
278
+ "scaling_global_batch_size": 192,
279
+ "scaling_grid_data_demos_per_task": 1000,
280
+ "scaling_grid_model_size": "n3",
281
+ "scaling_grid_protocol": "direct_5x5_seed0_first_pass_v20260711",
282
+ "scaling_lr_action_head": 0.0003,
283
+ "scaling_lr_encoder": 0.0003,
284
+ "scaling_lr_lookup_model": "n3",
285
+ "scaling_lr_lookup_status": "confirmed",
286
+ "scaling_lr_param_groups": [
287
+ "encoder",
288
+ "predictor",
289
+ "action_head"
290
+ ],
291
+ "scaling_lr_predictor": 0.0003,
292
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
293
+ "scaling_lr_table_version": "n_lookup_v20260712",
294
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
295
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
296
+ "scaling_per_rank_microbatch": 48,
297
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
298
+ "scaling_recipe": "nodyn",
299
+ "scaling_run_name": "e1_grid_d1000_n3_nodyn_seed0_100k",
300
+ "scaling_secondary_holdout_metric": "action_rmse",
301
+ "scaling_suite": "e1_grid",
302
+ "seed": 0,
303
+ "seq_len": 10,
304
+ "sequence_model": {
305
+ "backbone_type": "qwen2",
306
+ "hidden_dim": 1024,
307
+ "params": {
308
+ "attn_impl": "sdpa",
309
+ "dropout": 0.1,
310
+ "ffn_hidden_size": 2816,
311
+ "input_norm": false,
312
+ "max_context_len": 1024,
313
+ "model_dtype": "float32",
314
+ "n_heads": 8,
315
+ "n_kv_heads": null,
316
+ "n_layers": 8
317
+ },
318
+ "type": "continuous_transformer"
319
+ },
320
+ "use_proprio": true,
321
+ "warmup_steps": 5000,
322
+ "weight_decay": 0.0,
323
+ "z_bottleneck": {
324
+ "dim": null
325
+ },
326
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz"
327
+ }
04_robocasa_scaling/scaling_n3_d1000_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n3_d1000_seed0",
3
+ "original_run_name": "e1_grid_d1000_n3_nodyn_seed0_100k_cuda0-4-5-7",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 1000,
7
+ "N": 3,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 100000
13
+ }
04_robocasa_scaling/scaling_n3_d100_seed0/config.json ADDED
@@ -0,0 +1,327 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 256,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 1024,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 8,
13
+ "n_layers": 4,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/E1_scaling_grid/configs_world2/e1_grid_d100_n3_nodyn_seed0_10k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_data": false,
85
+ "dynamics": {
86
+ "enabled": false,
87
+ "params": {},
88
+ "type": "none"
89
+ },
90
+ "dynamics_decoder_type": "film",
91
+ "enable_pred_next": false,
92
+ "encoder": {
93
+ "params": {
94
+ "cnn_depth": 48,
95
+ "cnn_kernel": 5,
96
+ "cnn_mults": [
97
+ 2,
98
+ 3,
99
+ 4,
100
+ 8,
101
+ 16
102
+ ],
103
+ "d_model": 1024,
104
+ "dropout": 0.0,
105
+ "mlp_ratio": 4,
106
+ "n_fusion_layers": 4,
107
+ "n_heads": 8,
108
+ "readout_depth": 1,
109
+ "readout_mlp_ratio": 2,
110
+ "readout_queries": 4,
111
+ "rope_theta": 10000.0,
112
+ "use_proprio": true
113
+ },
114
+ "type": "attn_fusion_latent_token"
115
+ },
116
+ "encoder_lr": 0.0003,
117
+ "eval_at_start": false,
118
+ "eval_batch_size": 64,
119
+ "eval_crops_per_demo": 8,
120
+ "eval_every": 500,
121
+ "eval_num_workers": 24,
122
+ "eval_seed": 0,
123
+ "ffn_hidden_size": 1024,
124
+ "filter_key": "wl23_mg100_seed0",
125
+ "grad_accum_steps": 2,
126
+ "grad_clip": 1.0,
127
+ "hdf5_paths": [
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
151
+ ],
152
+ "holdout_action_mean_samples": 1,
153
+ "holdout_crops_per_demo": 8,
154
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
155
+ "holdout_per_task_metrics": true,
156
+ "holdout_rmse_samplers": [
157
+ "deterministic",
158
+ "stochastic"
159
+ ],
160
+ "holdout_sample_deterministic": null,
161
+ "holdout_sample_prefix_horizon": 4,
162
+ "holdout_sample_rmse": true,
163
+ "holdout_size": 2300,
164
+ "image_emb_dim": 512,
165
+ "init_device": "meta",
166
+ "input_norm": true,
167
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
168
+ "lang_emb_mode": "clip",
169
+ "lang_encoder_device": "auto",
170
+ "lang_obs_emb_dim": 256,
171
+ "log_every": 25,
172
+ "lora": false,
173
+ "lora_alpha": 32,
174
+ "lora_dropout": 0.0,
175
+ "lora_rank": 16,
176
+ "lora_target": [
177
+ "q_proj",
178
+ "k_proj",
179
+ "v_proj",
180
+ "o_proj",
181
+ "gate_proj",
182
+ "up_proj",
183
+ "down_proj"
184
+ ],
185
+ "lr": 0.0003,
186
+ "max_context_len": 1024,
187
+ "max_steps": 10000,
188
+ "min_lr_ratio": 0.1,
189
+ "model": {
190
+ "action_chunk_len": 10,
191
+ "latent_dim": 1024
192
+ },
193
+ "model_dtype": "float32",
194
+ "n_heads": 16,
195
+ "n_kv_heads": null,
196
+ "n_layers": 2,
197
+ "norm_bias_weight_decay": null,
198
+ "norm_fit_batches": 0,
199
+ "num_workers": 24,
200
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
201
+ "obs_spec": {
202
+ "image_channels": 3,
203
+ "image_hw": [
204
+ 128,
205
+ 128
206
+ ],
207
+ "image_keys": [
208
+ "robot0_agentview_left_image",
209
+ "robot0_agentview_right_image",
210
+ "robot0_eye_in_hand_image"
211
+ ],
212
+ "lang_dim": 768,
213
+ "layout": "NHWC",
214
+ "low_dim_dims": [
215
+ 3,
216
+ 4,
217
+ 3,
218
+ 4,
219
+ 2
220
+ ],
221
+ "low_dim_keys": [
222
+ "robot0_base_to_eef_pos",
223
+ "robot0_base_to_eef_quat",
224
+ "robot0_base_pos",
225
+ "robot0_base_quat",
226
+ "robot0_gripper_qpos"
227
+ ]
228
+ },
229
+ "obs_stride": 4,
230
+ "output_dir": "${EXPERIMENT_ROOT}/E1_scaling_grid/runs/e1_grid_d100_n3_nodyn_seed0_10k_cuda2-3",
231
+ "persistent_workers": true,
232
+ "precision": "bf16",
233
+ "pred_next_image_key_weights": {},
234
+ "pred_next_image_weight": 1.0,
235
+ "pred_next_lang_weight": 0.1,
236
+ "pred_next_mode": "terminal",
237
+ "pred_next_obs_offset": 1,
238
+ "pred_next_proprio_weight": 0.15,
239
+ "pred_next_steps": 4,
240
+ "pred_warmup_steps": 0,
241
+ "predictor_lr": 0.0003,
242
+ "prefetch_factor": 4,
243
+ "prepare_lang_cache_only": false,
244
+ "proprio_emb_dim": 128,
245
+ "recon": {
246
+ "enabled": false,
247
+ "params": {
248
+ "depth": 48,
249
+ "kernel_size": 5,
250
+ "mults": [
251
+ 2,
252
+ 3,
253
+ 4,
254
+ 4
255
+ ]
256
+ }
257
+ },
258
+ "recon_image_weight": 1.0,
259
+ "recon_lang_weight": 0.1,
260
+ "recon_obs": false,
261
+ "recon_proprio_weight": 1.0,
262
+ "resume": null,
263
+ "resume_final_lr": null,
264
+ "resume_lr_schedule": "checkpoint",
265
+ "robomimic_src": null,
266
+ "save_every": 1000,
267
+ "save_holdout_traces": true,
268
+ "save_latest_checkpoint": true,
269
+ "scaling_action_head_layers": 4,
270
+ "scaling_action_head_width": 256,
271
+ "scaling_analysis_suites": [
272
+ "e1_grid"
273
+ ],
274
+ "scaling_backbone_layers": 8,
275
+ "scaling_encoder_layers": 4,
276
+ "scaling_expected_world_size": 2,
277
+ "scaling_family_width": 1024,
278
+ "scaling_global_batch_size": 192,
279
+ "scaling_grid_data_demos_per_task": 100,
280
+ "scaling_grid_model_size": "n3",
281
+ "scaling_grid_protocol": "direct_5x5_seed0_first_pass_v20260711",
282
+ "scaling_lr_action_head": 0.0003,
283
+ "scaling_lr_encoder": 0.0003,
284
+ "scaling_lr_lookup_model": "n3",
285
+ "scaling_lr_lookup_status": "confirmed",
286
+ "scaling_lr_param_groups": [
287
+ "encoder",
288
+ "predictor",
289
+ "action_head"
290
+ ],
291
+ "scaling_lr_predictor": 0.0003,
292
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
293
+ "scaling_lr_table_version": "n_lookup_v20260711",
294
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
295
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
296
+ "scaling_per_rank_microbatch": 48,
297
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
298
+ "scaling_recipe": "nodyn",
299
+ "scaling_run_name": "e1_grid_d100_n3_nodyn_seed0_10k",
300
+ "scaling_secondary_holdout_metric": "action_rmse",
301
+ "scaling_suite": "e1_grid",
302
+ "seed": 0,
303
+ "seq_len": 10,
304
+ "sequence_model": {
305
+ "backbone_type": "qwen2",
306
+ "hidden_dim": 1024,
307
+ "params": {
308
+ "attn_impl": "sdpa",
309
+ "dropout": 0.1,
310
+ "ffn_hidden_size": 2816,
311
+ "input_norm": false,
312
+ "max_context_len": 1024,
313
+ "model_dtype": "float32",
314
+ "n_heads": 8,
315
+ "n_kv_heads": null,
316
+ "n_layers": 8
317
+ },
318
+ "type": "continuous_transformer"
319
+ },
320
+ "use_proprio": true,
321
+ "warmup_steps": 500,
322
+ "weight_decay": 0.0,
323
+ "z_bottleneck": {
324
+ "dim": null
325
+ },
326
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz"
327
+ }
04_robocasa_scaling/scaling_n3_d100_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n3_d100_seed0",
3
+ "original_run_name": "e1_grid_d100_n3_nodyn_seed0_10k_cuda2-3",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 100,
7
+ "N": 3,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 10000
13
+ }
04_robocasa_scaling/scaling_n3_d2900_seed0/config.json ADDED
@@ -0,0 +1,327 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 256,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 1024,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 8,
13
+ "n_layers": 4,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/E1_scaling_grid/configs_world4/e1_grid_d2900_n3_nodyn_seed0_280k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_data": false,
85
+ "dynamics": {
86
+ "enabled": false,
87
+ "params": {},
88
+ "type": "none"
89
+ },
90
+ "dynamics_decoder_type": "film",
91
+ "enable_pred_next": false,
92
+ "encoder": {
93
+ "params": {
94
+ "cnn_depth": 48,
95
+ "cnn_kernel": 5,
96
+ "cnn_mults": [
97
+ 2,
98
+ 3,
99
+ 4,
100
+ 8,
101
+ 16
102
+ ],
103
+ "d_model": 1024,
104
+ "dropout": 0.0,
105
+ "mlp_ratio": 4,
106
+ "n_fusion_layers": 4,
107
+ "n_heads": 8,
108
+ "readout_depth": 1,
109
+ "readout_mlp_ratio": 2,
110
+ "readout_queries": 4,
111
+ "rope_theta": 10000.0,
112
+ "use_proprio": true
113
+ },
114
+ "type": "attn_fusion_latent_token"
115
+ },
116
+ "encoder_lr": 0.0003,
117
+ "eval_at_start": false,
118
+ "eval_batch_size": 64,
119
+ "eval_crops_per_demo": 8,
120
+ "eval_every": 14000,
121
+ "eval_num_workers": 24,
122
+ "eval_seed": 0,
123
+ "ffn_hidden_size": 1024,
124
+ "filter_key": "wl23_mg2900_seed0",
125
+ "grad_accum_steps": 1,
126
+ "grad_clip": 1.0,
127
+ "hdf5_paths": [
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
151
+ ],
152
+ "holdout_action_mean_samples": 1,
153
+ "holdout_crops_per_demo": 8,
154
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
155
+ "holdout_per_task_metrics": true,
156
+ "holdout_rmse_samplers": [
157
+ "deterministic",
158
+ "stochastic"
159
+ ],
160
+ "holdout_sample_deterministic": null,
161
+ "holdout_sample_prefix_horizon": 4,
162
+ "holdout_sample_rmse": true,
163
+ "holdout_size": 2300,
164
+ "image_emb_dim": 512,
165
+ "init_device": "meta",
166
+ "input_norm": true,
167
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
168
+ "lang_emb_mode": "clip",
169
+ "lang_encoder_device": "auto",
170
+ "lang_obs_emb_dim": 256,
171
+ "log_every": 25,
172
+ "lora": false,
173
+ "lora_alpha": 32,
174
+ "lora_dropout": 0.0,
175
+ "lora_rank": 16,
176
+ "lora_target": [
177
+ "q_proj",
178
+ "k_proj",
179
+ "v_proj",
180
+ "o_proj",
181
+ "gate_proj",
182
+ "up_proj",
183
+ "down_proj"
184
+ ],
185
+ "lr": 0.0003,
186
+ "max_context_len": 1024,
187
+ "max_steps": 280000,
188
+ "min_lr_ratio": 0.1,
189
+ "model": {
190
+ "action_chunk_len": 10,
191
+ "latent_dim": 1024
192
+ },
193
+ "model_dtype": "float32",
194
+ "n_heads": 16,
195
+ "n_kv_heads": null,
196
+ "n_layers": 2,
197
+ "norm_bias_weight_decay": null,
198
+ "norm_fit_batches": 0,
199
+ "num_workers": 24,
200
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
201
+ "obs_spec": {
202
+ "image_channels": 3,
203
+ "image_hw": [
204
+ 128,
205
+ 128
206
+ ],
207
+ "image_keys": [
208
+ "robot0_agentview_left_image",
209
+ "robot0_agentview_right_image",
210
+ "robot0_eye_in_hand_image"
211
+ ],
212
+ "lang_dim": 768,
213
+ "layout": "NHWC",
214
+ "low_dim_dims": [
215
+ 3,
216
+ 4,
217
+ 3,
218
+ 4,
219
+ 2
220
+ ],
221
+ "low_dim_keys": [
222
+ "robot0_base_to_eef_pos",
223
+ "robot0_base_to_eef_quat",
224
+ "robot0_base_pos",
225
+ "robot0_base_quat",
226
+ "robot0_gripper_qpos"
227
+ ]
228
+ },
229
+ "obs_stride": 4,
230
+ "output_dir": "${EXPERIMENT_ROOT}/E1_scaling_grid/runs/e1_grid_d2900_n3_nodyn_seed0_280k_cuda1-2-3-6",
231
+ "persistent_workers": true,
232
+ "precision": "bf16",
233
+ "pred_next_image_key_weights": {},
234
+ "pred_next_image_weight": 1.0,
235
+ "pred_next_lang_weight": 0.1,
236
+ "pred_next_mode": "terminal",
237
+ "pred_next_obs_offset": 1,
238
+ "pred_next_proprio_weight": 0.15,
239
+ "pred_next_steps": 4,
240
+ "pred_warmup_steps": 0,
241
+ "predictor_lr": 0.0003,
242
+ "prefetch_factor": 4,
243
+ "prepare_lang_cache_only": false,
244
+ "proprio_emb_dim": 128,
245
+ "recon": {
246
+ "enabled": false,
247
+ "params": {
248
+ "depth": 48,
249
+ "kernel_size": 5,
250
+ "mults": [
251
+ 2,
252
+ 3,
253
+ 4,
254
+ 4
255
+ ]
256
+ }
257
+ },
258
+ "recon_image_weight": 1.0,
259
+ "recon_lang_weight": 0.1,
260
+ "recon_obs": false,
261
+ "recon_proprio_weight": 1.0,
262
+ "resume": null,
263
+ "resume_final_lr": null,
264
+ "resume_lr_schedule": "checkpoint",
265
+ "robomimic_src": null,
266
+ "save_every": 28000,
267
+ "save_holdout_traces": true,
268
+ "save_latest_checkpoint": true,
269
+ "scaling_action_head_layers": 4,
270
+ "scaling_action_head_width": 256,
271
+ "scaling_analysis_suites": [
272
+ "e1_grid"
273
+ ],
274
+ "scaling_backbone_layers": 8,
275
+ "scaling_encoder_layers": 4,
276
+ "scaling_expected_world_size": 4,
277
+ "scaling_family_width": 1024,
278
+ "scaling_global_batch_size": 192,
279
+ "scaling_grid_data_demos_per_task": 2900,
280
+ "scaling_grid_model_size": "n3",
281
+ "scaling_grid_protocol": "direct_5x5_seed0_first_pass_v20260711",
282
+ "scaling_lr_action_head": 0.0003,
283
+ "scaling_lr_encoder": 0.0003,
284
+ "scaling_lr_lookup_model": "n3",
285
+ "scaling_lr_lookup_status": "confirmed",
286
+ "scaling_lr_param_groups": [
287
+ "encoder",
288
+ "predictor",
289
+ "action_head"
290
+ ],
291
+ "scaling_lr_predictor": 0.0003,
292
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
293
+ "scaling_lr_table_version": "n_lookup_v20260711",
294
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
295
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
296
+ "scaling_per_rank_microbatch": 48,
297
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
298
+ "scaling_recipe": "nodyn",
299
+ "scaling_run_name": "e1_grid_d2900_n3_nodyn_seed0_280k",
300
+ "scaling_secondary_holdout_metric": "action_rmse",
301
+ "scaling_suite": "e1_grid",
302
+ "seed": 0,
303
+ "seq_len": 10,
304
+ "sequence_model": {
305
+ "backbone_type": "qwen2",
306
+ "hidden_dim": 1024,
307
+ "params": {
308
+ "attn_impl": "sdpa",
309
+ "dropout": 0.1,
310
+ "ffn_hidden_size": 2816,
311
+ "input_norm": false,
312
+ "max_context_len": 1024,
313
+ "model_dtype": "float32",
314
+ "n_heads": 8,
315
+ "n_kv_heads": null,
316
+ "n_layers": 8
317
+ },
318
+ "type": "continuous_transformer"
319
+ },
320
+ "use_proprio": true,
321
+ "warmup_steps": 14000,
322
+ "weight_decay": 0.0,
323
+ "z_bottleneck": {
324
+ "dim": null
325
+ },
326
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz"
327
+ }
04_robocasa_scaling/scaling_n3_d2900_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n3_d2900_seed0",
3
+ "original_run_name": "e1_grid_d2900_n3_nodyn_seed0_280k_cuda1-2-3-6",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 2900,
7
+ "N": 3,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 280000
13
+ }
04_robocasa_scaling/scaling_n3_d300_seed0/config.json ADDED
@@ -0,0 +1,328 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 256,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 1024,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 8,
13
+ "n_layers": 4,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/E1_scaling_grid/configs_world1/e1_grid_d300_n3_nodyn_seed0_30k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_data": false,
85
+ "dynamics": {
86
+ "enabled": false,
87
+ "params": {},
88
+ "type": "none"
89
+ },
90
+ "dynamics_decoder_type": "film",
91
+ "enable_pred_next": false,
92
+ "encoder": {
93
+ "params": {
94
+ "cnn_depth": 48,
95
+ "cnn_kernel": 5,
96
+ "cnn_mults": [
97
+ 2,
98
+ 3,
99
+ 4,
100
+ 8,
101
+ 16
102
+ ],
103
+ "d_model": 1024,
104
+ "dropout": 0.0,
105
+ "mlp_ratio": 4,
106
+ "n_fusion_layers": 4,
107
+ "n_heads": 8,
108
+ "readout_depth": 1,
109
+ "readout_mlp_ratio": 2,
110
+ "readout_queries": 4,
111
+ "rope_theta": 10000.0,
112
+ "use_proprio": true
113
+ },
114
+ "type": "attn_fusion_latent_token"
115
+ },
116
+ "encoder_lr": 0.0003,
117
+ "eval_at_start": false,
118
+ "eval_batch_size": 64,
119
+ "eval_crops_per_demo": 8,
120
+ "eval_every": 1500,
121
+ "eval_num_workers": 24,
122
+ "eval_seed": 0,
123
+ "ffn_hidden_size": 1024,
124
+ "filter_key": "300_demos",
125
+ "grad_accum_steps": 4,
126
+ "grad_clip": 1.0,
127
+ "hdf5_paths": [
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
151
+ ],
152
+ "holdout_action_mean_samples": 1,
153
+ "holdout_crops_per_demo": 8,
154
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
155
+ "holdout_per_task_metrics": true,
156
+ "holdout_rmse_samplers": [
157
+ "deterministic",
158
+ "stochastic"
159
+ ],
160
+ "holdout_sample_deterministic": null,
161
+ "holdout_sample_prefix_horizon": 4,
162
+ "holdout_sample_rmse": true,
163
+ "holdout_size": 2300,
164
+ "image_emb_dim": 512,
165
+ "init_device": "meta",
166
+ "input_norm": true,
167
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
168
+ "lang_emb_mode": "clip",
169
+ "lang_encoder_device": "auto",
170
+ "lang_obs_emb_dim": 256,
171
+ "log_every": 25,
172
+ "lora": false,
173
+ "lora_alpha": 32,
174
+ "lora_dropout": 0.0,
175
+ "lora_rank": 16,
176
+ "lora_target": [
177
+ "q_proj",
178
+ "k_proj",
179
+ "v_proj",
180
+ "o_proj",
181
+ "gate_proj",
182
+ "up_proj",
183
+ "down_proj"
184
+ ],
185
+ "lr": 0.0003,
186
+ "max_context_len": 1024,
187
+ "max_steps": 30000,
188
+ "min_lr_ratio": 0.1,
189
+ "model": {
190
+ "action_chunk_len": 10,
191
+ "latent_dim": 1024
192
+ },
193
+ "model_dtype": "float32",
194
+ "n_heads": 16,
195
+ "n_kv_heads": null,
196
+ "n_layers": 2,
197
+ "norm_bias_weight_decay": null,
198
+ "norm_fit_batches": 0,
199
+ "num_workers": 24,
200
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
201
+ "obs_spec": {
202
+ "image_channels": 3,
203
+ "image_hw": [
204
+ 128,
205
+ 128
206
+ ],
207
+ "image_keys": [
208
+ "robot0_agentview_left_image",
209
+ "robot0_agentview_right_image",
210
+ "robot0_eye_in_hand_image"
211
+ ],
212
+ "lang_dim": 768,
213
+ "layout": "NHWC",
214
+ "low_dim_dims": [
215
+ 3,
216
+ 4,
217
+ 3,
218
+ 4,
219
+ 2
220
+ ],
221
+ "low_dim_keys": [
222
+ "robot0_base_to_eef_pos",
223
+ "robot0_base_to_eef_quat",
224
+ "robot0_base_pos",
225
+ "robot0_base_quat",
226
+ "robot0_gripper_qpos"
227
+ ]
228
+ },
229
+ "obs_stride": 4,
230
+ "output_dir": "${EXPERIMENT_ROOT}/E1_scaling_grid/runs/e1_grid_d300_n3_nodyn_seed0_30k_cuda0",
231
+ "persistent_workers": true,
232
+ "precision": "bf16",
233
+ "pred_next_image_key_weights": {},
234
+ "pred_next_image_weight": 1.0,
235
+ "pred_next_lang_weight": 0.1,
236
+ "pred_next_mode": "terminal",
237
+ "pred_next_obs_offset": 1,
238
+ "pred_next_proprio_weight": 0.15,
239
+ "pred_next_steps": 4,
240
+ "pred_warmup_steps": 0,
241
+ "predictor_lr": 0.0003,
242
+ "prefetch_factor": 4,
243
+ "prepare_lang_cache_only": false,
244
+ "proprio_emb_dim": 128,
245
+ "recon": {
246
+ "enabled": false,
247
+ "params": {
248
+ "depth": 48,
249
+ "kernel_size": 5,
250
+ "mults": [
251
+ 2,
252
+ 3,
253
+ 4,
254
+ 4
255
+ ]
256
+ }
257
+ },
258
+ "recon_image_weight": 1.0,
259
+ "recon_lang_weight": 0.1,
260
+ "recon_obs": false,
261
+ "recon_proprio_weight": 1.0,
262
+ "resume": null,
263
+ "resume_final_lr": null,
264
+ "resume_lr_schedule": "checkpoint",
265
+ "robomimic_src": null,
266
+ "save_every": 3000,
267
+ "save_holdout_traces": true,
268
+ "save_latest_checkpoint": true,
269
+ "scaling_action_head_layers": 4,
270
+ "scaling_action_head_width": 256,
271
+ "scaling_analysis_suites": [
272
+ "e1_grid"
273
+ ],
274
+ "scaling_backbone_layers": 8,
275
+ "scaling_encoder_layers": 4,
276
+ "scaling_expected_world_size": 1,
277
+ "scaling_family_width": 1024,
278
+ "scaling_global_batch_size": 192,
279
+ "scaling_grid_data_demos_per_task": 300,
280
+ "scaling_grid_model_size": "n3",
281
+ "scaling_grid_protocol": "direct_5x5_seed0_first_pass_v20260711",
282
+ "scaling_lr_action_head": 0.0003,
283
+ "scaling_lr_encoder": 0.0003,
284
+ "scaling_lr_lookup_model": "n3",
285
+ "scaling_lr_lookup_status": "confirmed",
286
+ "scaling_lr_param_groups": [
287
+ "encoder",
288
+ "predictor",
289
+ "action_head"
290
+ ],
291
+ "scaling_lr_predictor": 0.0003,
292
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
293
+ "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter",
294
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
295
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
296
+ "scaling_model_size": "n3",
297
+ "scaling_per_rank_microbatch": 48,
298
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
299
+ "scaling_recipe": "nodyn",
300
+ "scaling_run_name": "e1_grid_d300_n3_nodyn_seed0_30k",
301
+ "scaling_secondary_holdout_metric": "action_rmse",
302
+ "scaling_suite": "e1_grid",
303
+ "seed": 0,
304
+ "seq_len": 10,
305
+ "sequence_model": {
306
+ "backbone_type": "qwen2",
307
+ "hidden_dim": 1024,
308
+ "params": {
309
+ "attn_impl": "sdpa",
310
+ "dropout": 0.1,
311
+ "ffn_hidden_size": 2816,
312
+ "input_norm": false,
313
+ "max_context_len": 1024,
314
+ "model_dtype": "float32",
315
+ "n_heads": 8,
316
+ "n_kv_heads": null,
317
+ "n_layers": 8
318
+ },
319
+ "type": "continuous_transformer"
320
+ },
321
+ "use_proprio": true,
322
+ "warmup_steps": 1500,
323
+ "weight_decay": 0.0,
324
+ "z_bottleneck": {
325
+ "dim": null
326
+ },
327
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz"
328
+ }
04_robocasa_scaling/scaling_n3_d300_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n3_d300_seed0",
3
+ "original_run_name": "e1_grid_d300_n3_nodyn_seed0_30k_cuda0",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 300,
7
+ "N": 3,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 30000
13
+ }
04_robocasa_scaling/scaling_n3_d50_seed0/config.json ADDED
@@ -0,0 +1,327 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 256,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 1024,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 8,
13
+ "n_layers": 4,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/E1_scaling_grid/configs_world1/e1_grid_d50_n3_nodyn_seed0_5k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_data": false,
85
+ "dynamics": {
86
+ "enabled": false,
87
+ "params": {},
88
+ "type": "none"
89
+ },
90
+ "dynamics_decoder_type": "film",
91
+ "enable_pred_next": false,
92
+ "encoder": {
93
+ "params": {
94
+ "cnn_depth": 48,
95
+ "cnn_kernel": 5,
96
+ "cnn_mults": [
97
+ 2,
98
+ 3,
99
+ 4,
100
+ 8,
101
+ 16
102
+ ],
103
+ "d_model": 1024,
104
+ "dropout": 0.0,
105
+ "mlp_ratio": 4,
106
+ "n_fusion_layers": 4,
107
+ "n_heads": 8,
108
+ "readout_depth": 1,
109
+ "readout_mlp_ratio": 2,
110
+ "readout_queries": 4,
111
+ "rope_theta": 10000.0,
112
+ "use_proprio": true
113
+ },
114
+ "type": "attn_fusion_latent_token"
115
+ },
116
+ "encoder_lr": 0.0003,
117
+ "eval_at_start": false,
118
+ "eval_batch_size": 64,
119
+ "eval_crops_per_demo": 8,
120
+ "eval_every": 250,
121
+ "eval_num_workers": 24,
122
+ "eval_seed": 0,
123
+ "ffn_hidden_size": 1024,
124
+ "filter_key": "wl23_mg50_seed0",
125
+ "grad_accum_steps": 4,
126
+ "grad_clip": 1.0,
127
+ "hdf5_paths": [
128
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
129
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
130
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
151
+ ],
152
+ "holdout_action_mean_samples": 1,
153
+ "holdout_crops_per_demo": 8,
154
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
155
+ "holdout_per_task_metrics": true,
156
+ "holdout_rmse_samplers": [
157
+ "deterministic",
158
+ "stochastic"
159
+ ],
160
+ "holdout_sample_deterministic": null,
161
+ "holdout_sample_prefix_horizon": 4,
162
+ "holdout_sample_rmse": true,
163
+ "holdout_size": 2300,
164
+ "image_emb_dim": 512,
165
+ "init_device": "meta",
166
+ "input_norm": true,
167
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
168
+ "lang_emb_mode": "clip",
169
+ "lang_encoder_device": "auto",
170
+ "lang_obs_emb_dim": 256,
171
+ "log_every": 25,
172
+ "lora": false,
173
+ "lora_alpha": 32,
174
+ "lora_dropout": 0.0,
175
+ "lora_rank": 16,
176
+ "lora_target": [
177
+ "q_proj",
178
+ "k_proj",
179
+ "v_proj",
180
+ "o_proj",
181
+ "gate_proj",
182
+ "up_proj",
183
+ "down_proj"
184
+ ],
185
+ "lr": 0.0003,
186
+ "max_context_len": 1024,
187
+ "max_steps": 5000,
188
+ "min_lr_ratio": 0.1,
189
+ "model": {
190
+ "action_chunk_len": 10,
191
+ "latent_dim": 1024
192
+ },
193
+ "model_dtype": "float32",
194
+ "n_heads": 16,
195
+ "n_kv_heads": null,
196
+ "n_layers": 2,
197
+ "norm_bias_weight_decay": null,
198
+ "norm_fit_batches": 0,
199
+ "num_workers": 24,
200
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
201
+ "obs_spec": {
202
+ "image_channels": 3,
203
+ "image_hw": [
204
+ 128,
205
+ 128
206
+ ],
207
+ "image_keys": [
208
+ "robot0_agentview_left_image",
209
+ "robot0_agentview_right_image",
210
+ "robot0_eye_in_hand_image"
211
+ ],
212
+ "lang_dim": 768,
213
+ "layout": "NHWC",
214
+ "low_dim_dims": [
215
+ 3,
216
+ 4,
217
+ 3,
218
+ 4,
219
+ 2
220
+ ],
221
+ "low_dim_keys": [
222
+ "robot0_base_to_eef_pos",
223
+ "robot0_base_to_eef_quat",
224
+ "robot0_base_pos",
225
+ "robot0_base_quat",
226
+ "robot0_gripper_qpos"
227
+ ]
228
+ },
229
+ "obs_stride": 4,
230
+ "output_dir": "${EXPERIMENT_ROOT}/E1_scaling_grid/runs/e1_grid_d50_n3_nodyn_seed0_5k_cuda6",
231
+ "persistent_workers": true,
232
+ "precision": "bf16",
233
+ "pred_next_image_key_weights": {},
234
+ "pred_next_image_weight": 1.0,
235
+ "pred_next_lang_weight": 0.1,
236
+ "pred_next_mode": "terminal",
237
+ "pred_next_obs_offset": 1,
238
+ "pred_next_proprio_weight": 0.15,
239
+ "pred_next_steps": 4,
240
+ "pred_warmup_steps": 0,
241
+ "predictor_lr": 0.0003,
242
+ "prefetch_factor": 4,
243
+ "prepare_lang_cache_only": false,
244
+ "proprio_emb_dim": 128,
245
+ "recon": {
246
+ "enabled": false,
247
+ "params": {
248
+ "depth": 48,
249
+ "kernel_size": 5,
250
+ "mults": [
251
+ 2,
252
+ 3,
253
+ 4,
254
+ 4
255
+ ]
256
+ }
257
+ },
258
+ "recon_image_weight": 1.0,
259
+ "recon_lang_weight": 0.1,
260
+ "recon_obs": false,
261
+ "recon_proprio_weight": 1.0,
262
+ "resume": null,
263
+ "resume_final_lr": null,
264
+ "resume_lr_schedule": "checkpoint",
265
+ "robomimic_src": null,
266
+ "save_every": 500,
267
+ "save_holdout_traces": true,
268
+ "save_latest_checkpoint": true,
269
+ "scaling_action_head_layers": 4,
270
+ "scaling_action_head_width": 256,
271
+ "scaling_analysis_suites": [
272
+ "e1_grid"
273
+ ],
274
+ "scaling_backbone_layers": 8,
275
+ "scaling_encoder_layers": 4,
276
+ "scaling_expected_world_size": 1,
277
+ "scaling_family_width": 1024,
278
+ "scaling_global_batch_size": 192,
279
+ "scaling_grid_data_demos_per_task": 50,
280
+ "scaling_grid_model_size": "n3",
281
+ "scaling_grid_protocol": "direct_5x5_seed0_first_pass_v20260711",
282
+ "scaling_lr_action_head": 0.0003,
283
+ "scaling_lr_encoder": 0.0003,
284
+ "scaling_lr_lookup_model": "n3",
285
+ "scaling_lr_lookup_status": "confirmed",
286
+ "scaling_lr_param_groups": [
287
+ "encoder",
288
+ "predictor",
289
+ "action_head"
290
+ ],
291
+ "scaling_lr_predictor": 0.0003,
292
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
293
+ "scaling_lr_table_version": "n_lookup_v20260711",
294
+ "scaling_model_axis": "full_model_excluding_cnn_stem",
295
+ "scaling_model_family": "equiscale_encoder_backbone_head_v20260708",
296
+ "scaling_per_rank_microbatch": 48,
297
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
298
+ "scaling_recipe": "nodyn",
299
+ "scaling_run_name": "e1_grid_d50_n3_nodyn_seed0_5k",
300
+ "scaling_secondary_holdout_metric": "action_rmse",
301
+ "scaling_suite": "e1_grid",
302
+ "seed": 0,
303
+ "seq_len": 10,
304
+ "sequence_model": {
305
+ "backbone_type": "qwen2",
306
+ "hidden_dim": 1024,
307
+ "params": {
308
+ "attn_impl": "sdpa",
309
+ "dropout": 0.1,
310
+ "ffn_hidden_size": 2816,
311
+ "input_norm": false,
312
+ "max_context_len": 1024,
313
+ "model_dtype": "float32",
314
+ "n_heads": 8,
315
+ "n_kv_heads": null,
316
+ "n_layers": 8
317
+ },
318
+ "type": "continuous_transformer"
319
+ },
320
+ "use_proprio": true,
321
+ "warmup_steps": 250,
322
+ "weight_decay": 0.0,
323
+ "z_bottleneck": {
324
+ "dim": null
325
+ },
326
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz"
327
+ }
04_robocasa_scaling/scaling_n3_d50_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "scaling_n3_d50_seed0",
3
+ "original_run_name": "e1_grid_d50_n3_nodyn_seed0_5k_cuda6",
4
+ "paper_section": 4,
5
+ "parameters": {
6
+ "D": 50,
7
+ "N": 3,
8
+ "K": 1,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 5000
13
+ }
05_token_interface/README.md ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Section 5 — Token interface
2
+
3
+ Paper: **Do more tokens per timestep improve performance?**
4
+
5
+ Ten final checkpoints: N2, K∈{4,50}, seed 0, C_train=10, across all five data settings.
6
+
7
+ | Run | Training steps | Checkpoint file |
8
+ |---|---:|---|
9
+ | [multitoken_n2_k4_d50_seed0](multitoken_n2_k4_d50_seed0/) | 5,000 | `checkpoint_step_00005000.pt` |
10
+ | [rawtoken_n2_k50_d50_seed0](rawtoken_n2_k50_d50_seed0/) | 5,000 | `checkpoint_step_00005000.pt` |
11
+ | [multitoken_n2_k4_d100_seed0](multitoken_n2_k4_d100_seed0/) | 10,000 | `checkpoint_step_00010000.pt` |
12
+ | [rawtoken_n2_k50_d100_seed0](rawtoken_n2_k50_d100_seed0/) | 10,000 | `checkpoint_step_00010000.pt` |
13
+ | [multitoken_n2_k4_d300_seed0](multitoken_n2_k4_d300_seed0/) | 30,000 | `checkpoint_step_00030000.pt` |
14
+ | [rawtoken_n2_k50_d300_seed0](rawtoken_n2_k50_d300_seed0/) | 30,000 | `checkpoint_step_00030000.pt` |
15
+ | [multitoken_n2_k4_d1000_seed0](multitoken_n2_k4_d1000_seed0/) | 100,000 | `checkpoint_step_00100000.pt` |
16
+ | [rawtoken_n2_k50_d1000_seed0](rawtoken_n2_k50_d1000_seed0/) | 100,000 | `checkpoint_step_00100000.pt` |
17
+ | [multitoken_n2_k4_d2900_seed0](multitoken_n2_k4_d2900_seed0/) | 280,000 | `checkpoint_step_00280000.pt` |
18
+ | [rawtoken_n2_k50_d2900_seed0](rawtoken_n2_k50_d2900_seed0/) | 280,000 | `checkpoint_step_00280000.pt` |
19
+
20
+ The K=1 controls reuse the N2 / seed-0 checkpoints in [Section 4](../04_robocasa_scaling/README.md).
05_token_interface/multitoken_n2_k4_d1000_seed0/config.json ADDED
@@ -0,0 +1,337 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 192,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 768,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 6,
13
+ "n_layers": 2,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/E_multitoken_n2/configs_world4/multitoken_d1000_n2_k4_nodyn_seed0_100k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_batch_size": 1,
85
+ "dry_run_data": false,
86
+ "dynamics": {
87
+ "enabled": false,
88
+ "params": {},
89
+ "type": "none"
90
+ },
91
+ "dynamics_decoder_type": "film",
92
+ "enable_pred_next": false,
93
+ "encoder": {
94
+ "params": {
95
+ "cnn_depth": 48,
96
+ "cnn_kernel": 5,
97
+ "cnn_mults": [
98
+ 2,
99
+ 3,
100
+ 4,
101
+ 8,
102
+ 16
103
+ ],
104
+ "d_model": 768,
105
+ "dropout": 0.0,
106
+ "latent_token_self_attn": "full",
107
+ "mlp_ratio": 4,
108
+ "n_fusion_layers": 2,
109
+ "n_heads": 6,
110
+ "readout_depth": 1,
111
+ "readout_mlp_ratio": 2,
112
+ "readout_queries": 4,
113
+ "retained_tokens_per_frame": 4,
114
+ "rope_theta": 10000.0,
115
+ "use_proprio": true
116
+ },
117
+ "type": "attn_fusion_multi_token"
118
+ },
119
+ "encoder_lr": 0.000425,
120
+ "eval_at_start": false,
121
+ "eval_batch_size": 64,
122
+ "eval_crops_per_demo": 8,
123
+ "eval_every": 5000,
124
+ "eval_num_workers": 24,
125
+ "eval_seed": 0,
126
+ "ffn_hidden_size": 1024,
127
+ "filter_key": "wl23_mg1000_seed0",
128
+ "grad_accum_steps": 1,
129
+ "grad_clip": 1.0,
130
+ "hdf5_paths": [
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
151
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
152
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
153
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
154
+ ],
155
+ "holdout_action_mean_samples": 1,
156
+ "holdout_crops_per_demo": 8,
157
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
158
+ "holdout_per_task_metrics": true,
159
+ "holdout_rmse_samplers": [
160
+ "deterministic",
161
+ "stochastic"
162
+ ],
163
+ "holdout_sample_deterministic": null,
164
+ "holdout_sample_prefix_horizon": 4,
165
+ "holdout_sample_rmse": true,
166
+ "holdout_size": 2300,
167
+ "image_emb_dim": 512,
168
+ "init_device": "meta",
169
+ "input_norm": true,
170
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
171
+ "lang_emb_mode": "clip",
172
+ "lang_encoder_device": "auto",
173
+ "lang_obs_emb_dim": 256,
174
+ "log_every": 25,
175
+ "lora": false,
176
+ "lora_alpha": 32,
177
+ "lora_dropout": 0.0,
178
+ "lora_rank": 16,
179
+ "lora_target": [
180
+ "q_proj",
181
+ "k_proj",
182
+ "v_proj",
183
+ "o_proj",
184
+ "gate_proj",
185
+ "up_proj",
186
+ "down_proj"
187
+ ],
188
+ "lr": 0.0003,
189
+ "max_context_len": 1024,
190
+ "max_steps": 100000,
191
+ "min_lr_ratio": 0.1,
192
+ "model": {
193
+ "action_chunk_len": 10,
194
+ "latent_dim": 768
195
+ },
196
+ "model_dtype": "float32",
197
+ "n_heads": 16,
198
+ "n_kv_heads": null,
199
+ "n_layers": 2,
200
+ "norm_bias_weight_decay": null,
201
+ "norm_fit_batches": 0,
202
+ "num_workers": 24,
203
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
204
+ "obs_spec": {
205
+ "image_channels": 3,
206
+ "image_hw": [
207
+ 128,
208
+ 128
209
+ ],
210
+ "image_keys": [
211
+ "robot0_agentview_left_image",
212
+ "robot0_agentview_right_image",
213
+ "robot0_eye_in_hand_image"
214
+ ],
215
+ "lang_dim": 768,
216
+ "layout": "NHWC",
217
+ "low_dim_dims": [
218
+ 3,
219
+ 4,
220
+ 3,
221
+ 4,
222
+ 2
223
+ ],
224
+ "low_dim_keys": [
225
+ "robot0_base_to_eef_pos",
226
+ "robot0_base_to_eef_quat",
227
+ "robot0_base_pos",
228
+ "robot0_base_quat",
229
+ "robot0_gripper_qpos"
230
+ ]
231
+ },
232
+ "obs_stride": 4,
233
+ "output_dir": "${RELEASE_ROOT}/05_token_interface/multitoken_n2_k4_d1000_seed0",
234
+ "persistent_workers": true,
235
+ "precision": "bf16",
236
+ "pred_next_image_key_weights": {},
237
+ "pred_next_image_weight": 1.0,
238
+ "pred_next_lang_weight": 0.1,
239
+ "pred_next_mode": "terminal",
240
+ "pred_next_obs_offset": 1,
241
+ "pred_next_proprio_weight": 0.15,
242
+ "pred_next_steps": 4,
243
+ "pred_warmup_steps": 0,
244
+ "predictor_lr": 0.000425,
245
+ "prefetch_factor": 4,
246
+ "prepare_lang_cache_only": false,
247
+ "proprio_emb_dim": 128,
248
+ "recon": {
249
+ "enabled": false,
250
+ "params": {
251
+ "depth": 48,
252
+ "kernel_size": 5,
253
+ "mults": [
254
+ 2,
255
+ 3,
256
+ 4,
257
+ 4
258
+ ]
259
+ }
260
+ },
261
+ "recon_image_weight": 1.0,
262
+ "recon_lang_weight": 0.1,
263
+ "recon_obs": false,
264
+ "recon_proprio_weight": 1.0,
265
+ "resume": null,
266
+ "resume_final_lr": null,
267
+ "resume_lr_schedule": "checkpoint",
268
+ "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic",
269
+ "save_every": 10000,
270
+ "save_holdout_traces": true,
271
+ "save_latest_checkpoint": true,
272
+ "scaling_action_head_d_model": 192,
273
+ "scaling_action_head_layers": 2,
274
+ "scaling_action_head_width": 192,
275
+ "scaling_analysis_suites": [
276
+ "world_tokens_per_frame_ablation"
277
+ ],
278
+ "scaling_backbone_d_model": 768,
279
+ "scaling_backbone_layers": 4,
280
+ "scaling_encoder_d_model": 768,
281
+ "scaling_encoder_layers": 2,
282
+ "scaling_expected_world_size": 4,
283
+ "scaling_family_width": 768,
284
+ "scaling_global_batch_size": 192,
285
+ "scaling_grid_data_demos_per_task": 1000,
286
+ "scaling_grid_model_size": "n2",
287
+ "scaling_grid_protocol": "multitoken_d1000_n2_v1",
288
+ "scaling_latent_dim": 768,
289
+ "scaling_lr_action_head": 0.0003,
290
+ "scaling_lr_encoder": 0.000425,
291
+ "scaling_lr_lookup_model": "n2",
292
+ "scaling_lr_lookup_status": "confirmed",
293
+ "scaling_lr_param_groups": [
294
+ "encoder",
295
+ "predictor",
296
+ "action_head"
297
+ ],
298
+ "scaling_lr_predictor": 0.000425,
299
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
300
+ "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter",
301
+ "scaling_model_axis": "retained_world_tokens_per_frame",
302
+ "scaling_model_family": "n2_multitoken_interface_v1",
303
+ "scaling_model_size": "n2",
304
+ "scaling_per_rank_microbatch": 48,
305
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
306
+ "scaling_recipe": "multitoken_causal_nodyn",
307
+ "scaling_run_name": "multitoken_d1000_n2_k4_nodyn_seed0_100k",
308
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10",
309
+ "scaling_suite": "world_tokens_per_frame_ablation",
310
+ "seed": 0,
311
+ "seq_len": 10,
312
+ "sequence_model": {
313
+ "backbone_type": "qwen2",
314
+ "hidden_dim": 768,
315
+ "params": {
316
+ "attn_impl": "sdpa",
317
+ "dropout": 0.1,
318
+ "ffn_hidden_size": 2048,
319
+ "frame_readout_index": -1,
320
+ "input_norm": false,
321
+ "max_context_len": 1024,
322
+ "model_dtype": "float32",
323
+ "n_heads": 6,
324
+ "n_kv_heads": null,
325
+ "n_layers": 4,
326
+ "tokens_per_frame": 4
327
+ },
328
+ "type": "frame_major_continuous_transformer"
329
+ },
330
+ "use_proprio": true,
331
+ "warmup_steps": 5000,
332
+ "weight_decay": 0.0,
333
+ "z_bottleneck": {
334
+ "dim": null
335
+ },
336
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz"
337
+ }
05_token_interface/multitoken_n2_k4_d1000_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "multitoken_n2_k4_d1000_seed0",
3
+ "original_run_name": "multitoken_d1000_n2_k4_nodyn_seed0_100k_cuda4-5-6-7",
4
+ "paper_section": 5,
5
+ "parameters": {
6
+ "D": 1000,
7
+ "N": 2,
8
+ "K": 4,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 100000
13
+ }
05_token_interface/multitoken_n2_k4_d100_seed0/config.json ADDED
@@ -0,0 +1,333 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 192,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 768,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 6,
13
+ "n_layers": 2,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/E_multitoken_n2/configs_world1/multitoken_d100_n2_k4_nodyn_seed0_10k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_batch_size": 1,
85
+ "dry_run_data": false,
86
+ "dynamics": {
87
+ "enabled": false,
88
+ "params": {},
89
+ "type": "none"
90
+ },
91
+ "dynamics_decoder_type": "film",
92
+ "enable_pred_next": false,
93
+ "encoder": {
94
+ "params": {
95
+ "cnn_depth": 48,
96
+ "cnn_kernel": 5,
97
+ "cnn_mults": [
98
+ 2,
99
+ 3,
100
+ 4,
101
+ 8,
102
+ 16
103
+ ],
104
+ "d_model": 768,
105
+ "dropout": 0.0,
106
+ "latent_token_self_attn": "full",
107
+ "mlp_ratio": 4,
108
+ "n_fusion_layers": 2,
109
+ "n_heads": 6,
110
+ "readout_depth": 1,
111
+ "readout_mlp_ratio": 2,
112
+ "readout_queries": 4,
113
+ "retained_tokens_per_frame": 4,
114
+ "rope_theta": 10000.0,
115
+ "use_proprio": true
116
+ },
117
+ "type": "attn_fusion_multi_token"
118
+ },
119
+ "encoder_lr": 0.000425,
120
+ "eval_at_start": false,
121
+ "eval_batch_size": 64,
122
+ "eval_crops_per_demo": 8,
123
+ "eval_every": 500,
124
+ "eval_num_workers": 24,
125
+ "eval_seed": 0,
126
+ "ffn_hidden_size": 1024,
127
+ "filter_key": "wl23_mg100_seed0",
128
+ "grad_accum_steps": 4,
129
+ "grad_clip": 1.0,
130
+ "hdf5_paths": [
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
151
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
152
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
153
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
154
+ ],
155
+ "holdout_action_mean_samples": 1,
156
+ "holdout_crops_per_demo": 8,
157
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
158
+ "holdout_per_task_metrics": true,
159
+ "holdout_rmse_samplers": [
160
+ "deterministic",
161
+ "stochastic"
162
+ ],
163
+ "holdout_sample_deterministic": null,
164
+ "holdout_sample_prefix_horizon": 4,
165
+ "holdout_sample_rmse": true,
166
+ "holdout_size": 2300,
167
+ "image_emb_dim": 512,
168
+ "init_device": "meta",
169
+ "input_norm": true,
170
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
171
+ "lang_emb_mode": "clip",
172
+ "lang_encoder_device": "auto",
173
+ "lang_obs_emb_dim": 256,
174
+ "log_every": 25,
175
+ "lora": false,
176
+ "lora_alpha": 32,
177
+ "lora_dropout": 0.0,
178
+ "lora_rank": 16,
179
+ "lora_target": [
180
+ "q_proj",
181
+ "k_proj",
182
+ "v_proj",
183
+ "o_proj",
184
+ "gate_proj",
185
+ "up_proj",
186
+ "down_proj"
187
+ ],
188
+ "lr": 0.0003,
189
+ "max_context_len": 1024,
190
+ "max_steps": 10000,
191
+ "min_lr_ratio": 0.1,
192
+ "model": {
193
+ "action_chunk_len": 10,
194
+ "latent_dim": 768
195
+ },
196
+ "model_dtype": "float32",
197
+ "n_heads": 16,
198
+ "n_kv_heads": null,
199
+ "n_layers": 2,
200
+ "norm_bias_weight_decay": null,
201
+ "norm_fit_batches": 0,
202
+ "num_workers": 24,
203
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
204
+ "obs_spec": {
205
+ "image_channels": 3,
206
+ "image_hw": [
207
+ 128,
208
+ 128
209
+ ],
210
+ "image_keys": [
211
+ "robot0_agentview_left_image",
212
+ "robot0_agentview_right_image",
213
+ "robot0_eye_in_hand_image"
214
+ ],
215
+ "lang_dim": 768,
216
+ "layout": "NHWC",
217
+ "low_dim_dims": [
218
+ 3,
219
+ 4,
220
+ 3,
221
+ 4,
222
+ 2
223
+ ],
224
+ "low_dim_keys": [
225
+ "robot0_base_to_eef_pos",
226
+ "robot0_base_to_eef_quat",
227
+ "robot0_base_pos",
228
+ "robot0_base_quat",
229
+ "robot0_gripper_qpos"
230
+ ]
231
+ },
232
+ "obs_stride": 4,
233
+ "output_dir": "${RELEASE_ROOT}/05_token_interface/multitoken_n2_k4_d100_seed0",
234
+ "persistent_workers": true,
235
+ "precision": "bf16",
236
+ "pred_next_image_key_weights": {},
237
+ "pred_next_image_weight": 1.0,
238
+ "pred_next_lang_weight": 0.1,
239
+ "pred_next_mode": "terminal",
240
+ "pred_next_obs_offset": 1,
241
+ "pred_next_proprio_weight": 0.15,
242
+ "pred_next_steps": 4,
243
+ "pred_warmup_steps": 0,
244
+ "predictor_lr": 0.000425,
245
+ "prefetch_factor": 4,
246
+ "prepare_lang_cache_only": false,
247
+ "proprio_emb_dim": 128,
248
+ "recon": {
249
+ "enabled": false,
250
+ "params": {
251
+ "depth": 48,
252
+ "kernel_size": 5,
253
+ "mults": [
254
+ 2,
255
+ 3,
256
+ 4,
257
+ 4
258
+ ]
259
+ }
260
+ },
261
+ "recon_image_weight": 1.0,
262
+ "recon_lang_weight": 0.1,
263
+ "recon_obs": false,
264
+ "recon_proprio_weight": 1.0,
265
+ "resume": null,
266
+ "resume_final_lr": null,
267
+ "resume_lr_schedule": "checkpoint",
268
+ "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic",
269
+ "save_every": 1000,
270
+ "save_holdout_traces": true,
271
+ "save_latest_checkpoint": true,
272
+ "scaling_action_head_layers": 2,
273
+ "scaling_action_head_width": 192,
274
+ "scaling_analysis_suites": [
275
+ "world_tokens_per_frame_ablation"
276
+ ],
277
+ "scaling_backbone_layers": 4,
278
+ "scaling_encoder_layers": 2,
279
+ "scaling_expected_world_size": 1,
280
+ "scaling_family_width": 768,
281
+ "scaling_global_batch_size": 192,
282
+ "scaling_grid_data_demos_per_task": 100,
283
+ "scaling_grid_model_size": "n2",
284
+ "scaling_grid_protocol": "multitoken_d100_n2_v1",
285
+ "scaling_lr_action_head": 0.0003,
286
+ "scaling_lr_encoder": 0.000425,
287
+ "scaling_lr_lookup_model": "n2",
288
+ "scaling_lr_lookup_status": "confirmed",
289
+ "scaling_lr_param_groups": [
290
+ "encoder",
291
+ "predictor",
292
+ "action_head"
293
+ ],
294
+ "scaling_lr_predictor": 0.000425,
295
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
296
+ "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter",
297
+ "scaling_model_axis": "retained_world_tokens_per_frame",
298
+ "scaling_model_family": "n2_multitoken_interface_v1",
299
+ "scaling_model_size": "n2",
300
+ "scaling_per_rank_microbatch": 48,
301
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
302
+ "scaling_recipe": "multitoken_causal_nodyn",
303
+ "scaling_run_name": "multitoken_d100_n2_k4_nodyn_seed0_10k",
304
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10",
305
+ "scaling_suite": "world_tokens_per_frame_ablation",
306
+ "seed": 0,
307
+ "seq_len": 10,
308
+ "sequence_model": {
309
+ "backbone_type": "qwen2",
310
+ "hidden_dim": 768,
311
+ "params": {
312
+ "attn_impl": "sdpa",
313
+ "dropout": 0.1,
314
+ "ffn_hidden_size": 2048,
315
+ "frame_readout_index": -1,
316
+ "input_norm": false,
317
+ "max_context_len": 1024,
318
+ "model_dtype": "float32",
319
+ "n_heads": 6,
320
+ "n_kv_heads": null,
321
+ "n_layers": 4,
322
+ "tokens_per_frame": 4
323
+ },
324
+ "type": "frame_major_continuous_transformer"
325
+ },
326
+ "use_proprio": true,
327
+ "warmup_steps": 500,
328
+ "weight_decay": 0.0,
329
+ "z_bottleneck": {
330
+ "dim": null
331
+ },
332
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz"
333
+ }
05_token_interface/multitoken_n2_k4_d100_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "multitoken_n2_k4_d100_seed0",
3
+ "original_run_name": "multitoken_d100_n2_k4_nodyn_seed0_10k_cuda7",
4
+ "paper_section": 5,
5
+ "parameters": {
6
+ "D": 100,
7
+ "N": 2,
8
+ "K": 4,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 10000
13
+ }
05_token_interface/multitoken_n2_k4_d2900_seed0/config.json ADDED
@@ -0,0 +1,333 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_chunk_len": 10,
3
+ "action_decoder_emb": 256,
4
+ "action_head": {
5
+ "params": {
6
+ "activation": "gelu",
7
+ "d_model": 192,
8
+ "denoising_steps": 20,
9
+ "dim_feedforward": 768,
10
+ "dropout": 0.0,
11
+ "h_adaln_bottleneck": false,
12
+ "n_heads": 6,
13
+ "n_layers": 2,
14
+ "use_h_cross_attn": false,
15
+ "use_obs_cross_attn": false
16
+ },
17
+ "type": "diffusion_dit"
18
+ },
19
+ "action_head_lr": 0.0003,
20
+ "action_spec": {
21
+ "dim": 12,
22
+ "discrete_dims": [
23
+ 6,
24
+ 11
25
+ ]
26
+ },
27
+ "action_weight": 1.0,
28
+ "adamw_beta1": 0.9,
29
+ "adamw_beta2": 0.95,
30
+ "adamw_eps": 1e-08,
31
+ "allow_holdout_overlap": false,
32
+ "attn_backend": "auto",
33
+ "attn_impl": "eager",
34
+ "backbone_lr": null,
35
+ "backbone_type": "qwen2",
36
+ "batch_size": 48,
37
+ "cnn_depth": 48,
38
+ "cnn_kernel": 5,
39
+ "cnn_mults": [
40
+ 2,
41
+ 3,
42
+ 4,
43
+ 4
44
+ ],
45
+ "config": "${EXPERIMENT_ROOT}/E_multitoken_n2/configs_world4/multitoken_d2900_n2_k4_nodyn_seed0_280k.yaml",
46
+ "crops_per_demo": 8,
47
+ "d_model": 2048,
48
+ "dataset": [
49
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
50
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
51
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
52
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
53
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
54
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
55
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
56
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
57
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
58
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
59
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
60
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
61
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
62
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
63
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
64
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
65
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
66
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
67
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
68
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
69
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
70
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5",
71
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5"
72
+ ],
73
+ "ddp_timeout_minutes": 180.0,
74
+ "denoising_steps": 20,
75
+ "device": "cuda",
76
+ "diffusion_mlp_dims": [
77
+ 1024,
78
+ 1024,
79
+ 1024
80
+ ],
81
+ "diffusion_residual_style": true,
82
+ "diffusion_time_dim": 16,
83
+ "dropout": 0.1,
84
+ "dry_run_batch_size": 1,
85
+ "dry_run_data": false,
86
+ "dynamics": {
87
+ "enabled": false,
88
+ "params": {},
89
+ "type": "none"
90
+ },
91
+ "dynamics_decoder_type": "film",
92
+ "enable_pred_next": false,
93
+ "encoder": {
94
+ "params": {
95
+ "cnn_depth": 48,
96
+ "cnn_kernel": 5,
97
+ "cnn_mults": [
98
+ 2,
99
+ 3,
100
+ 4,
101
+ 8,
102
+ 16
103
+ ],
104
+ "d_model": 768,
105
+ "dropout": 0.0,
106
+ "latent_token_self_attn": "full",
107
+ "mlp_ratio": 4,
108
+ "n_fusion_layers": 2,
109
+ "n_heads": 6,
110
+ "readout_depth": 1,
111
+ "readout_mlp_ratio": 2,
112
+ "readout_queries": 4,
113
+ "retained_tokens_per_frame": 4,
114
+ "rope_theta": 10000.0,
115
+ "use_proprio": true
116
+ },
117
+ "type": "attn_fusion_multi_token"
118
+ },
119
+ "encoder_lr": 0.000425,
120
+ "eval_at_start": false,
121
+ "eval_batch_size": 64,
122
+ "eval_crops_per_demo": 8,
123
+ "eval_every": 14000,
124
+ "eval_num_workers": 24,
125
+ "eval_seed": 0,
126
+ "ffn_hidden_size": 1024,
127
+ "filter_key": "wl23_mg2900_seed0",
128
+ "grad_accum_steps": 1,
129
+ "grad_clip": 1.0,
130
+ "hdf5_paths": [
131
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5",
132
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5",
133
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5",
134
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5",
135
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5",
136
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5",
137
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5",
138
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5",
139
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5",
140
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5",
141
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5",
142
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5",
143
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5",
144
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5",
145
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5",
146
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5",
147
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5",
148
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5",
149
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5",
150
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5",
151
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5",
152
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5",
153
+ "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5"
154
+ ],
155
+ "holdout_action_mean_samples": 1,
156
+ "holdout_crops_per_demo": 8,
157
+ "holdout_filter_key": "wl23_holdout100pt_seed0",
158
+ "holdout_per_task_metrics": true,
159
+ "holdout_rmse_samplers": [
160
+ "deterministic",
161
+ "stochastic"
162
+ ],
163
+ "holdout_sample_deterministic": null,
164
+ "holdout_sample_prefix_horizon": 4,
165
+ "holdout_sample_rmse": true,
166
+ "holdout_size": 2300,
167
+ "image_emb_dim": 512,
168
+ "init_device": "meta",
169
+ "input_norm": true,
170
+ "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz",
171
+ "lang_emb_mode": "clip",
172
+ "lang_encoder_device": "auto",
173
+ "lang_obs_emb_dim": 256,
174
+ "log_every": 25,
175
+ "lora": false,
176
+ "lora_alpha": 32,
177
+ "lora_dropout": 0.0,
178
+ "lora_rank": 16,
179
+ "lora_target": [
180
+ "q_proj",
181
+ "k_proj",
182
+ "v_proj",
183
+ "o_proj",
184
+ "gate_proj",
185
+ "up_proj",
186
+ "down_proj"
187
+ ],
188
+ "lr": 0.0003,
189
+ "max_context_len": 1024,
190
+ "max_steps": 280000,
191
+ "min_lr_ratio": 0.1,
192
+ "model": {
193
+ "action_chunk_len": 10,
194
+ "latent_dim": 768
195
+ },
196
+ "model_dtype": "float32",
197
+ "n_heads": 16,
198
+ "n_kv_heads": null,
199
+ "n_layers": 2,
200
+ "norm_bias_weight_decay": null,
201
+ "norm_fit_batches": 0,
202
+ "num_workers": 24,
203
+ "objective": "robocasa_lang_as_obs_image_state_diffusion_action",
204
+ "obs_spec": {
205
+ "image_channels": 3,
206
+ "image_hw": [
207
+ 128,
208
+ 128
209
+ ],
210
+ "image_keys": [
211
+ "robot0_agentview_left_image",
212
+ "robot0_agentview_right_image",
213
+ "robot0_eye_in_hand_image"
214
+ ],
215
+ "lang_dim": 768,
216
+ "layout": "NHWC",
217
+ "low_dim_dims": [
218
+ 3,
219
+ 4,
220
+ 3,
221
+ 4,
222
+ 2
223
+ ],
224
+ "low_dim_keys": [
225
+ "robot0_base_to_eef_pos",
226
+ "robot0_base_to_eef_quat",
227
+ "robot0_base_pos",
228
+ "robot0_base_quat",
229
+ "robot0_gripper_qpos"
230
+ ]
231
+ },
232
+ "obs_stride": 4,
233
+ "output_dir": "${RELEASE_ROOT}/05_token_interface/multitoken_n2_k4_d2900_seed0",
234
+ "persistent_workers": true,
235
+ "precision": "bf16",
236
+ "pred_next_image_key_weights": {},
237
+ "pred_next_image_weight": 1.0,
238
+ "pred_next_lang_weight": 0.1,
239
+ "pred_next_mode": "terminal",
240
+ "pred_next_obs_offset": 1,
241
+ "pred_next_proprio_weight": 0.15,
242
+ "pred_next_steps": 4,
243
+ "pred_warmup_steps": 0,
244
+ "predictor_lr": 0.000425,
245
+ "prefetch_factor": 4,
246
+ "prepare_lang_cache_only": false,
247
+ "proprio_emb_dim": 128,
248
+ "recon": {
249
+ "enabled": false,
250
+ "params": {
251
+ "depth": 48,
252
+ "kernel_size": 5,
253
+ "mults": [
254
+ 2,
255
+ 3,
256
+ 4,
257
+ 4
258
+ ]
259
+ }
260
+ },
261
+ "recon_image_weight": 1.0,
262
+ "recon_lang_weight": 0.1,
263
+ "recon_obs": false,
264
+ "recon_proprio_weight": 1.0,
265
+ "resume": null,
266
+ "resume_final_lr": null,
267
+ "resume_lr_schedule": "checkpoint",
268
+ "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic",
269
+ "save_every": 28000,
270
+ "save_holdout_traces": true,
271
+ "save_latest_checkpoint": true,
272
+ "scaling_action_head_layers": 2,
273
+ "scaling_action_head_width": 192,
274
+ "scaling_analysis_suites": [
275
+ "world_tokens_per_frame_ablation"
276
+ ],
277
+ "scaling_backbone_layers": 4,
278
+ "scaling_encoder_layers": 2,
279
+ "scaling_expected_world_size": 4,
280
+ "scaling_family_width": 768,
281
+ "scaling_global_batch_size": 192,
282
+ "scaling_grid_data_demos_per_task": 2900,
283
+ "scaling_grid_model_size": "n2",
284
+ "scaling_grid_protocol": "multitoken_d2900_n2_v1",
285
+ "scaling_lr_action_head": 0.0003,
286
+ "scaling_lr_encoder": 0.000425,
287
+ "scaling_lr_lookup_model": "n2",
288
+ "scaling_lr_lookup_status": "confirmed",
289
+ "scaling_lr_param_groups": [
290
+ "encoder",
291
+ "predictor",
292
+ "action_head"
293
+ ],
294
+ "scaling_lr_predictor": 0.000425,
295
+ "scaling_lr_rule": "discrete_n_lookup_v20260711",
296
+ "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter",
297
+ "scaling_model_axis": "retained_world_tokens_per_frame",
298
+ "scaling_model_family": "n2_multitoken_interface_v1",
299
+ "scaling_model_size": "n2",
300
+ "scaling_per_rank_microbatch": 48,
301
+ "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss",
302
+ "scaling_recipe": "multitoken_causal_nodyn",
303
+ "scaling_run_name": "multitoken_d2900_n2_k4_nodyn_seed0_280k",
304
+ "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10",
305
+ "scaling_suite": "world_tokens_per_frame_ablation",
306
+ "seed": 0,
307
+ "seq_len": 10,
308
+ "sequence_model": {
309
+ "backbone_type": "qwen2",
310
+ "hidden_dim": 768,
311
+ "params": {
312
+ "attn_impl": "sdpa",
313
+ "dropout": 0.1,
314
+ "ffn_hidden_size": 2048,
315
+ "frame_readout_index": -1,
316
+ "input_norm": false,
317
+ "max_context_len": 1024,
318
+ "model_dtype": "float32",
319
+ "n_heads": 6,
320
+ "n_kv_heads": null,
321
+ "n_layers": 4,
322
+ "tokens_per_frame": 4
323
+ },
324
+ "type": "frame_major_continuous_transformer"
325
+ },
326
+ "use_proprio": true,
327
+ "warmup_steps": 14000,
328
+ "weight_decay": 0.0,
329
+ "z_bottleneck": {
330
+ "dim": null
331
+ },
332
+ "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz"
333
+ }
05_token_interface/multitoken_n2_k4_d2900_seed0/run_info.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "multitoken_n2_k4_d2900_seed0",
3
+ "original_run_name": "multitoken_d2900_n2_k4_nodyn_seed0_280k_cuda4-5-6-7",
4
+ "paper_section": 5,
5
+ "parameters": {
6
+ "D": 2900,
7
+ "N": 2,
8
+ "K": 4,
9
+ "C_train": 10,
10
+ "seed": 0
11
+ },
12
+ "final_training_step": 280000
13
+ }