Ratstar216 commited on
Commit
78d3c17
·
verified ·
1 Parent(s): 94e3779

Add files using upload-large-folder tool

Browse files
Files changed (38) hide show
  1. checkpoints/step_004000/manifest.json +155 -0
  2. checkpoints/step_004000/metadata/args.json +42 -0
  3. checkpoints/step_004000/metadata/asset_hashes.json +5 -0
  4. checkpoints/step_004000/metadata/assets.lock.json +32 -0
  5. checkpoints/step_004000/metadata/checkpoint_load.json +22 -0
  6. checkpoints/step_004000/metadata/config.yaml +270 -0
  7. checkpoints/step_004000/metadata/dataset_stats.json +1074 -0
  8. checkpoints/step_004000/metadata/environment.txt +113 -0
  9. checkpoints/step_004000/metadata/parameters.json +218 -0
  10. checkpoints/step_004000/metadata/pretraining_report.md +53 -0
  11. checkpoints/step_004000/metadata/provenance.json +50 -0
  12. checkpoints/step_004000/metadata/qwen_cache.json +11 -0
  13. checkpoints/step_004000/metadata/sampling.json +14 -0
  14. checkpoints/step_004000/metadata/source/recovery_checkpoints.py +123 -0
  15. checkpoints/step_004000/metadata/source/resolution_adapt/README.md +357 -0
  16. checkpoints/step_004000/metadata/source/resolution_adapt/__init__.py +1 -0
  17. checkpoints/step_004000/metadata/source/resolution_adapt/assets.lock.json +32 -0
  18. checkpoints/step_004000/metadata/source/resolution_adapt/checkpoints.py +217 -0
  19. checkpoints/step_004000/metadata/source/resolution_adapt/data.py +180 -0
  20. checkpoints/step_004000/metadata/source/resolution_adapt/model.py +188 -0
  21. checkpoints/step_004000/metadata/source/resolution_adapt/offline.py +38 -0
  22. checkpoints/step_004000/metadata/source/resolution_adapt/preflight_data.py +114 -0
  23. checkpoints/step_004000/metadata/source/resolution_adapt/preflight_model.py +45 -0
  24. checkpoints/step_004000/metadata/source/resolution_adapt/prepare_assets.py +57 -0
  25. checkpoints/step_004000/metadata/source/resolution_adapt/released_stage2.yaml +271 -0
  26. checkpoints/step_004000/metadata/source/resolution_adapt/requirements-training.lock +341 -0
  27. checkpoints/step_004000/metadata/source/resolution_adapt/requirements-training.txt +29 -0
  28. checkpoints/step_004000/metadata/source/resolution_adapt/telemetry.py +58 -0
  29. checkpoints/step_004000/metadata/source/resolution_adapt/test_pipeline.py +284 -0
  30. checkpoints/step_004000/metadata/source/resolution_adapt/test_video_loading.py +211 -0
  31. checkpoints/step_004000/metadata/source/train_resolution_adapt.py +468 -0
  32. checkpoints/step_004000/metadata/source/training_support.py +357 -0
  33. checkpoints/step_004000/metadata/split.json +0 -0
  34. checkpoints/step_004000/metadata/tabular_audit.json +136 -0
  35. checkpoints/step_004000/metadata/video_audit.json +7 -0
  36. checkpoints/step_004000/metadata/wandb.json +6 -0
  37. checkpoints/step_004000/resume/state.pt +3 -0
  38. checkpoints/step_004000/weights/adaptation.pt +3 -0
checkpoints/step_004000/manifest.json ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "step": 4000,
3
+ "final": false,
4
+ "files": {
5
+ "metadata/args.json": {
6
+ "size": 1389,
7
+ "sha256": "1369a024d5fffe22dfbfe1400b7eef7de7ba7948a34ee631cd3e3c2ddf0570bc"
8
+ },
9
+ "metadata/asset_hashes.json": {
10
+ "size": 390,
11
+ "sha256": "e8c07d54adefa2fff39a36d6335f9ac1d92bed5814e68125bb4b1b1e7c2f9e8a"
12
+ },
13
+ "metadata/assets.lock.json": {
14
+ "size": 968,
15
+ "sha256": "3ff22d9d5dcdb747c3739a34e401b4e6dab21051261a31b8f25f3e8c33d4470a"
16
+ },
17
+ "metadata/checkpoint_load.json": {
18
+ "size": 415,
19
+ "sha256": "12f43b5b3b45c967277d8b5df707a2e6acab6acfd7da3b116fb77cfc0c168b47"
20
+ },
21
+ "metadata/config.yaml": {
22
+ "size": 6690,
23
+ "sha256": "35a87f80d7436ea1ed629f4b9d63a90846c3d09bcd90d7c82bc35833b76dd88d"
24
+ },
25
+ "metadata/dataset_stats.json": {
26
+ "size": 22880,
27
+ "sha256": "b107cd929518d0477535f99834e0cc13f470e25cd61bbce69b315619ba5140d4"
28
+ },
29
+ "metadata/environment.txt": {
30
+ "size": 2106,
31
+ "sha256": "2de6cecede55f1f43ed21f751e381c6f78ce9026e0b962a1da407c8257de54a9"
32
+ },
33
+ "metadata/parameters.json": {
34
+ "size": 5337,
35
+ "sha256": "c735a0a91c71e6a31c05bee272b447a585932f396ea92ad3fcfc95c8f8e9d3eb"
36
+ },
37
+ "metadata/pretraining_report.md": {
38
+ "size": 13688,
39
+ "sha256": "d0ab4563c377ee9a79bbdfcc7829b55b2cf9dfbd7d786eecffa181be5b21c676"
40
+ },
41
+ "metadata/provenance.json": {
42
+ "size": 1873,
43
+ "sha256": "c7c3edebc43685c22e6ce25e013553084d7d3a5718cd50412197299b3c03f5d6"
44
+ },
45
+ "metadata/qwen_cache.json": {
46
+ "size": 216,
47
+ "sha256": "264991ae378d3e8388732252e960f972ba2d01cc28301e5e47c84379e8809b40"
48
+ },
49
+ "metadata/sampling.json": {
50
+ "size": 229,
51
+ "sha256": "ad8969c84ed4d9ce1736cb271ebef09e0cc945ec5baeefd86ad294c5fb749cd4"
52
+ },
53
+ "metadata/source/recovery_checkpoints.py": {
54
+ "size": 5395,
55
+ "sha256": "73f5b3cd8c1825f5c9c064637a0b5433801297dbd31dfa2104438610650254e7"
56
+ },
57
+ "metadata/source/resolution_adapt/README.md": {
58
+ "size": 19088,
59
+ "sha256": "53327de34252a7a6ebc2604b9e86e6e1aaec7619c3b256b99ce6df12465d6173"
60
+ },
61
+ "metadata/source/resolution_adapt/__init__.py": {
62
+ "size": 85,
63
+ "sha256": "b9eacc14b8dec315086ef582e49a9a6d11c013b3aa602a201e5a75fde143bbbf"
64
+ },
65
+ "metadata/source/resolution_adapt/assets.lock.json": {
66
+ "size": 968,
67
+ "sha256": "3ff22d9d5dcdb747c3739a34e401b4e6dab21051261a31b8f25f3e8c33d4470a"
68
+ },
69
+ "metadata/source/resolution_adapt/checkpoints.py": {
70
+ "size": 11510,
71
+ "sha256": "016644cd764234f7f1a0da857b67c985b4796ec37b0725a38b0720cf4f32e1c1"
72
+ },
73
+ "metadata/source/resolution_adapt/data.py": {
74
+ "size": 8391,
75
+ "sha256": "e890eb65a989ee849c79a3eeaca93740ec1d907b3e85ab7ab78cb6e04d61c6f7"
76
+ },
77
+ "metadata/source/resolution_adapt/model.py": {
78
+ "size": 10445,
79
+ "sha256": "30376dc071cf1ff004348e2fed07f5e026541acbf471b894ed3a69e6b2a37f49"
80
+ },
81
+ "metadata/source/resolution_adapt/offline.py": {
82
+ "size": 2253,
83
+ "sha256": "ed5b420dda803618d559e6ecf236afb1105cdd4d191b89b7e3670e14344dfb98"
84
+ },
85
+ "metadata/source/resolution_adapt/preflight_data.py": {
86
+ "size": 7135,
87
+ "sha256": "b8b4b0835b4a5d21bf23e8057d0c1e2e00575b6f7839bbc55203d7addc55e420"
88
+ },
89
+ "metadata/source/resolution_adapt/preflight_model.py": {
90
+ "size": 2622,
91
+ "sha256": "7348384848bacf373f93a06efb5f538eea2e402a27fefc07d7c2e2995338c06f"
92
+ },
93
+ "metadata/source/resolution_adapt/prepare_assets.py": {
94
+ "size": 3414,
95
+ "sha256": "d6243b15acce8d439069d39ca1905fa2d4828e932f65529c3fe46d3dc04f6360"
96
+ },
97
+ "metadata/source/resolution_adapt/released_stage2.yaml": {
98
+ "size": 6886,
99
+ "sha256": "9c4a8d9eb8767e71e7ebd6edb921be810c7867512d9ebbf0f91a53dca21e9da1"
100
+ },
101
+ "metadata/source/resolution_adapt/requirements-training.lock": {
102
+ "size": 8037,
103
+ "sha256": "b8300068205c5a0f6f7ca2a3a52c10f3f26f4183de08cb9676df5fc9d9537c38"
104
+ },
105
+ "metadata/source/resolution_adapt/requirements-training.txt": {
106
+ "size": 652,
107
+ "sha256": "0f9d4aad5b1f89bda809706f964c8a9128d28cecb6bbea5138e4f4d43cb8ec31"
108
+ },
109
+ "metadata/source/resolution_adapt/telemetry.py": {
110
+ "size": 2261,
111
+ "sha256": "bda6d3f2ad96aa2844fa56deebdcac46476caf5440c5afa199dd5daeffc03196"
112
+ },
113
+ "metadata/source/resolution_adapt/test_pipeline.py": {
114
+ "size": 15265,
115
+ "sha256": "e0317527b23a866566b58a90bce4239f35df08066e6434ce6d999817599a73c2"
116
+ },
117
+ "metadata/source/resolution_adapt/test_video_loading.py": {
118
+ "size": 8465,
119
+ "sha256": "d43efef062717b163ca8849bbb2b089a2401b8f77bcefe7a522a8126f58cdeb3"
120
+ },
121
+ "metadata/source/train_resolution_adapt.py": {
122
+ "size": 30344,
123
+ "sha256": "e4fcfcb1b6e4b704a91de4b9dbea12ce2f858deaf6255b6b3a90228b17f2a858"
124
+ },
125
+ "metadata/source/training_support.py": {
126
+ "size": 19567,
127
+ "sha256": "30c31bf36d93e1e53e15eb6d511919fb1b65ab84c864b5987169ea16ae57c9b0"
128
+ },
129
+ "metadata/split.json": {
130
+ "size": 9038397,
131
+ "sha256": "2a78a2a60677149f0fdeb51c2e80ec47966c6b336bd78b5f6943e71e04c15d02"
132
+ },
133
+ "metadata/tabular_audit.json": {
134
+ "size": 3285,
135
+ "sha256": "6596934ec2c3e23bfa27b4b2278d1f4d6935751ff118af4d964cb94274839595"
136
+ },
137
+ "metadata/video_audit.json": {
138
+ "size": 255,
139
+ "sha256": "dba4a05568c5229b2f1b88aab7a0e74eea8032bfcefa4b5e2d78973ebbced164"
140
+ },
141
+ "metadata/wandb.json": {
142
+ "size": 159,
143
+ "sha256": "3aa8ececbf4ec6d9633e03f0e4e8361c3996da7ef3c12a92d8d2c295b49dea0b"
144
+ },
145
+ "resume/state.pt": {
146
+ "size": 6672739984,
147
+ "sha256": "d0c3e5d11137de82ae6898ea4288a95c05068fed58e30df7f1e70e05596181c4"
148
+ },
149
+ "weights/adaptation.pt": {
150
+ "size": 3336327138,
151
+ "sha256": "7e652091d431c95ce5df1bab7c3fd01de0edbab3c254763c180a6392fa6e7170"
152
+ }
153
+ },
154
+ "created_unix": 1791326174.4687986
155
+ }
checkpoints/step_004000/metadata/args.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "mode": "train",
3
+ "output": "/content/PARC2026_final/.runtime/lit-resolution-resume-2000",
4
+ "dataset_root": "/tmp/resadapt-dataset",
5
+ "released_dir": "/tmp/resadapt-assets/released/imagewam/lit_stage2",
6
+ "imagewam_root": "/tmp/resadapt-imagewam",
7
+ "flux_root": "/tmp/resadapt-flux2",
8
+ "ae_path": "/tmp/resadapt-assets/vae/ae.safetensors",
9
+ "qwen_cache": "/tmp/resadapt-assets/qwen_cache",
10
+ "resume": "/content/PARC2026_final/.runtime/lit-resolution-resume-2000/checkpoints/step_002000",
11
+ "max_steps": 10000,
12
+ "warmup_steps": 500,
13
+ "batch_size": 1,
14
+ "gradient_accumulation": 8,
15
+ "lora_rank": 16,
16
+ "unfreeze_last_flux_blocks": 0,
17
+ "checkpoint_every": 2000,
18
+ "seed": 42,
19
+ "eval_every": 200,
20
+ "validation_samples_per_pool": 8,
21
+ "log_every": 10,
22
+ "lora_alpha": 16.0,
23
+ "lora_dropout": 0.0,
24
+ "lr_flux_lora": 3e-05,
25
+ "lr_action": 1e-05,
26
+ "lr_interface": 1e-05,
27
+ "lr_pose": 1e-05,
28
+ "lr_proprio": 1e-05,
29
+ "lr_other": 1e-05,
30
+ "lr_unfrozen_flux": 1e-06,
31
+ "weight_decay": 0.01,
32
+ "familiar_libero_weight": 0.5,
33
+ "plus_weight": 0.4,
34
+ "libero90_weight": 0.1,
35
+ "validation_episode_fraction": 0.05,
36
+ "hf_repo": "Ratstar216/LIT-ImageWAM-ResolutionAdapt-128to224",
37
+ "wandb_project": "parc2026-lit-imagewam",
38
+ "wandb_run_name": "lit-stage2-resadapt-128to224-lora-r16",
39
+ "wandb_group": "lit-stage2-resadapt-128to224",
40
+ "upload_once": false,
41
+ "preflight_save": false
42
+ }
checkpoints/step_004000/metadata/asset_hashes.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "/tmp/resadapt-assets/released/imagewam/lit_stage2/model.pt": "f403225e76809034307caf23e5a8f155385a51adb69d98990cff14125743ec97",
3
+ "/tmp/resadapt-assets/released/imagewam/lit_stage2/dataset_stats.json": "b107cd929518d0477535f99834e0cc13f470e25cd61bbce69b315619ba5140d4",
4
+ "/tmp/resadapt-assets/vae/ae.safetensors": "868fe7b343cc8f3a19dbcfcafbc3d5f888802be3f89bd81b65b3621a066ce8f3"
5
+ }
checkpoints/step_004000/metadata/assets.lock.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "linjianman/LIT": {
3
+ "revision": "765a2b2bf136f7039ab18978920c682f1aae65d2",
4
+ "files": {
5
+ "imagewam/lit_stage2/config.yaml": {
6
+ "size": 6886,
7
+ "sha256": "9c4a8d9eb8767e71e7ebd6edb921be810c7867512d9ebbf0f91a53dca21e9da1"
8
+ },
9
+ "imagewam/lit_stage2/dataset_stats.json": {
10
+ "size": 22880,
11
+ "sha256": "b107cd929518d0477535f99834e0cc13f470e25cd61bbce69b315619ba5140d4"
12
+ },
13
+ "imagewam/lit_stage2/model.pt": {
14
+ "size": 9373131513,
15
+ "sha256": "f403225e76809034307caf23e5a8f155385a51adb69d98990cff14125743ec97"
16
+ }
17
+ }
18
+ },
19
+ "black-forest-labs/FLUX.2-dev": {
20
+ "revision": "26afe3a78bb242c0a8bb181dcc8937bb16e5c66c",
21
+ "files": {
22
+ "ae.safetensors": {
23
+ "size": 336211292,
24
+ "sha256": "868fe7b343cc8f3a19dbcfcafbc3d5f888802be3f89bd81b65b3621a066ce8f3"
25
+ }
26
+ }
27
+ },
28
+ "Qwen/Qwen3-4B": {
29
+ "revision": "1cfa9a7208912126459214e8b04321603b3df60c",
30
+ "files": {}
31
+ }
32
+ }
checkpoints/step_004000/metadata/checkpoint_load.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "mot": {
3
+ "missing_keys": [],
4
+ "unexpected_keys": []
5
+ },
6
+ "proprio_encoder": {
7
+ "missing_keys": [],
8
+ "unexpected_keys": []
9
+ },
10
+ "semantic_visual_aggregator": {
11
+ "missing_keys": [],
12
+ "unexpected_keys": []
13
+ },
14
+ "semantic_visual_pose_decoder": {
15
+ "missing_keys": [],
16
+ "unexpected_keys": []
17
+ },
18
+ "semantic_visual_pose_norm": {
19
+ "missing_keys": [],
20
+ "unexpected_keys": []
21
+ }
22
+ }
checkpoints/step_004000/metadata/config.yaml ADDED
@@ -0,0 +1,270 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ output_dir: /content/PARC2026_final/.runtime/lit-resolution-resume-2000
2
+ batch_size: 1
3
+ num_workers: 0
4
+ prefetch_factor: 2
5
+ persistent_workers: true
6
+ lr_scheduler_type: cosine
7
+ learning_rate: null
8
+ warmup_steps: 500
9
+ num_epochs: 10
10
+ max_steps: 10000
11
+ log_every: 10
12
+ save_every: 2000
13
+ keep_latest_state_only: true
14
+ eval_every: 100
15
+ eval_num_inference_steps: 10
16
+ eval_num_samples: 8
17
+ rank_timer_every: 10
18
+ rank_timer_sync_cuda: true
19
+ qwen_cache_batch_size: 32
20
+ qwen_cache_overwrite: false
21
+ qwen_cache_save_workers: 8
22
+ flux2_qwen3_model_spec: null
23
+ precache_arrow: true
24
+ precache_num_workers: 4
25
+ build_lerobot_meta_cache: true
26
+ build_norm_stats: false
27
+ precache_fuse_norm_stats: true
28
+ force_rebuild_norm_stats: false
29
+ norm_stats_use_arrow_cache: true
30
+ norm_stats_bulk_read_arrow: true
31
+ norm_stats_cache_enabled: true
32
+ norm_stats_cache_dir: null
33
+ precache_profile: false
34
+ precache_profile_min_sec: 0.0
35
+ gradient_accumulation_steps: 8
36
+ mixed_precision: bf16
37
+ seed: 42
38
+ max_grad_norm: 1.0
39
+ weight_decay: 0.01
40
+ resume: /content/PARC2026_final/.runtime/lit-resolution-resume-2000/checkpoints/step_002000
41
+ stage1_checkpoint: null
42
+ wandb:
43
+ enabled: true
44
+ workspace: null
45
+ project: parc2026-lit-imagewam
46
+ name: lit-stage2-resadapt-128to224-lora-r16
47
+ group: lit-stage2-resadapt-128to224
48
+ mode: online
49
+ data:
50
+ train:
51
+ _target_: imagewam.datasets.lerobot.robot_video_dataset.RobotVideoDataset
52
+ dataset_dirs:
53
+ - /tmp/resadapt-dataset
54
+ shape_meta:
55
+ images:
56
+ - key: image
57
+ raw_shape:
58
+ - 3
59
+ - 256
60
+ - 256
61
+ shape:
62
+ - 3
63
+ - 224
64
+ - 224
65
+ lerobot_key: observation.images.front
66
+ - key: wrist_image
67
+ raw_shape:
68
+ - 3
69
+ - 256
70
+ - 256
71
+ shape:
72
+ - 3
73
+ - 224
74
+ - 224
75
+ lerobot_key: observation.images.wrist
76
+ action:
77
+ - key: default
78
+ raw_shape: 7
79
+ shape: 7
80
+ state:
81
+ - key: default
82
+ raw_shape: 8
83
+ shape: 8
84
+ num_frames: 17
85
+ global_sample_stride: 1
86
+ action_video_freq_ratio: 1
87
+ video_size:
88
+ - 224
89
+ - 448
90
+ camera_key: null
91
+ val_set_proportion: 0.0
92
+ is_training_set: true
93
+ skip_padding_as_possible: false
94
+ concat_multi_camera: horizontal
95
+ video_augmentation:
96
+ _target_: imagewam.datasets.lerobot.transforms.image.VideoAugmentation
97
+ p: 0.65
98
+ augment_types:
99
+ - corrupt_only
100
+ - color_only
101
+ - both
102
+ - both
103
+ color_jitter:
104
+ brightness: 0.3
105
+ contrast: 0.3
106
+ saturation: 0.25
107
+ hue: 0.04
108
+ gamma:
109
+ range:
110
+ - 0.8
111
+ - 1.25
112
+ gaussian_noise:
113
+ std: 0.015
114
+ random_resized_crop: null
115
+ rotate: null
116
+ exposure:
117
+ ev_range:
118
+ - -0.25
119
+ - 0.25
120
+ require_text_cache: false
121
+ endpoint_frames_only: true
122
+ qwen_text_cache_dir: /tmp/resadapt-assets/qwen_cache
123
+ qwen_context_len: 512
124
+ pretrained_norm_stats: /tmp/resadapt-assets/released/imagewam/lit_stage2/dataset_stats.json
125
+ processor:
126
+ _target_: imagewam.datasets.lerobot.processors.imagewam_processor.ImageWAMProcessor
127
+ shape_meta:
128
+ images:
129
+ - key: image
130
+ raw_shape:
131
+ - 3
132
+ - 256
133
+ - 256
134
+ shape:
135
+ - 3
136
+ - 224
137
+ - 224
138
+ lerobot_key: observation.images.front
139
+ - key: wrist_image
140
+ raw_shape:
141
+ - 3
142
+ - 256
143
+ - 256
144
+ shape:
145
+ - 3
146
+ - 224
147
+ - 224
148
+ lerobot_key: observation.images.wrist
149
+ action:
150
+ - key: default
151
+ raw_shape: 7
152
+ shape: 7
153
+ state:
154
+ - key: default
155
+ raw_shape: 8
156
+ shape: 8
157
+ num_obs_steps: 17
158
+ image_obs_steps: 2
159
+ num_output_cameras: 2
160
+ action_output_dim: 7
161
+ proprio_output_dim: 8
162
+ delta_action_dim_mask:
163
+ default:
164
+ - true
165
+ - true
166
+ - true
167
+ - true
168
+ - true
169
+ - true
170
+ - false
171
+ action_state_transforms:
172
+ - _target_: training_support.DatasetGripperToImageWAM
173
+ use_stepwise_action_norm: false
174
+ norm_default_mode: min/max
175
+ norm_exception_mode: null
176
+ action_state_merger:
177
+ _target_: imagewam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign
178
+ train_transforms:
179
+ - _target_: imagewam.datasets.lerobot.transforms.image.ToTensor
180
+ - _target_: resolution_adapt.data.ResolutionBottleneck
181
+ val_transforms:
182
+ - _target_: imagewam.datasets.lerobot.transforms.image.ToTensor
183
+ - _target_: resolution_adapt.data.ResolutionBottleneck
184
+ text_embedding_cache_dir: null
185
+ context_len: 128
186
+ qwen_text_cache_format: qwen3_flux2
187
+ lerobot_backend: v3
188
+ lerobot_v3_video_backend: pyav
189
+ lerobot_tolerance_s: 0.0001
190
+ vision_free: false
191
+ model:
192
+ _target_: imagewam.runtime.create_imagewam_flux2_klein
193
+ flux2_src_path: /tmp/resadapt-flux2
194
+ flux2_model_path: null
195
+ ae_model_path: /tmp/resadapt-assets/vae/ae.safetensors
196
+ variant: klein-base-4b
197
+ qwen3_model_spec: null
198
+ qwen_context_len: 512
199
+ load_text_encoder: false
200
+ proprio_dim: 8
201
+ mot_checkpoint_mixed_attn: true
202
+ mot_gqa_implementation: repeat
203
+ mot_force_flash_attention: false
204
+ pack_proprio_after_text: true
205
+ action_dit_pretrained_path: null
206
+ goal_prior_stage: stage2
207
+ goal_prior:
208
+ num_latents: 100
209
+ num_pose_tokens: 8
210
+ num_layer_groups: 5
211
+ latent_dim: 768
212
+ inner_dim: 512
213
+ lambda_pose: 0.3
214
+ context_token_dropout: 0.05
215
+ context_blackout_prob: 0.1
216
+ zero_init_value: false
217
+ syn_gate_bias_init: -2.0
218
+ gate_pose_tokens: false
219
+ latent_layout: free
220
+ grid_hw:
221
+ - 7
222
+ - 7
223
+ grid_window: 1
224
+ action_sees_ref: true
225
+ p_both: 0.55
226
+ p_ref_only: 0.15
227
+ p_syn_only: 0.3
228
+ flux2_lora_config:
229
+ enabled: true
230
+ rank: 16
231
+ alpha: 16.0
232
+ dropout: 0.0
233
+ save_lora_merged: false
234
+ save_trainable_only: true
235
+ target_suffixes:
236
+ - qkv
237
+ - proj
238
+ - linear1
239
+ - linear2
240
+ - img_mlp.0
241
+ - img_mlp.2
242
+ - txt_mlp.0
243
+ - txt_mlp.2
244
+ action_dit_config:
245
+ action_dim: 7
246
+ hidden_dim: 1024
247
+ num_heads: 24
248
+ attn_head_dim: 128
249
+ num_layers_double: 5
250
+ num_layers_single: 20
251
+ mlp_ratio: 4.0
252
+ max_action_horizon: 64
253
+ use_gradient_checkpointing: true
254
+ video_scheduler:
255
+ train_shift: 5.0
256
+ infer_shift: 5.0
257
+ num_train_timesteps: 1000
258
+ action_scheduler:
259
+ train_shift: 5.0
260
+ infer_shift: 5.0
261
+ num_train_timesteps: 1000
262
+ loss:
263
+ lambda_video: 0.5
264
+ lambda_action: 1.0
265
+ lambda_pose: 0.3
266
+ initialization:
267
+ repo: linjianman/LIT
268
+ revision: 765a2b2bf136f7039ab18978920c682f1aae65d2
269
+ path: imagewam/lit_stage2/model.pt
270
+ local_path: /tmp/resadapt-assets/released/imagewam/lit_stage2/model.pt
checkpoints/step_004000/metadata/dataset_stats.json ADDED
@@ -0,0 +1,1074 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "state": {
3
+ "default": {
4
+ "stepwise_min": [
5
+ [
6
+ -0.48278069496154785,
7
+ -0.3309336006641388,
8
+ 0.008128181099891663,
9
+ 1.002794623374939,
10
+ -3.6312508583068848,
11
+ -1.842738389968872,
12
+ -0.005453015677630901,
13
+ -0.042015016078948975
14
+ ]
15
+ ],
16
+ "stepwise_max": [
17
+ [
18
+ 0.2103137969970703,
19
+ 0.3904264271259308,
20
+ 1.472778081893921,
21
+ 3.7248642444610596,
22
+ 3.5618896484375,
23
+ 1.3863215446472168,
24
+ 0.04232141748070717,
25
+ 0.0013126095291227102
26
+ ]
27
+ ],
28
+ "global_min": [
29
+ -0.48278069496154785,
30
+ -0.3309336006641388,
31
+ 0.008128181099891663,
32
+ 1.002794623374939,
33
+ -3.6312508583068848,
34
+ -1.842738389968872,
35
+ -0.005453015677630901,
36
+ -0.042015016078948975
37
+ ],
38
+ "global_max": [
39
+ 0.2103137969970703,
40
+ 0.3904264271259308,
41
+ 1.472778081893921,
42
+ 3.7248642444610596,
43
+ 3.5618896484375,
44
+ 1.3863215446472168,
45
+ 0.04232141748070717,
46
+ 0.0013126095291227102
47
+ ],
48
+ "stepwise_q01": [
49
+ [
50
+ -0.48195967078208923,
51
+ -0.3300417959690094,
52
+ 0.008850866928696632,
53
+ 1.0129998922348022,
54
+ -3.619534969329834,
55
+ -1.8389623165130615,
56
+ -0.0013291656505316496,
57
+ -0.04173403978347778
58
+ ]
59
+ ],
60
+ "stepwise_q99": [
61
+ [
62
+ 0.20942535996437073,
63
+ 0.39041632413864136,
64
+ 1.4693706035614014,
65
+ 3.721859931945801,
66
+ 3.5605900287628174,
67
+ 1.3660067319869995,
68
+ 0.04197758436203003,
69
+ 0.0009995142463594675
70
+ ]
71
+ ],
72
+ "global_q01": [
73
+ -0.48195967078208923,
74
+ -0.3300417959690094,
75
+ 0.008850866928696632,
76
+ 1.0129998922348022,
77
+ -3.619534969329834,
78
+ -1.8389623165130615,
79
+ -0.0013291656505316496,
80
+ -0.04173403978347778
81
+ ],
82
+ "global_q99": [
83
+ 0.20942535996437073,
84
+ 0.39041632413864136,
85
+ 1.4693706035614014,
86
+ 3.721859931945801,
87
+ 3.5605900287628174,
88
+ 1.3660067319869995,
89
+ 0.04197758436203003,
90
+ 0.0009995142463594675
91
+ ],
92
+ "stepwise_mean": [
93
+ [
94
+ -0.052439481019973755,
95
+ 0.040308527648448944,
96
+ 0.7768229842185974,
97
+ 2.996931314468384,
98
+ -0.13651959598064423,
99
+ -0.11046145856380463,
100
+ 0.026323523372411728,
101
+ -0.026523755863308907
102
+ ]
103
+ ],
104
+ "stepwise_std": [
105
+ [
106
+ 0.10999669134616852,
107
+ 0.15002697706222534,
108
+ 0.38846009969711304,
109
+ 0.3232516348361969,
110
+ 0.7607998847961426,
111
+ 0.3058409094810486,
112
+ 0.014585155062377453,
113
+ 0.014462423510849476
114
+ ]
115
+ ],
116
+ "global_mean": [
117
+ -0.052439481019973755,
118
+ 0.040308527648448944,
119
+ 0.7768229842185974,
120
+ 2.996931314468384,
121
+ -0.13651959598064423,
122
+ -0.11046145856380463,
123
+ 0.026323523372411728,
124
+ -0.026523755863308907
125
+ ],
126
+ "global_std": [
127
+ 0.10999669134616852,
128
+ 0.15002697706222534,
129
+ 0.38846009969711304,
130
+ 0.3232516348361969,
131
+ 0.7607998847961426,
132
+ 0.3058409094810486,
133
+ 0.014585155062377453,
134
+ 0.014462423510849476
135
+ ]
136
+ }
137
+ },
138
+ "action": {
139
+ "default": {
140
+ "stepwise_min": [
141
+ [
142
+ -0.9375,
143
+ -0.9375,
144
+ -0.9375,
145
+ -0.24214285612106323,
146
+ -0.375,
147
+ -0.3642857074737549,
148
+ 0.0
149
+ ],
150
+ [
151
+ -0.9375,
152
+ -0.9375,
153
+ -0.9375,
154
+ -0.24214285612106323,
155
+ -0.375,
156
+ -0.3642857074737549,
157
+ 0.0
158
+ ],
159
+ [
160
+ -0.9375,
161
+ -0.9375,
162
+ -0.9375,
163
+ -0.24214285612106323,
164
+ -0.375,
165
+ -0.3642857074737549,
166
+ 0.0
167
+ ],
168
+ [
169
+ -0.9375,
170
+ -0.9375,
171
+ -0.9375,
172
+ -0.24214285612106323,
173
+ -0.375,
174
+ -0.3642857074737549,
175
+ 0.0
176
+ ],
177
+ [
178
+ -0.9375,
179
+ -0.9375,
180
+ -0.9375,
181
+ -0.24214285612106323,
182
+ -0.375,
183
+ -0.3642857074737549,
184
+ 0.0
185
+ ],
186
+ [
187
+ -0.9375,
188
+ -0.9375,
189
+ -0.9375,
190
+ -0.24214285612106323,
191
+ -0.375,
192
+ -0.3642857074737549,
193
+ 0.0
194
+ ],
195
+ [
196
+ -0.9375,
197
+ -0.9375,
198
+ -0.9375,
199
+ -0.24214285612106323,
200
+ -0.375,
201
+ -0.3642857074737549,
202
+ 0.0
203
+ ],
204
+ [
205
+ -0.9375,
206
+ -0.9375,
207
+ -0.9375,
208
+ -0.24214285612106323,
209
+ -0.375,
210
+ -0.3642857074737549,
211
+ 0.0
212
+ ],
213
+ [
214
+ -0.9375,
215
+ -0.9375,
216
+ -0.9375,
217
+ -0.24214285612106323,
218
+ -0.375,
219
+ -0.3642857074737549,
220
+ 0.0
221
+ ],
222
+ [
223
+ -0.9375,
224
+ -0.9375,
225
+ -0.9375,
226
+ -0.24214285612106323,
227
+ -0.375,
228
+ -0.3642857074737549,
229
+ 0.0
230
+ ],
231
+ [
232
+ -0.9375,
233
+ -0.9375,
234
+ -0.9375,
235
+ -0.24214285612106323,
236
+ -0.375,
237
+ -0.3642857074737549,
238
+ 0.0
239
+ ],
240
+ [
241
+ -0.9375,
242
+ -0.9375,
243
+ -0.9375,
244
+ -0.24214285612106323,
245
+ -0.375,
246
+ -0.3642857074737549,
247
+ 0.0
248
+ ],
249
+ [
250
+ -0.9375,
251
+ -0.9375,
252
+ -0.9375,
253
+ -0.24214285612106323,
254
+ -0.375,
255
+ -0.3642857074737549,
256
+ 0.0
257
+ ],
258
+ [
259
+ -0.9375,
260
+ -0.9375,
261
+ -0.9375,
262
+ -0.24214285612106323,
263
+ -0.375,
264
+ -0.3642857074737549,
265
+ 0.0
266
+ ],
267
+ [
268
+ -0.9375,
269
+ -0.9375,
270
+ -0.9375,
271
+ -0.24214285612106323,
272
+ -0.375,
273
+ -0.3642857074737549,
274
+ 0.0
275
+ ],
276
+ [
277
+ -0.9375,
278
+ -0.9375,
279
+ -0.9375,
280
+ -0.23678570985794067,
281
+ -0.375,
282
+ -0.3642857074737549,
283
+ 0.0
284
+ ]
285
+ ],
286
+ "stepwise_max": [
287
+ [
288
+ 0.9375,
289
+ 0.9375,
290
+ 0.9375,
291
+ 0.3557142913341522,
292
+ 0.375,
293
+ 0.375,
294
+ 1.0
295
+ ],
296
+ [
297
+ 0.9375,
298
+ 0.9375,
299
+ 0.9375,
300
+ 0.3557142913341522,
301
+ 0.375,
302
+ 0.375,
303
+ 1.0
304
+ ],
305
+ [
306
+ 0.9375,
307
+ 0.9375,
308
+ 0.9375,
309
+ 0.3557142913341522,
310
+ 0.375,
311
+ 0.375,
312
+ 1.0
313
+ ],
314
+ [
315
+ 0.9375,
316
+ 0.9375,
317
+ 0.9375,
318
+ 0.3557142913341522,
319
+ 0.375,
320
+ 0.375,
321
+ 1.0
322
+ ],
323
+ [
324
+ 0.9375,
325
+ 0.9375,
326
+ 0.9375,
327
+ 0.3557142913341522,
328
+ 0.375,
329
+ 0.375,
330
+ 1.0
331
+ ],
332
+ [
333
+ 0.9375,
334
+ 0.9375,
335
+ 0.9375,
336
+ 0.3557142913341522,
337
+ 0.375,
338
+ 0.375,
339
+ 1.0
340
+ ],
341
+ [
342
+ 0.9375,
343
+ 0.9375,
344
+ 0.9375,
345
+ 0.3557142913341522,
346
+ 0.375,
347
+ 0.375,
348
+ 1.0
349
+ ],
350
+ [
351
+ 0.9375,
352
+ 0.9375,
353
+ 0.9375,
354
+ 0.3557142913341522,
355
+ 0.375,
356
+ 0.375,
357
+ 1.0
358
+ ],
359
+ [
360
+ 0.9375,
361
+ 0.9375,
362
+ 0.9375,
363
+ 0.3557142913341522,
364
+ 0.375,
365
+ 0.375,
366
+ 1.0
367
+ ],
368
+ [
369
+ 0.9375,
370
+ 0.9375,
371
+ 0.9375,
372
+ 0.3557142913341522,
373
+ 0.375,
374
+ 0.375,
375
+ 1.0
376
+ ],
377
+ [
378
+ 0.9375,
379
+ 0.9375,
380
+ 0.9375,
381
+ 0.3557142913341522,
382
+ 0.375,
383
+ 0.375,
384
+ 1.0
385
+ ],
386
+ [
387
+ 0.9375,
388
+ 0.9375,
389
+ 0.9375,
390
+ 0.3557142913341522,
391
+ 0.375,
392
+ 0.375,
393
+ 1.0
394
+ ],
395
+ [
396
+ 0.9375,
397
+ 0.9375,
398
+ 0.9375,
399
+ 0.3557142913341522,
400
+ 0.375,
401
+ 0.375,
402
+ 1.0
403
+ ],
404
+ [
405
+ 0.9375,
406
+ 0.9375,
407
+ 0.9375,
408
+ 0.3557142913341522,
409
+ 0.375,
410
+ 0.375,
411
+ 1.0
412
+ ],
413
+ [
414
+ 0.9375,
415
+ 0.9375,
416
+ 0.9375,
417
+ 0.3557142913341522,
418
+ 0.375,
419
+ 0.375,
420
+ 1.0
421
+ ],
422
+ [
423
+ 0.9375,
424
+ 0.9375,
425
+ 0.9375,
426
+ 0.3557142913341522,
427
+ 0.375,
428
+ 0.375,
429
+ 1.0
430
+ ]
431
+ ],
432
+ "global_min": [
433
+ -0.9375,
434
+ -0.9375,
435
+ -0.9375,
436
+ -0.24214285612106323,
437
+ -0.375,
438
+ -0.3642857074737549,
439
+ 0.0
440
+ ],
441
+ "global_max": [
442
+ 0.9375,
443
+ 0.9375,
444
+ 0.9375,
445
+ 0.3557142913341522,
446
+ 0.375,
447
+ 0.375,
448
+ 1.0
449
+ ],
450
+ "stepwise_q01": [
451
+ [
452
+ -0.9375,
453
+ -0.9375,
454
+ -0.9375,
455
+ -0.2326499968767166,
456
+ -0.375,
457
+ -0.3642857074737549,
458
+ 0.0
459
+ ],
460
+ [
461
+ -0.9375,
462
+ -0.9375,
463
+ -0.9375,
464
+ -0.2326499968767166,
465
+ -0.375,
466
+ -0.3642857074737549,
467
+ 0.0
468
+ ],
469
+ [
470
+ -0.9375,
471
+ -0.9375,
472
+ -0.9375,
473
+ -0.2326499968767166,
474
+ -0.375,
475
+ -0.3642857074737549,
476
+ 0.0
477
+ ],
478
+ [
479
+ -0.9375,
480
+ -0.9375,
481
+ -0.9375,
482
+ -0.2326499968767166,
483
+ -0.375,
484
+ -0.3642857074737549,
485
+ 0.0
486
+ ],
487
+ [
488
+ -0.9375,
489
+ -0.9375,
490
+ -0.9375,
491
+ -0.2326499968767166,
492
+ -0.375,
493
+ -0.3642857074737549,
494
+ 0.0
495
+ ],
496
+ [
497
+ -0.9375,
498
+ -0.9375,
499
+ -0.9375,
500
+ -0.2326499968767166,
501
+ -0.375,
502
+ -0.3642857074737549,
503
+ 0.0
504
+ ],
505
+ [
506
+ -0.9375,
507
+ -0.9375,
508
+ -0.9375,
509
+ -0.2326499968767166,
510
+ -0.375,
511
+ -0.3642857074737549,
512
+ 0.0
513
+ ],
514
+ [
515
+ -0.9375,
516
+ -0.9375,
517
+ -0.9375,
518
+ -0.2326499968767166,
519
+ -0.375,
520
+ -0.3642857074737549,
521
+ 0.0
522
+ ],
523
+ [
524
+ -0.9375,
525
+ -0.9375,
526
+ -0.9375,
527
+ -0.2326499968767166,
528
+ -0.375,
529
+ -0.3642857074737549,
530
+ 0.0
531
+ ],
532
+ [
533
+ -0.9375,
534
+ -0.9375,
535
+ -0.9375,
536
+ -0.2326499968767166,
537
+ -0.375,
538
+ -0.3642857074737549,
539
+ 0.0
540
+ ],
541
+ [
542
+ -0.9375,
543
+ -0.9375,
544
+ -0.9375,
545
+ -0.2326499968767166,
546
+ -0.375,
547
+ -0.3642857074737549,
548
+ 0.0
549
+ ],
550
+ [
551
+ -0.9375,
552
+ -0.9375,
553
+ -0.9375,
554
+ -0.2326499968767166,
555
+ -0.375,
556
+ -0.3642857074737549,
557
+ 0.0
558
+ ],
559
+ [
560
+ -0.9375,
561
+ -0.9375,
562
+ -0.9375,
563
+ -0.2326499968767166,
564
+ -0.375,
565
+ -0.3642857074737549,
566
+ 0.0
567
+ ],
568
+ [
569
+ -0.9375,
570
+ -0.9375,
571
+ -0.9375,
572
+ -0.2326499968767166,
573
+ -0.375,
574
+ -0.3642857074737549,
575
+ 0.0
576
+ ],
577
+ [
578
+ -0.9375,
579
+ -0.9375,
580
+ -0.9375,
581
+ -0.2326499968767166,
582
+ -0.375,
583
+ -0.3642857074737549,
584
+ 0.0
585
+ ],
586
+ [
587
+ -0.9375,
588
+ -0.9375,
589
+ -0.9375,
590
+ -0.2326499968767166,
591
+ -0.375,
592
+ -0.3642857074737549,
593
+ 0.0
594
+ ]
595
+ ],
596
+ "stepwise_q99": [
597
+ [
598
+ 0.9375,
599
+ 0.9375,
600
+ 0.9375,
601
+ 0.35076430439949036,
602
+ 0.375,
603
+ 0.375,
604
+ 1.0
605
+ ],
606
+ [
607
+ 0.9375,
608
+ 0.9375,
609
+ 0.9375,
610
+ 0.35076430439949036,
611
+ 0.375,
612
+ 0.375,
613
+ 1.0
614
+ ],
615
+ [
616
+ 0.9375,
617
+ 0.9375,
618
+ 0.9375,
619
+ 0.35076430439949036,
620
+ 0.375,
621
+ 0.375,
622
+ 1.0
623
+ ],
624
+ [
625
+ 0.9375,
626
+ 0.9375,
627
+ 0.9375,
628
+ 0.35076430439949036,
629
+ 0.375,
630
+ 0.375,
631
+ 1.0
632
+ ],
633
+ [
634
+ 0.9375,
635
+ 0.9375,
636
+ 0.9375,
637
+ 0.35076430439949036,
638
+ 0.375,
639
+ 0.375,
640
+ 1.0
641
+ ],
642
+ [
643
+ 0.9375,
644
+ 0.9375,
645
+ 0.9375,
646
+ 0.35076430439949036,
647
+ 0.375,
648
+ 0.375,
649
+ 1.0
650
+ ],
651
+ [
652
+ 0.9375,
653
+ 0.9375,
654
+ 0.9375,
655
+ 0.35076430439949036,
656
+ 0.375,
657
+ 0.375,
658
+ 1.0
659
+ ],
660
+ [
661
+ 0.9375,
662
+ 0.9375,
663
+ 0.9375,
664
+ 0.35076430439949036,
665
+ 0.375,
666
+ 0.375,
667
+ 1.0
668
+ ],
669
+ [
670
+ 0.9375,
671
+ 0.9375,
672
+ 0.9375,
673
+ 0.35076430439949036,
674
+ 0.375,
675
+ 0.375,
676
+ 1.0
677
+ ],
678
+ [
679
+ 0.9375,
680
+ 0.9375,
681
+ 0.9375,
682
+ 0.35076430439949036,
683
+ 0.375,
684
+ 0.375,
685
+ 1.0
686
+ ],
687
+ [
688
+ 0.9375,
689
+ 0.9375,
690
+ 0.9375,
691
+ 0.35076430439949036,
692
+ 0.375,
693
+ 0.375,
694
+ 1.0
695
+ ],
696
+ [
697
+ 0.9375,
698
+ 0.9375,
699
+ 0.9375,
700
+ 0.35076430439949036,
701
+ 0.375,
702
+ 0.375,
703
+ 1.0
704
+ ],
705
+ [
706
+ 0.9375,
707
+ 0.9375,
708
+ 0.9375,
709
+ 0.35076430439949036,
710
+ 0.375,
711
+ 0.375,
712
+ 1.0
713
+ ],
714
+ [
715
+ 0.9375,
716
+ 0.9375,
717
+ 0.9375,
718
+ 0.35076430439949036,
719
+ 0.375,
720
+ 0.375,
721
+ 1.0
722
+ ],
723
+ [
724
+ 0.9375,
725
+ 0.9375,
726
+ 0.9375,
727
+ 0.35076430439949036,
728
+ 0.375,
729
+ 0.375,
730
+ 1.0
731
+ ],
732
+ [
733
+ 0.9375,
734
+ 0.9375,
735
+ 0.9375,
736
+ 0.35076430439949036,
737
+ 0.375,
738
+ 0.375,
739
+ 1.0
740
+ ]
741
+ ],
742
+ "global_q01": [
743
+ -0.9375,
744
+ -0.9375,
745
+ -0.9375,
746
+ -0.2326499968767166,
747
+ -0.375,
748
+ -0.3642857074737549,
749
+ 0.0
750
+ ],
751
+ "global_q99": [
752
+ 0.9375,
753
+ 0.9375,
754
+ 0.9375,
755
+ 0.35076430439949036,
756
+ 0.375,
757
+ 0.375,
758
+ 1.0
759
+ ],
760
+ "stepwise_mean": [
761
+ [
762
+ 0.07509180158376694,
763
+ 0.0942603200674057,
764
+ -0.10477570444345474,
765
+ -0.0006721731624566019,
766
+ 0.006502428557723761,
767
+ -0.004753417335450649,
768
+ 0.5156609416007996
769
+ ],
770
+ [
771
+ 0.0742194727063179,
772
+ 0.0943378284573555,
773
+ -0.10369047522544861,
774
+ -0.000646332511678338,
775
+ 0.006517794448882341,
776
+ -0.0048326049000024796,
777
+ 0.5151447653770447
778
+ ],
779
+ [
780
+ 0.07300770282745361,
781
+ 0.09434358030557632,
782
+ -0.10265536606311798,
783
+ -0.000612989766523242,
784
+ 0.006519764196127653,
785
+ -0.004911692347377539,
786
+ 0.5146285891532898
787
+ ],
788
+ [
789
+ 0.07130849361419678,
790
+ 0.09427326172590256,
791
+ -0.10167480260133743,
792
+ -0.0005704726208932698,
793
+ 0.006502891890704632,
794
+ -0.004996591247618198,
795
+ 0.5141124129295349
796
+ ],
797
+ [
798
+ 0.06903650611639023,
799
+ 0.09413644671440125,
800
+ -0.10072868317365646,
801
+ -0.0005176868289709091,
802
+ 0.006469315383583307,
803
+ -0.005088216625154018,
804
+ 0.51359623670578
805
+ ],
806
+ [
807
+ 0.06624066829681396,
808
+ 0.09398345649242401,
809
+ -0.09977640211582184,
810
+ -0.00045814085751771927,
811
+ 0.006425761617720127,
812
+ -0.005182641092687845,
813
+ 0.5130800008773804
814
+ ],
815
+ [
816
+ 0.0630466639995575,
817
+ 0.09385841339826584,
818
+ -0.09878652542829514,
819
+ -0.0003933484840672463,
820
+ 0.006386647932231426,
821
+ -0.0052768453024327755,
822
+ 0.5125638246536255
823
+ ],
824
+ [
825
+ 0.059566862881183624,
826
+ 0.0937969759106636,
827
+ -0.09773147106170654,
828
+ -0.0003290225868113339,
829
+ 0.006361101288348436,
830
+ -0.005367835983633995,
831
+ 0.5120476484298706
832
+ ],
833
+ [
834
+ 0.055894214659929276,
835
+ 0.09381034225225449,
836
+ -0.09658003598451614,
837
+ -0.00026784592773765326,
838
+ 0.006349042057991028,
839
+ -0.0054576038382947445,
840
+ 0.5115315318107605
841
+ ],
842
+ [
843
+ 0.05210842192173004,
844
+ 0.09390032291412354,
845
+ -0.09531887620687485,
846
+ -0.00021082810417283326,
847
+ 0.00635516969487071,
848
+ -0.005544654093682766,
849
+ 0.5110153555870056
850
+ ],
851
+ [
852
+ 0.04825857654213905,
853
+ 0.09406367689371109,
854
+ -0.09393264353275299,
855
+ -0.00015976655413396657,
856
+ 0.006377693731337786,
857
+ -0.005628861952573061,
858
+ 0.5104991793632507
859
+ ],
860
+ [
861
+ 0.04438157379627228,
862
+ 0.09429564327001572,
863
+ -0.0924036055803299,
864
+ -0.00011604859173530713,
865
+ 0.006416680756956339,
866
+ -0.005709674209356308,
867
+ 0.5099830031394958
868
+ ],
869
+ [
870
+ 0.040517039597034454,
871
+ 0.09459187835454941,
872
+ -0.09071968495845795,
873
+ -8.002247341210023e-05,
874
+ 0.006473013199865818,
875
+ -0.0057876212522387505,
876
+ 0.5094667673110962
877
+ ],
878
+ [
879
+ 0.0366998165845871,
880
+ 0.09495285153388977,
881
+ -0.08885672688484192,
882
+ -5.3317955462262034e-05,
883
+ 0.00654600327834487,
884
+ -0.00586199015378952,
885
+ 0.5089505910873413
886
+ ],
887
+ [
888
+ 0.0329657644033432,
889
+ 0.09536924213171005,
890
+ -0.08679967373609543,
891
+ -3.667248893179931e-05,
892
+ 0.006634271237999201,
893
+ -0.0059333425015211105,
894
+ 0.5084344148635864
895
+ ],
896
+ [
897
+ 0.029352381825447083,
898
+ 0.09583891928195953,
899
+ -0.08452724665403366,
900
+ -2.956177922897041e-05,
901
+ 0.006736791227012873,
902
+ -0.006001695990562439,
903
+ 0.5079182386398315
904
+ ]
905
+ ],
906
+ "stepwise_std": [
907
+ [
908
+ 0.35524481534957886,
909
+ 0.37755152583122253,
910
+ 0.4687630534172058,
911
+ 0.03886449709534645,
912
+ 0.06502853333950043,
913
+ 0.07652278244495392,
914
+ 0.5014081597328186
915
+ ],
916
+ [
917
+ 0.35528722405433655,
918
+ 0.37743693590164185,
919
+ 0.46985360980033875,
920
+ 0.03892502188682556,
921
+ 0.06516389548778534,
922
+ 0.07675494253635406,
923
+ 0.5014236569404602
924
+ ],
925
+ [
926
+ 0.35516318678855896,
927
+ 0.3772093653678894,
928
+ 0.4709104001522064,
929
+ 0.0389784574508667,
930
+ 0.06527172774076462,
931
+ 0.07698068022727966,
932
+ 0.5014386177062988
933
+ ],
934
+ [
935
+ 0.354702353477478,
936
+ 0.37677210569381714,
937
+ 0.47192680835723877,
938
+ 0.039021968841552734,
939
+ 0.0653405413031578,
940
+ 0.07719580829143524,
941
+ 0.5014529824256897
942
+ ],
943
+ [
944
+ 0.35369622707366943,
945
+ 0.37603849172592163,
946
+ 0.47290918231010437,
947
+ 0.039052922278642654,
948
+ 0.06536615639925003,
949
+ 0.07739786058664322,
950
+ 0.5014666318893433
951
+ ],
952
+ [
953
+ 0.3520404100418091,
954
+ 0.37497878074645996,
955
+ 0.47387173771858215,
956
+ 0.039072245359420776,
957
+ 0.06535857915878296,
958
+ 0.07758834213018417,
959
+ 0.5014797449111938
960
+ ],
961
+ [
962
+ 0.34978508949279785,
963
+ 0.3736053705215454,
964
+ 0.4748258590698242,
965
+ 0.039080556482076645,
966
+ 0.06534324586391449,
967
+ 0.07776463776826859,
968
+ 0.5014922022819519
969
+ ],
970
+ [
971
+ 0.3470368981361389,
972
+ 0.3719497621059418,
973
+ 0.47577232122421265,
974
+ 0.03907886520028114,
975
+ 0.06533455848693848,
976
+ 0.07792745530605316,
977
+ 0.5015040040016174
978
+ ],
979
+ [
980
+ 0.34389936923980713,
981
+ 0.37007132172584534,
982
+ 0.4767095148563385,
983
+ 0.039068553596735,
984
+ 0.0653340294957161,
985
+ 0.07807470113039017,
986
+ 0.5015152096748352
987
+ ],
988
+ [
989
+ 0.34047427773475647,
990
+ 0.3680059611797333,
991
+ 0.4776283800601959,
992
+ 0.039051979780197144,
993
+ 0.06534522771835327,
994
+ 0.0782046914100647,
995
+ 0.5015258193016052
996
+ ],
997
+ [
998
+ 0.33682477474212646,
999
+ 0.3657948970794678,
1000
+ 0.4785217344760895,
1001
+ 0.03903175890445709,
1002
+ 0.06536738574504852,
1003
+ 0.07831670343875885,
1004
+ 0.5015357732772827
1005
+ ],
1006
+ [
1007
+ 0.33300837874412537,
1008
+ 0.36347952485084534,
1009
+ 0.4793775975704193,
1010
+ 0.0390087328851223,
1011
+ 0.06540162116289139,
1012
+ 0.07841045409440994,
1013
+ 0.5015451312065125
1014
+ ],
1015
+ [
1016
+ 0.3290861248970032,
1017
+ 0.3610839545726776,
1018
+ 0.48018527030944824,
1019
+ 0.03898213058710098,
1020
+ 0.06545032560825348,
1021
+ 0.07848569005727768,
1022
+ 0.5015538930892944
1023
+ ],
1024
+ [
1025
+ 0.3251110315322876,
1026
+ 0.35864168405532837,
1027
+ 0.48092642426490784,
1028
+ 0.038955289870500565,
1029
+ 0.06551124900579453,
1030
+ 0.07854204624891281,
1031
+ 0.5015619993209839
1032
+ ],
1033
+ [
1034
+ 0.3211345374584198,
1035
+ 0.35618487000465393,
1036
+ 0.4815789461135864,
1037
+ 0.0389283113181591,
1038
+ 0.06558437645435333,
1039
+ 0.07858172059059143,
1040
+ 0.5015695095062256
1041
+ ],
1042
+ [
1043
+ 0.31721803545951843,
1044
+ 0.3537396192550659,
1045
+ 0.48211759328842163,
1046
+ 0.038900382816791534,
1047
+ 0.06566666066646576,
1048
+ 0.07860506325960159,
1049
+ 0.5015763640403748
1050
+ ]
1051
+ ],
1052
+ "global_mean": [
1053
+ 0.055730998516082764,
1054
+ 0.09436333179473877,
1055
+ -0.0961848646402359,
1056
+ -0.0003221393853891641,
1057
+ 0.0064733983017504215,
1058
+ -0.005395954940468073,
1059
+ 0.5117895603179932
1060
+ ],
1061
+ "global_std": [
1062
+ 0.3424273431301117,
1063
+ 0.3689939081668854,
1064
+ 0.4760483205318451,
1065
+ 0.03900081664323807,
1066
+ 0.06536699831485748,
1067
+ 0.0778384581208229,
1068
+ 0.5015087723731995
1069
+ ]
1070
+ }
1071
+ },
1072
+ "num_episodes": 1712,
1073
+ "num_transition": 277713
1074
+ }
checkpoints/step_004000/metadata/environment.txt ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Farama-Notifications==0.0.6
2
+ GitPython==3.2.0
3
+ Jinja2==3.1.6
4
+ MarkupSafe==3.0.4
5
+ PyYAML==6.0.3
6
+ Pygments==2.21.0
7
+ accelerate==1.12.0
8
+ aiohappyeyeballs==2.7.1
9
+ aiohttp==3.14.4
10
+ aiosignal==1.4.0
11
+ annotated-types==0.8.0
12
+ antlr4-python3-runtime==4.9.3
13
+ anyio==4.15.1
14
+ attrs==26.1.0
15
+ av==16.0.1
16
+ boto3==1.35.99
17
+ botocore==1.35.99
18
+ certifi==2026.7.22
19
+ charset-normalizer==3.5.2
20
+ click==8.5.0
21
+ cloudpickle==3.1.2
22
+ cuda-bindings==13.4.3
23
+ cuda-pathfinder==1.8.3
24
+ cuda-toolkit==13.0.2
25
+ datasets==4.8.5
26
+ diffusers==0.35.2
27
+ dill==0.4.1
28
+ draccus==0.10.0
29
+ einops==0.8.1
30
+ filelock==4.0.12
31
+ frozenlist==1.8.0
32
+ fsspec==2026.2.0
33
+ gitdb==4.0.12
34
+ gymnasium==1.2.1
35
+ h11==0.16.0
36
+ hf-xet==1.6.0
37
+ httpcore==1.0.9
38
+ httpx==0.28.1
39
+ huggingface-hub==0.35.3
40
+ hydra-core==1.3.2
41
+ idna==3.20
42
+ imageio==2.37.0
43
+ importlib_metadata==9.0.1
44
+ iniconfig==2.3.0
45
+ jmespath==1.1.0
46
+ jsonlines==4.0.0
47
+ lerobot==0.4.4
48
+ markdown-it-py==4.2.0
49
+ mdurl==0.1.2
50
+ mergedeep==1.3.4
51
+ mpmath==1.3.0
52
+ multidict==6.9.1
53
+ multiprocess==0.70.19
54
+ mypy_extensions==1.1.0
55
+ networkx==3.7
56
+ numpy==1.26.4
57
+ nvidia-cublas==13.1.0.3
58
+ nvidia-cuda-cupti==13.0.85
59
+ nvidia-cuda-nvrtc==13.0.88
60
+ nvidia-cuda-runtime==13.0.96
61
+ nvidia-cudnn-cu13==9.19.0.56
62
+ nvidia-cufft==12.0.0.61
63
+ nvidia-cufile==1.15.1.6
64
+ nvidia-curand==10.4.0.35
65
+ nvidia-cusolver==12.0.4.66
66
+ nvidia-cusparse==12.6.3.3
67
+ nvidia-cusparselt-cu13==0.8.0
68
+ nvidia-nccl-cu13==2.28.9
69
+ nvidia-nvjitlink==13.0.88
70
+ nvidia-nvshmem-cu13==3.4.5
71
+ nvidia-nvtx==13.0.85
72
+ omegaconf==2.3.0
73
+ opencv-python-headless==4.11.0.86
74
+ packaging==26.3
75
+ pandas==3.0.6
76
+ pillow==12.0.0
77
+ platformdirs==4.12.3
78
+ pluggy==1.6.0
79
+ propcache==0.5.4
80
+ protobuf==7.36.2
81
+ psutil==7.2.2
82
+ pyarrow==23.0.0
83
+ pydantic==2.13.5
84
+ pydantic_core==2.46.5
85
+ pytest==8.4.2
86
+ python-dateutil==2.9.0.post0
87
+ pyyaml-include==1.4.1
88
+ regex==2026.9.29
89
+ requests==2.34.2
90
+ rich==14.2.0
91
+ s3transfer==0.10.4
92
+ safetensors==0.6.2
93
+ sentry-sdk==2.71.0
94
+ setuptools==81.0.0
95
+ six==1.17.0
96
+ smmap==5.0.3
97
+ sympy==1.14.0
98
+ termcolor==3.2.0
99
+ tokenizers==0.22.2
100
+ toml==0.10.2
101
+ torch==2.11.0
102
+ torchvision==0.26.0
103
+ tqdm==4.70.1
104
+ transformers==4.56.1
105
+ triton==3.6.0
106
+ typing-inspect==0.9.0
107
+ typing-inspection==0.4.4
108
+ typing_extensions==4.16.0
109
+ urllib3==2.8.0
110
+ wandb==0.28.1
111
+ xxhash==4.0.1
112
+ yarl==1.25.1
113
+ zipp==4.1.1
checkpoints/step_004000/metadata/parameters.json ADDED
@@ -0,0 +1,218 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "components": {
3
+ "flux_lora": {
4
+ "total": 23101440,
5
+ "trainable": 23101440,
6
+ "frozen": 0
7
+ },
8
+ "flux_dense": {
9
+ "total": 3875544576,
10
+ "trainable": 0,
11
+ "frozen": 3875544576
12
+ },
13
+ "action_expert": {
14
+ "total": 642012416,
15
+ "trainable": 642012416,
16
+ "frozen": 0
17
+ },
18
+ "lit_interface": {
19
+ "total": 165439365,
20
+ "trainable": 165439365,
21
+ "frozen": 0
22
+ },
23
+ "pose_decoder": {
24
+ "total": 3414536,
25
+ "trainable": 3414536,
26
+ "frozen": 0
27
+ },
28
+ "proprio_encoder": {
29
+ "total": 69120,
30
+ "trainable": 69120,
31
+ "frozen": 0
32
+ },
33
+ "other_stage2": {
34
+ "total": 0,
35
+ "trainable": 0,
36
+ "frozen": 0
37
+ },
38
+ "qwen": {
39
+ "total": 0,
40
+ "trainable": 0,
41
+ "frozen": 0
42
+ },
43
+ "vae": {
44
+ "total": 84046115,
45
+ "trainable": 0,
46
+ "frozen": 84046115
47
+ }
48
+ },
49
+ "totals": {
50
+ "total": 4793627568,
51
+ "trainable": 834036877,
52
+ "frozen": 3959590691,
53
+ "lora": 23101440,
54
+ "dense_trainable": 810935437
55
+ },
56
+ "optimizer_groups": [
57
+ {
58
+ "name": "flux_lora",
59
+ "tensors": 160,
60
+ "parameters": 23101440,
61
+ "lr": 3e-05,
62
+ "weight_decay": 0.01
63
+ },
64
+ {
65
+ "name": "action_expert",
66
+ "tensors": 34,
67
+ "parameters": 124003584,
68
+ "lr": 1e-05,
69
+ "weight_decay": 0.01
70
+ },
71
+ {
72
+ "name": "action_expert",
73
+ "tensors": 18,
74
+ "parameters": 113247232,
75
+ "lr": 1e-05,
76
+ "weight_decay": 0.01
77
+ },
78
+ {
79
+ "name": "action_expert",
80
+ "tensors": 20,
81
+ "parameters": 125830400,
82
+ "lr": 1e-05,
83
+ "weight_decay": 0.01
84
+ },
85
+ {
86
+ "name": "action_expert",
87
+ "tensors": 20,
88
+ "parameters": 125830400,
89
+ "lr": 1e-05,
90
+ "weight_decay": 0.01
91
+ },
92
+ {
93
+ "name": "action_expert",
94
+ "tensors": 20,
95
+ "parameters": 125830400,
96
+ "lr": 1e-05,
97
+ "weight_decay": 0.01
98
+ },
99
+ {
100
+ "name": "action_expert",
101
+ "tensors": 6,
102
+ "parameters": 27270400,
103
+ "lr": 1e-05,
104
+ "weight_decay": 0.01
105
+ },
106
+ {
107
+ "name": "proprio_encoder",
108
+ "tensors": 2,
109
+ "parameters": 69120,
110
+ "lr": 1e-05,
111
+ "weight_decay": 0.01
112
+ },
113
+ {
114
+ "name": "lit_interface",
115
+ "tensors": 190,
116
+ "parameters": 127648132,
117
+ "lr": 1e-05,
118
+ "weight_decay": 0.01
119
+ },
120
+ {
121
+ "name": "lit_interface",
122
+ "tensors": 51,
123
+ "parameters": 37791233,
124
+ "lr": 1e-05,
125
+ "weight_decay": 0.01
126
+ },
127
+ {
128
+ "name": "pose_decoder",
129
+ "tensors": 8,
130
+ "parameters": 3414536,
131
+ "lr": 1e-05,
132
+ "weight_decay": 0.01
133
+ }
134
+ ],
135
+ "lora_targets": [
136
+ "double_blocks.0.img_attn.qkv",
137
+ "double_blocks.0.img_attn.proj",
138
+ "double_blocks.0.img_mlp.0",
139
+ "double_blocks.0.img_mlp.2",
140
+ "double_blocks.0.txt_attn.qkv",
141
+ "double_blocks.0.txt_attn.proj",
142
+ "double_blocks.0.txt_mlp.0",
143
+ "double_blocks.0.txt_mlp.2",
144
+ "double_blocks.1.img_attn.qkv",
145
+ "double_blocks.1.img_attn.proj",
146
+ "double_blocks.1.img_mlp.0",
147
+ "double_blocks.1.img_mlp.2",
148
+ "double_blocks.1.txt_attn.qkv",
149
+ "double_blocks.1.txt_attn.proj",
150
+ "double_blocks.1.txt_mlp.0",
151
+ "double_blocks.1.txt_mlp.2",
152
+ "double_blocks.2.img_attn.qkv",
153
+ "double_blocks.2.img_attn.proj",
154
+ "double_blocks.2.img_mlp.0",
155
+ "double_blocks.2.img_mlp.2",
156
+ "double_blocks.2.txt_attn.qkv",
157
+ "double_blocks.2.txt_attn.proj",
158
+ "double_blocks.2.txt_mlp.0",
159
+ "double_blocks.2.txt_mlp.2",
160
+ "double_blocks.3.img_attn.qkv",
161
+ "double_blocks.3.img_attn.proj",
162
+ "double_blocks.3.img_mlp.0",
163
+ "double_blocks.3.img_mlp.2",
164
+ "double_blocks.3.txt_attn.qkv",
165
+ "double_blocks.3.txt_attn.proj",
166
+ "double_blocks.3.txt_mlp.0",
167
+ "double_blocks.3.txt_mlp.2",
168
+ "double_blocks.4.img_attn.qkv",
169
+ "double_blocks.4.img_attn.proj",
170
+ "double_blocks.4.img_mlp.0",
171
+ "double_blocks.4.img_mlp.2",
172
+ "double_blocks.4.txt_attn.qkv",
173
+ "double_blocks.4.txt_attn.proj",
174
+ "double_blocks.4.txt_mlp.0",
175
+ "double_blocks.4.txt_mlp.2",
176
+ "single_blocks.0.linear1",
177
+ "single_blocks.0.linear2",
178
+ "single_blocks.1.linear1",
179
+ "single_blocks.1.linear2",
180
+ "single_blocks.2.linear1",
181
+ "single_blocks.2.linear2",
182
+ "single_blocks.3.linear1",
183
+ "single_blocks.3.linear2",
184
+ "single_blocks.4.linear1",
185
+ "single_blocks.4.linear2",
186
+ "single_blocks.5.linear1",
187
+ "single_blocks.5.linear2",
188
+ "single_blocks.6.linear1",
189
+ "single_blocks.6.linear2",
190
+ "single_blocks.7.linear1",
191
+ "single_blocks.7.linear2",
192
+ "single_blocks.8.linear1",
193
+ "single_blocks.8.linear2",
194
+ "single_blocks.9.linear1",
195
+ "single_blocks.9.linear2",
196
+ "single_blocks.10.linear1",
197
+ "single_blocks.10.linear2",
198
+ "single_blocks.11.linear1",
199
+ "single_blocks.11.linear2",
200
+ "single_blocks.12.linear1",
201
+ "single_blocks.12.linear2",
202
+ "single_blocks.13.linear1",
203
+ "single_blocks.13.linear2",
204
+ "single_blocks.14.linear1",
205
+ "single_blocks.14.linear2",
206
+ "single_blocks.15.linear1",
207
+ "single_blocks.15.linear2",
208
+ "single_blocks.16.linear1",
209
+ "single_blocks.16.linear2",
210
+ "single_blocks.17.linear1",
211
+ "single_blocks.17.linear2",
212
+ "single_blocks.18.linear1",
213
+ "single_blocks.18.linear2",
214
+ "single_blocks.19.linear1",
215
+ "single_blocks.19.linear2"
216
+ ],
217
+ "qwen_cached_and_not_instantiated": true
218
+ }
checkpoints/step_004000/metadata/pretraining_report.md ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # LIT ImageWAM pre-training report
2
+
3
+ Inspected on 2026-10-03. All tabular/video checks and both full-sized model checks passed. The production DeepSpeed/fused AdamW path passed Stage-1 bridging, one Stage-2 update, and training-subset evaluation; checkpoint serialization and FP32 optimizer-state verification passed before launch. A subsequent multi-update activation-checkpoint benchmark exposed a second-update Stage-2 OOM that the initial one-update preflight missed; bounded optimizer groups resolve it and passed four consecutive full-sized updates.
4
+
5
+ 1. **Code.** Released ImageWAM LIT fork `jianmanlincjx/ImageWAM`, commit `1792d100099771fe404ebca5dcd606725141585b`; `imagewam.runtime.create_imagewam_flux2_klein`, `ImageWAM.training_loss`, and `Wan22Trainer`. FLUX source commit `50fe5162777813d869182b139e83b10743caef15`. Configs and normalization come from `linjianman/LIT` revision `765a2b2bf136f7039ab18978920c682f1aae65d2`, `imagewam/lit_stage{1,2}`. Local adapters add gripper conversion, task sampling, online logging, and resumable uploads.
6
+
7
+ 2. **Schema.** LeRobot v3; `observation.images.front` and `.wrist`: RGB 256×256 video; `observation.state`: float32[8]; `action`: float32[7]; float32 `timestamp`; int64 `frame_index`, `episode_index`, `index`, `task_index`. Instruction strings live in `meta/tasks.parquet`; episode intervals and video offsets in `meta/episodes/...parquet`; provenance in `meta/source_episodes.jsonl`. Frequency is 20 Hz.
8
+
9
+ 3. **Tasks and episodes.** 111 distinct instruction labels, 19,961 episodes, 3,080,750 frames. Per-label episodes range 34–649; frames range 4,447–137,237. Every label's episode/frame totals and source breakdown are in [task_distribution.csv](task_distribution.csv). These are instruction labels, not the 10,030 perturbation configurations of the LIBERO-Plus evaluation benchmark.
10
+
11
+ 4. **Composition.** NVIDIA LIBERO: 5,614 episodes / 842,714 frames / 111 labels; Plus: 14,347 episodes / 2,238,036 frames / 40 labels. NVIDIA includes LIBERO-90 (3,921 episodes), LIBERO-10 (379), Goal (428), Object (454), Spatial (432). Plus lacks per-episode perturbation-axis labels in the merged provenance. Stored Plus shares are 71.88% of episodes and 72.65% of frames. The original released ImageWAM recipe used four LIBERO suites without LIBERO-90 or Plus; this requested mixture changes training coverage deliberately.
12
+
13
+ 5. **Images.** Decode RGB at recorded video timestamps. Convert uint8 to float32 [0,1], then bilinear antialiased aspect-preserving resize into 128×128 per camera, symmetric zero padding for nonsquare images. Observed sources are square, so no padding is applied. No image rotation, crop, or later upsampling. The stored orientation matches an inspected original `yuanty/LIBERO-fastwam` training image. Concatenate front then wrist to 128×256 and normalize to [-1,1]. Apply the released color/noise augmentation only after downsampling; disable its random crop and ±8° rotation to retain the complete view. Current and t+16 future images receive the same preprocessing.
14
+
15
+ 6. **State/action.** State is EEF xyz, axis-angle orientation xyz, and two signed finger joint positions. Preserve all eight state values and the six arm action channels. Arm actions are LIBERO OSC control commands; do not treat them as absolute poses or integrate them as unscaled metric displacements. Convert dataset gripper −1=open/+1=close to released ImageWAM 1=open/0=close with **(1−g)/2 before normalization**. Use the inverse 1−2g at deployment. The goal is the observed 8D state at **t+16**, not the episode's final state. Actions cover t through t+15.
16
+
17
+ 7. **Inconsistencies and audit.** The dataset card's 0/1 gripper prose contradicts its merge formula. Measured next-step finger-width changes confirm the stored −1/open and +1/close convention in both sources. Released action statistics instead expect 0/close and 1/open; direct reuse without the conversion would invalidate labels. NVIDIA states use Arrow fixed-size lists; Plus uses variable lists, with all rows verified width 8. Use the native v3 loader with explicit features to reconcile storage types without changing values. Current torchvision removed the VideoReader API used by LeRobot; a direct PyAV shim preserves RGB decoding with strict timestamp tolerance. Selected-row queries replace expensive whole-column reads. Stage 1 skips video decoding, and random data-error fallback is disabled. Keep released global min/max normalization in both phases; expanded mixture extrema extend beyond [-1,1] (state approximately [-2.29,1.82], action [-1.28,1.06]), entirely inside the processor's existing [-5,5] clamp. Do not clip to [-1,1], change rotation parameterization, or substitute the dataset's OpenPI normalization.
18
+
19
+ All 3,080,750 tabular rows passed null, finite-value, vector-width, index, episode-contiguity, instruction, and frame/timestamp checks. Maximum timestamp deviation from frame_index/20: 7.63e−7 seconds. Video audit checks all 108 files, both camera intervals in all 19,961 episodes, and first/middle/last RGB frames of every video file. This is a sampled image decode audit, not exhaustive frame corruption detection. The dataset has no independent simulator replay, success, or collision certification; do not infer these from structural validity.
20
+
21
+ The general LIT README describes a latent-only visual firewall, H=10, and quantile normalization; the released **ImageWAM** config/code instead use H=16, global min/max, and direct reference plus synthetic latent channels. The user explicitly selected the released ImageWAM recipe: 55% both, 15% reference only, 30% synthetic only. Preserve that behavior and its 100×768 latents, 8 pose tokens, 512 inner width, and 5 parameter groups. Stage-2 loss also includes the upstream image objective; it is not action-plus-pose alone.
22
+
23
+ 8. **Sampling.** Shuffle uniform cycles over all 111 task IDs, then choose a source uniformly when both exist, an eligible episode uniformly, and a valid anchor uniformly. Require t+16<episode_length, so neither action nor goal crosses an episode or is padded. There are 2,761,374 valid anchors. Exact task counts differ by at most one per cycle; a probe produced exactly 100 anchors for every task and reproduced resume offsets. Expected Plus share becomes **40/(2×111)=18.02%**, because 71 labels have only NVIDIA episodes. Sampler passes round down to whole effective batches (2,761,216 Stage-1 draws / 2,761,360 Stage-2 draws), avoiding partial accumulation updates at an epoch boundary. Sampling uses replacement within tasks; an epoch does not mean exhaustive coverage.
24
+
25
+ 9. **Stage 1.** Vision-free action prior; frozen FLUX backbone, trained action expert and goal encoder, cached Qwen3 embeddings. Follow released initialization from the FLUX-derived ActionDiT initializer, rather than an already trained action checkpoint. **10,000 optimizer updates**, fused AdamW lr 2e−4, betas (0.9,0.95), decay .01, 2,000-step warmup, cosine decay to 1% of peak, gradient clip 1.0, seed 42, BF16, gradient checkpointing. Released launcher uses 8 GPUs × local batch 64; use one GPU × microbatch 256 × accumulation 2, preserving effective batch **512**. Stage-1 pose/auxiliary loss is explicitly zero; the observed terminal pose is conditioning, not a prediction objective.
26
+
27
+ 10. **Stage 2 and operation.** Strict Stage-1 bridge initialization; FLUX.2 Klein base 4B backbone, action expert, latent interface, and pose decoder train; Qwen3 and VAE remain frozen. Fused AdamW with released lr **1e−4 for all trainable parameters**, decay .01, betas (0.9,0.95), clip 1.0, BF16, checkpointing; loss = action + .5×image + .3×pose. Preserve **34,720 optimizer updates** of the released 10-epoch run, warmup **1,736** updates. Bound optimizer groups to 128 million parameters (39 groups, maximum observed 126,181,762) to prevent a 17.46 GiB DeepSpeed flatten/cast temporary. Every group retains the released lr/betas/decay and shared global clipping; masters and moments remain FP32. One GPU × microbatch 80 × accumulation 1 preserves the original 8×10 effective batch **80**. This gives 2,777,600 draws (~1.006 sampler epochs) on the larger mixture; reusing ten mixture epochs would greatly increase the released update budget. The max_steps override intentionally takes precedence over the retained released num_epochs metadata.
28
+
29
+ Separate online W&B runs in `star1/parc2026-lit-imagewam`, group `lit-libero-nvidia-plus-128`. Log every 10 optimizer updates: accumulated-update mean total/action/image/pose losses, lr, optimizer_step, epoch, grad norm, samples/s, and allocated/reserved/peak GPU memory. Cache all 111 Qwen instructions to avoid occupying training VRAM with the extra language model. The released runtime reuses the training dataset for evaluation. Preserve Stage-2 evaluation every 100 updates with 8 samples, reporting training-subset action/pose/image diagnostics, not held-out validation or simulator success. Save weight checkpoints every 5,000 updates and at each phase's final update; retain only the latest local optimizer/RNG state for resume. Keep only the two newest uploaded local checkpoints per phase and all final checkpoints; older complete snapshots remain on Hugging Face. Completed Stage-1 optimizer state is retired when Stage 2 starts. If disk cannot hold two full FP32 optimizer states, obsolete local state is reclaimed before the next write; independently saved weight checkpoints remain available. Stage-1 final: 10,000. Stage-2 final: 34,720. An independent retrying uploader sends immutable weight-only folders plus configs, preprocessing, audits, task distribution, and pinned revisions to private model repo `Ratstar216/LIT-ImageWAM-LIBERO-NVIDIA-PLUS-128`, using Hugging Face resumable large-folder caching and content addressing. Stage 2 starts only after Stage 1 completes successfully.
30
+
31
+ ## Changes from the released configuration
32
+
33
+ | Setting | Released | This run | Reason |
34
+ |---|---|---|---|
35
+ | Training GPUs | 8 | 1 RTX PRO 6000 Blackwell 96 GB | Available hardware |
36
+ | Local microbatch / accumulation | Stage 1: 64 / 1; Stage 2: 10 / 1 (8 GPUs) | Stage 1: 256 / 2; Stage 2: 80 / 1 (1 GPU) | Preserve effective batch 512 / 80 |
37
+ | Camera resize | 224×224 | 128×128, letterbox | Competition requirement; preserve complete image |
38
+ | Geometric augmentation | crop scale .92–1, rotation ±8° | Disabled | Avoid dropping task-relevant pixels |
39
+ | Stage-2 stopping | 10 original-data epochs (34,720 updates) | Explicit 34,720 updates | Preserve released optimizer budget on larger mixture |
40
+ | Warmup Stage 2 | 5% (1,736 updates) | Explicit 1,736 | Preserve released warmup |
41
+ | Stage-2 workers | 24 | 8 | Single-GPU loader; limit decoding/prefetch RAM |
42
+ | Checkpoint cadence | Stage 1: 2,000; Stage 2: 1,000 | 5,000 plus final | Requested upload cadence; avoid filling local disk with redundant full-backbone checkpoints |
43
+ | AdamW kernel | Default foreach | Fused | Full Stage-2 preflight OOM on 17.46 GiB temporary; preserve FP32 master weights/moments and all optimizer hyperparameters |
44
+ | Optimizer grouping, Stage 2 | One group of 4.686 billion parameters | Groups capped at 128 million (39 groups) | Multi-update check revealed a 17.46 GiB gradient cast OOM on update 2; preserve all parameter hyperparameters and FP32 masters/moments |
45
+ | W&B mode | Offline | Online | User requirement |
46
+ | Dataset backend/keys | v2, image/wrist_image | v3, explicit front/wrist mapping | Requested dataset format |
47
+ | Sampler | Uniform frame permutation | Task/source/episode/valid-anchor balance | User requirement |
48
+
49
+ Environment uses installed PyTorch 2.11.0+cu130 and torchvision 0.26.0+cu130 rather than the fork's cu118 environment, because Blackwell compute capability 12.0 needs a compatible CUDA/PyTorch build. BF16 support was verified. Stage-1 microbatch 256 preflight used 56.11 GiB peak allocated / 75.74 GiB reserved with a finite forward/backward/AdamW update. Stage-2 microbatch 80 plain-optimizer model preflight used 43.93 GiB allocated / 44.01 GiB reserved. The production optimizer uses the released DeepSpeed ZeRO-1 recipe and FP32 master weights/moments; its complete memory/checkpoint preflight is recorded separately. Default AdamW exhausted VRAM during its 17.46 GiB square-root temporary allocation; fused AdamW completed a full Stage-2 update, strict Stage-1 bridge load, and training-set evaluation. A four-step FP32 kernel comparison passed rtol/atol 1e−6 (maximum absolute difference 9.54e−7). Transformers is pinned to the fork's FLUX-compatible 4.56.1. Exact package versions and resolved configs are saved with every upload.
50
+
51
+ Sources: [released LIT ImageWAM code](https://github.com/jianmanlincjx/ImageWAM/tree/1792d100099771fe404ebca5dcd606725141585b), [released LIT checkpoints/configs](https://huggingface.co/linjianman/LIT/tree/765a2b2bf136f7039ab18978920c682f1aae65d2/imagewam), [requested dataset](https://huggingface.co/datasets/Ratstar216/LIBERO_NVIDIA_PLUS_LeRobot_v3), [original ImageWAM data](https://huggingface.co/datasets/yuanty/LIBERO-fastwam).
52
+
53
+ The completed selective-checkpoint benchmark is documented in [checkpoint_benchmark_report.md](checkpoint_benchmark_report.md). Stage-1 full checkpointing measured 30.85 seconds/update; checkpointing 12 of 25 layers with microbatch 64 and accumulation 8 measured 26.54 seconds/update at 78.99 GiB peak allocated, with identical first-update loss and sampled gradients. Production retains its original activation policy. Corrected Stage-2 full checkpointing measured 9.48 seconds/update at 86.10 GiB peak allocated across three measured updates after warm-up. All tested Stage-2 selective policies exceeded VRAM.
checkpoints/step_004000/metadata/provenance.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "experiment": "LIT ImageWAM Stage-2 Resolution Adaptation",
3
+ "input_information": [
4
+ 128,
5
+ 128
6
+ ],
7
+ "model_processing_per_camera": [
8
+ 224,
9
+ 224
10
+ ],
11
+ "combined": [
12
+ 224,
13
+ 448
14
+ ],
15
+ "source": [
16
+ 256,
17
+ 256
18
+ ],
19
+ "method": "FLUX.2 LoRA + full ActionDiT/LIT-interface fine-tuning",
20
+ "dataset": "Ratstar216/LIBERO_NVIDIA_PLUS_LeRobot_v3",
21
+ "dataset_revision": "c289c195d1fcb7eb6406b5c771a11465e81e8aef",
22
+ "released_lit_revision": "765a2b2bf136f7039ab18978920c682f1aae65d2",
23
+ "initialization": "linjianman/LIT/imagewam/lit_stage2/model.pt",
24
+ "normalization": "released Stage-2 dataset_stats.json",
25
+ "imagewam_commit": "1792d100099771fe404ebca5dcd606725141585b",
26
+ "flux_commit": "50fe5162777813d869182b139e83b10743caef15",
27
+ "competition_commit": "986b06942acb88b28a41ef0904c58418df04316d",
28
+ "python": "3.12.3",
29
+ "torch": "2.11.0+cu130",
30
+ "cuda": "13.0",
31
+ "created_unix": 1791307396.9788215,
32
+ "sampling_weights": [
33
+ 0.5,
34
+ 0.4,
35
+ 0.1
36
+ ],
37
+ "wandb": {
38
+ "id": "db6a83524459",
39
+ "project": "parc2026-lit-imagewam",
40
+ "name": "lit-stage2-resadapt-128to224-lora-r16",
41
+ "group": "lit-stage2-resadapt-128to224",
42
+ "url": "https://wandb.ai/star1/parc2026-lit-imagewam/runs/db6a83524459"
43
+ },
44
+ "preprocessing": "RGB 256 -> bilinear antialiased 128 -> 224; photometric only; front then wrist; [-1,1]; both endpoints; no stored rotation",
45
+ "goal": "state[t+16]; action[t:t+16]; t+16 < episode length",
46
+ "video_loading": "independent timestamp seeks; up to 8 attempts on invalid video data; replacement stays within pool/task/split; failures logged and counters checkpointed",
47
+ "gripper": "(1-g)/2 exactly once before released normalization",
48
+ "qwen": "frozen offline cached embeddings; encoder not instantiated",
49
+ "validation": "held-out episodes, equal sample count per pool; overall weighted by experiment mixture"
50
+ }
checkpoints/step_004000/metadata/qwen_cache.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "repo": "Qwen/Qwen3-4B",
3
+ "revision": "1cfa9a7208912126459214e8b04321603b3df60c",
4
+ "imagewam_commit": "1792d100099771fe404ebca5dcd606725141585b",
5
+ "context_len": 512,
6
+ "layers": [
7
+ 9,
8
+ 18,
9
+ 27
10
+ ]
11
+ }
checkpoints/step_004000/metadata/sampling.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "draws": 32000,
3
+ "counts": {
4
+ "familiar": 16110,
5
+ "plus": 12708,
6
+ "libero90": 3182
7
+ },
8
+ "video_decode_failures": {},
9
+ "percentages": {
10
+ "familiar": 50.34375,
11
+ "plus": 39.7125,
12
+ "libero90": 9.94375
13
+ }
14
+ }
checkpoints/step_004000/metadata/source/recovery_checkpoints.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Crash-safe local recovery snapshots and the persistent six-hour clock.
2
+
3
+ Only complete optimizer updates are saved. Queue files are hard links so
4
+ uploading a 66 GB ZeRO checkpoint does not require a second local copy.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import json
9
+ import os
10
+ import shutil
11
+ import time
12
+ from pathlib import Path
13
+
14
+ RECOVERY_REPO = "Ratstar216/LIT-ImageWAM-LIBERO-NVIDIA-PLUS-128-recovery"
15
+ MODEL_REPO = "Ratstar216/LIT-ImageWAM-LIBERO-NVIDIA-PLUS-128"
16
+ INTERVAL_SECONDS = 6 * 60 * 60
17
+
18
+
19
+ def atomic_json(path: Path, payload):
20
+ path.parent.mkdir(parents=True, exist_ok=True)
21
+ temporary = path.with_name(path.name + ".tmp")
22
+ with temporary.open("w") as stream:
23
+ json.dump(payload, stream, indent=2)
24
+ stream.write("\n")
25
+ stream.flush()
26
+ os.fsync(stream.fileno())
27
+ temporary.replace(path)
28
+
29
+
30
+ class RecoveryClock:
31
+ def __init__(self, path: Path, interval=INTERVAL_SECONDS, now=None):
32
+ self.path, self.interval = path, float(interval)
33
+ if not path.exists():
34
+ now = time.time() if now is None else now
35
+ atomic_json(path, {"interval_seconds": self.interval,
36
+ "next_due_unix": now + self.interval})
37
+
38
+ def due(self, now=None):
39
+ now = time.time() if now is None else now
40
+ return now >= json.loads(self.path.read_text())["next_due_unix"]
41
+
42
+ def completed(self, now=None):
43
+ # Fixed wall-clock cadence: saves at 5,000 updates do not postpone 6h.
44
+ now = time.time() if now is None else now
45
+ payload = json.loads(self.path.read_text())
46
+ while payload["next_due_unix"] <= now:
47
+ payload["next_due_unix"] += self.interval
48
+ payload["last_saved_unix"] = now
49
+ atomic_json(self.path, payload)
50
+
51
+
52
+ def complete_states(state_root: Path):
53
+ # Legacy checkpoints wrote trainer_state.json last too. Hidden pending
54
+ # directories never match this glob and cannot be resumed accidentally.
55
+ return sorted(p for p in state_root.glob("step_*")
56
+ if p.is_dir() and (p / "trainer_state.json").is_file())
57
+
58
+
59
+ def queue_snapshot(state: Path, stage_dir: Path, metadata_dir: Path,
60
+ repo=RECOVERY_REPO):
61
+ progress = json.loads((state / "trainer_state.json").read_text())
62
+ stage = stage_dir.name
63
+ step = int(progress["global_step"])
64
+ folder = stage_dir.parent / "recovery_uploads" / stage / f"step_{step:06d}"
65
+ receipt = stage_dir / "checkpoints/recovery/remote_latest.json"
66
+ if receipt.exists() and json.loads(receipt.read_text())["optimizer_step"] >= step:
67
+ return folder
68
+ if (folder / "READY").exists():
69
+ return folder
70
+ # A failed/incomplete staging operation is safe to retry; it was never READY.
71
+ if folder.exists():
72
+ shutil.rmtree(folder)
73
+ folder.mkdir(parents=True)
74
+ files = []
75
+ for source in sorted(state.rglob("*")):
76
+ if not source.is_file():
77
+ continue
78
+ relative = Path("state") / source.relative_to(state)
79
+ target = folder / relative
80
+ target.parent.mkdir(parents=True, exist_ok=True)
81
+ os.link(source, target)
82
+ files.append({"path": relative.as_posix(), "size": source.stat().st_size})
83
+ for name in ["config.yaml", "dataset_stats.json"]:
84
+ source = stage_dir / name
85
+ if source.exists():
86
+ shutil.copy2(source, folder / name)
87
+ clock = stage_dir / "checkpoints/recovery/clock.json"
88
+ if clock.exists():
89
+ shutil.copy2(clock, folder / "recovery_clock.json")
90
+ shutil.copytree(metadata_dir, folder / "reproduction")
91
+ manifest = {"format": 1, "stage": stage, "optimizer_step": step,
92
+ "epoch": progress["epoch"], "batch_in_epoch": progress["batch_in_epoch"],
93
+ "repo": repo, "remote_prefix": f"{stage}/{folder.name}",
94
+ "created_at_unix": time.time(), "state_files": files,
95
+ "state_bytes": sum(item["size"] for item in files),
96
+ "contents": "model, FP32 AdamW master weights and moments, scheduler, RNG, sampler progress",
97
+ "resume_requires": "same pinned model/data, optimizer groups, world size and prepared runtime"}
98
+ atomic_json(folder / "manifest.json", manifest)
99
+ (folder / "READY").touch()
100
+ atomic_json(stage_dir / "checkpoints/recovery/latest_local.json", manifest)
101
+ return folder
102
+
103
+
104
+ def make_state_space(stage_dir: Path, estimated_bytes: int, reserve=5 * 2**30):
105
+ """Never delete the only complete recoverable copy to make room.
106
+
107
+ A Stage-2 state may not fit twice on Colab. An uploaded receipt permits
108
+ retirement of the old *local* state while its remote snapshot stays live.
109
+ Pending uploads retain their hard links and are never deleted here.
110
+ """
111
+ if shutil.disk_usage(stage_dir).free >= estimated_bytes + reserve:
112
+ return
113
+ receipt_path = stage_dir / "checkpoints/recovery/remote_latest.json"
114
+ covered_step = -1
115
+ if receipt_path.exists():
116
+ covered_step = json.loads(receipt_path.read_text())["optimizer_step"]
117
+ for old in complete_states(stage_dir / "checkpoints/state"):
118
+ step = json.loads((old / "trainer_state.json").read_text())["global_step"]
119
+ if step <= covered_step:
120
+ shutil.rmtree(old)
121
+ if shutil.disk_usage(stage_dir).free < estimated_bytes + reserve:
122
+ raise OSError("Insufficient disk for a full recovery checkpoint; kept all states "
123
+ "without a verified Hugging Face replacement. See recovery_upload.log.")
checkpoints/step_004000/metadata/source/resolution_adapt/README.md ADDED
@@ -0,0 +1,357 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # LIT ImageWAM Stage-2 Resolution Adaptation
2
+
3
+ This experiment starts from **released `linjianman/LIT/imagewam/lit_stage2/model.pt`**.
4
+ It preserves **224×224 processing per camera** while limiting both current and
5
+ future images to **128×128 information**. It never runs Stage 1. The old native-128
6
+ pipeline remains separate.
7
+
8
+ The real CUDA preflight passed on an RTX PRO 6000 Blackwell (96 GB), using
9
+ microbatch 1 and accumulation 8. The released checkpoint loaded strictly, real
10
+ processed dataset samples matched raw expectations, one optimizer update passed,
11
+ and a separate process resumed for one more discarded update. Held-out validation
12
+ also passed. No full training run or production model upload was launched.
13
+ Sandboxed commands could not access CUDA; these tests ran outside the sandbox.
14
+ See the [verification report](../../../research/lit_resolution_adapt/report.md).
15
+
16
+ ## Reproducible assets and environment
17
+
18
+ Pinned sources:
19
+
20
+ | Asset | Revision |
21
+ | --- | --- |
22
+ | ImageWAM `feat/goal-prior-bottleneck-fix` implementation | `1792d100099771fe404ebca5dcd606725141585b` |
23
+ | FLUX source | `50fe5162777813d869182b139e83b10743caef15` |
24
+ | Released LIT Stage-2 checkpoint/config/stats | `765a2b2bf136f7039ab18978920c682f1aae65d2` |
25
+ | `Ratstar216/LIBERO_NVIDIA_PLUS_LeRobot_v3` | `c289c195d1fcb7eb6406b5c771a11465e81e8aef` |
26
+ | FLUX.2-dev autoencoder | `26afe3a78bb242c0a8bb181dcc8937bb16e5c66c` |
27
+ | Qwen3-4B | `1cfa9a7208912126459214e8b04321603b3df60c` |
28
+
29
+ `assets.lock.json` records the released model, stats, config and AE hashes. The
30
+ launcher checks full model/stat/AE hashes, clean source commits, pinned episode
31
+ provenance, and Qwen-cache provenance before model allocation. Dataset downloads
32
+ use a pinned snapshot; run metadata records that revision and source-provenance
33
+ hash. The existing all-row/video audits are included with checkpoints. These
34
+ audits establish dataset structure, not simulator success or collision safety.
35
+
36
+ From the repository root, prepare a dedicated GPU environment (Python 3.12):
37
+
38
+ ```bash
39
+ uv venv --python 3.12 .runtime/lit-training/venv
40
+ uv pip install --python .runtime/lit-training/venv/bin/python \
41
+ -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.lock
42
+ uv pip install --python .runtime/lit-training/venv/bin/python --no-deps lerobot==0.4.4
43
+ ```
44
+
45
+ The explicit LeRobot overlay preserves this repository's Torch 2.11 / torchvision
46
+ 0.26 / direct-PyAV path despite LeRobot's tighter package bounds. Its 0.4.4 loader
47
+ supplies explicit Arrow features, supporting the audited fixed/variable state
48
+ lists. The locked dependency set resolves with uv; a clean GPU installation of
49
+ that entire set has **not** been executed here. CPU and GPU checks ran on Python 3.13.15, Torch 2.11.0+cu130,
50
+ Transformers 4.56.1, Diffusers 0.35.2 and LeRobot 0.4.4, using the existing
51
+ environment plus temporary uv packages. The complete measured environment is
52
+ recorded in `research/lit_resolution_adapt/gpu_environment.txt`.
53
+
54
+ Authenticate using `HF_TOKEN`/existing HF login and `WANDB_API_KEY`/existing W&B
55
+ login. No keys belong in commands or config files. `WANDB_ENTITY` is optional.
56
+ The following **explicitly downloads the full dataset and model assets** and
57
+ caches frozen Qwen embeddings; it does not train. It may take substantial storage
58
+ and download time. Existing clean pinned checkouts/downloads are reused; dirty
59
+ source checkouts cause an error rather than being overwritten.
60
+
61
+ ```bash
62
+ .runtime/lit-training/venv/bin/python \
63
+ examples/lit_imagewam_libero/resolution_adapt/prepare_assets.py \
64
+ --runtime .runtime/lit-training --cache-text
65
+ ```
66
+
67
+ Cached Qwen contexts replace the text encoder during training; no Qwen parameters
68
+ are instantiated or optimized. The cache generator reuses the released script,
69
+ 512-token context, hidden layers 9/18/27, and released prompt template. It writes
70
+ `qwen_cache/provenance.json`. Existing caches without provenance should be
71
+ regenerated from the pinned local Qwen model, not blindly relabeled.
72
+
73
+ ## Preflights and launch
74
+
75
+ CPU tests with the actual upstream LoRA implementation:
76
+
77
+ ```bash
78
+ PYTHONPATH=.runtime/lit-training/upstream/src:.runtime/lit-training/flux2/src \
79
+ .runtime/lit-training/venv/bin/python -m pytest \
80
+ examples/lit_imagewam_libero/resolution_adapt/test_pipeline.py \
81
+ examples/lit_imagewam_libero/resolution_adapt/test_video_loading.py -q
82
+
83
+ .runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
84
+ --mode sampler --output .runtime/lit-resolution-sampler
85
+ ```
86
+
87
+ Representative raw RGB/data checks and debug PNGs can run without a GPU. The
88
+ optional `--stream-from-hub` reads pinned metadata/tabular shards and streams
89
+ endpoint frames without downloading all videos:
90
+
91
+ ```bash
92
+ .runtime/lit-training/venv/bin/python \
93
+ examples/lit_imagewam_libero/resolution_adapt/preflight_data.py \
94
+ --dataset-root .runtime/lit-training/dataset --output .runtime/lit-resolution-data \
95
+ --stats .runtime/lit-training/released/imagewam/lit_stage2/dataset_stats.json
96
+ ```
97
+
98
+ Actual checkpoint **metadata** keys/shapes can be checked without full tensor
99
+ allocation; this uses remote range reads and is not a substitute for GPU loading:
100
+
101
+ ```bash
102
+ .runtime/lit-training/venv/bin/python \
103
+ examples/lit_imagewam_libero/resolution_adapt/preflight_model.py \
104
+ --imagewam-root .runtime/lit-training/upstream \
105
+ --flux-root .runtime/lit-training/flux2 --output .runtime/lit-resolution-architecture
106
+ ```
107
+
108
+ Run one real accumulated optimizer update in a **separate disposable output**:
109
+
110
+ ```bash
111
+ .runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
112
+ --mode preflight --output .runtime/lit-resolution-preflight \
113
+ --batch-size 1 --gradient-accumulation 8
114
+ ```
115
+
116
+ This loads the exact Stage-2 model before LoRA, compares real processed samples
117
+ against raw t+16 state/action/image expectations, prints missing/unexpected keys,
118
+ checks optimizer coverage, runs the released losses/backward/clipping/AdamW/
119
+ scheduler, and checks LoRA/ActionDiT/LIT changes against frozen FLUX/AE probes.
120
+ It writes `one_step.json` only on success and never saves production weights.
121
+ Qwen cannot change because training only loads its frozen cached contexts.
122
+ Use a new output directory when repeating preflight.
123
+ To additionally exercise full-size checkpoint recovery, pass `--preflight-save` to
124
+ the first preflight command, then run:
125
+
126
+ ```bash
127
+ .runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
128
+ --mode preflight --output .runtime/lit-resolution-preflight --resume
129
+ ```
130
+
131
+ This takes just one more discarded update in a separate process and saves another
132
+ local recovery checkpoint; neither is uploaded. Each full preflight recovery
133
+ snapshot is approximately 10.02 GB. The tested sequence restored all 529 Adam
134
+ states, scheduler step 1, sampler draw 8, split and W&B identity, then advanced to
135
+ step 2/draw 16. The same production resume path is used.
136
+
137
+ For held-out losses without optimizer updates:
138
+
139
+ ```bash
140
+ .runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
141
+ --mode validate --output .runtime/lit-resolution-preflight --resume
142
+ ```
143
+
144
+
145
+ Validate online services for the production output directory:
146
+
147
+ ```bash
148
+ .runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
149
+ --mode services --output .runtime/lit-resolution-adapt
150
+ ```
151
+
152
+ This creates/ensures a private HF repo, uploads and downloads a tiny uniquely named
153
+ metadata probe, and creates a separate `-preflight` W&B run. Training checks the
154
+ HF service receipt; W&B failures leave local metrics and allow continued work.
155
+
156
+ **Exact training command (only run after GPU preflight succeeds):**
157
+
158
+ ```bash
159
+ .runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
160
+ --mode train --output .runtime/lit-resolution-adapt \
161
+ --max-steps 10000 --warmup-steps 500 \
162
+ --batch-size 1 --gradient-accumulation 8 \
163
+ --lora-rank 16 --lora-alpha 16 --lora-dropout 0 \
164
+ --lr-flux-lora 3e-5 --lr-action 1e-5 --lr-interface 1e-5 \
165
+ --lr-pose 1e-5 --lr-proprio 1e-5 --lr-other 1e-5 \
166
+ --unfreeze-last-flux-blocks 0 --lr-unfrozen-flux 1e-6 \
167
+ --checkpoint-every 2000 \
168
+ --hf-repo Ratstar216/LIT-ImageWAM-ResolutionAdapt-128to224 \
169
+ --wandb-project parc2026-lit-imagewam \
170
+ --wandb-run-name lit-stage2-resadapt-128to224-lora-r16
171
+ ```
172
+
173
+ **Exact local resume command:**
174
+
175
+ ```bash
176
+ .runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
177
+ --mode train --output .runtime/lit-resolution-adapt --resume
178
+ ```
179
+
180
+ Resume reuses the persisted arguments, schedule, split, sampling RNG/counters,
181
+ optimizer moments, model/LoRA weights, global step, Python/NumPy/Torch/CUDA RNG,
182
+ and W&B ID. CLI hyperparameter changes are intentionally ignored on resume.
183
+ Checkpoint saves occur at complete optimizer boundaries, so accumulation
184
+ position is zero and BF16 needs no scaler. The synchronous loader has no prefetched
185
+ unconsumed samples. Logical epoch = consumed draws / eligible training anchors;
186
+ sampling is with replacement, not a dataset traversal. File locks prevent two
187
+ trainers or uploaders using the same output.
188
+
189
+ Video decoding seeks independently to each requested timestamp. This avoids
190
+ decoding unused frames between the current and future endpoints, where an invalid
191
+ AV1 packet could previously stop training even when both endpoint frames were
192
+ valid. Required frames still have to meet the exact timestamp tolerance.
193
+ If invalid video data prevents loading a required frame, the loader makes up to
194
+ eight attempts, resampling an eligible episode/anchor within the same pool, task
195
+ and split. Other errors, including timestamp mismatches, missing files and
196
+ transform failures, still stop the run. Rejected samples are recorded in
197
+ `video_decode_failures.jsonl`; failure counters appear in metrics and checkpoints.
198
+ The replacement RNG is part of the sampler recovery state. Existing checkpoints
199
+ without failure counters remain resumable. Validation follows the same bounded
200
+ recovery within held-out episodes and reports failure counts per pool.
201
+
202
+ After a decoder crash, use the local resume command above. It restores the
203
+ latest complete checkpoint; optimizer updates after that checkpoint are replayed.
204
+ The previously written loss logs do not contain model or optimizer state.
205
+
206
+ Remote restore into a new output (prepare identical assets at their original
207
+ paths first):
208
+
209
+ ```bash
210
+ .runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
211
+ --mode restore --output .runtime/lit-resolution-restored
212
+ .runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
213
+ --mode services --output .runtime/lit-resolution-restored
214
+ .runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
215
+ --mode train --output .runtime/lit-resolution-restored --resume
216
+ ```
217
+
218
+ The restore pins the remote commit, chooses the latest completed snapshot, and
219
+ verifies every file before exposing it. Same asset paths are currently required;
220
+ portable path remapping is not automatic.
221
+
222
+ ### Fresh Colab resume from step 2,000
223
+
224
+ [`scripts/resume_lit_resolution_adapt_colab.sh`](../../../scripts/resume_lit_resolution_adapt_colab.sh)
225
+ rebuilds the locked Python 3.12 environment and pinned assets on a GPU runtime,
226
+ generates the frozen Qwen cache, and restores exactly `step_002000` through the
227
+ verified restore API. It creates symlinks at the checkpoint's original
228
+ `/tmp/resadapt-*` asset paths, preserving the saved arguments and schedule.
229
+ The service check and training then run in the same process sequence. The first
230
+ training update is step 2,001; the saved target is 10,000 optimizer steps.
231
+ Re-running the launcher on the same VM resumes the latest complete local
232
+ checkpoint in its output directory. The initial restore is about 10 GB and uses
233
+ additional space for the download cache; the dataset and frozen model assets are
234
+ downloaded separately.
235
+
236
+ Use an existing HF/W&B login, or load Colab Secrets in a Python notebook cell:
237
+
238
+ ```python
239
+ import os
240
+ from google.colab import userdata
241
+ os.environ["HF_TOKEN"] = userdata.get("HF_TOKEN")
242
+ os.environ["WANDB_API_KEY"] = userdata.get("WANDB_API_KEY")
243
+ ```
244
+
245
+ In another notebook cell, start the launcher on its own tmux socket so a new
246
+ server inherits the notebook's authentication environment:
247
+
248
+ ```python
249
+ %cd /content/PARC2026_final
250
+ !tmux -L lit-resume new-session -d -s lit-resume -c /content/PARC2026_final 'bash scripts/resume_lit_resolution_adapt_colab.sh'
251
+ ```
252
+
253
+ Check progress without attaching an interactive terminal:
254
+
255
+ ```python
256
+ !tmux -L lit-resume list-sessions
257
+ !tail -n 40 /content/PARC2026_final/.runtime/lit-resolution-resume-2000/colab-resume.log
258
+ !nvidia-smi
259
+ ```
260
+
261
+ From a Colab terminal, attach with `tmux -L lit-resume attach -t lit-resume` and
262
+ detach with **Ctrl+B**, then **D**. New checkpoints are uploaded by the existing
263
+ independent uploader; check `upload.log` and `upload_summary.json` in the output
264
+ directory. tmux does not prevent Colab runtime termination. A deleted VM loses
265
+ its local assets and any updates not saved and uploaded to persistent storage.
266
+
267
+ ## Preserved behavior and deliberate implementation choices
268
+
269
+ - Both endpoint cameras: decoded RGB 256 → antialiased bilinear 128 → bilinear
270
+ 224; photometric augmentation afterward; front then wrist → 224×448; [-1,1].
271
+ No stored-image rotation, flip or random crop. Both shape_meta copies agree.
272
+ `competition_cameras()` separately handles simulator 128 images and the existing
273
+ adapter's 180° rotation; stored data never uses that function.
274
+ - Actions are `[t:t+16]`; pose is normalized observed state at `t+16`, not the
275
+ episode end. Only `t+16 < L` anchors are eligible. The existing gripper transform
276
+ converts `(1-g)/2` once before released normalization. Eight state values and the
277
+ first six arm controls retain their original semantics.
278
+ - Sampling: pool 50/40/10 → uniform task → uniform eligible episode → uniform
279
+ valid anchor. Deterministic approximately 5% episode holdout per pool/task;
280
+ duplicate source identities stay together. The custom sampler explicitly
281
+ excludes validation. Small strata use at least one validation episode, so the
282
+ realized fraction need not be exactly 5%.
283
+ - Offline validation samples held-out episodes separately per pool, disables
284
+ photometric augmentation, preserves training RNG, and uses fixed diagnostic
285
+ seeds. Overall loss is the **50/40/10-weighted mean** of pool means. These are
286
+ offline losses, not simulator success metrics or training-subset diagnostics.
287
+ - Released 100×768 latent interface, 8 pose tokens, 512 inner width, 5 interface
288
+ parameter groups, 16-action horizon, 55/15/30 `both/ref_only/syn_only` gating,
289
+ context dropout/blackout, pose supervision and action-generation objective stay
290
+ intact. Total loss = action + 0.5×video + 0.3×pose. Returned upstream component
291
+ metrics are already weighted; logging divides by the weights for unweighted
292
+ values rather than multiplying them again.
293
+ - The released `require_text_cache` option controls T5, not Qwen; it remains
294
+ false, while `qwen_text_cache_dir` enables Qwen loading and a batch-shape check
295
+ enforces `[B,512,7680]` contexts.
296
+ - The released Stage-2 `apply_trainable_policy()` would unfreeze all FLUX weights;
297
+ this launcher applies an explicit policy and never calls that method afterward.
298
+ The existing trainer also expects a Stage-1 bridge and uses a different recovery
299
+ clock/retention policy. A small independent loop is used while reusing its model,
300
+ dataset hooks, gripper transform, atomic JSON helper, and resumable HF upload API.
301
+ - Original FLUX is frozen; LoRA wraps the released target suffixes only inside its
302
+ transformer. ActionDiT and all LIT/pose/proprio modules train densely. The final
303
+ N blocks option traverses double blocks followed by single blocks and uses the
304
+ dense-FLUX LR independently of LoRA. AdamW uses FP32 trainable parameters and
305
+ moments, BF16 autocast, betas (0.9,0.95), decay .01, clip 1.0 and cosine decay.
306
+ Groups are capped at 128M elements as in the repository's memory-safe recipe.
307
+ - Default W&B project `parc2026-lit-imagewam`, group
308
+ `lit-stage2-resadapt-128to224`, run `lit-stage2-resadapt-128to224-lora-r16`.
309
+ `WANDB_PROJECT`, `WANDB_RUN_NAME`, `WANDB_ENTITY` are supported. A stable UUID is
310
+ persisted, `resume=allow` reconnects, and every metric is written locally first.
311
+ Connection attempts time out; a failed connection is retried later. Historical
312
+ local JSONL metrics are retained but are not automatically replayed to W&B.
313
+
314
+ ## Checkpoints and offline reconstruction
315
+
316
+ Private repo: https://huggingface.co/Ratstar216/LIT-ImageWAM-ResolutionAdapt-128to224
317
+
318
+ Each immutable `checkpoints/step_002000/` (then 004000, 006000, 008000, 010000) contains:
319
+
320
+ ```text
321
+ weights/adaptation.pt # LoRA plus ALL dense trainable Stage-2 parameters
322
+ resume/state.pt # AdamW, scheduler, RNG, sampler/counters, optimizer step
323
+ metadata/ # config, args, split, stats, hashes, revisions, audits, source, W&B identity
324
+ manifest.json # byte lengths and SHA-256 checksums
325
+ COMPLETE # manifest hash, written last
326
+ ```
327
+
328
+ A non-cadence normal final update is saved too. No frozen base is re-uploaded in
329
+ each snapshot. Local writes are fsynced in a hidden pending directory, verified,
330
+ then renamed. The independent single upload worker scans complete folders, reuses
331
+ HF large-folder transfer caches, verifies remote sizes, and publishes the remote
332
+ completion marker in a final commit. Persistent receipts are rechecked on restart;
333
+ conflicting remote manifests never get overwritten. Training does not wait for
334
+ uploads to finish, including at its final step. Inspect `upload_summary.json` and
335
+ `upload.log`; allow pending uploads to finish before stopping the host.
336
+
337
+ Retain all adaptation weights; retain the latest two full local recovery states
338
+ and every final state. Older recovery data is pruned only after a verified upload
339
+ and never during upload. Failed uploads preserve the full local checkpoint. Disk
340
+ exhaustion fails the save without deleting unbacked checkpoints. Abandoned hidden
341
+ pending directories are cleaned only while holding the trainer lock.
342
+
343
+ Restart/check the independent uploader without training:
344
+
345
+ ```bash
346
+ .runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
347
+ --mode upload --output .runtime/lit-resolution-adapt
348
+ ```
349
+
350
+ `resolution_adapt.offline.load_offline()` reconstructs the exact released Stage-2
351
+ base, injects identical LoRA, loads the dense adaptation, merges LoRA with its
352
+ actual alpha/rank, then attaches locally packaged AE/Qwen assets. It imports no
353
+ training dataset, W&B, or HF API and uses `local_files_only=True` for Qwen.
354
+ The caller must package pinned ImageWAM/FLUX sources, base weights, stats, AE and
355
+ Qwen locally. This is a reconstruction interface, not a tested final submission
356
+ ZIP. A merged export should avoid packaging both the base and redundant adapted
357
+ weights; recheck the 20GB ZIP limit and L4 inference budget before submission.
checkpoints/step_004000/metadata/source/resolution_adapt/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Stage-2 128-information / 224-processing adaptation (training tools optional)."""
checkpoints/step_004000/metadata/source/resolution_adapt/assets.lock.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "linjianman/LIT": {
3
+ "revision": "765a2b2bf136f7039ab18978920c682f1aae65d2",
4
+ "files": {
5
+ "imagewam/lit_stage2/config.yaml": {
6
+ "size": 6886,
7
+ "sha256": "9c4a8d9eb8767e71e7ebd6edb921be810c7867512d9ebbf0f91a53dca21e9da1"
8
+ },
9
+ "imagewam/lit_stage2/dataset_stats.json": {
10
+ "size": 22880,
11
+ "sha256": "b107cd929518d0477535f99834e0cc13f470e25cd61bbce69b315619ba5140d4"
12
+ },
13
+ "imagewam/lit_stage2/model.pt": {
14
+ "size": 9373131513,
15
+ "sha256": "f403225e76809034307caf23e5a8f155385a51adb69d98990cff14125743ec97"
16
+ }
17
+ }
18
+ },
19
+ "black-forest-labs/FLUX.2-dev": {
20
+ "revision": "26afe3a78bb242c0a8bb181dcc8937bb16e5c66c",
21
+ "files": {
22
+ "ae.safetensors": {
23
+ "size": 336211292,
24
+ "sha256": "868fe7b343cc8f3a19dbcfcafbc3d5f888802be3f89bd81b65b3621a066ce8f3"
25
+ }
26
+ }
27
+ },
28
+ "Qwen/Qwen3-4B": {
29
+ "revision": "1cfa9a7208912126459214e8b04321603b3df60c",
30
+ "files": {}
31
+ }
32
+ }
checkpoints/step_004000/metadata/source/resolution_adapt/checkpoints.py ADDED
@@ -0,0 +1,217 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Immutable atomic snapshots; persistent single-worker retry queue and safe retention."""
2
+ from __future__ import annotations
3
+
4
+ import hashlib
5
+ import json
6
+ import os
7
+ import random
8
+ import shutil
9
+ import time
10
+ import uuid
11
+ from pathlib import Path
12
+
13
+ import numpy as np
14
+ import torch
15
+ from recovery_checkpoints import atomic_json
16
+ from .model import trainable_state, load_adaptation
17
+
18
+
19
+ def sha256(path: Path) -> str:
20
+ h = hashlib.sha256()
21
+ with path.open('rb') as stream:
22
+ for block in iter(lambda: stream.read(8*1024*1024),b''):
23
+ h.update(block)
24
+ return h.hexdigest()
25
+
26
+
27
+ def rng_state() -> dict:
28
+ return {'python':random.getstate(),'numpy':np.random.get_state(),'torch':torch.get_rng_state(),
29
+ 'cuda':torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None}
30
+
31
+
32
+ def restore_rng(state: dict) -> None:
33
+ random.setstate(state['python']); np.random.set_state(state['numpy']); torch.set_rng_state(state['torch'])
34
+ if state['cuda'] is not None:
35
+ torch.cuda.set_rng_state_all(state['cuda'])
36
+
37
+
38
+ def checkpoint_due(step: int, every: int, max_steps: int) -> bool:
39
+ return step > 0 and (step % every == 0 or step == max_steps)
40
+
41
+
42
+ def validate_checkpoint(folder: Path, *, resume=True) -> dict:
43
+ if not (folder/'COMPLETE').is_file():
44
+ raise ValueError(f'Incomplete checkpoint: {folder}')
45
+ manifest = json.loads((folder/'manifest.json').read_text())
46
+ if (folder/'COMPLETE').read_text().strip() != sha256(folder/'manifest.json'):
47
+ raise ValueError('Manifest hash mismatch')
48
+ for name,info in manifest['files'].items():
49
+ if not resume and name.startswith('resume/'):
50
+ continue
51
+ path = folder/name
52
+ if not path.is_file() or path.stat().st_size != info['size'] or sha256(path) != info['sha256']:
53
+ raise ValueError(f'Checkpoint integrity failure: {path}')
54
+ return manifest
55
+
56
+
57
+ def save_checkpoint(root: Path, step: int, model, optimizer, scheduler, sampler,
58
+ metadata_dir: Path, *, final=False) -> Path:
59
+ destination = root/f'step_{step:06d}'
60
+ if destination.exists():
61
+ raise FileExistsError(f'Immutable checkpoint already exists: {destination}')
62
+ root.mkdir(parents=True,exist_ok=True)
63
+ pending = root/f'.pending_{step:06d}_{uuid.uuid4().hex}'
64
+ pending.mkdir()
65
+ (pending/'weights').mkdir(); (pending/'resume').mkdir()
66
+ torch.save(trainable_state(model),pending/'weights/adaptation.pt')
67
+ torch.save({'optimizer':optimizer.state_dict(),'scheduler':scheduler.state_dict(),
68
+ 'global_step':step,'rng':rng_state(),'sampler':sampler.state_dict(),
69
+ 'accumulation_position':0},pending/'resume/state.pt')
70
+ shutil.copytree(metadata_dir,pending/'metadata')
71
+ atomic_json(pending/'metadata/sampling.json',sampler.distribution_report())
72
+ files = {}
73
+ for path in sorted(pending.rglob('*')):
74
+ if path.is_file():
75
+ with path.open('rb') as stream:
76
+ os.fsync(stream.fileno())
77
+ files[str(path.relative_to(pending))] = {'size':path.stat().st_size,'sha256':sha256(path)}
78
+ atomic_json(pending/'manifest.json',{'step':step,'final':final,'files':files,'created_unix':time.time()})
79
+ with (pending/'COMPLETE').open('w') as stream:
80
+ stream.write(sha256(pending/'manifest.json')+'\n'); stream.flush(); os.fsync(stream.fileno())
81
+ validate_checkpoint(pending)
82
+ pending.rename(destination)
83
+ fd = os.open(root,os.O_RDONLY)
84
+ try: os.fsync(fd)
85
+ finally: os.close(fd)
86
+ return destination
87
+
88
+
89
+ def latest_checkpoint(root: Path) -> Path:
90
+ errors = []
91
+ for folder in sorted(root.glob('step_*'),reverse=True):
92
+ if not (folder/'resume/state.pt').exists():
93
+ continue
94
+ try:
95
+ validate_checkpoint(folder)
96
+ return folder
97
+ except (ValueError, OSError) as exc:
98
+ errors.append(str(exc))
99
+ raise ValueError(f'No complete valid resume checkpoint; {errors}')
100
+
101
+
102
+ def resume_checkpoint(folder: Path, model, optimizer, scheduler, sampler) -> int:
103
+ validate_checkpoint(folder)
104
+ load_adaptation(model,folder/'weights/adaptation.pt')
105
+ # Only trusted, locally generated recovery files (contains Python/NumPy RNG tuples).
106
+ state = torch.load(folder/'resume/state.pt',map_location='cpu',weights_only=False)
107
+ if state['accumulation_position'] != 0:
108
+ raise ValueError('Only optimizer-boundary resumes are supported')
109
+ optimizer.load_state_dict(state['optimizer']); scheduler.load_state_dict(state['scheduler'])
110
+ sampler.load_state_dict(state['sampler']); restore_rng(state['rng'])
111
+ return int(state['global_step'])
112
+
113
+
114
+ def prune_uploaded(root: Path) -> None:
115
+ complete = sorted(p for p in root.glob('step_*') if (p/'resume/state.pt').is_file())
116
+ for folder in complete[:-2]:
117
+ receipt = root.parent/'upload_state'/f'{folder.name}.json'
118
+ if not receipt.exists(): continue
119
+ status = json.loads(receipt.read_text())
120
+ manifest = json.loads((folder/'manifest.json').read_text())
121
+ if manifest['final'] or not status.get('uploaded') or status.get('uploading'): continue
122
+ shutil.rmtree(folder/'resume')
123
+
124
+
125
+ def upload_once(run: Path, repo: str, api=None) -> dict:
126
+ """Resumable large-folder transfer, verified receipt; never delete remote history."""
127
+ from huggingface_hub import HfApi, hf_hub_download
128
+ api = api or HfApi()
129
+ info = api.model_info(repo)
130
+ if not info.private:
131
+ raise RuntimeError('Checkpoint repository must be private')
132
+ root = run/'checkpoints'
133
+ statuses = run/'upload_state'; statuses.mkdir(exist_ok=True)
134
+ summary = {'pending':0,'failures':0,'completed':0,'duration_seconds':0.0}
135
+ for folder in sorted(root.glob('step_*')):
136
+ if not (folder/'COMPLETE').exists(): continue
137
+ receipt_path = statuses/f'{folder.name}.json'
138
+ old = json.loads(receipt_path.read_text()) if receipt_path.exists() else {}
139
+ prefix = f'checkpoints/{folder.name}'
140
+ start = time.monotonic()
141
+ try:
142
+ # Recheck remotely even after restart with a local success receipt.
143
+ remote_paths = set(api.list_repo_files(repo))
144
+ if f'{prefix}/COMPLETE' in remote_paths:
145
+ marker = Path(hf_hub_download(repo,f'{prefix}/COMPLETE',force_download=True))
146
+ if marker.read_text() != (folder/'COMPLETE').read_text():
147
+ raise RuntimeError('Immutable remote checkpoint conflicts with local checkpoint')
148
+ atomic_json(receipt_path,{**old,'uploaded':True,'uploading':False,'repo':repo,'step':int(folder.name[5:]),
149
+ 'failures':old.get('failures',0),'verified_unix':time.time()})
150
+ continue
151
+ manifest = validate_checkpoint(folder)
152
+ if f'{prefix}/manifest.json' in remote_paths:
153
+ remote = Path(hf_hub_download(repo,f'{prefix}/manifest.json',force_download=True))
154
+ if sha256(remote) != sha256(folder/'manifest.json'):
155
+ raise RuntimeError('Refusing to overwrite a different immutable remote manifest')
156
+ atomic_json(receipt_path,{**old,'uploaded':False,'uploading':True})
157
+ # Reuse repository's upload_large_folder approach, including HF retry cache.
158
+ api.upload_large_folder(repo_id=repo,repo_type='model',folder_path=run,
159
+ allow_patterns=[f'{prefix}/**'],ignore_patterns=['**/COMPLETE','**/.cache/**'],
160
+ num_workers=2,print_report=False)
161
+ remote = {item.path:item.size for item in api.list_repo_tree(repo,path_in_repo=prefix,recursive=True)
162
+ if hasattr(item,'size')}
163
+ for name,entry in manifest['files'].items():
164
+ if remote.get(f'{prefix}/{name}') != entry['size']:
165
+ raise RuntimeError(f'Upload verification failed: {name}')
166
+ # A separate final commit is the only remote completion signal.
167
+ api.upload_file(repo_id=repo,path_or_fileobj=folder/'COMPLETE',path_in_repo=f'{prefix}/COMPLETE',
168
+ commit_message=f'Complete immutable {folder.name}')
169
+ marker = Path(hf_hub_download(repo,f'{prefix}/COMPLETE',force_download=True))
170
+ if marker.read_text() != (folder/'COMPLETE').read_text(): raise RuntimeError('Remote marker mismatch')
171
+ elapsed = time.monotonic()-start
172
+ atomic_json(receipt_path,{'uploaded':True,'uploading':False,'repo':repo,'step':manifest['step'],
173
+ 'duration_seconds':elapsed,'failures':old.get('failures',0),'verified_unix':time.time()})
174
+ summary['completed'] += 1; summary['duration_seconds'] += elapsed
175
+ except Exception as exc:
176
+ # Store only exception class, avoiding credential-bearing request diagnostics.
177
+ atomic_json(receipt_path,{'uploaded':False,'uploading':False,'failures':old.get('failures',0)+1,
178
+ 'error':type(exc).__name__,'last_attempt_unix':time.time()})
179
+ print(f'Upload pending {folder.name}: {type(exc).__name__}',flush=True)
180
+ summary['failures'] += 1; summary['pending'] += 1
181
+ prune_uploaded(root)
182
+ receipts = [json.loads(p.read_text()) for p in statuses.glob('step_*.json')]
183
+ summary.update(pending=sum(not r.get('uploaded',False) for r in receipts),
184
+ failures=sum(r.get('failures',0) for r in receipts),
185
+ completed=sum(r.get('uploaded',False) for r in receipts),
186
+ duration_seconds=sum(r.get('duration_seconds',0.) for r in receipts),
187
+ last_completed_step=max((r.get('step',0) for r in receipts if r.get('uploaded')),default=0))
188
+ atomic_json(run/'upload_summary.json',summary)
189
+ return summary
190
+
191
+
192
+ def restore_remote(run: Path, repo: str, step: str | None = None) -> Path:
193
+ """Download only a complete immutable recovery snapshot; verify before exposing it."""
194
+ from huggingface_hub import HfApi,hf_hub_download
195
+ if (run/'metadata').exists(): raise FileExistsError('Restore into a new run directory')
196
+ api=HfApi()
197
+ # Resolve one commit so concurrent newer uploads cannot change this restore.
198
+ revision=api.model_info(repo).sha
199
+ paths=api.list_repo_files(repo,revision=revision)
200
+ candidates=sorted(p.rsplit('/',1)[0] for p in paths if p.startswith('checkpoints/step_') and p.endswith('/COMPLETE'))
201
+ if step is not None: candidates=[p for p in candidates if p.endswith(step)]
202
+ if not candidates: raise ValueError('No complete remote checkpoint')
203
+ prefix=candidates[-1];root=run/'checkpoints';root.mkdir(parents=True,exist_ok=True)
204
+ destination=root/Path(prefix).name
205
+ if destination.exists(): raise FileExistsError(destination)
206
+ pending=root/f'.pending_restore_{uuid.uuid4().hex}';pending.mkdir()
207
+ manifest_path=hf_hub_download(repo,f'{prefix}/manifest.json',revision=revision)
208
+ manifest=json.loads(Path(manifest_path).read_text())
209
+ for name in [*manifest['files'],'manifest.json','COMPLETE']:
210
+ relative=Path(name)
211
+ if relative.is_absolute() or '..' in relative.parts: raise ValueError('Unsafe manifest path')
212
+ target=pending/relative;target.parent.mkdir(parents=True,exist_ok=True)
213
+ shutil.copy2(hf_hub_download(repo,f'{prefix}/{name}',revision=revision),target)
214
+ validate_checkpoint(pending);pending.rename(destination)
215
+ if (run/'metadata').exists(): raise FileExistsError('Restore into a new run directory')
216
+ shutil.copytree(destination/'metadata',run/'metadata')
217
+ return destination
checkpoints/step_004000/metadata/source/resolution_adapt/data.py ADDED
@@ -0,0 +1,180 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Deterministic provenance sampling and shared current/future image transform."""
2
+ from __future__ import annotations
3
+
4
+ import copy
5
+ import hashlib
6
+ import json
7
+ import math
8
+ import random
9
+ from collections import Counter, defaultdict
10
+ from pathlib import Path
11
+
12
+ import torch
13
+ import torch.nn.functional as F
14
+
15
+ DATASET = 'Ratstar216/LIBERO_NVIDIA_PLUS_LeRobot_v3'
16
+ DATASET_REVISION = 'c289c195d1fcb7eb6406b5c771a11465e81e8aef'
17
+ POOLS = ('familiar', 'plus', 'libero90')
18
+ WEIGHTS = (0.5, 0.4, 0.1)
19
+
20
+
21
+ def pool_of(ep: dict) -> str:
22
+ if ep['source_repo'] == 'lerobot/libero_plus':
23
+ return 'plus'
24
+ if ep['source_repo'] == 'nvidia/LIBERO_LeRobot_v3':
25
+ if ep['source_suite'] == 'libero_90':
26
+ return 'libero90'
27
+ if ep['source_suite'] in {'libero_spatial', 'libero_object', 'libero_goal', 'libero_10'}:
28
+ return 'familiar'
29
+ raise ValueError(f"Unknown source provenance: {ep['source_repo']}, {ep['source_suite']}")
30
+
31
+
32
+ def source_identity(ep: dict) -> tuple:
33
+ return ep['source_repo'], ep['source_suite'], ep['source_episode_index']
34
+
35
+
36
+ def digest(value) -> str:
37
+ return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
38
+
39
+
40
+ def split_episodes(episodes: list[dict], fraction=0.05, seed=42) -> dict:
41
+ if not 0 < fraction < 1:
42
+ raise ValueError('Validation fraction must be in (0,1)')
43
+ if len({e['episode_index'] for e in episodes}) != len(episodes):
44
+ raise ValueError('Duplicate merged episode ID')
45
+ # Group duplicate source identities together; never leak an exact source episode.
46
+ groups = defaultdict(dict)
47
+ for ep in episodes:
48
+ pool = pool_of(ep)
49
+ key = source_identity(ep)
50
+ groups[pool, ep['task_index']].setdefault(key, []).append(ep)
51
+ rows = []
52
+ for (pool, task), identities in sorted(groups.items()):
53
+ keys = sorted(identities, key=lambda k: digest([seed, k]))
54
+ eligible = [k for k in keys if any(e['length'] > 16 for e in identities[k])]
55
+ if len(eligible) < 2:
56
+ raise ValueError(f'Cannot stratify eligible episodes for {pool}/{task}')
57
+ nval = min(len(eligible)-1, max(1, round(len(eligible)*fraction)))
58
+ val = set(eligible[:nval])
59
+ for key in keys:
60
+ for ep in identities[key]:
61
+ rows.append({**ep, 'pool': pool, 'split': 'validation' if key in val else 'train'})
62
+ rows.sort(key=lambda e: e['episode_index'])
63
+ manifest = {'dataset': DATASET, 'dataset_revision': DATASET_REVISION,
64
+ 'fraction': fraction, 'seed': seed, 'provenance_sha256': digest(episodes), 'episodes': rows}
65
+ verify_split(manifest)
66
+ return manifest
67
+
68
+
69
+ def verify_split(manifest: dict) -> None:
70
+ sets = {side: {source_identity(e) for e in manifest['episodes'] if e['split'] == side}
71
+ for side in ('train', 'validation')}
72
+ if sets['train'] & sets['validation']:
73
+ raise ValueError('Source episode leakage')
74
+ ids = {side: {e['episode_index'] for e in manifest['episodes'] if e['split'] == side}
75
+ for side in sets}
76
+ if ids['train'] & ids['validation']:
77
+ raise ValueError('Merged episode leakage')
78
+
79
+
80
+ class PoolSampler:
81
+ """One synchronous draw per sample, with recorded video failures on retry."""
82
+ def __init__(self, manifest: dict, *, split='train', weights=WEIGHTS, seed=42):
83
+ verify_split(manifest)
84
+ if len(weights) != 3 or any(not math.isfinite(w) or w <= 0 for w in weights):
85
+ raise ValueError('Three positive finite pool weights required')
86
+ self.weights = tuple(w/sum(weights) for w in weights)
87
+ self.manifest_hash = digest(manifest)
88
+ self.split = split
89
+ self.rng = random.Random(seed)
90
+ self.groups = {p: defaultdict(list) for p in POOLS}
91
+ self.counts = Counter({p: 0 for p in POOLS})
92
+ self.task_counts = Counter()
93
+ self.video_decode_failures = Counter()
94
+ self.draws = 0
95
+ for ep in manifest['episodes']:
96
+ if ep['split'] == split and ep['length'] > 16:
97
+ self.groups[ep['pool']][ep['task_index']].append(ep)
98
+ if any(not self.groups[p] for p in POOLS):
99
+ raise ValueError(f'Empty eligible pool in {split}')
100
+ self.tasks = {p: sorted(self.groups[p]) for p in POOLS}
101
+
102
+ def draw(self) -> tuple[int, dict]:
103
+ pool = self.rng.choices(POOLS, self.weights)[0]
104
+ task = self.rng.choice(self.tasks[pool])
105
+ self.counts[pool] += 1
106
+ self.task_counts[pool, task] += 1
107
+ self.draws += 1
108
+ return self._draw_anchor(pool, task)
109
+
110
+ def _draw_anchor(self, pool, task) -> tuple[int, dict]:
111
+ ep = self.rng.choice(self.groups[pool][task])
112
+ t = self.rng.randrange(ep['length'] - 16)
113
+ return ep['dataset_from_index'] + t, {**ep, 'anchor': t}
114
+
115
+ def replacement(self, episode: dict) -> tuple[int, dict]:
116
+ """Replace only the episode/anchor; keep this draw's pool, task and split."""
117
+ if episode['split'] != self.split:
118
+ raise ValueError('Replacement sample split mismatch')
119
+ return self._draw_anchor(episode['pool'], episode['task_index'])
120
+
121
+ def state_dict(self) -> dict:
122
+ return copy.deepcopy({'rng': self.rng.getstate(), 'counts': self.counts,
123
+ 'task_counts': self.task_counts, 'draws': self.draws,
124
+ 'video_decode_failures': self.video_decode_failures,
125
+ 'manifest_hash': self.manifest_hash, 'weights': self.weights, 'split': self.split})
126
+
127
+ def load_state_dict(self, state: dict) -> None:
128
+ for key in ('manifest_hash', 'weights', 'split'):
129
+ if state[key] != getattr(self, key):
130
+ raise ValueError(f'Sampler resume mismatch: {key}')
131
+ self.rng.setstate(state['rng'])
132
+ self.counts, self.task_counts = Counter(state['counts']), Counter(state['task_counts'])
133
+ # Checkpoints created before video recovery did not have this counter.
134
+ self.video_decode_failures = Counter(state.get('video_decode_failures', {}))
135
+ self.draws = state['draws']
136
+
137
+ def distribution_report(self, check=False) -> dict:
138
+ report = {'draws': self.draws, 'counts': dict(self.counts),
139
+ 'video_decode_failures': dict(self.video_decode_failures),
140
+ 'percentages': {p: 100*self.counts[p]/max(1,self.draws) for p in POOLS}}
141
+ if check and self.draws >= 10000:
142
+ for pool, prob in zip(POOLS, self.weights, strict=True):
143
+ tolerance = max(0.01, 6*math.sqrt(prob*(1-prob)/self.draws))
144
+ if abs(self.counts[pool]/self.draws-prob) > tolerance:
145
+ raise ValueError(f'Unexpected pool distribution: {report}')
146
+ n = self.counts[pool]
147
+ p = 1/len(self.tasks[pool])
148
+ for task in self.tasks[pool]:
149
+ if abs(self.task_counts[pool, task]-n*p) > max(10, 7*math.sqrt(n*p*(1-p))):
150
+ raise ValueError(f'Unexpected task distribution: {pool}/{task}')
151
+ return report
152
+
153
+
154
+ def resize_rgb(x: torch.Tensor, size: int) -> torch.Tensor:
155
+ shape = x.shape
156
+ return F.interpolate(x.reshape(-1, 3, *shape[-2:]), size=(size,size), mode='bilinear',
157
+ align_corners=False, antialias=True).reshape(*shape[:-2], size,size)
158
+
159
+
160
+ class ResolutionBottleneck(torch.nn.Module):
161
+ """After released ToTensor, before photometric augmentation; all temporal frames."""
162
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
163
+ if x.shape[-3:] != (3,256,256) or not x.is_floating_point():
164
+ raise ValueError(f'Expected float RGB [...,3,256,256], got {x.shape}/{x.dtype}')
165
+ if not bool(torch.isfinite(x).all()) or x.min() < 0 or x.max() > 1:
166
+ raise ValueError('Expected finite RGB [0,1]')
167
+ return resize_rgb(resize_rgb(x,128),224)
168
+
169
+
170
+ def competition_cameras(front: torch.Tensor, wrist: torch.Tensor, *, rotate_180=True) -> torch.Tensor:
171
+ """Simulator uint8 HWC -> normalized CHW. Stored training videos never rotate."""
172
+ cameras = []
173
+ for image in (front,wrist):
174
+ if image.shape != (128,128,3) or image.dtype != torch.uint8:
175
+ raise ValueError('Competition camera must be uint8 [128,128,3]')
176
+ x = image.permute(2,0,1).float()/255
177
+ if rotate_180:
178
+ x = x.flip((-2,-1))
179
+ cameras.append(resize_rgb(x,224))
180
+ return torch.cat(cameras,dim=-1)*2-1
checkpoints/step_004000/metadata/source/resolution_adapt/model.py ADDED
@@ -0,0 +1,188 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Strict Stage-2 warm start and audited hybrid PEFT; usable entirely offline."""
2
+ from __future__ import annotations
3
+
4
+ from collections import defaultdict
5
+ from pathlib import Path
6
+ import torch
7
+
8
+ IMAGEWAM_COMMIT = '1792d100099771fe404ebca5dcd606725141585b'
9
+ FLUX_COMMIT = '50fe5162777813d869182b139e83b10743caef15'
10
+ LIT_REVISION = '765a2b2bf136f7039ab18978920c682f1aae65d2'
11
+ TARGETS = ('qkv','proj','linear1','linear2','img_mlp.0','img_mlp.2','txt_mlp.0','txt_mlp.2')
12
+
13
+
14
+ def configure_peft(model, args) -> tuple[list[dict], dict]:
15
+ from imagewam.models.backbones.lora import apply_lora_to_linear_suffixes
16
+ model.requires_grad_(True)
17
+ transformer = model.video_expert.transformer
18
+ # Freeze all original video-expert parameters, not just the selected linears.
19
+ model.video_expert.requires_grad_(False)
20
+ model.vae.requires_grad_(False)
21
+ qwen = getattr(getattr(model,'text_encoder',None),'model',None)
22
+ if qwen is not None:
23
+ qwen.requires_grad_(False)
24
+ qwen.eval()
25
+ matches = [n for n,m in transformer.named_modules()
26
+ if isinstance(m,torch.nn.Linear) and any(n.endswith(s) for s in TARGETS)]
27
+ if not matches:
28
+ raise RuntimeError('No supported FLUX LoRA targets found')
29
+ result = apply_lora_to_linear_suffixes(transformer,target_suffixes=TARGETS,
30
+ rank=args.lora_rank,alpha=args.lora_alpha,dropout=args.lora_dropout)
31
+ if result.replaced != len(matches) or not result.replaced:
32
+ raise RuntimeError('FLUX LoRA injection mismatch')
33
+ # Upstream creates adapter tensors on CPU; move only the new tensors.
34
+ for module in transformer.modules():
35
+ if hasattr(module,'lora_A'):
36
+ module.lora_A = torch.nn.Parameter(module.lora_A.to(module.base.weight.device))
37
+ module.lora_B = torch.nn.Parameter(module.lora_B.to(module.base.weight.device))
38
+ blocks = list(transformer.double_blocks)+list(transformer.single_blocks)
39
+ n = args.unfreeze_last_flux_blocks
40
+ if not 0 <= n <= len(blocks):
41
+ raise ValueError(f'unfreeze_last_flux_blocks must be in [0,{len(blocks)}]')
42
+ if n:
43
+ for block in blocks[-n:]:
44
+ block.requires_grad_(True)
45
+ model.video_expert.flux2_lora_enabled = True
46
+ model.video_expert.flux2_lora_target_suffixes = TARGETS
47
+ # Do not call upstream apply_trainable_policy: its Stage-2 branch unfreezes FLUX.
48
+ owners = {}
49
+ def mark(module, group):
50
+ if module is not None:
51
+ for p in module.parameters():
52
+ owners[id(p)] = group
53
+ mark(model.video_expert,'flux_dense')
54
+ for name,p in transformer.named_parameters():
55
+ if name.endswith(('.lora_A','.lora_B')):
56
+ owners[id(p)] = 'flux_lora'
57
+ mark(model.action_expert,'action_expert')
58
+ mark(model.semantic_visual_aggregator,'lit_interface')
59
+ mark(model.semantic_visual_pose_norm,'pose_decoder')
60
+ mark(model.semantic_visual_pose_decoder,'pose_decoder')
61
+ mark(model.proprio_encoder,'proprio_encoder')
62
+ mark(model.vae,'vae')
63
+ mark(qwen,'qwen')
64
+ rates = {'flux_lora':args.lr_flux_lora,'flux_dense':args.lr_unfrozen_flux,
65
+ 'action_expert':args.lr_action,'lit_interface':args.lr_interface,
66
+ 'pose_decoder':args.lr_pose,'proprio_encoder':args.lr_proprio,'other_stage2':args.lr_other}
67
+ grouped = defaultdict(list)
68
+ report = {k:{'total':0,'trainable':0,'frozen':0} for k in (*rates,'qwen','vae')}
69
+ for name,p in model.named_parameters():
70
+ group = owners.get(id(p),'other_stage2')
71
+ row = report[group]
72
+ row['total'] += p.numel()
73
+ row['trainable' if p.requires_grad else 'frozen'] += p.numel()
74
+ if p.requires_grad:
75
+ if group not in rates:
76
+ raise RuntimeError(f'Frozen-only module trains: {name}')
77
+ # FP32 master parameter and AdamW moments; BF16 forward via autocast.
78
+ p.data = p.data.float()
79
+ grouped[group].append(p)
80
+ # SimpleNamespace text encoders are not registered PyTorch children.
81
+ if qwen is not None and not report['qwen']['total']:
82
+ report['qwen'] = {'total':sum(p.numel() for p in qwen.parameters()),'trainable':0,
83
+ 'frozen':sum(p.numel() for p in qwen.parameters())}
84
+ for key in ('flux_lora','action_expert','lit_interface','pose_decoder','proprio_encoder'):
85
+ if report[key]['trainable'] == 0 or report[key]['frozen']:
86
+ raise RuntimeError(f'Required fully trainable group failed: {key}')
87
+ if report['vae']['trainable'] or report['qwen']['trainable'] or (n == 0 and report['flux_dense']['trainable']):
88
+ raise RuntimeError('Freeze policy violated')
89
+ groups = []
90
+ # Bound optimizer temporaries with the repository's 128M-element policy.
91
+ for name,parameters in grouped.items():
92
+ part, count = [],0
93
+ for p in parameters:
94
+ if part and count+p.numel() > 128_000_000:
95
+ groups.append({'params':part,'name':name,'lr':rates[name],'weight_decay':args.weight_decay})
96
+ part,count = [],0
97
+ part.append(p); count += p.numel()
98
+ if part:
99
+ groups.append({'params':part,'name':name,'lr':rates[name],'weight_decay':args.weight_decay})
100
+ included = [id(p) for g in groups for p in g['params']]
101
+ expected = {id(p) for p in model.parameters() if p.requires_grad}
102
+ if len(included) != len(set(included)) or set(included) != expected:
103
+ raise RuntimeError('Optimizer coverage mismatch')
104
+ totals = {k:sum(row[k] for row in report.values()) for k in ('total','trainable','frozen')}
105
+ totals['lora'] = report['flux_lora']['trainable']
106
+ totals['dense_trainable'] = totals['trainable']-totals['lora']
107
+ table = [{'name':g['name'],'tensors':len(g['params']),'parameters':sum(p.numel() for p in g['params']),
108
+ 'lr':g['lr'],'weight_decay':g['weight_decay']} for g in groups]
109
+ return groups, {'components':report,'totals':totals,'optimizer_groups':table,'lora_targets':matches,
110
+ 'qwen_cached_and_not_instantiated':qwen is None}
111
+
112
+
113
+ def trainable_state(model) -> dict:
114
+ return {name:p.detach().cpu().clone() for name,p in model.named_parameters() if p.requires_grad}
115
+
116
+
117
+ def load_adaptation(model, path: Path) -> None:
118
+ """After released base + identical LoRA injection; no network/logging dependencies."""
119
+ state = torch.load(path,map_location='cpu',weights_only=True)
120
+ expected = {n:p for n,p in model.named_parameters() if p.requires_grad}
121
+ if state.keys() != expected.keys():
122
+ raise ValueError('Adaptation parameter names do not match the configured base + PEFT model')
123
+ with torch.no_grad():
124
+ for name,p in expected.items():
125
+ if p.shape != state[name].shape:
126
+ raise ValueError(f'Adaptation shape mismatch: {name}')
127
+ p.copy_(state[name])
128
+
129
+
130
+ def create_stage2_meta(config: dict):
131
+ """Same construction as existing offline model_loader, with released config fields.
132
+
133
+ Meta initialization avoids both a second 4B base copy and any action initializer.
134
+ Every trainable tensor must be assigned from the full released Stage-2 payload.
135
+ """
136
+ from flux2.model import Flux2, Klein4BParams
137
+ from flux2.autoencoder import AutoEncoder, AutoEncoderParams
138
+ from imagewam.models.backbones.flux2_video_expert import Flux2VideoExpert
139
+ from imagewam.models.backbones.action_dit_flux2 import ActionDiTFlux2
140
+ from imagewam.models.backbones.mot import MoT
141
+ from imagewam.models.backbones.imagewam import ImageWAM
142
+ with torch.device('meta'):
143
+ video = Flux2VideoExpert(Flux2(Klein4BParams()).to(torch.bfloat16),'klein-base-4b')
144
+ action = ActionDiTFlux2(**config['action_dit_config']).to(torch.bfloat16)
145
+ mot = MoT(mixtures={'video':video,'action':action},
146
+ mot_checkpoint_mixed_attn=config['mot_checkpoint_mixed_attn'],
147
+ gqa_implementation=config['mot_gqa_implementation'],
148
+ force_flash_attention=config['mot_force_flash_attention'])
149
+ vae = AutoEncoder(AutoEncoderParams()).to(torch.bfloat16)
150
+ v,a = config['video_scheduler'],config['action_scheduler']
151
+ model = ImageWAM(video,action,mot,vae,text_dim=7680,proprio_dim=8,device='meta',
152
+ torch_dtype=torch.bfloat16,stack='flux2',qwen_context_len=512,pack_proprio_after_text=True,
153
+ goal_prior_stage='stage2',goal_prior=config['goal_prior'],
154
+ video_train_shift=v['train_shift'],video_infer_shift=v['infer_shift'],
155
+ video_num_train_timesteps=v['num_train_timesteps'],
156
+ action_train_shift=a['train_shift'],action_infer_shift=a['infer_shift'],
157
+ action_num_train_timesteps=a['num_train_timesteps'],
158
+ loss_lambda_video=0.5,loss_lambda_action=1.0,loss_lambda_pose=0.3)
159
+ return model
160
+
161
+
162
+ def build_stage2(config: dict, checkpoint: Path, ae_path: Path, device='cuda'):
163
+ from safetensors.torch import load_file
164
+ model = create_stage2_meta(config)
165
+ payload = torch.load(checkpoint,map_location='cpu',mmap=True,weights_only=True)
166
+ required = {'mot','proprio_encoder','semantic_visual_aggregator',
167
+ 'semantic_visual_pose_norm','semantic_visual_pose_decoder'}
168
+ if set(payload) != required|{'step','torch_dtype','goal_prior_stage'} or payload['goal_prior_stage'] != 'stage2':
169
+ raise ValueError('Expected complete released Stage-2 payload, without LoRA or Stage-1 bridging')
170
+ cache,report = {},{}
171
+ for name in sorted(required):
172
+ state = {}
173
+ for key,value in payload[name].items():
174
+ identity = (value.data_ptr(),tuple(value.shape),tuple(value.stride()))
175
+ if identity not in cache:
176
+ cache[identity] = value.to(device=device,dtype=torch.bfloat16)
177
+ state[key] = cache[identity]
178
+ result = getattr(model,name).load_state_dict(state,strict=False,assign=True)
179
+ report[name] = {'missing_keys':list(result.missing_keys),'unexpected_keys':list(result.unexpected_keys)}
180
+ print('Released checkpoint keys:',report,flush=True)
181
+ if any(r['missing_keys'] or r['unexpected_keys'] for r in report.values()):
182
+ raise RuntimeError(f'Stage-2 load mismatch: {report}')
183
+ model.vae.load_state_dict(load_file(str(ae_path),device=device),strict=True,assign=True)
184
+ model.to(device=device,dtype=torch.bfloat16)
185
+ model.device = torch.device(device)
186
+ if any(p.is_meta for p in model.parameters()):
187
+ raise RuntimeError('Uninitialized model tensor after strict Stage-2 load')
188
+ return model,report
checkpoints/step_004000/metadata/source/resolution_adapt/offline.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Offline reconstruction: released Stage-2 + adaptation weights + local AE/Qwen."""
2
+ from __future__ import annotations
3
+ import json
4
+ import hashlib
5
+ import sys
6
+ from pathlib import Path
7
+ from types import SimpleNamespace
8
+ import torch
9
+ import yaml
10
+
11
+
12
+ def load_offline(checkpoint: Path, *, released_model: Path, autoencoder: Path,
13
+ qwen: Path, imagewam_source: Path, flux_source: Path, device='cuda'):
14
+ """Requires packaged local assets only. No W&B, HF API, or training dataset imports."""
15
+ sys.path[:0] = [str(imagewam_source/'src'),str(flux_source/'src')]
16
+ from .model import build_stage2,configure_peft,load_adaptation
17
+ metadata=checkpoint/'metadata'
18
+ lock=json.loads((metadata/'assets.lock.json').read_text())
19
+ for path,expected in [(released_model,lock['linjianman/LIT']['files']['imagewam/lit_stage2/model.pt']['sha256']),
20
+ (autoencoder,lock['black-forest-labs/FLUX.2-dev']['files']['ae.safetensors']['sha256'])]:
21
+ with path.open('rb') as stream:
22
+ actual=hashlib.file_digest(stream,'sha256').hexdigest()
23
+ if actual!=expected: raise ValueError(f'Offline base asset hash mismatch: {path}')
24
+ config=yaml.safe_load((metadata/'config.yaml').read_text())
25
+ args=SimpleNamespace(**json.loads((metadata/'args.json').read_text()))
26
+ model,_=build_stage2(config['model'],released_model,autoencoder,device=device)
27
+ configure_peft(model,args)
28
+ load_adaptation(model,checkpoint/'weights/adaptation.pt')
29
+ # Merge using the upstream implementation, which honors alpha/rank exactly.
30
+ from imagewam.models.backbones.lora import merge_lora_linear_layers
31
+ merge_lora_linear_layers(model.video_expert.transformer)
32
+ model.to(device=device,dtype=torch.bfloat16).eval().requires_grad_(False)
33
+ from transformers import AutoModelForCausalLM,AutoTokenizer
34
+ encoder=AutoModelForCausalLM.from_pretrained(qwen,local_files_only=True,torch_dtype=torch.bfloat16,
35
+ attn_implementation='sdpa').to(device).eval().requires_grad_(False)
36
+ model.text_encoder=SimpleNamespace(model=encoder,tokenizer=AutoTokenizer.from_pretrained(qwen,local_files_only=True),
37
+ max_length=512)
38
+ return model
checkpoints/step_004000/metadata/source/resolution_adapt/preflight_data.py ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Read-only representative endpoint audit; optional bounded HTTP video streaming."""
2
+ from __future__ import annotations
3
+ import argparse
4
+ import json
5
+ import sys
6
+ from pathlib import Path
7
+
8
+ import numpy as np
9
+ import torch
10
+ from PIL import Image, ImageDraw
11
+
12
+ sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
13
+ from resolution_adapt.data import DATASET,DATASET_REVISION,POOLS,pool_of,ResolutionBottleneck,resize_rgb
14
+ from resolution_adapt.model import LIT_REVISION
15
+ from training_support import DatasetGripperToImageWAM
16
+
17
+
18
+ def audit(root: Path, output: Path, *, stream=False, stats_path=None, processed=None):
19
+ import av
20
+ import pyarrow.parquet as pq
21
+ from huggingface_hub import hf_hub_download,hf_hub_url,get_token
22
+ output.mkdir(parents=True,exist_ok=True)
23
+ def file(name):
24
+ return Path(hf_hub_download(DATASET,name,repo_type='dataset',revision=DATASET_REVISION,
25
+ local_dir=str(root))) if stream else root/name
26
+ provenance=[json.loads(line) for line in file('meta/source_episodes.jsonl').read_text().splitlines()]
27
+ info=json.loads(file('meta/info.json').read_text())
28
+ ep_rows=pq.read_table(file('meta/episodes/chunk-000/file-000.parquet')).to_pylist()
29
+ ep_by_id={e['episode_index']:e for e in ep_rows}
30
+ if stats_path is None:
31
+ stats_path=hf_hub_download('linjianman/LIT','imagewam/lit_stage2/dataset_stats.json',revision=LIT_REVISION)
32
+ stats=json.loads(Path(stats_path).read_text())
33
+ def normalize(value,category):
34
+ st=stats[category]['default']; low=torch.tensor(st['global_min']); high=torch.tensor(st['global_max'])
35
+ span=high-low; constant=span<1e-4; span[constant]=2
36
+ scale=2/span; offset=-1-scale*low; offset[constant]=-low[constant]
37
+ return (value*scale+offset).clamp(-5,5)
38
+ report=[]; selected=[next(e for e in provenance if pool_of(e)==p and e['length']>32) for p in POOLS]
39
+ for prov in selected:
40
+ pool=pool_of(prov); ep=ep_by_id[prov['episode_index']]
41
+ shard=info['data_path'].format(chunk_index=ep['data/chunk_index'],file_index=ep['data/file_index'])
42
+ rows=pq.read_table(file(shard),filters=[('episode_index','=',ep['episode_index'])]).to_pylist()
43
+ rows.sort(key=lambda r:r['frame_index'])
44
+ t=0
45
+ states=torch.tensor([r['observation.state'] for r in rows[:17]],dtype=torch.float32)
46
+ actions=torch.tensor([r['action'] for r in rows[:16]],dtype=torch.float32)
47
+ assert states.shape==(17,8) and actions.shape==(16,7)
48
+ assert all(r['episode_index']==prov['episode_index'] for r in rows[:17])
49
+ original=actions.clone()
50
+ converted=DatasetGripperToImageWAM().forward({'action':{'default':actions}})['action']['default']
51
+ assert torch.equal(converted[:,:6],original[:,:6])
52
+ assert torch.equal(converted[:,-1],(1-original[:,-1])/2)
53
+ cameras=[]; panels=[]
54
+ for camera in ('front','wrist'):
55
+ key=f'observation.images.{camera}'
56
+ video=info['video_path'].format(video_key=key,chunk_index=ep[f'videos/{key}/chunk_index'],
57
+ file_index=ep[f'videos/{key}/file_index'])
58
+ timestamps=[ep[f'videos/{key}/from_timestamp']+rows[i]['timestamp'] for i in (0,16)]
59
+ options={'rw_timeout':'30000000'}
60
+ location=str(root/video)
61
+ if stream:
62
+ location=hf_hub_url(DATASET,video,repo_type='dataset',revision=DATASET_REVISION)
63
+ token=get_token()
64
+ if token: options['headers']='Authorization: Bearer '+token+'\r\n'
65
+ with av.open(location,options=options) as container:
66
+ video_stream=container.streams.video[0]
67
+ container.seek(int(timestamps[0]/video_stream.time_base),stream=video_stream,backward=True)
68
+ decoded={}
69
+ for frame in container.decode(video_stream):
70
+ timestamp=float(frame.pts*video_stream.time_base)
71
+ for i,target in enumerate(timestamps):
72
+ if abs(timestamp-target)<1e-4: decoded[i]=frame.to_ndarray(format='rgb24')
73
+ if len(decoded)==2: break
74
+ if timestamp > timestamps[-1]+.1: raise ValueError('Missing endpoint video frame')
75
+ raw=torch.from_numpy(np.stack([decoded[i] for i in (0,1)])).permute(0,3,1,2)
76
+ assert raw.shape==(2,3,256,256) and raw.dtype==torch.uint8
77
+ source=raw.float()/255; degraded=resize_rgb(source,128); final=ResolutionBottleneck()(source)
78
+ assert degraded.shape==(2,3,128,128) and final.shape==(2,3,224,224)
79
+ cameras.append(final)
80
+ panels.append((camera,source[0],degraded[0],final[0]))
81
+ combined=torch.cat(cameras,dim=-1)*2-1
82
+ assert combined.shape==(2,3,224,448) and -1<=combined.min()<=combined.max()<=1
83
+ normalized_action=normalize(converted,'action'); pose=normalize(states[16],'state')
84
+ if processed is not None:
85
+ augment=processed.video_augmentation; processed.video_augmentation=None
86
+ try: sample=processed[prov['dataset_from_index']+t]
87
+ finally: processed.video_augmentation=augment
88
+ torch.testing.assert_close(sample['goal_pose'],pose)
89
+ torch.testing.assert_close(sample['action'],normalized_action)
90
+ torch.testing.assert_close(sample['video'],combined.permute(1,0,2,3),atol=.02,rtol=0)
91
+ assert not sample['goal_pose_is_pad'].any()
92
+ canvas=Image.new('RGB',(720,560),'white'); draw=ImageDraw.Draw(canvas)
93
+ for row,(camera,*images) in enumerate(panels):
94
+ x=0
95
+ for label,img in zip(('source 256','bottleneck 128','model 224'),images,strict=True):
96
+ array=(img.permute(1,2,0).numpy()*255).round().clip(0,255).astype(np.uint8)
97
+ canvas.paste(Image.fromarray(array),(x,row*280+22))
98
+ draw.text((x,row*280),f'{pool} {camera}: {label}',fill='black'); x+=img.shape[-1]+20
99
+ canvas.save(output/f'{pool}.png')
100
+ report.append({'pool':pool,'episode':prov['episode_index'],'anchor':t,'pose_frame':t+16,
101
+ 'raw':[3,256,256],'degraded':[3,128,128],'model':[3,224,224],'combined':[3,224,448],
102
+ 'raw_rgb_range':[0,255],'float_range':[0,1],'normalized_image_range':[float(combined.min()),float(combined.max())],
103
+ 'state_shape':[8],'action_shape':list(converted.shape),'pose_shape':list(pose.shape),
104
+ 'action_normalized_range':[float(normalized_action.min()),float(normalized_action.max())],
105
+ 'gripper_exactly_once':True,'stored_rotation':False,'processed_loader_compared':processed is not None})
106
+ (output/'dataset_preflight.json').write_text(json.dumps(report,indent=2)+'\n')
107
+ return report
108
+
109
+
110
+ if __name__=='__main__':
111
+ p=argparse.ArgumentParser();p.add_argument('--dataset-root',type=Path,required=True)
112
+ p.add_argument('--output',type=Path,required=True);p.add_argument('--stream-from-hub',action='store_true')
113
+ p.add_argument('--stats',type=Path)
114
+ a=p.parse_args(); print(json.dumps(audit(a.dataset_root,a.output,stream=a.stream_from_hub,stats_path=a.stats),indent=2))
checkpoints/step_004000/metadata/source/resolution_adapt/preflight_model.py ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CPU architecture counts and actual remote checkpoint key/shape audit via range reads.
2
+
3
+ This deliberately does NOT claim to validate tensor bytes or a real forward pass.
4
+ """
5
+ from __future__ import annotations
6
+ import argparse
7
+ import json
8
+ import sys
9
+ from pathlib import Path
10
+ import torch
11
+ import yaml
12
+
13
+ sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
14
+ from resolution_adapt.model import create_stage2_meta,configure_peft,LIT_REVISION
15
+ from train_resolution_adapt import parser as training_parser,check_sources
16
+
17
+
18
+ def main():
19
+ p=argparse.ArgumentParser()
20
+ p.add_argument('--imagewam-root',type=Path,required=True); p.add_argument('--flux-root',type=Path,required=True)
21
+ p.add_argument('--output',type=Path,required=True); args=p.parse_args(); args.output.mkdir(parents=True,exist_ok=True)
22
+ check_sources(args)
23
+ from huggingface_hub import HfFileSystem
24
+ from torch._subclasses.fake_tensor import FakeTensorMode
25
+ model=create_stage2_meta(yaml.safe_load((Path(__file__).parent/'released_stage2.yaml').read_text())['model'])
26
+ with HfFileSystem().open(f'linjianman/LIT@{LIT_REVISION}/imagewam/lit_stage2/model.pt','rb',block_size=1024*1024) as stream:
27
+ with FakeTensorMode(): payload=torch.load(stream,map_location='cpu',weights_only=True)
28
+ required={'mot','proprio_encoder','semantic_visual_aggregator','semantic_visual_pose_norm','semantic_visual_pose_decoder'}
29
+ if set(payload)!=required|{'step','torch_dtype','goal_prior_stage'} or payload['goal_prior_stage']!='stage2':
30
+ raise ValueError('Not a complete released Stage-2 payload')
31
+ report={}
32
+ for name in sorted(required):
33
+ expected=getattr(model,name).state_dict(); actual=payload[name]
34
+ report[name]={'missing_keys':sorted(set(expected)-set(actual)),'unexpected_keys':sorted(set(actual)-set(expected)),
35
+ 'shape_mismatches':[k for k in expected.keys() & actual.keys() if expected[k].shape!=actual[k].shape]}
36
+ if any(report[name].values()): raise ValueError(report[name])
37
+ report['verification']='Actual pinned checkpoint metadata only; range reads with FakeTensorMode. GPU tensor load and backward remain required.'
38
+ (args.output/'checkpoint_keys.json').write_text(json.dumps(report,indent=2)+'\n')
39
+ _,counts=configure_peft(model,training_parser().parse_args([]))
40
+ counts['verification']='Exact pinned architecture on meta device; no checkpoint tensor bytes loaded'
41
+ (args.output/'parameter_report.json').write_text(json.dumps(counts,indent=2)+'\n')
42
+ print(json.dumps(report,indent=2));print(json.dumps(counts['totals'],indent=2))
43
+
44
+
45
+ if __name__=='__main__': main()
checkpoints/step_004000/metadata/source/resolution_adapt/prepare_assets.py ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Explicit asset downloads and optional frozen Qwen caching; never starts training."""
2
+ from __future__ import annotations
3
+ import argparse
4
+ import hashlib
5
+ import json
6
+ import subprocess
7
+ import sys
8
+ from pathlib import Path
9
+
10
+ sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
11
+ from resolution_adapt.data import DATASET,DATASET_REVISION
12
+ from resolution_adapt.model import IMAGEWAM_COMMIT,FLUX_COMMIT,LIT_REVISION
13
+
14
+
15
+ def main():
16
+ p=argparse.ArgumentParser(); p.add_argument('--runtime',type=Path,required=True)
17
+ p.add_argument('--cache-text',action='store_true'); args=p.parse_args()
18
+ root=args.runtime.resolve();root.mkdir(parents=True,exist_ok=True)
19
+ for name,url,commit in [('upstream','https://github.com/jianmanlincjx/ImageWAM.git',IMAGEWAM_COMMIT),
20
+ ('flux2','https://github.com/black-forest-labs/flux2.git',FLUX_COMMIT)]:
21
+ target=root/name
22
+ if not target.exists(): subprocess.run(['git','clone',url,str(target)],check=True)
23
+ if subprocess.check_output(['git','-C',str(target),'status','--porcelain'],text=True):
24
+ raise ValueError(f'Refusing to change dirty checkout: {target}')
25
+ subprocess.run(['git','-C',str(target),'checkout',commit],check=True)
26
+ from huggingface_hub import snapshot_download,hf_hub_download
27
+ lock=json.loads((Path(__file__).parent/'assets.lock.json').read_text())
28
+ snapshot_download(DATASET,repo_type='dataset',revision=DATASET_REVISION,local_dir=root/'dataset')
29
+ snapshot_download('linjianman/LIT',revision=LIT_REVISION,allow_patterns=['imagewam/lit_stage2/*'],local_dir=root/'released')
30
+ ae=lock['black-forest-labs/FLUX.2-dev']
31
+ hf_hub_download('black-forest-labs/FLUX.2-dev','ae.safetensors',revision=ae['revision'],
32
+ local_dir=root/'models/FLUX.2-dev')
33
+ snapshot_download('Qwen/Qwen3-4B',revision=lock['Qwen/Qwen3-4B']['revision'],local_dir=root/'models/Qwen3-4B')
34
+ receipt={'dataset':DATASET,'dataset_revision':DATASET_REVISION,'models':lock,
35
+ 'source_provenance_sha256':hashlib.sha256((root/'dataset/meta/source_episodes.jsonl').read_bytes()).hexdigest()}
36
+ (root/'assets.json').write_text(json.dumps(receipt,indent=2)+'\n')
37
+ if args.cache_text:
38
+ from train_resolution_adapt import parser,make_config
39
+ import yaml
40
+ cfg_args=parser().parse_args([])
41
+ cfg_args.dataset_root=root/'dataset';cfg_args.flux_root=root/'flux2';cfg_args.qwen_cache=root/'qwen_cache'
42
+ cfg=make_config(cfg_args)
43
+ cfg['flux2_qwen3_model_spec']=str(root/'models/Qwen3-4B')
44
+ cfg['hydra']={'run':{'dir':str(root/'cache_config/logs')}}
45
+ cfg['qwen_cache_batch_size']=1
46
+ cfg['qwen_cache_overwrite']=True # Never relabel an unverified old cache as pinned.
47
+ path=root/'cache_config';path.mkdir(exist_ok=True)
48
+ (path/'cache.yaml').write_text(yaml.safe_dump(cfg))
49
+ subprocess.run([sys.executable,str(root/'upstream/scripts/flux2/precompute_flux2_qwen3_embeds.py'),
50
+ '--config-path',str(path),'--config-name','cache'],check=True)
51
+ cache=root/'qwen_cache'
52
+ (cache/'provenance.json').write_text(json.dumps({'repo':'Qwen/Qwen3-4B','revision':lock['Qwen/Qwen3-4B']['revision'],
53
+ 'imagewam_commit':IMAGEWAM_COMMIT,'context_len':512,'layers':[9,18,27]},indent=2)+'\n')
54
+ print('Assets prepared. No training launched.')
55
+
56
+
57
+ if __name__=='__main__': main()
checkpoints/step_004000/metadata/source/resolution_adapt/released_stage2.yaml ADDED
@@ -0,0 +1,271 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ output_dir: ./runs/libero_flux2_klein_4b_goal_prior_stage2/2026-08-18_04-16-51
2
+ batch_size: 10
3
+ num_workers: 24
4
+ prefetch_factor: 2
5
+ persistent_workers: true
6
+ lr_scheduler_type: cosine
7
+ learning_rate: 0.0001
8
+ warmup_steps: null
9
+ num_epochs: 10
10
+ max_steps: null
11
+ log_every: 10
12
+ save_every: 1000
13
+ keep_latest_state_only: true
14
+ eval_every: 100
15
+ eval_num_inference_steps: 10
16
+ eval_num_samples: 8
17
+ rank_timer_every: 10
18
+ rank_timer_sync_cuda: true
19
+ qwen_cache_batch_size: 32
20
+ qwen_cache_overwrite: false
21
+ qwen_cache_save_workers: 8
22
+ flux2_qwen3_model_spec: null
23
+ precache_arrow: true
24
+ precache_num_workers: 4
25
+ build_lerobot_meta_cache: true
26
+ build_norm_stats: true
27
+ precache_fuse_norm_stats: true
28
+ force_rebuild_norm_stats: false
29
+ norm_stats_use_arrow_cache: true
30
+ norm_stats_bulk_read_arrow: true
31
+ norm_stats_cache_enabled: true
32
+ norm_stats_cache_dir: null
33
+ precache_profile: false
34
+ precache_profile_min_sec: 0.0
35
+ gradient_accumulation_steps: 1
36
+ mixed_precision: bf16
37
+ seed: 42
38
+ max_grad_norm: 1.0
39
+ weight_decay: 0.01
40
+ resume: null
41
+ stage1_checkpoint: /data2/JM/Code/ImageWAM-v2/runs/libero_flux2_klein_4b_goal_prior_stage1/2026-08-17_13-21-30/checkpoints/weights/step_010000.pt
42
+ wandb:
43
+ enabled: true
44
+ workspace: arisilin
45
+ project: imagewam
46
+ name: libero_flux2_klein_4b_goal_prior_stage2
47
+ group: null
48
+ mode: offline
49
+ data:
50
+ train:
51
+ _target_: imagewam.datasets.lerobot.robot_video_dataset.RobotVideoDataset
52
+ dataset_dirs:
53
+ - /data2/JM/Code/ImageWAM/data/libero_mujoco3.3.2/libero_spatial_no_noops_lerobot
54
+ - /data2/JM/Code/ImageWAM/data/libero_mujoco3.3.2/libero_object_no_noops_lerobot
55
+ - /data2/JM/Code/ImageWAM/data/libero_mujoco3.3.2/libero_goal_no_noops_lerobot
56
+ - /data2/JM/Code/ImageWAM/data/libero_mujoco3.3.2/libero_10_no_noops_lerobot
57
+ shape_meta:
58
+ images:
59
+ - key: image
60
+ raw_shape:
61
+ - 3
62
+ - 512
63
+ - 512
64
+ shape:
65
+ - 3
66
+ - 224
67
+ - 224
68
+ - key: wrist_image
69
+ raw_shape:
70
+ - 3
71
+ - 512
72
+ - 512
73
+ shape:
74
+ - 3
75
+ - 224
76
+ - 224
77
+ action:
78
+ - key: default
79
+ raw_shape: 7
80
+ shape: 7
81
+ state:
82
+ - key: default
83
+ raw_shape: 8
84
+ shape: 8
85
+ num_frames: 17
86
+ global_sample_stride: 1
87
+ action_video_freq_ratio: 1
88
+ video_size:
89
+ - 224
90
+ - 448
91
+ camera_key: null
92
+ val_set_proportion: 0.0
93
+ is_training_set: true
94
+ skip_padding_as_possible: false
95
+ concat_multi_camera: horizontal
96
+ video_augmentation:
97
+ _target_: imagewam.datasets.lerobot.transforms.image.VideoAugmentation
98
+ p: 0.65
99
+ augment_types:
100
+ - corrupt_only
101
+ - color_only
102
+ - both
103
+ - both
104
+ color_jitter:
105
+ brightness: 0.3
106
+ contrast: 0.3
107
+ saturation: 0.25
108
+ hue: 0.04
109
+ gamma:
110
+ range:
111
+ - 0.8
112
+ - 1.25
113
+ gaussian_noise:
114
+ std: 0.015
115
+ random_resized_crop:
116
+ scale:
117
+ - 0.92
118
+ - 1.0
119
+ ratio: preserve
120
+ rotate:
121
+ degrees: 8.0
122
+ fill: mean
123
+ exposure:
124
+ ev_range:
125
+ - -0.25
126
+ - 0.25
127
+ require_text_cache: false
128
+ endpoint_frames_only: true
129
+ qwen_text_cache_dir: /data2/JM/Code/ImageWAM/data/libero_mujoco3.3.2/flux2_qwen3_cache_4b
130
+ qwen_context_len: 512
131
+ pretrained_norm_stats: /data2/JM/Code/ImageWAM-v2/runs/libero_flux2_klein_4b_base_imagewam/2026-08-13_00-24-52/dataset_stats.json
132
+ processor:
133
+ _target_: imagewam.datasets.lerobot.processors.imagewam_processor.ImageWAMProcessor
134
+ shape_meta:
135
+ images:
136
+ - key: image
137
+ raw_shape:
138
+ - 3
139
+ - 512
140
+ - 512
141
+ shape:
142
+ - 3
143
+ - 224
144
+ - 224
145
+ - key: wrist_image
146
+ raw_shape:
147
+ - 3
148
+ - 512
149
+ - 512
150
+ shape:
151
+ - 3
152
+ - 224
153
+ - 224
154
+ action:
155
+ - key: default
156
+ raw_shape: 7
157
+ shape: 7
158
+ state:
159
+ - key: default
160
+ raw_shape: 8
161
+ shape: 8
162
+ num_obs_steps: 17
163
+ image_obs_steps: 2
164
+ num_output_cameras: 2
165
+ action_output_dim: 7
166
+ proprio_output_dim: 8
167
+ delta_action_dim_mask:
168
+ default:
169
+ - true
170
+ - true
171
+ - true
172
+ - true
173
+ - true
174
+ - true
175
+ - false
176
+ action_state_transforms: null
177
+ use_stepwise_action_norm: false
178
+ norm_default_mode: min/max
179
+ norm_exception_mode: null
180
+ action_state_merger:
181
+ _target_: imagewam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign
182
+ train_transforms:
183
+ - _target_: imagewam.datasets.lerobot.transforms.image.ToTensor
184
+ - _target_: torchvision.transforms.Resize
185
+ size:
186
+ - 224
187
+ - 224
188
+ val_transforms:
189
+ - _target_: imagewam.datasets.lerobot.transforms.image.ToTensor
190
+ - _target_: torchvision.transforms.Resize
191
+ size:
192
+ - 224
193
+ - 224
194
+ text_embedding_cache_dir: null
195
+ context_len: 128
196
+ qwen_text_cache_format: qwen3_flux2
197
+ model:
198
+ _target_: imagewam.runtime.create_imagewam_flux2_klein
199
+ flux2_src_path: /data2/JM/Code/ImageWAM/third_party/flux2
200
+ flux2_model_path: /data2/JM/Code/ImageWAM/checkpoints/flux2/FLUX.2-klein-base-4B/flux-2-klein-base-4b.safetensors
201
+ ae_model_path: /data2/JM/Code/ImageWAM/checkpoints/flux2/FLUX.2-dev/ae.safetensors
202
+ variant: klein-base-4b
203
+ qwen3_model_spec: Qwen/Qwen3-4B
204
+ qwen_context_len: 512
205
+ load_text_encoder: false
206
+ proprio_dim: 8
207
+ mot_checkpoint_mixed_attn: true
208
+ mot_gqa_implementation: repeat
209
+ mot_force_flash_attention: false
210
+ pack_proprio_after_text: true
211
+ action_dit_pretrained_path: checkpoints/action_dit_flux2_4b_libero_init.pt
212
+ goal_prior_stage: stage2
213
+ goal_prior:
214
+ num_latents: 100
215
+ num_pose_tokens: 8
216
+ num_layer_groups: 5
217
+ latent_dim: 768
218
+ inner_dim: 512
219
+ lambda_pose: 0.3
220
+ context_token_dropout: 0.05
221
+ context_blackout_prob: 0.1
222
+ zero_init_value: false
223
+ syn_gate_bias_init: -2.0
224
+ gate_pose_tokens: false
225
+ latent_layout: free
226
+ grid_hw:
227
+ - 7
228
+ - 7
229
+ grid_window: 1
230
+ action_sees_ref: true
231
+ p_both: 0.55
232
+ p_ref_only: 0.15
233
+ p_syn_only: 0.3
234
+ flux2_lora_config:
235
+ enabled: false
236
+ rank: 16
237
+ alpha: 16.0
238
+ dropout: 0.0
239
+ save_lora_merged: false
240
+ save_trainable_only: false
241
+ target_suffixes:
242
+ - qkv
243
+ - proj
244
+ - linear1
245
+ - linear2
246
+ - img_mlp.0
247
+ - img_mlp.2
248
+ - txt_mlp.0
249
+ - txt_mlp.2
250
+ action_dit_config:
251
+ action_dim: 7
252
+ hidden_dim: 1024
253
+ num_heads: 24
254
+ attn_head_dim: 128
255
+ num_layers_double: 5
256
+ num_layers_single: 20
257
+ mlp_ratio: 4.0
258
+ max_action_horizon: 64
259
+ use_gradient_checkpointing: true
260
+ video_scheduler:
261
+ train_shift: 5.0
262
+ infer_shift: 5.0
263
+ num_train_timesteps: 1000
264
+ action_scheduler:
265
+ train_shift: 5.0
266
+ infer_shift: 5.0
267
+ num_train_timesteps: 1000
268
+ loss:
269
+ lambda_video: 0.5
270
+ lambda_action: 1.0
271
+ lambda_pose: 0.3
checkpoints/step_004000/metadata/source/resolution_adapt/requirements-training.lock ADDED
@@ -0,0 +1,341 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # This file was autogenerated by uv via the following command:
2
+ # uv pip compile examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt --python-version 3.12 --output-file examples/lit_imagewam_libero/resolution_adapt/requirements-training.lock
3
+ accelerate==1.12.0
4
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
5
+ aiohappyeyeballs==2.7.1
6
+ # via aiohttp
7
+ aiohttp==3.14.4
8
+ # via fsspec
9
+ aiosignal==1.4.0
10
+ # via aiohttp
11
+ annotated-types==0.8.0
12
+ # via pydantic
13
+ antlr4-python3-runtime==4.9.3
14
+ # via
15
+ # hydra-core
16
+ # omegaconf
17
+ anyio==4.15.1
18
+ # via httpx
19
+ attrs==26.1.0
20
+ # via
21
+ # aiohttp
22
+ # jsonlines
23
+ av==16.0.1
24
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
25
+ boto3==1.35.99
26
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
27
+ botocore==1.35.99
28
+ # via
29
+ # boto3
30
+ # s3transfer
31
+ certifi==2026.7.22
32
+ # via
33
+ # httpcore
34
+ # httpx
35
+ # requests
36
+ # sentry-sdk
37
+ charset-normalizer==3.5.2
38
+ # via requests
39
+ click==8.5.0
40
+ # via wandb
41
+ cloudpickle==3.1.2
42
+ # via gymnasium
43
+ cuda-bindings==13.4.3
44
+ # via torch
45
+ cuda-pathfinder==1.8.3
46
+ # via cuda-bindings
47
+ cuda-toolkit==13.0.2
48
+ # via torch
49
+ datasets==4.8.5
50
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
51
+ diffusers==0.35.2
52
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
53
+ dill==0.4.1
54
+ # via
55
+ # datasets
56
+ # multiprocess
57
+ draccus==0.10.0
58
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
59
+ einops==0.8.1
60
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
61
+ farama-notifications==0.0.6
62
+ # via gymnasium
63
+ filelock==4.0.12
64
+ # via
65
+ # datasets
66
+ # diffusers
67
+ # huggingface-hub
68
+ # torch
69
+ # transformers
70
+ frozenlist==1.8.0
71
+ # via
72
+ # aiohttp
73
+ # aiosignal
74
+ fsspec==2026.2.0
75
+ # via
76
+ # datasets
77
+ # huggingface-hub
78
+ # torch
79
+ gitdb==4.0.12
80
+ # via gitpython
81
+ gitpython==3.2.0
82
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
83
+ gymnasium==1.2.1
84
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
85
+ h11==0.16.0
86
+ # via httpcore
87
+ hf-xet==1.6.0
88
+ # via huggingface-hub
89
+ httpcore==1.0.9
90
+ # via httpx
91
+ httpx==0.28.1
92
+ # via datasets
93
+ huggingface-hub==0.35.3
94
+ # via
95
+ # -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
96
+ # accelerate
97
+ # datasets
98
+ # diffusers
99
+ # tokenizers
100
+ # transformers
101
+ hydra-core==1.3.2
102
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
103
+ idna==3.20
104
+ # via
105
+ # anyio
106
+ # httpx
107
+ # requests
108
+ # yarl
109
+ imageio==2.37.0
110
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
111
+ importlib-metadata==9.0.1
112
+ # via diffusers
113
+ iniconfig==2.3.0
114
+ # via pytest
115
+ jinja2==3.1.6
116
+ # via torch
117
+ jmespath==1.1.0
118
+ # via
119
+ # boto3
120
+ # botocore
121
+ jsonlines==4.0.0
122
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
123
+ markdown-it-py==4.2.0
124
+ # via rich
125
+ markupsafe==3.0.4
126
+ # via jinja2
127
+ mdurl==0.1.2
128
+ # via markdown-it-py
129
+ mergedeep==1.3.4
130
+ # via draccus
131
+ mpmath==1.3.0
132
+ # via sympy
133
+ multidict==6.9.1
134
+ # via
135
+ # aiohttp
136
+ # yarl
137
+ multiprocess==0.70.19
138
+ # via datasets
139
+ mypy-extensions==1.1.0
140
+ # via typing-inspect
141
+ networkx==3.7
142
+ # via torch
143
+ numpy==1.26.4
144
+ # via
145
+ # -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
146
+ # accelerate
147
+ # datasets
148
+ # diffusers
149
+ # gymnasium
150
+ # imageio
151
+ # opencv-python-headless
152
+ # pandas
153
+ # torchvision
154
+ # transformers
155
+ nvidia-cublas==13.1.0.3
156
+ # via
157
+ # cuda-toolkit
158
+ # nvidia-cudnn-cu13
159
+ # nvidia-cusolver
160
+ nvidia-cuda-cupti==13.0.85
161
+ # via cuda-toolkit
162
+ nvidia-cuda-nvrtc==13.0.88
163
+ # via cuda-toolkit
164
+ nvidia-cuda-runtime==13.0.96
165
+ # via cuda-toolkit
166
+ nvidia-cudnn-cu13==9.19.0.56
167
+ # via torch
168
+ nvidia-cufft==12.0.0.61
169
+ # via cuda-toolkit
170
+ nvidia-cufile==1.15.1.6
171
+ # via cuda-toolkit
172
+ nvidia-curand==10.4.0.35
173
+ # via cuda-toolkit
174
+ nvidia-cusolver==12.0.4.66
175
+ # via cuda-toolkit
176
+ nvidia-cusparse==12.6.3.3
177
+ # via
178
+ # cuda-toolkit
179
+ # nvidia-cusolver
180
+ nvidia-cusparselt-cu13==0.8.0
181
+ # via torch
182
+ nvidia-nccl-cu13==2.28.9
183
+ # via torch
184
+ nvidia-nvjitlink==13.0.88
185
+ # via
186
+ # cuda-toolkit
187
+ # nvidia-cufft
188
+ # nvidia-cusolver
189
+ # nvidia-cusparse
190
+ nvidia-nvshmem-cu13==3.4.5
191
+ # via torch
192
+ nvidia-nvtx==13.0.85
193
+ # via cuda-toolkit
194
+ omegaconf==2.3.0
195
+ # via
196
+ # -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
197
+ # hydra-core
198
+ opencv-python-headless==4.11.0.86
199
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
200
+ packaging==26.3
201
+ # via
202
+ # accelerate
203
+ # datasets
204
+ # huggingface-hub
205
+ # hydra-core
206
+ # pytest
207
+ # transformers
208
+ # wandb
209
+ pandas==3.0.6
210
+ # via datasets
211
+ pillow==12.0.0
212
+ # via
213
+ # -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
214
+ # diffusers
215
+ # imageio
216
+ # torchvision
217
+ platformdirs==4.12.3
218
+ # via wandb
219
+ pluggy==1.6.0
220
+ # via pytest
221
+ propcache==0.5.4
222
+ # via
223
+ # aiohttp
224
+ # yarl
225
+ protobuf==7.36.2
226
+ # via wandb
227
+ psutil==7.2.2
228
+ # via accelerate
229
+ pyarrow==23.0.0
230
+ # via
231
+ # -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
232
+ # datasets
233
+ pydantic==2.13.5
234
+ # via wandb
235
+ pydantic-core==2.46.5
236
+ # via pydantic
237
+ pygments==2.21.0
238
+ # via
239
+ # pytest
240
+ # rich
241
+ pytest==8.4.2
242
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
243
+ python-dateutil==2.9.0.post0
244
+ # via
245
+ # botocore
246
+ # pandas
247
+ pyyaml==6.0.3
248
+ # via
249
+ # accelerate
250
+ # datasets
251
+ # draccus
252
+ # huggingface-hub
253
+ # omegaconf
254
+ # pyyaml-include
255
+ # transformers
256
+ # wandb
257
+ pyyaml-include==1.4.1
258
+ # via draccus
259
+ regex==2026.9.29
260
+ # via
261
+ # diffusers
262
+ # transformers
263
+ requests==2.34.2
264
+ # via
265
+ # datasets
266
+ # diffusers
267
+ # huggingface-hub
268
+ # transformers
269
+ # wandb
270
+ rich==14.2.0
271
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
272
+ s3transfer==0.10.4
273
+ # via boto3
274
+ safetensors==0.6.2
275
+ # via
276
+ # -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
277
+ # accelerate
278
+ # diffusers
279
+ # transformers
280
+ sentry-sdk==2.71.0
281
+ # via wandb
282
+ setuptools==81.0.0
283
+ # via torch
284
+ six==1.17.0
285
+ # via python-dateutil
286
+ smmap==5.0.3
287
+ # via gitdb
288
+ sympy==1.14.0
289
+ # via torch
290
+ termcolor==3.2.0
291
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
292
+ tokenizers==0.22.2
293
+ # via transformers
294
+ toml==0.10.2
295
+ # via draccus
296
+ torch==2.11.0
297
+ # via
298
+ # -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
299
+ # accelerate
300
+ # torchvision
301
+ torchvision==0.26.0
302
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
303
+ tqdm==4.70.1
304
+ # via
305
+ # datasets
306
+ # huggingface-hub
307
+ # transformers
308
+ transformers==4.56.1
309
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
310
+ triton==3.6.0
311
+ # via torch
312
+ typing-extensions==4.16.0
313
+ # via
314
+ # aiohttp
315
+ # aiosignal
316
+ # anyio
317
+ # gymnasium
318
+ # huggingface-hub
319
+ # pydantic
320
+ # pydantic-core
321
+ # torch
322
+ # typing-inspect
323
+ # typing-inspection
324
+ # wandb
325
+ typing-inspect==0.9.0
326
+ # via draccus
327
+ typing-inspection==0.4.4
328
+ # via pydantic
329
+ urllib3==2.8.0
330
+ # via
331
+ # botocore
332
+ # requests
333
+ # sentry-sdk
334
+ wandb==0.28.1
335
+ # via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
336
+ xxhash==4.0.1
337
+ # via datasets
338
+ yarl==1.25.1
339
+ # via aiohttp
340
+ zipp==4.1.1
341
+ # via importlib-metadata
checkpoints/step_004000/metadata/source/resolution_adapt/requirements-training.txt ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # CUDA wheels below match the repository training stack.
2
+ # Install lerobot==0.4.4 separately with --no-deps: its bounds exclude the
3
+ # repository's audited torch 2.11 / torchvision 0.26 / direct-PyAV stack.
4
+ accelerate==1.12.0
5
+ av==16.0.1
6
+ datasets==4.8.5
7
+ diffusers==0.35.2
8
+ einops==0.8.1
9
+ huggingface-hub==0.35.3
10
+ hydra-core==1.3.2
11
+ omegaconf==2.3.0
12
+ numpy==1.26.4
13
+ pillow==12.0.0
14
+ pyarrow==23.0.0
15
+ safetensors==0.6.2
16
+ transformers==4.56.1
17
+ wandb==0.28.1
18
+ jsonlines==4.0.0
19
+ termcolor==3.2.0
20
+ rich==14.2.0
21
+ draccus==0.10.0
22
+ opencv-python-headless==4.11.0.86
23
+ gymnasium==1.2.1
24
+ pytest==8.4.2
25
+ torch==2.11.0
26
+ torchvision==0.26.0
27
+ imageio==2.37.0
28
+ boto3==1.35.99
29
+ gitpython==3.2.0
checkpoints/step_004000/metadata/source/resolution_adapt/telemetry.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Local-first metrics with bounded W&B connection attempts and stable run identity."""
2
+ from __future__ import annotations
3
+ import json
4
+ import os
5
+ import time
6
+ import warnings
7
+ from pathlib import Path
8
+ from functools import wraps
9
+ from .checkpoints import rng_state, restore_rng
10
+
11
+
12
+ def preserve_training_rng(fn):
13
+ @wraps(fn)
14
+ def call(*args, **kwargs):
15
+ state = rng_state()
16
+ try: return fn(*args, **kwargs)
17
+ finally: restore_rng(state)
18
+ return call
19
+
20
+
21
+ class Metrics:
22
+ def __init__(self, root: Path, identity: dict, config: dict, *, online=True):
23
+ self.root,self.identity,self.config = root,identity,config
24
+ self.online,self.run,self.next_retry = online,None,0.0
25
+ self.connect()
26
+
27
+ @preserve_training_rng
28
+ def connect(self):
29
+ if not self.online or self.run is not None or time.monotonic() < self.next_retry: return
30
+ self.next_retry = time.monotonic()+60
31
+ try:
32
+ import wandb
33
+ self.run = wandb.init(project=self.identity['project'],entity=os.getenv('WANDB_ENTITY'),
34
+ name=self.identity['name'],group=self.identity['group'],id=self.identity['id'],
35
+ resume='allow',mode='online',dir=str(self.root),config=self.config,
36
+ settings=wandb.Settings(init_timeout=20))
37
+ if self.run is not None:
38
+ (self.root/'wandb_url.txt').write_text(self.run.url+'\n')
39
+ except Exception as exc:
40
+ warnings.warn(f'W&B unavailable ({type(exc).__name__}); metrics retained locally; will retry')
41
+
42
+ @preserve_training_rng
43
+ def log(self, step: int, values: dict):
44
+ row = {'optimizer_step':step,**values}
45
+ with (self.root/'metrics.jsonl').open('a') as stream:
46
+ stream.write(json.dumps(row)+'\n'); stream.flush()
47
+ self.connect()
48
+ if self.run is not None:
49
+ try:
50
+ self.run.log(row,step=step)
51
+ except Exception as exc:
52
+ warnings.warn(f'W&B log failed ({type(exc).__name__}); local metrics preserved')
53
+ self.run = None
54
+
55
+ def finish(self):
56
+ if self.run is not None:
57
+ try: self.run.finish()
58
+ except Exception as exc: warnings.warn(f'W&B finish failed ({type(exc).__name__})')
checkpoints/step_004000/metadata/source/resolution_adapt/test_pipeline.py ADDED
@@ -0,0 +1,284 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import copy
4
+ import importlib.util
5
+ import json
6
+ import random
7
+ import sys
8
+ from pathlib import Path
9
+ from types import SimpleNamespace
10
+
11
+ import numpy as np
12
+ import pytest
13
+ import torch
14
+
15
+ HERE = Path(__file__).resolve().parents[1]
16
+ sys.path.insert(0,str(HERE))
17
+ from resolution_adapt.data import (PoolSampler, ResolutionBottleneck, competition_cameras,
18
+ resize_rgb, split_episodes, verify_split)
19
+ from resolution_adapt.checkpoints import (checkpoint_due, latest_checkpoint, prune_uploaded, resume_checkpoint,
20
+ save_checkpoint, validate_checkpoint)
21
+ from training_support import DatasetGripperToImageWAM
22
+ from train_resolution_adapt import make_config, parser, scheduler_for
23
+
24
+
25
+ def episodes():
26
+ rows=[]
27
+ for pool,(repo,suite) in enumerate([('nvidia/LIBERO_LeRobot_v3','libero_goal'),
28
+ ('lerobot/libero_plus',None),('nvidia/LIBERO_LeRobot_v3','libero_90')]):
29
+ for task in range(4):
30
+ for ep in range(12):
31
+ idx=len(rows)
32
+ rows.append(dict(episode_index=idx,source_repo=repo,source_suite=suite,
33
+ source_episode_index=task*12+ep,task_index=task,task=f'task {task}',
34
+ length=32,dataset_from_index=idx*32))
35
+ return rows
36
+
37
+
38
+ def test_split_distribution_resume():
39
+ manifest=split_episodes(episodes())
40
+ assert manifest == split_episodes(episodes())
41
+ verify_split(manifest)
42
+ train={e['episode_index'] for e in manifest['episodes'] if e['split']=='train'}
43
+ sampler=PoolSampler(manifest)
44
+ for _ in range(100000):
45
+ index,ep=sampler.draw()
46
+ assert ep['episode_index'] in train
47
+ assert ep['anchor']+16 < ep['length']
48
+ assert index == ep['dataset_from_index']+ep['anchor']
49
+ sampler.distribution_report(check=True)
50
+ state=sampler.state_dict()
51
+ expected=[sampler.draw() for _ in range(20)]
52
+ restored=PoolSampler(manifest); restored.load_state_dict(state)
53
+ assert [restored.draw() for _ in range(20)] == expected
54
+ assert restored.counts == sampler.counts
55
+
56
+
57
+ def test_duplicate_source_cannot_leak():
58
+ rows=episodes(); extra={**rows[0],'episode_index':9999,'dataset_from_index':999999}
59
+ rows.append(extra)
60
+ m=split_episodes(rows)
61
+ first=next(e for e in m['episodes'] if e['episode_index']==0)
62
+ duplicate=next(e for e in m['episodes'] if e['episode_index']==9999)
63
+ assert first['split']==duplicate['split']
64
+ duplicate['split']='validation' if first['split']=='train' else 'train'
65
+ with pytest.raises(ValueError,match='leakage'): verify_split(m)
66
+
67
+
68
+ def test_current_future_resolution_and_orientation():
69
+ # RGB upper-left red marker and asymmetric blue corner detect rotation/channel swaps.
70
+ x=torch.zeros(2,3,256,256); x[:,0,:80,:80]=1; x[:,2,180:,180:]=.6
71
+ y=ResolutionBottleneck()(x)
72
+ assert y.shape==(2,3,224,224)
73
+ torch.testing.assert_close(y,resize_rgb(resize_rgb(x,128),224),rtol=0,atol=0)
74
+ assert y[0,0,10,10] == 1 and y[0,2,10,10] == 0
75
+ assert 0 <= y.min() <= y.max() <= 1
76
+ combined=torch.cat((y,y*.5),-1)*2-1
77
+ assert combined.shape==(2,3,224,448) and -1 <= combined.min() <= combined.max() <= 1
78
+ raw=(resize_rgb(x[0],128).permute(1,2,0)*255).to(torch.uint8)
79
+ sim=competition_cameras(raw,raw,rotate_180=True)
80
+ unrotated=competition_cameras(raw,raw,rotate_180=False)
81
+ assert sim.shape==(3,224,448)
82
+ torch.testing.assert_close(sim[:,:,:224],unrotated[:,:,:224].flip((-2,-1)),atol=2e-5,rtol=1e-5)
83
+ with pytest.raises(ValueError): ResolutionBottleneck()(x[:,:,:128,:128])
84
+
85
+
86
+ def test_gripper_once_and_config():
87
+ action=torch.randn(16,7); action[:8,-1]=-1; action[8:,-1]=1
88
+ original=action.clone()
89
+ transform=DatasetGripperToImageWAM()
90
+ batch=transform.forward({'action':{'default':action}})
91
+ assert torch.equal(batch['action']['default'][:,:6],original[:,:6])
92
+ assert (batch['action']['default'][:8,-1]==1).all()
93
+ assert (batch['action']['default'][8:,-1]==0).all()
94
+ with pytest.raises(ValueError): transform.forward(batch)
95
+ cfg=make_config(parser().parse_args([])); data=cfg['data']['train']
96
+ assert data['processor']['shape_meta']==data['shape_meta']
97
+ assert data['video_size']==[224,448]
98
+ assert len(data['processor']['action_state_transforms'])==1
99
+ assert data['endpoint_frames_only'] and data['num_frames']==17
100
+ # This upstream flag enables the unrelated T5 cache, not the Qwen cache.
101
+ assert data['require_text_cache'] is False
102
+ assert data['qwen_text_cache_dir'] and data['qwen_text_cache_format']=='qwen3_flux2'
103
+ assert cfg['stage1_checkpoint'] is None
104
+ assert cfg['model']['goal_prior']['num_latents']==100
105
+ assert cfg['model']['goal_prior']['num_layer_groups']==5
106
+ assert cfg['model']['loss']=={'lambda_video':.5,'lambda_action':1.,'lambda_pose':.3}
107
+
108
+
109
+ @pytest.mark.parametrize('accumulation',[1,3,8])
110
+ def test_cadence(accumulation):
111
+ triggers=[]; step=0
112
+ for micro in range(10000*accumulation):
113
+ if (micro+1)%accumulation: continue
114
+ step+=1
115
+ if checkpoint_due(step,2000,10000): triggers.append(step)
116
+ assert triggers==[2000,4000,6000,8000,10000]
117
+ assert checkpoint_due(2011,2000,2011)
118
+
119
+
120
+ def test_exact_resume_and_corruption(tmp_path):
121
+ args=parser().parse_args(['--max-steps','20','--warmup-steps','3'])
122
+ torch.manual_seed(5); random.seed(5); np.random.seed(5)
123
+ model=torch.nn.Linear(3,2); optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3)
124
+ scheduler=scheduler_for(optimizer,args); sampler=PoolSampler(split_episodes(episodes()))
125
+ metadata=tmp_path/'metadata'; metadata.mkdir()
126
+ (metadata/'split.json').write_text(json.dumps(split_episodes(episodes())))
127
+ (metadata/'wandb.json').write_text(json.dumps({'id':'stable-run'}))
128
+ def step():
129
+ sampler.draw(); optimizer.zero_grad()
130
+ x=torch.randn(2,3)*(random.random()+np.random.rand())
131
+ model(x).square().mean().backward(); optimizer.step(); scheduler.step()
132
+ for _ in range(3): step()
133
+ root=tmp_path/'checkpoints'
134
+ folder=save_checkpoint(root,3,model,optimizer,scheduler,sampler,metadata)
135
+ saved_moments=copy.deepcopy(optimizer.state_dict())
136
+ for _ in range(4): step()
137
+ expected=copy.deepcopy(model.state_dict()); expected_sampler=sampler.state_dict(); expected_lr=scheduler.get_last_lr()
138
+ assert resume_checkpoint(folder,model,optimizer,scheduler,sampler)==3
139
+ for key,state in optimizer.state_dict()['state'].items():
140
+ for name,value in state.items(): torch.testing.assert_close(value,saved_moments['state'][key][name],rtol=0,atol=0)
141
+ for _ in range(4): step()
142
+ for key,value in model.state_dict().items(): torch.testing.assert_close(value,expected[key],rtol=0,atol=0)
143
+ assert sampler.state_dict()==expected_sampler and scheduler.get_last_lr()==expected_lr
144
+ assert json.loads((folder/'metadata/wandb.json').read_text())['id']=='stable-run'
145
+ assert (folder/'metadata/split.json').read_bytes()==(metadata/'split.json').read_bytes()
146
+ (root/'.pending_broken').mkdir()
147
+ assert latest_checkpoint(root)==folder
148
+ with pytest.raises(FileExistsError): save_checkpoint(root,3,model,optimizer,scheduler,sampler,metadata)
149
+ (folder/'weights/adaptation.pt').write_bytes(b'corrupt')
150
+ with pytest.raises(ValueError): validate_checkpoint(folder)
151
+ with pytest.raises(ValueError): latest_checkpoint(root)
152
+
153
+
154
+ def test_retention_requires_receipt_and_protects_final(tmp_path):
155
+ root=tmp_path/'checkpoints'; root.mkdir(); status=tmp_path/'upload_state'; status.mkdir()
156
+ for step in range(1,6):
157
+ folder=root/f'step_{step:06d}'; (folder/'resume').mkdir(parents=True)
158
+ (folder/'resume/state.pt').touch(); (folder/'manifest.json').write_text(json.dumps({'final':step==2}))
159
+ (status/'step_000001.json').write_text(json.dumps({'uploaded':False}))
160
+ (status/'step_000002.json').write_text(json.dumps({'uploaded':True}))
161
+ (status/'step_000003.json').write_text(json.dumps({'uploaded':True,'uploading':True}))
162
+ prune_uploaded(root)
163
+ assert all((p/'resume').exists() for p in root.iterdir())
164
+ (status/'step_000003.json').write_text(json.dumps({'uploaded':True,'uploading':False}))
165
+ prune_uploaded(root)
166
+ assert not (root/'step_000003/resume').exists()
167
+ assert (root/'step_000001/resume').exists() and (root/'step_000002/resume').exists()
168
+
169
+
170
+ @pytest.mark.parametrize('unfreeze',[0,1])
171
+ def test_actual_upstream_lora_groups_and_update(unfreeze):
172
+ pytest.importorskip('imagewam.models.backbones.lora')
173
+ from resolution_adapt.model import configure_peft
174
+ class Block(torch.nn.Module):
175
+ def __init__(self):
176
+ super().__init__(); self.qkv=torch.nn.Linear(4,4); self.proj=torch.nn.Linear(4,4)
177
+ def forward(self,x): return self.proj(self.qkv(x))
178
+ class Toy(torch.nn.Module):
179
+ def __init__(self):
180
+ super().__init__()
181
+ self.video_expert=torch.nn.Module(); transformer=torch.nn.Module()
182
+ transformer.double_blocks=torch.nn.ModuleList([Block()])
183
+ transformer.single_blocks=torch.nn.ModuleList([Block()])
184
+ self.video_expert.transformer=transformer
185
+ for name in ('action_expert','semantic_visual_aggregator','semantic_visual_pose_norm',
186
+ 'semantic_visual_pose_decoder','proprio_encoder','vae'):
187
+ setattr(self,name,torch.nn.Linear(4,4))
188
+ self.text_encoder=SimpleNamespace(model=torch.nn.Linear(4,4))
189
+ torch.manual_seed(4); model=Toy()
190
+ args=parser().parse_args(['--unfreeze-last-flux-blocks',str(unfreeze)])
191
+ groups,report=configure_peft(model,args)
192
+ assert report['totals']['lora']>0
193
+ assert report['components']['qwen']['trainable']==0 and report['components']['vae']['trainable']==0
194
+ for g in groups:
195
+ assert g['lr']==(3e-5 if g['name']=='flux_lora' else 1e-6 if g['name']=='flux_dense' else 1e-5)
196
+ assert bool(report['components']['flux_dense']['trainable']) == bool(unfreeze)
197
+ before={n:p.detach().clone() for n,p in model.named_parameters()}
198
+ opt=torch.optim.AdamW(groups)
199
+ x=torch.randn(2,4)
200
+ for block in [*model.video_expert.transformer.double_blocks,*model.video_expert.transformer.single_blocks]: x=block(x)
201
+ for name in ('action_expert','semantic_visual_aggregator','semantic_visual_pose_norm',
202
+ 'semantic_visual_pose_decoder','proprio_encoder'): x=getattr(model,name)(x)
203
+ x.square().mean().backward(); torch.nn.utils.clip_grad_norm_(model.parameters(),1,error_if_nonfinite=True);opt.step()
204
+ for name,p in model.named_parameters():
205
+ if not p.requires_grad: assert torch.equal(p,before[name])
206
+ assert any(not torch.equal(p,before[n]) for n,p in model.named_parameters() if n.endswith('lora_B'))
207
+ for prefix in ('action_expert','semantic_visual_aggregator'):
208
+ assert any(not torch.equal(p,before[n]) for n,p in model.named_parameters() if n.startswith(prefix))
209
+
210
+
211
+ def test_logging_failure_preserves_local_metrics(tmp_path,monkeypatch):
212
+ from resolution_adapt.telemetry import Metrics
213
+ class Failing:
214
+ @staticmethod
215
+ def init(**kwargs): raise ConnectionError('temporary')
216
+ monkeypatch.setitem(sys.modules,'wandb',Failing)
217
+ with pytest.warns(UserWarning,match='retained locally'):
218
+ logger=Metrics(tmp_path,{'id':'stable','project':'test','name':'test','group':'test'},{})
219
+ logger.log(7,{'train/loss':.3}); logger.finish()
220
+ assert json.loads((tmp_path/'metrics.jsonl').read_text())['optimizer_step']==7
221
+
222
+
223
+ def test_upload_retry_verified_receipt_and_idempotence(tmp_path,monkeypatch):
224
+ import huggingface_hub
225
+ from resolution_adapt.checkpoints import upload_once
226
+ model=torch.nn.Linear(2,1);opt=torch.optim.AdamW(model.parameters());sched=torch.optim.lr_scheduler.LambdaLR(opt,lambda s:1)
227
+ sampler=PoolSampler(split_episodes(episodes()))
228
+ metadata=tmp_path/'metadata';metadata.mkdir();(metadata/'identity.json').write_text('{}')
229
+ folder=save_checkpoint(tmp_path/'checkpoints',2000,model,opt,sched,sampler,metadata)
230
+ class Hub:
231
+ def __init__(self): self.files={};self.fail=True;self.uploads=0
232
+ def model_info(self,repo): return SimpleNamespace(private=True)
233
+ def list_repo_files(self,repo): return list(self.files)
234
+ def upload_large_folder(self,**kwargs):
235
+ self.uploads+=1
236
+ for f in folder.rglob('*'):
237
+ if f.is_file() and f.name!='COMPLETE':self.files[str(f.relative_to(tmp_path))]=f.read_bytes()
238
+ if self.fail: self.fail=False;raise ConnectionError('interrupted')
239
+ def list_repo_tree(self,repo,**kwargs):return [SimpleNamespace(path=p,size=len(v)) for p,v in self.files.items()]
240
+ def upload_file(self,**kwargs):self.files[kwargs['path_in_repo']]=Path(kwargs['path_or_fileobj']).read_bytes()
241
+ hub=Hub()
242
+ def download(repo,name,**kwargs):
243
+ dest=tmp_path/'fake_download';dest.write_bytes(hub.files[name]);return str(dest)
244
+ monkeypatch.setattr(huggingface_hub,'hf_hub_download',download)
245
+ upload_once(tmp_path,'test/repo',api=hub)
246
+ receipt=tmp_path/'upload_state/step_002000.json'
247
+ assert not json.loads(receipt.read_text())['uploaded']
248
+ assert (folder/'resume/state.pt').exists()
249
+ upload_once(tmp_path,'test/repo',api=hub)
250
+ assert json.loads(receipt.read_text())['uploaded']
251
+ assert hub.uploads==2
252
+ upload_once(tmp_path,'test/repo',api=hub)
253
+ assert hub.uploads==2
254
+ hub.files['checkpoints/step_002000/COMPLETE']=b'conflicting manifest hash'
255
+ upload_once(tmp_path,'test/repo',api=hub)
256
+ assert not json.loads(receipt.read_text())['uploaded']
257
+ assert hub.uploads==2 # collision never overwrites remote state
258
+
259
+
260
+ def test_remote_restore_verifies_before_exposing(tmp_path,monkeypatch):
261
+ import huggingface_hub
262
+ from resolution_adapt.checkpoints import restore_remote
263
+ origin=tmp_path/'origin';origin.mkdir()
264
+ metadata=origin/'metadata';metadata.mkdir();(metadata/'wandb.json').write_text('{"id":"stable"}')
265
+ model=torch.nn.Linear(2,1);opt=torch.optim.AdamW(model.parameters())
266
+ sched=torch.optim.lr_scheduler.LambdaLR(opt,lambda s:1);sampler=PoolSampler(split_episodes(episodes()))
267
+ folder=save_checkpoint(origin/'checkpoints',2000,model,opt,sched,sampler,metadata)
268
+ files={str(p.relative_to(origin)):p.read_bytes() for p in folder.rglob('*') if p.is_file()}
269
+ class Hub:
270
+ def model_info(self,repo): return SimpleNamespace(sha='pinned-remote-commit')
271
+ def list_repo_files(self,repo,revision):
272
+ assert revision=='pinned-remote-commit';return list(files)
273
+ monkeypatch.setattr(huggingface_hub,'HfApi',Hub)
274
+ def download(repo,name,revision):
275
+ assert revision=='pinned-remote-commit'
276
+ cache=tmp_path/'download';cache.write_bytes(files[name]);return str(cache)
277
+ monkeypatch.setattr(huggingface_hub,'hf_hub_download',download)
278
+ restored=restore_remote(tmp_path/'restored','test/repo')
279
+ validate_checkpoint(restored)
280
+ assert json.loads((tmp_path/'restored/metadata/wandb.json').read_text())['id']=='stable'
281
+ files['checkpoints/step_002000/resume/state.pt']=b'bad transfer'
282
+ with pytest.raises(ValueError): restore_remote(tmp_path/'corrupt','test/repo')
283
+ assert not (tmp_path/'corrupt/checkpoints/step_002000').exists()
284
+ assert not (tmp_path/'corrupt/metadata').exists()
checkpoints/step_004000/metadata/source/resolution_adapt/test_video_loading.py ADDED
@@ -0,0 +1,211 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import sys
5
+ from fractions import Fraction
6
+ from pathlib import Path
7
+ from types import SimpleNamespace
8
+
9
+ import numpy as np
10
+ import pytest
11
+ import torch
12
+
13
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
14
+ from resolution_adapt.data import POOLS, PoolSampler, split_episodes
15
+ from training_support import VideoDecodeError, decode_video_frames
16
+ from train_resolution_adapt import load_sample
17
+
18
+
19
+ @pytest.fixture
20
+ def sampler():
21
+ episodes = []
22
+ for repo, suite in [('nvidia/LIBERO_LeRobot_v3', 'libero_goal'),
23
+ ('lerobot/libero_plus', None),
24
+ ('nvidia/LIBERO_LeRobot_v3', 'libero_90')]:
25
+ for task in range(2):
26
+ for source_episode in range(6):
27
+ idx = len(episodes)
28
+ episodes.append(dict(episode_index=idx, source_repo=repo, source_suite=suite,
29
+ source_episode_index=task*6+source_episode, task_index=task,
30
+ length=32, dataset_from_index=idx*32))
31
+ return PoolSampler(split_episodes(episodes))
32
+
33
+
34
+ def broken_video():
35
+ # The released dataset wraps the decoder exception in a RuntimeError.
36
+ try:
37
+ raise VideoDecodeError('wrist.mp4', [1., 1.8], 'Invalid packet')
38
+ except VideoDecodeError as exc:
39
+ raise RuntimeError('Failed to load a valid sample after 1 attempts') from exc
40
+
41
+
42
+ def test_replacement_preserves_distribution_split_horizon_and_resume(sampler, tmp_path):
43
+ index, episode = sampler.draw()
44
+ counts = sampler.counts.copy(), sampler.task_counts.copy(), sampler.draws
45
+ seen = []
46
+
47
+ class Dataset:
48
+ def __getitem__(self, idx):
49
+ seen.append(idx)
50
+ if len(seen) <= 2:
51
+ broken_video()
52
+ return idx
53
+
54
+ log = tmp_path/'failures.jsonl'
55
+ with pytest.warns(UserWarning, match='Rejected video sample'):
56
+ accepted = load_sample(Dataset(), sampler, index, episode, failure_log=log)
57
+ assert accepted == seen[-1]
58
+ assert (sampler.counts, sampler.task_counts, sampler.draws) == counts
59
+ eligible = sampler.groups[episode['pool']][episode['task_index']]
60
+ for idx in seen:
61
+ ep = next(ep for ep in eligible if ep['dataset_from_index'] <= idx < ep['dataset_from_index']+ep['length'])
62
+ assert ep['split'] == 'train'
63
+ assert idx-ep['dataset_from_index']+16 < ep['length']
64
+ records = [json.loads(line) for line in log.read_text().splitlines()]
65
+ assert [row['index'] for row in records] == seen[:2]
66
+ assert all(row['video_path'] == 'wrist.mp4' and row['split'] == 'train' for row in records)
67
+ assert sampler.video_decode_failures[episode['pool']] == 2
68
+ state = sampler.state_dict()
69
+ expected = [sampler.draw() for _ in range(20)]
70
+ sampler.load_state_dict(state)
71
+ assert [sampler.draw() for _ in range(20)] == expected
72
+ assert sampler.video_decode_failures[episode['pool']] == 2
73
+
74
+
75
+ def test_old_checkpoint_loads_without_failure_counters(sampler):
76
+ state = sampler.state_dict()
77
+ del state['video_decode_failures']
78
+ sampler.load_state_dict(state)
79
+ assert sum(sampler.video_decode_failures.values()) == 0
80
+
81
+
82
+ def test_persistent_video_failure_is_bounded(sampler):
83
+ seen = []
84
+
85
+ class Dataset:
86
+ def __getitem__(self, index):
87
+ seen.append(index)
88
+ broken_video()
89
+
90
+ with pytest.warns(UserWarning), pytest.raises(RuntimeError, match='after 3 attempts'):
91
+ load_sample(Dataset(), sampler, *sampler.draw(), max_attempts=3)
92
+ assert len(seen) == 3
93
+ assert sum(sampler.video_decode_failures.values()) == 3
94
+
95
+
96
+ @pytest.mark.parametrize('error', [ValueError('Video timestamp mismatch'),
97
+ FileNotFoundError('Missing shard'),
98
+ RuntimeError('Missing text context')])
99
+ def test_other_failures_are_not_resampled(sampler, error):
100
+ seen = []
101
+
102
+ class Dataset:
103
+ def __getitem__(self, index):
104
+ seen.append(index)
105
+ raise error
106
+
107
+ with pytest.raises(type(error)) as raised:
108
+ load_sample(Dataset(), sampler, *sampler.draw())
109
+ assert raised.value is error
110
+ assert len(seen) == 1 and not sampler.video_decode_failures
111
+
112
+
113
+ def test_validation_replacement_stays_held_out(sampler):
114
+ manifest_episodes = []
115
+ # Reuse the same full manifest via a fresh fixture-style synthetic split.
116
+ # Every pool/task has multiple held-out episodes for this retry check.
117
+ for pool in POOLS:
118
+ for task in sampler.tasks[pool]:
119
+ for ep in sampler.groups[pool][task][:2]:
120
+ manifest_episodes.append({**ep, 'split':'validation'})
121
+ validation = PoolSampler({'episodes':manifest_episodes}, split='validation')
122
+ index, episode = validation.draw()
123
+
124
+ class Dataset:
125
+ attempts = 0
126
+
127
+ def __getitem__(self, idx):
128
+ self.attempts += 1
129
+ if self.attempts == 1:
130
+ broken_video()
131
+ return idx
132
+
133
+ with pytest.warns(UserWarning):
134
+ idx = load_sample(Dataset(), validation, index, episode)
135
+ assert any(ep['dataset_from_index'] <= idx < ep['dataset_from_index']+ep['length']-16
136
+ for ep in validation.groups[episode['pool']][episode['task_index']])
137
+
138
+
139
+ def fake_container(av, *, frame_offset=0):
140
+ class Container:
141
+ def __init__(self):
142
+ self.stream = SimpleNamespace(time_base=Fraction(1, 20),
143
+ codec_context=SimpleNamespace(thread_count=0))
144
+ self.streams = SimpleNamespace(video=[self.stream])
145
+ self.seeks = []
146
+
147
+ def __enter__(self):
148
+ return self
149
+
150
+ def __exit__(self, *args):
151
+ pass
152
+
153
+ def seek(self, offset, **kwargs):
154
+ self.seeks.append(offset)
155
+ self.position = offset//2*2
156
+
157
+ def decode(self, stream):
158
+ for pts in range(self.position, 20):
159
+ if pts == 15:
160
+ raise av.error.InvalidDataError(1094995529, 'Invalid packet')
161
+ yield SimpleNamespace(pts=pts+frame_offset, is_corrupt=False,
162
+ to_ndarray=lambda format, value=pts: np.full((2, 2, 3), value, dtype=np.uint8))
163
+ return Container()
164
+
165
+
166
+ def test_unused_corrupt_packet_is_not_decoded(monkeypatch):
167
+ av = pytest.importorskip('av')
168
+ container = fake_container(av)
169
+ monkeypatch.setattr(av, 'open', lambda path: container)
170
+ frames = decode_video_frames('wrist.mp4', [0., .8, 0.], 1e-4)
171
+ assert container.seeks == [0, 16]
172
+ torch.testing.assert_close(frames[:, 0, 0, 0], torch.tensor([0., 16/255, 0.]), rtol=0, atol=0)
173
+
174
+
175
+ def test_needed_corrupt_packet_has_video_context(monkeypatch):
176
+ av = pytest.importorskip('av')
177
+ monkeypatch.setattr(av, 'open', lambda path: fake_container(av))
178
+ with pytest.raises(VideoDecodeError) as raised:
179
+ decode_video_frames('wrist.mp4', [.75], 1e-4)
180
+ assert raised.value.video_path == 'wrist.mp4'
181
+ assert raised.value.timestamps == [.75]
182
+ assert isinstance(raised.value.__cause__, av.error.InvalidDataError)
183
+
184
+
185
+ def test_timestamp_mismatch_is_not_substituted(monkeypatch):
186
+ av = pytest.importorskip('av')
187
+ monkeypatch.setattr(av, 'open', lambda path: fake_container(av, frame_offset=1))
188
+ with pytest.raises(ValueError, match='timestamp mismatch'):
189
+ decode_video_frames('wrist.mp4', [.8], 1e-4)
190
+
191
+
192
+ def test_real_video_matches_sequential_rgb_for_unsorted_duplicate_requests(tmp_path):
193
+ av = pytest.importorskip('av')
194
+ path = tmp_path/'frames.mp4'
195
+ with av.open(str(path), mode='w') as container:
196
+ stream = container.add_stream('mpeg4', rate=20)
197
+ stream.width = stream.height = 16
198
+ stream.pix_fmt = 'yuv420p'
199
+ stream.codec_context.gop_size = 2
200
+ stream.codec_context.max_b_frames = 0
201
+ for idx in range(20):
202
+ array = np.full((16, 16, 3), idx*10, dtype=np.uint8)
203
+ for packet in stream.encode(av.VideoFrame.from_ndarray(array, format='rgb24')):
204
+ container.mux(packet)
205
+ for packet in stream.encode():
206
+ container.mux(packet)
207
+ with av.open(str(path)) as container:
208
+ all_frames = [frame.to_ndarray(format='rgb24') for frame in container.decode(video=0)]
209
+ expected = torch.from_numpy(np.stack([all_frames[i] for i in [16, 0, 16, 6]])).permute(0, 3, 1, 2).float()/255
210
+ actual = decode_video_frames(path, [.8, 0., .8, .3], 1e-4)
211
+ torch.testing.assert_close(actual, expected, rtol=0, atol=0)
checkpoints/step_004000/metadata/source/train_resolution_adapt.py ADDED
@@ -0,0 +1,468 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Explicit launcher for released LIT Stage-2 resolution adaptation. No default training."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import copy
6
+ import json
7
+ import math
8
+ import os
9
+ import platform
10
+ import random
11
+ import shutil
12
+ import subprocess
13
+ import sys
14
+ import time
15
+ import uuid
16
+ import warnings
17
+ from collections import Counter
18
+ from pathlib import Path
19
+
20
+ import numpy as np
21
+ import torch
22
+ import yaml
23
+
24
+ HERE = Path(__file__).resolve().parent
25
+ ROOT = HERE.parents[1]
26
+ sys.path.insert(0,str(HERE))
27
+ from resolution_adapt.data import DATASET, DATASET_REVISION, POOLS, PoolSampler, digest, split_episodes
28
+ from resolution_adapt.model import IMAGEWAM_COMMIT, FLUX_COMMIT, LIT_REVISION, build_stage2, configure_peft
29
+ from resolution_adapt.checkpoints import (atomic_json, checkpoint_due, latest_checkpoint, resume_checkpoint,
30
+ restore_rng, rng_state, save_checkpoint, sha256, upload_once)
31
+ from resolution_adapt.telemetry import Metrics
32
+ from training_support import VideoDecodeError
33
+
34
+
35
+ def parser():
36
+ p = argparse.ArgumentParser(description=__doc__)
37
+ p.add_argument('--mode',choices=['sampler','services','preflight','train','upload','restore','validate'],default='preflight')
38
+ p.add_argument('--output',type=Path,default=ROOT/'.runtime/lit-resolution-adapt')
39
+ p.add_argument('--dataset-root',type=Path,default=ROOT/'.runtime/lit-training/dataset')
40
+ p.add_argument('--released-dir',type=Path,default=ROOT/'.runtime/lit-training/released/imagewam/lit_stage2')
41
+ p.add_argument('--imagewam-root',type=Path,default=ROOT/'.runtime/lit-training/upstream')
42
+ p.add_argument('--flux-root',type=Path,default=ROOT/'.runtime/lit-training/flux2')
43
+ p.add_argument('--ae-path',type=Path,default=ROOT/'.runtime/lit-training/models/FLUX.2-dev/ae.safetensors')
44
+ p.add_argument('--qwen-cache',type=Path,default=ROOT/'.runtime/lit-training/qwen_cache')
45
+ p.add_argument('--resume',nargs='?',const='latest')
46
+ for name,default in [('max-steps',10000),('warmup-steps',500),('batch-size',1),('gradient-accumulation',8),
47
+ ('lora-rank',16),('unfreeze-last-flux-blocks',0),('checkpoint-every',2000),('seed',42),
48
+ ('eval-every',200),('validation-samples-per-pool',8),('log-every',10)]:
49
+ p.add_argument('--'+name,type=int,default=default)
50
+ for name,default in [('lora-alpha',16.),('lora-dropout',0.),('lr-flux-lora',3e-5),('lr-action',1e-5),
51
+ ('lr-interface',1e-5),('lr-pose',1e-5),('lr-proprio',1e-5),('lr-other',1e-5),
52
+ ('lr-unfrozen-flux',1e-6),('weight-decay',0.01),('familiar-libero-weight',.5),
53
+ ('plus-weight',.4),('libero90-weight',.1),('validation-episode-fraction',.05)]:
54
+ p.add_argument('--'+name,type=float,default=default)
55
+ p.add_argument('--hf-repo',default='Ratstar216/LIT-ImageWAM-ResolutionAdapt-128to224')
56
+ p.add_argument('--wandb-project',default=os.getenv('WANDB_PROJECT','parc2026-lit-imagewam'))
57
+ p.add_argument('--wandb-run-name',default=os.getenv('WANDB_RUN_NAME','lit-stage2-resadapt-128to224-lora-r16'))
58
+ p.add_argument('--wandb-group',default='lit-stage2-resadapt-128to224')
59
+ p.add_argument('--upload-once',action='store_true')
60
+ p.add_argument('--preflight-save',action='store_true',help='Save a local full recovery snapshot after the discarded preflight update; never upload it')
61
+ return p
62
+
63
+
64
+ def serialized_args(args):
65
+ return {k:str(v) if isinstance(v,Path) else v for k,v in vars(args).items()}
66
+
67
+
68
+ def make_config(args):
69
+ cfg = yaml.safe_load((HERE/'resolution_adapt/released_stage2.yaml').read_text())
70
+ cfg.update(max_steps=args.max_steps,warmup_steps=args.warmup_steps,batch_size=args.batch_size,
71
+ gradient_accumulation_steps=args.gradient_accumulation,save_every=args.checkpoint_every,
72
+ stage1_checkpoint=None,resume=args.resume,build_norm_stats=False,force_rebuild_norm_stats=False,
73
+ output_dir=str(args.output),num_workers=0,learning_rate=None)
74
+ cfg['model']['flux2_lora_config'].update(enabled=True,rank=args.lora_rank,alpha=args.lora_alpha,
75
+ dropout=args.lora_dropout,save_lora_merged=False,save_trainable_only=True)
76
+ cfg['model'].update(flux2_src_path=str(args.flux_root),flux2_model_path=None,
77
+ action_dit_pretrained_path=None,ae_model_path=str(args.ae_path),qwen3_model_spec=None)
78
+ cfg['initialization'] = {'repo':'linjianman/LIT','revision':LIT_REVISION,
79
+ 'path':'imagewam/lit_stage2/model.pt','local_path':str(args.released_dir/'model.pt')}
80
+ cfg['wandb'].update(enabled=True,mode='online',workspace=os.getenv('WANDB_ENTITY'),
81
+ project=args.wandb_project,name=args.wandb_run_name,group=args.wandb_group)
82
+ data = cfg['data']['train']
83
+ data.update(dataset_dirs=[str(args.dataset_root)],lerobot_backend='v3',lerobot_v3_video_backend='pyav',
84
+ lerobot_tolerance_s=1e-4,qwen_text_cache_dir=str(args.qwen_cache),require_text_cache=False,
85
+ pretrained_norm_stats=str(args.released_dir/'dataset_stats.json'),vision_free=False)
86
+ for image,key in zip(data['shape_meta']['images'],('front','wrist'),strict=True):
87
+ image.update(raw_shape=[3,256,256],shape=[3,224,224],lerobot_key=f'observation.images.{key}')
88
+ transforms = [{'_target_':'imagewam.datasets.lerobot.transforms.image.ToTensor'},
89
+ {'_target_':'resolution_adapt.data.ResolutionBottleneck'}]
90
+ data['processor'].update(shape_meta=copy.deepcopy(data['shape_meta']),train_transforms=transforms,
91
+ val_transforms=copy.deepcopy(transforms),
92
+ action_state_transforms=[{'_target_':'training_support.DatasetGripperToImageWAM'}])
93
+ data['video_augmentation']['random_resized_crop'] = None
94
+ data['video_augmentation']['rotate'] = None
95
+ return cfg
96
+
97
+
98
+ def scheduler_for(optimizer,args):
99
+ def scale(step):
100
+ if step < args.warmup_steps:
101
+ return (step+1)/max(1,args.warmup_steps)
102
+ progress = (step-args.warmup_steps)/max(1,args.max_steps-args.warmup_steps)
103
+ return .5*(1+math.cos(math.pi*min(1.,progress)))
104
+ return torch.optim.lr_scheduler.LambdaLR(optimizer,scale)
105
+
106
+
107
+ def services(args):
108
+ from huggingface_hub import HfApi, hf_hub_download
109
+ api = HfApi(); api.whoami()
110
+ api.create_repo(args.hf_repo,private=True,repo_type='model',exist_ok=True)
111
+ if not api.model_info(args.hf_repo).private:
112
+ api.update_repo_settings(args.hf_repo,private=True)
113
+ if not api.model_info(args.hf_repo).private: raise RuntimeError('HF repository is not private')
114
+ token = uuid.uuid4().hex
115
+ content = json.dumps({'experiment':'LIT ImageWAM Stage-2 Resolution Adaptation','probe':token}).encode()
116
+ path = f'preflight/{token}.json'
117
+ api.upload_file(repo_id=args.hf_repo,path_or_fileobj=content,path_in_repo=path)
118
+ if Path(hf_hub_download(args.hf_repo,path)).read_bytes() != content: raise RuntimeError('HF test mismatch')
119
+ identity = {'id':'preflight-'+token[:12],'project':args.wandb_project,'name':args.wandb_run_name+'-preflight',
120
+ 'group':args.wandb_group}
121
+ logs = args.output/'service_logs'; logs.mkdir(exist_ok=True)
122
+ metrics = Metrics(logs,identity,{'test':'connectivity only'})
123
+ metrics.log(0,{'preflight/connectivity':1})
124
+ report = {'hf':{'passed':True,'repo':args.hf_repo,'private':True,'test_path':path},
125
+ 'wandb':{'passed':metrics.run is not None,'id':identity['id'],
126
+ 'url':metrics.run.url if metrics.run is not None else None}}
127
+ metrics.finish(); atomic_json(args.output/'services.json',report)
128
+ print(json.dumps(report,indent=2))
129
+
130
+
131
+ def check_sources(args):
132
+ for root,commit in ((args.imagewam_root,IMAGEWAM_COMMIT),(args.flux_root,FLUX_COMMIT)):
133
+ actual = subprocess.check_output(['git','-C',str(root),'rev-parse','HEAD'],text=True).strip()
134
+ if actual != commit: raise ValueError(f'Wrong source revision: {root} {actual}')
135
+ if subprocess.check_output(['git','-C',str(root),'status','--porcelain','--untracked-files=no'],text=True):
136
+ raise ValueError(f'Pinned source has tracked modifications: {root}')
137
+ sys.path[:0] = [str(args.imagewam_root/'src'),str(args.flux_root/'src')]
138
+
139
+
140
+ def load_sample(dataset, sampler, index, episode, *, max_attempts=8, failure_log=None):
141
+ """Bounded video-data recovery; other dataset/transform failures still stop."""
142
+ if max_attempts <= 0:
143
+ raise ValueError('max_attempts must be positive')
144
+ for attempt in range(1, max_attempts + 1):
145
+ try:
146
+ return dataset[index]
147
+ except Exception as exc:
148
+ error = exc
149
+ while not isinstance(error, VideoDecodeError) and error.__cause__ is not None:
150
+ error = error.__cause__
151
+ if not isinstance(error, VideoDecodeError):
152
+ raise
153
+ sampler.video_decode_failures[episode['pool']] += 1
154
+ record = {'index':index, 'episode_index':episode['episode_index'],
155
+ 'anchor':episode['anchor'], 'pool':episode['pool'],
156
+ 'task_index':episode['task_index'], 'split':episode['split'],
157
+ 'sampler_draws':sampler.draws, 'attempt':attempt,
158
+ 'video_path':error.video_path, 'timestamps':error.timestamps,
159
+ 'error':str(error)}
160
+ if failure_log is not None:
161
+ with Path(failure_log).open('a') as stream:
162
+ stream.write(json.dumps(record) + '\n')
163
+ warnings.warn(f"Rejected video sample {index} ({episode['pool']}/"
164
+ f"task {episode['task_index']}, attempt {attempt}/{max_attempts}): {error}")
165
+ if attempt == max_attempts:
166
+ raise RuntimeError(f'Video loading failed after {max_attempts} attempts '
167
+ f"for {episode['pool']}/task {episode['task_index']}") from exc
168
+ index, episode = sampler.replacement(episode)
169
+
170
+
171
+ def batch_for(dataset,sampler,size,*,failure_log=None):
172
+ samples = [load_sample(dataset,sampler,*sampler.draw(),failure_log=failure_log) for _ in range(size)]
173
+ from torch.utils.data import default_collate
174
+ batch = default_collate(samples)
175
+ if batch['video'].shape != (size,3,2,224,448) or batch['action'].shape != (size,16,7):
176
+ raise ValueError('Incorrect Stage-2 batch geometry')
177
+ if batch['goal_pose'].shape != (size,8): raise ValueError('Incorrect pose target')
178
+ if batch['text_hidden_states'].shape != (size,512,7680):
179
+ raise ValueError('Missing/incompatible released Qwen contexts')
180
+ for key in ('action_is_pad','goal_pose_is_pad','proprio_is_pad'):
181
+ if key in batch and batch[key].any(): raise ValueError(f'Padded training sample: {key}')
182
+ return batch
183
+
184
+
185
+ def loss_metrics(loss,metrics):
186
+ result = {'total_loss':float(loss.detach())}
187
+ for component,weight in (('action',1.),('video',.5),('pose',.3)):
188
+ value = float(metrics['loss_'+component])
189
+ if not math.isfinite(value): raise FloatingPointError(f'Nonfinite {component} loss')
190
+ result[component+'_loss_weighted'] = value
191
+ result[component+'_loss'] = value/weight
192
+ if not math.isfinite(result['total_loss']): raise FloatingPointError('Nonfinite total loss')
193
+ return result
194
+
195
+
196
+ @torch.no_grad()
197
+ def validate(model,dataset,manifest,args):
198
+ state = rng_state(); augment = dataset.video_augmentation
199
+ dataset.video_augmentation = None
200
+ model.eval()
201
+ report = {}
202
+ try:
203
+ random.seed(args.seed+1); np.random.seed(args.seed+1); torch.manual_seed(args.seed+1)
204
+ sampler = PoolSampler(manifest,split='validation',seed=args.seed+1)
205
+ overall = Counter()
206
+ for pool in POOLS:
207
+ values = Counter()
208
+ for _ in range(args.validation_samples_per_pool):
209
+ task = sampler.rng.choice(sampler.tasks[pool]); ep = sampler.rng.choice(sampler.groups[pool][task])
210
+ anchor = sampler.rng.randrange(ep['length']-16)
211
+ index = ep['dataset_from_index']+anchor
212
+ from torch.utils.data import default_collate
213
+ sample = load_sample(dataset,sampler,index,{**ep,'anchor':anchor},
214
+ failure_log=args.output/'video_decode_failures.jsonl')
215
+ with torch.autocast('cuda',dtype=torch.bfloat16):
216
+ loss,metrics = model.training_loss(default_collate([sample]))
217
+ values.update(loss_metrics(loss,metrics))
218
+ report[f'validation/{pool}/video_decode_failures'] = sampler.video_decode_failures[pool]
219
+ for k,v in values.items():
220
+ report[f'validation/{pool}/{k}'] = v/args.validation_samples_per_pool
221
+ # Overall is explicitly weighted by the requested experiment mixture.
222
+ overall[k] += v/args.validation_samples_per_pool*sampler.weights[POOLS.index(pool)]
223
+ report.update({f'validation/overall/{k}':v for k,v in overall.items()})
224
+ return report
225
+ finally:
226
+ dataset.video_augmentation = augment; model.train(); model.vae.eval(); restore_rng(state)
227
+
228
+
229
+ def run(args):
230
+ if not torch.cuda.is_available(): raise RuntimeError('CUDA unavailable: real Stage-2 preflight/training cannot run')
231
+ if not torch.cuda.is_bf16_supported(): raise RuntimeError('BF16 GPU required')
232
+ check_sources(args)
233
+ from omegaconf import OmegaConf
234
+ from hydra.utils import instantiate
235
+ from imagewam.utils.config_resolvers import register_default_resolvers
236
+ from imagewam.utils import misc
237
+ from training_support import install_dataset_hooks
238
+ register_default_resolvers(); install_dataset_hooks(False)
239
+ misc.register_work_dir(str(args.output))
240
+ random.seed(args.seed); np.random.seed(args.seed); torch.manual_seed(args.seed)
241
+ cfg = make_config(args)
242
+ for name in ('tabular_audit.json','video_audit.json'):
243
+ audited = json.loads((ROOT/'research/lit_training'/name).read_text())
244
+ if audited['errors']: raise ValueError(f'Existing dataset audit failed: {name}')
245
+ metadata = args.output/'metadata'; metadata.mkdir(exist_ok=True)
246
+ path = metadata/'split.json'
247
+ source_path = args.dataset_root/'meta/source_episodes.jsonl'
248
+ if sha256(source_path) != 'f2e0e560a3decbed38cb6d5c998bf659b84d1e2d60b236f81e8ebb23415b9969':
249
+ raise ValueError('Dataset provenance does not match the pinned audited revision')
250
+ episodes = [json.loads(s) for s in source_path.read_text().splitlines()]
251
+ if args.resume:
252
+ manifest = json.loads(path.read_text())
253
+ if manifest['provenance_sha256'] != digest(episodes): raise ValueError('Dataset changed since split creation')
254
+ else:
255
+ manifest = split_episodes(episodes,args.validation_episode_fraction,args.seed); atomic_json(path,manifest)
256
+ sampler = PoolSampler(manifest,weights=(args.familiar_libero_weight,args.plus_weight,args.libero90_weight),seed=args.seed)
257
+ # Never recalculate normalization. Verify the config really is the pinned release.
258
+ if yaml.safe_load((args.released_dir/'config.yaml').read_text()) != yaml.safe_load((HERE/'resolution_adapt/released_stage2.yaml').read_text()):
259
+ raise ValueError('Released config differs from pinned reference')
260
+ asset_lock = json.loads((HERE/'resolution_adapt/assets.lock.json').read_text())
261
+ asset_hashes = {}
262
+ for repo,name,local in [('linjianman/LIT','imagewam/lit_stage2/model.pt',args.released_dir/'model.pt'),
263
+ ('linjianman/LIT','imagewam/lit_stage2/dataset_stats.json',args.released_dir/'dataset_stats.json'),
264
+ ('black-forest-labs/FLUX.2-dev','ae.safetensors',args.ae_path)]:
265
+ value = sha256(local); expected = asset_lock[repo]['files'][name]['sha256']
266
+ if value != expected: raise ValueError(f'Asset hash mismatch: {local}')
267
+ asset_hashes[str(local)] = value
268
+ cache_provenance = json.loads((args.qwen_cache/'provenance.json').read_text())
269
+ if cache_provenance['revision'] != asset_lock['Qwen/Qwen3-4B']['revision']:
270
+ raise ValueError('Qwen cache revision mismatch; rebuild with prepare_assets.py --cache-text')
271
+ dataset = instantiate(OmegaConf.create(cfg['data']['train']))
272
+ if len(dataset) != 3080750:
273
+ raise ValueError('Dataset length differs from the audited full snapshot; global sampler indices would be unsafe')
274
+ if args.mode == 'preflight':
275
+ from resolution_adapt.preflight_data import audit
276
+ audit(args.dataset_root,args.output/'data_preflight',stats_path=args.released_dir/'dataset_stats.json',processed=dataset)
277
+ model,key_report = build_stage2(cfg['model'],args.released_dir/'model.pt',args.ae_path)
278
+ groups,report = configure_peft(model,args)
279
+ print(json.dumps(report['totals'],indent=2),flush=True)
280
+ print(f'{"group":24s} {"tensors":>8s} {"parameters":>15s} {"LR":>12s} {"decay":>8s}')
281
+ for row in report['optimizer_groups']:
282
+ print(f'{row["name"]:24s} {row["tensors"]:8d} {row["parameters"]:15,d} {row["lr"]:12.2g} {row["weight_decay"]:8.3g}',flush=True)
283
+ optimizer = torch.optim.AdamW(groups,betas=(.9,.95),fused=True)
284
+ scheduler = scheduler_for(optimizer,args)
285
+ identity_path = metadata/'wandb.json'
286
+ if args.resume:
287
+ identity = json.loads(identity_path.read_text())
288
+ else:
289
+ identity = {'id':uuid.uuid4().hex[:12],'project':args.wandb_project,'name':args.wandb_run_name,'group':args.wandb_group}
290
+ atomic_json(identity_path,identity)
291
+ (metadata/'config.yaml').write_text(yaml.safe_dump(cfg,sort_keys=False))
292
+ atomic_json(metadata/'args.json',serialized_args(args))
293
+ atomic_json(metadata/'parameters.json',report); atomic_json(metadata/'checkpoint_load.json',key_report)
294
+ shutil.copy2(args.released_dir/'dataset_stats.json',metadata/'dataset_stats.json')
295
+ provenance = {'experiment':'LIT ImageWAM Stage-2 Resolution Adaptation','input_information':[128,128],
296
+ 'model_processing_per_camera':[224,224],'combined':[224,448],'source':[256,256],
297
+ 'method':'FLUX.2 LoRA + full ActionDiT/LIT-interface fine-tuning',
298
+ 'dataset':DATASET,'dataset_revision':DATASET_REVISION,'released_lit_revision':LIT_REVISION,
299
+ 'initialization':'linjianman/LIT/imagewam/lit_stage2/model.pt','normalization':'released Stage-2 dataset_stats.json',
300
+ 'imagewam_commit':IMAGEWAM_COMMIT,'flux_commit':FLUX_COMMIT,
301
+ 'competition_commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),
302
+ 'python':platform.python_version(),'torch':torch.__version__,'cuda':torch.version.cuda,
303
+ 'created_unix':time.time(),'sampling_weights':sampler.weights,'wandb':identity,
304
+ 'preprocessing':'RGB 256 -> bilinear antialiased 128 -> 224; photometric only; front then wrist; [-1,1]; both endpoints; no stored rotation',
305
+ 'goal':'state[t+16]; action[t:t+16]; t+16 < episode length',
306
+ 'video_loading':'independent timestamp seeks; up to 8 attempts on invalid video data; '
307
+ 'replacement stays within pool/task/split; failures logged and counters checkpointed',
308
+ 'gripper':'(1-g)/2 exactly once before released normalization',
309
+ 'qwen':'frozen offline cached embeddings; encoder not instantiated',
310
+ 'validation':'held-out episodes, equal sample count per pool; overall weighted by experiment mixture'}
311
+ atomic_json(metadata/'provenance.json',provenance)
312
+ from importlib.metadata import distributions
313
+ (metadata/'environment.txt').write_text('\n'.join(sorted(f'{d.metadata["Name"]}=={d.version}' for d in distributions())))
314
+ atomic_json(metadata/'asset_hashes.json',asset_hashes)
315
+ atomic_json(metadata/'assets.lock.json',asset_lock)
316
+ atomic_json(metadata/'qwen_cache.json',cache_provenance)
317
+ for audit in ('tabular_audit.json','video_audit.json','pretraining_report.md'):
318
+ source = ROOT/'research/lit_training'/audit
319
+ if source.exists(): shutil.copy2(source,metadata/audit)
320
+ shutil.copytree(HERE/'resolution_adapt',metadata/'source/resolution_adapt',dirs_exist_ok=True,ignore=shutil.ignore_patterns('__pycache__'))
321
+ for source in (Path(__file__),HERE/'training_support.py',HERE/'recovery_checkpoints.py'):
322
+ shutil.copy2(source,metadata/'source'/source.name)
323
+ step = 0
324
+ if args.resume:
325
+ checkpoint = latest_checkpoint(args.output/'checkpoints') if args.resume == 'latest' else Path(args.resume)
326
+ step = resume_checkpoint(checkpoint,model,optimizer,scheduler,sampler)
327
+ initial_step = step
328
+ initial_sampler_draws = sampler.draws
329
+ initial_scheduler_step = scheduler.last_epoch
330
+ metrics = Metrics(args.output,identity,provenance,online=args.mode == 'train')
331
+ if (args.output/'wandb_url.txt').exists():
332
+ provenance['wandb']['url'] = (args.output/'wandb_url.txt').read_text().strip()
333
+ atomic_json(metadata/'provenance.json',provenance)
334
+ metrics.log(step,{**{'model/'+k:v for k,v in report['totals'].items()},'model/lora_rank':args.lora_rank})
335
+ model.train(); model.vae.eval()
336
+ params = [p for g in groups for p in g['params']]
337
+ limit = step+1 if args.mode == 'preflight' else args.max_steps
338
+ # A preflight update is discarded and can never become a production checkpoint.
339
+ probes = {}
340
+ if args.mode == 'preflight':
341
+ for category,module in [('flux',model.video_expert),('action',model.action_expert),
342
+ ('lit',model.semantic_visual_aggregator),('vae',model.vae)]:
343
+ for name,p in module.named_parameters():
344
+ key = 'lora' if name.endswith('lora_B') else category
345
+ if key not in probes and (key != 'flux' or not p.requires_grad):
346
+ probes[key] = (p,p.detach().clone())
347
+ logical_epoch_size = sum(e['length']-16 for e in manifest['episodes'] if e['split']=='train' and e['length']>16)
348
+ last_counts = sampler.counts.copy()
349
+ try:
350
+ if args.mode == 'validate':
351
+ result = validate(model,dataset,manifest,args)
352
+ metrics.log(step,result)
353
+ atomic_json(args.output/'validation.json',{'optimizer_step':step,**result})
354
+ return
355
+ while step < limit:
356
+ begin = time.monotonic(); totals = Counter(); optimizer.zero_grad(set_to_none=True)
357
+ for _ in range(args.gradient_accumulation):
358
+ batch = batch_for(dataset,sampler,args.batch_size,
359
+ failure_log=args.output/'video_decode_failures.jsonl')
360
+ with torch.autocast('cuda',dtype=torch.bfloat16): loss,raw = model.training_loss(batch)
361
+ totals.update(loss_metrics(loss,raw)); (loss/args.gradient_accumulation).backward()
362
+ norm = torch.nn.utils.clip_grad_norm_(params,1.0,error_if_nonfinite=True)
363
+ optimizer.step(); scheduler.step(); step += 1
364
+ if step % args.log_every == 0 or step == 1 or step == limit:
365
+ row = {'train/'+k:v/args.gradient_accumulation for k,v in totals.items()}
366
+ row.update({'train/gradient_norm':float(norm),'performance/samples_per_sec':args.batch_size*args.gradient_accumulation/(time.monotonic()-begin),
367
+ 'performance/effective_batch_size':args.batch_size*args.gradient_accumulation,'sampling/draws':sampler.draws,'sampling/logical_epoch':sampler.draws/logical_epoch_size})
368
+ row['sampling/video_decode_failures'] = sum(sampler.video_decode_failures.values())
369
+ delta = sum(sampler.counts[p]-last_counts[p] for p in POOLS)
370
+ for pool in POOLS:
371
+ row[f'sampling/{pool}/count'] = sampler.counts[pool]
372
+ row[f'sampling/{pool}/percent'] = 100*sampler.counts[pool]/sampler.draws
373
+ row[f'sampling/{pool}/interval_percent'] = 100*(sampler.counts[pool]-last_counts[pool])/max(1,delta)
374
+ last_counts = sampler.counts.copy()
375
+ for i,g in enumerate(optimizer.param_groups): row[f'lr/{g["name"]}/{i}'] = g['lr']
376
+ for name,fn in [('allocated',torch.cuda.memory_allocated),('reserved',torch.cuda.memory_reserved),
377
+ ('max_allocated',torch.cuda.max_memory_allocated),('max_reserved',torch.cuda.max_memory_reserved)]:
378
+ row['gpu/'+name] = fn()
379
+ try: sampler.distribution_report(check=True)
380
+ except ValueError as exc: warnings.warn(str(exc))
381
+ if (args.output/'upload_summary.json').exists():
382
+ row.update({'checkpoint/upload_'+k:v for k,v in json.loads((args.output/'upload_summary.json').read_text()).items()})
383
+ metrics.log(step,row)
384
+ if args.mode == 'train' and args.eval_every and step % args.eval_every == 0:
385
+ metrics.log(step,validate(model,dataset,manifest,args))
386
+ if args.mode == 'train' and checkpoint_due(step,args.checkpoint_every,args.max_steps):
387
+ if (args.output/'wandb_url.txt').exists():
388
+ provenance['wandb']['url'] = (args.output/'wandb_url.txt').read_text().strip()
389
+ atomic_json(metadata/'provenance.json',provenance)
390
+ save_checkpoint(args.output/'checkpoints',step,model,optimizer,scheduler,sampler,metadata,final=step == args.max_steps)
391
+ metrics.log(step,{'checkpoint/step':step,'checkpoint/local_complete':1})
392
+ if args.mode == 'preflight':
393
+ changes = {name:not torch.equal(p,before) for name,(p,before) in probes.items()}
394
+ if not all(changes.get(k) for k in ('lora','action','lit')) or changes.get('flux') or changes.get('vae'):
395
+ raise AssertionError(f'One-step parameter change audit failed: {changes}')
396
+ if args.preflight_save:
397
+ save_checkpoint(args.output/'checkpoints',step,model,optimizer,scheduler,sampler,metadata,final=True)
398
+ atomic_json(args.output/'one_step.json',{'passed':True,'changes':changes,
399
+ 'losses_mean':{k:v/args.gradient_accumulation for k,v in totals.items()},
400
+ 'initial_step':initial_step,'final_step':step,'initial_sampler_draws':initial_sampler_draws,
401
+ 'final_sampler_draws':sampler.draws,'initial_scheduler_step':initial_scheduler_step,
402
+ 'final_scheduler_step':scheduler.last_epoch,'wandb_id':identity['id'],
403
+ 'qwen':'not instantiated; frozen cached contexts','production_steps':0,
404
+ 'gradient_norm':float(norm),'gpu_max_allocated':torch.cuda.max_memory_allocated(),
405
+ 'gpu_max_reserved':torch.cuda.max_memory_reserved(),
406
+ 'recovery_saved_locally':args.preflight_save})
407
+ atomic_json(metadata/'sampling_final.json',sampler.distribution_report())
408
+ finally:
409
+ metrics.finish()
410
+
411
+
412
+ def main():
413
+ args = parser().parse_args(); args.output = args.output.resolve(); args.output.mkdir(parents=True,exist_ok=True)
414
+ if args.mode == 'restore':
415
+ from resolution_adapt.checkpoints import restore_remote
416
+ print(restore_remote(args.output,args.hf_repo)); return
417
+ if args.mode == 'upload':
418
+ import fcntl
419
+ with (args.output/'uploader.lock').open('w') as lock:
420
+ fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
421
+ while True:
422
+ try: upload_once(args.output,args.hf_repo)
423
+ except Exception as exc: print('Uploader retry:',type(exc).__name__,flush=True)
424
+ if args.upload_once: break
425
+ time.sleep(30)
426
+ return
427
+ if args.mode == 'services': services(args); return
428
+ if args.mode == 'sampler':
429
+ episodes = [json.loads(s) for s in (args.dataset_root/'meta/source_episodes.jsonl').read_text().splitlines()]
430
+ manifest = split_episodes(episodes,args.validation_episode_fraction,args.seed)
431
+ sampler = PoolSampler(manifest,weights=(args.familiar_libero_weight,args.plus_weight,args.libero90_weight),seed=args.seed)
432
+ for _ in range(100000): sampler.draw()
433
+ report = sampler.distribution_report(check=True)
434
+ report['episodes'] = {side:dict(Counter(e['pool'] for e in manifest['episodes'] if e['split']==side)) for side in ('train','validation')}
435
+ atomic_json(args.output/'split.json',manifest); atomic_json(args.output/'sampler_report.json',report)
436
+ print(json.dumps(report,indent=2)); return
437
+ import fcntl
438
+ with (args.output/'trainer.lock').open('w') as lock:
439
+ fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
440
+ if args.resume:
441
+ saved = json.loads((args.output/'metadata/args.json').read_text())
442
+ mode,resume,output = args.mode,args.resume,args.output
443
+ for k,v in saved.items():
444
+ setattr(args,k,Path(v) if isinstance(getattr(args,k),Path) else v)
445
+ args.mode,args.resume,args.output = mode,resume,output
446
+ print('Resume uses persisted configuration and split, including original schedule.',flush=True)
447
+ elif (args.output/'metadata/args.json').exists():
448
+ raise ValueError('Run exists: use --resume or a new --output (including after preflight)')
449
+ if args.max_steps <= 0 or not 0 <= args.warmup_steps < args.max_steps:
450
+ raise ValueError('Require max_steps > warmup_steps >= 0')
451
+ for key in ('batch_size','gradient_accumulation','checkpoint_every','log_every','validation_samples_per_pool'):
452
+ if getattr(args,key) <= 0: raise ValueError(f'{key} must be positive')
453
+ # The exclusive training lock proves these are abandoned writes from this run.
454
+ for pending in (args.output/'checkpoints').glob('.pending_*'):
455
+ if pending.is_dir() and not pending.is_symlink(): shutil.rmtree(pending)
456
+ if args.mode == 'train':
457
+ service_report = args.output/'services.json'
458
+ if not service_report.exists() or not json.loads(service_report.read_text())['hf']['passed'] or json.loads(service_report.read_text())['hf']['repo'] != args.hf_repo:
459
+ raise RuntimeError('Run --mode services on this output directory before long training')
460
+ # Independent bounded worker; a slow Hub cannot block optimizer progress.
461
+ log = (args.output/'upload.log').open('a')
462
+ subprocess.Popen([sys.executable,str(Path(__file__).resolve()),'--mode','upload','--output',str(args.output),
463
+ '--hf-repo',args.hf_repo],stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
464
+ log.close()
465
+ run(args)
466
+
467
+
468
+ if __name__ == '__main__': main()
checkpoints/step_004000/metadata/source/training_support.py ADDED
@@ -0,0 +1,357 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Small, explicit adapters for the released LIT ImageWAM training code."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import os
6
+ import shutil
7
+ import time
8
+ from pathlib import Path
9
+
10
+ import numpy as np
11
+ import torch
12
+ from torch.utils.data import Sampler
13
+ from recovery_checkpoints import RecoveryClock, complete_states, make_state_space, queue_snapshot
14
+
15
+
16
+ class VideoDecodeError(RuntimeError):
17
+ """Invalid video data, with enough context to audit a rejected sample."""
18
+ def __init__(self, video_path, timestamps, detail):
19
+ self.video_path = str(video_path)
20
+ self.timestamps = [float(t) for t in timestamps]
21
+ super().__init__(f"{detail}: {self.video_path}, timestamps={self.timestamps}")
22
+
23
+
24
+ def decode_video_frames(video_path, timestamps, tolerance_s, backend=None):
25
+ """Seek to each requested frame; do not decode unused endpoint interiors.
26
+
27
+ A damaged packet between two valid endpoints must not invalidate their
28
+ sample. Each seek still starts at a keyframe and decodes any dependencies;
29
+ invalid data needed for a requested frame raises an auditable error.
30
+ """
31
+ import av
32
+
33
+ targets, inverse = np.unique(np.asarray(timestamps, dtype=np.float64), return_inverse=True)
34
+ if not len(targets):
35
+ raise ValueError("No video timestamps requested")
36
+ decoded = []
37
+ try:
38
+ with av.open(str(video_path)) as container:
39
+ stream = container.streams.video[0]
40
+ stream.thread_type = "SLICE"
41
+ stream.codec_context.thread_count = 1
42
+ for target in targets:
43
+ container.seek(int(target / stream.time_base), stream=stream,
44
+ backward=True, any_frame=False)
45
+ for frame in container.decode(stream):
46
+ timestamp = float(frame.pts * stream.time_base)
47
+ if timestamp < target - tolerance_s:
48
+ continue
49
+ if abs(timestamp - target) > tolerance_s:
50
+ raise ValueError(f"Video timestamp mismatch: {video_path}, "
51
+ f"requested={target}, decoded={timestamp}")
52
+ if frame.is_corrupt:
53
+ raise VideoDecodeError(video_path, timestamps,
54
+ f"Corrupt decoded frame at {timestamp}")
55
+ decoded.append(frame.to_ndarray(format="rgb24"))
56
+ break
57
+ else:
58
+ raise ValueError(f"No video frame decoded: {video_path}, requested={target}")
59
+ except av.error.InvalidDataError as exc:
60
+ raise VideoDecodeError(video_path, timestamps, str(exc)) from exc
61
+ images = np.stack([decoded[i] for i in inverse])
62
+ return torch.from_numpy(images).permute(0, 3, 1, 2).float() / 255
63
+
64
+
65
+ def install_dataset_hooks(vision_free: bool):
66
+ """Use row queries and direct PyAV with current torchvision.
67
+
68
+ torchvision 0.26 removed VideoReader. Decode the same RGB frames directly
69
+ in PyAV and enforce exact timestamp tolerance, without random fallback.
70
+ """
71
+ import lerobot.datasets.lerobot_dataset as dataset_module
72
+ import lerobot.datasets.video_utils as video_module
73
+ cls = dataset_module.LeRobotDataset
74
+
75
+ def row_queries(self, query_indices):
76
+ result = {}
77
+ for key, indices in query_indices.items():
78
+ if key in self.meta.video_keys: continue
79
+ relative = indices if self._absolute_to_relative_idx is None else [self._absolute_to_relative_idx[i] for i in indices]
80
+ result[key] = torch.stack(self.hf_dataset[relative][key])
81
+ return result
82
+ cls._query_hf_dataset = row_queries
83
+ if not hasattr(cls, "_parc_original_query_videos"):
84
+ cls._parc_original_query_videos = cls._query_videos
85
+ if vision_free:
86
+ cls._query_videos = lambda self, query_timestamps, ep_idx: {}
87
+ else:
88
+ cls._query_videos = cls._parc_original_query_videos
89
+
90
+ dataset_module.decode_video_frames = decode_video_frames
91
+ video_module.decode_video_frames = decode_video_frames
92
+ video_module._imagewam_tail_clamp_patched = True
93
+ import imagewam.datasets.lerobot.base_lerobot_dataset as base_module
94
+ base_module.MAX_GETITEM_ATTEMPT = 1
95
+ from imagewam.datasets.lerobot.robot_video_dataset import RobotVideoDataset
96
+ def strict_getitem(self, index):
97
+ result = self._get(index)
98
+ self._mem_trim.tick()
99
+ return result
100
+ RobotVideoDataset.__getitem__ = strict_getitem
101
+
102
+
103
+ class DatasetGripperToImageWAM:
104
+ """Dataset -1=open/+1=close -> released 1=open/0=close."""
105
+ def forward(self, batch):
106
+ action = batch["action"]["default"].clone()
107
+ if not bool(((action[..., -1] == -1) | (action[..., -1] == 1)).all()):
108
+ raise ValueError("Unexpected dataset gripper label")
109
+ action[..., -1] = (1 - action[..., -1]) / 2
110
+ batch["action"]["default"] = action
111
+ return batch
112
+
113
+ def backward(self, batch):
114
+ batch["action"]["default"][..., -1] = 1 - 2 * batch["action"]["default"][..., -1]
115
+ return batch
116
+
117
+
118
+ class TaskBalancedSampler(Sampler[int]):
119
+ """Uniform shuffled task cycles, then source/episode/valid t+16 anchor.
120
+
121
+ Deterministic resume skips consumed microbatches, including accumulation.
122
+ Never samples a padded goal or crosses an episode boundary.
123
+ """
124
+ def __init__(self, dataset, seed, batch_size, num_processes):
125
+ self.seed, self.batch_size, self.num_processes = int(seed), int(batch_size), int(num_processes)
126
+ self.epoch = self.epoch_offset = self.resume_batch_offset = 0
127
+ root = Path(dataset.lerobot_dataset.dataset_dirs[0])
128
+ self.groups = {}
129
+ self.horizon = dataset.num_frames - 1
130
+ self.count = 0
131
+ for line in (root / "meta/source_episodes.jsonl").open():
132
+ ep = json.loads(line)
133
+ eligible = ep["length"] - self.horizon
134
+ if eligible <= 0:
135
+ continue
136
+ source = ep["source_repo"]
137
+ self.groups.setdefault(ep["task_index"], {}).setdefault(source, []).append((ep["dataset_from_index"], eligible))
138
+ self.count += eligible
139
+ self.task_ids = sorted(self.groups)
140
+ if len(self.task_ids) != 111 or len(dataset) != 3080750:
141
+ raise ValueError("Sampler requires the fully audited pinned 111-task dataset")
142
+ effective = int(getattr(dataset, "_parc_effective_batch", self.batch_size*self.num_processes))
143
+ self.count = self.count // effective * effective
144
+
145
+ def set_epoch(self, epoch): self.epoch = int(epoch)
146
+ def set_epoch_offset(self, epoch): self.epoch_offset = int(epoch)
147
+ def set_resume_batch_offset(self, batch): self.resume_batch_offset = int(batch)
148
+ def clear_resume_batch_offset(self): self.resume_batch_offset = 0
149
+ def __len__(self): return self.count
150
+
151
+ def __iter__(self):
152
+ rng = np.random.default_rng(self.seed + self.epoch + self.epoch_offset)
153
+ skip = self.resume_batch_offset * self.batch_size * self.num_processes if self.epoch == 0 else 0
154
+ produced = 0
155
+ while produced < self.count:
156
+ for task in rng.permutation(self.task_ids):
157
+ if produced >= self.count: return
158
+ sources = sorted(self.groups[int(task)])
159
+ source = sources[int(rng.integers(len(sources)))]
160
+ episodes = self.groups[int(task)][source]
161
+ start, eligible = episodes[int(rng.integers(len(episodes)))]
162
+ anchor = start + int(rng.integers(eligible))
163
+ if produced >= skip: yield anchor
164
+ produced += 1
165
+
166
+
167
+ def install_training_hooks(cfg, metadata_dir: Path):
168
+ """Extend logging/checkpoints without changing the released model or loss."""
169
+ import imagewam.trainer as module
170
+ install_dataset_hooks(bool(cfg.data.train.vision_free))
171
+ module.ResumableEpochSampler = TaskBalancedSampler
172
+ base = module.Wan22Trainer
173
+
174
+ class PARCTrainer(base):
175
+ def _build_loader(self, dataset, worker_init_fn=None):
176
+ dataset._parc_effective_batch = self.batch_size * self.gradient_accumulation_steps * self.accelerator.num_processes
177
+ return super()._build_loader(dataset, worker_init_fn=worker_init_fn)
178
+
179
+ def __init__(self, model, *args, **kwargs):
180
+ original_loss = model.training_loss
181
+ self._update_metrics = {}
182
+ self._update_microbatches = 0
183
+ self._metric_step = -1
184
+ def loss_with_update_metrics(sample, *loss_args, **loss_kwargs):
185
+ loss, metrics = original_loss(sample, *loss_args, **loss_kwargs)
186
+ if not bool(torch.isfinite(loss)):
187
+ raise FloatingPointError("Nonfinite LIT loss; refusing an optimizer update")
188
+ if not torch.is_grad_enabled():
189
+ return loss, metrics
190
+ if self._metric_step != self.global_step:
191
+ self._metric_step = self.global_step
192
+ self._update_microbatches = 0
193
+ self._update_metrics = {}
194
+ self._update_microbatches += 1
195
+ values = {"loss":float(loss.detach()), **metrics}
196
+ for key, value in values.items():
197
+ self._update_metrics[key] = self._update_metrics.get(key, 0.0) + float(value)
198
+ return loss, metrics
199
+ model.training_loss = loss_with_update_metrics
200
+ original_init = torch.optim.AdamW.__init__
201
+ max_group_numel = int(kwargs["cfg"].get("optimizer_max_group_numel", 0))
202
+ def fused_init(optimizer, parameters, *optimizer_args, **optimizer_kwargs):
203
+ optimizer_kwargs["fused"] = True
204
+ if max_group_numel:
205
+ # ZeRO-1 casts/flattens a complete optimizer group at once.
206
+ # Bound those temporaries without changing any parameter's
207
+ # AdamW settings or the global gradient clipping rule.
208
+ groups, current, count = [], [], 0
209
+ for parameter in parameters:
210
+ if isinstance(parameter, dict):
211
+ raise ValueError("Expected the released flat parameter list")
212
+ if current and count + parameter.numel() > max_group_numel:
213
+ groups.append({"params": current})
214
+ current, count = [], 0
215
+ current.append(parameter)
216
+ count += parameter.numel()
217
+ if current: groups.append({"params": current})
218
+ parameters = groups
219
+ original_init(optimizer, parameters, *optimizer_args, **optimizer_kwargs)
220
+ use_fused = bool(kwargs["cfg"].get("fused_adamw", False))
221
+ if use_fused: torch.optim.AdamW.__init__ = fused_init
222
+ try:
223
+ super().__init__(model, *args, **kwargs)
224
+ finally:
225
+ torch.optim.AdamW.__init__ = original_init
226
+ if self.wandb_enabled:
227
+ # Publish the first real optimizer update to verify online
228
+ # telemetry (also after resume), then use the ten-update cadence.
229
+ self.log_every = 1
230
+ self._recovery_clock = None
231
+ if not self.cfg.get("preflight", False) and self.cfg.get("recovery_interval_seconds", 0):
232
+ self._recovery_clock = RecoveryClock(
233
+ Path(self.output_dir) / "checkpoints/recovery/clock.json",
234
+ interval=float(self.cfg.recovery_interval_seconds))
235
+
236
+ def _init_wandb(self):
237
+ super()._init_wandb()
238
+ if self.wandb_run is not None:
239
+ from omegaconf import OmegaConf
240
+ self.wandb_run.config.update(OmegaConf.to_container(self.cfg, resolve=True))
241
+ (Path(self.output_dir) / "wandb_url.txt").write_text(self.wandb_run.url + "\n")
242
+
243
+ def _maybe_log_omnigen2_forward_profile(self, device):
244
+ super()._maybe_log_omnigen2_forward_profile(device)
245
+ # This released hook runs after backward on every microbatch.
246
+ # Force the upcoming optimizer boundary to call _wandb_log when
247
+ # a recovery is due; ordinary telemetry retains its cadence.
248
+ if self._recovery_clock and (self._recovery_clock.due()
249
+ or (Path(self.output_dir).parent / "recovery-save.request").exists()
250
+ or (Path(self.output_dir).parent / "save-and-stop.request").exists()):
251
+ self.log_every = 1
252
+
253
+ def _wandb_log(self, payload):
254
+ if "eval/val_loss" in payload:
255
+ payload["eval/training_subset_loss"] = payload.pop("eval/val_loss")
256
+ if "train/loss" in payload and self._update_microbatches:
257
+ for key, value in self._update_metrics.items():
258
+ payload["train/" + key] = value / self._update_microbatches
259
+ payload.update({"optimizer_step": self.global_step, "epoch": self.epoch,
260
+ "performance/effective_batch_size": self.batch_size * self.gradient_accumulation_steps * self.accelerator.num_processes})
261
+ if "performance/samples_per_sec" in payload:
262
+ payload["performance/samples_per_sec"] *= self.gradient_accumulation_steps
263
+ if torch.cuda.is_available():
264
+ payload.update({"gpu/allocated_gib": torch.cuda.memory_allocated()/2**30,
265
+ "gpu/reserved_gib": torch.cuda.memory_reserved()/2**30,
266
+ "gpu/peak_allocated_gib": torch.cuda.max_memory_allocated()/2**30,
267
+ "gpu/peak_reserved_gib": torch.cuda.max_memory_reserved()/2**30})
268
+ # Stage 1 has no pose objective; explicit zero rather than missing.
269
+ if "train/loss" in payload and self.cfg.model.goal_prior_stage == "stage1":
270
+ payload["train/loss_pose"] = 0.0
271
+ payload["train/loss_video"] = 0.0
272
+ if "train/loss_pose" in payload:
273
+ payload["train/pose_loss_unweighted"] = payload["train/loss_pose"] / float(self.cfg.model.loss.lambda_pose)
274
+ name = "latest_train_metrics.json" if "train/loss" in payload else "latest_eval_metrics.json"
275
+ destination = Path(self.output_dir) / name
276
+ temporary = destination.with_suffix(".tmp")
277
+ temporary.write_text(json.dumps(payload, indent=2) + "\n")
278
+ temporary.replace(destination)
279
+ super()._wandb_log(payload)
280
+ if "train/loss" in payload and self.global_step > 0:
281
+ self.log_every = int(self.cfg.log_every)
282
+ request = Path(self.output_dir).parent / "save-and-stop.request"
283
+ if request.exists() and not self.cfg.get("preflight", False):
284
+ # Logging is called after a complete optimizer update and
285
+ # before the next sample, so state and sampler progress agree.
286
+ checkpoint = self.save_checkpoint()
287
+ (Path(self.output_dir).parent / "save-and-stop.result.json").write_text(
288
+ json.dumps({"optimizer_step": self.global_step,
289
+ "stage": self.cfg.model.goal_prior_stage, **checkpoint}, indent=2) + "\n")
290
+ request.unlink()
291
+ self._finish_wandb()
292
+ raise SystemExit(75)
293
+ force_recovery = Path(self.output_dir).parent / "recovery-save.request"
294
+ if self._recovery_clock and (self._recovery_clock.due() or force_recovery.exists()):
295
+ checkpoint = self._save_recovery_state()
296
+ self._recovery_clock.completed()
297
+ force_recovery.unlink(missing_ok=True)
298
+ print(f"Rolling recovery checkpoint: update {self.global_step}, {checkpoint}", flush=True)
299
+
300
+ def _save_recovery_state(self):
301
+ stage_dir = Path(self.output_dir)
302
+ step_tag = f"step_{self.global_step:06d}"
303
+ state = Path(self.state_dir) / step_tag
304
+ pending = Path(self.state_dir) / (".pending_" + step_tag)
305
+ self.accelerator.wait_for_everyone()
306
+ if self.accelerator.is_main_process:
307
+ if not (state / "trainer_state.json").exists():
308
+ shutil.rmtree(pending, ignore_errors=True)
309
+ previous = complete_states(Path(self.state_dir))
310
+ estimated = (sum(p.stat().st_size for p in previous[-1].rglob("*") if p.is_file())
311
+ if previous else (17_100_000_000 if self.cfg.model.goal_prior_stage == "stage1" else 66_000_000_000))
312
+ make_state_space(stage_dir, int(estimated * 1.05))
313
+ pending.mkdir(parents=True)
314
+ self.accelerator.wait_for_everyone()
315
+ if not (state / "trainer_state.json").exists():
316
+ self.accelerator.save_state(output_dir=str(pending))
317
+ if self.accelerator.is_main_process:
318
+ self._save_trainer_state(str(pending))
319
+ (pending / "COMPLETE").touch()
320
+ pending.rename(state)
321
+ self.accelerator.wait_for_everyone()
322
+ if self.accelerator.is_main_process:
323
+ if not self.cfg.get("preflight", False) and self.cfg.get("recovery_interval_seconds", 0):
324
+ queue_snapshot(state, stage_dir, metadata_dir, repo=str(self.cfg.recovery_repo))
325
+ self._prune_old_state_checkpoints(keep_step_tag=step_tag)
326
+ self.accelerator.wait_for_everyone()
327
+ return str(state)
328
+
329
+ def save_checkpoint(self):
330
+ # Timed recovery saves can coincide with 5,000-update/final saves.
331
+ # Reuse the state but still create the independently scheduled weights.
332
+ state_path = self._save_recovery_state()
333
+ weights_path = Path(self.weights_dir) / f"step_{self.global_step:06d}.pt"
334
+ if self.accelerator.is_main_process and not weights_path.exists():
335
+ pending_tag = f".pending_step_{self.global_step:06d}"
336
+ self._save_weights_checkpoint(step_tag=pending_tag)
337
+ (Path(self.weights_dir) / (pending_tag + ".pt")).replace(weights_path)
338
+ self.accelerator.wait_for_everyone()
339
+ result = {"weights_path": str(weights_path), "state_path": state_path}
340
+ if not self.cfg.get("preflight", False) and self.accelerator.is_main_process and (self.global_step % 5000 == 0 or self.global_step == self.max_steps):
341
+ stage = self.cfg.model.goal_prior_stage
342
+ folder = Path(self.output_dir).parent / "uploads" / stage / f"step_{self.global_step:06d}"
343
+ folder.mkdir(parents=True, exist_ok=True)
344
+ target = folder / "model.pt"
345
+ if not target.exists(): os.link(result["weights_path"], target)
346
+ for name in ["config.yaml", "dataset_stats.json"]:
347
+ shutil.copy2(Path(self.output_dir) / name, folder / name)
348
+ shutil.copytree(metadata_dir, folder / "reproduction", dirs_exist_ok=True)
349
+ (folder / "checkpoint.json").write_text(json.dumps({"stage":stage,"optimizer_step":self.global_step,
350
+ "final":self.global_step==self.max_steps,"created_at_unix":time.time()},indent=2))
351
+ # The independent uploader scans only fully staged folders.
352
+ (folder / "READY").touch()
353
+ return result
354
+
355
+ module.Wan22Trainer = PARCTrainer
356
+ import imagewam.runtime as runtime
357
+ runtime.Wan22Trainer = PARCTrainer
checkpoints/step_004000/metadata/split.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoints/step_004000/metadata/tabular_audit.json ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tasks": 111,
3
+ "episodes": 19961,
4
+ "frames": 3080750,
5
+ "fps": 20,
6
+ "valid_h16_goal_anchors": 2761374,
7
+ "timestamp_max_error_seconds": 7.629394538355427e-07,
8
+ "sources": {
9
+ "nvidia": {
10
+ "frames": 842714,
11
+ "tasks": 111,
12
+ "state_min": [
13
+ -0.4862295389175415,
14
+ -0.3970998525619507,
15
+ 0.008128180168569088,
16
+ -0.7480751872062683,
17
+ -4.363129138946533,
18
+ -3.212415933609009,
19
+ -0.0025667562149465084,
20
+ -0.04255744442343712
21
+ ],
22
+ "state_max": [
23
+ 0.21031762659549713,
24
+ 0.48814651370048523,
25
+ 1.3660105466842651,
26
+ 4.843389987945557,
27
+ 4.042375564575195,
28
+ 2.4014885425567627,
29
+ 0.04639676958322525,
30
+ 0.0017311422852799296
31
+ ],
32
+ "action_min": [
33
+ -0.9375,
34
+ -0.9375,
35
+ -0.9375,
36
+ -0.3257142901420593,
37
+ -0.375,
38
+ -0.375,
39
+ -1.0
40
+ ],
41
+ "action_max": [
42
+ 0.9375,
43
+ 0.9375,
44
+ 0.9375,
45
+ 0.375,
46
+ 0.375,
47
+ 0.375,
48
+ 1.0
49
+ ],
50
+ "gripper_counts": {
51
+ "-1.0": 444702,
52
+ "1.0": 398012
53
+ },
54
+ "gripper_dynamics": {
55
+ "-1": {
56
+ "count": 440070,
57
+ "mean_next_width_delta": 0.0004565622366499156,
58
+ "median_width": 0.0791521966457367
59
+ },
60
+ "1": {
61
+ "count": 397030,
62
+ "mean_next_width_delta": -0.0007894165464676917,
63
+ "median_width": 0.02936265431344509
64
+ }
65
+ }
66
+ },
67
+ "plus": {
68
+ "frames": 2238036,
69
+ "tasks": 40,
70
+ "state_min": [
71
+ -0.4856736361980438,
72
+ -0.3313629925251007,
73
+ 0.008128180168569088,
74
+ 0.8415665030479431,
75
+ -3.616978406906128,
76
+ -1.842738389968872,
77
+ -0.002542249858379364,
78
+ -0.042119648307561874
79
+ ],
80
+ "state_max": [
81
+ 0.21031762659549713,
82
+ 0.39128610491752625,
83
+ 1.3660105466842651,
84
+ 3.7743608951568604,
85
+ 3.5752453804016113,
86
+ 1.3747878074645996,
87
+ 0.042380549013614655,
88
+ 0.0010137537028640509
89
+ ],
90
+ "action_min": [
91
+ -0.9375,
92
+ -0.9375,
93
+ -0.9375,
94
+ -0.2582142949104309,
95
+ -0.375,
96
+ -0.3642857074737549,
97
+ -1.0
98
+ ],
99
+ "action_max": [
100
+ 0.9375,
101
+ 0.9375,
102
+ 0.9375,
103
+ 0.3557142913341522,
104
+ 0.375,
105
+ 0.375,
106
+ 1.0
107
+ ],
108
+ "gripper_counts": {
109
+ "-1.0": 1158433,
110
+ "1.0": 1079603
111
+ },
112
+ "gripper_dynamics": {
113
+ "-1": {
114
+ "count": 1145046,
115
+ "mean_next_width_delta": 0.0006007962510921061,
116
+ "median_width": 0.07898755371570587
117
+ },
118
+ "1": {
119
+ "count": 1078643,
120
+ "mean_next_width_delta": -0.0008084022556431592,
121
+ "median_width": 0.0340743213891983
122
+ }
123
+ }
124
+ }
125
+ },
126
+ "suites": {
127
+ "('nvidia/LIBERO_LeRobot_v3', 'libero_10')": 379,
128
+ "('nvidia/LIBERO_LeRobot_v3', 'libero_90')": 3921,
129
+ "('nvidia/LIBERO_LeRobot_v3', 'libero_goal')": 428,
130
+ "('nvidia/LIBERO_LeRobot_v3', 'libero_object')": 454,
131
+ "('nvidia/LIBERO_LeRobot_v3', 'libero_spatial')": 432,
132
+ "('lerobot/libero_plus', None)": 14347
133
+ },
134
+ "errors": [],
135
+ "image_audit": "pending; tabular audit does not certify images or simulator replay"
136
+ }
checkpoints/step_004000/metadata/video_audit.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "video_files_checked": 108,
3
+ "expected_files": 108,
4
+ "errors": [],
5
+ "decoded_frames": 324,
6
+ "scope": "all file metadata, all episode camera intervals, first/middle/last decoded RGB frame of every file; not every video frame or simulator replay"
7
+ }
checkpoints/step_004000/metadata/wandb.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "id": "db6a83524459",
3
+ "project": "parc2026-lit-imagewam",
4
+ "name": "lit-stage2-resadapt-128to224-lora-r16",
5
+ "group": "lit-stage2-resadapt-128to224"
6
+ }
checkpoints/step_004000/resume/state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0c3e5d11137de82ae6898ea4288a95c05068fed58e30df7f1e70e05596181c4
3
+ size 6672739984
checkpoints/step_004000/weights/adaptation.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e652091d431c95ce5df1bab7c3fd01de0edbab3c254763c180a6392fa6e7170
3
+ size 3336327138