Add files using upload-large-folder tool
Browse files- checkpoints/step_004000/manifest.json +155 -0
- checkpoints/step_004000/metadata/args.json +42 -0
- checkpoints/step_004000/metadata/asset_hashes.json +5 -0
- checkpoints/step_004000/metadata/assets.lock.json +32 -0
- checkpoints/step_004000/metadata/checkpoint_load.json +22 -0
- checkpoints/step_004000/metadata/config.yaml +270 -0
- checkpoints/step_004000/metadata/dataset_stats.json +1074 -0
- checkpoints/step_004000/metadata/environment.txt +113 -0
- checkpoints/step_004000/metadata/parameters.json +218 -0
- checkpoints/step_004000/metadata/pretraining_report.md +53 -0
- checkpoints/step_004000/metadata/provenance.json +50 -0
- checkpoints/step_004000/metadata/qwen_cache.json +11 -0
- checkpoints/step_004000/metadata/sampling.json +14 -0
- checkpoints/step_004000/metadata/source/recovery_checkpoints.py +123 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/README.md +357 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/__init__.py +1 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/assets.lock.json +32 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/checkpoints.py +217 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/data.py +180 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/model.py +188 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/offline.py +38 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/preflight_data.py +114 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/preflight_model.py +45 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/prepare_assets.py +57 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/released_stage2.yaml +271 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/requirements-training.lock +341 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/requirements-training.txt +29 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/telemetry.py +58 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/test_pipeline.py +284 -0
- checkpoints/step_004000/metadata/source/resolution_adapt/test_video_loading.py +211 -0
- checkpoints/step_004000/metadata/source/train_resolution_adapt.py +468 -0
- checkpoints/step_004000/metadata/source/training_support.py +357 -0
- checkpoints/step_004000/metadata/split.json +0 -0
- checkpoints/step_004000/metadata/tabular_audit.json +136 -0
- checkpoints/step_004000/metadata/video_audit.json +7 -0
- checkpoints/step_004000/metadata/wandb.json +6 -0
- checkpoints/step_004000/resume/state.pt +3 -0
- checkpoints/step_004000/weights/adaptation.pt +3 -0
checkpoints/step_004000/manifest.json
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 4000,
|
| 3 |
+
"final": false,
|
| 4 |
+
"files": {
|
| 5 |
+
"metadata/args.json": {
|
| 6 |
+
"size": 1389,
|
| 7 |
+
"sha256": "1369a024d5fffe22dfbfe1400b7eef7de7ba7948a34ee631cd3e3c2ddf0570bc"
|
| 8 |
+
},
|
| 9 |
+
"metadata/asset_hashes.json": {
|
| 10 |
+
"size": 390,
|
| 11 |
+
"sha256": "e8c07d54adefa2fff39a36d6335f9ac1d92bed5814e68125bb4b1b1e7c2f9e8a"
|
| 12 |
+
},
|
| 13 |
+
"metadata/assets.lock.json": {
|
| 14 |
+
"size": 968,
|
| 15 |
+
"sha256": "3ff22d9d5dcdb747c3739a34e401b4e6dab21051261a31b8f25f3e8c33d4470a"
|
| 16 |
+
},
|
| 17 |
+
"metadata/checkpoint_load.json": {
|
| 18 |
+
"size": 415,
|
| 19 |
+
"sha256": "12f43b5b3b45c967277d8b5df707a2e6acab6acfd7da3b116fb77cfc0c168b47"
|
| 20 |
+
},
|
| 21 |
+
"metadata/config.yaml": {
|
| 22 |
+
"size": 6690,
|
| 23 |
+
"sha256": "35a87f80d7436ea1ed629f4b9d63a90846c3d09bcd90d7c82bc35833b76dd88d"
|
| 24 |
+
},
|
| 25 |
+
"metadata/dataset_stats.json": {
|
| 26 |
+
"size": 22880,
|
| 27 |
+
"sha256": "b107cd929518d0477535f99834e0cc13f470e25cd61bbce69b315619ba5140d4"
|
| 28 |
+
},
|
| 29 |
+
"metadata/environment.txt": {
|
| 30 |
+
"size": 2106,
|
| 31 |
+
"sha256": "2de6cecede55f1f43ed21f751e381c6f78ce9026e0b962a1da407c8257de54a9"
|
| 32 |
+
},
|
| 33 |
+
"metadata/parameters.json": {
|
| 34 |
+
"size": 5337,
|
| 35 |
+
"sha256": "c735a0a91c71e6a31c05bee272b447a585932f396ea92ad3fcfc95c8f8e9d3eb"
|
| 36 |
+
},
|
| 37 |
+
"metadata/pretraining_report.md": {
|
| 38 |
+
"size": 13688,
|
| 39 |
+
"sha256": "d0ab4563c377ee9a79bbdfcc7829b55b2cf9dfbd7d786eecffa181be5b21c676"
|
| 40 |
+
},
|
| 41 |
+
"metadata/provenance.json": {
|
| 42 |
+
"size": 1873,
|
| 43 |
+
"sha256": "c7c3edebc43685c22e6ce25e013553084d7d3a5718cd50412197299b3c03f5d6"
|
| 44 |
+
},
|
| 45 |
+
"metadata/qwen_cache.json": {
|
| 46 |
+
"size": 216,
|
| 47 |
+
"sha256": "264991ae378d3e8388732252e960f972ba2d01cc28301e5e47c84379e8809b40"
|
| 48 |
+
},
|
| 49 |
+
"metadata/sampling.json": {
|
| 50 |
+
"size": 229,
|
| 51 |
+
"sha256": "ad8969c84ed4d9ce1736cb271ebef09e0cc945ec5baeefd86ad294c5fb749cd4"
|
| 52 |
+
},
|
| 53 |
+
"metadata/source/recovery_checkpoints.py": {
|
| 54 |
+
"size": 5395,
|
| 55 |
+
"sha256": "73f5b3cd8c1825f5c9c064637a0b5433801297dbd31dfa2104438610650254e7"
|
| 56 |
+
},
|
| 57 |
+
"metadata/source/resolution_adapt/README.md": {
|
| 58 |
+
"size": 19088,
|
| 59 |
+
"sha256": "53327de34252a7a6ebc2604b9e86e6e1aaec7619c3b256b99ce6df12465d6173"
|
| 60 |
+
},
|
| 61 |
+
"metadata/source/resolution_adapt/__init__.py": {
|
| 62 |
+
"size": 85,
|
| 63 |
+
"sha256": "b9eacc14b8dec315086ef582e49a9a6d11c013b3aa602a201e5a75fde143bbbf"
|
| 64 |
+
},
|
| 65 |
+
"metadata/source/resolution_adapt/assets.lock.json": {
|
| 66 |
+
"size": 968,
|
| 67 |
+
"sha256": "3ff22d9d5dcdb747c3739a34e401b4e6dab21051261a31b8f25f3e8c33d4470a"
|
| 68 |
+
},
|
| 69 |
+
"metadata/source/resolution_adapt/checkpoints.py": {
|
| 70 |
+
"size": 11510,
|
| 71 |
+
"sha256": "016644cd764234f7f1a0da857b67c985b4796ec37b0725a38b0720cf4f32e1c1"
|
| 72 |
+
},
|
| 73 |
+
"metadata/source/resolution_adapt/data.py": {
|
| 74 |
+
"size": 8391,
|
| 75 |
+
"sha256": "e890eb65a989ee849c79a3eeaca93740ec1d907b3e85ab7ab78cb6e04d61c6f7"
|
| 76 |
+
},
|
| 77 |
+
"metadata/source/resolution_adapt/model.py": {
|
| 78 |
+
"size": 10445,
|
| 79 |
+
"sha256": "30376dc071cf1ff004348e2fed07f5e026541acbf471b894ed3a69e6b2a37f49"
|
| 80 |
+
},
|
| 81 |
+
"metadata/source/resolution_adapt/offline.py": {
|
| 82 |
+
"size": 2253,
|
| 83 |
+
"sha256": "ed5b420dda803618d559e6ecf236afb1105cdd4d191b89b7e3670e14344dfb98"
|
| 84 |
+
},
|
| 85 |
+
"metadata/source/resolution_adapt/preflight_data.py": {
|
| 86 |
+
"size": 7135,
|
| 87 |
+
"sha256": "b8b4b0835b4a5d21bf23e8057d0c1e2e00575b6f7839bbc55203d7addc55e420"
|
| 88 |
+
},
|
| 89 |
+
"metadata/source/resolution_adapt/preflight_model.py": {
|
| 90 |
+
"size": 2622,
|
| 91 |
+
"sha256": "7348384848bacf373f93a06efb5f538eea2e402a27fefc07d7c2e2995338c06f"
|
| 92 |
+
},
|
| 93 |
+
"metadata/source/resolution_adapt/prepare_assets.py": {
|
| 94 |
+
"size": 3414,
|
| 95 |
+
"sha256": "d6243b15acce8d439069d39ca1905fa2d4828e932f65529c3fe46d3dc04f6360"
|
| 96 |
+
},
|
| 97 |
+
"metadata/source/resolution_adapt/released_stage2.yaml": {
|
| 98 |
+
"size": 6886,
|
| 99 |
+
"sha256": "9c4a8d9eb8767e71e7ebd6edb921be810c7867512d9ebbf0f91a53dca21e9da1"
|
| 100 |
+
},
|
| 101 |
+
"metadata/source/resolution_adapt/requirements-training.lock": {
|
| 102 |
+
"size": 8037,
|
| 103 |
+
"sha256": "b8300068205c5a0f6f7ca2a3a52c10f3f26f4183de08cb9676df5fc9d9537c38"
|
| 104 |
+
},
|
| 105 |
+
"metadata/source/resolution_adapt/requirements-training.txt": {
|
| 106 |
+
"size": 652,
|
| 107 |
+
"sha256": "0f9d4aad5b1f89bda809706f964c8a9128d28cecb6bbea5138e4f4d43cb8ec31"
|
| 108 |
+
},
|
| 109 |
+
"metadata/source/resolution_adapt/telemetry.py": {
|
| 110 |
+
"size": 2261,
|
| 111 |
+
"sha256": "bda6d3f2ad96aa2844fa56deebdcac46476caf5440c5afa199dd5daeffc03196"
|
| 112 |
+
},
|
| 113 |
+
"metadata/source/resolution_adapt/test_pipeline.py": {
|
| 114 |
+
"size": 15265,
|
| 115 |
+
"sha256": "e0317527b23a866566b58a90bce4239f35df08066e6434ce6d999817599a73c2"
|
| 116 |
+
},
|
| 117 |
+
"metadata/source/resolution_adapt/test_video_loading.py": {
|
| 118 |
+
"size": 8465,
|
| 119 |
+
"sha256": "d43efef062717b163ca8849bbb2b089a2401b8f77bcefe7a522a8126f58cdeb3"
|
| 120 |
+
},
|
| 121 |
+
"metadata/source/train_resolution_adapt.py": {
|
| 122 |
+
"size": 30344,
|
| 123 |
+
"sha256": "e4fcfcb1b6e4b704a91de4b9dbea12ce2f858deaf6255b6b3a90228b17f2a858"
|
| 124 |
+
},
|
| 125 |
+
"metadata/source/training_support.py": {
|
| 126 |
+
"size": 19567,
|
| 127 |
+
"sha256": "30c31bf36d93e1e53e15eb6d511919fb1b65ab84c864b5987169ea16ae57c9b0"
|
| 128 |
+
},
|
| 129 |
+
"metadata/split.json": {
|
| 130 |
+
"size": 9038397,
|
| 131 |
+
"sha256": "2a78a2a60677149f0fdeb51c2e80ec47966c6b336bd78b5f6943e71e04c15d02"
|
| 132 |
+
},
|
| 133 |
+
"metadata/tabular_audit.json": {
|
| 134 |
+
"size": 3285,
|
| 135 |
+
"sha256": "6596934ec2c3e23bfa27b4b2278d1f4d6935751ff118af4d964cb94274839595"
|
| 136 |
+
},
|
| 137 |
+
"metadata/video_audit.json": {
|
| 138 |
+
"size": 255,
|
| 139 |
+
"sha256": "dba4a05568c5229b2f1b88aab7a0e74eea8032bfcefa4b5e2d78973ebbced164"
|
| 140 |
+
},
|
| 141 |
+
"metadata/wandb.json": {
|
| 142 |
+
"size": 159,
|
| 143 |
+
"sha256": "3aa8ececbf4ec6d9633e03f0e4e8361c3996da7ef3c12a92d8d2c295b49dea0b"
|
| 144 |
+
},
|
| 145 |
+
"resume/state.pt": {
|
| 146 |
+
"size": 6672739984,
|
| 147 |
+
"sha256": "d0c3e5d11137de82ae6898ea4288a95c05068fed58e30df7f1e70e05596181c4"
|
| 148 |
+
},
|
| 149 |
+
"weights/adaptation.pt": {
|
| 150 |
+
"size": 3336327138,
|
| 151 |
+
"sha256": "7e652091d431c95ce5df1bab7c3fd01de0edbab3c254763c180a6392fa6e7170"
|
| 152 |
+
}
|
| 153 |
+
},
|
| 154 |
+
"created_unix": 1791326174.4687986
|
| 155 |
+
}
|
checkpoints/step_004000/metadata/args.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"mode": "train",
|
| 3 |
+
"output": "/content/PARC2026_final/.runtime/lit-resolution-resume-2000",
|
| 4 |
+
"dataset_root": "/tmp/resadapt-dataset",
|
| 5 |
+
"released_dir": "/tmp/resadapt-assets/released/imagewam/lit_stage2",
|
| 6 |
+
"imagewam_root": "/tmp/resadapt-imagewam",
|
| 7 |
+
"flux_root": "/tmp/resadapt-flux2",
|
| 8 |
+
"ae_path": "/tmp/resadapt-assets/vae/ae.safetensors",
|
| 9 |
+
"qwen_cache": "/tmp/resadapt-assets/qwen_cache",
|
| 10 |
+
"resume": "/content/PARC2026_final/.runtime/lit-resolution-resume-2000/checkpoints/step_002000",
|
| 11 |
+
"max_steps": 10000,
|
| 12 |
+
"warmup_steps": 500,
|
| 13 |
+
"batch_size": 1,
|
| 14 |
+
"gradient_accumulation": 8,
|
| 15 |
+
"lora_rank": 16,
|
| 16 |
+
"unfreeze_last_flux_blocks": 0,
|
| 17 |
+
"checkpoint_every": 2000,
|
| 18 |
+
"seed": 42,
|
| 19 |
+
"eval_every": 200,
|
| 20 |
+
"validation_samples_per_pool": 8,
|
| 21 |
+
"log_every": 10,
|
| 22 |
+
"lora_alpha": 16.0,
|
| 23 |
+
"lora_dropout": 0.0,
|
| 24 |
+
"lr_flux_lora": 3e-05,
|
| 25 |
+
"lr_action": 1e-05,
|
| 26 |
+
"lr_interface": 1e-05,
|
| 27 |
+
"lr_pose": 1e-05,
|
| 28 |
+
"lr_proprio": 1e-05,
|
| 29 |
+
"lr_other": 1e-05,
|
| 30 |
+
"lr_unfrozen_flux": 1e-06,
|
| 31 |
+
"weight_decay": 0.01,
|
| 32 |
+
"familiar_libero_weight": 0.5,
|
| 33 |
+
"plus_weight": 0.4,
|
| 34 |
+
"libero90_weight": 0.1,
|
| 35 |
+
"validation_episode_fraction": 0.05,
|
| 36 |
+
"hf_repo": "Ratstar216/LIT-ImageWAM-ResolutionAdapt-128to224",
|
| 37 |
+
"wandb_project": "parc2026-lit-imagewam",
|
| 38 |
+
"wandb_run_name": "lit-stage2-resadapt-128to224-lora-r16",
|
| 39 |
+
"wandb_group": "lit-stage2-resadapt-128to224",
|
| 40 |
+
"upload_once": false,
|
| 41 |
+
"preflight_save": false
|
| 42 |
+
}
|
checkpoints/step_004000/metadata/asset_hashes.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"/tmp/resadapt-assets/released/imagewam/lit_stage2/model.pt": "f403225e76809034307caf23e5a8f155385a51adb69d98990cff14125743ec97",
|
| 3 |
+
"/tmp/resadapt-assets/released/imagewam/lit_stage2/dataset_stats.json": "b107cd929518d0477535f99834e0cc13f470e25cd61bbce69b315619ba5140d4",
|
| 4 |
+
"/tmp/resadapt-assets/vae/ae.safetensors": "868fe7b343cc8f3a19dbcfcafbc3d5f888802be3f89bd81b65b3621a066ce8f3"
|
| 5 |
+
}
|
checkpoints/step_004000/metadata/assets.lock.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"linjianman/LIT": {
|
| 3 |
+
"revision": "765a2b2bf136f7039ab18978920c682f1aae65d2",
|
| 4 |
+
"files": {
|
| 5 |
+
"imagewam/lit_stage2/config.yaml": {
|
| 6 |
+
"size": 6886,
|
| 7 |
+
"sha256": "9c4a8d9eb8767e71e7ebd6edb921be810c7867512d9ebbf0f91a53dca21e9da1"
|
| 8 |
+
},
|
| 9 |
+
"imagewam/lit_stage2/dataset_stats.json": {
|
| 10 |
+
"size": 22880,
|
| 11 |
+
"sha256": "b107cd929518d0477535f99834e0cc13f470e25cd61bbce69b315619ba5140d4"
|
| 12 |
+
},
|
| 13 |
+
"imagewam/lit_stage2/model.pt": {
|
| 14 |
+
"size": 9373131513,
|
| 15 |
+
"sha256": "f403225e76809034307caf23e5a8f155385a51adb69d98990cff14125743ec97"
|
| 16 |
+
}
|
| 17 |
+
}
|
| 18 |
+
},
|
| 19 |
+
"black-forest-labs/FLUX.2-dev": {
|
| 20 |
+
"revision": "26afe3a78bb242c0a8bb181dcc8937bb16e5c66c",
|
| 21 |
+
"files": {
|
| 22 |
+
"ae.safetensors": {
|
| 23 |
+
"size": 336211292,
|
| 24 |
+
"sha256": "868fe7b343cc8f3a19dbcfcafbc3d5f888802be3f89bd81b65b3621a066ce8f3"
|
| 25 |
+
}
|
| 26 |
+
}
|
| 27 |
+
},
|
| 28 |
+
"Qwen/Qwen3-4B": {
|
| 29 |
+
"revision": "1cfa9a7208912126459214e8b04321603b3df60c",
|
| 30 |
+
"files": {}
|
| 31 |
+
}
|
| 32 |
+
}
|
checkpoints/step_004000/metadata/checkpoint_load.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"mot": {
|
| 3 |
+
"missing_keys": [],
|
| 4 |
+
"unexpected_keys": []
|
| 5 |
+
},
|
| 6 |
+
"proprio_encoder": {
|
| 7 |
+
"missing_keys": [],
|
| 8 |
+
"unexpected_keys": []
|
| 9 |
+
},
|
| 10 |
+
"semantic_visual_aggregator": {
|
| 11 |
+
"missing_keys": [],
|
| 12 |
+
"unexpected_keys": []
|
| 13 |
+
},
|
| 14 |
+
"semantic_visual_pose_decoder": {
|
| 15 |
+
"missing_keys": [],
|
| 16 |
+
"unexpected_keys": []
|
| 17 |
+
},
|
| 18 |
+
"semantic_visual_pose_norm": {
|
| 19 |
+
"missing_keys": [],
|
| 20 |
+
"unexpected_keys": []
|
| 21 |
+
}
|
| 22 |
+
}
|
checkpoints/step_004000/metadata/config.yaml
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
output_dir: /content/PARC2026_final/.runtime/lit-resolution-resume-2000
|
| 2 |
+
batch_size: 1
|
| 3 |
+
num_workers: 0
|
| 4 |
+
prefetch_factor: 2
|
| 5 |
+
persistent_workers: true
|
| 6 |
+
lr_scheduler_type: cosine
|
| 7 |
+
learning_rate: null
|
| 8 |
+
warmup_steps: 500
|
| 9 |
+
num_epochs: 10
|
| 10 |
+
max_steps: 10000
|
| 11 |
+
log_every: 10
|
| 12 |
+
save_every: 2000
|
| 13 |
+
keep_latest_state_only: true
|
| 14 |
+
eval_every: 100
|
| 15 |
+
eval_num_inference_steps: 10
|
| 16 |
+
eval_num_samples: 8
|
| 17 |
+
rank_timer_every: 10
|
| 18 |
+
rank_timer_sync_cuda: true
|
| 19 |
+
qwen_cache_batch_size: 32
|
| 20 |
+
qwen_cache_overwrite: false
|
| 21 |
+
qwen_cache_save_workers: 8
|
| 22 |
+
flux2_qwen3_model_spec: null
|
| 23 |
+
precache_arrow: true
|
| 24 |
+
precache_num_workers: 4
|
| 25 |
+
build_lerobot_meta_cache: true
|
| 26 |
+
build_norm_stats: false
|
| 27 |
+
precache_fuse_norm_stats: true
|
| 28 |
+
force_rebuild_norm_stats: false
|
| 29 |
+
norm_stats_use_arrow_cache: true
|
| 30 |
+
norm_stats_bulk_read_arrow: true
|
| 31 |
+
norm_stats_cache_enabled: true
|
| 32 |
+
norm_stats_cache_dir: null
|
| 33 |
+
precache_profile: false
|
| 34 |
+
precache_profile_min_sec: 0.0
|
| 35 |
+
gradient_accumulation_steps: 8
|
| 36 |
+
mixed_precision: bf16
|
| 37 |
+
seed: 42
|
| 38 |
+
max_grad_norm: 1.0
|
| 39 |
+
weight_decay: 0.01
|
| 40 |
+
resume: /content/PARC2026_final/.runtime/lit-resolution-resume-2000/checkpoints/step_002000
|
| 41 |
+
stage1_checkpoint: null
|
| 42 |
+
wandb:
|
| 43 |
+
enabled: true
|
| 44 |
+
workspace: null
|
| 45 |
+
project: parc2026-lit-imagewam
|
| 46 |
+
name: lit-stage2-resadapt-128to224-lora-r16
|
| 47 |
+
group: lit-stage2-resadapt-128to224
|
| 48 |
+
mode: online
|
| 49 |
+
data:
|
| 50 |
+
train:
|
| 51 |
+
_target_: imagewam.datasets.lerobot.robot_video_dataset.RobotVideoDataset
|
| 52 |
+
dataset_dirs:
|
| 53 |
+
- /tmp/resadapt-dataset
|
| 54 |
+
shape_meta:
|
| 55 |
+
images:
|
| 56 |
+
- key: image
|
| 57 |
+
raw_shape:
|
| 58 |
+
- 3
|
| 59 |
+
- 256
|
| 60 |
+
- 256
|
| 61 |
+
shape:
|
| 62 |
+
- 3
|
| 63 |
+
- 224
|
| 64 |
+
- 224
|
| 65 |
+
lerobot_key: observation.images.front
|
| 66 |
+
- key: wrist_image
|
| 67 |
+
raw_shape:
|
| 68 |
+
- 3
|
| 69 |
+
- 256
|
| 70 |
+
- 256
|
| 71 |
+
shape:
|
| 72 |
+
- 3
|
| 73 |
+
- 224
|
| 74 |
+
- 224
|
| 75 |
+
lerobot_key: observation.images.wrist
|
| 76 |
+
action:
|
| 77 |
+
- key: default
|
| 78 |
+
raw_shape: 7
|
| 79 |
+
shape: 7
|
| 80 |
+
state:
|
| 81 |
+
- key: default
|
| 82 |
+
raw_shape: 8
|
| 83 |
+
shape: 8
|
| 84 |
+
num_frames: 17
|
| 85 |
+
global_sample_stride: 1
|
| 86 |
+
action_video_freq_ratio: 1
|
| 87 |
+
video_size:
|
| 88 |
+
- 224
|
| 89 |
+
- 448
|
| 90 |
+
camera_key: null
|
| 91 |
+
val_set_proportion: 0.0
|
| 92 |
+
is_training_set: true
|
| 93 |
+
skip_padding_as_possible: false
|
| 94 |
+
concat_multi_camera: horizontal
|
| 95 |
+
video_augmentation:
|
| 96 |
+
_target_: imagewam.datasets.lerobot.transforms.image.VideoAugmentation
|
| 97 |
+
p: 0.65
|
| 98 |
+
augment_types:
|
| 99 |
+
- corrupt_only
|
| 100 |
+
- color_only
|
| 101 |
+
- both
|
| 102 |
+
- both
|
| 103 |
+
color_jitter:
|
| 104 |
+
brightness: 0.3
|
| 105 |
+
contrast: 0.3
|
| 106 |
+
saturation: 0.25
|
| 107 |
+
hue: 0.04
|
| 108 |
+
gamma:
|
| 109 |
+
range:
|
| 110 |
+
- 0.8
|
| 111 |
+
- 1.25
|
| 112 |
+
gaussian_noise:
|
| 113 |
+
std: 0.015
|
| 114 |
+
random_resized_crop: null
|
| 115 |
+
rotate: null
|
| 116 |
+
exposure:
|
| 117 |
+
ev_range:
|
| 118 |
+
- -0.25
|
| 119 |
+
- 0.25
|
| 120 |
+
require_text_cache: false
|
| 121 |
+
endpoint_frames_only: true
|
| 122 |
+
qwen_text_cache_dir: /tmp/resadapt-assets/qwen_cache
|
| 123 |
+
qwen_context_len: 512
|
| 124 |
+
pretrained_norm_stats: /tmp/resadapt-assets/released/imagewam/lit_stage2/dataset_stats.json
|
| 125 |
+
processor:
|
| 126 |
+
_target_: imagewam.datasets.lerobot.processors.imagewam_processor.ImageWAMProcessor
|
| 127 |
+
shape_meta:
|
| 128 |
+
images:
|
| 129 |
+
- key: image
|
| 130 |
+
raw_shape:
|
| 131 |
+
- 3
|
| 132 |
+
- 256
|
| 133 |
+
- 256
|
| 134 |
+
shape:
|
| 135 |
+
- 3
|
| 136 |
+
- 224
|
| 137 |
+
- 224
|
| 138 |
+
lerobot_key: observation.images.front
|
| 139 |
+
- key: wrist_image
|
| 140 |
+
raw_shape:
|
| 141 |
+
- 3
|
| 142 |
+
- 256
|
| 143 |
+
- 256
|
| 144 |
+
shape:
|
| 145 |
+
- 3
|
| 146 |
+
- 224
|
| 147 |
+
- 224
|
| 148 |
+
lerobot_key: observation.images.wrist
|
| 149 |
+
action:
|
| 150 |
+
- key: default
|
| 151 |
+
raw_shape: 7
|
| 152 |
+
shape: 7
|
| 153 |
+
state:
|
| 154 |
+
- key: default
|
| 155 |
+
raw_shape: 8
|
| 156 |
+
shape: 8
|
| 157 |
+
num_obs_steps: 17
|
| 158 |
+
image_obs_steps: 2
|
| 159 |
+
num_output_cameras: 2
|
| 160 |
+
action_output_dim: 7
|
| 161 |
+
proprio_output_dim: 8
|
| 162 |
+
delta_action_dim_mask:
|
| 163 |
+
default:
|
| 164 |
+
- true
|
| 165 |
+
- true
|
| 166 |
+
- true
|
| 167 |
+
- true
|
| 168 |
+
- true
|
| 169 |
+
- true
|
| 170 |
+
- false
|
| 171 |
+
action_state_transforms:
|
| 172 |
+
- _target_: training_support.DatasetGripperToImageWAM
|
| 173 |
+
use_stepwise_action_norm: false
|
| 174 |
+
norm_default_mode: min/max
|
| 175 |
+
norm_exception_mode: null
|
| 176 |
+
action_state_merger:
|
| 177 |
+
_target_: imagewam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign
|
| 178 |
+
train_transforms:
|
| 179 |
+
- _target_: imagewam.datasets.lerobot.transforms.image.ToTensor
|
| 180 |
+
- _target_: resolution_adapt.data.ResolutionBottleneck
|
| 181 |
+
val_transforms:
|
| 182 |
+
- _target_: imagewam.datasets.lerobot.transforms.image.ToTensor
|
| 183 |
+
- _target_: resolution_adapt.data.ResolutionBottleneck
|
| 184 |
+
text_embedding_cache_dir: null
|
| 185 |
+
context_len: 128
|
| 186 |
+
qwen_text_cache_format: qwen3_flux2
|
| 187 |
+
lerobot_backend: v3
|
| 188 |
+
lerobot_v3_video_backend: pyav
|
| 189 |
+
lerobot_tolerance_s: 0.0001
|
| 190 |
+
vision_free: false
|
| 191 |
+
model:
|
| 192 |
+
_target_: imagewam.runtime.create_imagewam_flux2_klein
|
| 193 |
+
flux2_src_path: /tmp/resadapt-flux2
|
| 194 |
+
flux2_model_path: null
|
| 195 |
+
ae_model_path: /tmp/resadapt-assets/vae/ae.safetensors
|
| 196 |
+
variant: klein-base-4b
|
| 197 |
+
qwen3_model_spec: null
|
| 198 |
+
qwen_context_len: 512
|
| 199 |
+
load_text_encoder: false
|
| 200 |
+
proprio_dim: 8
|
| 201 |
+
mot_checkpoint_mixed_attn: true
|
| 202 |
+
mot_gqa_implementation: repeat
|
| 203 |
+
mot_force_flash_attention: false
|
| 204 |
+
pack_proprio_after_text: true
|
| 205 |
+
action_dit_pretrained_path: null
|
| 206 |
+
goal_prior_stage: stage2
|
| 207 |
+
goal_prior:
|
| 208 |
+
num_latents: 100
|
| 209 |
+
num_pose_tokens: 8
|
| 210 |
+
num_layer_groups: 5
|
| 211 |
+
latent_dim: 768
|
| 212 |
+
inner_dim: 512
|
| 213 |
+
lambda_pose: 0.3
|
| 214 |
+
context_token_dropout: 0.05
|
| 215 |
+
context_blackout_prob: 0.1
|
| 216 |
+
zero_init_value: false
|
| 217 |
+
syn_gate_bias_init: -2.0
|
| 218 |
+
gate_pose_tokens: false
|
| 219 |
+
latent_layout: free
|
| 220 |
+
grid_hw:
|
| 221 |
+
- 7
|
| 222 |
+
- 7
|
| 223 |
+
grid_window: 1
|
| 224 |
+
action_sees_ref: true
|
| 225 |
+
p_both: 0.55
|
| 226 |
+
p_ref_only: 0.15
|
| 227 |
+
p_syn_only: 0.3
|
| 228 |
+
flux2_lora_config:
|
| 229 |
+
enabled: true
|
| 230 |
+
rank: 16
|
| 231 |
+
alpha: 16.0
|
| 232 |
+
dropout: 0.0
|
| 233 |
+
save_lora_merged: false
|
| 234 |
+
save_trainable_only: true
|
| 235 |
+
target_suffixes:
|
| 236 |
+
- qkv
|
| 237 |
+
- proj
|
| 238 |
+
- linear1
|
| 239 |
+
- linear2
|
| 240 |
+
- img_mlp.0
|
| 241 |
+
- img_mlp.2
|
| 242 |
+
- txt_mlp.0
|
| 243 |
+
- txt_mlp.2
|
| 244 |
+
action_dit_config:
|
| 245 |
+
action_dim: 7
|
| 246 |
+
hidden_dim: 1024
|
| 247 |
+
num_heads: 24
|
| 248 |
+
attn_head_dim: 128
|
| 249 |
+
num_layers_double: 5
|
| 250 |
+
num_layers_single: 20
|
| 251 |
+
mlp_ratio: 4.0
|
| 252 |
+
max_action_horizon: 64
|
| 253 |
+
use_gradient_checkpointing: true
|
| 254 |
+
video_scheduler:
|
| 255 |
+
train_shift: 5.0
|
| 256 |
+
infer_shift: 5.0
|
| 257 |
+
num_train_timesteps: 1000
|
| 258 |
+
action_scheduler:
|
| 259 |
+
train_shift: 5.0
|
| 260 |
+
infer_shift: 5.0
|
| 261 |
+
num_train_timesteps: 1000
|
| 262 |
+
loss:
|
| 263 |
+
lambda_video: 0.5
|
| 264 |
+
lambda_action: 1.0
|
| 265 |
+
lambda_pose: 0.3
|
| 266 |
+
initialization:
|
| 267 |
+
repo: linjianman/LIT
|
| 268 |
+
revision: 765a2b2bf136f7039ab18978920c682f1aae65d2
|
| 269 |
+
path: imagewam/lit_stage2/model.pt
|
| 270 |
+
local_path: /tmp/resadapt-assets/released/imagewam/lit_stage2/model.pt
|
checkpoints/step_004000/metadata/dataset_stats.json
ADDED
|
@@ -0,0 +1,1074 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"state": {
|
| 3 |
+
"default": {
|
| 4 |
+
"stepwise_min": [
|
| 5 |
+
[
|
| 6 |
+
-0.48278069496154785,
|
| 7 |
+
-0.3309336006641388,
|
| 8 |
+
0.008128181099891663,
|
| 9 |
+
1.002794623374939,
|
| 10 |
+
-3.6312508583068848,
|
| 11 |
+
-1.842738389968872,
|
| 12 |
+
-0.005453015677630901,
|
| 13 |
+
-0.042015016078948975
|
| 14 |
+
]
|
| 15 |
+
],
|
| 16 |
+
"stepwise_max": [
|
| 17 |
+
[
|
| 18 |
+
0.2103137969970703,
|
| 19 |
+
0.3904264271259308,
|
| 20 |
+
1.472778081893921,
|
| 21 |
+
3.7248642444610596,
|
| 22 |
+
3.5618896484375,
|
| 23 |
+
1.3863215446472168,
|
| 24 |
+
0.04232141748070717,
|
| 25 |
+
0.0013126095291227102
|
| 26 |
+
]
|
| 27 |
+
],
|
| 28 |
+
"global_min": [
|
| 29 |
+
-0.48278069496154785,
|
| 30 |
+
-0.3309336006641388,
|
| 31 |
+
0.008128181099891663,
|
| 32 |
+
1.002794623374939,
|
| 33 |
+
-3.6312508583068848,
|
| 34 |
+
-1.842738389968872,
|
| 35 |
+
-0.005453015677630901,
|
| 36 |
+
-0.042015016078948975
|
| 37 |
+
],
|
| 38 |
+
"global_max": [
|
| 39 |
+
0.2103137969970703,
|
| 40 |
+
0.3904264271259308,
|
| 41 |
+
1.472778081893921,
|
| 42 |
+
3.7248642444610596,
|
| 43 |
+
3.5618896484375,
|
| 44 |
+
1.3863215446472168,
|
| 45 |
+
0.04232141748070717,
|
| 46 |
+
0.0013126095291227102
|
| 47 |
+
],
|
| 48 |
+
"stepwise_q01": [
|
| 49 |
+
[
|
| 50 |
+
-0.48195967078208923,
|
| 51 |
+
-0.3300417959690094,
|
| 52 |
+
0.008850866928696632,
|
| 53 |
+
1.0129998922348022,
|
| 54 |
+
-3.619534969329834,
|
| 55 |
+
-1.8389623165130615,
|
| 56 |
+
-0.0013291656505316496,
|
| 57 |
+
-0.04173403978347778
|
| 58 |
+
]
|
| 59 |
+
],
|
| 60 |
+
"stepwise_q99": [
|
| 61 |
+
[
|
| 62 |
+
0.20942535996437073,
|
| 63 |
+
0.39041632413864136,
|
| 64 |
+
1.4693706035614014,
|
| 65 |
+
3.721859931945801,
|
| 66 |
+
3.5605900287628174,
|
| 67 |
+
1.3660067319869995,
|
| 68 |
+
0.04197758436203003,
|
| 69 |
+
0.0009995142463594675
|
| 70 |
+
]
|
| 71 |
+
],
|
| 72 |
+
"global_q01": [
|
| 73 |
+
-0.48195967078208923,
|
| 74 |
+
-0.3300417959690094,
|
| 75 |
+
0.008850866928696632,
|
| 76 |
+
1.0129998922348022,
|
| 77 |
+
-3.619534969329834,
|
| 78 |
+
-1.8389623165130615,
|
| 79 |
+
-0.0013291656505316496,
|
| 80 |
+
-0.04173403978347778
|
| 81 |
+
],
|
| 82 |
+
"global_q99": [
|
| 83 |
+
0.20942535996437073,
|
| 84 |
+
0.39041632413864136,
|
| 85 |
+
1.4693706035614014,
|
| 86 |
+
3.721859931945801,
|
| 87 |
+
3.5605900287628174,
|
| 88 |
+
1.3660067319869995,
|
| 89 |
+
0.04197758436203003,
|
| 90 |
+
0.0009995142463594675
|
| 91 |
+
],
|
| 92 |
+
"stepwise_mean": [
|
| 93 |
+
[
|
| 94 |
+
-0.052439481019973755,
|
| 95 |
+
0.040308527648448944,
|
| 96 |
+
0.7768229842185974,
|
| 97 |
+
2.996931314468384,
|
| 98 |
+
-0.13651959598064423,
|
| 99 |
+
-0.11046145856380463,
|
| 100 |
+
0.026323523372411728,
|
| 101 |
+
-0.026523755863308907
|
| 102 |
+
]
|
| 103 |
+
],
|
| 104 |
+
"stepwise_std": [
|
| 105 |
+
[
|
| 106 |
+
0.10999669134616852,
|
| 107 |
+
0.15002697706222534,
|
| 108 |
+
0.38846009969711304,
|
| 109 |
+
0.3232516348361969,
|
| 110 |
+
0.7607998847961426,
|
| 111 |
+
0.3058409094810486,
|
| 112 |
+
0.014585155062377453,
|
| 113 |
+
0.014462423510849476
|
| 114 |
+
]
|
| 115 |
+
],
|
| 116 |
+
"global_mean": [
|
| 117 |
+
-0.052439481019973755,
|
| 118 |
+
0.040308527648448944,
|
| 119 |
+
0.7768229842185974,
|
| 120 |
+
2.996931314468384,
|
| 121 |
+
-0.13651959598064423,
|
| 122 |
+
-0.11046145856380463,
|
| 123 |
+
0.026323523372411728,
|
| 124 |
+
-0.026523755863308907
|
| 125 |
+
],
|
| 126 |
+
"global_std": [
|
| 127 |
+
0.10999669134616852,
|
| 128 |
+
0.15002697706222534,
|
| 129 |
+
0.38846009969711304,
|
| 130 |
+
0.3232516348361969,
|
| 131 |
+
0.7607998847961426,
|
| 132 |
+
0.3058409094810486,
|
| 133 |
+
0.014585155062377453,
|
| 134 |
+
0.014462423510849476
|
| 135 |
+
]
|
| 136 |
+
}
|
| 137 |
+
},
|
| 138 |
+
"action": {
|
| 139 |
+
"default": {
|
| 140 |
+
"stepwise_min": [
|
| 141 |
+
[
|
| 142 |
+
-0.9375,
|
| 143 |
+
-0.9375,
|
| 144 |
+
-0.9375,
|
| 145 |
+
-0.24214285612106323,
|
| 146 |
+
-0.375,
|
| 147 |
+
-0.3642857074737549,
|
| 148 |
+
0.0
|
| 149 |
+
],
|
| 150 |
+
[
|
| 151 |
+
-0.9375,
|
| 152 |
+
-0.9375,
|
| 153 |
+
-0.9375,
|
| 154 |
+
-0.24214285612106323,
|
| 155 |
+
-0.375,
|
| 156 |
+
-0.3642857074737549,
|
| 157 |
+
0.0
|
| 158 |
+
],
|
| 159 |
+
[
|
| 160 |
+
-0.9375,
|
| 161 |
+
-0.9375,
|
| 162 |
+
-0.9375,
|
| 163 |
+
-0.24214285612106323,
|
| 164 |
+
-0.375,
|
| 165 |
+
-0.3642857074737549,
|
| 166 |
+
0.0
|
| 167 |
+
],
|
| 168 |
+
[
|
| 169 |
+
-0.9375,
|
| 170 |
+
-0.9375,
|
| 171 |
+
-0.9375,
|
| 172 |
+
-0.24214285612106323,
|
| 173 |
+
-0.375,
|
| 174 |
+
-0.3642857074737549,
|
| 175 |
+
0.0
|
| 176 |
+
],
|
| 177 |
+
[
|
| 178 |
+
-0.9375,
|
| 179 |
+
-0.9375,
|
| 180 |
+
-0.9375,
|
| 181 |
+
-0.24214285612106323,
|
| 182 |
+
-0.375,
|
| 183 |
+
-0.3642857074737549,
|
| 184 |
+
0.0
|
| 185 |
+
],
|
| 186 |
+
[
|
| 187 |
+
-0.9375,
|
| 188 |
+
-0.9375,
|
| 189 |
+
-0.9375,
|
| 190 |
+
-0.24214285612106323,
|
| 191 |
+
-0.375,
|
| 192 |
+
-0.3642857074737549,
|
| 193 |
+
0.0
|
| 194 |
+
],
|
| 195 |
+
[
|
| 196 |
+
-0.9375,
|
| 197 |
+
-0.9375,
|
| 198 |
+
-0.9375,
|
| 199 |
+
-0.24214285612106323,
|
| 200 |
+
-0.375,
|
| 201 |
+
-0.3642857074737549,
|
| 202 |
+
0.0
|
| 203 |
+
],
|
| 204 |
+
[
|
| 205 |
+
-0.9375,
|
| 206 |
+
-0.9375,
|
| 207 |
+
-0.9375,
|
| 208 |
+
-0.24214285612106323,
|
| 209 |
+
-0.375,
|
| 210 |
+
-0.3642857074737549,
|
| 211 |
+
0.0
|
| 212 |
+
],
|
| 213 |
+
[
|
| 214 |
+
-0.9375,
|
| 215 |
+
-0.9375,
|
| 216 |
+
-0.9375,
|
| 217 |
+
-0.24214285612106323,
|
| 218 |
+
-0.375,
|
| 219 |
+
-0.3642857074737549,
|
| 220 |
+
0.0
|
| 221 |
+
],
|
| 222 |
+
[
|
| 223 |
+
-0.9375,
|
| 224 |
+
-0.9375,
|
| 225 |
+
-0.9375,
|
| 226 |
+
-0.24214285612106323,
|
| 227 |
+
-0.375,
|
| 228 |
+
-0.3642857074737549,
|
| 229 |
+
0.0
|
| 230 |
+
],
|
| 231 |
+
[
|
| 232 |
+
-0.9375,
|
| 233 |
+
-0.9375,
|
| 234 |
+
-0.9375,
|
| 235 |
+
-0.24214285612106323,
|
| 236 |
+
-0.375,
|
| 237 |
+
-0.3642857074737549,
|
| 238 |
+
0.0
|
| 239 |
+
],
|
| 240 |
+
[
|
| 241 |
+
-0.9375,
|
| 242 |
+
-0.9375,
|
| 243 |
+
-0.9375,
|
| 244 |
+
-0.24214285612106323,
|
| 245 |
+
-0.375,
|
| 246 |
+
-0.3642857074737549,
|
| 247 |
+
0.0
|
| 248 |
+
],
|
| 249 |
+
[
|
| 250 |
+
-0.9375,
|
| 251 |
+
-0.9375,
|
| 252 |
+
-0.9375,
|
| 253 |
+
-0.24214285612106323,
|
| 254 |
+
-0.375,
|
| 255 |
+
-0.3642857074737549,
|
| 256 |
+
0.0
|
| 257 |
+
],
|
| 258 |
+
[
|
| 259 |
+
-0.9375,
|
| 260 |
+
-0.9375,
|
| 261 |
+
-0.9375,
|
| 262 |
+
-0.24214285612106323,
|
| 263 |
+
-0.375,
|
| 264 |
+
-0.3642857074737549,
|
| 265 |
+
0.0
|
| 266 |
+
],
|
| 267 |
+
[
|
| 268 |
+
-0.9375,
|
| 269 |
+
-0.9375,
|
| 270 |
+
-0.9375,
|
| 271 |
+
-0.24214285612106323,
|
| 272 |
+
-0.375,
|
| 273 |
+
-0.3642857074737549,
|
| 274 |
+
0.0
|
| 275 |
+
],
|
| 276 |
+
[
|
| 277 |
+
-0.9375,
|
| 278 |
+
-0.9375,
|
| 279 |
+
-0.9375,
|
| 280 |
+
-0.23678570985794067,
|
| 281 |
+
-0.375,
|
| 282 |
+
-0.3642857074737549,
|
| 283 |
+
0.0
|
| 284 |
+
]
|
| 285 |
+
],
|
| 286 |
+
"stepwise_max": [
|
| 287 |
+
[
|
| 288 |
+
0.9375,
|
| 289 |
+
0.9375,
|
| 290 |
+
0.9375,
|
| 291 |
+
0.3557142913341522,
|
| 292 |
+
0.375,
|
| 293 |
+
0.375,
|
| 294 |
+
1.0
|
| 295 |
+
],
|
| 296 |
+
[
|
| 297 |
+
0.9375,
|
| 298 |
+
0.9375,
|
| 299 |
+
0.9375,
|
| 300 |
+
0.3557142913341522,
|
| 301 |
+
0.375,
|
| 302 |
+
0.375,
|
| 303 |
+
1.0
|
| 304 |
+
],
|
| 305 |
+
[
|
| 306 |
+
0.9375,
|
| 307 |
+
0.9375,
|
| 308 |
+
0.9375,
|
| 309 |
+
0.3557142913341522,
|
| 310 |
+
0.375,
|
| 311 |
+
0.375,
|
| 312 |
+
1.0
|
| 313 |
+
],
|
| 314 |
+
[
|
| 315 |
+
0.9375,
|
| 316 |
+
0.9375,
|
| 317 |
+
0.9375,
|
| 318 |
+
0.3557142913341522,
|
| 319 |
+
0.375,
|
| 320 |
+
0.375,
|
| 321 |
+
1.0
|
| 322 |
+
],
|
| 323 |
+
[
|
| 324 |
+
0.9375,
|
| 325 |
+
0.9375,
|
| 326 |
+
0.9375,
|
| 327 |
+
0.3557142913341522,
|
| 328 |
+
0.375,
|
| 329 |
+
0.375,
|
| 330 |
+
1.0
|
| 331 |
+
],
|
| 332 |
+
[
|
| 333 |
+
0.9375,
|
| 334 |
+
0.9375,
|
| 335 |
+
0.9375,
|
| 336 |
+
0.3557142913341522,
|
| 337 |
+
0.375,
|
| 338 |
+
0.375,
|
| 339 |
+
1.0
|
| 340 |
+
],
|
| 341 |
+
[
|
| 342 |
+
0.9375,
|
| 343 |
+
0.9375,
|
| 344 |
+
0.9375,
|
| 345 |
+
0.3557142913341522,
|
| 346 |
+
0.375,
|
| 347 |
+
0.375,
|
| 348 |
+
1.0
|
| 349 |
+
],
|
| 350 |
+
[
|
| 351 |
+
0.9375,
|
| 352 |
+
0.9375,
|
| 353 |
+
0.9375,
|
| 354 |
+
0.3557142913341522,
|
| 355 |
+
0.375,
|
| 356 |
+
0.375,
|
| 357 |
+
1.0
|
| 358 |
+
],
|
| 359 |
+
[
|
| 360 |
+
0.9375,
|
| 361 |
+
0.9375,
|
| 362 |
+
0.9375,
|
| 363 |
+
0.3557142913341522,
|
| 364 |
+
0.375,
|
| 365 |
+
0.375,
|
| 366 |
+
1.0
|
| 367 |
+
],
|
| 368 |
+
[
|
| 369 |
+
0.9375,
|
| 370 |
+
0.9375,
|
| 371 |
+
0.9375,
|
| 372 |
+
0.3557142913341522,
|
| 373 |
+
0.375,
|
| 374 |
+
0.375,
|
| 375 |
+
1.0
|
| 376 |
+
],
|
| 377 |
+
[
|
| 378 |
+
0.9375,
|
| 379 |
+
0.9375,
|
| 380 |
+
0.9375,
|
| 381 |
+
0.3557142913341522,
|
| 382 |
+
0.375,
|
| 383 |
+
0.375,
|
| 384 |
+
1.0
|
| 385 |
+
],
|
| 386 |
+
[
|
| 387 |
+
0.9375,
|
| 388 |
+
0.9375,
|
| 389 |
+
0.9375,
|
| 390 |
+
0.3557142913341522,
|
| 391 |
+
0.375,
|
| 392 |
+
0.375,
|
| 393 |
+
1.0
|
| 394 |
+
],
|
| 395 |
+
[
|
| 396 |
+
0.9375,
|
| 397 |
+
0.9375,
|
| 398 |
+
0.9375,
|
| 399 |
+
0.3557142913341522,
|
| 400 |
+
0.375,
|
| 401 |
+
0.375,
|
| 402 |
+
1.0
|
| 403 |
+
],
|
| 404 |
+
[
|
| 405 |
+
0.9375,
|
| 406 |
+
0.9375,
|
| 407 |
+
0.9375,
|
| 408 |
+
0.3557142913341522,
|
| 409 |
+
0.375,
|
| 410 |
+
0.375,
|
| 411 |
+
1.0
|
| 412 |
+
],
|
| 413 |
+
[
|
| 414 |
+
0.9375,
|
| 415 |
+
0.9375,
|
| 416 |
+
0.9375,
|
| 417 |
+
0.3557142913341522,
|
| 418 |
+
0.375,
|
| 419 |
+
0.375,
|
| 420 |
+
1.0
|
| 421 |
+
],
|
| 422 |
+
[
|
| 423 |
+
0.9375,
|
| 424 |
+
0.9375,
|
| 425 |
+
0.9375,
|
| 426 |
+
0.3557142913341522,
|
| 427 |
+
0.375,
|
| 428 |
+
0.375,
|
| 429 |
+
1.0
|
| 430 |
+
]
|
| 431 |
+
],
|
| 432 |
+
"global_min": [
|
| 433 |
+
-0.9375,
|
| 434 |
+
-0.9375,
|
| 435 |
+
-0.9375,
|
| 436 |
+
-0.24214285612106323,
|
| 437 |
+
-0.375,
|
| 438 |
+
-0.3642857074737549,
|
| 439 |
+
0.0
|
| 440 |
+
],
|
| 441 |
+
"global_max": [
|
| 442 |
+
0.9375,
|
| 443 |
+
0.9375,
|
| 444 |
+
0.9375,
|
| 445 |
+
0.3557142913341522,
|
| 446 |
+
0.375,
|
| 447 |
+
0.375,
|
| 448 |
+
1.0
|
| 449 |
+
],
|
| 450 |
+
"stepwise_q01": [
|
| 451 |
+
[
|
| 452 |
+
-0.9375,
|
| 453 |
+
-0.9375,
|
| 454 |
+
-0.9375,
|
| 455 |
+
-0.2326499968767166,
|
| 456 |
+
-0.375,
|
| 457 |
+
-0.3642857074737549,
|
| 458 |
+
0.0
|
| 459 |
+
],
|
| 460 |
+
[
|
| 461 |
+
-0.9375,
|
| 462 |
+
-0.9375,
|
| 463 |
+
-0.9375,
|
| 464 |
+
-0.2326499968767166,
|
| 465 |
+
-0.375,
|
| 466 |
+
-0.3642857074737549,
|
| 467 |
+
0.0
|
| 468 |
+
],
|
| 469 |
+
[
|
| 470 |
+
-0.9375,
|
| 471 |
+
-0.9375,
|
| 472 |
+
-0.9375,
|
| 473 |
+
-0.2326499968767166,
|
| 474 |
+
-0.375,
|
| 475 |
+
-0.3642857074737549,
|
| 476 |
+
0.0
|
| 477 |
+
],
|
| 478 |
+
[
|
| 479 |
+
-0.9375,
|
| 480 |
+
-0.9375,
|
| 481 |
+
-0.9375,
|
| 482 |
+
-0.2326499968767166,
|
| 483 |
+
-0.375,
|
| 484 |
+
-0.3642857074737549,
|
| 485 |
+
0.0
|
| 486 |
+
],
|
| 487 |
+
[
|
| 488 |
+
-0.9375,
|
| 489 |
+
-0.9375,
|
| 490 |
+
-0.9375,
|
| 491 |
+
-0.2326499968767166,
|
| 492 |
+
-0.375,
|
| 493 |
+
-0.3642857074737549,
|
| 494 |
+
0.0
|
| 495 |
+
],
|
| 496 |
+
[
|
| 497 |
+
-0.9375,
|
| 498 |
+
-0.9375,
|
| 499 |
+
-0.9375,
|
| 500 |
+
-0.2326499968767166,
|
| 501 |
+
-0.375,
|
| 502 |
+
-0.3642857074737549,
|
| 503 |
+
0.0
|
| 504 |
+
],
|
| 505 |
+
[
|
| 506 |
+
-0.9375,
|
| 507 |
+
-0.9375,
|
| 508 |
+
-0.9375,
|
| 509 |
+
-0.2326499968767166,
|
| 510 |
+
-0.375,
|
| 511 |
+
-0.3642857074737549,
|
| 512 |
+
0.0
|
| 513 |
+
],
|
| 514 |
+
[
|
| 515 |
+
-0.9375,
|
| 516 |
+
-0.9375,
|
| 517 |
+
-0.9375,
|
| 518 |
+
-0.2326499968767166,
|
| 519 |
+
-0.375,
|
| 520 |
+
-0.3642857074737549,
|
| 521 |
+
0.0
|
| 522 |
+
],
|
| 523 |
+
[
|
| 524 |
+
-0.9375,
|
| 525 |
+
-0.9375,
|
| 526 |
+
-0.9375,
|
| 527 |
+
-0.2326499968767166,
|
| 528 |
+
-0.375,
|
| 529 |
+
-0.3642857074737549,
|
| 530 |
+
0.0
|
| 531 |
+
],
|
| 532 |
+
[
|
| 533 |
+
-0.9375,
|
| 534 |
+
-0.9375,
|
| 535 |
+
-0.9375,
|
| 536 |
+
-0.2326499968767166,
|
| 537 |
+
-0.375,
|
| 538 |
+
-0.3642857074737549,
|
| 539 |
+
0.0
|
| 540 |
+
],
|
| 541 |
+
[
|
| 542 |
+
-0.9375,
|
| 543 |
+
-0.9375,
|
| 544 |
+
-0.9375,
|
| 545 |
+
-0.2326499968767166,
|
| 546 |
+
-0.375,
|
| 547 |
+
-0.3642857074737549,
|
| 548 |
+
0.0
|
| 549 |
+
],
|
| 550 |
+
[
|
| 551 |
+
-0.9375,
|
| 552 |
+
-0.9375,
|
| 553 |
+
-0.9375,
|
| 554 |
+
-0.2326499968767166,
|
| 555 |
+
-0.375,
|
| 556 |
+
-0.3642857074737549,
|
| 557 |
+
0.0
|
| 558 |
+
],
|
| 559 |
+
[
|
| 560 |
+
-0.9375,
|
| 561 |
+
-0.9375,
|
| 562 |
+
-0.9375,
|
| 563 |
+
-0.2326499968767166,
|
| 564 |
+
-0.375,
|
| 565 |
+
-0.3642857074737549,
|
| 566 |
+
0.0
|
| 567 |
+
],
|
| 568 |
+
[
|
| 569 |
+
-0.9375,
|
| 570 |
+
-0.9375,
|
| 571 |
+
-0.9375,
|
| 572 |
+
-0.2326499968767166,
|
| 573 |
+
-0.375,
|
| 574 |
+
-0.3642857074737549,
|
| 575 |
+
0.0
|
| 576 |
+
],
|
| 577 |
+
[
|
| 578 |
+
-0.9375,
|
| 579 |
+
-0.9375,
|
| 580 |
+
-0.9375,
|
| 581 |
+
-0.2326499968767166,
|
| 582 |
+
-0.375,
|
| 583 |
+
-0.3642857074737549,
|
| 584 |
+
0.0
|
| 585 |
+
],
|
| 586 |
+
[
|
| 587 |
+
-0.9375,
|
| 588 |
+
-0.9375,
|
| 589 |
+
-0.9375,
|
| 590 |
+
-0.2326499968767166,
|
| 591 |
+
-0.375,
|
| 592 |
+
-0.3642857074737549,
|
| 593 |
+
0.0
|
| 594 |
+
]
|
| 595 |
+
],
|
| 596 |
+
"stepwise_q99": [
|
| 597 |
+
[
|
| 598 |
+
0.9375,
|
| 599 |
+
0.9375,
|
| 600 |
+
0.9375,
|
| 601 |
+
0.35076430439949036,
|
| 602 |
+
0.375,
|
| 603 |
+
0.375,
|
| 604 |
+
1.0
|
| 605 |
+
],
|
| 606 |
+
[
|
| 607 |
+
0.9375,
|
| 608 |
+
0.9375,
|
| 609 |
+
0.9375,
|
| 610 |
+
0.35076430439949036,
|
| 611 |
+
0.375,
|
| 612 |
+
0.375,
|
| 613 |
+
1.0
|
| 614 |
+
],
|
| 615 |
+
[
|
| 616 |
+
0.9375,
|
| 617 |
+
0.9375,
|
| 618 |
+
0.9375,
|
| 619 |
+
0.35076430439949036,
|
| 620 |
+
0.375,
|
| 621 |
+
0.375,
|
| 622 |
+
1.0
|
| 623 |
+
],
|
| 624 |
+
[
|
| 625 |
+
0.9375,
|
| 626 |
+
0.9375,
|
| 627 |
+
0.9375,
|
| 628 |
+
0.35076430439949036,
|
| 629 |
+
0.375,
|
| 630 |
+
0.375,
|
| 631 |
+
1.0
|
| 632 |
+
],
|
| 633 |
+
[
|
| 634 |
+
0.9375,
|
| 635 |
+
0.9375,
|
| 636 |
+
0.9375,
|
| 637 |
+
0.35076430439949036,
|
| 638 |
+
0.375,
|
| 639 |
+
0.375,
|
| 640 |
+
1.0
|
| 641 |
+
],
|
| 642 |
+
[
|
| 643 |
+
0.9375,
|
| 644 |
+
0.9375,
|
| 645 |
+
0.9375,
|
| 646 |
+
0.35076430439949036,
|
| 647 |
+
0.375,
|
| 648 |
+
0.375,
|
| 649 |
+
1.0
|
| 650 |
+
],
|
| 651 |
+
[
|
| 652 |
+
0.9375,
|
| 653 |
+
0.9375,
|
| 654 |
+
0.9375,
|
| 655 |
+
0.35076430439949036,
|
| 656 |
+
0.375,
|
| 657 |
+
0.375,
|
| 658 |
+
1.0
|
| 659 |
+
],
|
| 660 |
+
[
|
| 661 |
+
0.9375,
|
| 662 |
+
0.9375,
|
| 663 |
+
0.9375,
|
| 664 |
+
0.35076430439949036,
|
| 665 |
+
0.375,
|
| 666 |
+
0.375,
|
| 667 |
+
1.0
|
| 668 |
+
],
|
| 669 |
+
[
|
| 670 |
+
0.9375,
|
| 671 |
+
0.9375,
|
| 672 |
+
0.9375,
|
| 673 |
+
0.35076430439949036,
|
| 674 |
+
0.375,
|
| 675 |
+
0.375,
|
| 676 |
+
1.0
|
| 677 |
+
],
|
| 678 |
+
[
|
| 679 |
+
0.9375,
|
| 680 |
+
0.9375,
|
| 681 |
+
0.9375,
|
| 682 |
+
0.35076430439949036,
|
| 683 |
+
0.375,
|
| 684 |
+
0.375,
|
| 685 |
+
1.0
|
| 686 |
+
],
|
| 687 |
+
[
|
| 688 |
+
0.9375,
|
| 689 |
+
0.9375,
|
| 690 |
+
0.9375,
|
| 691 |
+
0.35076430439949036,
|
| 692 |
+
0.375,
|
| 693 |
+
0.375,
|
| 694 |
+
1.0
|
| 695 |
+
],
|
| 696 |
+
[
|
| 697 |
+
0.9375,
|
| 698 |
+
0.9375,
|
| 699 |
+
0.9375,
|
| 700 |
+
0.35076430439949036,
|
| 701 |
+
0.375,
|
| 702 |
+
0.375,
|
| 703 |
+
1.0
|
| 704 |
+
],
|
| 705 |
+
[
|
| 706 |
+
0.9375,
|
| 707 |
+
0.9375,
|
| 708 |
+
0.9375,
|
| 709 |
+
0.35076430439949036,
|
| 710 |
+
0.375,
|
| 711 |
+
0.375,
|
| 712 |
+
1.0
|
| 713 |
+
],
|
| 714 |
+
[
|
| 715 |
+
0.9375,
|
| 716 |
+
0.9375,
|
| 717 |
+
0.9375,
|
| 718 |
+
0.35076430439949036,
|
| 719 |
+
0.375,
|
| 720 |
+
0.375,
|
| 721 |
+
1.0
|
| 722 |
+
],
|
| 723 |
+
[
|
| 724 |
+
0.9375,
|
| 725 |
+
0.9375,
|
| 726 |
+
0.9375,
|
| 727 |
+
0.35076430439949036,
|
| 728 |
+
0.375,
|
| 729 |
+
0.375,
|
| 730 |
+
1.0
|
| 731 |
+
],
|
| 732 |
+
[
|
| 733 |
+
0.9375,
|
| 734 |
+
0.9375,
|
| 735 |
+
0.9375,
|
| 736 |
+
0.35076430439949036,
|
| 737 |
+
0.375,
|
| 738 |
+
0.375,
|
| 739 |
+
1.0
|
| 740 |
+
]
|
| 741 |
+
],
|
| 742 |
+
"global_q01": [
|
| 743 |
+
-0.9375,
|
| 744 |
+
-0.9375,
|
| 745 |
+
-0.9375,
|
| 746 |
+
-0.2326499968767166,
|
| 747 |
+
-0.375,
|
| 748 |
+
-0.3642857074737549,
|
| 749 |
+
0.0
|
| 750 |
+
],
|
| 751 |
+
"global_q99": [
|
| 752 |
+
0.9375,
|
| 753 |
+
0.9375,
|
| 754 |
+
0.9375,
|
| 755 |
+
0.35076430439949036,
|
| 756 |
+
0.375,
|
| 757 |
+
0.375,
|
| 758 |
+
1.0
|
| 759 |
+
],
|
| 760 |
+
"stepwise_mean": [
|
| 761 |
+
[
|
| 762 |
+
0.07509180158376694,
|
| 763 |
+
0.0942603200674057,
|
| 764 |
+
-0.10477570444345474,
|
| 765 |
+
-0.0006721731624566019,
|
| 766 |
+
0.006502428557723761,
|
| 767 |
+
-0.004753417335450649,
|
| 768 |
+
0.5156609416007996
|
| 769 |
+
],
|
| 770 |
+
[
|
| 771 |
+
0.0742194727063179,
|
| 772 |
+
0.0943378284573555,
|
| 773 |
+
-0.10369047522544861,
|
| 774 |
+
-0.000646332511678338,
|
| 775 |
+
0.006517794448882341,
|
| 776 |
+
-0.0048326049000024796,
|
| 777 |
+
0.5151447653770447
|
| 778 |
+
],
|
| 779 |
+
[
|
| 780 |
+
0.07300770282745361,
|
| 781 |
+
0.09434358030557632,
|
| 782 |
+
-0.10265536606311798,
|
| 783 |
+
-0.000612989766523242,
|
| 784 |
+
0.006519764196127653,
|
| 785 |
+
-0.004911692347377539,
|
| 786 |
+
0.5146285891532898
|
| 787 |
+
],
|
| 788 |
+
[
|
| 789 |
+
0.07130849361419678,
|
| 790 |
+
0.09427326172590256,
|
| 791 |
+
-0.10167480260133743,
|
| 792 |
+
-0.0005704726208932698,
|
| 793 |
+
0.006502891890704632,
|
| 794 |
+
-0.004996591247618198,
|
| 795 |
+
0.5141124129295349
|
| 796 |
+
],
|
| 797 |
+
[
|
| 798 |
+
0.06903650611639023,
|
| 799 |
+
0.09413644671440125,
|
| 800 |
+
-0.10072868317365646,
|
| 801 |
+
-0.0005176868289709091,
|
| 802 |
+
0.006469315383583307,
|
| 803 |
+
-0.005088216625154018,
|
| 804 |
+
0.51359623670578
|
| 805 |
+
],
|
| 806 |
+
[
|
| 807 |
+
0.06624066829681396,
|
| 808 |
+
0.09398345649242401,
|
| 809 |
+
-0.09977640211582184,
|
| 810 |
+
-0.00045814085751771927,
|
| 811 |
+
0.006425761617720127,
|
| 812 |
+
-0.005182641092687845,
|
| 813 |
+
0.5130800008773804
|
| 814 |
+
],
|
| 815 |
+
[
|
| 816 |
+
0.0630466639995575,
|
| 817 |
+
0.09385841339826584,
|
| 818 |
+
-0.09878652542829514,
|
| 819 |
+
-0.0003933484840672463,
|
| 820 |
+
0.006386647932231426,
|
| 821 |
+
-0.0052768453024327755,
|
| 822 |
+
0.5125638246536255
|
| 823 |
+
],
|
| 824 |
+
[
|
| 825 |
+
0.059566862881183624,
|
| 826 |
+
0.0937969759106636,
|
| 827 |
+
-0.09773147106170654,
|
| 828 |
+
-0.0003290225868113339,
|
| 829 |
+
0.006361101288348436,
|
| 830 |
+
-0.005367835983633995,
|
| 831 |
+
0.5120476484298706
|
| 832 |
+
],
|
| 833 |
+
[
|
| 834 |
+
0.055894214659929276,
|
| 835 |
+
0.09381034225225449,
|
| 836 |
+
-0.09658003598451614,
|
| 837 |
+
-0.00026784592773765326,
|
| 838 |
+
0.006349042057991028,
|
| 839 |
+
-0.0054576038382947445,
|
| 840 |
+
0.5115315318107605
|
| 841 |
+
],
|
| 842 |
+
[
|
| 843 |
+
0.05210842192173004,
|
| 844 |
+
0.09390032291412354,
|
| 845 |
+
-0.09531887620687485,
|
| 846 |
+
-0.00021082810417283326,
|
| 847 |
+
0.00635516969487071,
|
| 848 |
+
-0.005544654093682766,
|
| 849 |
+
0.5110153555870056
|
| 850 |
+
],
|
| 851 |
+
[
|
| 852 |
+
0.04825857654213905,
|
| 853 |
+
0.09406367689371109,
|
| 854 |
+
-0.09393264353275299,
|
| 855 |
+
-0.00015976655413396657,
|
| 856 |
+
0.006377693731337786,
|
| 857 |
+
-0.005628861952573061,
|
| 858 |
+
0.5104991793632507
|
| 859 |
+
],
|
| 860 |
+
[
|
| 861 |
+
0.04438157379627228,
|
| 862 |
+
0.09429564327001572,
|
| 863 |
+
-0.0924036055803299,
|
| 864 |
+
-0.00011604859173530713,
|
| 865 |
+
0.006416680756956339,
|
| 866 |
+
-0.005709674209356308,
|
| 867 |
+
0.5099830031394958
|
| 868 |
+
],
|
| 869 |
+
[
|
| 870 |
+
0.040517039597034454,
|
| 871 |
+
0.09459187835454941,
|
| 872 |
+
-0.09071968495845795,
|
| 873 |
+
-8.002247341210023e-05,
|
| 874 |
+
0.006473013199865818,
|
| 875 |
+
-0.0057876212522387505,
|
| 876 |
+
0.5094667673110962
|
| 877 |
+
],
|
| 878 |
+
[
|
| 879 |
+
0.0366998165845871,
|
| 880 |
+
0.09495285153388977,
|
| 881 |
+
-0.08885672688484192,
|
| 882 |
+
-5.3317955462262034e-05,
|
| 883 |
+
0.00654600327834487,
|
| 884 |
+
-0.00586199015378952,
|
| 885 |
+
0.5089505910873413
|
| 886 |
+
],
|
| 887 |
+
[
|
| 888 |
+
0.0329657644033432,
|
| 889 |
+
0.09536924213171005,
|
| 890 |
+
-0.08679967373609543,
|
| 891 |
+
-3.667248893179931e-05,
|
| 892 |
+
0.006634271237999201,
|
| 893 |
+
-0.0059333425015211105,
|
| 894 |
+
0.5084344148635864
|
| 895 |
+
],
|
| 896 |
+
[
|
| 897 |
+
0.029352381825447083,
|
| 898 |
+
0.09583891928195953,
|
| 899 |
+
-0.08452724665403366,
|
| 900 |
+
-2.956177922897041e-05,
|
| 901 |
+
0.006736791227012873,
|
| 902 |
+
-0.006001695990562439,
|
| 903 |
+
0.5079182386398315
|
| 904 |
+
]
|
| 905 |
+
],
|
| 906 |
+
"stepwise_std": [
|
| 907 |
+
[
|
| 908 |
+
0.35524481534957886,
|
| 909 |
+
0.37755152583122253,
|
| 910 |
+
0.4687630534172058,
|
| 911 |
+
0.03886449709534645,
|
| 912 |
+
0.06502853333950043,
|
| 913 |
+
0.07652278244495392,
|
| 914 |
+
0.5014081597328186
|
| 915 |
+
],
|
| 916 |
+
[
|
| 917 |
+
0.35528722405433655,
|
| 918 |
+
0.37743693590164185,
|
| 919 |
+
0.46985360980033875,
|
| 920 |
+
0.03892502188682556,
|
| 921 |
+
0.06516389548778534,
|
| 922 |
+
0.07675494253635406,
|
| 923 |
+
0.5014236569404602
|
| 924 |
+
],
|
| 925 |
+
[
|
| 926 |
+
0.35516318678855896,
|
| 927 |
+
0.3772093653678894,
|
| 928 |
+
0.4709104001522064,
|
| 929 |
+
0.0389784574508667,
|
| 930 |
+
0.06527172774076462,
|
| 931 |
+
0.07698068022727966,
|
| 932 |
+
0.5014386177062988
|
| 933 |
+
],
|
| 934 |
+
[
|
| 935 |
+
0.354702353477478,
|
| 936 |
+
0.37677210569381714,
|
| 937 |
+
0.47192680835723877,
|
| 938 |
+
0.039021968841552734,
|
| 939 |
+
0.0653405413031578,
|
| 940 |
+
0.07719580829143524,
|
| 941 |
+
0.5014529824256897
|
| 942 |
+
],
|
| 943 |
+
[
|
| 944 |
+
0.35369622707366943,
|
| 945 |
+
0.37603849172592163,
|
| 946 |
+
0.47290918231010437,
|
| 947 |
+
0.039052922278642654,
|
| 948 |
+
0.06536615639925003,
|
| 949 |
+
0.07739786058664322,
|
| 950 |
+
0.5014666318893433
|
| 951 |
+
],
|
| 952 |
+
[
|
| 953 |
+
0.3520404100418091,
|
| 954 |
+
0.37497878074645996,
|
| 955 |
+
0.47387173771858215,
|
| 956 |
+
0.039072245359420776,
|
| 957 |
+
0.06535857915878296,
|
| 958 |
+
0.07758834213018417,
|
| 959 |
+
0.5014797449111938
|
| 960 |
+
],
|
| 961 |
+
[
|
| 962 |
+
0.34978508949279785,
|
| 963 |
+
0.3736053705215454,
|
| 964 |
+
0.4748258590698242,
|
| 965 |
+
0.039080556482076645,
|
| 966 |
+
0.06534324586391449,
|
| 967 |
+
0.07776463776826859,
|
| 968 |
+
0.5014922022819519
|
| 969 |
+
],
|
| 970 |
+
[
|
| 971 |
+
0.3470368981361389,
|
| 972 |
+
0.3719497621059418,
|
| 973 |
+
0.47577232122421265,
|
| 974 |
+
0.03907886520028114,
|
| 975 |
+
0.06533455848693848,
|
| 976 |
+
0.07792745530605316,
|
| 977 |
+
0.5015040040016174
|
| 978 |
+
],
|
| 979 |
+
[
|
| 980 |
+
0.34389936923980713,
|
| 981 |
+
0.37007132172584534,
|
| 982 |
+
0.4767095148563385,
|
| 983 |
+
0.039068553596735,
|
| 984 |
+
0.0653340294957161,
|
| 985 |
+
0.07807470113039017,
|
| 986 |
+
0.5015152096748352
|
| 987 |
+
],
|
| 988 |
+
[
|
| 989 |
+
0.34047427773475647,
|
| 990 |
+
0.3680059611797333,
|
| 991 |
+
0.4776283800601959,
|
| 992 |
+
0.039051979780197144,
|
| 993 |
+
0.06534522771835327,
|
| 994 |
+
0.0782046914100647,
|
| 995 |
+
0.5015258193016052
|
| 996 |
+
],
|
| 997 |
+
[
|
| 998 |
+
0.33682477474212646,
|
| 999 |
+
0.3657948970794678,
|
| 1000 |
+
0.4785217344760895,
|
| 1001 |
+
0.03903175890445709,
|
| 1002 |
+
0.06536738574504852,
|
| 1003 |
+
0.07831670343875885,
|
| 1004 |
+
0.5015357732772827
|
| 1005 |
+
],
|
| 1006 |
+
[
|
| 1007 |
+
0.33300837874412537,
|
| 1008 |
+
0.36347952485084534,
|
| 1009 |
+
0.4793775975704193,
|
| 1010 |
+
0.0390087328851223,
|
| 1011 |
+
0.06540162116289139,
|
| 1012 |
+
0.07841045409440994,
|
| 1013 |
+
0.5015451312065125
|
| 1014 |
+
],
|
| 1015 |
+
[
|
| 1016 |
+
0.3290861248970032,
|
| 1017 |
+
0.3610839545726776,
|
| 1018 |
+
0.48018527030944824,
|
| 1019 |
+
0.03898213058710098,
|
| 1020 |
+
0.06545032560825348,
|
| 1021 |
+
0.07848569005727768,
|
| 1022 |
+
0.5015538930892944
|
| 1023 |
+
],
|
| 1024 |
+
[
|
| 1025 |
+
0.3251110315322876,
|
| 1026 |
+
0.35864168405532837,
|
| 1027 |
+
0.48092642426490784,
|
| 1028 |
+
0.038955289870500565,
|
| 1029 |
+
0.06551124900579453,
|
| 1030 |
+
0.07854204624891281,
|
| 1031 |
+
0.5015619993209839
|
| 1032 |
+
],
|
| 1033 |
+
[
|
| 1034 |
+
0.3211345374584198,
|
| 1035 |
+
0.35618487000465393,
|
| 1036 |
+
0.4815789461135864,
|
| 1037 |
+
0.0389283113181591,
|
| 1038 |
+
0.06558437645435333,
|
| 1039 |
+
0.07858172059059143,
|
| 1040 |
+
0.5015695095062256
|
| 1041 |
+
],
|
| 1042 |
+
[
|
| 1043 |
+
0.31721803545951843,
|
| 1044 |
+
0.3537396192550659,
|
| 1045 |
+
0.48211759328842163,
|
| 1046 |
+
0.038900382816791534,
|
| 1047 |
+
0.06566666066646576,
|
| 1048 |
+
0.07860506325960159,
|
| 1049 |
+
0.5015763640403748
|
| 1050 |
+
]
|
| 1051 |
+
],
|
| 1052 |
+
"global_mean": [
|
| 1053 |
+
0.055730998516082764,
|
| 1054 |
+
0.09436333179473877,
|
| 1055 |
+
-0.0961848646402359,
|
| 1056 |
+
-0.0003221393853891641,
|
| 1057 |
+
0.0064733983017504215,
|
| 1058 |
+
-0.005395954940468073,
|
| 1059 |
+
0.5117895603179932
|
| 1060 |
+
],
|
| 1061 |
+
"global_std": [
|
| 1062 |
+
0.3424273431301117,
|
| 1063 |
+
0.3689939081668854,
|
| 1064 |
+
0.4760483205318451,
|
| 1065 |
+
0.03900081664323807,
|
| 1066 |
+
0.06536699831485748,
|
| 1067 |
+
0.0778384581208229,
|
| 1068 |
+
0.5015087723731995
|
| 1069 |
+
]
|
| 1070 |
+
}
|
| 1071 |
+
},
|
| 1072 |
+
"num_episodes": 1712,
|
| 1073 |
+
"num_transition": 277713
|
| 1074 |
+
}
|
checkpoints/step_004000/metadata/environment.txt
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Farama-Notifications==0.0.6
|
| 2 |
+
GitPython==3.2.0
|
| 3 |
+
Jinja2==3.1.6
|
| 4 |
+
MarkupSafe==3.0.4
|
| 5 |
+
PyYAML==6.0.3
|
| 6 |
+
Pygments==2.21.0
|
| 7 |
+
accelerate==1.12.0
|
| 8 |
+
aiohappyeyeballs==2.7.1
|
| 9 |
+
aiohttp==3.14.4
|
| 10 |
+
aiosignal==1.4.0
|
| 11 |
+
annotated-types==0.8.0
|
| 12 |
+
antlr4-python3-runtime==4.9.3
|
| 13 |
+
anyio==4.15.1
|
| 14 |
+
attrs==26.1.0
|
| 15 |
+
av==16.0.1
|
| 16 |
+
boto3==1.35.99
|
| 17 |
+
botocore==1.35.99
|
| 18 |
+
certifi==2026.7.22
|
| 19 |
+
charset-normalizer==3.5.2
|
| 20 |
+
click==8.5.0
|
| 21 |
+
cloudpickle==3.1.2
|
| 22 |
+
cuda-bindings==13.4.3
|
| 23 |
+
cuda-pathfinder==1.8.3
|
| 24 |
+
cuda-toolkit==13.0.2
|
| 25 |
+
datasets==4.8.5
|
| 26 |
+
diffusers==0.35.2
|
| 27 |
+
dill==0.4.1
|
| 28 |
+
draccus==0.10.0
|
| 29 |
+
einops==0.8.1
|
| 30 |
+
filelock==4.0.12
|
| 31 |
+
frozenlist==1.8.0
|
| 32 |
+
fsspec==2026.2.0
|
| 33 |
+
gitdb==4.0.12
|
| 34 |
+
gymnasium==1.2.1
|
| 35 |
+
h11==0.16.0
|
| 36 |
+
hf-xet==1.6.0
|
| 37 |
+
httpcore==1.0.9
|
| 38 |
+
httpx==0.28.1
|
| 39 |
+
huggingface-hub==0.35.3
|
| 40 |
+
hydra-core==1.3.2
|
| 41 |
+
idna==3.20
|
| 42 |
+
imageio==2.37.0
|
| 43 |
+
importlib_metadata==9.0.1
|
| 44 |
+
iniconfig==2.3.0
|
| 45 |
+
jmespath==1.1.0
|
| 46 |
+
jsonlines==4.0.0
|
| 47 |
+
lerobot==0.4.4
|
| 48 |
+
markdown-it-py==4.2.0
|
| 49 |
+
mdurl==0.1.2
|
| 50 |
+
mergedeep==1.3.4
|
| 51 |
+
mpmath==1.3.0
|
| 52 |
+
multidict==6.9.1
|
| 53 |
+
multiprocess==0.70.19
|
| 54 |
+
mypy_extensions==1.1.0
|
| 55 |
+
networkx==3.7
|
| 56 |
+
numpy==1.26.4
|
| 57 |
+
nvidia-cublas==13.1.0.3
|
| 58 |
+
nvidia-cuda-cupti==13.0.85
|
| 59 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 60 |
+
nvidia-cuda-runtime==13.0.96
|
| 61 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 62 |
+
nvidia-cufft==12.0.0.61
|
| 63 |
+
nvidia-cufile==1.15.1.6
|
| 64 |
+
nvidia-curand==10.4.0.35
|
| 65 |
+
nvidia-cusolver==12.0.4.66
|
| 66 |
+
nvidia-cusparse==12.6.3.3
|
| 67 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 68 |
+
nvidia-nccl-cu13==2.28.9
|
| 69 |
+
nvidia-nvjitlink==13.0.88
|
| 70 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 71 |
+
nvidia-nvtx==13.0.85
|
| 72 |
+
omegaconf==2.3.0
|
| 73 |
+
opencv-python-headless==4.11.0.86
|
| 74 |
+
packaging==26.3
|
| 75 |
+
pandas==3.0.6
|
| 76 |
+
pillow==12.0.0
|
| 77 |
+
platformdirs==4.12.3
|
| 78 |
+
pluggy==1.6.0
|
| 79 |
+
propcache==0.5.4
|
| 80 |
+
protobuf==7.36.2
|
| 81 |
+
psutil==7.2.2
|
| 82 |
+
pyarrow==23.0.0
|
| 83 |
+
pydantic==2.13.5
|
| 84 |
+
pydantic_core==2.46.5
|
| 85 |
+
pytest==8.4.2
|
| 86 |
+
python-dateutil==2.9.0.post0
|
| 87 |
+
pyyaml-include==1.4.1
|
| 88 |
+
regex==2026.9.29
|
| 89 |
+
requests==2.34.2
|
| 90 |
+
rich==14.2.0
|
| 91 |
+
s3transfer==0.10.4
|
| 92 |
+
safetensors==0.6.2
|
| 93 |
+
sentry-sdk==2.71.0
|
| 94 |
+
setuptools==81.0.0
|
| 95 |
+
six==1.17.0
|
| 96 |
+
smmap==5.0.3
|
| 97 |
+
sympy==1.14.0
|
| 98 |
+
termcolor==3.2.0
|
| 99 |
+
tokenizers==0.22.2
|
| 100 |
+
toml==0.10.2
|
| 101 |
+
torch==2.11.0
|
| 102 |
+
torchvision==0.26.0
|
| 103 |
+
tqdm==4.70.1
|
| 104 |
+
transformers==4.56.1
|
| 105 |
+
triton==3.6.0
|
| 106 |
+
typing-inspect==0.9.0
|
| 107 |
+
typing-inspection==0.4.4
|
| 108 |
+
typing_extensions==4.16.0
|
| 109 |
+
urllib3==2.8.0
|
| 110 |
+
wandb==0.28.1
|
| 111 |
+
xxhash==4.0.1
|
| 112 |
+
yarl==1.25.1
|
| 113 |
+
zipp==4.1.1
|
checkpoints/step_004000/metadata/parameters.json
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"components": {
|
| 3 |
+
"flux_lora": {
|
| 4 |
+
"total": 23101440,
|
| 5 |
+
"trainable": 23101440,
|
| 6 |
+
"frozen": 0
|
| 7 |
+
},
|
| 8 |
+
"flux_dense": {
|
| 9 |
+
"total": 3875544576,
|
| 10 |
+
"trainable": 0,
|
| 11 |
+
"frozen": 3875544576
|
| 12 |
+
},
|
| 13 |
+
"action_expert": {
|
| 14 |
+
"total": 642012416,
|
| 15 |
+
"trainable": 642012416,
|
| 16 |
+
"frozen": 0
|
| 17 |
+
},
|
| 18 |
+
"lit_interface": {
|
| 19 |
+
"total": 165439365,
|
| 20 |
+
"trainable": 165439365,
|
| 21 |
+
"frozen": 0
|
| 22 |
+
},
|
| 23 |
+
"pose_decoder": {
|
| 24 |
+
"total": 3414536,
|
| 25 |
+
"trainable": 3414536,
|
| 26 |
+
"frozen": 0
|
| 27 |
+
},
|
| 28 |
+
"proprio_encoder": {
|
| 29 |
+
"total": 69120,
|
| 30 |
+
"trainable": 69120,
|
| 31 |
+
"frozen": 0
|
| 32 |
+
},
|
| 33 |
+
"other_stage2": {
|
| 34 |
+
"total": 0,
|
| 35 |
+
"trainable": 0,
|
| 36 |
+
"frozen": 0
|
| 37 |
+
},
|
| 38 |
+
"qwen": {
|
| 39 |
+
"total": 0,
|
| 40 |
+
"trainable": 0,
|
| 41 |
+
"frozen": 0
|
| 42 |
+
},
|
| 43 |
+
"vae": {
|
| 44 |
+
"total": 84046115,
|
| 45 |
+
"trainable": 0,
|
| 46 |
+
"frozen": 84046115
|
| 47 |
+
}
|
| 48 |
+
},
|
| 49 |
+
"totals": {
|
| 50 |
+
"total": 4793627568,
|
| 51 |
+
"trainable": 834036877,
|
| 52 |
+
"frozen": 3959590691,
|
| 53 |
+
"lora": 23101440,
|
| 54 |
+
"dense_trainable": 810935437
|
| 55 |
+
},
|
| 56 |
+
"optimizer_groups": [
|
| 57 |
+
{
|
| 58 |
+
"name": "flux_lora",
|
| 59 |
+
"tensors": 160,
|
| 60 |
+
"parameters": 23101440,
|
| 61 |
+
"lr": 3e-05,
|
| 62 |
+
"weight_decay": 0.01
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"name": "action_expert",
|
| 66 |
+
"tensors": 34,
|
| 67 |
+
"parameters": 124003584,
|
| 68 |
+
"lr": 1e-05,
|
| 69 |
+
"weight_decay": 0.01
|
| 70 |
+
},
|
| 71 |
+
{
|
| 72 |
+
"name": "action_expert",
|
| 73 |
+
"tensors": 18,
|
| 74 |
+
"parameters": 113247232,
|
| 75 |
+
"lr": 1e-05,
|
| 76 |
+
"weight_decay": 0.01
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"name": "action_expert",
|
| 80 |
+
"tensors": 20,
|
| 81 |
+
"parameters": 125830400,
|
| 82 |
+
"lr": 1e-05,
|
| 83 |
+
"weight_decay": 0.01
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "action_expert",
|
| 87 |
+
"tensors": 20,
|
| 88 |
+
"parameters": 125830400,
|
| 89 |
+
"lr": 1e-05,
|
| 90 |
+
"weight_decay": 0.01
|
| 91 |
+
},
|
| 92 |
+
{
|
| 93 |
+
"name": "action_expert",
|
| 94 |
+
"tensors": 20,
|
| 95 |
+
"parameters": 125830400,
|
| 96 |
+
"lr": 1e-05,
|
| 97 |
+
"weight_decay": 0.01
|
| 98 |
+
},
|
| 99 |
+
{
|
| 100 |
+
"name": "action_expert",
|
| 101 |
+
"tensors": 6,
|
| 102 |
+
"parameters": 27270400,
|
| 103 |
+
"lr": 1e-05,
|
| 104 |
+
"weight_decay": 0.01
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"name": "proprio_encoder",
|
| 108 |
+
"tensors": 2,
|
| 109 |
+
"parameters": 69120,
|
| 110 |
+
"lr": 1e-05,
|
| 111 |
+
"weight_decay": 0.01
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"name": "lit_interface",
|
| 115 |
+
"tensors": 190,
|
| 116 |
+
"parameters": 127648132,
|
| 117 |
+
"lr": 1e-05,
|
| 118 |
+
"weight_decay": 0.01
|
| 119 |
+
},
|
| 120 |
+
{
|
| 121 |
+
"name": "lit_interface",
|
| 122 |
+
"tensors": 51,
|
| 123 |
+
"parameters": 37791233,
|
| 124 |
+
"lr": 1e-05,
|
| 125 |
+
"weight_decay": 0.01
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"name": "pose_decoder",
|
| 129 |
+
"tensors": 8,
|
| 130 |
+
"parameters": 3414536,
|
| 131 |
+
"lr": 1e-05,
|
| 132 |
+
"weight_decay": 0.01
|
| 133 |
+
}
|
| 134 |
+
],
|
| 135 |
+
"lora_targets": [
|
| 136 |
+
"double_blocks.0.img_attn.qkv",
|
| 137 |
+
"double_blocks.0.img_attn.proj",
|
| 138 |
+
"double_blocks.0.img_mlp.0",
|
| 139 |
+
"double_blocks.0.img_mlp.2",
|
| 140 |
+
"double_blocks.0.txt_attn.qkv",
|
| 141 |
+
"double_blocks.0.txt_attn.proj",
|
| 142 |
+
"double_blocks.0.txt_mlp.0",
|
| 143 |
+
"double_blocks.0.txt_mlp.2",
|
| 144 |
+
"double_blocks.1.img_attn.qkv",
|
| 145 |
+
"double_blocks.1.img_attn.proj",
|
| 146 |
+
"double_blocks.1.img_mlp.0",
|
| 147 |
+
"double_blocks.1.img_mlp.2",
|
| 148 |
+
"double_blocks.1.txt_attn.qkv",
|
| 149 |
+
"double_blocks.1.txt_attn.proj",
|
| 150 |
+
"double_blocks.1.txt_mlp.0",
|
| 151 |
+
"double_blocks.1.txt_mlp.2",
|
| 152 |
+
"double_blocks.2.img_attn.qkv",
|
| 153 |
+
"double_blocks.2.img_attn.proj",
|
| 154 |
+
"double_blocks.2.img_mlp.0",
|
| 155 |
+
"double_blocks.2.img_mlp.2",
|
| 156 |
+
"double_blocks.2.txt_attn.qkv",
|
| 157 |
+
"double_blocks.2.txt_attn.proj",
|
| 158 |
+
"double_blocks.2.txt_mlp.0",
|
| 159 |
+
"double_blocks.2.txt_mlp.2",
|
| 160 |
+
"double_blocks.3.img_attn.qkv",
|
| 161 |
+
"double_blocks.3.img_attn.proj",
|
| 162 |
+
"double_blocks.3.img_mlp.0",
|
| 163 |
+
"double_blocks.3.img_mlp.2",
|
| 164 |
+
"double_blocks.3.txt_attn.qkv",
|
| 165 |
+
"double_blocks.3.txt_attn.proj",
|
| 166 |
+
"double_blocks.3.txt_mlp.0",
|
| 167 |
+
"double_blocks.3.txt_mlp.2",
|
| 168 |
+
"double_blocks.4.img_attn.qkv",
|
| 169 |
+
"double_blocks.4.img_attn.proj",
|
| 170 |
+
"double_blocks.4.img_mlp.0",
|
| 171 |
+
"double_blocks.4.img_mlp.2",
|
| 172 |
+
"double_blocks.4.txt_attn.qkv",
|
| 173 |
+
"double_blocks.4.txt_attn.proj",
|
| 174 |
+
"double_blocks.4.txt_mlp.0",
|
| 175 |
+
"double_blocks.4.txt_mlp.2",
|
| 176 |
+
"single_blocks.0.linear1",
|
| 177 |
+
"single_blocks.0.linear2",
|
| 178 |
+
"single_blocks.1.linear1",
|
| 179 |
+
"single_blocks.1.linear2",
|
| 180 |
+
"single_blocks.2.linear1",
|
| 181 |
+
"single_blocks.2.linear2",
|
| 182 |
+
"single_blocks.3.linear1",
|
| 183 |
+
"single_blocks.3.linear2",
|
| 184 |
+
"single_blocks.4.linear1",
|
| 185 |
+
"single_blocks.4.linear2",
|
| 186 |
+
"single_blocks.5.linear1",
|
| 187 |
+
"single_blocks.5.linear2",
|
| 188 |
+
"single_blocks.6.linear1",
|
| 189 |
+
"single_blocks.6.linear2",
|
| 190 |
+
"single_blocks.7.linear1",
|
| 191 |
+
"single_blocks.7.linear2",
|
| 192 |
+
"single_blocks.8.linear1",
|
| 193 |
+
"single_blocks.8.linear2",
|
| 194 |
+
"single_blocks.9.linear1",
|
| 195 |
+
"single_blocks.9.linear2",
|
| 196 |
+
"single_blocks.10.linear1",
|
| 197 |
+
"single_blocks.10.linear2",
|
| 198 |
+
"single_blocks.11.linear1",
|
| 199 |
+
"single_blocks.11.linear2",
|
| 200 |
+
"single_blocks.12.linear1",
|
| 201 |
+
"single_blocks.12.linear2",
|
| 202 |
+
"single_blocks.13.linear1",
|
| 203 |
+
"single_blocks.13.linear2",
|
| 204 |
+
"single_blocks.14.linear1",
|
| 205 |
+
"single_blocks.14.linear2",
|
| 206 |
+
"single_blocks.15.linear1",
|
| 207 |
+
"single_blocks.15.linear2",
|
| 208 |
+
"single_blocks.16.linear1",
|
| 209 |
+
"single_blocks.16.linear2",
|
| 210 |
+
"single_blocks.17.linear1",
|
| 211 |
+
"single_blocks.17.linear2",
|
| 212 |
+
"single_blocks.18.linear1",
|
| 213 |
+
"single_blocks.18.linear2",
|
| 214 |
+
"single_blocks.19.linear1",
|
| 215 |
+
"single_blocks.19.linear2"
|
| 216 |
+
],
|
| 217 |
+
"qwen_cached_and_not_instantiated": true
|
| 218 |
+
}
|
checkpoints/step_004000/metadata/pretraining_report.md
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# LIT ImageWAM pre-training report
|
| 2 |
+
|
| 3 |
+
Inspected on 2026-10-03. All tabular/video checks and both full-sized model checks passed. The production DeepSpeed/fused AdamW path passed Stage-1 bridging, one Stage-2 update, and training-subset evaluation; checkpoint serialization and FP32 optimizer-state verification passed before launch. A subsequent multi-update activation-checkpoint benchmark exposed a second-update Stage-2 OOM that the initial one-update preflight missed; bounded optimizer groups resolve it and passed four consecutive full-sized updates.
|
| 4 |
+
|
| 5 |
+
1. **Code.** Released ImageWAM LIT fork `jianmanlincjx/ImageWAM`, commit `1792d100099771fe404ebca5dcd606725141585b`; `imagewam.runtime.create_imagewam_flux2_klein`, `ImageWAM.training_loss`, and `Wan22Trainer`. FLUX source commit `50fe5162777813d869182b139e83b10743caef15`. Configs and normalization come from `linjianman/LIT` revision `765a2b2bf136f7039ab18978920c682f1aae65d2`, `imagewam/lit_stage{1,2}`. Local adapters add gripper conversion, task sampling, online logging, and resumable uploads.
|
| 6 |
+
|
| 7 |
+
2. **Schema.** LeRobot v3; `observation.images.front` and `.wrist`: RGB 256×256 video; `observation.state`: float32[8]; `action`: float32[7]; float32 `timestamp`; int64 `frame_index`, `episode_index`, `index`, `task_index`. Instruction strings live in `meta/tasks.parquet`; episode intervals and video offsets in `meta/episodes/...parquet`; provenance in `meta/source_episodes.jsonl`. Frequency is 20 Hz.
|
| 8 |
+
|
| 9 |
+
3. **Tasks and episodes.** 111 distinct instruction labels, 19,961 episodes, 3,080,750 frames. Per-label episodes range 34–649; frames range 4,447–137,237. Every label's episode/frame totals and source breakdown are in [task_distribution.csv](task_distribution.csv). These are instruction labels, not the 10,030 perturbation configurations of the LIBERO-Plus evaluation benchmark.
|
| 10 |
+
|
| 11 |
+
4. **Composition.** NVIDIA LIBERO: 5,614 episodes / 842,714 frames / 111 labels; Plus: 14,347 episodes / 2,238,036 frames / 40 labels. NVIDIA includes LIBERO-90 (3,921 episodes), LIBERO-10 (379), Goal (428), Object (454), Spatial (432). Plus lacks per-episode perturbation-axis labels in the merged provenance. Stored Plus shares are 71.88% of episodes and 72.65% of frames. The original released ImageWAM recipe used four LIBERO suites without LIBERO-90 or Plus; this requested mixture changes training coverage deliberately.
|
| 12 |
+
|
| 13 |
+
5. **Images.** Decode RGB at recorded video timestamps. Convert uint8 to float32 [0,1], then bilinear antialiased aspect-preserving resize into 128×128 per camera, symmetric zero padding for nonsquare images. Observed sources are square, so no padding is applied. No image rotation, crop, or later upsampling. The stored orientation matches an inspected original `yuanty/LIBERO-fastwam` training image. Concatenate front then wrist to 128×256 and normalize to [-1,1]. Apply the released color/noise augmentation only after downsampling; disable its random crop and ±8° rotation to retain the complete view. Current and t+16 future images receive the same preprocessing.
|
| 14 |
+
|
| 15 |
+
6. **State/action.** State is EEF xyz, axis-angle orientation xyz, and two signed finger joint positions. Preserve all eight state values and the six arm action channels. Arm actions are LIBERO OSC control commands; do not treat them as absolute poses or integrate them as unscaled metric displacements. Convert dataset gripper −1=open/+1=close to released ImageWAM 1=open/0=close with **(1−g)/2 before normalization**. Use the inverse 1−2g at deployment. The goal is the observed 8D state at **t+16**, not the episode's final state. Actions cover t through t+15.
|
| 16 |
+
|
| 17 |
+
7. **Inconsistencies and audit.** The dataset card's 0/1 gripper prose contradicts its merge formula. Measured next-step finger-width changes confirm the stored −1/open and +1/close convention in both sources. Released action statistics instead expect 0/close and 1/open; direct reuse without the conversion would invalidate labels. NVIDIA states use Arrow fixed-size lists; Plus uses variable lists, with all rows verified width 8. Use the native v3 loader with explicit features to reconcile storage types without changing values. Current torchvision removed the VideoReader API used by LeRobot; a direct PyAV shim preserves RGB decoding with strict timestamp tolerance. Selected-row queries replace expensive whole-column reads. Stage 1 skips video decoding, and random data-error fallback is disabled. Keep released global min/max normalization in both phases; expanded mixture extrema extend beyond [-1,1] (state approximately [-2.29,1.82], action [-1.28,1.06]), entirely inside the processor's existing [-5,5] clamp. Do not clip to [-1,1], change rotation parameterization, or substitute the dataset's OpenPI normalization.
|
| 18 |
+
|
| 19 |
+
All 3,080,750 tabular rows passed null, finite-value, vector-width, index, episode-contiguity, instruction, and frame/timestamp checks. Maximum timestamp deviation from frame_index/20: 7.63e−7 seconds. Video audit checks all 108 files, both camera intervals in all 19,961 episodes, and first/middle/last RGB frames of every video file. This is a sampled image decode audit, not exhaustive frame corruption detection. The dataset has no independent simulator replay, success, or collision certification; do not infer these from structural validity.
|
| 20 |
+
|
| 21 |
+
The general LIT README describes a latent-only visual firewall, H=10, and quantile normalization; the released **ImageWAM** config/code instead use H=16, global min/max, and direct reference plus synthetic latent channels. The user explicitly selected the released ImageWAM recipe: 55% both, 15% reference only, 30% synthetic only. Preserve that behavior and its 100×768 latents, 8 pose tokens, 512 inner width, and 5 parameter groups. Stage-2 loss also includes the upstream image objective; it is not action-plus-pose alone.
|
| 22 |
+
|
| 23 |
+
8. **Sampling.** Shuffle uniform cycles over all 111 task IDs, then choose a source uniformly when both exist, an eligible episode uniformly, and a valid anchor uniformly. Require t+16<episode_length, so neither action nor goal crosses an episode or is padded. There are 2,761,374 valid anchors. Exact task counts differ by at most one per cycle; a probe produced exactly 100 anchors for every task and reproduced resume offsets. Expected Plus share becomes **40/(2×111)=18.02%**, because 71 labels have only NVIDIA episodes. Sampler passes round down to whole effective batches (2,761,216 Stage-1 draws / 2,761,360 Stage-2 draws), avoiding partial accumulation updates at an epoch boundary. Sampling uses replacement within tasks; an epoch does not mean exhaustive coverage.
|
| 24 |
+
|
| 25 |
+
9. **Stage 1.** Vision-free action prior; frozen FLUX backbone, trained action expert and goal encoder, cached Qwen3 embeddings. Follow released initialization from the FLUX-derived ActionDiT initializer, rather than an already trained action checkpoint. **10,000 optimizer updates**, fused AdamW lr 2e−4, betas (0.9,0.95), decay .01, 2,000-step warmup, cosine decay to 1% of peak, gradient clip 1.0, seed 42, BF16, gradient checkpointing. Released launcher uses 8 GPUs × local batch 64; use one GPU × microbatch 256 × accumulation 2, preserving effective batch **512**. Stage-1 pose/auxiliary loss is explicitly zero; the observed terminal pose is conditioning, not a prediction objective.
|
| 26 |
+
|
| 27 |
+
10. **Stage 2 and operation.** Strict Stage-1 bridge initialization; FLUX.2 Klein base 4B backbone, action expert, latent interface, and pose decoder train; Qwen3 and VAE remain frozen. Fused AdamW with released lr **1e−4 for all trainable parameters**, decay .01, betas (0.9,0.95), clip 1.0, BF16, checkpointing; loss = action + .5×image + .3×pose. Preserve **34,720 optimizer updates** of the released 10-epoch run, warmup **1,736** updates. Bound optimizer groups to 128 million parameters (39 groups, maximum observed 126,181,762) to prevent a 17.46 GiB DeepSpeed flatten/cast temporary. Every group retains the released lr/betas/decay and shared global clipping; masters and moments remain FP32. One GPU × microbatch 80 × accumulation 1 preserves the original 8×10 effective batch **80**. This gives 2,777,600 draws (~1.006 sampler epochs) on the larger mixture; reusing ten mixture epochs would greatly increase the released update budget. The max_steps override intentionally takes precedence over the retained released num_epochs metadata.
|
| 28 |
+
|
| 29 |
+
Separate online W&B runs in `star1/parc2026-lit-imagewam`, group `lit-libero-nvidia-plus-128`. Log every 10 optimizer updates: accumulated-update mean total/action/image/pose losses, lr, optimizer_step, epoch, grad norm, samples/s, and allocated/reserved/peak GPU memory. Cache all 111 Qwen instructions to avoid occupying training VRAM with the extra language model. The released runtime reuses the training dataset for evaluation. Preserve Stage-2 evaluation every 100 updates with 8 samples, reporting training-subset action/pose/image diagnostics, not held-out validation or simulator success. Save weight checkpoints every 5,000 updates and at each phase's final update; retain only the latest local optimizer/RNG state for resume. Keep only the two newest uploaded local checkpoints per phase and all final checkpoints; older complete snapshots remain on Hugging Face. Completed Stage-1 optimizer state is retired when Stage 2 starts. If disk cannot hold two full FP32 optimizer states, obsolete local state is reclaimed before the next write; independently saved weight checkpoints remain available. Stage-1 final: 10,000. Stage-2 final: 34,720. An independent retrying uploader sends immutable weight-only folders plus configs, preprocessing, audits, task distribution, and pinned revisions to private model repo `Ratstar216/LIT-ImageWAM-LIBERO-NVIDIA-PLUS-128`, using Hugging Face resumable large-folder caching and content addressing. Stage 2 starts only after Stage 1 completes successfully.
|
| 30 |
+
|
| 31 |
+
## Changes from the released configuration
|
| 32 |
+
|
| 33 |
+
| Setting | Released | This run | Reason |
|
| 34 |
+
|---|---|---|---|
|
| 35 |
+
| Training GPUs | 8 | 1 RTX PRO 6000 Blackwell 96 GB | Available hardware |
|
| 36 |
+
| Local microbatch / accumulation | Stage 1: 64 / 1; Stage 2: 10 / 1 (8 GPUs) | Stage 1: 256 / 2; Stage 2: 80 / 1 (1 GPU) | Preserve effective batch 512 / 80 |
|
| 37 |
+
| Camera resize | 224×224 | 128×128, letterbox | Competition requirement; preserve complete image |
|
| 38 |
+
| Geometric augmentation | crop scale .92–1, rotation ±8° | Disabled | Avoid dropping task-relevant pixels |
|
| 39 |
+
| Stage-2 stopping | 10 original-data epochs (34,720 updates) | Explicit 34,720 updates | Preserve released optimizer budget on larger mixture |
|
| 40 |
+
| Warmup Stage 2 | 5% (1,736 updates) | Explicit 1,736 | Preserve released warmup |
|
| 41 |
+
| Stage-2 workers | 24 | 8 | Single-GPU loader; limit decoding/prefetch RAM |
|
| 42 |
+
| Checkpoint cadence | Stage 1: 2,000; Stage 2: 1,000 | 5,000 plus final | Requested upload cadence; avoid filling local disk with redundant full-backbone checkpoints |
|
| 43 |
+
| AdamW kernel | Default foreach | Fused | Full Stage-2 preflight OOM on 17.46 GiB temporary; preserve FP32 master weights/moments and all optimizer hyperparameters |
|
| 44 |
+
| Optimizer grouping, Stage 2 | One group of 4.686 billion parameters | Groups capped at 128 million (39 groups) | Multi-update check revealed a 17.46 GiB gradient cast OOM on update 2; preserve all parameter hyperparameters and FP32 masters/moments |
|
| 45 |
+
| W&B mode | Offline | Online | User requirement |
|
| 46 |
+
| Dataset backend/keys | v2, image/wrist_image | v3, explicit front/wrist mapping | Requested dataset format |
|
| 47 |
+
| Sampler | Uniform frame permutation | Task/source/episode/valid-anchor balance | User requirement |
|
| 48 |
+
|
| 49 |
+
Environment uses installed PyTorch 2.11.0+cu130 and torchvision 0.26.0+cu130 rather than the fork's cu118 environment, because Blackwell compute capability 12.0 needs a compatible CUDA/PyTorch build. BF16 support was verified. Stage-1 microbatch 256 preflight used 56.11 GiB peak allocated / 75.74 GiB reserved with a finite forward/backward/AdamW update. Stage-2 microbatch 80 plain-optimizer model preflight used 43.93 GiB allocated / 44.01 GiB reserved. The production optimizer uses the released DeepSpeed ZeRO-1 recipe and FP32 master weights/moments; its complete memory/checkpoint preflight is recorded separately. Default AdamW exhausted VRAM during its 17.46 GiB square-root temporary allocation; fused AdamW completed a full Stage-2 update, strict Stage-1 bridge load, and training-set evaluation. A four-step FP32 kernel comparison passed rtol/atol 1e−6 (maximum absolute difference 9.54e−7). Transformers is pinned to the fork's FLUX-compatible 4.56.1. Exact package versions and resolved configs are saved with every upload.
|
| 50 |
+
|
| 51 |
+
Sources: [released LIT ImageWAM code](https://github.com/jianmanlincjx/ImageWAM/tree/1792d100099771fe404ebca5dcd606725141585b), [released LIT checkpoints/configs](https://huggingface.co/linjianman/LIT/tree/765a2b2bf136f7039ab18978920c682f1aae65d2/imagewam), [requested dataset](https://huggingface.co/datasets/Ratstar216/LIBERO_NVIDIA_PLUS_LeRobot_v3), [original ImageWAM data](https://huggingface.co/datasets/yuanty/LIBERO-fastwam).
|
| 52 |
+
|
| 53 |
+
The completed selective-checkpoint benchmark is documented in [checkpoint_benchmark_report.md](checkpoint_benchmark_report.md). Stage-1 full checkpointing measured 30.85 seconds/update; checkpointing 12 of 25 layers with microbatch 64 and accumulation 8 measured 26.54 seconds/update at 78.99 GiB peak allocated, with identical first-update loss and sampled gradients. Production retains its original activation policy. Corrected Stage-2 full checkpointing measured 9.48 seconds/update at 86.10 GiB peak allocated across three measured updates after warm-up. All tested Stage-2 selective policies exceeded VRAM.
|
checkpoints/step_004000/metadata/provenance.json
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "LIT ImageWAM Stage-2 Resolution Adaptation",
|
| 3 |
+
"input_information": [
|
| 4 |
+
128,
|
| 5 |
+
128
|
| 6 |
+
],
|
| 7 |
+
"model_processing_per_camera": [
|
| 8 |
+
224,
|
| 9 |
+
224
|
| 10 |
+
],
|
| 11 |
+
"combined": [
|
| 12 |
+
224,
|
| 13 |
+
448
|
| 14 |
+
],
|
| 15 |
+
"source": [
|
| 16 |
+
256,
|
| 17 |
+
256
|
| 18 |
+
],
|
| 19 |
+
"method": "FLUX.2 LoRA + full ActionDiT/LIT-interface fine-tuning",
|
| 20 |
+
"dataset": "Ratstar216/LIBERO_NVIDIA_PLUS_LeRobot_v3",
|
| 21 |
+
"dataset_revision": "c289c195d1fcb7eb6406b5c771a11465e81e8aef",
|
| 22 |
+
"released_lit_revision": "765a2b2bf136f7039ab18978920c682f1aae65d2",
|
| 23 |
+
"initialization": "linjianman/LIT/imagewam/lit_stage2/model.pt",
|
| 24 |
+
"normalization": "released Stage-2 dataset_stats.json",
|
| 25 |
+
"imagewam_commit": "1792d100099771fe404ebca5dcd606725141585b",
|
| 26 |
+
"flux_commit": "50fe5162777813d869182b139e83b10743caef15",
|
| 27 |
+
"competition_commit": "986b06942acb88b28a41ef0904c58418df04316d",
|
| 28 |
+
"python": "3.12.3",
|
| 29 |
+
"torch": "2.11.0+cu130",
|
| 30 |
+
"cuda": "13.0",
|
| 31 |
+
"created_unix": 1791307396.9788215,
|
| 32 |
+
"sampling_weights": [
|
| 33 |
+
0.5,
|
| 34 |
+
0.4,
|
| 35 |
+
0.1
|
| 36 |
+
],
|
| 37 |
+
"wandb": {
|
| 38 |
+
"id": "db6a83524459",
|
| 39 |
+
"project": "parc2026-lit-imagewam",
|
| 40 |
+
"name": "lit-stage2-resadapt-128to224-lora-r16",
|
| 41 |
+
"group": "lit-stage2-resadapt-128to224",
|
| 42 |
+
"url": "https://wandb.ai/star1/parc2026-lit-imagewam/runs/db6a83524459"
|
| 43 |
+
},
|
| 44 |
+
"preprocessing": "RGB 256 -> bilinear antialiased 128 -> 224; photometric only; front then wrist; [-1,1]; both endpoints; no stored rotation",
|
| 45 |
+
"goal": "state[t+16]; action[t:t+16]; t+16 < episode length",
|
| 46 |
+
"video_loading": "independent timestamp seeks; up to 8 attempts on invalid video data; replacement stays within pool/task/split; failures logged and counters checkpointed",
|
| 47 |
+
"gripper": "(1-g)/2 exactly once before released normalization",
|
| 48 |
+
"qwen": "frozen offline cached embeddings; encoder not instantiated",
|
| 49 |
+
"validation": "held-out episodes, equal sample count per pool; overall weighted by experiment mixture"
|
| 50 |
+
}
|
checkpoints/step_004000/metadata/qwen_cache.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"repo": "Qwen/Qwen3-4B",
|
| 3 |
+
"revision": "1cfa9a7208912126459214e8b04321603b3df60c",
|
| 4 |
+
"imagewam_commit": "1792d100099771fe404ebca5dcd606725141585b",
|
| 5 |
+
"context_len": 512,
|
| 6 |
+
"layers": [
|
| 7 |
+
9,
|
| 8 |
+
18,
|
| 9 |
+
27
|
| 10 |
+
]
|
| 11 |
+
}
|
checkpoints/step_004000/metadata/sampling.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"draws": 32000,
|
| 3 |
+
"counts": {
|
| 4 |
+
"familiar": 16110,
|
| 5 |
+
"plus": 12708,
|
| 6 |
+
"libero90": 3182
|
| 7 |
+
},
|
| 8 |
+
"video_decode_failures": {},
|
| 9 |
+
"percentages": {
|
| 10 |
+
"familiar": 50.34375,
|
| 11 |
+
"plus": 39.7125,
|
| 12 |
+
"libero90": 9.94375
|
| 13 |
+
}
|
| 14 |
+
}
|
checkpoints/step_004000/metadata/source/recovery_checkpoints.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Crash-safe local recovery snapshots and the persistent six-hour clock.
|
| 2 |
+
|
| 3 |
+
Only complete optimizer updates are saved. Queue files are hard links so
|
| 4 |
+
uploading a 66 GB ZeRO checkpoint does not require a second local copy.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
import shutil
|
| 11 |
+
import time
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
RECOVERY_REPO = "Ratstar216/LIT-ImageWAM-LIBERO-NVIDIA-PLUS-128-recovery"
|
| 15 |
+
MODEL_REPO = "Ratstar216/LIT-ImageWAM-LIBERO-NVIDIA-PLUS-128"
|
| 16 |
+
INTERVAL_SECONDS = 6 * 60 * 60
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def atomic_json(path: Path, payload):
|
| 20 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 21 |
+
temporary = path.with_name(path.name + ".tmp")
|
| 22 |
+
with temporary.open("w") as stream:
|
| 23 |
+
json.dump(payload, stream, indent=2)
|
| 24 |
+
stream.write("\n")
|
| 25 |
+
stream.flush()
|
| 26 |
+
os.fsync(stream.fileno())
|
| 27 |
+
temporary.replace(path)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class RecoveryClock:
|
| 31 |
+
def __init__(self, path: Path, interval=INTERVAL_SECONDS, now=None):
|
| 32 |
+
self.path, self.interval = path, float(interval)
|
| 33 |
+
if not path.exists():
|
| 34 |
+
now = time.time() if now is None else now
|
| 35 |
+
atomic_json(path, {"interval_seconds": self.interval,
|
| 36 |
+
"next_due_unix": now + self.interval})
|
| 37 |
+
|
| 38 |
+
def due(self, now=None):
|
| 39 |
+
now = time.time() if now is None else now
|
| 40 |
+
return now >= json.loads(self.path.read_text())["next_due_unix"]
|
| 41 |
+
|
| 42 |
+
def completed(self, now=None):
|
| 43 |
+
# Fixed wall-clock cadence: saves at 5,000 updates do not postpone 6h.
|
| 44 |
+
now = time.time() if now is None else now
|
| 45 |
+
payload = json.loads(self.path.read_text())
|
| 46 |
+
while payload["next_due_unix"] <= now:
|
| 47 |
+
payload["next_due_unix"] += self.interval
|
| 48 |
+
payload["last_saved_unix"] = now
|
| 49 |
+
atomic_json(self.path, payload)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def complete_states(state_root: Path):
|
| 53 |
+
# Legacy checkpoints wrote trainer_state.json last too. Hidden pending
|
| 54 |
+
# directories never match this glob and cannot be resumed accidentally.
|
| 55 |
+
return sorted(p for p in state_root.glob("step_*")
|
| 56 |
+
if p.is_dir() and (p / "trainer_state.json").is_file())
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def queue_snapshot(state: Path, stage_dir: Path, metadata_dir: Path,
|
| 60 |
+
repo=RECOVERY_REPO):
|
| 61 |
+
progress = json.loads((state / "trainer_state.json").read_text())
|
| 62 |
+
stage = stage_dir.name
|
| 63 |
+
step = int(progress["global_step"])
|
| 64 |
+
folder = stage_dir.parent / "recovery_uploads" / stage / f"step_{step:06d}"
|
| 65 |
+
receipt = stage_dir / "checkpoints/recovery/remote_latest.json"
|
| 66 |
+
if receipt.exists() and json.loads(receipt.read_text())["optimizer_step"] >= step:
|
| 67 |
+
return folder
|
| 68 |
+
if (folder / "READY").exists():
|
| 69 |
+
return folder
|
| 70 |
+
# A failed/incomplete staging operation is safe to retry; it was never READY.
|
| 71 |
+
if folder.exists():
|
| 72 |
+
shutil.rmtree(folder)
|
| 73 |
+
folder.mkdir(parents=True)
|
| 74 |
+
files = []
|
| 75 |
+
for source in sorted(state.rglob("*")):
|
| 76 |
+
if not source.is_file():
|
| 77 |
+
continue
|
| 78 |
+
relative = Path("state") / source.relative_to(state)
|
| 79 |
+
target = folder / relative
|
| 80 |
+
target.parent.mkdir(parents=True, exist_ok=True)
|
| 81 |
+
os.link(source, target)
|
| 82 |
+
files.append({"path": relative.as_posix(), "size": source.stat().st_size})
|
| 83 |
+
for name in ["config.yaml", "dataset_stats.json"]:
|
| 84 |
+
source = stage_dir / name
|
| 85 |
+
if source.exists():
|
| 86 |
+
shutil.copy2(source, folder / name)
|
| 87 |
+
clock = stage_dir / "checkpoints/recovery/clock.json"
|
| 88 |
+
if clock.exists():
|
| 89 |
+
shutil.copy2(clock, folder / "recovery_clock.json")
|
| 90 |
+
shutil.copytree(metadata_dir, folder / "reproduction")
|
| 91 |
+
manifest = {"format": 1, "stage": stage, "optimizer_step": step,
|
| 92 |
+
"epoch": progress["epoch"], "batch_in_epoch": progress["batch_in_epoch"],
|
| 93 |
+
"repo": repo, "remote_prefix": f"{stage}/{folder.name}",
|
| 94 |
+
"created_at_unix": time.time(), "state_files": files,
|
| 95 |
+
"state_bytes": sum(item["size"] for item in files),
|
| 96 |
+
"contents": "model, FP32 AdamW master weights and moments, scheduler, RNG, sampler progress",
|
| 97 |
+
"resume_requires": "same pinned model/data, optimizer groups, world size and prepared runtime"}
|
| 98 |
+
atomic_json(folder / "manifest.json", manifest)
|
| 99 |
+
(folder / "READY").touch()
|
| 100 |
+
atomic_json(stage_dir / "checkpoints/recovery/latest_local.json", manifest)
|
| 101 |
+
return folder
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def make_state_space(stage_dir: Path, estimated_bytes: int, reserve=5 * 2**30):
|
| 105 |
+
"""Never delete the only complete recoverable copy to make room.
|
| 106 |
+
|
| 107 |
+
A Stage-2 state may not fit twice on Colab. An uploaded receipt permits
|
| 108 |
+
retirement of the old *local* state while its remote snapshot stays live.
|
| 109 |
+
Pending uploads retain their hard links and are never deleted here.
|
| 110 |
+
"""
|
| 111 |
+
if shutil.disk_usage(stage_dir).free >= estimated_bytes + reserve:
|
| 112 |
+
return
|
| 113 |
+
receipt_path = stage_dir / "checkpoints/recovery/remote_latest.json"
|
| 114 |
+
covered_step = -1
|
| 115 |
+
if receipt_path.exists():
|
| 116 |
+
covered_step = json.loads(receipt_path.read_text())["optimizer_step"]
|
| 117 |
+
for old in complete_states(stage_dir / "checkpoints/state"):
|
| 118 |
+
step = json.loads((old / "trainer_state.json").read_text())["global_step"]
|
| 119 |
+
if step <= covered_step:
|
| 120 |
+
shutil.rmtree(old)
|
| 121 |
+
if shutil.disk_usage(stage_dir).free < estimated_bytes + reserve:
|
| 122 |
+
raise OSError("Insufficient disk for a full recovery checkpoint; kept all states "
|
| 123 |
+
"without a verified Hugging Face replacement. See recovery_upload.log.")
|
checkpoints/step_004000/metadata/source/resolution_adapt/README.md
ADDED
|
@@ -0,0 +1,357 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# LIT ImageWAM Stage-2 Resolution Adaptation
|
| 2 |
+
|
| 3 |
+
This experiment starts from **released `linjianman/LIT/imagewam/lit_stage2/model.pt`**.
|
| 4 |
+
It preserves **224×224 processing per camera** while limiting both current and
|
| 5 |
+
future images to **128×128 information**. It never runs Stage 1. The old native-128
|
| 6 |
+
pipeline remains separate.
|
| 7 |
+
|
| 8 |
+
The real CUDA preflight passed on an RTX PRO 6000 Blackwell (96 GB), using
|
| 9 |
+
microbatch 1 and accumulation 8. The released checkpoint loaded strictly, real
|
| 10 |
+
processed dataset samples matched raw expectations, one optimizer update passed,
|
| 11 |
+
and a separate process resumed for one more discarded update. Held-out validation
|
| 12 |
+
also passed. No full training run or production model upload was launched.
|
| 13 |
+
Sandboxed commands could not access CUDA; these tests ran outside the sandbox.
|
| 14 |
+
See the [verification report](../../../research/lit_resolution_adapt/report.md).
|
| 15 |
+
|
| 16 |
+
## Reproducible assets and environment
|
| 17 |
+
|
| 18 |
+
Pinned sources:
|
| 19 |
+
|
| 20 |
+
| Asset | Revision |
|
| 21 |
+
| --- | --- |
|
| 22 |
+
| ImageWAM `feat/goal-prior-bottleneck-fix` implementation | `1792d100099771fe404ebca5dcd606725141585b` |
|
| 23 |
+
| FLUX source | `50fe5162777813d869182b139e83b10743caef15` |
|
| 24 |
+
| Released LIT Stage-2 checkpoint/config/stats | `765a2b2bf136f7039ab18978920c682f1aae65d2` |
|
| 25 |
+
| `Ratstar216/LIBERO_NVIDIA_PLUS_LeRobot_v3` | `c289c195d1fcb7eb6406b5c771a11465e81e8aef` |
|
| 26 |
+
| FLUX.2-dev autoencoder | `26afe3a78bb242c0a8bb181dcc8937bb16e5c66c` |
|
| 27 |
+
| Qwen3-4B | `1cfa9a7208912126459214e8b04321603b3df60c` |
|
| 28 |
+
|
| 29 |
+
`assets.lock.json` records the released model, stats, config and AE hashes. The
|
| 30 |
+
launcher checks full model/stat/AE hashes, clean source commits, pinned episode
|
| 31 |
+
provenance, and Qwen-cache provenance before model allocation. Dataset downloads
|
| 32 |
+
use a pinned snapshot; run metadata records that revision and source-provenance
|
| 33 |
+
hash. The existing all-row/video audits are included with checkpoints. These
|
| 34 |
+
audits establish dataset structure, not simulator success or collision safety.
|
| 35 |
+
|
| 36 |
+
From the repository root, prepare a dedicated GPU environment (Python 3.12):
|
| 37 |
+
|
| 38 |
+
```bash
|
| 39 |
+
uv venv --python 3.12 .runtime/lit-training/venv
|
| 40 |
+
uv pip install --python .runtime/lit-training/venv/bin/python \
|
| 41 |
+
-r examples/lit_imagewam_libero/resolution_adapt/requirements-training.lock
|
| 42 |
+
uv pip install --python .runtime/lit-training/venv/bin/python --no-deps lerobot==0.4.4
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
The explicit LeRobot overlay preserves this repository's Torch 2.11 / torchvision
|
| 46 |
+
0.26 / direct-PyAV path despite LeRobot's tighter package bounds. Its 0.4.4 loader
|
| 47 |
+
supplies explicit Arrow features, supporting the audited fixed/variable state
|
| 48 |
+
lists. The locked dependency set resolves with uv; a clean GPU installation of
|
| 49 |
+
that entire set has **not** been executed here. CPU and GPU checks ran on Python 3.13.15, Torch 2.11.0+cu130,
|
| 50 |
+
Transformers 4.56.1, Diffusers 0.35.2 and LeRobot 0.4.4, using the existing
|
| 51 |
+
environment plus temporary uv packages. The complete measured environment is
|
| 52 |
+
recorded in `research/lit_resolution_adapt/gpu_environment.txt`.
|
| 53 |
+
|
| 54 |
+
Authenticate using `HF_TOKEN`/existing HF login and `WANDB_API_KEY`/existing W&B
|
| 55 |
+
login. No keys belong in commands or config files. `WANDB_ENTITY` is optional.
|
| 56 |
+
The following **explicitly downloads the full dataset and model assets** and
|
| 57 |
+
caches frozen Qwen embeddings; it does not train. It may take substantial storage
|
| 58 |
+
and download time. Existing clean pinned checkouts/downloads are reused; dirty
|
| 59 |
+
source checkouts cause an error rather than being overwritten.
|
| 60 |
+
|
| 61 |
+
```bash
|
| 62 |
+
.runtime/lit-training/venv/bin/python \
|
| 63 |
+
examples/lit_imagewam_libero/resolution_adapt/prepare_assets.py \
|
| 64 |
+
--runtime .runtime/lit-training --cache-text
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
Cached Qwen contexts replace the text encoder during training; no Qwen parameters
|
| 68 |
+
are instantiated or optimized. The cache generator reuses the released script,
|
| 69 |
+
512-token context, hidden layers 9/18/27, and released prompt template. It writes
|
| 70 |
+
`qwen_cache/provenance.json`. Existing caches without provenance should be
|
| 71 |
+
regenerated from the pinned local Qwen model, not blindly relabeled.
|
| 72 |
+
|
| 73 |
+
## Preflights and launch
|
| 74 |
+
|
| 75 |
+
CPU tests with the actual upstream LoRA implementation:
|
| 76 |
+
|
| 77 |
+
```bash
|
| 78 |
+
PYTHONPATH=.runtime/lit-training/upstream/src:.runtime/lit-training/flux2/src \
|
| 79 |
+
.runtime/lit-training/venv/bin/python -m pytest \
|
| 80 |
+
examples/lit_imagewam_libero/resolution_adapt/test_pipeline.py \
|
| 81 |
+
examples/lit_imagewam_libero/resolution_adapt/test_video_loading.py -q
|
| 82 |
+
|
| 83 |
+
.runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
|
| 84 |
+
--mode sampler --output .runtime/lit-resolution-sampler
|
| 85 |
+
```
|
| 86 |
+
|
| 87 |
+
Representative raw RGB/data checks and debug PNGs can run without a GPU. The
|
| 88 |
+
optional `--stream-from-hub` reads pinned metadata/tabular shards and streams
|
| 89 |
+
endpoint frames without downloading all videos:
|
| 90 |
+
|
| 91 |
+
```bash
|
| 92 |
+
.runtime/lit-training/venv/bin/python \
|
| 93 |
+
examples/lit_imagewam_libero/resolution_adapt/preflight_data.py \
|
| 94 |
+
--dataset-root .runtime/lit-training/dataset --output .runtime/lit-resolution-data \
|
| 95 |
+
--stats .runtime/lit-training/released/imagewam/lit_stage2/dataset_stats.json
|
| 96 |
+
```
|
| 97 |
+
|
| 98 |
+
Actual checkpoint **metadata** keys/shapes can be checked without full tensor
|
| 99 |
+
allocation; this uses remote range reads and is not a substitute for GPU loading:
|
| 100 |
+
|
| 101 |
+
```bash
|
| 102 |
+
.runtime/lit-training/venv/bin/python \
|
| 103 |
+
examples/lit_imagewam_libero/resolution_adapt/preflight_model.py \
|
| 104 |
+
--imagewam-root .runtime/lit-training/upstream \
|
| 105 |
+
--flux-root .runtime/lit-training/flux2 --output .runtime/lit-resolution-architecture
|
| 106 |
+
```
|
| 107 |
+
|
| 108 |
+
Run one real accumulated optimizer update in a **separate disposable output**:
|
| 109 |
+
|
| 110 |
+
```bash
|
| 111 |
+
.runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
|
| 112 |
+
--mode preflight --output .runtime/lit-resolution-preflight \
|
| 113 |
+
--batch-size 1 --gradient-accumulation 8
|
| 114 |
+
```
|
| 115 |
+
|
| 116 |
+
This loads the exact Stage-2 model before LoRA, compares real processed samples
|
| 117 |
+
against raw t+16 state/action/image expectations, prints missing/unexpected keys,
|
| 118 |
+
checks optimizer coverage, runs the released losses/backward/clipping/AdamW/
|
| 119 |
+
scheduler, and checks LoRA/ActionDiT/LIT changes against frozen FLUX/AE probes.
|
| 120 |
+
It writes `one_step.json` only on success and never saves production weights.
|
| 121 |
+
Qwen cannot change because training only loads its frozen cached contexts.
|
| 122 |
+
Use a new output directory when repeating preflight.
|
| 123 |
+
To additionally exercise full-size checkpoint recovery, pass `--preflight-save` to
|
| 124 |
+
the first preflight command, then run:
|
| 125 |
+
|
| 126 |
+
```bash
|
| 127 |
+
.runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
|
| 128 |
+
--mode preflight --output .runtime/lit-resolution-preflight --resume
|
| 129 |
+
```
|
| 130 |
+
|
| 131 |
+
This takes just one more discarded update in a separate process and saves another
|
| 132 |
+
local recovery checkpoint; neither is uploaded. Each full preflight recovery
|
| 133 |
+
snapshot is approximately 10.02 GB. The tested sequence restored all 529 Adam
|
| 134 |
+
states, scheduler step 1, sampler draw 8, split and W&B identity, then advanced to
|
| 135 |
+
step 2/draw 16. The same production resume path is used.
|
| 136 |
+
|
| 137 |
+
For held-out losses without optimizer updates:
|
| 138 |
+
|
| 139 |
+
```bash
|
| 140 |
+
.runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
|
| 141 |
+
--mode validate --output .runtime/lit-resolution-preflight --resume
|
| 142 |
+
```
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
Validate online services for the production output directory:
|
| 146 |
+
|
| 147 |
+
```bash
|
| 148 |
+
.runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
|
| 149 |
+
--mode services --output .runtime/lit-resolution-adapt
|
| 150 |
+
```
|
| 151 |
+
|
| 152 |
+
This creates/ensures a private HF repo, uploads and downloads a tiny uniquely named
|
| 153 |
+
metadata probe, and creates a separate `-preflight` W&B run. Training checks the
|
| 154 |
+
HF service receipt; W&B failures leave local metrics and allow continued work.
|
| 155 |
+
|
| 156 |
+
**Exact training command (only run after GPU preflight succeeds):**
|
| 157 |
+
|
| 158 |
+
```bash
|
| 159 |
+
.runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
|
| 160 |
+
--mode train --output .runtime/lit-resolution-adapt \
|
| 161 |
+
--max-steps 10000 --warmup-steps 500 \
|
| 162 |
+
--batch-size 1 --gradient-accumulation 8 \
|
| 163 |
+
--lora-rank 16 --lora-alpha 16 --lora-dropout 0 \
|
| 164 |
+
--lr-flux-lora 3e-5 --lr-action 1e-5 --lr-interface 1e-5 \
|
| 165 |
+
--lr-pose 1e-5 --lr-proprio 1e-5 --lr-other 1e-5 \
|
| 166 |
+
--unfreeze-last-flux-blocks 0 --lr-unfrozen-flux 1e-6 \
|
| 167 |
+
--checkpoint-every 2000 \
|
| 168 |
+
--hf-repo Ratstar216/LIT-ImageWAM-ResolutionAdapt-128to224 \
|
| 169 |
+
--wandb-project parc2026-lit-imagewam \
|
| 170 |
+
--wandb-run-name lit-stage2-resadapt-128to224-lora-r16
|
| 171 |
+
```
|
| 172 |
+
|
| 173 |
+
**Exact local resume command:**
|
| 174 |
+
|
| 175 |
+
```bash
|
| 176 |
+
.runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
|
| 177 |
+
--mode train --output .runtime/lit-resolution-adapt --resume
|
| 178 |
+
```
|
| 179 |
+
|
| 180 |
+
Resume reuses the persisted arguments, schedule, split, sampling RNG/counters,
|
| 181 |
+
optimizer moments, model/LoRA weights, global step, Python/NumPy/Torch/CUDA RNG,
|
| 182 |
+
and W&B ID. CLI hyperparameter changes are intentionally ignored on resume.
|
| 183 |
+
Checkpoint saves occur at complete optimizer boundaries, so accumulation
|
| 184 |
+
position is zero and BF16 needs no scaler. The synchronous loader has no prefetched
|
| 185 |
+
unconsumed samples. Logical epoch = consumed draws / eligible training anchors;
|
| 186 |
+
sampling is with replacement, not a dataset traversal. File locks prevent two
|
| 187 |
+
trainers or uploaders using the same output.
|
| 188 |
+
|
| 189 |
+
Video decoding seeks independently to each requested timestamp. This avoids
|
| 190 |
+
decoding unused frames between the current and future endpoints, where an invalid
|
| 191 |
+
AV1 packet could previously stop training even when both endpoint frames were
|
| 192 |
+
valid. Required frames still have to meet the exact timestamp tolerance.
|
| 193 |
+
If invalid video data prevents loading a required frame, the loader makes up to
|
| 194 |
+
eight attempts, resampling an eligible episode/anchor within the same pool, task
|
| 195 |
+
and split. Other errors, including timestamp mismatches, missing files and
|
| 196 |
+
transform failures, still stop the run. Rejected samples are recorded in
|
| 197 |
+
`video_decode_failures.jsonl`; failure counters appear in metrics and checkpoints.
|
| 198 |
+
The replacement RNG is part of the sampler recovery state. Existing checkpoints
|
| 199 |
+
without failure counters remain resumable. Validation follows the same bounded
|
| 200 |
+
recovery within held-out episodes and reports failure counts per pool.
|
| 201 |
+
|
| 202 |
+
After a decoder crash, use the local resume command above. It restores the
|
| 203 |
+
latest complete checkpoint; optimizer updates after that checkpoint are replayed.
|
| 204 |
+
The previously written loss logs do not contain model or optimizer state.
|
| 205 |
+
|
| 206 |
+
Remote restore into a new output (prepare identical assets at their original
|
| 207 |
+
paths first):
|
| 208 |
+
|
| 209 |
+
```bash
|
| 210 |
+
.runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
|
| 211 |
+
--mode restore --output .runtime/lit-resolution-restored
|
| 212 |
+
.runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
|
| 213 |
+
--mode services --output .runtime/lit-resolution-restored
|
| 214 |
+
.runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
|
| 215 |
+
--mode train --output .runtime/lit-resolution-restored --resume
|
| 216 |
+
```
|
| 217 |
+
|
| 218 |
+
The restore pins the remote commit, chooses the latest completed snapshot, and
|
| 219 |
+
verifies every file before exposing it. Same asset paths are currently required;
|
| 220 |
+
portable path remapping is not automatic.
|
| 221 |
+
|
| 222 |
+
### Fresh Colab resume from step 2,000
|
| 223 |
+
|
| 224 |
+
[`scripts/resume_lit_resolution_adapt_colab.sh`](../../../scripts/resume_lit_resolution_adapt_colab.sh)
|
| 225 |
+
rebuilds the locked Python 3.12 environment and pinned assets on a GPU runtime,
|
| 226 |
+
generates the frozen Qwen cache, and restores exactly `step_002000` through the
|
| 227 |
+
verified restore API. It creates symlinks at the checkpoint's original
|
| 228 |
+
`/tmp/resadapt-*` asset paths, preserving the saved arguments and schedule.
|
| 229 |
+
The service check and training then run in the same process sequence. The first
|
| 230 |
+
training update is step 2,001; the saved target is 10,000 optimizer steps.
|
| 231 |
+
Re-running the launcher on the same VM resumes the latest complete local
|
| 232 |
+
checkpoint in its output directory. The initial restore is about 10 GB and uses
|
| 233 |
+
additional space for the download cache; the dataset and frozen model assets are
|
| 234 |
+
downloaded separately.
|
| 235 |
+
|
| 236 |
+
Use an existing HF/W&B login, or load Colab Secrets in a Python notebook cell:
|
| 237 |
+
|
| 238 |
+
```python
|
| 239 |
+
import os
|
| 240 |
+
from google.colab import userdata
|
| 241 |
+
os.environ["HF_TOKEN"] = userdata.get("HF_TOKEN")
|
| 242 |
+
os.environ["WANDB_API_KEY"] = userdata.get("WANDB_API_KEY")
|
| 243 |
+
```
|
| 244 |
+
|
| 245 |
+
In another notebook cell, start the launcher on its own tmux socket so a new
|
| 246 |
+
server inherits the notebook's authentication environment:
|
| 247 |
+
|
| 248 |
+
```python
|
| 249 |
+
%cd /content/PARC2026_final
|
| 250 |
+
!tmux -L lit-resume new-session -d -s lit-resume -c /content/PARC2026_final 'bash scripts/resume_lit_resolution_adapt_colab.sh'
|
| 251 |
+
```
|
| 252 |
+
|
| 253 |
+
Check progress without attaching an interactive terminal:
|
| 254 |
+
|
| 255 |
+
```python
|
| 256 |
+
!tmux -L lit-resume list-sessions
|
| 257 |
+
!tail -n 40 /content/PARC2026_final/.runtime/lit-resolution-resume-2000/colab-resume.log
|
| 258 |
+
!nvidia-smi
|
| 259 |
+
```
|
| 260 |
+
|
| 261 |
+
From a Colab terminal, attach with `tmux -L lit-resume attach -t lit-resume` and
|
| 262 |
+
detach with **Ctrl+B**, then **D**. New checkpoints are uploaded by the existing
|
| 263 |
+
independent uploader; check `upload.log` and `upload_summary.json` in the output
|
| 264 |
+
directory. tmux does not prevent Colab runtime termination. A deleted VM loses
|
| 265 |
+
its local assets and any updates not saved and uploaded to persistent storage.
|
| 266 |
+
|
| 267 |
+
## Preserved behavior and deliberate implementation choices
|
| 268 |
+
|
| 269 |
+
- Both endpoint cameras: decoded RGB 256 → antialiased bilinear 128 → bilinear
|
| 270 |
+
224; photometric augmentation afterward; front then wrist → 224×448; [-1,1].
|
| 271 |
+
No stored-image rotation, flip or random crop. Both shape_meta copies agree.
|
| 272 |
+
`competition_cameras()` separately handles simulator 128 images and the existing
|
| 273 |
+
adapter's 180° rotation; stored data never uses that function.
|
| 274 |
+
- Actions are `[t:t+16]`; pose is normalized observed state at `t+16`, not the
|
| 275 |
+
episode end. Only `t+16 < L` anchors are eligible. The existing gripper transform
|
| 276 |
+
converts `(1-g)/2` once before released normalization. Eight state values and the
|
| 277 |
+
first six arm controls retain their original semantics.
|
| 278 |
+
- Sampling: pool 50/40/10 → uniform task → uniform eligible episode → uniform
|
| 279 |
+
valid anchor. Deterministic approximately 5% episode holdout per pool/task;
|
| 280 |
+
duplicate source identities stay together. The custom sampler explicitly
|
| 281 |
+
excludes validation. Small strata use at least one validation episode, so the
|
| 282 |
+
realized fraction need not be exactly 5%.
|
| 283 |
+
- Offline validation samples held-out episodes separately per pool, disables
|
| 284 |
+
photometric augmentation, preserves training RNG, and uses fixed diagnostic
|
| 285 |
+
seeds. Overall loss is the **50/40/10-weighted mean** of pool means. These are
|
| 286 |
+
offline losses, not simulator success metrics or training-subset diagnostics.
|
| 287 |
+
- Released 100×768 latent interface, 8 pose tokens, 512 inner width, 5 interface
|
| 288 |
+
parameter groups, 16-action horizon, 55/15/30 `both/ref_only/syn_only` gating,
|
| 289 |
+
context dropout/blackout, pose supervision and action-generation objective stay
|
| 290 |
+
intact. Total loss = action + 0.5×video + 0.3×pose. Returned upstream component
|
| 291 |
+
metrics are already weighted; logging divides by the weights for unweighted
|
| 292 |
+
values rather than multiplying them again.
|
| 293 |
+
- The released `require_text_cache` option controls T5, not Qwen; it remains
|
| 294 |
+
false, while `qwen_text_cache_dir` enables Qwen loading and a batch-shape check
|
| 295 |
+
enforces `[B,512,7680]` contexts.
|
| 296 |
+
- The released Stage-2 `apply_trainable_policy()` would unfreeze all FLUX weights;
|
| 297 |
+
this launcher applies an explicit policy and never calls that method afterward.
|
| 298 |
+
The existing trainer also expects a Stage-1 bridge and uses a different recovery
|
| 299 |
+
clock/retention policy. A small independent loop is used while reusing its model,
|
| 300 |
+
dataset hooks, gripper transform, atomic JSON helper, and resumable HF upload API.
|
| 301 |
+
- Original FLUX is frozen; LoRA wraps the released target suffixes only inside its
|
| 302 |
+
transformer. ActionDiT and all LIT/pose/proprio modules train densely. The final
|
| 303 |
+
N blocks option traverses double blocks followed by single blocks and uses the
|
| 304 |
+
dense-FLUX LR independently of LoRA. AdamW uses FP32 trainable parameters and
|
| 305 |
+
moments, BF16 autocast, betas (0.9,0.95), decay .01, clip 1.0 and cosine decay.
|
| 306 |
+
Groups are capped at 128M elements as in the repository's memory-safe recipe.
|
| 307 |
+
- Default W&B project `parc2026-lit-imagewam`, group
|
| 308 |
+
`lit-stage2-resadapt-128to224`, run `lit-stage2-resadapt-128to224-lora-r16`.
|
| 309 |
+
`WANDB_PROJECT`, `WANDB_RUN_NAME`, `WANDB_ENTITY` are supported. A stable UUID is
|
| 310 |
+
persisted, `resume=allow` reconnects, and every metric is written locally first.
|
| 311 |
+
Connection attempts time out; a failed connection is retried later. Historical
|
| 312 |
+
local JSONL metrics are retained but are not automatically replayed to W&B.
|
| 313 |
+
|
| 314 |
+
## Checkpoints and offline reconstruction
|
| 315 |
+
|
| 316 |
+
Private repo: https://huggingface.co/Ratstar216/LIT-ImageWAM-ResolutionAdapt-128to224
|
| 317 |
+
|
| 318 |
+
Each immutable `checkpoints/step_002000/` (then 004000, 006000, 008000, 010000) contains:
|
| 319 |
+
|
| 320 |
+
```text
|
| 321 |
+
weights/adaptation.pt # LoRA plus ALL dense trainable Stage-2 parameters
|
| 322 |
+
resume/state.pt # AdamW, scheduler, RNG, sampler/counters, optimizer step
|
| 323 |
+
metadata/ # config, args, split, stats, hashes, revisions, audits, source, W&B identity
|
| 324 |
+
manifest.json # byte lengths and SHA-256 checksums
|
| 325 |
+
COMPLETE # manifest hash, written last
|
| 326 |
+
```
|
| 327 |
+
|
| 328 |
+
A non-cadence normal final update is saved too. No frozen base is re-uploaded in
|
| 329 |
+
each snapshot. Local writes are fsynced in a hidden pending directory, verified,
|
| 330 |
+
then renamed. The independent single upload worker scans complete folders, reuses
|
| 331 |
+
HF large-folder transfer caches, verifies remote sizes, and publishes the remote
|
| 332 |
+
completion marker in a final commit. Persistent receipts are rechecked on restart;
|
| 333 |
+
conflicting remote manifests never get overwritten. Training does not wait for
|
| 334 |
+
uploads to finish, including at its final step. Inspect `upload_summary.json` and
|
| 335 |
+
`upload.log`; allow pending uploads to finish before stopping the host.
|
| 336 |
+
|
| 337 |
+
Retain all adaptation weights; retain the latest two full local recovery states
|
| 338 |
+
and every final state. Older recovery data is pruned only after a verified upload
|
| 339 |
+
and never during upload. Failed uploads preserve the full local checkpoint. Disk
|
| 340 |
+
exhaustion fails the save without deleting unbacked checkpoints. Abandoned hidden
|
| 341 |
+
pending directories are cleaned only while holding the trainer lock.
|
| 342 |
+
|
| 343 |
+
Restart/check the independent uploader without training:
|
| 344 |
+
|
| 345 |
+
```bash
|
| 346 |
+
.runtime/lit-training/venv/bin/python examples/lit_imagewam_libero/train_resolution_adapt.py \
|
| 347 |
+
--mode upload --output .runtime/lit-resolution-adapt
|
| 348 |
+
```
|
| 349 |
+
|
| 350 |
+
`resolution_adapt.offline.load_offline()` reconstructs the exact released Stage-2
|
| 351 |
+
base, injects identical LoRA, loads the dense adaptation, merges LoRA with its
|
| 352 |
+
actual alpha/rank, then attaches locally packaged AE/Qwen assets. It imports no
|
| 353 |
+
training dataset, W&B, or HF API and uses `local_files_only=True` for Qwen.
|
| 354 |
+
The caller must package pinned ImageWAM/FLUX sources, base weights, stats, AE and
|
| 355 |
+
Qwen locally. This is a reconstruction interface, not a tested final submission
|
| 356 |
+
ZIP. A merged export should avoid packaging both the base and redundant adapted
|
| 357 |
+
weights; recheck the 20GB ZIP limit and L4 inference budget before submission.
|
checkpoints/step_004000/metadata/source/resolution_adapt/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Stage-2 128-information / 224-processing adaptation (training tools optional)."""
|
checkpoints/step_004000/metadata/source/resolution_adapt/assets.lock.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"linjianman/LIT": {
|
| 3 |
+
"revision": "765a2b2bf136f7039ab18978920c682f1aae65d2",
|
| 4 |
+
"files": {
|
| 5 |
+
"imagewam/lit_stage2/config.yaml": {
|
| 6 |
+
"size": 6886,
|
| 7 |
+
"sha256": "9c4a8d9eb8767e71e7ebd6edb921be810c7867512d9ebbf0f91a53dca21e9da1"
|
| 8 |
+
},
|
| 9 |
+
"imagewam/lit_stage2/dataset_stats.json": {
|
| 10 |
+
"size": 22880,
|
| 11 |
+
"sha256": "b107cd929518d0477535f99834e0cc13f470e25cd61bbce69b315619ba5140d4"
|
| 12 |
+
},
|
| 13 |
+
"imagewam/lit_stage2/model.pt": {
|
| 14 |
+
"size": 9373131513,
|
| 15 |
+
"sha256": "f403225e76809034307caf23e5a8f155385a51adb69d98990cff14125743ec97"
|
| 16 |
+
}
|
| 17 |
+
}
|
| 18 |
+
},
|
| 19 |
+
"black-forest-labs/FLUX.2-dev": {
|
| 20 |
+
"revision": "26afe3a78bb242c0a8bb181dcc8937bb16e5c66c",
|
| 21 |
+
"files": {
|
| 22 |
+
"ae.safetensors": {
|
| 23 |
+
"size": 336211292,
|
| 24 |
+
"sha256": "868fe7b343cc8f3a19dbcfcafbc3d5f888802be3f89bd81b65b3621a066ce8f3"
|
| 25 |
+
}
|
| 26 |
+
}
|
| 27 |
+
},
|
| 28 |
+
"Qwen/Qwen3-4B": {
|
| 29 |
+
"revision": "1cfa9a7208912126459214e8b04321603b3df60c",
|
| 30 |
+
"files": {}
|
| 31 |
+
}
|
| 32 |
+
}
|
checkpoints/step_004000/metadata/source/resolution_adapt/checkpoints.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Immutable atomic snapshots; persistent single-worker retry queue and safe retention."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import hashlib
|
| 5 |
+
import json
|
| 6 |
+
import os
|
| 7 |
+
import random
|
| 8 |
+
import shutil
|
| 9 |
+
import time
|
| 10 |
+
import uuid
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
from recovery_checkpoints import atomic_json
|
| 16 |
+
from .model import trainable_state, load_adaptation
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def sha256(path: Path) -> str:
|
| 20 |
+
h = hashlib.sha256()
|
| 21 |
+
with path.open('rb') as stream:
|
| 22 |
+
for block in iter(lambda: stream.read(8*1024*1024),b''):
|
| 23 |
+
h.update(block)
|
| 24 |
+
return h.hexdigest()
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def rng_state() -> dict:
|
| 28 |
+
return {'python':random.getstate(),'numpy':np.random.get_state(),'torch':torch.get_rng_state(),
|
| 29 |
+
'cuda':torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None}
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def restore_rng(state: dict) -> None:
|
| 33 |
+
random.setstate(state['python']); np.random.set_state(state['numpy']); torch.set_rng_state(state['torch'])
|
| 34 |
+
if state['cuda'] is not None:
|
| 35 |
+
torch.cuda.set_rng_state_all(state['cuda'])
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def checkpoint_due(step: int, every: int, max_steps: int) -> bool:
|
| 39 |
+
return step > 0 and (step % every == 0 or step == max_steps)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def validate_checkpoint(folder: Path, *, resume=True) -> dict:
|
| 43 |
+
if not (folder/'COMPLETE').is_file():
|
| 44 |
+
raise ValueError(f'Incomplete checkpoint: {folder}')
|
| 45 |
+
manifest = json.loads((folder/'manifest.json').read_text())
|
| 46 |
+
if (folder/'COMPLETE').read_text().strip() != sha256(folder/'manifest.json'):
|
| 47 |
+
raise ValueError('Manifest hash mismatch')
|
| 48 |
+
for name,info in manifest['files'].items():
|
| 49 |
+
if not resume and name.startswith('resume/'):
|
| 50 |
+
continue
|
| 51 |
+
path = folder/name
|
| 52 |
+
if not path.is_file() or path.stat().st_size != info['size'] or sha256(path) != info['sha256']:
|
| 53 |
+
raise ValueError(f'Checkpoint integrity failure: {path}')
|
| 54 |
+
return manifest
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def save_checkpoint(root: Path, step: int, model, optimizer, scheduler, sampler,
|
| 58 |
+
metadata_dir: Path, *, final=False) -> Path:
|
| 59 |
+
destination = root/f'step_{step:06d}'
|
| 60 |
+
if destination.exists():
|
| 61 |
+
raise FileExistsError(f'Immutable checkpoint already exists: {destination}')
|
| 62 |
+
root.mkdir(parents=True,exist_ok=True)
|
| 63 |
+
pending = root/f'.pending_{step:06d}_{uuid.uuid4().hex}'
|
| 64 |
+
pending.mkdir()
|
| 65 |
+
(pending/'weights').mkdir(); (pending/'resume').mkdir()
|
| 66 |
+
torch.save(trainable_state(model),pending/'weights/adaptation.pt')
|
| 67 |
+
torch.save({'optimizer':optimizer.state_dict(),'scheduler':scheduler.state_dict(),
|
| 68 |
+
'global_step':step,'rng':rng_state(),'sampler':sampler.state_dict(),
|
| 69 |
+
'accumulation_position':0},pending/'resume/state.pt')
|
| 70 |
+
shutil.copytree(metadata_dir,pending/'metadata')
|
| 71 |
+
atomic_json(pending/'metadata/sampling.json',sampler.distribution_report())
|
| 72 |
+
files = {}
|
| 73 |
+
for path in sorted(pending.rglob('*')):
|
| 74 |
+
if path.is_file():
|
| 75 |
+
with path.open('rb') as stream:
|
| 76 |
+
os.fsync(stream.fileno())
|
| 77 |
+
files[str(path.relative_to(pending))] = {'size':path.stat().st_size,'sha256':sha256(path)}
|
| 78 |
+
atomic_json(pending/'manifest.json',{'step':step,'final':final,'files':files,'created_unix':time.time()})
|
| 79 |
+
with (pending/'COMPLETE').open('w') as stream:
|
| 80 |
+
stream.write(sha256(pending/'manifest.json')+'\n'); stream.flush(); os.fsync(stream.fileno())
|
| 81 |
+
validate_checkpoint(pending)
|
| 82 |
+
pending.rename(destination)
|
| 83 |
+
fd = os.open(root,os.O_RDONLY)
|
| 84 |
+
try: os.fsync(fd)
|
| 85 |
+
finally: os.close(fd)
|
| 86 |
+
return destination
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def latest_checkpoint(root: Path) -> Path:
|
| 90 |
+
errors = []
|
| 91 |
+
for folder in sorted(root.glob('step_*'),reverse=True):
|
| 92 |
+
if not (folder/'resume/state.pt').exists():
|
| 93 |
+
continue
|
| 94 |
+
try:
|
| 95 |
+
validate_checkpoint(folder)
|
| 96 |
+
return folder
|
| 97 |
+
except (ValueError, OSError) as exc:
|
| 98 |
+
errors.append(str(exc))
|
| 99 |
+
raise ValueError(f'No complete valid resume checkpoint; {errors}')
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def resume_checkpoint(folder: Path, model, optimizer, scheduler, sampler) -> int:
|
| 103 |
+
validate_checkpoint(folder)
|
| 104 |
+
load_adaptation(model,folder/'weights/adaptation.pt')
|
| 105 |
+
# Only trusted, locally generated recovery files (contains Python/NumPy RNG tuples).
|
| 106 |
+
state = torch.load(folder/'resume/state.pt',map_location='cpu',weights_only=False)
|
| 107 |
+
if state['accumulation_position'] != 0:
|
| 108 |
+
raise ValueError('Only optimizer-boundary resumes are supported')
|
| 109 |
+
optimizer.load_state_dict(state['optimizer']); scheduler.load_state_dict(state['scheduler'])
|
| 110 |
+
sampler.load_state_dict(state['sampler']); restore_rng(state['rng'])
|
| 111 |
+
return int(state['global_step'])
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def prune_uploaded(root: Path) -> None:
|
| 115 |
+
complete = sorted(p for p in root.glob('step_*') if (p/'resume/state.pt').is_file())
|
| 116 |
+
for folder in complete[:-2]:
|
| 117 |
+
receipt = root.parent/'upload_state'/f'{folder.name}.json'
|
| 118 |
+
if not receipt.exists(): continue
|
| 119 |
+
status = json.loads(receipt.read_text())
|
| 120 |
+
manifest = json.loads((folder/'manifest.json').read_text())
|
| 121 |
+
if manifest['final'] or not status.get('uploaded') or status.get('uploading'): continue
|
| 122 |
+
shutil.rmtree(folder/'resume')
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def upload_once(run: Path, repo: str, api=None) -> dict:
|
| 126 |
+
"""Resumable large-folder transfer, verified receipt; never delete remote history."""
|
| 127 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 128 |
+
api = api or HfApi()
|
| 129 |
+
info = api.model_info(repo)
|
| 130 |
+
if not info.private:
|
| 131 |
+
raise RuntimeError('Checkpoint repository must be private')
|
| 132 |
+
root = run/'checkpoints'
|
| 133 |
+
statuses = run/'upload_state'; statuses.mkdir(exist_ok=True)
|
| 134 |
+
summary = {'pending':0,'failures':0,'completed':0,'duration_seconds':0.0}
|
| 135 |
+
for folder in sorted(root.glob('step_*')):
|
| 136 |
+
if not (folder/'COMPLETE').exists(): continue
|
| 137 |
+
receipt_path = statuses/f'{folder.name}.json'
|
| 138 |
+
old = json.loads(receipt_path.read_text()) if receipt_path.exists() else {}
|
| 139 |
+
prefix = f'checkpoints/{folder.name}'
|
| 140 |
+
start = time.monotonic()
|
| 141 |
+
try:
|
| 142 |
+
# Recheck remotely even after restart with a local success receipt.
|
| 143 |
+
remote_paths = set(api.list_repo_files(repo))
|
| 144 |
+
if f'{prefix}/COMPLETE' in remote_paths:
|
| 145 |
+
marker = Path(hf_hub_download(repo,f'{prefix}/COMPLETE',force_download=True))
|
| 146 |
+
if marker.read_text() != (folder/'COMPLETE').read_text():
|
| 147 |
+
raise RuntimeError('Immutable remote checkpoint conflicts with local checkpoint')
|
| 148 |
+
atomic_json(receipt_path,{**old,'uploaded':True,'uploading':False,'repo':repo,'step':int(folder.name[5:]),
|
| 149 |
+
'failures':old.get('failures',0),'verified_unix':time.time()})
|
| 150 |
+
continue
|
| 151 |
+
manifest = validate_checkpoint(folder)
|
| 152 |
+
if f'{prefix}/manifest.json' in remote_paths:
|
| 153 |
+
remote = Path(hf_hub_download(repo,f'{prefix}/manifest.json',force_download=True))
|
| 154 |
+
if sha256(remote) != sha256(folder/'manifest.json'):
|
| 155 |
+
raise RuntimeError('Refusing to overwrite a different immutable remote manifest')
|
| 156 |
+
atomic_json(receipt_path,{**old,'uploaded':False,'uploading':True})
|
| 157 |
+
# Reuse repository's upload_large_folder approach, including HF retry cache.
|
| 158 |
+
api.upload_large_folder(repo_id=repo,repo_type='model',folder_path=run,
|
| 159 |
+
allow_patterns=[f'{prefix}/**'],ignore_patterns=['**/COMPLETE','**/.cache/**'],
|
| 160 |
+
num_workers=2,print_report=False)
|
| 161 |
+
remote = {item.path:item.size for item in api.list_repo_tree(repo,path_in_repo=prefix,recursive=True)
|
| 162 |
+
if hasattr(item,'size')}
|
| 163 |
+
for name,entry in manifest['files'].items():
|
| 164 |
+
if remote.get(f'{prefix}/{name}') != entry['size']:
|
| 165 |
+
raise RuntimeError(f'Upload verification failed: {name}')
|
| 166 |
+
# A separate final commit is the only remote completion signal.
|
| 167 |
+
api.upload_file(repo_id=repo,path_or_fileobj=folder/'COMPLETE',path_in_repo=f'{prefix}/COMPLETE',
|
| 168 |
+
commit_message=f'Complete immutable {folder.name}')
|
| 169 |
+
marker = Path(hf_hub_download(repo,f'{prefix}/COMPLETE',force_download=True))
|
| 170 |
+
if marker.read_text() != (folder/'COMPLETE').read_text(): raise RuntimeError('Remote marker mismatch')
|
| 171 |
+
elapsed = time.monotonic()-start
|
| 172 |
+
atomic_json(receipt_path,{'uploaded':True,'uploading':False,'repo':repo,'step':manifest['step'],
|
| 173 |
+
'duration_seconds':elapsed,'failures':old.get('failures',0),'verified_unix':time.time()})
|
| 174 |
+
summary['completed'] += 1; summary['duration_seconds'] += elapsed
|
| 175 |
+
except Exception as exc:
|
| 176 |
+
# Store only exception class, avoiding credential-bearing request diagnostics.
|
| 177 |
+
atomic_json(receipt_path,{'uploaded':False,'uploading':False,'failures':old.get('failures',0)+1,
|
| 178 |
+
'error':type(exc).__name__,'last_attempt_unix':time.time()})
|
| 179 |
+
print(f'Upload pending {folder.name}: {type(exc).__name__}',flush=True)
|
| 180 |
+
summary['failures'] += 1; summary['pending'] += 1
|
| 181 |
+
prune_uploaded(root)
|
| 182 |
+
receipts = [json.loads(p.read_text()) for p in statuses.glob('step_*.json')]
|
| 183 |
+
summary.update(pending=sum(not r.get('uploaded',False) for r in receipts),
|
| 184 |
+
failures=sum(r.get('failures',0) for r in receipts),
|
| 185 |
+
completed=sum(r.get('uploaded',False) for r in receipts),
|
| 186 |
+
duration_seconds=sum(r.get('duration_seconds',0.) for r in receipts),
|
| 187 |
+
last_completed_step=max((r.get('step',0) for r in receipts if r.get('uploaded')),default=0))
|
| 188 |
+
atomic_json(run/'upload_summary.json',summary)
|
| 189 |
+
return summary
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def restore_remote(run: Path, repo: str, step: str | None = None) -> Path:
|
| 193 |
+
"""Download only a complete immutable recovery snapshot; verify before exposing it."""
|
| 194 |
+
from huggingface_hub import HfApi,hf_hub_download
|
| 195 |
+
if (run/'metadata').exists(): raise FileExistsError('Restore into a new run directory')
|
| 196 |
+
api=HfApi()
|
| 197 |
+
# Resolve one commit so concurrent newer uploads cannot change this restore.
|
| 198 |
+
revision=api.model_info(repo).sha
|
| 199 |
+
paths=api.list_repo_files(repo,revision=revision)
|
| 200 |
+
candidates=sorted(p.rsplit('/',1)[0] for p in paths if p.startswith('checkpoints/step_') and p.endswith('/COMPLETE'))
|
| 201 |
+
if step is not None: candidates=[p for p in candidates if p.endswith(step)]
|
| 202 |
+
if not candidates: raise ValueError('No complete remote checkpoint')
|
| 203 |
+
prefix=candidates[-1];root=run/'checkpoints';root.mkdir(parents=True,exist_ok=True)
|
| 204 |
+
destination=root/Path(prefix).name
|
| 205 |
+
if destination.exists(): raise FileExistsError(destination)
|
| 206 |
+
pending=root/f'.pending_restore_{uuid.uuid4().hex}';pending.mkdir()
|
| 207 |
+
manifest_path=hf_hub_download(repo,f'{prefix}/manifest.json',revision=revision)
|
| 208 |
+
manifest=json.loads(Path(manifest_path).read_text())
|
| 209 |
+
for name in [*manifest['files'],'manifest.json','COMPLETE']:
|
| 210 |
+
relative=Path(name)
|
| 211 |
+
if relative.is_absolute() or '..' in relative.parts: raise ValueError('Unsafe manifest path')
|
| 212 |
+
target=pending/relative;target.parent.mkdir(parents=True,exist_ok=True)
|
| 213 |
+
shutil.copy2(hf_hub_download(repo,f'{prefix}/{name}',revision=revision),target)
|
| 214 |
+
validate_checkpoint(pending);pending.rename(destination)
|
| 215 |
+
if (run/'metadata').exists(): raise FileExistsError('Restore into a new run directory')
|
| 216 |
+
shutil.copytree(destination/'metadata',run/'metadata')
|
| 217 |
+
return destination
|
checkpoints/step_004000/metadata/source/resolution_adapt/data.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deterministic provenance sampling and shared current/future image transform."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import copy
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
import math
|
| 8 |
+
import random
|
| 9 |
+
from collections import Counter, defaultdict
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
import torch.nn.functional as F
|
| 14 |
+
|
| 15 |
+
DATASET = 'Ratstar216/LIBERO_NVIDIA_PLUS_LeRobot_v3'
|
| 16 |
+
DATASET_REVISION = 'c289c195d1fcb7eb6406b5c771a11465e81e8aef'
|
| 17 |
+
POOLS = ('familiar', 'plus', 'libero90')
|
| 18 |
+
WEIGHTS = (0.5, 0.4, 0.1)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def pool_of(ep: dict) -> str:
|
| 22 |
+
if ep['source_repo'] == 'lerobot/libero_plus':
|
| 23 |
+
return 'plus'
|
| 24 |
+
if ep['source_repo'] == 'nvidia/LIBERO_LeRobot_v3':
|
| 25 |
+
if ep['source_suite'] == 'libero_90':
|
| 26 |
+
return 'libero90'
|
| 27 |
+
if ep['source_suite'] in {'libero_spatial', 'libero_object', 'libero_goal', 'libero_10'}:
|
| 28 |
+
return 'familiar'
|
| 29 |
+
raise ValueError(f"Unknown source provenance: {ep['source_repo']}, {ep['source_suite']}")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def source_identity(ep: dict) -> tuple:
|
| 33 |
+
return ep['source_repo'], ep['source_suite'], ep['source_episode_index']
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def digest(value) -> str:
|
| 37 |
+
return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def split_episodes(episodes: list[dict], fraction=0.05, seed=42) -> dict:
|
| 41 |
+
if not 0 < fraction < 1:
|
| 42 |
+
raise ValueError('Validation fraction must be in (0,1)')
|
| 43 |
+
if len({e['episode_index'] for e in episodes}) != len(episodes):
|
| 44 |
+
raise ValueError('Duplicate merged episode ID')
|
| 45 |
+
# Group duplicate source identities together; never leak an exact source episode.
|
| 46 |
+
groups = defaultdict(dict)
|
| 47 |
+
for ep in episodes:
|
| 48 |
+
pool = pool_of(ep)
|
| 49 |
+
key = source_identity(ep)
|
| 50 |
+
groups[pool, ep['task_index']].setdefault(key, []).append(ep)
|
| 51 |
+
rows = []
|
| 52 |
+
for (pool, task), identities in sorted(groups.items()):
|
| 53 |
+
keys = sorted(identities, key=lambda k: digest([seed, k]))
|
| 54 |
+
eligible = [k for k in keys if any(e['length'] > 16 for e in identities[k])]
|
| 55 |
+
if len(eligible) < 2:
|
| 56 |
+
raise ValueError(f'Cannot stratify eligible episodes for {pool}/{task}')
|
| 57 |
+
nval = min(len(eligible)-1, max(1, round(len(eligible)*fraction)))
|
| 58 |
+
val = set(eligible[:nval])
|
| 59 |
+
for key in keys:
|
| 60 |
+
for ep in identities[key]:
|
| 61 |
+
rows.append({**ep, 'pool': pool, 'split': 'validation' if key in val else 'train'})
|
| 62 |
+
rows.sort(key=lambda e: e['episode_index'])
|
| 63 |
+
manifest = {'dataset': DATASET, 'dataset_revision': DATASET_REVISION,
|
| 64 |
+
'fraction': fraction, 'seed': seed, 'provenance_sha256': digest(episodes), 'episodes': rows}
|
| 65 |
+
verify_split(manifest)
|
| 66 |
+
return manifest
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def verify_split(manifest: dict) -> None:
|
| 70 |
+
sets = {side: {source_identity(e) for e in manifest['episodes'] if e['split'] == side}
|
| 71 |
+
for side in ('train', 'validation')}
|
| 72 |
+
if sets['train'] & sets['validation']:
|
| 73 |
+
raise ValueError('Source episode leakage')
|
| 74 |
+
ids = {side: {e['episode_index'] for e in manifest['episodes'] if e['split'] == side}
|
| 75 |
+
for side in sets}
|
| 76 |
+
if ids['train'] & ids['validation']:
|
| 77 |
+
raise ValueError('Merged episode leakage')
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
class PoolSampler:
|
| 81 |
+
"""One synchronous draw per sample, with recorded video failures on retry."""
|
| 82 |
+
def __init__(self, manifest: dict, *, split='train', weights=WEIGHTS, seed=42):
|
| 83 |
+
verify_split(manifest)
|
| 84 |
+
if len(weights) != 3 or any(not math.isfinite(w) or w <= 0 for w in weights):
|
| 85 |
+
raise ValueError('Three positive finite pool weights required')
|
| 86 |
+
self.weights = tuple(w/sum(weights) for w in weights)
|
| 87 |
+
self.manifest_hash = digest(manifest)
|
| 88 |
+
self.split = split
|
| 89 |
+
self.rng = random.Random(seed)
|
| 90 |
+
self.groups = {p: defaultdict(list) for p in POOLS}
|
| 91 |
+
self.counts = Counter({p: 0 for p in POOLS})
|
| 92 |
+
self.task_counts = Counter()
|
| 93 |
+
self.video_decode_failures = Counter()
|
| 94 |
+
self.draws = 0
|
| 95 |
+
for ep in manifest['episodes']:
|
| 96 |
+
if ep['split'] == split and ep['length'] > 16:
|
| 97 |
+
self.groups[ep['pool']][ep['task_index']].append(ep)
|
| 98 |
+
if any(not self.groups[p] for p in POOLS):
|
| 99 |
+
raise ValueError(f'Empty eligible pool in {split}')
|
| 100 |
+
self.tasks = {p: sorted(self.groups[p]) for p in POOLS}
|
| 101 |
+
|
| 102 |
+
def draw(self) -> tuple[int, dict]:
|
| 103 |
+
pool = self.rng.choices(POOLS, self.weights)[0]
|
| 104 |
+
task = self.rng.choice(self.tasks[pool])
|
| 105 |
+
self.counts[pool] += 1
|
| 106 |
+
self.task_counts[pool, task] += 1
|
| 107 |
+
self.draws += 1
|
| 108 |
+
return self._draw_anchor(pool, task)
|
| 109 |
+
|
| 110 |
+
def _draw_anchor(self, pool, task) -> tuple[int, dict]:
|
| 111 |
+
ep = self.rng.choice(self.groups[pool][task])
|
| 112 |
+
t = self.rng.randrange(ep['length'] - 16)
|
| 113 |
+
return ep['dataset_from_index'] + t, {**ep, 'anchor': t}
|
| 114 |
+
|
| 115 |
+
def replacement(self, episode: dict) -> tuple[int, dict]:
|
| 116 |
+
"""Replace only the episode/anchor; keep this draw's pool, task and split."""
|
| 117 |
+
if episode['split'] != self.split:
|
| 118 |
+
raise ValueError('Replacement sample split mismatch')
|
| 119 |
+
return self._draw_anchor(episode['pool'], episode['task_index'])
|
| 120 |
+
|
| 121 |
+
def state_dict(self) -> dict:
|
| 122 |
+
return copy.deepcopy({'rng': self.rng.getstate(), 'counts': self.counts,
|
| 123 |
+
'task_counts': self.task_counts, 'draws': self.draws,
|
| 124 |
+
'video_decode_failures': self.video_decode_failures,
|
| 125 |
+
'manifest_hash': self.manifest_hash, 'weights': self.weights, 'split': self.split})
|
| 126 |
+
|
| 127 |
+
def load_state_dict(self, state: dict) -> None:
|
| 128 |
+
for key in ('manifest_hash', 'weights', 'split'):
|
| 129 |
+
if state[key] != getattr(self, key):
|
| 130 |
+
raise ValueError(f'Sampler resume mismatch: {key}')
|
| 131 |
+
self.rng.setstate(state['rng'])
|
| 132 |
+
self.counts, self.task_counts = Counter(state['counts']), Counter(state['task_counts'])
|
| 133 |
+
# Checkpoints created before video recovery did not have this counter.
|
| 134 |
+
self.video_decode_failures = Counter(state.get('video_decode_failures', {}))
|
| 135 |
+
self.draws = state['draws']
|
| 136 |
+
|
| 137 |
+
def distribution_report(self, check=False) -> dict:
|
| 138 |
+
report = {'draws': self.draws, 'counts': dict(self.counts),
|
| 139 |
+
'video_decode_failures': dict(self.video_decode_failures),
|
| 140 |
+
'percentages': {p: 100*self.counts[p]/max(1,self.draws) for p in POOLS}}
|
| 141 |
+
if check and self.draws >= 10000:
|
| 142 |
+
for pool, prob in zip(POOLS, self.weights, strict=True):
|
| 143 |
+
tolerance = max(0.01, 6*math.sqrt(prob*(1-prob)/self.draws))
|
| 144 |
+
if abs(self.counts[pool]/self.draws-prob) > tolerance:
|
| 145 |
+
raise ValueError(f'Unexpected pool distribution: {report}')
|
| 146 |
+
n = self.counts[pool]
|
| 147 |
+
p = 1/len(self.tasks[pool])
|
| 148 |
+
for task in self.tasks[pool]:
|
| 149 |
+
if abs(self.task_counts[pool, task]-n*p) > max(10, 7*math.sqrt(n*p*(1-p))):
|
| 150 |
+
raise ValueError(f'Unexpected task distribution: {pool}/{task}')
|
| 151 |
+
return report
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def resize_rgb(x: torch.Tensor, size: int) -> torch.Tensor:
|
| 155 |
+
shape = x.shape
|
| 156 |
+
return F.interpolate(x.reshape(-1, 3, *shape[-2:]), size=(size,size), mode='bilinear',
|
| 157 |
+
align_corners=False, antialias=True).reshape(*shape[:-2], size,size)
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
class ResolutionBottleneck(torch.nn.Module):
|
| 161 |
+
"""After released ToTensor, before photometric augmentation; all temporal frames."""
|
| 162 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 163 |
+
if x.shape[-3:] != (3,256,256) or not x.is_floating_point():
|
| 164 |
+
raise ValueError(f'Expected float RGB [...,3,256,256], got {x.shape}/{x.dtype}')
|
| 165 |
+
if not bool(torch.isfinite(x).all()) or x.min() < 0 or x.max() > 1:
|
| 166 |
+
raise ValueError('Expected finite RGB [0,1]')
|
| 167 |
+
return resize_rgb(resize_rgb(x,128),224)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def competition_cameras(front: torch.Tensor, wrist: torch.Tensor, *, rotate_180=True) -> torch.Tensor:
|
| 171 |
+
"""Simulator uint8 HWC -> normalized CHW. Stored training videos never rotate."""
|
| 172 |
+
cameras = []
|
| 173 |
+
for image in (front,wrist):
|
| 174 |
+
if image.shape != (128,128,3) or image.dtype != torch.uint8:
|
| 175 |
+
raise ValueError('Competition camera must be uint8 [128,128,3]')
|
| 176 |
+
x = image.permute(2,0,1).float()/255
|
| 177 |
+
if rotate_180:
|
| 178 |
+
x = x.flip((-2,-1))
|
| 179 |
+
cameras.append(resize_rgb(x,224))
|
| 180 |
+
return torch.cat(cameras,dim=-1)*2-1
|
checkpoints/step_004000/metadata/source/resolution_adapt/model.py
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Strict Stage-2 warm start and audited hybrid PEFT; usable entirely offline."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from collections import defaultdict
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
IMAGEWAM_COMMIT = '1792d100099771fe404ebca5dcd606725141585b'
|
| 9 |
+
FLUX_COMMIT = '50fe5162777813d869182b139e83b10743caef15'
|
| 10 |
+
LIT_REVISION = '765a2b2bf136f7039ab18978920c682f1aae65d2'
|
| 11 |
+
TARGETS = ('qkv','proj','linear1','linear2','img_mlp.0','img_mlp.2','txt_mlp.0','txt_mlp.2')
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def configure_peft(model, args) -> tuple[list[dict], dict]:
|
| 15 |
+
from imagewam.models.backbones.lora import apply_lora_to_linear_suffixes
|
| 16 |
+
model.requires_grad_(True)
|
| 17 |
+
transformer = model.video_expert.transformer
|
| 18 |
+
# Freeze all original video-expert parameters, not just the selected linears.
|
| 19 |
+
model.video_expert.requires_grad_(False)
|
| 20 |
+
model.vae.requires_grad_(False)
|
| 21 |
+
qwen = getattr(getattr(model,'text_encoder',None),'model',None)
|
| 22 |
+
if qwen is not None:
|
| 23 |
+
qwen.requires_grad_(False)
|
| 24 |
+
qwen.eval()
|
| 25 |
+
matches = [n for n,m in transformer.named_modules()
|
| 26 |
+
if isinstance(m,torch.nn.Linear) and any(n.endswith(s) for s in TARGETS)]
|
| 27 |
+
if not matches:
|
| 28 |
+
raise RuntimeError('No supported FLUX LoRA targets found')
|
| 29 |
+
result = apply_lora_to_linear_suffixes(transformer,target_suffixes=TARGETS,
|
| 30 |
+
rank=args.lora_rank,alpha=args.lora_alpha,dropout=args.lora_dropout)
|
| 31 |
+
if result.replaced != len(matches) or not result.replaced:
|
| 32 |
+
raise RuntimeError('FLUX LoRA injection mismatch')
|
| 33 |
+
# Upstream creates adapter tensors on CPU; move only the new tensors.
|
| 34 |
+
for module in transformer.modules():
|
| 35 |
+
if hasattr(module,'lora_A'):
|
| 36 |
+
module.lora_A = torch.nn.Parameter(module.lora_A.to(module.base.weight.device))
|
| 37 |
+
module.lora_B = torch.nn.Parameter(module.lora_B.to(module.base.weight.device))
|
| 38 |
+
blocks = list(transformer.double_blocks)+list(transformer.single_blocks)
|
| 39 |
+
n = args.unfreeze_last_flux_blocks
|
| 40 |
+
if not 0 <= n <= len(blocks):
|
| 41 |
+
raise ValueError(f'unfreeze_last_flux_blocks must be in [0,{len(blocks)}]')
|
| 42 |
+
if n:
|
| 43 |
+
for block in blocks[-n:]:
|
| 44 |
+
block.requires_grad_(True)
|
| 45 |
+
model.video_expert.flux2_lora_enabled = True
|
| 46 |
+
model.video_expert.flux2_lora_target_suffixes = TARGETS
|
| 47 |
+
# Do not call upstream apply_trainable_policy: its Stage-2 branch unfreezes FLUX.
|
| 48 |
+
owners = {}
|
| 49 |
+
def mark(module, group):
|
| 50 |
+
if module is not None:
|
| 51 |
+
for p in module.parameters():
|
| 52 |
+
owners[id(p)] = group
|
| 53 |
+
mark(model.video_expert,'flux_dense')
|
| 54 |
+
for name,p in transformer.named_parameters():
|
| 55 |
+
if name.endswith(('.lora_A','.lora_B')):
|
| 56 |
+
owners[id(p)] = 'flux_lora'
|
| 57 |
+
mark(model.action_expert,'action_expert')
|
| 58 |
+
mark(model.semantic_visual_aggregator,'lit_interface')
|
| 59 |
+
mark(model.semantic_visual_pose_norm,'pose_decoder')
|
| 60 |
+
mark(model.semantic_visual_pose_decoder,'pose_decoder')
|
| 61 |
+
mark(model.proprio_encoder,'proprio_encoder')
|
| 62 |
+
mark(model.vae,'vae')
|
| 63 |
+
mark(qwen,'qwen')
|
| 64 |
+
rates = {'flux_lora':args.lr_flux_lora,'flux_dense':args.lr_unfrozen_flux,
|
| 65 |
+
'action_expert':args.lr_action,'lit_interface':args.lr_interface,
|
| 66 |
+
'pose_decoder':args.lr_pose,'proprio_encoder':args.lr_proprio,'other_stage2':args.lr_other}
|
| 67 |
+
grouped = defaultdict(list)
|
| 68 |
+
report = {k:{'total':0,'trainable':0,'frozen':0} for k in (*rates,'qwen','vae')}
|
| 69 |
+
for name,p in model.named_parameters():
|
| 70 |
+
group = owners.get(id(p),'other_stage2')
|
| 71 |
+
row = report[group]
|
| 72 |
+
row['total'] += p.numel()
|
| 73 |
+
row['trainable' if p.requires_grad else 'frozen'] += p.numel()
|
| 74 |
+
if p.requires_grad:
|
| 75 |
+
if group not in rates:
|
| 76 |
+
raise RuntimeError(f'Frozen-only module trains: {name}')
|
| 77 |
+
# FP32 master parameter and AdamW moments; BF16 forward via autocast.
|
| 78 |
+
p.data = p.data.float()
|
| 79 |
+
grouped[group].append(p)
|
| 80 |
+
# SimpleNamespace text encoders are not registered PyTorch children.
|
| 81 |
+
if qwen is not None and not report['qwen']['total']:
|
| 82 |
+
report['qwen'] = {'total':sum(p.numel() for p in qwen.parameters()),'trainable':0,
|
| 83 |
+
'frozen':sum(p.numel() for p in qwen.parameters())}
|
| 84 |
+
for key in ('flux_lora','action_expert','lit_interface','pose_decoder','proprio_encoder'):
|
| 85 |
+
if report[key]['trainable'] == 0 or report[key]['frozen']:
|
| 86 |
+
raise RuntimeError(f'Required fully trainable group failed: {key}')
|
| 87 |
+
if report['vae']['trainable'] or report['qwen']['trainable'] or (n == 0 and report['flux_dense']['trainable']):
|
| 88 |
+
raise RuntimeError('Freeze policy violated')
|
| 89 |
+
groups = []
|
| 90 |
+
# Bound optimizer temporaries with the repository's 128M-element policy.
|
| 91 |
+
for name,parameters in grouped.items():
|
| 92 |
+
part, count = [],0
|
| 93 |
+
for p in parameters:
|
| 94 |
+
if part and count+p.numel() > 128_000_000:
|
| 95 |
+
groups.append({'params':part,'name':name,'lr':rates[name],'weight_decay':args.weight_decay})
|
| 96 |
+
part,count = [],0
|
| 97 |
+
part.append(p); count += p.numel()
|
| 98 |
+
if part:
|
| 99 |
+
groups.append({'params':part,'name':name,'lr':rates[name],'weight_decay':args.weight_decay})
|
| 100 |
+
included = [id(p) for g in groups for p in g['params']]
|
| 101 |
+
expected = {id(p) for p in model.parameters() if p.requires_grad}
|
| 102 |
+
if len(included) != len(set(included)) or set(included) != expected:
|
| 103 |
+
raise RuntimeError('Optimizer coverage mismatch')
|
| 104 |
+
totals = {k:sum(row[k] for row in report.values()) for k in ('total','trainable','frozen')}
|
| 105 |
+
totals['lora'] = report['flux_lora']['trainable']
|
| 106 |
+
totals['dense_trainable'] = totals['trainable']-totals['lora']
|
| 107 |
+
table = [{'name':g['name'],'tensors':len(g['params']),'parameters':sum(p.numel() for p in g['params']),
|
| 108 |
+
'lr':g['lr'],'weight_decay':g['weight_decay']} for g in groups]
|
| 109 |
+
return groups, {'components':report,'totals':totals,'optimizer_groups':table,'lora_targets':matches,
|
| 110 |
+
'qwen_cached_and_not_instantiated':qwen is None}
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def trainable_state(model) -> dict:
|
| 114 |
+
return {name:p.detach().cpu().clone() for name,p in model.named_parameters() if p.requires_grad}
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def load_adaptation(model, path: Path) -> None:
|
| 118 |
+
"""After released base + identical LoRA injection; no network/logging dependencies."""
|
| 119 |
+
state = torch.load(path,map_location='cpu',weights_only=True)
|
| 120 |
+
expected = {n:p for n,p in model.named_parameters() if p.requires_grad}
|
| 121 |
+
if state.keys() != expected.keys():
|
| 122 |
+
raise ValueError('Adaptation parameter names do not match the configured base + PEFT model')
|
| 123 |
+
with torch.no_grad():
|
| 124 |
+
for name,p in expected.items():
|
| 125 |
+
if p.shape != state[name].shape:
|
| 126 |
+
raise ValueError(f'Adaptation shape mismatch: {name}')
|
| 127 |
+
p.copy_(state[name])
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def create_stage2_meta(config: dict):
|
| 131 |
+
"""Same construction as existing offline model_loader, with released config fields.
|
| 132 |
+
|
| 133 |
+
Meta initialization avoids both a second 4B base copy and any action initializer.
|
| 134 |
+
Every trainable tensor must be assigned from the full released Stage-2 payload.
|
| 135 |
+
"""
|
| 136 |
+
from flux2.model import Flux2, Klein4BParams
|
| 137 |
+
from flux2.autoencoder import AutoEncoder, AutoEncoderParams
|
| 138 |
+
from imagewam.models.backbones.flux2_video_expert import Flux2VideoExpert
|
| 139 |
+
from imagewam.models.backbones.action_dit_flux2 import ActionDiTFlux2
|
| 140 |
+
from imagewam.models.backbones.mot import MoT
|
| 141 |
+
from imagewam.models.backbones.imagewam import ImageWAM
|
| 142 |
+
with torch.device('meta'):
|
| 143 |
+
video = Flux2VideoExpert(Flux2(Klein4BParams()).to(torch.bfloat16),'klein-base-4b')
|
| 144 |
+
action = ActionDiTFlux2(**config['action_dit_config']).to(torch.bfloat16)
|
| 145 |
+
mot = MoT(mixtures={'video':video,'action':action},
|
| 146 |
+
mot_checkpoint_mixed_attn=config['mot_checkpoint_mixed_attn'],
|
| 147 |
+
gqa_implementation=config['mot_gqa_implementation'],
|
| 148 |
+
force_flash_attention=config['mot_force_flash_attention'])
|
| 149 |
+
vae = AutoEncoder(AutoEncoderParams()).to(torch.bfloat16)
|
| 150 |
+
v,a = config['video_scheduler'],config['action_scheduler']
|
| 151 |
+
model = ImageWAM(video,action,mot,vae,text_dim=7680,proprio_dim=8,device='meta',
|
| 152 |
+
torch_dtype=torch.bfloat16,stack='flux2',qwen_context_len=512,pack_proprio_after_text=True,
|
| 153 |
+
goal_prior_stage='stage2',goal_prior=config['goal_prior'],
|
| 154 |
+
video_train_shift=v['train_shift'],video_infer_shift=v['infer_shift'],
|
| 155 |
+
video_num_train_timesteps=v['num_train_timesteps'],
|
| 156 |
+
action_train_shift=a['train_shift'],action_infer_shift=a['infer_shift'],
|
| 157 |
+
action_num_train_timesteps=a['num_train_timesteps'],
|
| 158 |
+
loss_lambda_video=0.5,loss_lambda_action=1.0,loss_lambda_pose=0.3)
|
| 159 |
+
return model
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def build_stage2(config: dict, checkpoint: Path, ae_path: Path, device='cuda'):
|
| 163 |
+
from safetensors.torch import load_file
|
| 164 |
+
model = create_stage2_meta(config)
|
| 165 |
+
payload = torch.load(checkpoint,map_location='cpu',mmap=True,weights_only=True)
|
| 166 |
+
required = {'mot','proprio_encoder','semantic_visual_aggregator',
|
| 167 |
+
'semantic_visual_pose_norm','semantic_visual_pose_decoder'}
|
| 168 |
+
if set(payload) != required|{'step','torch_dtype','goal_prior_stage'} or payload['goal_prior_stage'] != 'stage2':
|
| 169 |
+
raise ValueError('Expected complete released Stage-2 payload, without LoRA or Stage-1 bridging')
|
| 170 |
+
cache,report = {},{}
|
| 171 |
+
for name in sorted(required):
|
| 172 |
+
state = {}
|
| 173 |
+
for key,value in payload[name].items():
|
| 174 |
+
identity = (value.data_ptr(),tuple(value.shape),tuple(value.stride()))
|
| 175 |
+
if identity not in cache:
|
| 176 |
+
cache[identity] = value.to(device=device,dtype=torch.bfloat16)
|
| 177 |
+
state[key] = cache[identity]
|
| 178 |
+
result = getattr(model,name).load_state_dict(state,strict=False,assign=True)
|
| 179 |
+
report[name] = {'missing_keys':list(result.missing_keys),'unexpected_keys':list(result.unexpected_keys)}
|
| 180 |
+
print('Released checkpoint keys:',report,flush=True)
|
| 181 |
+
if any(r['missing_keys'] or r['unexpected_keys'] for r in report.values()):
|
| 182 |
+
raise RuntimeError(f'Stage-2 load mismatch: {report}')
|
| 183 |
+
model.vae.load_state_dict(load_file(str(ae_path),device=device),strict=True,assign=True)
|
| 184 |
+
model.to(device=device,dtype=torch.bfloat16)
|
| 185 |
+
model.device = torch.device(device)
|
| 186 |
+
if any(p.is_meta for p in model.parameters()):
|
| 187 |
+
raise RuntimeError('Uninitialized model tensor after strict Stage-2 load')
|
| 188 |
+
return model,report
|
checkpoints/step_004000/metadata/source/resolution_adapt/offline.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Offline reconstruction: released Stage-2 + adaptation weights + local AE/Qwen."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import json
|
| 4 |
+
import hashlib
|
| 5 |
+
import sys
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from types import SimpleNamespace
|
| 8 |
+
import torch
|
| 9 |
+
import yaml
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def load_offline(checkpoint: Path, *, released_model: Path, autoencoder: Path,
|
| 13 |
+
qwen: Path, imagewam_source: Path, flux_source: Path, device='cuda'):
|
| 14 |
+
"""Requires packaged local assets only. No W&B, HF API, or training dataset imports."""
|
| 15 |
+
sys.path[:0] = [str(imagewam_source/'src'),str(flux_source/'src')]
|
| 16 |
+
from .model import build_stage2,configure_peft,load_adaptation
|
| 17 |
+
metadata=checkpoint/'metadata'
|
| 18 |
+
lock=json.loads((metadata/'assets.lock.json').read_text())
|
| 19 |
+
for path,expected in [(released_model,lock['linjianman/LIT']['files']['imagewam/lit_stage2/model.pt']['sha256']),
|
| 20 |
+
(autoencoder,lock['black-forest-labs/FLUX.2-dev']['files']['ae.safetensors']['sha256'])]:
|
| 21 |
+
with path.open('rb') as stream:
|
| 22 |
+
actual=hashlib.file_digest(stream,'sha256').hexdigest()
|
| 23 |
+
if actual!=expected: raise ValueError(f'Offline base asset hash mismatch: {path}')
|
| 24 |
+
config=yaml.safe_load((metadata/'config.yaml').read_text())
|
| 25 |
+
args=SimpleNamespace(**json.loads((metadata/'args.json').read_text()))
|
| 26 |
+
model,_=build_stage2(config['model'],released_model,autoencoder,device=device)
|
| 27 |
+
configure_peft(model,args)
|
| 28 |
+
load_adaptation(model,checkpoint/'weights/adaptation.pt')
|
| 29 |
+
# Merge using the upstream implementation, which honors alpha/rank exactly.
|
| 30 |
+
from imagewam.models.backbones.lora import merge_lora_linear_layers
|
| 31 |
+
merge_lora_linear_layers(model.video_expert.transformer)
|
| 32 |
+
model.to(device=device,dtype=torch.bfloat16).eval().requires_grad_(False)
|
| 33 |
+
from transformers import AutoModelForCausalLM,AutoTokenizer
|
| 34 |
+
encoder=AutoModelForCausalLM.from_pretrained(qwen,local_files_only=True,torch_dtype=torch.bfloat16,
|
| 35 |
+
attn_implementation='sdpa').to(device).eval().requires_grad_(False)
|
| 36 |
+
model.text_encoder=SimpleNamespace(model=encoder,tokenizer=AutoTokenizer.from_pretrained(qwen,local_files_only=True),
|
| 37 |
+
max_length=512)
|
| 38 |
+
return model
|
checkpoints/step_004000/metadata/source/resolution_adapt/preflight_data.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Read-only representative endpoint audit; optional bounded HTTP video streaming."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import argparse
|
| 4 |
+
import json
|
| 5 |
+
import sys
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import torch
|
| 10 |
+
from PIL import Image, ImageDraw
|
| 11 |
+
|
| 12 |
+
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
| 13 |
+
from resolution_adapt.data import DATASET,DATASET_REVISION,POOLS,pool_of,ResolutionBottleneck,resize_rgb
|
| 14 |
+
from resolution_adapt.model import LIT_REVISION
|
| 15 |
+
from training_support import DatasetGripperToImageWAM
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def audit(root: Path, output: Path, *, stream=False, stats_path=None, processed=None):
|
| 19 |
+
import av
|
| 20 |
+
import pyarrow.parquet as pq
|
| 21 |
+
from huggingface_hub import hf_hub_download,hf_hub_url,get_token
|
| 22 |
+
output.mkdir(parents=True,exist_ok=True)
|
| 23 |
+
def file(name):
|
| 24 |
+
return Path(hf_hub_download(DATASET,name,repo_type='dataset',revision=DATASET_REVISION,
|
| 25 |
+
local_dir=str(root))) if stream else root/name
|
| 26 |
+
provenance=[json.loads(line) for line in file('meta/source_episodes.jsonl').read_text().splitlines()]
|
| 27 |
+
info=json.loads(file('meta/info.json').read_text())
|
| 28 |
+
ep_rows=pq.read_table(file('meta/episodes/chunk-000/file-000.parquet')).to_pylist()
|
| 29 |
+
ep_by_id={e['episode_index']:e for e in ep_rows}
|
| 30 |
+
if stats_path is None:
|
| 31 |
+
stats_path=hf_hub_download('linjianman/LIT','imagewam/lit_stage2/dataset_stats.json',revision=LIT_REVISION)
|
| 32 |
+
stats=json.loads(Path(stats_path).read_text())
|
| 33 |
+
def normalize(value,category):
|
| 34 |
+
st=stats[category]['default']; low=torch.tensor(st['global_min']); high=torch.tensor(st['global_max'])
|
| 35 |
+
span=high-low; constant=span<1e-4; span[constant]=2
|
| 36 |
+
scale=2/span; offset=-1-scale*low; offset[constant]=-low[constant]
|
| 37 |
+
return (value*scale+offset).clamp(-5,5)
|
| 38 |
+
report=[]; selected=[next(e for e in provenance if pool_of(e)==p and e['length']>32) for p in POOLS]
|
| 39 |
+
for prov in selected:
|
| 40 |
+
pool=pool_of(prov); ep=ep_by_id[prov['episode_index']]
|
| 41 |
+
shard=info['data_path'].format(chunk_index=ep['data/chunk_index'],file_index=ep['data/file_index'])
|
| 42 |
+
rows=pq.read_table(file(shard),filters=[('episode_index','=',ep['episode_index'])]).to_pylist()
|
| 43 |
+
rows.sort(key=lambda r:r['frame_index'])
|
| 44 |
+
t=0
|
| 45 |
+
states=torch.tensor([r['observation.state'] for r in rows[:17]],dtype=torch.float32)
|
| 46 |
+
actions=torch.tensor([r['action'] for r in rows[:16]],dtype=torch.float32)
|
| 47 |
+
assert states.shape==(17,8) and actions.shape==(16,7)
|
| 48 |
+
assert all(r['episode_index']==prov['episode_index'] for r in rows[:17])
|
| 49 |
+
original=actions.clone()
|
| 50 |
+
converted=DatasetGripperToImageWAM().forward({'action':{'default':actions}})['action']['default']
|
| 51 |
+
assert torch.equal(converted[:,:6],original[:,:6])
|
| 52 |
+
assert torch.equal(converted[:,-1],(1-original[:,-1])/2)
|
| 53 |
+
cameras=[]; panels=[]
|
| 54 |
+
for camera in ('front','wrist'):
|
| 55 |
+
key=f'observation.images.{camera}'
|
| 56 |
+
video=info['video_path'].format(video_key=key,chunk_index=ep[f'videos/{key}/chunk_index'],
|
| 57 |
+
file_index=ep[f'videos/{key}/file_index'])
|
| 58 |
+
timestamps=[ep[f'videos/{key}/from_timestamp']+rows[i]['timestamp'] for i in (0,16)]
|
| 59 |
+
options={'rw_timeout':'30000000'}
|
| 60 |
+
location=str(root/video)
|
| 61 |
+
if stream:
|
| 62 |
+
location=hf_hub_url(DATASET,video,repo_type='dataset',revision=DATASET_REVISION)
|
| 63 |
+
token=get_token()
|
| 64 |
+
if token: options['headers']='Authorization: Bearer '+token+'\r\n'
|
| 65 |
+
with av.open(location,options=options) as container:
|
| 66 |
+
video_stream=container.streams.video[0]
|
| 67 |
+
container.seek(int(timestamps[0]/video_stream.time_base),stream=video_stream,backward=True)
|
| 68 |
+
decoded={}
|
| 69 |
+
for frame in container.decode(video_stream):
|
| 70 |
+
timestamp=float(frame.pts*video_stream.time_base)
|
| 71 |
+
for i,target in enumerate(timestamps):
|
| 72 |
+
if abs(timestamp-target)<1e-4: decoded[i]=frame.to_ndarray(format='rgb24')
|
| 73 |
+
if len(decoded)==2: break
|
| 74 |
+
if timestamp > timestamps[-1]+.1: raise ValueError('Missing endpoint video frame')
|
| 75 |
+
raw=torch.from_numpy(np.stack([decoded[i] for i in (0,1)])).permute(0,3,1,2)
|
| 76 |
+
assert raw.shape==(2,3,256,256) and raw.dtype==torch.uint8
|
| 77 |
+
source=raw.float()/255; degraded=resize_rgb(source,128); final=ResolutionBottleneck()(source)
|
| 78 |
+
assert degraded.shape==(2,3,128,128) and final.shape==(2,3,224,224)
|
| 79 |
+
cameras.append(final)
|
| 80 |
+
panels.append((camera,source[0],degraded[0],final[0]))
|
| 81 |
+
combined=torch.cat(cameras,dim=-1)*2-1
|
| 82 |
+
assert combined.shape==(2,3,224,448) and -1<=combined.min()<=combined.max()<=1
|
| 83 |
+
normalized_action=normalize(converted,'action'); pose=normalize(states[16],'state')
|
| 84 |
+
if processed is not None:
|
| 85 |
+
augment=processed.video_augmentation; processed.video_augmentation=None
|
| 86 |
+
try: sample=processed[prov['dataset_from_index']+t]
|
| 87 |
+
finally: processed.video_augmentation=augment
|
| 88 |
+
torch.testing.assert_close(sample['goal_pose'],pose)
|
| 89 |
+
torch.testing.assert_close(sample['action'],normalized_action)
|
| 90 |
+
torch.testing.assert_close(sample['video'],combined.permute(1,0,2,3),atol=.02,rtol=0)
|
| 91 |
+
assert not sample['goal_pose_is_pad'].any()
|
| 92 |
+
canvas=Image.new('RGB',(720,560),'white'); draw=ImageDraw.Draw(canvas)
|
| 93 |
+
for row,(camera,*images) in enumerate(panels):
|
| 94 |
+
x=0
|
| 95 |
+
for label,img in zip(('source 256','bottleneck 128','model 224'),images,strict=True):
|
| 96 |
+
array=(img.permute(1,2,0).numpy()*255).round().clip(0,255).astype(np.uint8)
|
| 97 |
+
canvas.paste(Image.fromarray(array),(x,row*280+22))
|
| 98 |
+
draw.text((x,row*280),f'{pool} {camera}: {label}',fill='black'); x+=img.shape[-1]+20
|
| 99 |
+
canvas.save(output/f'{pool}.png')
|
| 100 |
+
report.append({'pool':pool,'episode':prov['episode_index'],'anchor':t,'pose_frame':t+16,
|
| 101 |
+
'raw':[3,256,256],'degraded':[3,128,128],'model':[3,224,224],'combined':[3,224,448],
|
| 102 |
+
'raw_rgb_range':[0,255],'float_range':[0,1],'normalized_image_range':[float(combined.min()),float(combined.max())],
|
| 103 |
+
'state_shape':[8],'action_shape':list(converted.shape),'pose_shape':list(pose.shape),
|
| 104 |
+
'action_normalized_range':[float(normalized_action.min()),float(normalized_action.max())],
|
| 105 |
+
'gripper_exactly_once':True,'stored_rotation':False,'processed_loader_compared':processed is not None})
|
| 106 |
+
(output/'dataset_preflight.json').write_text(json.dumps(report,indent=2)+'\n')
|
| 107 |
+
return report
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
if __name__=='__main__':
|
| 111 |
+
p=argparse.ArgumentParser();p.add_argument('--dataset-root',type=Path,required=True)
|
| 112 |
+
p.add_argument('--output',type=Path,required=True);p.add_argument('--stream-from-hub',action='store_true')
|
| 113 |
+
p.add_argument('--stats',type=Path)
|
| 114 |
+
a=p.parse_args(); print(json.dumps(audit(a.dataset_root,a.output,stream=a.stream_from_hub,stats_path=a.stats),indent=2))
|
checkpoints/step_004000/metadata/source/resolution_adapt/preflight_model.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CPU architecture counts and actual remote checkpoint key/shape audit via range reads.
|
| 2 |
+
|
| 3 |
+
This deliberately does NOT claim to validate tensor bytes or a real forward pass.
|
| 4 |
+
"""
|
| 5 |
+
from __future__ import annotations
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
import torch
|
| 11 |
+
import yaml
|
| 12 |
+
|
| 13 |
+
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
| 14 |
+
from resolution_adapt.model import create_stage2_meta,configure_peft,LIT_REVISION
|
| 15 |
+
from train_resolution_adapt import parser as training_parser,check_sources
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def main():
|
| 19 |
+
p=argparse.ArgumentParser()
|
| 20 |
+
p.add_argument('--imagewam-root',type=Path,required=True); p.add_argument('--flux-root',type=Path,required=True)
|
| 21 |
+
p.add_argument('--output',type=Path,required=True); args=p.parse_args(); args.output.mkdir(parents=True,exist_ok=True)
|
| 22 |
+
check_sources(args)
|
| 23 |
+
from huggingface_hub import HfFileSystem
|
| 24 |
+
from torch._subclasses.fake_tensor import FakeTensorMode
|
| 25 |
+
model=create_stage2_meta(yaml.safe_load((Path(__file__).parent/'released_stage2.yaml').read_text())['model'])
|
| 26 |
+
with HfFileSystem().open(f'linjianman/LIT@{LIT_REVISION}/imagewam/lit_stage2/model.pt','rb',block_size=1024*1024) as stream:
|
| 27 |
+
with FakeTensorMode(): payload=torch.load(stream,map_location='cpu',weights_only=True)
|
| 28 |
+
required={'mot','proprio_encoder','semantic_visual_aggregator','semantic_visual_pose_norm','semantic_visual_pose_decoder'}
|
| 29 |
+
if set(payload)!=required|{'step','torch_dtype','goal_prior_stage'} or payload['goal_prior_stage']!='stage2':
|
| 30 |
+
raise ValueError('Not a complete released Stage-2 payload')
|
| 31 |
+
report={}
|
| 32 |
+
for name in sorted(required):
|
| 33 |
+
expected=getattr(model,name).state_dict(); actual=payload[name]
|
| 34 |
+
report[name]={'missing_keys':sorted(set(expected)-set(actual)),'unexpected_keys':sorted(set(actual)-set(expected)),
|
| 35 |
+
'shape_mismatches':[k for k in expected.keys() & actual.keys() if expected[k].shape!=actual[k].shape]}
|
| 36 |
+
if any(report[name].values()): raise ValueError(report[name])
|
| 37 |
+
report['verification']='Actual pinned checkpoint metadata only; range reads with FakeTensorMode. GPU tensor load and backward remain required.'
|
| 38 |
+
(args.output/'checkpoint_keys.json').write_text(json.dumps(report,indent=2)+'\n')
|
| 39 |
+
_,counts=configure_peft(model,training_parser().parse_args([]))
|
| 40 |
+
counts['verification']='Exact pinned architecture on meta device; no checkpoint tensor bytes loaded'
|
| 41 |
+
(args.output/'parameter_report.json').write_text(json.dumps(counts,indent=2)+'\n')
|
| 42 |
+
print(json.dumps(report,indent=2));print(json.dumps(counts['totals'],indent=2))
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
if __name__=='__main__': main()
|
checkpoints/step_004000/metadata/source/resolution_adapt/prepare_assets.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Explicit asset downloads and optional frozen Qwen caching; never starts training."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import argparse
|
| 4 |
+
import hashlib
|
| 5 |
+
import json
|
| 6 |
+
import subprocess
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
| 11 |
+
from resolution_adapt.data import DATASET,DATASET_REVISION
|
| 12 |
+
from resolution_adapt.model import IMAGEWAM_COMMIT,FLUX_COMMIT,LIT_REVISION
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def main():
|
| 16 |
+
p=argparse.ArgumentParser(); p.add_argument('--runtime',type=Path,required=True)
|
| 17 |
+
p.add_argument('--cache-text',action='store_true'); args=p.parse_args()
|
| 18 |
+
root=args.runtime.resolve();root.mkdir(parents=True,exist_ok=True)
|
| 19 |
+
for name,url,commit in [('upstream','https://github.com/jianmanlincjx/ImageWAM.git',IMAGEWAM_COMMIT),
|
| 20 |
+
('flux2','https://github.com/black-forest-labs/flux2.git',FLUX_COMMIT)]:
|
| 21 |
+
target=root/name
|
| 22 |
+
if not target.exists(): subprocess.run(['git','clone',url,str(target)],check=True)
|
| 23 |
+
if subprocess.check_output(['git','-C',str(target),'status','--porcelain'],text=True):
|
| 24 |
+
raise ValueError(f'Refusing to change dirty checkout: {target}')
|
| 25 |
+
subprocess.run(['git','-C',str(target),'checkout',commit],check=True)
|
| 26 |
+
from huggingface_hub import snapshot_download,hf_hub_download
|
| 27 |
+
lock=json.loads((Path(__file__).parent/'assets.lock.json').read_text())
|
| 28 |
+
snapshot_download(DATASET,repo_type='dataset',revision=DATASET_REVISION,local_dir=root/'dataset')
|
| 29 |
+
snapshot_download('linjianman/LIT',revision=LIT_REVISION,allow_patterns=['imagewam/lit_stage2/*'],local_dir=root/'released')
|
| 30 |
+
ae=lock['black-forest-labs/FLUX.2-dev']
|
| 31 |
+
hf_hub_download('black-forest-labs/FLUX.2-dev','ae.safetensors',revision=ae['revision'],
|
| 32 |
+
local_dir=root/'models/FLUX.2-dev')
|
| 33 |
+
snapshot_download('Qwen/Qwen3-4B',revision=lock['Qwen/Qwen3-4B']['revision'],local_dir=root/'models/Qwen3-4B')
|
| 34 |
+
receipt={'dataset':DATASET,'dataset_revision':DATASET_REVISION,'models':lock,
|
| 35 |
+
'source_provenance_sha256':hashlib.sha256((root/'dataset/meta/source_episodes.jsonl').read_bytes()).hexdigest()}
|
| 36 |
+
(root/'assets.json').write_text(json.dumps(receipt,indent=2)+'\n')
|
| 37 |
+
if args.cache_text:
|
| 38 |
+
from train_resolution_adapt import parser,make_config
|
| 39 |
+
import yaml
|
| 40 |
+
cfg_args=parser().parse_args([])
|
| 41 |
+
cfg_args.dataset_root=root/'dataset';cfg_args.flux_root=root/'flux2';cfg_args.qwen_cache=root/'qwen_cache'
|
| 42 |
+
cfg=make_config(cfg_args)
|
| 43 |
+
cfg['flux2_qwen3_model_spec']=str(root/'models/Qwen3-4B')
|
| 44 |
+
cfg['hydra']={'run':{'dir':str(root/'cache_config/logs')}}
|
| 45 |
+
cfg['qwen_cache_batch_size']=1
|
| 46 |
+
cfg['qwen_cache_overwrite']=True # Never relabel an unverified old cache as pinned.
|
| 47 |
+
path=root/'cache_config';path.mkdir(exist_ok=True)
|
| 48 |
+
(path/'cache.yaml').write_text(yaml.safe_dump(cfg))
|
| 49 |
+
subprocess.run([sys.executable,str(root/'upstream/scripts/flux2/precompute_flux2_qwen3_embeds.py'),
|
| 50 |
+
'--config-path',str(path),'--config-name','cache'],check=True)
|
| 51 |
+
cache=root/'qwen_cache'
|
| 52 |
+
(cache/'provenance.json').write_text(json.dumps({'repo':'Qwen/Qwen3-4B','revision':lock['Qwen/Qwen3-4B']['revision'],
|
| 53 |
+
'imagewam_commit':IMAGEWAM_COMMIT,'context_len':512,'layers':[9,18,27]},indent=2)+'\n')
|
| 54 |
+
print('Assets prepared. No training launched.')
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
if __name__=='__main__': main()
|
checkpoints/step_004000/metadata/source/resolution_adapt/released_stage2.yaml
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
output_dir: ./runs/libero_flux2_klein_4b_goal_prior_stage2/2026-08-18_04-16-51
|
| 2 |
+
batch_size: 10
|
| 3 |
+
num_workers: 24
|
| 4 |
+
prefetch_factor: 2
|
| 5 |
+
persistent_workers: true
|
| 6 |
+
lr_scheduler_type: cosine
|
| 7 |
+
learning_rate: 0.0001
|
| 8 |
+
warmup_steps: null
|
| 9 |
+
num_epochs: 10
|
| 10 |
+
max_steps: null
|
| 11 |
+
log_every: 10
|
| 12 |
+
save_every: 1000
|
| 13 |
+
keep_latest_state_only: true
|
| 14 |
+
eval_every: 100
|
| 15 |
+
eval_num_inference_steps: 10
|
| 16 |
+
eval_num_samples: 8
|
| 17 |
+
rank_timer_every: 10
|
| 18 |
+
rank_timer_sync_cuda: true
|
| 19 |
+
qwen_cache_batch_size: 32
|
| 20 |
+
qwen_cache_overwrite: false
|
| 21 |
+
qwen_cache_save_workers: 8
|
| 22 |
+
flux2_qwen3_model_spec: null
|
| 23 |
+
precache_arrow: true
|
| 24 |
+
precache_num_workers: 4
|
| 25 |
+
build_lerobot_meta_cache: true
|
| 26 |
+
build_norm_stats: true
|
| 27 |
+
precache_fuse_norm_stats: true
|
| 28 |
+
force_rebuild_norm_stats: false
|
| 29 |
+
norm_stats_use_arrow_cache: true
|
| 30 |
+
norm_stats_bulk_read_arrow: true
|
| 31 |
+
norm_stats_cache_enabled: true
|
| 32 |
+
norm_stats_cache_dir: null
|
| 33 |
+
precache_profile: false
|
| 34 |
+
precache_profile_min_sec: 0.0
|
| 35 |
+
gradient_accumulation_steps: 1
|
| 36 |
+
mixed_precision: bf16
|
| 37 |
+
seed: 42
|
| 38 |
+
max_grad_norm: 1.0
|
| 39 |
+
weight_decay: 0.01
|
| 40 |
+
resume: null
|
| 41 |
+
stage1_checkpoint: /data2/JM/Code/ImageWAM-v2/runs/libero_flux2_klein_4b_goal_prior_stage1/2026-08-17_13-21-30/checkpoints/weights/step_010000.pt
|
| 42 |
+
wandb:
|
| 43 |
+
enabled: true
|
| 44 |
+
workspace: arisilin
|
| 45 |
+
project: imagewam
|
| 46 |
+
name: libero_flux2_klein_4b_goal_prior_stage2
|
| 47 |
+
group: null
|
| 48 |
+
mode: offline
|
| 49 |
+
data:
|
| 50 |
+
train:
|
| 51 |
+
_target_: imagewam.datasets.lerobot.robot_video_dataset.RobotVideoDataset
|
| 52 |
+
dataset_dirs:
|
| 53 |
+
- /data2/JM/Code/ImageWAM/data/libero_mujoco3.3.2/libero_spatial_no_noops_lerobot
|
| 54 |
+
- /data2/JM/Code/ImageWAM/data/libero_mujoco3.3.2/libero_object_no_noops_lerobot
|
| 55 |
+
- /data2/JM/Code/ImageWAM/data/libero_mujoco3.3.2/libero_goal_no_noops_lerobot
|
| 56 |
+
- /data2/JM/Code/ImageWAM/data/libero_mujoco3.3.2/libero_10_no_noops_lerobot
|
| 57 |
+
shape_meta:
|
| 58 |
+
images:
|
| 59 |
+
- key: image
|
| 60 |
+
raw_shape:
|
| 61 |
+
- 3
|
| 62 |
+
- 512
|
| 63 |
+
- 512
|
| 64 |
+
shape:
|
| 65 |
+
- 3
|
| 66 |
+
- 224
|
| 67 |
+
- 224
|
| 68 |
+
- key: wrist_image
|
| 69 |
+
raw_shape:
|
| 70 |
+
- 3
|
| 71 |
+
- 512
|
| 72 |
+
- 512
|
| 73 |
+
shape:
|
| 74 |
+
- 3
|
| 75 |
+
- 224
|
| 76 |
+
- 224
|
| 77 |
+
action:
|
| 78 |
+
- key: default
|
| 79 |
+
raw_shape: 7
|
| 80 |
+
shape: 7
|
| 81 |
+
state:
|
| 82 |
+
- key: default
|
| 83 |
+
raw_shape: 8
|
| 84 |
+
shape: 8
|
| 85 |
+
num_frames: 17
|
| 86 |
+
global_sample_stride: 1
|
| 87 |
+
action_video_freq_ratio: 1
|
| 88 |
+
video_size:
|
| 89 |
+
- 224
|
| 90 |
+
- 448
|
| 91 |
+
camera_key: null
|
| 92 |
+
val_set_proportion: 0.0
|
| 93 |
+
is_training_set: true
|
| 94 |
+
skip_padding_as_possible: false
|
| 95 |
+
concat_multi_camera: horizontal
|
| 96 |
+
video_augmentation:
|
| 97 |
+
_target_: imagewam.datasets.lerobot.transforms.image.VideoAugmentation
|
| 98 |
+
p: 0.65
|
| 99 |
+
augment_types:
|
| 100 |
+
- corrupt_only
|
| 101 |
+
- color_only
|
| 102 |
+
- both
|
| 103 |
+
- both
|
| 104 |
+
color_jitter:
|
| 105 |
+
brightness: 0.3
|
| 106 |
+
contrast: 0.3
|
| 107 |
+
saturation: 0.25
|
| 108 |
+
hue: 0.04
|
| 109 |
+
gamma:
|
| 110 |
+
range:
|
| 111 |
+
- 0.8
|
| 112 |
+
- 1.25
|
| 113 |
+
gaussian_noise:
|
| 114 |
+
std: 0.015
|
| 115 |
+
random_resized_crop:
|
| 116 |
+
scale:
|
| 117 |
+
- 0.92
|
| 118 |
+
- 1.0
|
| 119 |
+
ratio: preserve
|
| 120 |
+
rotate:
|
| 121 |
+
degrees: 8.0
|
| 122 |
+
fill: mean
|
| 123 |
+
exposure:
|
| 124 |
+
ev_range:
|
| 125 |
+
- -0.25
|
| 126 |
+
- 0.25
|
| 127 |
+
require_text_cache: false
|
| 128 |
+
endpoint_frames_only: true
|
| 129 |
+
qwen_text_cache_dir: /data2/JM/Code/ImageWAM/data/libero_mujoco3.3.2/flux2_qwen3_cache_4b
|
| 130 |
+
qwen_context_len: 512
|
| 131 |
+
pretrained_norm_stats: /data2/JM/Code/ImageWAM-v2/runs/libero_flux2_klein_4b_base_imagewam/2026-08-13_00-24-52/dataset_stats.json
|
| 132 |
+
processor:
|
| 133 |
+
_target_: imagewam.datasets.lerobot.processors.imagewam_processor.ImageWAMProcessor
|
| 134 |
+
shape_meta:
|
| 135 |
+
images:
|
| 136 |
+
- key: image
|
| 137 |
+
raw_shape:
|
| 138 |
+
- 3
|
| 139 |
+
- 512
|
| 140 |
+
- 512
|
| 141 |
+
shape:
|
| 142 |
+
- 3
|
| 143 |
+
- 224
|
| 144 |
+
- 224
|
| 145 |
+
- key: wrist_image
|
| 146 |
+
raw_shape:
|
| 147 |
+
- 3
|
| 148 |
+
- 512
|
| 149 |
+
- 512
|
| 150 |
+
shape:
|
| 151 |
+
- 3
|
| 152 |
+
- 224
|
| 153 |
+
- 224
|
| 154 |
+
action:
|
| 155 |
+
- key: default
|
| 156 |
+
raw_shape: 7
|
| 157 |
+
shape: 7
|
| 158 |
+
state:
|
| 159 |
+
- key: default
|
| 160 |
+
raw_shape: 8
|
| 161 |
+
shape: 8
|
| 162 |
+
num_obs_steps: 17
|
| 163 |
+
image_obs_steps: 2
|
| 164 |
+
num_output_cameras: 2
|
| 165 |
+
action_output_dim: 7
|
| 166 |
+
proprio_output_dim: 8
|
| 167 |
+
delta_action_dim_mask:
|
| 168 |
+
default:
|
| 169 |
+
- true
|
| 170 |
+
- true
|
| 171 |
+
- true
|
| 172 |
+
- true
|
| 173 |
+
- true
|
| 174 |
+
- true
|
| 175 |
+
- false
|
| 176 |
+
action_state_transforms: null
|
| 177 |
+
use_stepwise_action_norm: false
|
| 178 |
+
norm_default_mode: min/max
|
| 179 |
+
norm_exception_mode: null
|
| 180 |
+
action_state_merger:
|
| 181 |
+
_target_: imagewam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign
|
| 182 |
+
train_transforms:
|
| 183 |
+
- _target_: imagewam.datasets.lerobot.transforms.image.ToTensor
|
| 184 |
+
- _target_: torchvision.transforms.Resize
|
| 185 |
+
size:
|
| 186 |
+
- 224
|
| 187 |
+
- 224
|
| 188 |
+
val_transforms:
|
| 189 |
+
- _target_: imagewam.datasets.lerobot.transforms.image.ToTensor
|
| 190 |
+
- _target_: torchvision.transforms.Resize
|
| 191 |
+
size:
|
| 192 |
+
- 224
|
| 193 |
+
- 224
|
| 194 |
+
text_embedding_cache_dir: null
|
| 195 |
+
context_len: 128
|
| 196 |
+
qwen_text_cache_format: qwen3_flux2
|
| 197 |
+
model:
|
| 198 |
+
_target_: imagewam.runtime.create_imagewam_flux2_klein
|
| 199 |
+
flux2_src_path: /data2/JM/Code/ImageWAM/third_party/flux2
|
| 200 |
+
flux2_model_path: /data2/JM/Code/ImageWAM/checkpoints/flux2/FLUX.2-klein-base-4B/flux-2-klein-base-4b.safetensors
|
| 201 |
+
ae_model_path: /data2/JM/Code/ImageWAM/checkpoints/flux2/FLUX.2-dev/ae.safetensors
|
| 202 |
+
variant: klein-base-4b
|
| 203 |
+
qwen3_model_spec: Qwen/Qwen3-4B
|
| 204 |
+
qwen_context_len: 512
|
| 205 |
+
load_text_encoder: false
|
| 206 |
+
proprio_dim: 8
|
| 207 |
+
mot_checkpoint_mixed_attn: true
|
| 208 |
+
mot_gqa_implementation: repeat
|
| 209 |
+
mot_force_flash_attention: false
|
| 210 |
+
pack_proprio_after_text: true
|
| 211 |
+
action_dit_pretrained_path: checkpoints/action_dit_flux2_4b_libero_init.pt
|
| 212 |
+
goal_prior_stage: stage2
|
| 213 |
+
goal_prior:
|
| 214 |
+
num_latents: 100
|
| 215 |
+
num_pose_tokens: 8
|
| 216 |
+
num_layer_groups: 5
|
| 217 |
+
latent_dim: 768
|
| 218 |
+
inner_dim: 512
|
| 219 |
+
lambda_pose: 0.3
|
| 220 |
+
context_token_dropout: 0.05
|
| 221 |
+
context_blackout_prob: 0.1
|
| 222 |
+
zero_init_value: false
|
| 223 |
+
syn_gate_bias_init: -2.0
|
| 224 |
+
gate_pose_tokens: false
|
| 225 |
+
latent_layout: free
|
| 226 |
+
grid_hw:
|
| 227 |
+
- 7
|
| 228 |
+
- 7
|
| 229 |
+
grid_window: 1
|
| 230 |
+
action_sees_ref: true
|
| 231 |
+
p_both: 0.55
|
| 232 |
+
p_ref_only: 0.15
|
| 233 |
+
p_syn_only: 0.3
|
| 234 |
+
flux2_lora_config:
|
| 235 |
+
enabled: false
|
| 236 |
+
rank: 16
|
| 237 |
+
alpha: 16.0
|
| 238 |
+
dropout: 0.0
|
| 239 |
+
save_lora_merged: false
|
| 240 |
+
save_trainable_only: false
|
| 241 |
+
target_suffixes:
|
| 242 |
+
- qkv
|
| 243 |
+
- proj
|
| 244 |
+
- linear1
|
| 245 |
+
- linear2
|
| 246 |
+
- img_mlp.0
|
| 247 |
+
- img_mlp.2
|
| 248 |
+
- txt_mlp.0
|
| 249 |
+
- txt_mlp.2
|
| 250 |
+
action_dit_config:
|
| 251 |
+
action_dim: 7
|
| 252 |
+
hidden_dim: 1024
|
| 253 |
+
num_heads: 24
|
| 254 |
+
attn_head_dim: 128
|
| 255 |
+
num_layers_double: 5
|
| 256 |
+
num_layers_single: 20
|
| 257 |
+
mlp_ratio: 4.0
|
| 258 |
+
max_action_horizon: 64
|
| 259 |
+
use_gradient_checkpointing: true
|
| 260 |
+
video_scheduler:
|
| 261 |
+
train_shift: 5.0
|
| 262 |
+
infer_shift: 5.0
|
| 263 |
+
num_train_timesteps: 1000
|
| 264 |
+
action_scheduler:
|
| 265 |
+
train_shift: 5.0
|
| 266 |
+
infer_shift: 5.0
|
| 267 |
+
num_train_timesteps: 1000
|
| 268 |
+
loss:
|
| 269 |
+
lambda_video: 0.5
|
| 270 |
+
lambda_action: 1.0
|
| 271 |
+
lambda_pose: 0.3
|
checkpoints/step_004000/metadata/source/resolution_adapt/requirements-training.lock
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# This file was autogenerated by uv via the following command:
|
| 2 |
+
# uv pip compile examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt --python-version 3.12 --output-file examples/lit_imagewam_libero/resolution_adapt/requirements-training.lock
|
| 3 |
+
accelerate==1.12.0
|
| 4 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 5 |
+
aiohappyeyeballs==2.7.1
|
| 6 |
+
# via aiohttp
|
| 7 |
+
aiohttp==3.14.4
|
| 8 |
+
# via fsspec
|
| 9 |
+
aiosignal==1.4.0
|
| 10 |
+
# via aiohttp
|
| 11 |
+
annotated-types==0.8.0
|
| 12 |
+
# via pydantic
|
| 13 |
+
antlr4-python3-runtime==4.9.3
|
| 14 |
+
# via
|
| 15 |
+
# hydra-core
|
| 16 |
+
# omegaconf
|
| 17 |
+
anyio==4.15.1
|
| 18 |
+
# via httpx
|
| 19 |
+
attrs==26.1.0
|
| 20 |
+
# via
|
| 21 |
+
# aiohttp
|
| 22 |
+
# jsonlines
|
| 23 |
+
av==16.0.1
|
| 24 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 25 |
+
boto3==1.35.99
|
| 26 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 27 |
+
botocore==1.35.99
|
| 28 |
+
# via
|
| 29 |
+
# boto3
|
| 30 |
+
# s3transfer
|
| 31 |
+
certifi==2026.7.22
|
| 32 |
+
# via
|
| 33 |
+
# httpcore
|
| 34 |
+
# httpx
|
| 35 |
+
# requests
|
| 36 |
+
# sentry-sdk
|
| 37 |
+
charset-normalizer==3.5.2
|
| 38 |
+
# via requests
|
| 39 |
+
click==8.5.0
|
| 40 |
+
# via wandb
|
| 41 |
+
cloudpickle==3.1.2
|
| 42 |
+
# via gymnasium
|
| 43 |
+
cuda-bindings==13.4.3
|
| 44 |
+
# via torch
|
| 45 |
+
cuda-pathfinder==1.8.3
|
| 46 |
+
# via cuda-bindings
|
| 47 |
+
cuda-toolkit==13.0.2
|
| 48 |
+
# via torch
|
| 49 |
+
datasets==4.8.5
|
| 50 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 51 |
+
diffusers==0.35.2
|
| 52 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 53 |
+
dill==0.4.1
|
| 54 |
+
# via
|
| 55 |
+
# datasets
|
| 56 |
+
# multiprocess
|
| 57 |
+
draccus==0.10.0
|
| 58 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 59 |
+
einops==0.8.1
|
| 60 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 61 |
+
farama-notifications==0.0.6
|
| 62 |
+
# via gymnasium
|
| 63 |
+
filelock==4.0.12
|
| 64 |
+
# via
|
| 65 |
+
# datasets
|
| 66 |
+
# diffusers
|
| 67 |
+
# huggingface-hub
|
| 68 |
+
# torch
|
| 69 |
+
# transformers
|
| 70 |
+
frozenlist==1.8.0
|
| 71 |
+
# via
|
| 72 |
+
# aiohttp
|
| 73 |
+
# aiosignal
|
| 74 |
+
fsspec==2026.2.0
|
| 75 |
+
# via
|
| 76 |
+
# datasets
|
| 77 |
+
# huggingface-hub
|
| 78 |
+
# torch
|
| 79 |
+
gitdb==4.0.12
|
| 80 |
+
# via gitpython
|
| 81 |
+
gitpython==3.2.0
|
| 82 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 83 |
+
gymnasium==1.2.1
|
| 84 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 85 |
+
h11==0.16.0
|
| 86 |
+
# via httpcore
|
| 87 |
+
hf-xet==1.6.0
|
| 88 |
+
# via huggingface-hub
|
| 89 |
+
httpcore==1.0.9
|
| 90 |
+
# via httpx
|
| 91 |
+
httpx==0.28.1
|
| 92 |
+
# via datasets
|
| 93 |
+
huggingface-hub==0.35.3
|
| 94 |
+
# via
|
| 95 |
+
# -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 96 |
+
# accelerate
|
| 97 |
+
# datasets
|
| 98 |
+
# diffusers
|
| 99 |
+
# tokenizers
|
| 100 |
+
# transformers
|
| 101 |
+
hydra-core==1.3.2
|
| 102 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 103 |
+
idna==3.20
|
| 104 |
+
# via
|
| 105 |
+
# anyio
|
| 106 |
+
# httpx
|
| 107 |
+
# requests
|
| 108 |
+
# yarl
|
| 109 |
+
imageio==2.37.0
|
| 110 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 111 |
+
importlib-metadata==9.0.1
|
| 112 |
+
# via diffusers
|
| 113 |
+
iniconfig==2.3.0
|
| 114 |
+
# via pytest
|
| 115 |
+
jinja2==3.1.6
|
| 116 |
+
# via torch
|
| 117 |
+
jmespath==1.1.0
|
| 118 |
+
# via
|
| 119 |
+
# boto3
|
| 120 |
+
# botocore
|
| 121 |
+
jsonlines==4.0.0
|
| 122 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 123 |
+
markdown-it-py==4.2.0
|
| 124 |
+
# via rich
|
| 125 |
+
markupsafe==3.0.4
|
| 126 |
+
# via jinja2
|
| 127 |
+
mdurl==0.1.2
|
| 128 |
+
# via markdown-it-py
|
| 129 |
+
mergedeep==1.3.4
|
| 130 |
+
# via draccus
|
| 131 |
+
mpmath==1.3.0
|
| 132 |
+
# via sympy
|
| 133 |
+
multidict==6.9.1
|
| 134 |
+
# via
|
| 135 |
+
# aiohttp
|
| 136 |
+
# yarl
|
| 137 |
+
multiprocess==0.70.19
|
| 138 |
+
# via datasets
|
| 139 |
+
mypy-extensions==1.1.0
|
| 140 |
+
# via typing-inspect
|
| 141 |
+
networkx==3.7
|
| 142 |
+
# via torch
|
| 143 |
+
numpy==1.26.4
|
| 144 |
+
# via
|
| 145 |
+
# -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 146 |
+
# accelerate
|
| 147 |
+
# datasets
|
| 148 |
+
# diffusers
|
| 149 |
+
# gymnasium
|
| 150 |
+
# imageio
|
| 151 |
+
# opencv-python-headless
|
| 152 |
+
# pandas
|
| 153 |
+
# torchvision
|
| 154 |
+
# transformers
|
| 155 |
+
nvidia-cublas==13.1.0.3
|
| 156 |
+
# via
|
| 157 |
+
# cuda-toolkit
|
| 158 |
+
# nvidia-cudnn-cu13
|
| 159 |
+
# nvidia-cusolver
|
| 160 |
+
nvidia-cuda-cupti==13.0.85
|
| 161 |
+
# via cuda-toolkit
|
| 162 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 163 |
+
# via cuda-toolkit
|
| 164 |
+
nvidia-cuda-runtime==13.0.96
|
| 165 |
+
# via cuda-toolkit
|
| 166 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 167 |
+
# via torch
|
| 168 |
+
nvidia-cufft==12.0.0.61
|
| 169 |
+
# via cuda-toolkit
|
| 170 |
+
nvidia-cufile==1.15.1.6
|
| 171 |
+
# via cuda-toolkit
|
| 172 |
+
nvidia-curand==10.4.0.35
|
| 173 |
+
# via cuda-toolkit
|
| 174 |
+
nvidia-cusolver==12.0.4.66
|
| 175 |
+
# via cuda-toolkit
|
| 176 |
+
nvidia-cusparse==12.6.3.3
|
| 177 |
+
# via
|
| 178 |
+
# cuda-toolkit
|
| 179 |
+
# nvidia-cusolver
|
| 180 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 181 |
+
# via torch
|
| 182 |
+
nvidia-nccl-cu13==2.28.9
|
| 183 |
+
# via torch
|
| 184 |
+
nvidia-nvjitlink==13.0.88
|
| 185 |
+
# via
|
| 186 |
+
# cuda-toolkit
|
| 187 |
+
# nvidia-cufft
|
| 188 |
+
# nvidia-cusolver
|
| 189 |
+
# nvidia-cusparse
|
| 190 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 191 |
+
# via torch
|
| 192 |
+
nvidia-nvtx==13.0.85
|
| 193 |
+
# via cuda-toolkit
|
| 194 |
+
omegaconf==2.3.0
|
| 195 |
+
# via
|
| 196 |
+
# -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 197 |
+
# hydra-core
|
| 198 |
+
opencv-python-headless==4.11.0.86
|
| 199 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 200 |
+
packaging==26.3
|
| 201 |
+
# via
|
| 202 |
+
# accelerate
|
| 203 |
+
# datasets
|
| 204 |
+
# huggingface-hub
|
| 205 |
+
# hydra-core
|
| 206 |
+
# pytest
|
| 207 |
+
# transformers
|
| 208 |
+
# wandb
|
| 209 |
+
pandas==3.0.6
|
| 210 |
+
# via datasets
|
| 211 |
+
pillow==12.0.0
|
| 212 |
+
# via
|
| 213 |
+
# -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 214 |
+
# diffusers
|
| 215 |
+
# imageio
|
| 216 |
+
# torchvision
|
| 217 |
+
platformdirs==4.12.3
|
| 218 |
+
# via wandb
|
| 219 |
+
pluggy==1.6.0
|
| 220 |
+
# via pytest
|
| 221 |
+
propcache==0.5.4
|
| 222 |
+
# via
|
| 223 |
+
# aiohttp
|
| 224 |
+
# yarl
|
| 225 |
+
protobuf==7.36.2
|
| 226 |
+
# via wandb
|
| 227 |
+
psutil==7.2.2
|
| 228 |
+
# via accelerate
|
| 229 |
+
pyarrow==23.0.0
|
| 230 |
+
# via
|
| 231 |
+
# -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 232 |
+
# datasets
|
| 233 |
+
pydantic==2.13.5
|
| 234 |
+
# via wandb
|
| 235 |
+
pydantic-core==2.46.5
|
| 236 |
+
# via pydantic
|
| 237 |
+
pygments==2.21.0
|
| 238 |
+
# via
|
| 239 |
+
# pytest
|
| 240 |
+
# rich
|
| 241 |
+
pytest==8.4.2
|
| 242 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 243 |
+
python-dateutil==2.9.0.post0
|
| 244 |
+
# via
|
| 245 |
+
# botocore
|
| 246 |
+
# pandas
|
| 247 |
+
pyyaml==6.0.3
|
| 248 |
+
# via
|
| 249 |
+
# accelerate
|
| 250 |
+
# datasets
|
| 251 |
+
# draccus
|
| 252 |
+
# huggingface-hub
|
| 253 |
+
# omegaconf
|
| 254 |
+
# pyyaml-include
|
| 255 |
+
# transformers
|
| 256 |
+
# wandb
|
| 257 |
+
pyyaml-include==1.4.1
|
| 258 |
+
# via draccus
|
| 259 |
+
regex==2026.9.29
|
| 260 |
+
# via
|
| 261 |
+
# diffusers
|
| 262 |
+
# transformers
|
| 263 |
+
requests==2.34.2
|
| 264 |
+
# via
|
| 265 |
+
# datasets
|
| 266 |
+
# diffusers
|
| 267 |
+
# huggingface-hub
|
| 268 |
+
# transformers
|
| 269 |
+
# wandb
|
| 270 |
+
rich==14.2.0
|
| 271 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 272 |
+
s3transfer==0.10.4
|
| 273 |
+
# via boto3
|
| 274 |
+
safetensors==0.6.2
|
| 275 |
+
# via
|
| 276 |
+
# -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 277 |
+
# accelerate
|
| 278 |
+
# diffusers
|
| 279 |
+
# transformers
|
| 280 |
+
sentry-sdk==2.71.0
|
| 281 |
+
# via wandb
|
| 282 |
+
setuptools==81.0.0
|
| 283 |
+
# via torch
|
| 284 |
+
six==1.17.0
|
| 285 |
+
# via python-dateutil
|
| 286 |
+
smmap==5.0.3
|
| 287 |
+
# via gitdb
|
| 288 |
+
sympy==1.14.0
|
| 289 |
+
# via torch
|
| 290 |
+
termcolor==3.2.0
|
| 291 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 292 |
+
tokenizers==0.22.2
|
| 293 |
+
# via transformers
|
| 294 |
+
toml==0.10.2
|
| 295 |
+
# via draccus
|
| 296 |
+
torch==2.11.0
|
| 297 |
+
# via
|
| 298 |
+
# -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 299 |
+
# accelerate
|
| 300 |
+
# torchvision
|
| 301 |
+
torchvision==0.26.0
|
| 302 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 303 |
+
tqdm==4.70.1
|
| 304 |
+
# via
|
| 305 |
+
# datasets
|
| 306 |
+
# huggingface-hub
|
| 307 |
+
# transformers
|
| 308 |
+
transformers==4.56.1
|
| 309 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 310 |
+
triton==3.6.0
|
| 311 |
+
# via torch
|
| 312 |
+
typing-extensions==4.16.0
|
| 313 |
+
# via
|
| 314 |
+
# aiohttp
|
| 315 |
+
# aiosignal
|
| 316 |
+
# anyio
|
| 317 |
+
# gymnasium
|
| 318 |
+
# huggingface-hub
|
| 319 |
+
# pydantic
|
| 320 |
+
# pydantic-core
|
| 321 |
+
# torch
|
| 322 |
+
# typing-inspect
|
| 323 |
+
# typing-inspection
|
| 324 |
+
# wandb
|
| 325 |
+
typing-inspect==0.9.0
|
| 326 |
+
# via draccus
|
| 327 |
+
typing-inspection==0.4.4
|
| 328 |
+
# via pydantic
|
| 329 |
+
urllib3==2.8.0
|
| 330 |
+
# via
|
| 331 |
+
# botocore
|
| 332 |
+
# requests
|
| 333 |
+
# sentry-sdk
|
| 334 |
+
wandb==0.28.1
|
| 335 |
+
# via -r examples/lit_imagewam_libero/resolution_adapt/requirements-training.txt
|
| 336 |
+
xxhash==4.0.1
|
| 337 |
+
# via datasets
|
| 338 |
+
yarl==1.25.1
|
| 339 |
+
# via aiohttp
|
| 340 |
+
zipp==4.1.1
|
| 341 |
+
# via importlib-metadata
|
checkpoints/step_004000/metadata/source/resolution_adapt/requirements-training.txt
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CUDA wheels below match the repository training stack.
|
| 2 |
+
# Install lerobot==0.4.4 separately with --no-deps: its bounds exclude the
|
| 3 |
+
# repository's audited torch 2.11 / torchvision 0.26 / direct-PyAV stack.
|
| 4 |
+
accelerate==1.12.0
|
| 5 |
+
av==16.0.1
|
| 6 |
+
datasets==4.8.5
|
| 7 |
+
diffusers==0.35.2
|
| 8 |
+
einops==0.8.1
|
| 9 |
+
huggingface-hub==0.35.3
|
| 10 |
+
hydra-core==1.3.2
|
| 11 |
+
omegaconf==2.3.0
|
| 12 |
+
numpy==1.26.4
|
| 13 |
+
pillow==12.0.0
|
| 14 |
+
pyarrow==23.0.0
|
| 15 |
+
safetensors==0.6.2
|
| 16 |
+
transformers==4.56.1
|
| 17 |
+
wandb==0.28.1
|
| 18 |
+
jsonlines==4.0.0
|
| 19 |
+
termcolor==3.2.0
|
| 20 |
+
rich==14.2.0
|
| 21 |
+
draccus==0.10.0
|
| 22 |
+
opencv-python-headless==4.11.0.86
|
| 23 |
+
gymnasium==1.2.1
|
| 24 |
+
pytest==8.4.2
|
| 25 |
+
torch==2.11.0
|
| 26 |
+
torchvision==0.26.0
|
| 27 |
+
imageio==2.37.0
|
| 28 |
+
boto3==1.35.99
|
| 29 |
+
gitpython==3.2.0
|
checkpoints/step_004000/metadata/source/resolution_adapt/telemetry.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Local-first metrics with bounded W&B connection attempts and stable run identity."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import json
|
| 4 |
+
import os
|
| 5 |
+
import time
|
| 6 |
+
import warnings
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from functools import wraps
|
| 9 |
+
from .checkpoints import rng_state, restore_rng
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def preserve_training_rng(fn):
|
| 13 |
+
@wraps(fn)
|
| 14 |
+
def call(*args, **kwargs):
|
| 15 |
+
state = rng_state()
|
| 16 |
+
try: return fn(*args, **kwargs)
|
| 17 |
+
finally: restore_rng(state)
|
| 18 |
+
return call
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class Metrics:
|
| 22 |
+
def __init__(self, root: Path, identity: dict, config: dict, *, online=True):
|
| 23 |
+
self.root,self.identity,self.config = root,identity,config
|
| 24 |
+
self.online,self.run,self.next_retry = online,None,0.0
|
| 25 |
+
self.connect()
|
| 26 |
+
|
| 27 |
+
@preserve_training_rng
|
| 28 |
+
def connect(self):
|
| 29 |
+
if not self.online or self.run is not None or time.monotonic() < self.next_retry: return
|
| 30 |
+
self.next_retry = time.monotonic()+60
|
| 31 |
+
try:
|
| 32 |
+
import wandb
|
| 33 |
+
self.run = wandb.init(project=self.identity['project'],entity=os.getenv('WANDB_ENTITY'),
|
| 34 |
+
name=self.identity['name'],group=self.identity['group'],id=self.identity['id'],
|
| 35 |
+
resume='allow',mode='online',dir=str(self.root),config=self.config,
|
| 36 |
+
settings=wandb.Settings(init_timeout=20))
|
| 37 |
+
if self.run is not None:
|
| 38 |
+
(self.root/'wandb_url.txt').write_text(self.run.url+'\n')
|
| 39 |
+
except Exception as exc:
|
| 40 |
+
warnings.warn(f'W&B unavailable ({type(exc).__name__}); metrics retained locally; will retry')
|
| 41 |
+
|
| 42 |
+
@preserve_training_rng
|
| 43 |
+
def log(self, step: int, values: dict):
|
| 44 |
+
row = {'optimizer_step':step,**values}
|
| 45 |
+
with (self.root/'metrics.jsonl').open('a') as stream:
|
| 46 |
+
stream.write(json.dumps(row)+'\n'); stream.flush()
|
| 47 |
+
self.connect()
|
| 48 |
+
if self.run is not None:
|
| 49 |
+
try:
|
| 50 |
+
self.run.log(row,step=step)
|
| 51 |
+
except Exception as exc:
|
| 52 |
+
warnings.warn(f'W&B log failed ({type(exc).__name__}); local metrics preserved')
|
| 53 |
+
self.run = None
|
| 54 |
+
|
| 55 |
+
def finish(self):
|
| 56 |
+
if self.run is not None:
|
| 57 |
+
try: self.run.finish()
|
| 58 |
+
except Exception as exc: warnings.warn(f'W&B finish failed ({type(exc).__name__})')
|
checkpoints/step_004000/metadata/source/resolution_adapt/test_pipeline.py
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import copy
|
| 4 |
+
import importlib.util
|
| 5 |
+
import json
|
| 6 |
+
import random
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from types import SimpleNamespace
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
import pytest
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
HERE = Path(__file__).resolve().parents[1]
|
| 16 |
+
sys.path.insert(0,str(HERE))
|
| 17 |
+
from resolution_adapt.data import (PoolSampler, ResolutionBottleneck, competition_cameras,
|
| 18 |
+
resize_rgb, split_episodes, verify_split)
|
| 19 |
+
from resolution_adapt.checkpoints import (checkpoint_due, latest_checkpoint, prune_uploaded, resume_checkpoint,
|
| 20 |
+
save_checkpoint, validate_checkpoint)
|
| 21 |
+
from training_support import DatasetGripperToImageWAM
|
| 22 |
+
from train_resolution_adapt import make_config, parser, scheduler_for
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def episodes():
|
| 26 |
+
rows=[]
|
| 27 |
+
for pool,(repo,suite) in enumerate([('nvidia/LIBERO_LeRobot_v3','libero_goal'),
|
| 28 |
+
('lerobot/libero_plus',None),('nvidia/LIBERO_LeRobot_v3','libero_90')]):
|
| 29 |
+
for task in range(4):
|
| 30 |
+
for ep in range(12):
|
| 31 |
+
idx=len(rows)
|
| 32 |
+
rows.append(dict(episode_index=idx,source_repo=repo,source_suite=suite,
|
| 33 |
+
source_episode_index=task*12+ep,task_index=task,task=f'task {task}',
|
| 34 |
+
length=32,dataset_from_index=idx*32))
|
| 35 |
+
return rows
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def test_split_distribution_resume():
|
| 39 |
+
manifest=split_episodes(episodes())
|
| 40 |
+
assert manifest == split_episodes(episodes())
|
| 41 |
+
verify_split(manifest)
|
| 42 |
+
train={e['episode_index'] for e in manifest['episodes'] if e['split']=='train'}
|
| 43 |
+
sampler=PoolSampler(manifest)
|
| 44 |
+
for _ in range(100000):
|
| 45 |
+
index,ep=sampler.draw()
|
| 46 |
+
assert ep['episode_index'] in train
|
| 47 |
+
assert ep['anchor']+16 < ep['length']
|
| 48 |
+
assert index == ep['dataset_from_index']+ep['anchor']
|
| 49 |
+
sampler.distribution_report(check=True)
|
| 50 |
+
state=sampler.state_dict()
|
| 51 |
+
expected=[sampler.draw() for _ in range(20)]
|
| 52 |
+
restored=PoolSampler(manifest); restored.load_state_dict(state)
|
| 53 |
+
assert [restored.draw() for _ in range(20)] == expected
|
| 54 |
+
assert restored.counts == sampler.counts
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def test_duplicate_source_cannot_leak():
|
| 58 |
+
rows=episodes(); extra={**rows[0],'episode_index':9999,'dataset_from_index':999999}
|
| 59 |
+
rows.append(extra)
|
| 60 |
+
m=split_episodes(rows)
|
| 61 |
+
first=next(e for e in m['episodes'] if e['episode_index']==0)
|
| 62 |
+
duplicate=next(e for e in m['episodes'] if e['episode_index']==9999)
|
| 63 |
+
assert first['split']==duplicate['split']
|
| 64 |
+
duplicate['split']='validation' if first['split']=='train' else 'train'
|
| 65 |
+
with pytest.raises(ValueError,match='leakage'): verify_split(m)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def test_current_future_resolution_and_orientation():
|
| 69 |
+
# RGB upper-left red marker and asymmetric blue corner detect rotation/channel swaps.
|
| 70 |
+
x=torch.zeros(2,3,256,256); x[:,0,:80,:80]=1; x[:,2,180:,180:]=.6
|
| 71 |
+
y=ResolutionBottleneck()(x)
|
| 72 |
+
assert y.shape==(2,3,224,224)
|
| 73 |
+
torch.testing.assert_close(y,resize_rgb(resize_rgb(x,128),224),rtol=0,atol=0)
|
| 74 |
+
assert y[0,0,10,10] == 1 and y[0,2,10,10] == 0
|
| 75 |
+
assert 0 <= y.min() <= y.max() <= 1
|
| 76 |
+
combined=torch.cat((y,y*.5),-1)*2-1
|
| 77 |
+
assert combined.shape==(2,3,224,448) and -1 <= combined.min() <= combined.max() <= 1
|
| 78 |
+
raw=(resize_rgb(x[0],128).permute(1,2,0)*255).to(torch.uint8)
|
| 79 |
+
sim=competition_cameras(raw,raw,rotate_180=True)
|
| 80 |
+
unrotated=competition_cameras(raw,raw,rotate_180=False)
|
| 81 |
+
assert sim.shape==(3,224,448)
|
| 82 |
+
torch.testing.assert_close(sim[:,:,:224],unrotated[:,:,:224].flip((-2,-1)),atol=2e-5,rtol=1e-5)
|
| 83 |
+
with pytest.raises(ValueError): ResolutionBottleneck()(x[:,:,:128,:128])
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def test_gripper_once_and_config():
|
| 87 |
+
action=torch.randn(16,7); action[:8,-1]=-1; action[8:,-1]=1
|
| 88 |
+
original=action.clone()
|
| 89 |
+
transform=DatasetGripperToImageWAM()
|
| 90 |
+
batch=transform.forward({'action':{'default':action}})
|
| 91 |
+
assert torch.equal(batch['action']['default'][:,:6],original[:,:6])
|
| 92 |
+
assert (batch['action']['default'][:8,-1]==1).all()
|
| 93 |
+
assert (batch['action']['default'][8:,-1]==0).all()
|
| 94 |
+
with pytest.raises(ValueError): transform.forward(batch)
|
| 95 |
+
cfg=make_config(parser().parse_args([])); data=cfg['data']['train']
|
| 96 |
+
assert data['processor']['shape_meta']==data['shape_meta']
|
| 97 |
+
assert data['video_size']==[224,448]
|
| 98 |
+
assert len(data['processor']['action_state_transforms'])==1
|
| 99 |
+
assert data['endpoint_frames_only'] and data['num_frames']==17
|
| 100 |
+
# This upstream flag enables the unrelated T5 cache, not the Qwen cache.
|
| 101 |
+
assert data['require_text_cache'] is False
|
| 102 |
+
assert data['qwen_text_cache_dir'] and data['qwen_text_cache_format']=='qwen3_flux2'
|
| 103 |
+
assert cfg['stage1_checkpoint'] is None
|
| 104 |
+
assert cfg['model']['goal_prior']['num_latents']==100
|
| 105 |
+
assert cfg['model']['goal_prior']['num_layer_groups']==5
|
| 106 |
+
assert cfg['model']['loss']=={'lambda_video':.5,'lambda_action':1.,'lambda_pose':.3}
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
@pytest.mark.parametrize('accumulation',[1,3,8])
|
| 110 |
+
def test_cadence(accumulation):
|
| 111 |
+
triggers=[]; step=0
|
| 112 |
+
for micro in range(10000*accumulation):
|
| 113 |
+
if (micro+1)%accumulation: continue
|
| 114 |
+
step+=1
|
| 115 |
+
if checkpoint_due(step,2000,10000): triggers.append(step)
|
| 116 |
+
assert triggers==[2000,4000,6000,8000,10000]
|
| 117 |
+
assert checkpoint_due(2011,2000,2011)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def test_exact_resume_and_corruption(tmp_path):
|
| 121 |
+
args=parser().parse_args(['--max-steps','20','--warmup-steps','3'])
|
| 122 |
+
torch.manual_seed(5); random.seed(5); np.random.seed(5)
|
| 123 |
+
model=torch.nn.Linear(3,2); optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3)
|
| 124 |
+
scheduler=scheduler_for(optimizer,args); sampler=PoolSampler(split_episodes(episodes()))
|
| 125 |
+
metadata=tmp_path/'metadata'; metadata.mkdir()
|
| 126 |
+
(metadata/'split.json').write_text(json.dumps(split_episodes(episodes())))
|
| 127 |
+
(metadata/'wandb.json').write_text(json.dumps({'id':'stable-run'}))
|
| 128 |
+
def step():
|
| 129 |
+
sampler.draw(); optimizer.zero_grad()
|
| 130 |
+
x=torch.randn(2,3)*(random.random()+np.random.rand())
|
| 131 |
+
model(x).square().mean().backward(); optimizer.step(); scheduler.step()
|
| 132 |
+
for _ in range(3): step()
|
| 133 |
+
root=tmp_path/'checkpoints'
|
| 134 |
+
folder=save_checkpoint(root,3,model,optimizer,scheduler,sampler,metadata)
|
| 135 |
+
saved_moments=copy.deepcopy(optimizer.state_dict())
|
| 136 |
+
for _ in range(4): step()
|
| 137 |
+
expected=copy.deepcopy(model.state_dict()); expected_sampler=sampler.state_dict(); expected_lr=scheduler.get_last_lr()
|
| 138 |
+
assert resume_checkpoint(folder,model,optimizer,scheduler,sampler)==3
|
| 139 |
+
for key,state in optimizer.state_dict()['state'].items():
|
| 140 |
+
for name,value in state.items(): torch.testing.assert_close(value,saved_moments['state'][key][name],rtol=0,atol=0)
|
| 141 |
+
for _ in range(4): step()
|
| 142 |
+
for key,value in model.state_dict().items(): torch.testing.assert_close(value,expected[key],rtol=0,atol=0)
|
| 143 |
+
assert sampler.state_dict()==expected_sampler and scheduler.get_last_lr()==expected_lr
|
| 144 |
+
assert json.loads((folder/'metadata/wandb.json').read_text())['id']=='stable-run'
|
| 145 |
+
assert (folder/'metadata/split.json').read_bytes()==(metadata/'split.json').read_bytes()
|
| 146 |
+
(root/'.pending_broken').mkdir()
|
| 147 |
+
assert latest_checkpoint(root)==folder
|
| 148 |
+
with pytest.raises(FileExistsError): save_checkpoint(root,3,model,optimizer,scheduler,sampler,metadata)
|
| 149 |
+
(folder/'weights/adaptation.pt').write_bytes(b'corrupt')
|
| 150 |
+
with pytest.raises(ValueError): validate_checkpoint(folder)
|
| 151 |
+
with pytest.raises(ValueError): latest_checkpoint(root)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def test_retention_requires_receipt_and_protects_final(tmp_path):
|
| 155 |
+
root=tmp_path/'checkpoints'; root.mkdir(); status=tmp_path/'upload_state'; status.mkdir()
|
| 156 |
+
for step in range(1,6):
|
| 157 |
+
folder=root/f'step_{step:06d}'; (folder/'resume').mkdir(parents=True)
|
| 158 |
+
(folder/'resume/state.pt').touch(); (folder/'manifest.json').write_text(json.dumps({'final':step==2}))
|
| 159 |
+
(status/'step_000001.json').write_text(json.dumps({'uploaded':False}))
|
| 160 |
+
(status/'step_000002.json').write_text(json.dumps({'uploaded':True}))
|
| 161 |
+
(status/'step_000003.json').write_text(json.dumps({'uploaded':True,'uploading':True}))
|
| 162 |
+
prune_uploaded(root)
|
| 163 |
+
assert all((p/'resume').exists() for p in root.iterdir())
|
| 164 |
+
(status/'step_000003.json').write_text(json.dumps({'uploaded':True,'uploading':False}))
|
| 165 |
+
prune_uploaded(root)
|
| 166 |
+
assert not (root/'step_000003/resume').exists()
|
| 167 |
+
assert (root/'step_000001/resume').exists() and (root/'step_000002/resume').exists()
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
@pytest.mark.parametrize('unfreeze',[0,1])
|
| 171 |
+
def test_actual_upstream_lora_groups_and_update(unfreeze):
|
| 172 |
+
pytest.importorskip('imagewam.models.backbones.lora')
|
| 173 |
+
from resolution_adapt.model import configure_peft
|
| 174 |
+
class Block(torch.nn.Module):
|
| 175 |
+
def __init__(self):
|
| 176 |
+
super().__init__(); self.qkv=torch.nn.Linear(4,4); self.proj=torch.nn.Linear(4,4)
|
| 177 |
+
def forward(self,x): return self.proj(self.qkv(x))
|
| 178 |
+
class Toy(torch.nn.Module):
|
| 179 |
+
def __init__(self):
|
| 180 |
+
super().__init__()
|
| 181 |
+
self.video_expert=torch.nn.Module(); transformer=torch.nn.Module()
|
| 182 |
+
transformer.double_blocks=torch.nn.ModuleList([Block()])
|
| 183 |
+
transformer.single_blocks=torch.nn.ModuleList([Block()])
|
| 184 |
+
self.video_expert.transformer=transformer
|
| 185 |
+
for name in ('action_expert','semantic_visual_aggregator','semantic_visual_pose_norm',
|
| 186 |
+
'semantic_visual_pose_decoder','proprio_encoder','vae'):
|
| 187 |
+
setattr(self,name,torch.nn.Linear(4,4))
|
| 188 |
+
self.text_encoder=SimpleNamespace(model=torch.nn.Linear(4,4))
|
| 189 |
+
torch.manual_seed(4); model=Toy()
|
| 190 |
+
args=parser().parse_args(['--unfreeze-last-flux-blocks',str(unfreeze)])
|
| 191 |
+
groups,report=configure_peft(model,args)
|
| 192 |
+
assert report['totals']['lora']>0
|
| 193 |
+
assert report['components']['qwen']['trainable']==0 and report['components']['vae']['trainable']==0
|
| 194 |
+
for g in groups:
|
| 195 |
+
assert g['lr']==(3e-5 if g['name']=='flux_lora' else 1e-6 if g['name']=='flux_dense' else 1e-5)
|
| 196 |
+
assert bool(report['components']['flux_dense']['trainable']) == bool(unfreeze)
|
| 197 |
+
before={n:p.detach().clone() for n,p in model.named_parameters()}
|
| 198 |
+
opt=torch.optim.AdamW(groups)
|
| 199 |
+
x=torch.randn(2,4)
|
| 200 |
+
for block in [*model.video_expert.transformer.double_blocks,*model.video_expert.transformer.single_blocks]: x=block(x)
|
| 201 |
+
for name in ('action_expert','semantic_visual_aggregator','semantic_visual_pose_norm',
|
| 202 |
+
'semantic_visual_pose_decoder','proprio_encoder'): x=getattr(model,name)(x)
|
| 203 |
+
x.square().mean().backward(); torch.nn.utils.clip_grad_norm_(model.parameters(),1,error_if_nonfinite=True);opt.step()
|
| 204 |
+
for name,p in model.named_parameters():
|
| 205 |
+
if not p.requires_grad: assert torch.equal(p,before[name])
|
| 206 |
+
assert any(not torch.equal(p,before[n]) for n,p in model.named_parameters() if n.endswith('lora_B'))
|
| 207 |
+
for prefix in ('action_expert','semantic_visual_aggregator'):
|
| 208 |
+
assert any(not torch.equal(p,before[n]) for n,p in model.named_parameters() if n.startswith(prefix))
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def test_logging_failure_preserves_local_metrics(tmp_path,monkeypatch):
|
| 212 |
+
from resolution_adapt.telemetry import Metrics
|
| 213 |
+
class Failing:
|
| 214 |
+
@staticmethod
|
| 215 |
+
def init(**kwargs): raise ConnectionError('temporary')
|
| 216 |
+
monkeypatch.setitem(sys.modules,'wandb',Failing)
|
| 217 |
+
with pytest.warns(UserWarning,match='retained locally'):
|
| 218 |
+
logger=Metrics(tmp_path,{'id':'stable','project':'test','name':'test','group':'test'},{})
|
| 219 |
+
logger.log(7,{'train/loss':.3}); logger.finish()
|
| 220 |
+
assert json.loads((tmp_path/'metrics.jsonl').read_text())['optimizer_step']==7
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def test_upload_retry_verified_receipt_and_idempotence(tmp_path,monkeypatch):
|
| 224 |
+
import huggingface_hub
|
| 225 |
+
from resolution_adapt.checkpoints import upload_once
|
| 226 |
+
model=torch.nn.Linear(2,1);opt=torch.optim.AdamW(model.parameters());sched=torch.optim.lr_scheduler.LambdaLR(opt,lambda s:1)
|
| 227 |
+
sampler=PoolSampler(split_episodes(episodes()))
|
| 228 |
+
metadata=tmp_path/'metadata';metadata.mkdir();(metadata/'identity.json').write_text('{}')
|
| 229 |
+
folder=save_checkpoint(tmp_path/'checkpoints',2000,model,opt,sched,sampler,metadata)
|
| 230 |
+
class Hub:
|
| 231 |
+
def __init__(self): self.files={};self.fail=True;self.uploads=0
|
| 232 |
+
def model_info(self,repo): return SimpleNamespace(private=True)
|
| 233 |
+
def list_repo_files(self,repo): return list(self.files)
|
| 234 |
+
def upload_large_folder(self,**kwargs):
|
| 235 |
+
self.uploads+=1
|
| 236 |
+
for f in folder.rglob('*'):
|
| 237 |
+
if f.is_file() and f.name!='COMPLETE':self.files[str(f.relative_to(tmp_path))]=f.read_bytes()
|
| 238 |
+
if self.fail: self.fail=False;raise ConnectionError('interrupted')
|
| 239 |
+
def list_repo_tree(self,repo,**kwargs):return [SimpleNamespace(path=p,size=len(v)) for p,v in self.files.items()]
|
| 240 |
+
def upload_file(self,**kwargs):self.files[kwargs['path_in_repo']]=Path(kwargs['path_or_fileobj']).read_bytes()
|
| 241 |
+
hub=Hub()
|
| 242 |
+
def download(repo,name,**kwargs):
|
| 243 |
+
dest=tmp_path/'fake_download';dest.write_bytes(hub.files[name]);return str(dest)
|
| 244 |
+
monkeypatch.setattr(huggingface_hub,'hf_hub_download',download)
|
| 245 |
+
upload_once(tmp_path,'test/repo',api=hub)
|
| 246 |
+
receipt=tmp_path/'upload_state/step_002000.json'
|
| 247 |
+
assert not json.loads(receipt.read_text())['uploaded']
|
| 248 |
+
assert (folder/'resume/state.pt').exists()
|
| 249 |
+
upload_once(tmp_path,'test/repo',api=hub)
|
| 250 |
+
assert json.loads(receipt.read_text())['uploaded']
|
| 251 |
+
assert hub.uploads==2
|
| 252 |
+
upload_once(tmp_path,'test/repo',api=hub)
|
| 253 |
+
assert hub.uploads==2
|
| 254 |
+
hub.files['checkpoints/step_002000/COMPLETE']=b'conflicting manifest hash'
|
| 255 |
+
upload_once(tmp_path,'test/repo',api=hub)
|
| 256 |
+
assert not json.loads(receipt.read_text())['uploaded']
|
| 257 |
+
assert hub.uploads==2 # collision never overwrites remote state
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def test_remote_restore_verifies_before_exposing(tmp_path,monkeypatch):
|
| 261 |
+
import huggingface_hub
|
| 262 |
+
from resolution_adapt.checkpoints import restore_remote
|
| 263 |
+
origin=tmp_path/'origin';origin.mkdir()
|
| 264 |
+
metadata=origin/'metadata';metadata.mkdir();(metadata/'wandb.json').write_text('{"id":"stable"}')
|
| 265 |
+
model=torch.nn.Linear(2,1);opt=torch.optim.AdamW(model.parameters())
|
| 266 |
+
sched=torch.optim.lr_scheduler.LambdaLR(opt,lambda s:1);sampler=PoolSampler(split_episodes(episodes()))
|
| 267 |
+
folder=save_checkpoint(origin/'checkpoints',2000,model,opt,sched,sampler,metadata)
|
| 268 |
+
files={str(p.relative_to(origin)):p.read_bytes() for p in folder.rglob('*') if p.is_file()}
|
| 269 |
+
class Hub:
|
| 270 |
+
def model_info(self,repo): return SimpleNamespace(sha='pinned-remote-commit')
|
| 271 |
+
def list_repo_files(self,repo,revision):
|
| 272 |
+
assert revision=='pinned-remote-commit';return list(files)
|
| 273 |
+
monkeypatch.setattr(huggingface_hub,'HfApi',Hub)
|
| 274 |
+
def download(repo,name,revision):
|
| 275 |
+
assert revision=='pinned-remote-commit'
|
| 276 |
+
cache=tmp_path/'download';cache.write_bytes(files[name]);return str(cache)
|
| 277 |
+
monkeypatch.setattr(huggingface_hub,'hf_hub_download',download)
|
| 278 |
+
restored=restore_remote(tmp_path/'restored','test/repo')
|
| 279 |
+
validate_checkpoint(restored)
|
| 280 |
+
assert json.loads((tmp_path/'restored/metadata/wandb.json').read_text())['id']=='stable'
|
| 281 |
+
files['checkpoints/step_002000/resume/state.pt']=b'bad transfer'
|
| 282 |
+
with pytest.raises(ValueError): restore_remote(tmp_path/'corrupt','test/repo')
|
| 283 |
+
assert not (tmp_path/'corrupt/checkpoints/step_002000').exists()
|
| 284 |
+
assert not (tmp_path/'corrupt/metadata').exists()
|
checkpoints/step_004000/metadata/source/resolution_adapt/test_video_loading.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import sys
|
| 5 |
+
from fractions import Fraction
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from types import SimpleNamespace
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import pytest
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 14 |
+
from resolution_adapt.data import POOLS, PoolSampler, split_episodes
|
| 15 |
+
from training_support import VideoDecodeError, decode_video_frames
|
| 16 |
+
from train_resolution_adapt import load_sample
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@pytest.fixture
|
| 20 |
+
def sampler():
|
| 21 |
+
episodes = []
|
| 22 |
+
for repo, suite in [('nvidia/LIBERO_LeRobot_v3', 'libero_goal'),
|
| 23 |
+
('lerobot/libero_plus', None),
|
| 24 |
+
('nvidia/LIBERO_LeRobot_v3', 'libero_90')]:
|
| 25 |
+
for task in range(2):
|
| 26 |
+
for source_episode in range(6):
|
| 27 |
+
idx = len(episodes)
|
| 28 |
+
episodes.append(dict(episode_index=idx, source_repo=repo, source_suite=suite,
|
| 29 |
+
source_episode_index=task*6+source_episode, task_index=task,
|
| 30 |
+
length=32, dataset_from_index=idx*32))
|
| 31 |
+
return PoolSampler(split_episodes(episodes))
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def broken_video():
|
| 35 |
+
# The released dataset wraps the decoder exception in a RuntimeError.
|
| 36 |
+
try:
|
| 37 |
+
raise VideoDecodeError('wrist.mp4', [1., 1.8], 'Invalid packet')
|
| 38 |
+
except VideoDecodeError as exc:
|
| 39 |
+
raise RuntimeError('Failed to load a valid sample after 1 attempts') from exc
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def test_replacement_preserves_distribution_split_horizon_and_resume(sampler, tmp_path):
|
| 43 |
+
index, episode = sampler.draw()
|
| 44 |
+
counts = sampler.counts.copy(), sampler.task_counts.copy(), sampler.draws
|
| 45 |
+
seen = []
|
| 46 |
+
|
| 47 |
+
class Dataset:
|
| 48 |
+
def __getitem__(self, idx):
|
| 49 |
+
seen.append(idx)
|
| 50 |
+
if len(seen) <= 2:
|
| 51 |
+
broken_video()
|
| 52 |
+
return idx
|
| 53 |
+
|
| 54 |
+
log = tmp_path/'failures.jsonl'
|
| 55 |
+
with pytest.warns(UserWarning, match='Rejected video sample'):
|
| 56 |
+
accepted = load_sample(Dataset(), sampler, index, episode, failure_log=log)
|
| 57 |
+
assert accepted == seen[-1]
|
| 58 |
+
assert (sampler.counts, sampler.task_counts, sampler.draws) == counts
|
| 59 |
+
eligible = sampler.groups[episode['pool']][episode['task_index']]
|
| 60 |
+
for idx in seen:
|
| 61 |
+
ep = next(ep for ep in eligible if ep['dataset_from_index'] <= idx < ep['dataset_from_index']+ep['length'])
|
| 62 |
+
assert ep['split'] == 'train'
|
| 63 |
+
assert idx-ep['dataset_from_index']+16 < ep['length']
|
| 64 |
+
records = [json.loads(line) for line in log.read_text().splitlines()]
|
| 65 |
+
assert [row['index'] for row in records] == seen[:2]
|
| 66 |
+
assert all(row['video_path'] == 'wrist.mp4' and row['split'] == 'train' for row in records)
|
| 67 |
+
assert sampler.video_decode_failures[episode['pool']] == 2
|
| 68 |
+
state = sampler.state_dict()
|
| 69 |
+
expected = [sampler.draw() for _ in range(20)]
|
| 70 |
+
sampler.load_state_dict(state)
|
| 71 |
+
assert [sampler.draw() for _ in range(20)] == expected
|
| 72 |
+
assert sampler.video_decode_failures[episode['pool']] == 2
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def test_old_checkpoint_loads_without_failure_counters(sampler):
|
| 76 |
+
state = sampler.state_dict()
|
| 77 |
+
del state['video_decode_failures']
|
| 78 |
+
sampler.load_state_dict(state)
|
| 79 |
+
assert sum(sampler.video_decode_failures.values()) == 0
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def test_persistent_video_failure_is_bounded(sampler):
|
| 83 |
+
seen = []
|
| 84 |
+
|
| 85 |
+
class Dataset:
|
| 86 |
+
def __getitem__(self, index):
|
| 87 |
+
seen.append(index)
|
| 88 |
+
broken_video()
|
| 89 |
+
|
| 90 |
+
with pytest.warns(UserWarning), pytest.raises(RuntimeError, match='after 3 attempts'):
|
| 91 |
+
load_sample(Dataset(), sampler, *sampler.draw(), max_attempts=3)
|
| 92 |
+
assert len(seen) == 3
|
| 93 |
+
assert sum(sampler.video_decode_failures.values()) == 3
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
@pytest.mark.parametrize('error', [ValueError('Video timestamp mismatch'),
|
| 97 |
+
FileNotFoundError('Missing shard'),
|
| 98 |
+
RuntimeError('Missing text context')])
|
| 99 |
+
def test_other_failures_are_not_resampled(sampler, error):
|
| 100 |
+
seen = []
|
| 101 |
+
|
| 102 |
+
class Dataset:
|
| 103 |
+
def __getitem__(self, index):
|
| 104 |
+
seen.append(index)
|
| 105 |
+
raise error
|
| 106 |
+
|
| 107 |
+
with pytest.raises(type(error)) as raised:
|
| 108 |
+
load_sample(Dataset(), sampler, *sampler.draw())
|
| 109 |
+
assert raised.value is error
|
| 110 |
+
assert len(seen) == 1 and not sampler.video_decode_failures
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def test_validation_replacement_stays_held_out(sampler):
|
| 114 |
+
manifest_episodes = []
|
| 115 |
+
# Reuse the same full manifest via a fresh fixture-style synthetic split.
|
| 116 |
+
# Every pool/task has multiple held-out episodes for this retry check.
|
| 117 |
+
for pool in POOLS:
|
| 118 |
+
for task in sampler.tasks[pool]:
|
| 119 |
+
for ep in sampler.groups[pool][task][:2]:
|
| 120 |
+
manifest_episodes.append({**ep, 'split':'validation'})
|
| 121 |
+
validation = PoolSampler({'episodes':manifest_episodes}, split='validation')
|
| 122 |
+
index, episode = validation.draw()
|
| 123 |
+
|
| 124 |
+
class Dataset:
|
| 125 |
+
attempts = 0
|
| 126 |
+
|
| 127 |
+
def __getitem__(self, idx):
|
| 128 |
+
self.attempts += 1
|
| 129 |
+
if self.attempts == 1:
|
| 130 |
+
broken_video()
|
| 131 |
+
return idx
|
| 132 |
+
|
| 133 |
+
with pytest.warns(UserWarning):
|
| 134 |
+
idx = load_sample(Dataset(), validation, index, episode)
|
| 135 |
+
assert any(ep['dataset_from_index'] <= idx < ep['dataset_from_index']+ep['length']-16
|
| 136 |
+
for ep in validation.groups[episode['pool']][episode['task_index']])
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def fake_container(av, *, frame_offset=0):
|
| 140 |
+
class Container:
|
| 141 |
+
def __init__(self):
|
| 142 |
+
self.stream = SimpleNamespace(time_base=Fraction(1, 20),
|
| 143 |
+
codec_context=SimpleNamespace(thread_count=0))
|
| 144 |
+
self.streams = SimpleNamespace(video=[self.stream])
|
| 145 |
+
self.seeks = []
|
| 146 |
+
|
| 147 |
+
def __enter__(self):
|
| 148 |
+
return self
|
| 149 |
+
|
| 150 |
+
def __exit__(self, *args):
|
| 151 |
+
pass
|
| 152 |
+
|
| 153 |
+
def seek(self, offset, **kwargs):
|
| 154 |
+
self.seeks.append(offset)
|
| 155 |
+
self.position = offset//2*2
|
| 156 |
+
|
| 157 |
+
def decode(self, stream):
|
| 158 |
+
for pts in range(self.position, 20):
|
| 159 |
+
if pts == 15:
|
| 160 |
+
raise av.error.InvalidDataError(1094995529, 'Invalid packet')
|
| 161 |
+
yield SimpleNamespace(pts=pts+frame_offset, is_corrupt=False,
|
| 162 |
+
to_ndarray=lambda format, value=pts: np.full((2, 2, 3), value, dtype=np.uint8))
|
| 163 |
+
return Container()
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def test_unused_corrupt_packet_is_not_decoded(monkeypatch):
|
| 167 |
+
av = pytest.importorskip('av')
|
| 168 |
+
container = fake_container(av)
|
| 169 |
+
monkeypatch.setattr(av, 'open', lambda path: container)
|
| 170 |
+
frames = decode_video_frames('wrist.mp4', [0., .8, 0.], 1e-4)
|
| 171 |
+
assert container.seeks == [0, 16]
|
| 172 |
+
torch.testing.assert_close(frames[:, 0, 0, 0], torch.tensor([0., 16/255, 0.]), rtol=0, atol=0)
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def test_needed_corrupt_packet_has_video_context(monkeypatch):
|
| 176 |
+
av = pytest.importorskip('av')
|
| 177 |
+
monkeypatch.setattr(av, 'open', lambda path: fake_container(av))
|
| 178 |
+
with pytest.raises(VideoDecodeError) as raised:
|
| 179 |
+
decode_video_frames('wrist.mp4', [.75], 1e-4)
|
| 180 |
+
assert raised.value.video_path == 'wrist.mp4'
|
| 181 |
+
assert raised.value.timestamps == [.75]
|
| 182 |
+
assert isinstance(raised.value.__cause__, av.error.InvalidDataError)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def test_timestamp_mismatch_is_not_substituted(monkeypatch):
|
| 186 |
+
av = pytest.importorskip('av')
|
| 187 |
+
monkeypatch.setattr(av, 'open', lambda path: fake_container(av, frame_offset=1))
|
| 188 |
+
with pytest.raises(ValueError, match='timestamp mismatch'):
|
| 189 |
+
decode_video_frames('wrist.mp4', [.8], 1e-4)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def test_real_video_matches_sequential_rgb_for_unsorted_duplicate_requests(tmp_path):
|
| 193 |
+
av = pytest.importorskip('av')
|
| 194 |
+
path = tmp_path/'frames.mp4'
|
| 195 |
+
with av.open(str(path), mode='w') as container:
|
| 196 |
+
stream = container.add_stream('mpeg4', rate=20)
|
| 197 |
+
stream.width = stream.height = 16
|
| 198 |
+
stream.pix_fmt = 'yuv420p'
|
| 199 |
+
stream.codec_context.gop_size = 2
|
| 200 |
+
stream.codec_context.max_b_frames = 0
|
| 201 |
+
for idx in range(20):
|
| 202 |
+
array = np.full((16, 16, 3), idx*10, dtype=np.uint8)
|
| 203 |
+
for packet in stream.encode(av.VideoFrame.from_ndarray(array, format='rgb24')):
|
| 204 |
+
container.mux(packet)
|
| 205 |
+
for packet in stream.encode():
|
| 206 |
+
container.mux(packet)
|
| 207 |
+
with av.open(str(path)) as container:
|
| 208 |
+
all_frames = [frame.to_ndarray(format='rgb24') for frame in container.decode(video=0)]
|
| 209 |
+
expected = torch.from_numpy(np.stack([all_frames[i] for i in [16, 0, 16, 6]])).permute(0, 3, 1, 2).float()/255
|
| 210 |
+
actual = decode_video_frames(path, [.8, 0., .8, .3], 1e-4)
|
| 211 |
+
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
checkpoints/step_004000/metadata/source/train_resolution_adapt.py
ADDED
|
@@ -0,0 +1,468 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Explicit launcher for released LIT Stage-2 resolution adaptation. No default training."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import copy
|
| 6 |
+
import json
|
| 7 |
+
import math
|
| 8 |
+
import os
|
| 9 |
+
import platform
|
| 10 |
+
import random
|
| 11 |
+
import shutil
|
| 12 |
+
import subprocess
|
| 13 |
+
import sys
|
| 14 |
+
import time
|
| 15 |
+
import uuid
|
| 16 |
+
import warnings
|
| 17 |
+
from collections import Counter
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
import torch
|
| 22 |
+
import yaml
|
| 23 |
+
|
| 24 |
+
HERE = Path(__file__).resolve().parent
|
| 25 |
+
ROOT = HERE.parents[1]
|
| 26 |
+
sys.path.insert(0,str(HERE))
|
| 27 |
+
from resolution_adapt.data import DATASET, DATASET_REVISION, POOLS, PoolSampler, digest, split_episodes
|
| 28 |
+
from resolution_adapt.model import IMAGEWAM_COMMIT, FLUX_COMMIT, LIT_REVISION, build_stage2, configure_peft
|
| 29 |
+
from resolution_adapt.checkpoints import (atomic_json, checkpoint_due, latest_checkpoint, resume_checkpoint,
|
| 30 |
+
restore_rng, rng_state, save_checkpoint, sha256, upload_once)
|
| 31 |
+
from resolution_adapt.telemetry import Metrics
|
| 32 |
+
from training_support import VideoDecodeError
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def parser():
|
| 36 |
+
p = argparse.ArgumentParser(description=__doc__)
|
| 37 |
+
p.add_argument('--mode',choices=['sampler','services','preflight','train','upload','restore','validate'],default='preflight')
|
| 38 |
+
p.add_argument('--output',type=Path,default=ROOT/'.runtime/lit-resolution-adapt')
|
| 39 |
+
p.add_argument('--dataset-root',type=Path,default=ROOT/'.runtime/lit-training/dataset')
|
| 40 |
+
p.add_argument('--released-dir',type=Path,default=ROOT/'.runtime/lit-training/released/imagewam/lit_stage2')
|
| 41 |
+
p.add_argument('--imagewam-root',type=Path,default=ROOT/'.runtime/lit-training/upstream')
|
| 42 |
+
p.add_argument('--flux-root',type=Path,default=ROOT/'.runtime/lit-training/flux2')
|
| 43 |
+
p.add_argument('--ae-path',type=Path,default=ROOT/'.runtime/lit-training/models/FLUX.2-dev/ae.safetensors')
|
| 44 |
+
p.add_argument('--qwen-cache',type=Path,default=ROOT/'.runtime/lit-training/qwen_cache')
|
| 45 |
+
p.add_argument('--resume',nargs='?',const='latest')
|
| 46 |
+
for name,default in [('max-steps',10000),('warmup-steps',500),('batch-size',1),('gradient-accumulation',8),
|
| 47 |
+
('lora-rank',16),('unfreeze-last-flux-blocks',0),('checkpoint-every',2000),('seed',42),
|
| 48 |
+
('eval-every',200),('validation-samples-per-pool',8),('log-every',10)]:
|
| 49 |
+
p.add_argument('--'+name,type=int,default=default)
|
| 50 |
+
for name,default in [('lora-alpha',16.),('lora-dropout',0.),('lr-flux-lora',3e-5),('lr-action',1e-5),
|
| 51 |
+
('lr-interface',1e-5),('lr-pose',1e-5),('lr-proprio',1e-5),('lr-other',1e-5),
|
| 52 |
+
('lr-unfrozen-flux',1e-6),('weight-decay',0.01),('familiar-libero-weight',.5),
|
| 53 |
+
('plus-weight',.4),('libero90-weight',.1),('validation-episode-fraction',.05)]:
|
| 54 |
+
p.add_argument('--'+name,type=float,default=default)
|
| 55 |
+
p.add_argument('--hf-repo',default='Ratstar216/LIT-ImageWAM-ResolutionAdapt-128to224')
|
| 56 |
+
p.add_argument('--wandb-project',default=os.getenv('WANDB_PROJECT','parc2026-lit-imagewam'))
|
| 57 |
+
p.add_argument('--wandb-run-name',default=os.getenv('WANDB_RUN_NAME','lit-stage2-resadapt-128to224-lora-r16'))
|
| 58 |
+
p.add_argument('--wandb-group',default='lit-stage2-resadapt-128to224')
|
| 59 |
+
p.add_argument('--upload-once',action='store_true')
|
| 60 |
+
p.add_argument('--preflight-save',action='store_true',help='Save a local full recovery snapshot after the discarded preflight update; never upload it')
|
| 61 |
+
return p
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def serialized_args(args):
|
| 65 |
+
return {k:str(v) if isinstance(v,Path) else v for k,v in vars(args).items()}
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def make_config(args):
|
| 69 |
+
cfg = yaml.safe_load((HERE/'resolution_adapt/released_stage2.yaml').read_text())
|
| 70 |
+
cfg.update(max_steps=args.max_steps,warmup_steps=args.warmup_steps,batch_size=args.batch_size,
|
| 71 |
+
gradient_accumulation_steps=args.gradient_accumulation,save_every=args.checkpoint_every,
|
| 72 |
+
stage1_checkpoint=None,resume=args.resume,build_norm_stats=False,force_rebuild_norm_stats=False,
|
| 73 |
+
output_dir=str(args.output),num_workers=0,learning_rate=None)
|
| 74 |
+
cfg['model']['flux2_lora_config'].update(enabled=True,rank=args.lora_rank,alpha=args.lora_alpha,
|
| 75 |
+
dropout=args.lora_dropout,save_lora_merged=False,save_trainable_only=True)
|
| 76 |
+
cfg['model'].update(flux2_src_path=str(args.flux_root),flux2_model_path=None,
|
| 77 |
+
action_dit_pretrained_path=None,ae_model_path=str(args.ae_path),qwen3_model_spec=None)
|
| 78 |
+
cfg['initialization'] = {'repo':'linjianman/LIT','revision':LIT_REVISION,
|
| 79 |
+
'path':'imagewam/lit_stage2/model.pt','local_path':str(args.released_dir/'model.pt')}
|
| 80 |
+
cfg['wandb'].update(enabled=True,mode='online',workspace=os.getenv('WANDB_ENTITY'),
|
| 81 |
+
project=args.wandb_project,name=args.wandb_run_name,group=args.wandb_group)
|
| 82 |
+
data = cfg['data']['train']
|
| 83 |
+
data.update(dataset_dirs=[str(args.dataset_root)],lerobot_backend='v3',lerobot_v3_video_backend='pyav',
|
| 84 |
+
lerobot_tolerance_s=1e-4,qwen_text_cache_dir=str(args.qwen_cache),require_text_cache=False,
|
| 85 |
+
pretrained_norm_stats=str(args.released_dir/'dataset_stats.json'),vision_free=False)
|
| 86 |
+
for image,key in zip(data['shape_meta']['images'],('front','wrist'),strict=True):
|
| 87 |
+
image.update(raw_shape=[3,256,256],shape=[3,224,224],lerobot_key=f'observation.images.{key}')
|
| 88 |
+
transforms = [{'_target_':'imagewam.datasets.lerobot.transforms.image.ToTensor'},
|
| 89 |
+
{'_target_':'resolution_adapt.data.ResolutionBottleneck'}]
|
| 90 |
+
data['processor'].update(shape_meta=copy.deepcopy(data['shape_meta']),train_transforms=transforms,
|
| 91 |
+
val_transforms=copy.deepcopy(transforms),
|
| 92 |
+
action_state_transforms=[{'_target_':'training_support.DatasetGripperToImageWAM'}])
|
| 93 |
+
data['video_augmentation']['random_resized_crop'] = None
|
| 94 |
+
data['video_augmentation']['rotate'] = None
|
| 95 |
+
return cfg
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def scheduler_for(optimizer,args):
|
| 99 |
+
def scale(step):
|
| 100 |
+
if step < args.warmup_steps:
|
| 101 |
+
return (step+1)/max(1,args.warmup_steps)
|
| 102 |
+
progress = (step-args.warmup_steps)/max(1,args.max_steps-args.warmup_steps)
|
| 103 |
+
return .5*(1+math.cos(math.pi*min(1.,progress)))
|
| 104 |
+
return torch.optim.lr_scheduler.LambdaLR(optimizer,scale)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def services(args):
|
| 108 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 109 |
+
api = HfApi(); api.whoami()
|
| 110 |
+
api.create_repo(args.hf_repo,private=True,repo_type='model',exist_ok=True)
|
| 111 |
+
if not api.model_info(args.hf_repo).private:
|
| 112 |
+
api.update_repo_settings(args.hf_repo,private=True)
|
| 113 |
+
if not api.model_info(args.hf_repo).private: raise RuntimeError('HF repository is not private')
|
| 114 |
+
token = uuid.uuid4().hex
|
| 115 |
+
content = json.dumps({'experiment':'LIT ImageWAM Stage-2 Resolution Adaptation','probe':token}).encode()
|
| 116 |
+
path = f'preflight/{token}.json'
|
| 117 |
+
api.upload_file(repo_id=args.hf_repo,path_or_fileobj=content,path_in_repo=path)
|
| 118 |
+
if Path(hf_hub_download(args.hf_repo,path)).read_bytes() != content: raise RuntimeError('HF test mismatch')
|
| 119 |
+
identity = {'id':'preflight-'+token[:12],'project':args.wandb_project,'name':args.wandb_run_name+'-preflight',
|
| 120 |
+
'group':args.wandb_group}
|
| 121 |
+
logs = args.output/'service_logs'; logs.mkdir(exist_ok=True)
|
| 122 |
+
metrics = Metrics(logs,identity,{'test':'connectivity only'})
|
| 123 |
+
metrics.log(0,{'preflight/connectivity':1})
|
| 124 |
+
report = {'hf':{'passed':True,'repo':args.hf_repo,'private':True,'test_path':path},
|
| 125 |
+
'wandb':{'passed':metrics.run is not None,'id':identity['id'],
|
| 126 |
+
'url':metrics.run.url if metrics.run is not None else None}}
|
| 127 |
+
metrics.finish(); atomic_json(args.output/'services.json',report)
|
| 128 |
+
print(json.dumps(report,indent=2))
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def check_sources(args):
|
| 132 |
+
for root,commit in ((args.imagewam_root,IMAGEWAM_COMMIT),(args.flux_root,FLUX_COMMIT)):
|
| 133 |
+
actual = subprocess.check_output(['git','-C',str(root),'rev-parse','HEAD'],text=True).strip()
|
| 134 |
+
if actual != commit: raise ValueError(f'Wrong source revision: {root} {actual}')
|
| 135 |
+
if subprocess.check_output(['git','-C',str(root),'status','--porcelain','--untracked-files=no'],text=True):
|
| 136 |
+
raise ValueError(f'Pinned source has tracked modifications: {root}')
|
| 137 |
+
sys.path[:0] = [str(args.imagewam_root/'src'),str(args.flux_root/'src')]
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def load_sample(dataset, sampler, index, episode, *, max_attempts=8, failure_log=None):
|
| 141 |
+
"""Bounded video-data recovery; other dataset/transform failures still stop."""
|
| 142 |
+
if max_attempts <= 0:
|
| 143 |
+
raise ValueError('max_attempts must be positive')
|
| 144 |
+
for attempt in range(1, max_attempts + 1):
|
| 145 |
+
try:
|
| 146 |
+
return dataset[index]
|
| 147 |
+
except Exception as exc:
|
| 148 |
+
error = exc
|
| 149 |
+
while not isinstance(error, VideoDecodeError) and error.__cause__ is not None:
|
| 150 |
+
error = error.__cause__
|
| 151 |
+
if not isinstance(error, VideoDecodeError):
|
| 152 |
+
raise
|
| 153 |
+
sampler.video_decode_failures[episode['pool']] += 1
|
| 154 |
+
record = {'index':index, 'episode_index':episode['episode_index'],
|
| 155 |
+
'anchor':episode['anchor'], 'pool':episode['pool'],
|
| 156 |
+
'task_index':episode['task_index'], 'split':episode['split'],
|
| 157 |
+
'sampler_draws':sampler.draws, 'attempt':attempt,
|
| 158 |
+
'video_path':error.video_path, 'timestamps':error.timestamps,
|
| 159 |
+
'error':str(error)}
|
| 160 |
+
if failure_log is not None:
|
| 161 |
+
with Path(failure_log).open('a') as stream:
|
| 162 |
+
stream.write(json.dumps(record) + '\n')
|
| 163 |
+
warnings.warn(f"Rejected video sample {index} ({episode['pool']}/"
|
| 164 |
+
f"task {episode['task_index']}, attempt {attempt}/{max_attempts}): {error}")
|
| 165 |
+
if attempt == max_attempts:
|
| 166 |
+
raise RuntimeError(f'Video loading failed after {max_attempts} attempts '
|
| 167 |
+
f"for {episode['pool']}/task {episode['task_index']}") from exc
|
| 168 |
+
index, episode = sampler.replacement(episode)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def batch_for(dataset,sampler,size,*,failure_log=None):
|
| 172 |
+
samples = [load_sample(dataset,sampler,*sampler.draw(),failure_log=failure_log) for _ in range(size)]
|
| 173 |
+
from torch.utils.data import default_collate
|
| 174 |
+
batch = default_collate(samples)
|
| 175 |
+
if batch['video'].shape != (size,3,2,224,448) or batch['action'].shape != (size,16,7):
|
| 176 |
+
raise ValueError('Incorrect Stage-2 batch geometry')
|
| 177 |
+
if batch['goal_pose'].shape != (size,8): raise ValueError('Incorrect pose target')
|
| 178 |
+
if batch['text_hidden_states'].shape != (size,512,7680):
|
| 179 |
+
raise ValueError('Missing/incompatible released Qwen contexts')
|
| 180 |
+
for key in ('action_is_pad','goal_pose_is_pad','proprio_is_pad'):
|
| 181 |
+
if key in batch and batch[key].any(): raise ValueError(f'Padded training sample: {key}')
|
| 182 |
+
return batch
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def loss_metrics(loss,metrics):
|
| 186 |
+
result = {'total_loss':float(loss.detach())}
|
| 187 |
+
for component,weight in (('action',1.),('video',.5),('pose',.3)):
|
| 188 |
+
value = float(metrics['loss_'+component])
|
| 189 |
+
if not math.isfinite(value): raise FloatingPointError(f'Nonfinite {component} loss')
|
| 190 |
+
result[component+'_loss_weighted'] = value
|
| 191 |
+
result[component+'_loss'] = value/weight
|
| 192 |
+
if not math.isfinite(result['total_loss']): raise FloatingPointError('Nonfinite total loss')
|
| 193 |
+
return result
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
@torch.no_grad()
|
| 197 |
+
def validate(model,dataset,manifest,args):
|
| 198 |
+
state = rng_state(); augment = dataset.video_augmentation
|
| 199 |
+
dataset.video_augmentation = None
|
| 200 |
+
model.eval()
|
| 201 |
+
report = {}
|
| 202 |
+
try:
|
| 203 |
+
random.seed(args.seed+1); np.random.seed(args.seed+1); torch.manual_seed(args.seed+1)
|
| 204 |
+
sampler = PoolSampler(manifest,split='validation',seed=args.seed+1)
|
| 205 |
+
overall = Counter()
|
| 206 |
+
for pool in POOLS:
|
| 207 |
+
values = Counter()
|
| 208 |
+
for _ in range(args.validation_samples_per_pool):
|
| 209 |
+
task = sampler.rng.choice(sampler.tasks[pool]); ep = sampler.rng.choice(sampler.groups[pool][task])
|
| 210 |
+
anchor = sampler.rng.randrange(ep['length']-16)
|
| 211 |
+
index = ep['dataset_from_index']+anchor
|
| 212 |
+
from torch.utils.data import default_collate
|
| 213 |
+
sample = load_sample(dataset,sampler,index,{**ep,'anchor':anchor},
|
| 214 |
+
failure_log=args.output/'video_decode_failures.jsonl')
|
| 215 |
+
with torch.autocast('cuda',dtype=torch.bfloat16):
|
| 216 |
+
loss,metrics = model.training_loss(default_collate([sample]))
|
| 217 |
+
values.update(loss_metrics(loss,metrics))
|
| 218 |
+
report[f'validation/{pool}/video_decode_failures'] = sampler.video_decode_failures[pool]
|
| 219 |
+
for k,v in values.items():
|
| 220 |
+
report[f'validation/{pool}/{k}'] = v/args.validation_samples_per_pool
|
| 221 |
+
# Overall is explicitly weighted by the requested experiment mixture.
|
| 222 |
+
overall[k] += v/args.validation_samples_per_pool*sampler.weights[POOLS.index(pool)]
|
| 223 |
+
report.update({f'validation/overall/{k}':v for k,v in overall.items()})
|
| 224 |
+
return report
|
| 225 |
+
finally:
|
| 226 |
+
dataset.video_augmentation = augment; model.train(); model.vae.eval(); restore_rng(state)
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def run(args):
|
| 230 |
+
if not torch.cuda.is_available(): raise RuntimeError('CUDA unavailable: real Stage-2 preflight/training cannot run')
|
| 231 |
+
if not torch.cuda.is_bf16_supported(): raise RuntimeError('BF16 GPU required')
|
| 232 |
+
check_sources(args)
|
| 233 |
+
from omegaconf import OmegaConf
|
| 234 |
+
from hydra.utils import instantiate
|
| 235 |
+
from imagewam.utils.config_resolvers import register_default_resolvers
|
| 236 |
+
from imagewam.utils import misc
|
| 237 |
+
from training_support import install_dataset_hooks
|
| 238 |
+
register_default_resolvers(); install_dataset_hooks(False)
|
| 239 |
+
misc.register_work_dir(str(args.output))
|
| 240 |
+
random.seed(args.seed); np.random.seed(args.seed); torch.manual_seed(args.seed)
|
| 241 |
+
cfg = make_config(args)
|
| 242 |
+
for name in ('tabular_audit.json','video_audit.json'):
|
| 243 |
+
audited = json.loads((ROOT/'research/lit_training'/name).read_text())
|
| 244 |
+
if audited['errors']: raise ValueError(f'Existing dataset audit failed: {name}')
|
| 245 |
+
metadata = args.output/'metadata'; metadata.mkdir(exist_ok=True)
|
| 246 |
+
path = metadata/'split.json'
|
| 247 |
+
source_path = args.dataset_root/'meta/source_episodes.jsonl'
|
| 248 |
+
if sha256(source_path) != 'f2e0e560a3decbed38cb6d5c998bf659b84d1e2d60b236f81e8ebb23415b9969':
|
| 249 |
+
raise ValueError('Dataset provenance does not match the pinned audited revision')
|
| 250 |
+
episodes = [json.loads(s) for s in source_path.read_text().splitlines()]
|
| 251 |
+
if args.resume:
|
| 252 |
+
manifest = json.loads(path.read_text())
|
| 253 |
+
if manifest['provenance_sha256'] != digest(episodes): raise ValueError('Dataset changed since split creation')
|
| 254 |
+
else:
|
| 255 |
+
manifest = split_episodes(episodes,args.validation_episode_fraction,args.seed); atomic_json(path,manifest)
|
| 256 |
+
sampler = PoolSampler(manifest,weights=(args.familiar_libero_weight,args.plus_weight,args.libero90_weight),seed=args.seed)
|
| 257 |
+
# Never recalculate normalization. Verify the config really is the pinned release.
|
| 258 |
+
if yaml.safe_load((args.released_dir/'config.yaml').read_text()) != yaml.safe_load((HERE/'resolution_adapt/released_stage2.yaml').read_text()):
|
| 259 |
+
raise ValueError('Released config differs from pinned reference')
|
| 260 |
+
asset_lock = json.loads((HERE/'resolution_adapt/assets.lock.json').read_text())
|
| 261 |
+
asset_hashes = {}
|
| 262 |
+
for repo,name,local in [('linjianman/LIT','imagewam/lit_stage2/model.pt',args.released_dir/'model.pt'),
|
| 263 |
+
('linjianman/LIT','imagewam/lit_stage2/dataset_stats.json',args.released_dir/'dataset_stats.json'),
|
| 264 |
+
('black-forest-labs/FLUX.2-dev','ae.safetensors',args.ae_path)]:
|
| 265 |
+
value = sha256(local); expected = asset_lock[repo]['files'][name]['sha256']
|
| 266 |
+
if value != expected: raise ValueError(f'Asset hash mismatch: {local}')
|
| 267 |
+
asset_hashes[str(local)] = value
|
| 268 |
+
cache_provenance = json.loads((args.qwen_cache/'provenance.json').read_text())
|
| 269 |
+
if cache_provenance['revision'] != asset_lock['Qwen/Qwen3-4B']['revision']:
|
| 270 |
+
raise ValueError('Qwen cache revision mismatch; rebuild with prepare_assets.py --cache-text')
|
| 271 |
+
dataset = instantiate(OmegaConf.create(cfg['data']['train']))
|
| 272 |
+
if len(dataset) != 3080750:
|
| 273 |
+
raise ValueError('Dataset length differs from the audited full snapshot; global sampler indices would be unsafe')
|
| 274 |
+
if args.mode == 'preflight':
|
| 275 |
+
from resolution_adapt.preflight_data import audit
|
| 276 |
+
audit(args.dataset_root,args.output/'data_preflight',stats_path=args.released_dir/'dataset_stats.json',processed=dataset)
|
| 277 |
+
model,key_report = build_stage2(cfg['model'],args.released_dir/'model.pt',args.ae_path)
|
| 278 |
+
groups,report = configure_peft(model,args)
|
| 279 |
+
print(json.dumps(report['totals'],indent=2),flush=True)
|
| 280 |
+
print(f'{"group":24s} {"tensors":>8s} {"parameters":>15s} {"LR":>12s} {"decay":>8s}')
|
| 281 |
+
for row in report['optimizer_groups']:
|
| 282 |
+
print(f'{row["name"]:24s} {row["tensors"]:8d} {row["parameters"]:15,d} {row["lr"]:12.2g} {row["weight_decay"]:8.3g}',flush=True)
|
| 283 |
+
optimizer = torch.optim.AdamW(groups,betas=(.9,.95),fused=True)
|
| 284 |
+
scheduler = scheduler_for(optimizer,args)
|
| 285 |
+
identity_path = metadata/'wandb.json'
|
| 286 |
+
if args.resume:
|
| 287 |
+
identity = json.loads(identity_path.read_text())
|
| 288 |
+
else:
|
| 289 |
+
identity = {'id':uuid.uuid4().hex[:12],'project':args.wandb_project,'name':args.wandb_run_name,'group':args.wandb_group}
|
| 290 |
+
atomic_json(identity_path,identity)
|
| 291 |
+
(metadata/'config.yaml').write_text(yaml.safe_dump(cfg,sort_keys=False))
|
| 292 |
+
atomic_json(metadata/'args.json',serialized_args(args))
|
| 293 |
+
atomic_json(metadata/'parameters.json',report); atomic_json(metadata/'checkpoint_load.json',key_report)
|
| 294 |
+
shutil.copy2(args.released_dir/'dataset_stats.json',metadata/'dataset_stats.json')
|
| 295 |
+
provenance = {'experiment':'LIT ImageWAM Stage-2 Resolution Adaptation','input_information':[128,128],
|
| 296 |
+
'model_processing_per_camera':[224,224],'combined':[224,448],'source':[256,256],
|
| 297 |
+
'method':'FLUX.2 LoRA + full ActionDiT/LIT-interface fine-tuning',
|
| 298 |
+
'dataset':DATASET,'dataset_revision':DATASET_REVISION,'released_lit_revision':LIT_REVISION,
|
| 299 |
+
'initialization':'linjianman/LIT/imagewam/lit_stage2/model.pt','normalization':'released Stage-2 dataset_stats.json',
|
| 300 |
+
'imagewam_commit':IMAGEWAM_COMMIT,'flux_commit':FLUX_COMMIT,
|
| 301 |
+
'competition_commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),
|
| 302 |
+
'python':platform.python_version(),'torch':torch.__version__,'cuda':torch.version.cuda,
|
| 303 |
+
'created_unix':time.time(),'sampling_weights':sampler.weights,'wandb':identity,
|
| 304 |
+
'preprocessing':'RGB 256 -> bilinear antialiased 128 -> 224; photometric only; front then wrist; [-1,1]; both endpoints; no stored rotation',
|
| 305 |
+
'goal':'state[t+16]; action[t:t+16]; t+16 < episode length',
|
| 306 |
+
'video_loading':'independent timestamp seeks; up to 8 attempts on invalid video data; '
|
| 307 |
+
'replacement stays within pool/task/split; failures logged and counters checkpointed',
|
| 308 |
+
'gripper':'(1-g)/2 exactly once before released normalization',
|
| 309 |
+
'qwen':'frozen offline cached embeddings; encoder not instantiated',
|
| 310 |
+
'validation':'held-out episodes, equal sample count per pool; overall weighted by experiment mixture'}
|
| 311 |
+
atomic_json(metadata/'provenance.json',provenance)
|
| 312 |
+
from importlib.metadata import distributions
|
| 313 |
+
(metadata/'environment.txt').write_text('\n'.join(sorted(f'{d.metadata["Name"]}=={d.version}' for d in distributions())))
|
| 314 |
+
atomic_json(metadata/'asset_hashes.json',asset_hashes)
|
| 315 |
+
atomic_json(metadata/'assets.lock.json',asset_lock)
|
| 316 |
+
atomic_json(metadata/'qwen_cache.json',cache_provenance)
|
| 317 |
+
for audit in ('tabular_audit.json','video_audit.json','pretraining_report.md'):
|
| 318 |
+
source = ROOT/'research/lit_training'/audit
|
| 319 |
+
if source.exists(): shutil.copy2(source,metadata/audit)
|
| 320 |
+
shutil.copytree(HERE/'resolution_adapt',metadata/'source/resolution_adapt',dirs_exist_ok=True,ignore=shutil.ignore_patterns('__pycache__'))
|
| 321 |
+
for source in (Path(__file__),HERE/'training_support.py',HERE/'recovery_checkpoints.py'):
|
| 322 |
+
shutil.copy2(source,metadata/'source'/source.name)
|
| 323 |
+
step = 0
|
| 324 |
+
if args.resume:
|
| 325 |
+
checkpoint = latest_checkpoint(args.output/'checkpoints') if args.resume == 'latest' else Path(args.resume)
|
| 326 |
+
step = resume_checkpoint(checkpoint,model,optimizer,scheduler,sampler)
|
| 327 |
+
initial_step = step
|
| 328 |
+
initial_sampler_draws = sampler.draws
|
| 329 |
+
initial_scheduler_step = scheduler.last_epoch
|
| 330 |
+
metrics = Metrics(args.output,identity,provenance,online=args.mode == 'train')
|
| 331 |
+
if (args.output/'wandb_url.txt').exists():
|
| 332 |
+
provenance['wandb']['url'] = (args.output/'wandb_url.txt').read_text().strip()
|
| 333 |
+
atomic_json(metadata/'provenance.json',provenance)
|
| 334 |
+
metrics.log(step,{**{'model/'+k:v for k,v in report['totals'].items()},'model/lora_rank':args.lora_rank})
|
| 335 |
+
model.train(); model.vae.eval()
|
| 336 |
+
params = [p for g in groups for p in g['params']]
|
| 337 |
+
limit = step+1 if args.mode == 'preflight' else args.max_steps
|
| 338 |
+
# A preflight update is discarded and can never become a production checkpoint.
|
| 339 |
+
probes = {}
|
| 340 |
+
if args.mode == 'preflight':
|
| 341 |
+
for category,module in [('flux',model.video_expert),('action',model.action_expert),
|
| 342 |
+
('lit',model.semantic_visual_aggregator),('vae',model.vae)]:
|
| 343 |
+
for name,p in module.named_parameters():
|
| 344 |
+
key = 'lora' if name.endswith('lora_B') else category
|
| 345 |
+
if key not in probes and (key != 'flux' or not p.requires_grad):
|
| 346 |
+
probes[key] = (p,p.detach().clone())
|
| 347 |
+
logical_epoch_size = sum(e['length']-16 for e in manifest['episodes'] if e['split']=='train' and e['length']>16)
|
| 348 |
+
last_counts = sampler.counts.copy()
|
| 349 |
+
try:
|
| 350 |
+
if args.mode == 'validate':
|
| 351 |
+
result = validate(model,dataset,manifest,args)
|
| 352 |
+
metrics.log(step,result)
|
| 353 |
+
atomic_json(args.output/'validation.json',{'optimizer_step':step,**result})
|
| 354 |
+
return
|
| 355 |
+
while step < limit:
|
| 356 |
+
begin = time.monotonic(); totals = Counter(); optimizer.zero_grad(set_to_none=True)
|
| 357 |
+
for _ in range(args.gradient_accumulation):
|
| 358 |
+
batch = batch_for(dataset,sampler,args.batch_size,
|
| 359 |
+
failure_log=args.output/'video_decode_failures.jsonl')
|
| 360 |
+
with torch.autocast('cuda',dtype=torch.bfloat16): loss,raw = model.training_loss(batch)
|
| 361 |
+
totals.update(loss_metrics(loss,raw)); (loss/args.gradient_accumulation).backward()
|
| 362 |
+
norm = torch.nn.utils.clip_grad_norm_(params,1.0,error_if_nonfinite=True)
|
| 363 |
+
optimizer.step(); scheduler.step(); step += 1
|
| 364 |
+
if step % args.log_every == 0 or step == 1 or step == limit:
|
| 365 |
+
row = {'train/'+k:v/args.gradient_accumulation for k,v in totals.items()}
|
| 366 |
+
row.update({'train/gradient_norm':float(norm),'performance/samples_per_sec':args.batch_size*args.gradient_accumulation/(time.monotonic()-begin),
|
| 367 |
+
'performance/effective_batch_size':args.batch_size*args.gradient_accumulation,'sampling/draws':sampler.draws,'sampling/logical_epoch':sampler.draws/logical_epoch_size})
|
| 368 |
+
row['sampling/video_decode_failures'] = sum(sampler.video_decode_failures.values())
|
| 369 |
+
delta = sum(sampler.counts[p]-last_counts[p] for p in POOLS)
|
| 370 |
+
for pool in POOLS:
|
| 371 |
+
row[f'sampling/{pool}/count'] = sampler.counts[pool]
|
| 372 |
+
row[f'sampling/{pool}/percent'] = 100*sampler.counts[pool]/sampler.draws
|
| 373 |
+
row[f'sampling/{pool}/interval_percent'] = 100*(sampler.counts[pool]-last_counts[pool])/max(1,delta)
|
| 374 |
+
last_counts = sampler.counts.copy()
|
| 375 |
+
for i,g in enumerate(optimizer.param_groups): row[f'lr/{g["name"]}/{i}'] = g['lr']
|
| 376 |
+
for name,fn in [('allocated',torch.cuda.memory_allocated),('reserved',torch.cuda.memory_reserved),
|
| 377 |
+
('max_allocated',torch.cuda.max_memory_allocated),('max_reserved',torch.cuda.max_memory_reserved)]:
|
| 378 |
+
row['gpu/'+name] = fn()
|
| 379 |
+
try: sampler.distribution_report(check=True)
|
| 380 |
+
except ValueError as exc: warnings.warn(str(exc))
|
| 381 |
+
if (args.output/'upload_summary.json').exists():
|
| 382 |
+
row.update({'checkpoint/upload_'+k:v for k,v in json.loads((args.output/'upload_summary.json').read_text()).items()})
|
| 383 |
+
metrics.log(step,row)
|
| 384 |
+
if args.mode == 'train' and args.eval_every and step % args.eval_every == 0:
|
| 385 |
+
metrics.log(step,validate(model,dataset,manifest,args))
|
| 386 |
+
if args.mode == 'train' and checkpoint_due(step,args.checkpoint_every,args.max_steps):
|
| 387 |
+
if (args.output/'wandb_url.txt').exists():
|
| 388 |
+
provenance['wandb']['url'] = (args.output/'wandb_url.txt').read_text().strip()
|
| 389 |
+
atomic_json(metadata/'provenance.json',provenance)
|
| 390 |
+
save_checkpoint(args.output/'checkpoints',step,model,optimizer,scheduler,sampler,metadata,final=step == args.max_steps)
|
| 391 |
+
metrics.log(step,{'checkpoint/step':step,'checkpoint/local_complete':1})
|
| 392 |
+
if args.mode == 'preflight':
|
| 393 |
+
changes = {name:not torch.equal(p,before) for name,(p,before) in probes.items()}
|
| 394 |
+
if not all(changes.get(k) for k in ('lora','action','lit')) or changes.get('flux') or changes.get('vae'):
|
| 395 |
+
raise AssertionError(f'One-step parameter change audit failed: {changes}')
|
| 396 |
+
if args.preflight_save:
|
| 397 |
+
save_checkpoint(args.output/'checkpoints',step,model,optimizer,scheduler,sampler,metadata,final=True)
|
| 398 |
+
atomic_json(args.output/'one_step.json',{'passed':True,'changes':changes,
|
| 399 |
+
'losses_mean':{k:v/args.gradient_accumulation for k,v in totals.items()},
|
| 400 |
+
'initial_step':initial_step,'final_step':step,'initial_sampler_draws':initial_sampler_draws,
|
| 401 |
+
'final_sampler_draws':sampler.draws,'initial_scheduler_step':initial_scheduler_step,
|
| 402 |
+
'final_scheduler_step':scheduler.last_epoch,'wandb_id':identity['id'],
|
| 403 |
+
'qwen':'not instantiated; frozen cached contexts','production_steps':0,
|
| 404 |
+
'gradient_norm':float(norm),'gpu_max_allocated':torch.cuda.max_memory_allocated(),
|
| 405 |
+
'gpu_max_reserved':torch.cuda.max_memory_reserved(),
|
| 406 |
+
'recovery_saved_locally':args.preflight_save})
|
| 407 |
+
atomic_json(metadata/'sampling_final.json',sampler.distribution_report())
|
| 408 |
+
finally:
|
| 409 |
+
metrics.finish()
|
| 410 |
+
|
| 411 |
+
|
| 412 |
+
def main():
|
| 413 |
+
args = parser().parse_args(); args.output = args.output.resolve(); args.output.mkdir(parents=True,exist_ok=True)
|
| 414 |
+
if args.mode == 'restore':
|
| 415 |
+
from resolution_adapt.checkpoints import restore_remote
|
| 416 |
+
print(restore_remote(args.output,args.hf_repo)); return
|
| 417 |
+
if args.mode == 'upload':
|
| 418 |
+
import fcntl
|
| 419 |
+
with (args.output/'uploader.lock').open('w') as lock:
|
| 420 |
+
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
| 421 |
+
while True:
|
| 422 |
+
try: upload_once(args.output,args.hf_repo)
|
| 423 |
+
except Exception as exc: print('Uploader retry:',type(exc).__name__,flush=True)
|
| 424 |
+
if args.upload_once: break
|
| 425 |
+
time.sleep(30)
|
| 426 |
+
return
|
| 427 |
+
if args.mode == 'services': services(args); return
|
| 428 |
+
if args.mode == 'sampler':
|
| 429 |
+
episodes = [json.loads(s) for s in (args.dataset_root/'meta/source_episodes.jsonl').read_text().splitlines()]
|
| 430 |
+
manifest = split_episodes(episodes,args.validation_episode_fraction,args.seed)
|
| 431 |
+
sampler = PoolSampler(manifest,weights=(args.familiar_libero_weight,args.plus_weight,args.libero90_weight),seed=args.seed)
|
| 432 |
+
for _ in range(100000): sampler.draw()
|
| 433 |
+
report = sampler.distribution_report(check=True)
|
| 434 |
+
report['episodes'] = {side:dict(Counter(e['pool'] for e in manifest['episodes'] if e['split']==side)) for side in ('train','validation')}
|
| 435 |
+
atomic_json(args.output/'split.json',manifest); atomic_json(args.output/'sampler_report.json',report)
|
| 436 |
+
print(json.dumps(report,indent=2)); return
|
| 437 |
+
import fcntl
|
| 438 |
+
with (args.output/'trainer.lock').open('w') as lock:
|
| 439 |
+
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
| 440 |
+
if args.resume:
|
| 441 |
+
saved = json.loads((args.output/'metadata/args.json').read_text())
|
| 442 |
+
mode,resume,output = args.mode,args.resume,args.output
|
| 443 |
+
for k,v in saved.items():
|
| 444 |
+
setattr(args,k,Path(v) if isinstance(getattr(args,k),Path) else v)
|
| 445 |
+
args.mode,args.resume,args.output = mode,resume,output
|
| 446 |
+
print('Resume uses persisted configuration and split, including original schedule.',flush=True)
|
| 447 |
+
elif (args.output/'metadata/args.json').exists():
|
| 448 |
+
raise ValueError('Run exists: use --resume or a new --output (including after preflight)')
|
| 449 |
+
if args.max_steps <= 0 or not 0 <= args.warmup_steps < args.max_steps:
|
| 450 |
+
raise ValueError('Require max_steps > warmup_steps >= 0')
|
| 451 |
+
for key in ('batch_size','gradient_accumulation','checkpoint_every','log_every','validation_samples_per_pool'):
|
| 452 |
+
if getattr(args,key) <= 0: raise ValueError(f'{key} must be positive')
|
| 453 |
+
# The exclusive training lock proves these are abandoned writes from this run.
|
| 454 |
+
for pending in (args.output/'checkpoints').glob('.pending_*'):
|
| 455 |
+
if pending.is_dir() and not pending.is_symlink(): shutil.rmtree(pending)
|
| 456 |
+
if args.mode == 'train':
|
| 457 |
+
service_report = args.output/'services.json'
|
| 458 |
+
if not service_report.exists() or not json.loads(service_report.read_text())['hf']['passed'] or json.loads(service_report.read_text())['hf']['repo'] != args.hf_repo:
|
| 459 |
+
raise RuntimeError('Run --mode services on this output directory before long training')
|
| 460 |
+
# Independent bounded worker; a slow Hub cannot block optimizer progress.
|
| 461 |
+
log = (args.output/'upload.log').open('a')
|
| 462 |
+
subprocess.Popen([sys.executable,str(Path(__file__).resolve()),'--mode','upload','--output',str(args.output),
|
| 463 |
+
'--hf-repo',args.hf_repo],stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
| 464 |
+
log.close()
|
| 465 |
+
run(args)
|
| 466 |
+
|
| 467 |
+
|
| 468 |
+
if __name__ == '__main__': main()
|
checkpoints/step_004000/metadata/source/training_support.py
ADDED
|
@@ -0,0 +1,357 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Small, explicit adapters for the released LIT ImageWAM training code."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
import shutil
|
| 7 |
+
import time
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import torch
|
| 12 |
+
from torch.utils.data import Sampler
|
| 13 |
+
from recovery_checkpoints import RecoveryClock, complete_states, make_state_space, queue_snapshot
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class VideoDecodeError(RuntimeError):
|
| 17 |
+
"""Invalid video data, with enough context to audit a rejected sample."""
|
| 18 |
+
def __init__(self, video_path, timestamps, detail):
|
| 19 |
+
self.video_path = str(video_path)
|
| 20 |
+
self.timestamps = [float(t) for t in timestamps]
|
| 21 |
+
super().__init__(f"{detail}: {self.video_path}, timestamps={self.timestamps}")
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def decode_video_frames(video_path, timestamps, tolerance_s, backend=None):
|
| 25 |
+
"""Seek to each requested frame; do not decode unused endpoint interiors.
|
| 26 |
+
|
| 27 |
+
A damaged packet between two valid endpoints must not invalidate their
|
| 28 |
+
sample. Each seek still starts at a keyframe and decodes any dependencies;
|
| 29 |
+
invalid data needed for a requested frame raises an auditable error.
|
| 30 |
+
"""
|
| 31 |
+
import av
|
| 32 |
+
|
| 33 |
+
targets, inverse = np.unique(np.asarray(timestamps, dtype=np.float64), return_inverse=True)
|
| 34 |
+
if not len(targets):
|
| 35 |
+
raise ValueError("No video timestamps requested")
|
| 36 |
+
decoded = []
|
| 37 |
+
try:
|
| 38 |
+
with av.open(str(video_path)) as container:
|
| 39 |
+
stream = container.streams.video[0]
|
| 40 |
+
stream.thread_type = "SLICE"
|
| 41 |
+
stream.codec_context.thread_count = 1
|
| 42 |
+
for target in targets:
|
| 43 |
+
container.seek(int(target / stream.time_base), stream=stream,
|
| 44 |
+
backward=True, any_frame=False)
|
| 45 |
+
for frame in container.decode(stream):
|
| 46 |
+
timestamp = float(frame.pts * stream.time_base)
|
| 47 |
+
if timestamp < target - tolerance_s:
|
| 48 |
+
continue
|
| 49 |
+
if abs(timestamp - target) > tolerance_s:
|
| 50 |
+
raise ValueError(f"Video timestamp mismatch: {video_path}, "
|
| 51 |
+
f"requested={target}, decoded={timestamp}")
|
| 52 |
+
if frame.is_corrupt:
|
| 53 |
+
raise VideoDecodeError(video_path, timestamps,
|
| 54 |
+
f"Corrupt decoded frame at {timestamp}")
|
| 55 |
+
decoded.append(frame.to_ndarray(format="rgb24"))
|
| 56 |
+
break
|
| 57 |
+
else:
|
| 58 |
+
raise ValueError(f"No video frame decoded: {video_path}, requested={target}")
|
| 59 |
+
except av.error.InvalidDataError as exc:
|
| 60 |
+
raise VideoDecodeError(video_path, timestamps, str(exc)) from exc
|
| 61 |
+
images = np.stack([decoded[i] for i in inverse])
|
| 62 |
+
return torch.from_numpy(images).permute(0, 3, 1, 2).float() / 255
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def install_dataset_hooks(vision_free: bool):
|
| 66 |
+
"""Use row queries and direct PyAV with current torchvision.
|
| 67 |
+
|
| 68 |
+
torchvision 0.26 removed VideoReader. Decode the same RGB frames directly
|
| 69 |
+
in PyAV and enforce exact timestamp tolerance, without random fallback.
|
| 70 |
+
"""
|
| 71 |
+
import lerobot.datasets.lerobot_dataset as dataset_module
|
| 72 |
+
import lerobot.datasets.video_utils as video_module
|
| 73 |
+
cls = dataset_module.LeRobotDataset
|
| 74 |
+
|
| 75 |
+
def row_queries(self, query_indices):
|
| 76 |
+
result = {}
|
| 77 |
+
for key, indices in query_indices.items():
|
| 78 |
+
if key in self.meta.video_keys: continue
|
| 79 |
+
relative = indices if self._absolute_to_relative_idx is None else [self._absolute_to_relative_idx[i] for i in indices]
|
| 80 |
+
result[key] = torch.stack(self.hf_dataset[relative][key])
|
| 81 |
+
return result
|
| 82 |
+
cls._query_hf_dataset = row_queries
|
| 83 |
+
if not hasattr(cls, "_parc_original_query_videos"):
|
| 84 |
+
cls._parc_original_query_videos = cls._query_videos
|
| 85 |
+
if vision_free:
|
| 86 |
+
cls._query_videos = lambda self, query_timestamps, ep_idx: {}
|
| 87 |
+
else:
|
| 88 |
+
cls._query_videos = cls._parc_original_query_videos
|
| 89 |
+
|
| 90 |
+
dataset_module.decode_video_frames = decode_video_frames
|
| 91 |
+
video_module.decode_video_frames = decode_video_frames
|
| 92 |
+
video_module._imagewam_tail_clamp_patched = True
|
| 93 |
+
import imagewam.datasets.lerobot.base_lerobot_dataset as base_module
|
| 94 |
+
base_module.MAX_GETITEM_ATTEMPT = 1
|
| 95 |
+
from imagewam.datasets.lerobot.robot_video_dataset import RobotVideoDataset
|
| 96 |
+
def strict_getitem(self, index):
|
| 97 |
+
result = self._get(index)
|
| 98 |
+
self._mem_trim.tick()
|
| 99 |
+
return result
|
| 100 |
+
RobotVideoDataset.__getitem__ = strict_getitem
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
class DatasetGripperToImageWAM:
|
| 104 |
+
"""Dataset -1=open/+1=close -> released 1=open/0=close."""
|
| 105 |
+
def forward(self, batch):
|
| 106 |
+
action = batch["action"]["default"].clone()
|
| 107 |
+
if not bool(((action[..., -1] == -1) | (action[..., -1] == 1)).all()):
|
| 108 |
+
raise ValueError("Unexpected dataset gripper label")
|
| 109 |
+
action[..., -1] = (1 - action[..., -1]) / 2
|
| 110 |
+
batch["action"]["default"] = action
|
| 111 |
+
return batch
|
| 112 |
+
|
| 113 |
+
def backward(self, batch):
|
| 114 |
+
batch["action"]["default"][..., -1] = 1 - 2 * batch["action"]["default"][..., -1]
|
| 115 |
+
return batch
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
class TaskBalancedSampler(Sampler[int]):
|
| 119 |
+
"""Uniform shuffled task cycles, then source/episode/valid t+16 anchor.
|
| 120 |
+
|
| 121 |
+
Deterministic resume skips consumed microbatches, including accumulation.
|
| 122 |
+
Never samples a padded goal or crosses an episode boundary.
|
| 123 |
+
"""
|
| 124 |
+
def __init__(self, dataset, seed, batch_size, num_processes):
|
| 125 |
+
self.seed, self.batch_size, self.num_processes = int(seed), int(batch_size), int(num_processes)
|
| 126 |
+
self.epoch = self.epoch_offset = self.resume_batch_offset = 0
|
| 127 |
+
root = Path(dataset.lerobot_dataset.dataset_dirs[0])
|
| 128 |
+
self.groups = {}
|
| 129 |
+
self.horizon = dataset.num_frames - 1
|
| 130 |
+
self.count = 0
|
| 131 |
+
for line in (root / "meta/source_episodes.jsonl").open():
|
| 132 |
+
ep = json.loads(line)
|
| 133 |
+
eligible = ep["length"] - self.horizon
|
| 134 |
+
if eligible <= 0:
|
| 135 |
+
continue
|
| 136 |
+
source = ep["source_repo"]
|
| 137 |
+
self.groups.setdefault(ep["task_index"], {}).setdefault(source, []).append((ep["dataset_from_index"], eligible))
|
| 138 |
+
self.count += eligible
|
| 139 |
+
self.task_ids = sorted(self.groups)
|
| 140 |
+
if len(self.task_ids) != 111 or len(dataset) != 3080750:
|
| 141 |
+
raise ValueError("Sampler requires the fully audited pinned 111-task dataset")
|
| 142 |
+
effective = int(getattr(dataset, "_parc_effective_batch", self.batch_size*self.num_processes))
|
| 143 |
+
self.count = self.count // effective * effective
|
| 144 |
+
|
| 145 |
+
def set_epoch(self, epoch): self.epoch = int(epoch)
|
| 146 |
+
def set_epoch_offset(self, epoch): self.epoch_offset = int(epoch)
|
| 147 |
+
def set_resume_batch_offset(self, batch): self.resume_batch_offset = int(batch)
|
| 148 |
+
def clear_resume_batch_offset(self): self.resume_batch_offset = 0
|
| 149 |
+
def __len__(self): return self.count
|
| 150 |
+
|
| 151 |
+
def __iter__(self):
|
| 152 |
+
rng = np.random.default_rng(self.seed + self.epoch + self.epoch_offset)
|
| 153 |
+
skip = self.resume_batch_offset * self.batch_size * self.num_processes if self.epoch == 0 else 0
|
| 154 |
+
produced = 0
|
| 155 |
+
while produced < self.count:
|
| 156 |
+
for task in rng.permutation(self.task_ids):
|
| 157 |
+
if produced >= self.count: return
|
| 158 |
+
sources = sorted(self.groups[int(task)])
|
| 159 |
+
source = sources[int(rng.integers(len(sources)))]
|
| 160 |
+
episodes = self.groups[int(task)][source]
|
| 161 |
+
start, eligible = episodes[int(rng.integers(len(episodes)))]
|
| 162 |
+
anchor = start + int(rng.integers(eligible))
|
| 163 |
+
if produced >= skip: yield anchor
|
| 164 |
+
produced += 1
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def install_training_hooks(cfg, metadata_dir: Path):
|
| 168 |
+
"""Extend logging/checkpoints without changing the released model or loss."""
|
| 169 |
+
import imagewam.trainer as module
|
| 170 |
+
install_dataset_hooks(bool(cfg.data.train.vision_free))
|
| 171 |
+
module.ResumableEpochSampler = TaskBalancedSampler
|
| 172 |
+
base = module.Wan22Trainer
|
| 173 |
+
|
| 174 |
+
class PARCTrainer(base):
|
| 175 |
+
def _build_loader(self, dataset, worker_init_fn=None):
|
| 176 |
+
dataset._parc_effective_batch = self.batch_size * self.gradient_accumulation_steps * self.accelerator.num_processes
|
| 177 |
+
return super()._build_loader(dataset, worker_init_fn=worker_init_fn)
|
| 178 |
+
|
| 179 |
+
def __init__(self, model, *args, **kwargs):
|
| 180 |
+
original_loss = model.training_loss
|
| 181 |
+
self._update_metrics = {}
|
| 182 |
+
self._update_microbatches = 0
|
| 183 |
+
self._metric_step = -1
|
| 184 |
+
def loss_with_update_metrics(sample, *loss_args, **loss_kwargs):
|
| 185 |
+
loss, metrics = original_loss(sample, *loss_args, **loss_kwargs)
|
| 186 |
+
if not bool(torch.isfinite(loss)):
|
| 187 |
+
raise FloatingPointError("Nonfinite LIT loss; refusing an optimizer update")
|
| 188 |
+
if not torch.is_grad_enabled():
|
| 189 |
+
return loss, metrics
|
| 190 |
+
if self._metric_step != self.global_step:
|
| 191 |
+
self._metric_step = self.global_step
|
| 192 |
+
self._update_microbatches = 0
|
| 193 |
+
self._update_metrics = {}
|
| 194 |
+
self._update_microbatches += 1
|
| 195 |
+
values = {"loss":float(loss.detach()), **metrics}
|
| 196 |
+
for key, value in values.items():
|
| 197 |
+
self._update_metrics[key] = self._update_metrics.get(key, 0.0) + float(value)
|
| 198 |
+
return loss, metrics
|
| 199 |
+
model.training_loss = loss_with_update_metrics
|
| 200 |
+
original_init = torch.optim.AdamW.__init__
|
| 201 |
+
max_group_numel = int(kwargs["cfg"].get("optimizer_max_group_numel", 0))
|
| 202 |
+
def fused_init(optimizer, parameters, *optimizer_args, **optimizer_kwargs):
|
| 203 |
+
optimizer_kwargs["fused"] = True
|
| 204 |
+
if max_group_numel:
|
| 205 |
+
# ZeRO-1 casts/flattens a complete optimizer group at once.
|
| 206 |
+
# Bound those temporaries without changing any parameter's
|
| 207 |
+
# AdamW settings or the global gradient clipping rule.
|
| 208 |
+
groups, current, count = [], [], 0
|
| 209 |
+
for parameter in parameters:
|
| 210 |
+
if isinstance(parameter, dict):
|
| 211 |
+
raise ValueError("Expected the released flat parameter list")
|
| 212 |
+
if current and count + parameter.numel() > max_group_numel:
|
| 213 |
+
groups.append({"params": current})
|
| 214 |
+
current, count = [], 0
|
| 215 |
+
current.append(parameter)
|
| 216 |
+
count += parameter.numel()
|
| 217 |
+
if current: groups.append({"params": current})
|
| 218 |
+
parameters = groups
|
| 219 |
+
original_init(optimizer, parameters, *optimizer_args, **optimizer_kwargs)
|
| 220 |
+
use_fused = bool(kwargs["cfg"].get("fused_adamw", False))
|
| 221 |
+
if use_fused: torch.optim.AdamW.__init__ = fused_init
|
| 222 |
+
try:
|
| 223 |
+
super().__init__(model, *args, **kwargs)
|
| 224 |
+
finally:
|
| 225 |
+
torch.optim.AdamW.__init__ = original_init
|
| 226 |
+
if self.wandb_enabled:
|
| 227 |
+
# Publish the first real optimizer update to verify online
|
| 228 |
+
# telemetry (also after resume), then use the ten-update cadence.
|
| 229 |
+
self.log_every = 1
|
| 230 |
+
self._recovery_clock = None
|
| 231 |
+
if not self.cfg.get("preflight", False) and self.cfg.get("recovery_interval_seconds", 0):
|
| 232 |
+
self._recovery_clock = RecoveryClock(
|
| 233 |
+
Path(self.output_dir) / "checkpoints/recovery/clock.json",
|
| 234 |
+
interval=float(self.cfg.recovery_interval_seconds))
|
| 235 |
+
|
| 236 |
+
def _init_wandb(self):
|
| 237 |
+
super()._init_wandb()
|
| 238 |
+
if self.wandb_run is not None:
|
| 239 |
+
from omegaconf import OmegaConf
|
| 240 |
+
self.wandb_run.config.update(OmegaConf.to_container(self.cfg, resolve=True))
|
| 241 |
+
(Path(self.output_dir) / "wandb_url.txt").write_text(self.wandb_run.url + "\n")
|
| 242 |
+
|
| 243 |
+
def _maybe_log_omnigen2_forward_profile(self, device):
|
| 244 |
+
super()._maybe_log_omnigen2_forward_profile(device)
|
| 245 |
+
# This released hook runs after backward on every microbatch.
|
| 246 |
+
# Force the upcoming optimizer boundary to call _wandb_log when
|
| 247 |
+
# a recovery is due; ordinary telemetry retains its cadence.
|
| 248 |
+
if self._recovery_clock and (self._recovery_clock.due()
|
| 249 |
+
or (Path(self.output_dir).parent / "recovery-save.request").exists()
|
| 250 |
+
or (Path(self.output_dir).parent / "save-and-stop.request").exists()):
|
| 251 |
+
self.log_every = 1
|
| 252 |
+
|
| 253 |
+
def _wandb_log(self, payload):
|
| 254 |
+
if "eval/val_loss" in payload:
|
| 255 |
+
payload["eval/training_subset_loss"] = payload.pop("eval/val_loss")
|
| 256 |
+
if "train/loss" in payload and self._update_microbatches:
|
| 257 |
+
for key, value in self._update_metrics.items():
|
| 258 |
+
payload["train/" + key] = value / self._update_microbatches
|
| 259 |
+
payload.update({"optimizer_step": self.global_step, "epoch": self.epoch,
|
| 260 |
+
"performance/effective_batch_size": self.batch_size * self.gradient_accumulation_steps * self.accelerator.num_processes})
|
| 261 |
+
if "performance/samples_per_sec" in payload:
|
| 262 |
+
payload["performance/samples_per_sec"] *= self.gradient_accumulation_steps
|
| 263 |
+
if torch.cuda.is_available():
|
| 264 |
+
payload.update({"gpu/allocated_gib": torch.cuda.memory_allocated()/2**30,
|
| 265 |
+
"gpu/reserved_gib": torch.cuda.memory_reserved()/2**30,
|
| 266 |
+
"gpu/peak_allocated_gib": torch.cuda.max_memory_allocated()/2**30,
|
| 267 |
+
"gpu/peak_reserved_gib": torch.cuda.max_memory_reserved()/2**30})
|
| 268 |
+
# Stage 1 has no pose objective; explicit zero rather than missing.
|
| 269 |
+
if "train/loss" in payload and self.cfg.model.goal_prior_stage == "stage1":
|
| 270 |
+
payload["train/loss_pose"] = 0.0
|
| 271 |
+
payload["train/loss_video"] = 0.0
|
| 272 |
+
if "train/loss_pose" in payload:
|
| 273 |
+
payload["train/pose_loss_unweighted"] = payload["train/loss_pose"] / float(self.cfg.model.loss.lambda_pose)
|
| 274 |
+
name = "latest_train_metrics.json" if "train/loss" in payload else "latest_eval_metrics.json"
|
| 275 |
+
destination = Path(self.output_dir) / name
|
| 276 |
+
temporary = destination.with_suffix(".tmp")
|
| 277 |
+
temporary.write_text(json.dumps(payload, indent=2) + "\n")
|
| 278 |
+
temporary.replace(destination)
|
| 279 |
+
super()._wandb_log(payload)
|
| 280 |
+
if "train/loss" in payload and self.global_step > 0:
|
| 281 |
+
self.log_every = int(self.cfg.log_every)
|
| 282 |
+
request = Path(self.output_dir).parent / "save-and-stop.request"
|
| 283 |
+
if request.exists() and not self.cfg.get("preflight", False):
|
| 284 |
+
# Logging is called after a complete optimizer update and
|
| 285 |
+
# before the next sample, so state and sampler progress agree.
|
| 286 |
+
checkpoint = self.save_checkpoint()
|
| 287 |
+
(Path(self.output_dir).parent / "save-and-stop.result.json").write_text(
|
| 288 |
+
json.dumps({"optimizer_step": self.global_step,
|
| 289 |
+
"stage": self.cfg.model.goal_prior_stage, **checkpoint}, indent=2) + "\n")
|
| 290 |
+
request.unlink()
|
| 291 |
+
self._finish_wandb()
|
| 292 |
+
raise SystemExit(75)
|
| 293 |
+
force_recovery = Path(self.output_dir).parent / "recovery-save.request"
|
| 294 |
+
if self._recovery_clock and (self._recovery_clock.due() or force_recovery.exists()):
|
| 295 |
+
checkpoint = self._save_recovery_state()
|
| 296 |
+
self._recovery_clock.completed()
|
| 297 |
+
force_recovery.unlink(missing_ok=True)
|
| 298 |
+
print(f"Rolling recovery checkpoint: update {self.global_step}, {checkpoint}", flush=True)
|
| 299 |
+
|
| 300 |
+
def _save_recovery_state(self):
|
| 301 |
+
stage_dir = Path(self.output_dir)
|
| 302 |
+
step_tag = f"step_{self.global_step:06d}"
|
| 303 |
+
state = Path(self.state_dir) / step_tag
|
| 304 |
+
pending = Path(self.state_dir) / (".pending_" + step_tag)
|
| 305 |
+
self.accelerator.wait_for_everyone()
|
| 306 |
+
if self.accelerator.is_main_process:
|
| 307 |
+
if not (state / "trainer_state.json").exists():
|
| 308 |
+
shutil.rmtree(pending, ignore_errors=True)
|
| 309 |
+
previous = complete_states(Path(self.state_dir))
|
| 310 |
+
estimated = (sum(p.stat().st_size for p in previous[-1].rglob("*") if p.is_file())
|
| 311 |
+
if previous else (17_100_000_000 if self.cfg.model.goal_prior_stage == "stage1" else 66_000_000_000))
|
| 312 |
+
make_state_space(stage_dir, int(estimated * 1.05))
|
| 313 |
+
pending.mkdir(parents=True)
|
| 314 |
+
self.accelerator.wait_for_everyone()
|
| 315 |
+
if not (state / "trainer_state.json").exists():
|
| 316 |
+
self.accelerator.save_state(output_dir=str(pending))
|
| 317 |
+
if self.accelerator.is_main_process:
|
| 318 |
+
self._save_trainer_state(str(pending))
|
| 319 |
+
(pending / "COMPLETE").touch()
|
| 320 |
+
pending.rename(state)
|
| 321 |
+
self.accelerator.wait_for_everyone()
|
| 322 |
+
if self.accelerator.is_main_process:
|
| 323 |
+
if not self.cfg.get("preflight", False) and self.cfg.get("recovery_interval_seconds", 0):
|
| 324 |
+
queue_snapshot(state, stage_dir, metadata_dir, repo=str(self.cfg.recovery_repo))
|
| 325 |
+
self._prune_old_state_checkpoints(keep_step_tag=step_tag)
|
| 326 |
+
self.accelerator.wait_for_everyone()
|
| 327 |
+
return str(state)
|
| 328 |
+
|
| 329 |
+
def save_checkpoint(self):
|
| 330 |
+
# Timed recovery saves can coincide with 5,000-update/final saves.
|
| 331 |
+
# Reuse the state but still create the independently scheduled weights.
|
| 332 |
+
state_path = self._save_recovery_state()
|
| 333 |
+
weights_path = Path(self.weights_dir) / f"step_{self.global_step:06d}.pt"
|
| 334 |
+
if self.accelerator.is_main_process and not weights_path.exists():
|
| 335 |
+
pending_tag = f".pending_step_{self.global_step:06d}"
|
| 336 |
+
self._save_weights_checkpoint(step_tag=pending_tag)
|
| 337 |
+
(Path(self.weights_dir) / (pending_tag + ".pt")).replace(weights_path)
|
| 338 |
+
self.accelerator.wait_for_everyone()
|
| 339 |
+
result = {"weights_path": str(weights_path), "state_path": state_path}
|
| 340 |
+
if not self.cfg.get("preflight", False) and self.accelerator.is_main_process and (self.global_step % 5000 == 0 or self.global_step == self.max_steps):
|
| 341 |
+
stage = self.cfg.model.goal_prior_stage
|
| 342 |
+
folder = Path(self.output_dir).parent / "uploads" / stage / f"step_{self.global_step:06d}"
|
| 343 |
+
folder.mkdir(parents=True, exist_ok=True)
|
| 344 |
+
target = folder / "model.pt"
|
| 345 |
+
if not target.exists(): os.link(result["weights_path"], target)
|
| 346 |
+
for name in ["config.yaml", "dataset_stats.json"]:
|
| 347 |
+
shutil.copy2(Path(self.output_dir) / name, folder / name)
|
| 348 |
+
shutil.copytree(metadata_dir, folder / "reproduction", dirs_exist_ok=True)
|
| 349 |
+
(folder / "checkpoint.json").write_text(json.dumps({"stage":stage,"optimizer_step":self.global_step,
|
| 350 |
+
"final":self.global_step==self.max_steps,"created_at_unix":time.time()},indent=2))
|
| 351 |
+
# The independent uploader scans only fully staged folders.
|
| 352 |
+
(folder / "READY").touch()
|
| 353 |
+
return result
|
| 354 |
+
|
| 355 |
+
module.Wan22Trainer = PARCTrainer
|
| 356 |
+
import imagewam.runtime as runtime
|
| 357 |
+
runtime.Wan22Trainer = PARCTrainer
|
checkpoints/step_004000/metadata/split.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/step_004000/metadata/tabular_audit.json
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"tasks": 111,
|
| 3 |
+
"episodes": 19961,
|
| 4 |
+
"frames": 3080750,
|
| 5 |
+
"fps": 20,
|
| 6 |
+
"valid_h16_goal_anchors": 2761374,
|
| 7 |
+
"timestamp_max_error_seconds": 7.629394538355427e-07,
|
| 8 |
+
"sources": {
|
| 9 |
+
"nvidia": {
|
| 10 |
+
"frames": 842714,
|
| 11 |
+
"tasks": 111,
|
| 12 |
+
"state_min": [
|
| 13 |
+
-0.4862295389175415,
|
| 14 |
+
-0.3970998525619507,
|
| 15 |
+
0.008128180168569088,
|
| 16 |
+
-0.7480751872062683,
|
| 17 |
+
-4.363129138946533,
|
| 18 |
+
-3.212415933609009,
|
| 19 |
+
-0.0025667562149465084,
|
| 20 |
+
-0.04255744442343712
|
| 21 |
+
],
|
| 22 |
+
"state_max": [
|
| 23 |
+
0.21031762659549713,
|
| 24 |
+
0.48814651370048523,
|
| 25 |
+
1.3660105466842651,
|
| 26 |
+
4.843389987945557,
|
| 27 |
+
4.042375564575195,
|
| 28 |
+
2.4014885425567627,
|
| 29 |
+
0.04639676958322525,
|
| 30 |
+
0.0017311422852799296
|
| 31 |
+
],
|
| 32 |
+
"action_min": [
|
| 33 |
+
-0.9375,
|
| 34 |
+
-0.9375,
|
| 35 |
+
-0.9375,
|
| 36 |
+
-0.3257142901420593,
|
| 37 |
+
-0.375,
|
| 38 |
+
-0.375,
|
| 39 |
+
-1.0
|
| 40 |
+
],
|
| 41 |
+
"action_max": [
|
| 42 |
+
0.9375,
|
| 43 |
+
0.9375,
|
| 44 |
+
0.9375,
|
| 45 |
+
0.375,
|
| 46 |
+
0.375,
|
| 47 |
+
0.375,
|
| 48 |
+
1.0
|
| 49 |
+
],
|
| 50 |
+
"gripper_counts": {
|
| 51 |
+
"-1.0": 444702,
|
| 52 |
+
"1.0": 398012
|
| 53 |
+
},
|
| 54 |
+
"gripper_dynamics": {
|
| 55 |
+
"-1": {
|
| 56 |
+
"count": 440070,
|
| 57 |
+
"mean_next_width_delta": 0.0004565622366499156,
|
| 58 |
+
"median_width": 0.0791521966457367
|
| 59 |
+
},
|
| 60 |
+
"1": {
|
| 61 |
+
"count": 397030,
|
| 62 |
+
"mean_next_width_delta": -0.0007894165464676917,
|
| 63 |
+
"median_width": 0.02936265431344509
|
| 64 |
+
}
|
| 65 |
+
}
|
| 66 |
+
},
|
| 67 |
+
"plus": {
|
| 68 |
+
"frames": 2238036,
|
| 69 |
+
"tasks": 40,
|
| 70 |
+
"state_min": [
|
| 71 |
+
-0.4856736361980438,
|
| 72 |
+
-0.3313629925251007,
|
| 73 |
+
0.008128180168569088,
|
| 74 |
+
0.8415665030479431,
|
| 75 |
+
-3.616978406906128,
|
| 76 |
+
-1.842738389968872,
|
| 77 |
+
-0.002542249858379364,
|
| 78 |
+
-0.042119648307561874
|
| 79 |
+
],
|
| 80 |
+
"state_max": [
|
| 81 |
+
0.21031762659549713,
|
| 82 |
+
0.39128610491752625,
|
| 83 |
+
1.3660105466842651,
|
| 84 |
+
3.7743608951568604,
|
| 85 |
+
3.5752453804016113,
|
| 86 |
+
1.3747878074645996,
|
| 87 |
+
0.042380549013614655,
|
| 88 |
+
0.0010137537028640509
|
| 89 |
+
],
|
| 90 |
+
"action_min": [
|
| 91 |
+
-0.9375,
|
| 92 |
+
-0.9375,
|
| 93 |
+
-0.9375,
|
| 94 |
+
-0.2582142949104309,
|
| 95 |
+
-0.375,
|
| 96 |
+
-0.3642857074737549,
|
| 97 |
+
-1.0
|
| 98 |
+
],
|
| 99 |
+
"action_max": [
|
| 100 |
+
0.9375,
|
| 101 |
+
0.9375,
|
| 102 |
+
0.9375,
|
| 103 |
+
0.3557142913341522,
|
| 104 |
+
0.375,
|
| 105 |
+
0.375,
|
| 106 |
+
1.0
|
| 107 |
+
],
|
| 108 |
+
"gripper_counts": {
|
| 109 |
+
"-1.0": 1158433,
|
| 110 |
+
"1.0": 1079603
|
| 111 |
+
},
|
| 112 |
+
"gripper_dynamics": {
|
| 113 |
+
"-1": {
|
| 114 |
+
"count": 1145046,
|
| 115 |
+
"mean_next_width_delta": 0.0006007962510921061,
|
| 116 |
+
"median_width": 0.07898755371570587
|
| 117 |
+
},
|
| 118 |
+
"1": {
|
| 119 |
+
"count": 1078643,
|
| 120 |
+
"mean_next_width_delta": -0.0008084022556431592,
|
| 121 |
+
"median_width": 0.0340743213891983
|
| 122 |
+
}
|
| 123 |
+
}
|
| 124 |
+
}
|
| 125 |
+
},
|
| 126 |
+
"suites": {
|
| 127 |
+
"('nvidia/LIBERO_LeRobot_v3', 'libero_10')": 379,
|
| 128 |
+
"('nvidia/LIBERO_LeRobot_v3', 'libero_90')": 3921,
|
| 129 |
+
"('nvidia/LIBERO_LeRobot_v3', 'libero_goal')": 428,
|
| 130 |
+
"('nvidia/LIBERO_LeRobot_v3', 'libero_object')": 454,
|
| 131 |
+
"('nvidia/LIBERO_LeRobot_v3', 'libero_spatial')": 432,
|
| 132 |
+
"('lerobot/libero_plus', None)": 14347
|
| 133 |
+
},
|
| 134 |
+
"errors": [],
|
| 135 |
+
"image_audit": "pending; tabular audit does not certify images or simulator replay"
|
| 136 |
+
}
|
checkpoints/step_004000/metadata/video_audit.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"video_files_checked": 108,
|
| 3 |
+
"expected_files": 108,
|
| 4 |
+
"errors": [],
|
| 5 |
+
"decoded_frames": 324,
|
| 6 |
+
"scope": "all file metadata, all episode camera intervals, first/middle/last decoded RGB frame of every file; not every video frame or simulator replay"
|
| 7 |
+
}
|
checkpoints/step_004000/metadata/wandb.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "db6a83524459",
|
| 3 |
+
"project": "parc2026-lit-imagewam",
|
| 4 |
+
"name": "lit-stage2-resadapt-128to224-lora-r16",
|
| 5 |
+
"group": "lit-stage2-resadapt-128to224"
|
| 6 |
+
}
|
checkpoints/step_004000/resume/state.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d0c3e5d11137de82ae6898ea4288a95c05068fed58e30df7f1e70e05596181c4
|
| 3 |
+
size 6672739984
|
checkpoints/step_004000/weights/adaptation.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7e652091d431c95ce5df1bab7c3fd01de0edbab3c254763c180a6392fa6e7170
|
| 3 |
+
size 3336327138
|