diff --git a/spacedreamer_stack_cups_clean40_30ep/CPU_validation.json b/spacedreamer_stack_cups_clean40_30ep/CPU_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..32b9afe5ac5d1ca4afc55c80a4aa293dd546dae8 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/CPU_validation.json @@ -0,0 +1,23 @@ +{ + "status": "PASS_CPU_STRICT_LOAD_EXACT_SELECTED40_NORMALIZATION_PROMPT_AND_FROZEN_SOURCE", + "checkpoint_step": 4350, + "epochs": 30, + "state_dict_keys": 1661, + "strict_load_missing_keys": [], + "strict_load_unexpected_keys": [], + "all_parameters_materialized": true, + "selected40_FP32_statistics_file_identical": true, + "training_and_adapter_normalize_decode_bitwise_equal": true, + "all4_quantized_buffers_correct": true, + "context_shape": [ + 128, + 4096 + ], + "context_prompt_matches": true, + "shared_dependencies_at_revision": "64a7a5fad88d69ed041b00d233fc1d4374e4bf8e", + "shared_dependency_sha256": { + "spacedreamer_20epochs/assets/Wan2.2-TI2V-5B/Wan2.2_VAE.pth": "20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36", + "spacedreamer_20epochs/assets/metric_video_depth_anything_vitb.pth": "f6f58576b9680a112f2428d4f39aff92656d3ae85745b0164675ace8d5b1fade" + }, + "scope": "CPU checkpoint reconstruction and packaging; no new GPU sampling or robot success claimed." +} diff --git a/spacedreamer_stack_cups_clean40_30ep/README.md b/spacedreamer_stack_cups_clean40_30ep/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e3c145d436eb322ca925c1bb414f96e6c51b66c4 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/README.md @@ -0,0 +1,100 @@ +# SpaceDreamer stack_cups — clean40,30 epochs + +Final original `final_weights.pt` from **bifrost-2026111502493001-shouqx**: **30 epochs,4350 optimizer steps**,trained on the approved40 demonstrations from the new67 November3/4 stack_cups recordings (41 continuous segments,37273 unique rows). Fresh initialization from SpaceDreamer-v3 pretrained018126;16H200,global batch256. Right-arm6joint loss weight2;left joints and both grippers1. This is the new selected40-refit-normalization run. + +The model is **RgbGeometryActionMoT**,joint RGB/Depth/dense-point-map/Action prediction. It predicts **32×14 absolute joint/gripper commands**,not the RGB-only FastWAM40-step model. All final model parameters are contained in this weight file; distributed optimizer/RNG training states are retained on the original training storage and are not part of this inference release. + +## Files + +- `final_weights.pt`: unchanged final training export (`state_dict`,`config`,`initialization`). +- `training_config.json`: original training config; historical absolute dataset/output paths are provenance and are not needed by the inference adapter. +- `reference_norm_stats.json`: **exact FP32 statistics refit on selected40 only**. Use these for state normalization and action decoding. +- `text_context.pt`: exact cached128-token training instruction,`Stack the cups.` +- `checkpoint_metadata.json`: training step,weight size and SHA-256. +- `runtime/spacedreamer_galexea.py`: inference adapter exposing `infer_action()` through the original joint sampler. +- `runtime/run_inference.py`: saved-observation prediction CLI,with no robot I/O. +- `runtime/source/`: matching frozen training source;`training_reference/code/`: exact training recipe. +- `postflight_normalizer_audit.json`: independent explanation of the platform failure after training. +- `CPU_validation.json`,`manifest.json`: packaging checks and exact file digests. + +## Download this model and shared dependencies + +VAE,VDA and optional text encoder already reside in this repository under `spacedreamer_20epochs/assets`. They are reused rather than duplicated in this new folder. The standard cached instruction needs VAE and,if depth is not provided,VDA. UMT5/tokenizer are needed only for a different instruction. + +```python +from huggingface_hub import HfApi,snapshot_download +repo="qshou-coder/SpaceDreamer_Galexea" +prefix="spacedreamer_stack_cups_clean40_30ep" +shared="spacedreamer_20epochs/assets" +revision=HfApi().model_info(repo).sha # save/pin this revision +snapshot_download( + repo_id=repo,revision=revision,local_dir="SpaceDreamer_Galexea", + allow_patterns=[ + f"{prefix}/**", + f"{shared}/Wan2.2-TI2V-5B/Wan2.2_VAE.pth", + f"{shared}/metric_video_depth_anything_vitb.pth", + f"{shared}/calibrations.json", + ], +) +# For custom instructions,also download the existing shared UMT5 weights/tokenizer. +``` + +Known shared hashes: + +- VAE: `20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36`. +- Metric VDA: `f6f58576b9680a112f2428d4f39aff92656d3ae85745b0164675ace8d5b1fade`. + +The calibration manifest is historical training-camera metadata. No training datasets are required for prediction. Deployment camera geometry must correspond to the trained image/depth convention. + +## Runtime and inference + +`runtime/runtime_packages.json` records observed package versions. The checked runtime uses Python3.12,PyTorch2.9.0 and torchvision0.24.0. Install the CUDA build appropriate to your machine and the dependencies in `runtime/requirements.txt` in an isolated environment. The checkpoint uses BF16. No Wan diffusion backbone checkpoint is needed beyond this trained export and its VAE. + +From the downloaded repository root: + +```bash +export PYTHONPATH="$PWD/spacedreamer_stack_cups_clean40_30ep/runtime${PYTHONPATH:+:$PYTHONPATH}" +python spacedreamer_stack_cups_clean40_30ep/runtime/run_inference.py \ + --task-dir spacedreamer_stack_cups_clean40_30ep \ + --assets-dir spacedreamer_20epochs/assets \ + --observation observation.npz \ + --output predicted_actions.npy \ + --steps 10 --seed 0 +``` + +`observation.npz` contains your own `head_rgb` RGB,uint8 image and `state` float14;optionally `depth_m`. No training-camera images are published in this folder. + +```python +from spacedreamer_galexea import SpaceDreamerGalaxea +policy=SpaceDreamerGalaxea( + "spacedreamer_stack_cups_clean40_30ep", + "spacedreamer_20epochs/assets", + device="cuda:0", +) +policy.reset_episode() # once at the start of a new episode +commands=policy.infer_action( + head_rgb,state,depth_m=depth_m, + num_inference_steps=10,seed=None, +) +assert commands.shape==(32,14) +``` + +Ten sampling steps is the adapter's default,not a claim that it is optimal for this new checkpoint. If `depth_m` is omitted,the adapter advances causal metric VDA using the head image. Feed successive camera frames through `update_depth` to maintain its temporal history when replanning less often;reset the stream once per episode. The adapter performs the original joint multimodal sampling and returns physical actions;it does not implement a new action-only algorithm or execute robot commands. + +## Exact input/output contract + +- One **head-left RGB view**,not FastWAM's concatenated three-camera layout. +- Accepted image: raw RGB,uint8 **512×640**,or the exact training letterboxed **224×224** image. Arbitrary square resizing of a raw camera frame is not equivalent. The adapter applies the matching letterbox and model resize to256×256. +- `depth_m`,if supplied: finite metric z-depth,float224×224,aligned with that letterboxed RGB. The adapter applies the training float16 cache quantization,nearest resize and0.05–5m validity convention. +- State order: left_arm6,left_gripper,right_arm6,right_gripper. Joint values in radians;grippers keep original continuous raw units approximately0–100. +- State normalization and output decoding use the external **reference_norm_stats.json in FP32**,including native clipping;do not substitute the old81/148-demo statistics or rounded checkpoint buffer values. +- Returned array is float32 **32×14**,already denormalized **absolute original motion_target.position**. Do not add current state or inverse-normalize again. At the20Hz training rate,32 commands span approximately1.6 seconds. Execution prefix/replanning frequency is controlled separately. +- Current RGB/depth frame0 condition the model;point frame0 and future RGB/depth/point/action latents are sampled jointly. + +## Training completion and postflight issue + +All4350 optimizer steps completed;all logged loss/gradient values were finite. The final100-step mean action loss was0.0086685. The platform then marked the job JOB_FAILED because an export checker required bitwise equality between BF16-saved normalizer buffers converted back toFP32 and the original FP32 statistics. + +The independent audit found all four stored buffers **exactly equal to the correct selected40 statistics cast to BF16**,and the external inference statistics file byte-identical to the training normalizer. The failure is a postflight comparison false positive,not missing training steps or a switched normalizer. The weight file is uploaded unchanged. The inference adapter explicitly uses the original FP32 JSON for normalization/decoding. + +Packaging/CPU model reconstruction checks are documented in CPU_validation.json. This upload does not claim new GPU rollout acceptance or a real-robot success rate for this checkpoint. diff --git a/spacedreamer_stack_cups_clean40_30ep/checkpoint_metadata.json b/spacedreamer_stack_cups_clean40_30ep/checkpoint_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9b697d5d650c7731392e17bc9f95973ded96def0 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/checkpoint_metadata.json @@ -0,0 +1,18 @@ +{ + "task": "stack_cups", + "training_job": "bifrost-2026111502493001-shouqx", + "epochs": 30, + "global_step": 4350, + "checkpoint": "final_weights.pt", + "bytes": 11030029192, + "sha256": "b9c898a58d03a4b863d3ec9cc5b3b9c87e7ac00427a27478e61a17cbf269a6e7", + "action_dim": 14, + "action_horizon": 32, + "video_frames": 33, + "action_source": "absolute_original_motion_target_position", + "joint_order": "left_arm6,left_gripper,right_arm6,right_gripper", + "initialization": "Fresh fine-tune from original pretrained018126 on approved40 of new67 demos; selected40-refit normalization; right6joint loss2x,grippers1x.", + "optimizer_training_complete": true, + "platform_state": "JOB_FAILED", + "platform_failure_reason": "Postflight BF16-buffer versus FP32-statistics bitwise comparison false positive, independently audited; no missing optimizer steps." +} diff --git a/spacedreamer_stack_cups_clean40_30ep/final_weights.pt b/spacedreamer_stack_cups_clean40_30ep/final_weights.pt new file mode 100644 index 0000000000000000000000000000000000000000..2471a422f8886cfc28bc55593d5580e3c2af4741 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/final_weights.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9c898a58d03a4b863d3ec9cc5b3b9c87e7ac00427a27478e61a17cbf269a6e7 +size 11030029192 diff --git a/spacedreamer_stack_cups_clean40_30ep/initialization.json b/spacedreamer_stack_cups_clean40_30ep/initialization.json new file mode 100644 index 0000000000000000000000000000000000000000..ba2d10b24f3df0a867f4fc8f1c35ef034735e9b6 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/initialization.json @@ -0,0 +1,35 @@ +{ + "loaded_backbone_tensors": 1649, + "fresh_interface_tensors": [ + "action_normalizer.action_q01", + "action_normalizer.action_q99", + "action_normalizer.state_q01", + "action_normalizer.state_q99", + "action_state_encoder.0.bias", + "action_state_encoder.0.weight", + "action_state_encoder.2.bias", + "action_state_encoder.2.weight", + "mot.mixtures.action.action_encoder.bias", + "mot.mixtures.action.action_encoder.weight", + "mot.mixtures.action.head.bias", + "mot.mixtures.action.head.weight" + ], + "omitted_source_tensors": [ + "action_normalizer.count", + "action_normalizer.fitted", + "action_normalizer.mean", + "action_normalizer.std", + "camera_encoder.bias", + "camera_encoder.weight", + "camera_head.bias", + "camera_head.weight", + "camera_normalizer.count", + "camera_normalizer.fitted", + "camera_normalizer.mean", + "camera_normalizer.std", + "structured_type_embedding" + ], + "pretrained_action_semantics": "absolute_ee_xyz_so3log_grip", + "target_action_semantics": "absolute_joint_position_grip", + "optimizer_restored": false +} diff --git a/spacedreamer_stack_cups_clean40_30ep/manifest.json b/spacedreamer_stack_cups_clean40_30ep/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..04de80e325a95953b9c4d1c618d35c8c975edfc8 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/manifest.json @@ -0,0 +1,928 @@ +{ + "status": "VALIDATED_ORIGINAL_FINAL30EP_RELEASE", + "training_job": "bifrost-2026111502493001-shouqx", + "step": 4350, + "epochs": 30, + "files": [ + { + "path": "CPU_validation.json", + "bytes": 1008, + "sha256": "40f3bee5664f3afba5b173ea57168944b7f6c7362118cf311365906c3fafb0cb" + }, + { + "path": "README.md", + "bytes": 7458, + "sha256": "f75d0fe3b967edfbc5956a70466836a93edeb8266d801fa71cbd0e48b348ed3b" + }, + { + "path": "checkpoint_metadata.json", + "bytes": 856, + "sha256": "3709afdfb089858776b4c16d5430c53b6a31e31c4f6a419ef34591dceff750d3" + }, + { + "path": "final_weights.pt", + "bytes": 11030029192, + "sha256": "b9c898a58d03a4b863d3ec9cc5b3b9c87e7ac00427a27478e61a17cbf269a6e7" + }, + { + "path": "initialization.json", + "bytes": 1104, + "sha256": "e7d9654086d1758bc509f6b47877423cabc6314e3c32439443a54b2568a7d7a6" + }, + { + "path": "postflight_normalizer_audit.json", + "bytes": 2771, + "sha256": "e02b02ca0501d9ab43a11baec6e7c0400e2ab27154722c510325d529f11fc954" + }, + { + "path": "reference_norm_stats.json", + "bytes": 3360, + "sha256": "10f598cd5148e9ac85fa2b7b8c3b2f2c503127fcb307f33ad3e5277f8a9ff6f8" + }, + { + "path": "runtime/requirements.txt", + "bytes": 456, + "sha256": "93ed58aca470a8fec133f87b19e479f28744f29c848a0e8e26cd74a729afa16b" + }, + { + "path": "runtime/run_inference.py", + "bytes": 1427, + "sha256": "86c4c73bd13919001de9991b15883a2c7e6c4aff2f59a0c15aae51bb0b208e53" + }, + { + "path": "runtime/runtime_packages.json", + "bytes": 4403, + "sha256": "54120c1011271310e5babd300edcbff2b4ccab01fb1a3f84ed8f28f54b0f94e3" + }, + { + "path": "runtime/source/model_old/src/fastwam/__init__.py", + "bytes": 23, + "sha256": "c4149c5ff0f9abd1517360daf761e692feb8570eb7a0caa2dfde52d4412c3e23" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/__init__.py", + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/dataset_utils.py", + "bytes": 3735, + "sha256": "3bc17590049c7cb54061acf8d731b07892e7f530c5d7eeac2613fa77a285a392" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/base_lerobot_dataset.py", + "bytes": 18557, + "sha256": "9a0690381f0ce0058bb0d526ea4fd4982939a4dbe293dabfc83d5abc086f0512" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/constants.py", + "bytes": 1915, + "sha256": "11ac9260341e1560c5eb0dd7acb79ed63c74180b8ab4113c27622ff213db417f" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py", + "bytes": 7067, + "sha256": "7d7daab219332b5abdfd5ec67ccaf3502e8eb583a17a4e482c3f3d4f7a691e89" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py", + "bytes": 33439, + "sha256": "40f1b94e6ceb50765fb8dfa3d498887c2a0fc2eb49f09770c5f36190e7110c98" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py", + "bytes": 19421, + "sha256": "cb647a1e955c3279e39ce8d7af7a37ea7432bbd71534c183f9ceebc8557dd89f" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py", + "bytes": 55587, + "sha256": "6abe9eccf17077eee478805b574c25980cedf7a4f673a3dee1cb7184db9126f8" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/processors/base_processor.py", + "bytes": 10811, + "sha256": "1f47bc85fee11e3d16714b2f1e3a2dc874f4609a1706a788617156d34b48b58e" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/processors/fastwam_processor.py", + "bytes": 13316, + "sha256": "d71bd147c6594ee9d1a50c04d22fa69ccb914d601c4ad816233496f5b583fda5" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/robot_video_dataset.py", + "bytes": 12358, + "sha256": "a30583e16418c2fd848e1588f42a28889fdc7b3b9dbaeccffbedbfe7fca0b081" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/action_state_merger.py", + "bytes": 2522, + "sha256": "669d6991415fc5eca44b32b129d15fee1e06e2f17d59d90350c252598033b427" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/image.py", + "bytes": 725, + "sha256": "4d7ab61d0844918525244db653831f0f632730046b2714a6163ff57c441a16fa" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/misc.py", + "bytes": 422, + "sha256": "8b505bd9e3a54ab2db7efcd64f0929d2e0303f5db1123de1074fd01fa9887e2b" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/relative_action.py", + "bytes": 4032, + "sha256": "e555e0558e948c4eea3d1119e274fa5d923af30862eec4380fb8528f4a319c47" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/rotation.py", + "bytes": 5603, + "sha256": "0de34b252d60b956cda07c0c19baebbd3c8a28916face700c6664c5815e11e91" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/utils/normalizer.py", + "bytes": 11274, + "sha256": "5ac89a646cd6395e98feaa832f8fdf7c7e00857080af3d9c15710e243c43ad43" + }, + { + "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/utils/rotation.py", + "bytes": 21110, + "sha256": "6169162f98faf02a3fe5201e2b8a1069fe299840d036ee7cad8af5fa8f561542" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/__init__.py", + "bytes": 35, + "sha256": "ae4c5733f4bbf7caf31df2ae6f789516f6a10cd84a321f7d0375a7034253ac6d" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/__init__.py", + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/action_dit.py", + "bytes": 13347, + "sha256": "1301d9224149de43bb701f620a5d41858ecc63c6b19a573ec32edd45a3bdb0a2" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/attention_masks.py", + "bytes": 2846, + "sha256": "5c84fd14dfba35086c75fbd14244609d54e52f482a658ca4f02ddd63800bbb31" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/fastwam.py", + "bytes": 48807, + "sha256": "36a61b652944a89dfd59dbba94ce206a664e40ed27868ca6d7bdbf0642e3bd0a" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/fastwam_idm.py", + "bytes": 20427, + "sha256": "aae332d17645a6744aafd85edde8e8d7dacbb8b110de3988015ca330146c3c65" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/fastwam_joint.py", + "bytes": 9795, + "sha256": "10937e5102839c8e6103aab828e3355fab6015299bdf66e303b056703dc79bb8" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/__init__.py", + "bytes": 415, + "sha256": "f26116de229557a37a4812dc4ee75971f5f9e8b80240786ea4d0e5b0ff703cbf" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/gradient.py", + "bytes": 565, + "sha256": "ba5d8f7272eb029dc6cd2849ca99b70f6ad5abb838d21c818beb0590620dc793" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/io.py", + "bytes": 7751, + "sha256": "1c6c1bc8a975b47092912833194fce25e2e3ac886fbab72d73af3ff5a00b3c62" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/loader.py", + "bytes": 8279, + "sha256": "792f66bd0609c1365256aa24d43301340f81260c1651d89b04d2ae8224a08726" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/state_dict_converters.py", + "bytes": 5226, + "sha256": "81d213373e5056a8d8bf31768b9b04164138ea78d0b306e316334841499f4bf9" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/mot.py", + "bytes": 22491, + "sha256": "fe0dce22413e18ff2e9cb249f7f9714f25d828b214f1a19e73feee029d29944b" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/schedulers/scheduler_continuous.py", + "bytes": 4068, + "sha256": "2ececce284f7b9fe876965fbf5e11ca7b69caaf693979266b0f63062a589f181" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/wan22.py", + "bytes": 16945, + "sha256": "b72998999b4b2c5096773092339b8ca81e9e7d96f398b81c57d3d22d14972ac4" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/wan_video_dit.py", + "bytes": 31351, + "sha256": "c254ac7f2a43da59f103864140bb84474f5588290f5fb17a6ed426d0b64e02c9" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/wan_video_text_encoder.py", + "bytes": 11459, + "sha256": "aedf7f131556a39b99a273deec3a85d67f41b5367cef41f310f33a807ae60a87" + }, + { + "path": "runtime/source/model_old/src/fastwam/models/wan22/wan_video_vae.py", + "bytes": 50058, + "sha256": "31b3fb9f844a76f5db5af4854e14802aae640b85b61d0de03a4b53b52515abb7" + }, + { + "path": "runtime/source/model_old/src/fastwam/runtime.py", + "bytes": 18262, + "sha256": "e19cd2abf94f3af9f65a231158406b3d61887136a603bd3a954335d3623319b0" + }, + { + "path": "runtime/source/model_old/src/fastwam/trainer.py", + "bytes": 35849, + "sha256": "551a81bb01adcbbe9c1bc753d397b3e53df1b9d80cc28041cef56c3a1f32607c" + }, + { + "path": "runtime/source/model_old/src/fastwam/utils/__init__.py", + "bytes": 96, + "sha256": "2893c05b99b17162941c70a41859c2808f8ae390be367a26cac7d5ab93ee197b" + }, + { + "path": "runtime/source/model_old/src/fastwam/utils/config_resolvers.py", + "bytes": 2526, + "sha256": "01c48cbd42b86e636b70ccc3e193a81147162ba890777102b2c61086fa415340" + }, + { + "path": "runtime/source/model_old/src/fastwam/utils/fs.py", + "bytes": 76, + "sha256": "69e6df236b43360428a613686c8f37c531c0d16a3a2c5b06a725c46b49d657c5" + }, + { + "path": "runtime/source/model_old/src/fastwam/utils/logging_config.py", + "bytes": 4679, + "sha256": "cadc4f6ad487c3f37171a6defad36d23d9ab8bf262f73a461cd9ee9b03b757e7" + }, + { + "path": "runtime/source/model_old/src/fastwam/utils/misc.py", + "bytes": 1223, + "sha256": "05aa51247936a8d3faac27f90b78b65ff83a688259bed55d114b6a617ced532d" + }, + { + "path": "runtime/source/model_old/src/fastwam/utils/pytorch_utils.py", + "bytes": 5049, + "sha256": "ffc483e7fb435cd654326750eaffdbfc9243837c3140dcc74e45b24abb751a33" + }, + { + "path": "runtime/source/model_old/src/fastwam/utils/samplers.py", + "bytes": 1339, + "sha256": "4d3bd9b2d498f4ad8f2cf9765d60c4bf5368fa6fa2a7d42d0db6210f66899652" + }, + { + "path": "runtime/source/model_old/src/fastwam/utils/video_io.py", + "bytes": 819, + "sha256": "9e9898b021a44e87aa1b4e6d13e3755a9869a2e27a7eb53c72a6802808d5bd80" + }, + { + "path": "runtime/source/model_old/src/fastwam/utils/video_metrics.py", + "bytes": 3446, + "sha256": "aaa37e458cc9e4bb79a17923b5ff5900c97b4309f7abd50db44cf76e9143ffd6" + }, + { + "path": "runtime/source/multimodal/audit_robodojo_warmup_reproduction.py", + "bytes": 14938, + "sha256": "ab2d694fc3b00d3047296eac4a196ee7a30180fa7abed2547bf1f68b781a2405" + }, + { + "path": "runtime/source/multimodal/build_piper_hdf5_index.py", + "bytes": 1287, + "sha256": "c6fa001c3363d4306e85a5eab2fe4e847f18d69c899b068259e8a35f090e1b9a" + }, + { + "path": "runtime/source/multimodal/canonical_spacedreamer_data.py", + "bytes": 39499, + "sha256": "0b5f5c718c882f24012c3ae8a2070ffb53e386aea53a1ee60d8eb0a361906920" + }, + { + "path": "runtime/source/multimodal/controlled_directional_mot.py", + "bytes": 6970, + "sha256": "8f1c53f5a7570ed251057274136a733faa4ef653b46a7728ad00539670ab4503" + }, + { + "path": "runtime/source/multimodal/depth_preprocessing.py", + "bytes": 3254, + "sha256": "4fe296aee8e22ee56e488bf6fcbf6279e7f1ac385b286ae3b7f705d92940d482" + }, + { + "path": "runtime/source/multimodal/evaluate_spacedreamer_validation.py", + "bytes": 2147, + "sha256": "ec76991435cbe39bec9e3ea506810cabbf466a129a9ad4544bed906b4b7fe550" + }, + { + "path": "runtime/source/multimodal/finetune_galexea_rgb_action.py", + "bytes": 28110, + "sha256": "dc29a32aa0dc5afad4273f30e690016b3a65e0a385d4aa751b17e6db7c8d2d2a" + }, + { + "path": "runtime/source/multimodal/galexea_mcap.py", + "bytes": 18359, + "sha256": "2f2e60c5a81bd44a9c9d0272af79ebb2b9dd8004b12fbddffb2a7c8d4e84ace1" + }, + { + "path": "runtime/source/multimodal/joint_rgb_action_diffusion.py", + "bytes": 7935, + "sha256": "e71d7210e7bbf20e3807215e6b9feacc48f7c2f31b496e7d4e5a4af76f7323eb" + }, + { + "path": "runtime/source/multimodal/joint_rgb_action_diffusion_shared_timestep.py", + "bytes": 5510, + "sha256": "cfcf934e2c77d4d69a6a7280e0dcf264a55a42e885b99612cb89bf721a35a96f" + }, + { + "path": "runtime/source/multimodal/joint_rgb_depth_action_fastwam_diffusion.py", + "bytes": 4807, + "sha256": "8bc87f96c4ed0d6076f9c5be3367e290bb6863c3e9b9f5146539971c7b243541" + }, + { + "path": "runtime/source/multimodal/joint_rgb_depth_point_action_diffusion.py", + "bytes": 10866, + "sha256": "0874a83ebfe1f4452fb8eedace7293d750fd1aa5814fd93ee6bd68c9e12e453b" + }, + { + "path": "runtime/source/multimodal/joint_rgb_depth_point_action_fastwam_diffusion.py", + "bytes": 7018, + "sha256": "78d7d14c323178994391414e6cf7b102e97adddfb79266d4e66385a8e473d165" + }, + { + "path": "runtime/source/multimodal/joint_rgb_geometry_action_diffusion.py", + "bytes": 9126, + "sha256": "e94cf04caac709aa806bec600e38db472dae896b122ae23be8d9d35ef24fad56" + }, + { + "path": "runtime/source/multimodal/online_egoverse_action.py", + "bytes": 27776, + "sha256": "c484a0e11953ba78a154bb38c170ba890d8ef427a0ec515148312466ee966c16" + }, + { + "path": "runtime/source/multimodal/optimizer_step_scheduler.py", + "bytes": 5408, + "sha256": "84b09ff87013725d120bf2cd5357a71752d3e406724b57fdffe054de262698c7" + }, + { + "path": "runtime/source/multimodal/piper_hdf5.py", + "bytes": 23659, + "sha256": "544d6e39836339de5c69a22756df198f09cce52adcfdafab97e0d63e4e95d2a0" + }, + { + "path": "runtime/source/multimodal/post_training_test.py", + "bytes": 29116, + "sha256": "5a41fd8f38ec3baf6a0e69619d074fffa8acb30aaeed22c26612d0576120cbb8" + }, + { + "path": "runtime/source/multimodal/post_training_test_action_mot.py", + "bytes": 21288, + "sha256": "ba1718fddc337b11e9e7f7c4d7e68241cfe87db6cd5123cfcfd9ee7aa7df7cf1" + }, + { + "path": "runtime/source/multimodal/post_training_test_rgb_depth_point.py", + "bytes": 22379, + "sha256": "418af7e62aa7ae6115538a01364c0ec5ee7647321c7bea6701e820637385445e" + }, + { + "path": "runtime/source/multimodal/precompute_robodojo_exhaustive_rgb_latents.py", + "bytes": 14671, + "sha256": "943e83702c1c5eb1ee224d691d14c53aa87f8084c8824304aff91bbbf8d8d28a" + }, + { + "path": "runtime/source/multimodal/precompute_robodojo_geometry_latents.py", + "bytes": 10570, + "sha256": "27b2c7f5c27b865921f4403ac68e861d2cd02e4f928bb1e96400eea675d0163b" + }, + { + "path": "runtime/source/multimodal/precompute_robodojo_native_depth_latents_fastwam.py", + "bytes": 12125, + "sha256": "1596c08fd81ca38ef8d4dcc11dec36a7b9df17c57282b057571f44176b7e9ca5" + }, + { + "path": "runtime/source/multimodal/precompute_robodojo_native_point_latents_fastwam.py", + "bytes": 14509, + "sha256": "ff70aa582dc179104058a451fc4796d0b502394f8b30fe9026e0bd31666bb42e" + }, + { + "path": "runtime/source/multimodal/preflight_robodojo_ego_first_cam.py", + "bytes": 6302, + "sha256": "bae3d9a15a791cee000c85daca94c0c977895ff88be0da835c5f24361d615d53" + }, + { + "path": "runtime/source/multimodal/prepare_point_cloud_cache.py", + "bytes": 7450, + "sha256": "b6c3bcacc0ae1d912ae1370525d841896b4f6acba51d2ca53e0cab1a561fddde" + }, + { + "path": "runtime/source/multimodal/rgb_action_model_factory.py", + "bytes": 3791, + "sha256": "285364b62bf5f9ca488f003328150aacd95a7fd4a98e990965cfe2cad9f2965a" + }, + { + "path": "runtime/source/multimodal/rgb_depth_point_action_model_factory.py", + "bytes": 7916, + "sha256": "1ac6d134c3f94f4b04604d6b9a35cc4a98bafff728361affc5b0c69211e71ba3" + }, + { + "path": "runtime/source/multimodal/robodojo_ego_first_cam.py", + "bytes": 11763, + "sha256": "9543f5f29ea78db8c0763450c08fee8bd52a9fc1e9433826e161338b95b3a4f1" + }, + { + "path": "runtime/source/multimodal/robodojo_geometry_data.py", + "bytes": 12794, + "sha256": "37221e841d324cc8713a142515591cd375c09ce01d7cb896be5f18fd75e5332e" + }, + { + "path": "runtime/source/multimodal/robodojo_lerobot_v30.py", + "bytes": 17380, + "sha256": "4bfc2df11d6786bc78b5fa1452904f6cca66e1203b3a8e8524d81fca20ee9563" + }, + { + "path": "runtime/source/multimodal/robodojo_native_depth_action_model.py", + "bytes": 15497, + "sha256": "21ac2f0d5aabcde94fc2961db747449fd4d08e27ad7067b47daf460a813be40e" + }, + { + "path": "runtime/source/multimodal/robodojo_native_depth_data.py", + "bytes": 15063, + "sha256": "4e982175ceb1211f08386a339a973eb0d377c2a4ac1ff368738857a484820584" + }, + { + "path": "runtime/source/multimodal/robodojo_native_depth_point_action_model.py", + "bytes": 18092, + "sha256": "eec08ebb5ada5a278916bf6f414fe82e5749d5c5159f7d2e1c576b476f76e6fe" + }, + { + "path": "runtime/source/multimodal/robodojo_native_depth_point_data.py", + "bytes": 12559, + "sha256": "16895df58500abc5b9926b37a6f5face6f9d6e7d1f8fda15331efee165a5dc28" + }, + { + "path": "runtime/source/multimodal/robodojo_rgb_geometry_action_model.py", + "bytes": 7859, + "sha256": "5a496d410f42f206542e5111d1ac38003b646b0f50e55dfb78a2ee3e50bc5bc0" + }, + { + "path": "runtime/source/multimodal/robodojo_v2_checkpoint_contract.py", + "bytes": 6497, + "sha256": "f361f24c019c0e04989208457cc0b92f47dcb4567499902a1f62ac6af5afc89c" + }, + { + "path": "runtime/source/multimodal/scripts/audit_piper_pretrained018126.py", + "bytes": 3894, + "sha256": "5011329e2f3c0c7962c6c31df7fe0a946042dfaded19a08b51209e32dbbd9ab2" + }, + { + "path": "runtime/source/multimodal/scripts/audit_piper_pretrained018126_target_norm.py", + "bytes": 2207, + "sha256": "dd817495b883b408eb53a0f04ba298cc87f4084cbbdd8a0fa53582a46cbaf17b" + }, + { + "path": "runtime/source/multimodal/scripts/audit_robodojo_full35_training.py", + "bytes": 4532, + "sha256": "1f8dd936fb44781533ed3db581b82017d27d99a3b2147f79300af9b8bf283864" + }, + { + "path": "runtime/source/multimodal/scripts/audit_robodojo_full_normalizer_noaug_configs.py", + "bytes": 6127, + "sha256": "58a4bc39a9d59c1add35f44d8b3931117ba23be8f61bfb70163205a17d368917" + }, + { + "path": "runtime/source/multimodal/scripts/audit_source_normalizer_noaug_configs.py", + "bytes": 6930, + "sha256": "cb72a0c6361b40340382625022ef93f978e3b1bcc9938a5a447861ab0c3a55b5" + }, + { + "path": "runtime/source/multimodal/scripts/audit_spacedreamer_v3_robodojo_srs_resume5000.py", + "bytes": 2683, + "sha256": "9bf8dd49834503c92664df78d3a33e7b08244454d275595780550e69d719c53c" + }, + { + "path": "runtime/source/multimodal/scripts/audit_spacedreamer_v3_robodojo_srs_trail1.py", + "bytes": 3388, + "sha256": "c96e68f282fe17da9ae1d84d21b11eecb2dc8e5fb80e50b028c6eaa6169df3c3" + }, + { + "path": "runtime/source/multimodal/scripts/audit_target_normalizer_noaug_configs.py", + "bytes": 2826, + "sha256": "4b817fffe274c9e139ba79c1d07ec6ec50af4d92027cfe0a85c2fb39b9a47dc7" + }, + { + "path": "runtime/source/multimodal/scripts/build_egoverse_canonical_motion_cache.py", + "bytes": 12355, + "sha256": "a34371ef4c32998343a19646977450d5cad51f2514a010cc382579fd6c49697d" + }, + { + "path": "runtime/source/multimodal/scripts/build_spacedreamer_canonical_mix.py", + "bytes": 27136, + "sha256": "c8769ed42f9e36e62c734bfe960efcc57fe8657fc46af5852a9598ed067568b2" + }, + { + "path": "runtime/source/multimodal/scripts/build_spacedreamer_indexed_canonical_manifest.py", + "bytes": 12295, + "sha256": "a06fbeda5afc37f141cf97bfc4686613ed11e3fd86db112a7d7e6ba6e58b6a76" + }, + { + "path": "runtime/source/multimodal/scripts/evaluate_robodojo_train_action_generation.py", + "bytes": 23899, + "sha256": "cc950de9e1efae04e87af46600b8beee9742a2689267416cf929e718580b4ed4" + }, + { + "path": "runtime/source/multimodal/scripts/merge_spacedreamer_canonical_parts.py", + "bytes": 6149, + "sha256": "d43aa61829542c19efe07950dc52802e6c3086706fbaa1e81dcb19f821c63bd1" + }, + { + "path": "runtime/source/multimodal/scripts/prepare_robodojo_full35_base_cache.py", + "bytes": 1389, + "sha256": "a446bedc5b09d769761a87e572247c3df4fe7ade950d6ec8c3c3893e4496ae35" + }, + { + "path": "runtime/source/multimodal/scripts/run_spacedreamer_canonical_full_build.py", + "bytes": 4433, + "sha256": "5fe85a4edeaec5572c7bac49c2541c58b86393e34db5b8d2b66aeba4841c6d28" + }, + { + "path": "runtime/source/multimodal/scripts/smoke_robodojo_geometry_cached_dataset.py", + "bytes": 3709, + "sha256": "fba36fe8f2bf0d8802f5f4013867bd4a651771222daf07bfb42361aeeb32e909" + }, + { + "path": "runtime/source/multimodal/scripts/validate_spacedreamer_canonical_mix.py", + "bytes": 12597, + "sha256": "1397bd66a8ff17f59eb2668aa115e34ace8c44b05782d7805994923f01e84e88" + }, + { + "path": "runtime/source/multimodal/scripts/validate_spacedreamer_indexed_canonical.py", + "bytes": 17200, + "sha256": "1efcc093f42978254a1fbc6c26487261ff9bec25c38adcd8fff218af59d03ba4" + }, + { + "path": "runtime/source/multimodal/scripts/verify_robodojo_geometry_experiment.py", + "bytes": 7763, + "sha256": "ac10edb1850a3229f52c7c472bde0584a9d212095dd37ea98c87454729c4fb11" + }, + { + "path": "runtime/source/multimodal/scripts/visualize_depth_preprocessing.py", + "bytes": 6814, + "sha256": "9f20bbef948916866cdc2a2ac3a5010e549ba0787ed350b71435adc2aa66c28a" + }, + { + "path": "runtime/source/multimodal/spacedreamer_ego_dataset.py", + "bytes": 38628, + "sha256": "633e1c7de3799c0d05329bfe8edf97b1dced4541ed6cec3cea1afd02dc80726c" + }, + { + "path": "runtime/source/multimodal/train_galexea_rgb_state_joint_action_ablation.py", + "bytes": 28500, + "sha256": "ed1e3cef681348472320ac04a9bb7f7e1f26a203e25f2673cd40239187c84d3f" + }, + { + "path": "runtime/source/multimodal/train_joint_rgb_depth.py", + "bytes": 67200, + "sha256": "53b1f71eda541f36499b7be813b805068aa22245693cb9fea0e6c302fa0bebd8" + }, + { + "path": "runtime/source/multimodal/train_joint_rgb_depth_point.py", + "bytes": 40421, + "sha256": "012bcd542e7e43bc99b7606d76dc2af9a1816e9630246fcfa779b1d0c6a833fe" + }, + { + "path": "runtime/source/multimodal/train_joint_rgb_depth_point_action.py", + "bytes": 35857, + "sha256": "ddbcabac68c212ad3aab2afd1e825ce3de3120be32b14ac791670b2d8b66bf0d" + }, + { + "path": "runtime/source/multimodal/train_piper_rgb_state_joint_action_quantile_4way.py", + "bytes": 27604, + "sha256": "76353a68caeb954a3be02359a75c626072e93e617cb2c2b079363d72786ec0d9" + }, + { + "path": "runtime/source/multimodal/train_robodojo_rgb_geometry_action_absolute_ee.py", + "bytes": 12293, + "sha256": "ced9745b4ab2bca6b3bea5508f433247106af98dc708cd08bfe7a9400bbc19ce" + }, + { + "path": "runtime/source/multimodal/train_robodojo_rgb_native_depth_point_state_joint_action_fastwam.py", + "bytes": 13798, + "sha256": "9e2519a5dbe5a960a5cdedfa247630a6dfd28bef72fecfed0b4c4bbc3cbc18be" + }, + { + "path": "runtime/source/multimodal/train_robodojo_rgb_native_depth_state_joint_action_fastwam.py", + "bytes": 17614, + "sha256": "32fddebea5accffc5be727e61d7f7c4865a75522c61d2293bca1b503ebe4dd7f" + }, + { + "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_4way.py", + "bytes": 80135, + "sha256": "26b1ed4fc9c8c6b079a8b212aae78287a6b36f8bb35c3ed171d8bfa511eaed39" + }, + { + "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_ablation.py", + "bytes": 42461, + "sha256": "485b33691060a00f85beae7e180ac11f30881982406f2698622e0ed91a93969e" + }, + { + "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_4way.py", + "bytes": 7861, + "sha256": "e069cd1b6198e7c591ba179d5f26cf0b02bad98133cb5b8cf26ae558acc9caa0" + }, + { + "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way.py", + "bytes": 10551, + "sha256": "465b849da066d562e12b43a31557f74eb3faac71669ef52f87ac90a185185b8d" + }, + { + "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_quantile_shared_timestep_4way.py", + "bytes": 3857, + "sha256": "119231049ec988daf4fbaa4b4db7f8013d84fd47e23a8bf0596131b4e2911143" + }, + { + "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_ego_first_cam_meanstd_4way.py", + "bytes": 26471, + "sha256": "600eca1cd377a5399f1659260a798e548f368f1788566d2a5467ca9d66ed168e" + }, + { + "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_episode_anchor_4way.py", + "bytes": 7408, + "sha256": "d90b516158b2781c3b913098c6d488b05db1e30714269330ffce6daf8b55fd6d" + }, + { + "path": "runtime/source/multimodal/train_spacedreamer_ego.py", + "bytes": 68897, + "sha256": "c8f87e0a82f564a07dc58a09d4cb6220fc850d15654662307abf6cd55f8c0140" + }, + { + "path": "runtime/source/multimodal/wan_rgb_action_mot.py", + "bytes": 10421, + "sha256": "a33437c1dbeed4dcaa6cf70e5505bd6ddc89a2324e11d81f5e18bff56ae115e7" + }, + { + "path": "runtime/source/multimodal/wan_rgb_depth_initialization.py", + "bytes": 24878, + "sha256": "025d5c2cef3ddfced4ef0f3f4b25644cf68618e5e75be19f4eb20be40be79df3" + }, + { + "path": "runtime/source/multimodal/wan_rgb_depth_point_action_mot.py", + "bytes": 17358, + "sha256": "238447d95a5b07d876e4ddf235077ac40cd8a945af8b27f4147811c771868557" + }, + { + "path": "runtime/source/multimodal/wan_rgb_depth_point_initialization.py", + "bytes": 13728, + "sha256": "724f8ef02bb42754d2cc3fa3e2ed0e507e2a0451f96e37e2bb72a00da847ba7d" + }, + { + "path": "runtime/source/multimodal/world_point_cloud.py", + "bytes": 25523, + "sha256": "3323ae5e1779126182e5a27ffcd39336dac5feb3d4db4f3ab572e5483272180b" + }, + { + "path": "runtime/source/vda/utils/dc_utils.py", + "bytes": 3675, + "sha256": "45103ac573780e51ae54bbc3a010e391003c0e8bbd02746cb5f493d60d4e7907" + }, + { + "path": "runtime/source/vda/utils/util.py", + "bytes": 2449, + "sha256": "f02cf85f5a9543ce0f832a4fde1492076026650c8eb25c6a8cc4315bf67a8ed2" + }, + { + "path": "runtime/source/vda/video_depth_anything/dinov2.py", + "bytes": 15178, + "sha256": "d22df8d0da44c8525ebed898fd9c33b3acfcda919a84d577cc2a08bfd3d50ac1" + }, + { + "path": "runtime/source/vda/video_depth_anything/dinov2_layers/__init__.py", + "bytes": 382, + "sha256": "e3c28c7b775165b6844000eb69f44d104d6c3803ac7da5d59f6ac3ecfb46d9d8" + }, + { + "path": "runtime/source/vda/video_depth_anything/dinov2_layers/attention.py", + "bytes": 2343, + "sha256": "0d2410ce8b8b4499a6b0cdab939ae2bceab8971efd8660d999b0f3088f28cbf7" + }, + { + "path": "runtime/source/vda/video_depth_anything/dinov2_layers/block.py", + "bytes": 9332, + "sha256": "093266ea08084c7cce7b8ddd779ecfaaa13055735faa8a7d68b5924f385b395b" + }, + { + "path": "runtime/source/vda/video_depth_anything/dinov2_layers/drop_path.py", + "bytes": 1160, + "sha256": "949100aac4d9c19c9d65319dbd362886c5d46a72d7586ba4fd500e9f1d51570c" + }, + { + "path": "runtime/source/vda/video_depth_anything/dinov2_layers/layer_scale.py", + "bytes": 823, + "sha256": "356817b5d1390737128cbcc03a5333fd3060b73beae478772282ccb851d0ef85" + }, + { + "path": "runtime/source/vda/video_depth_anything/dinov2_layers/mlp.py", + "bytes": 1272, + "sha256": "c18c5b3f021408597a473bb710070ec3fb3b39f5a2e42e88ae83ced425a608b5" + }, + { + "path": "runtime/source/vda/video_depth_anything/dinov2_layers/patch_embed.py", + "bytes": 2832, + "sha256": "b3f82e19926c9cfb63f10136aca5e8a723bde1ad62dae1d04f90d405f9439baf" + }, + { + "path": "runtime/source/vda/video_depth_anything/dinov2_layers/swiglu_ffn.py", + "bytes": 1859, + "sha256": "df81d9901240214edc1831ce1b77558dcd6227b29a2f000fc5f546df55d2d09a" + }, + { + "path": "runtime/source/vda/video_depth_anything/dpt.py", + "bytes": 5467, + "sha256": "4751fac16206892889d74fd7f564f275e0efa3a903172c9350a1b55ed850c7d8" + }, + { + "path": "runtime/source/vda/video_depth_anything/dpt_temporal.py", + "bytes": 5909, + "sha256": "b80c448f4fe66d0ca7092be887c0ea3925a4ce7c03cffa0083480e7e2f55805d" + }, + { + "path": "runtime/source/vda/video_depth_anything/motion_module/attention.py", + "bytes": 16719, + "sha256": "15e4049641f04f73027ea520ada3ba2c81196cea1339f0d6a6063d944df61c8b" + }, + { + "path": "runtime/source/vda/video_depth_anything/motion_module/motion_module.py", + "bytes": 12744, + "sha256": "73d6a9cee40681532ae2603dd426405e840dc5c592a313005fa367f6fe4c69e8" + }, + { + "path": "runtime/source/vda/video_depth_anything/util/blocks.py", + "bytes": 4064, + "sha256": "8a9fde84bddbfa2498b5f5ef06895ddc941c1fc12c5919e9891fad481997b7dd" + }, + { + "path": "runtime/source/vda/video_depth_anything/util/transform.py", + "bytes": 6075, + "sha256": "a97df1dbb687457a40c387fdc3482edee7ddc5c142c7d2735ca6235a8ae2885f" + }, + { + "path": "runtime/source/vda/video_depth_anything/video_depth.py", + "bytes": 6712, + "sha256": "208e882d60a41434b3a0d9935a7025b01cce0756c17b3d889c6db0c4c945eb29" + }, + { + "path": "runtime/source/vda/video_depth_anything/video_depth_stream.py", + "bytes": 6833, + "sha256": "23ebab70525f318cf0bfeba38dfcac3f1c345ba50aaa1bb5c6dbae671b0caa6d" + }, + { + "path": "runtime/spacedreamer_galexea.py", + "bytes": 14685, + "sha256": "d694e516d3f6baca2a49df8086c95b5438147091c9eb27bc684f16ffd319942f" + }, + { + "path": "text_context.pt", + "bytes": 1050977, + "sha256": "49cf3918423e55ed12776ac91546bd4efb4142fd4645309b34b8526953a7cc92" + }, + { + "path": "training_complete.json", + "bytes": 358, + "sha256": "92e46d8dd3370060bacf464880b0fcec4b712629c83e06b0203cfec1baccb07a" + }, + { + "path": "training_config.json", + "bytes": 7131, + "sha256": "a94d0f7e603f5f905ee694c7bed920693bb0b32db77869f80878855fe8311718" + }, + { + "path": "training_reference/code/audit.py", + "bytes": 1956, + "sha256": "110519fba757e41fb284c9d8853e416f4b9312fe8d33d7d79519fded2a41cc83" + }, + { + "path": "training_reference/code/build_smoke_sample.py", + "bytes": 2493, + "sha256": "aa2976e044bab50de47268fff191dc73122308ce4d3188169a2cf06f046e6263" + }, + { + "path": "training_reference/code/cache_calibrations.py", + "bytes": 2398, + "sha256": "65c51b36d239cbbfd4f9dea9d40280f445bcc276bb5164ea9a71806fb1f36e19" + }, + { + "path": "training_reference/code/calibration.py", + "bytes": 3092, + "sha256": "39f4cfc0ab2a841a2e9c44f04af149aee9b2b52db4c42e6feab2b54b75d4c0ee" + }, + { + "path": "training_reference/code/check_global_batch.py", + "bytes": 2872, + "sha256": "de59449caa64e50eef846768454895d4d904daf6339f797be80ea959fbd23e6a" + }, + { + "path": "training_reference/code/cluster_entry.py", + "bytes": 7841, + "sha256": "d42963927ce7637ebd5ad2bee0102b595dff41b564aa318c81781de833d7cd98" + }, + { + "path": "training_reference/code/common.py", + "bytes": 5682, + "sha256": "54bdfd54a3dca23bfa7073d8d85f011b5009bd81623f0dac7d5a8c0ca561134d" + }, + { + "path": "training_reference/code/cpu_bootstrap.py", + "bytes": 315, + "sha256": "59ffd83c32036d4bd3a83b01f480058e51f3f5a04838500c85a197afe7d0ae50" + }, + { + "path": "training_reference/code/cpu_preflight.py", + "bytes": 3879, + "sha256": "53455315a2aa2141293e111e3cbd67b5bc9b600e5df3c69938c03c791713a5b3" + }, + { + "path": "training_reference/code/cuda_probe.py", + "bytes": 2547, + "sha256": "01949bf05a92ba781aa65f929d5a6bd7bb7972231524b89e3186ee18f14f12c2" + }, + { + "path": "training_reference/code/data.py", + "bytes": 14912, + "sha256": "697b7dd046b054e2db0c2e61f8350b02b3a973458ac9670edb295f3903c156d6" + }, + { + "path": "training_reference/code/decode_selected.py", + "bytes": 695, + "sha256": "6754d0f88170e5e87dd2ebe733c350c405debd3492801d71693ce31038bc15f8" + }, + { + "path": "training_reference/code/distributed.py", + "bytes": 1052, + "sha256": "cdafe58a0101d1adb508b44d06f1dfbf4c52d8fc73ba66cbcdbf7fd3d0dfa843" + }, + { + "path": "training_reference/code/model.py", + "bytes": 7629, + "sha256": "99bafc6873be3cca2d016df41c30c7741841701518256617038f36972969ef3e" + }, + { + "path": "training_reference/code/output_guard.py", + "bytes": 1453, + "sha256": "e9c1c6192dc71c478c7d612d544e238d8995d5b45bbf38145f3dd66ae41d4969" + }, + { + "path": "training_reference/code/prepare.py", + "bytes": 7614, + "sha256": "7e63fdae5a8b765a206c5c2196d507c96770fbdb73b9356cc1b9faa8438df088" + }, + { + "path": "training_reference/code/selection_data.py", + "bytes": 1297, + "sha256": "175ed3df31c6e13ed2276078e888e8605485d1aff26282a861a6bbbe4c879be3" + }, + { + "path": "training_reference/code/staging.py", + "bytes": 2327, + "sha256": "6d8f94c6e24db6373fd1129fdf7ee0387c03b9505b37c77f0c421b56c3cf7521" + }, + { + "path": "training_reference/code/storage_budget.py", + "bytes": 12441, + "sha256": "e7ed733b8ccce3480d59f46c54293cf469b34269e2bc301203ebb71a00b8bb2c" + }, + { + "path": "training_reference/code/submit.py", + "bytes": 4656, + "sha256": "70dd48052e9b5b8ecea25605811bba0f9e20d441e28b664fb3af860f787910c2" + }, + { + "path": "training_reference/code/submit20.py", + "bytes": 6964, + "sha256": "41763dfa7b49b466462fca56c2bf4d27dea2ee06c6287c47bfbebaf12ed4ca4a" + }, + { + "path": "training_reference/code/train.py", + "bytes": 14828, + "sha256": "28a320e6fa8a737ce71e880d5d1103246b1e054975c0db9723ed9a0170ed0788" + }, + { + "path": "training_reference/code/vae_runtime.py", + "bytes": 1005, + "sha256": "ab129de4f7d5aff6271176b6305d00ba78931b4510595a41d0f2c28cd86192fe" + } + ], + "total_bytes": 11033164188, + "shared_dependencies": { + "spacedreamer_20epochs/assets/Wan2.2-TI2V-5B/Wan2.2_VAE.pth": "20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36", + "spacedreamer_20epochs/assets/metric_video_depth_anything_vitb.pth": "f6f58576b9680a112f2428d4f39aff92656d3ae85745b0164675ace8d5b1fade" + } +} diff --git a/spacedreamer_stack_cups_clean40_30ep/postflight_normalizer_audit.json b/spacedreamer_stack_cups_clean40_30ep/postflight_normalizer_audit.json new file mode 100644 index 0000000000000000000000000000000000000000..228caa6e7acf0dd69978ae1202e0ed4cddab93da --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/postflight_normalizer_audit.json @@ -0,0 +1,64 @@ +{ + "status": "TRAINING_COMPLETE_POSTFLIGHT_BF16_COMPARISON_FALSE_POSITIVE_CONFIRMED", + "platform_state": "JOB_FAILED", + "job": "bifrost-2026111502493001-shouqx", + "epochs": 30, + "optimizer_steps": 4350, + "rows_of_training_metrics": 4350, + "all_loss_and_grad_metrics_finite": true, + "last_metrics": { + "step": 4350, + "epoch": 30.0, + "lr": 3.4098125841150466e-05, + "seconds": 38622.264301493764, + "point_prediction_grad_norm": 0.00035875628236681223, + "loss_rgb": 0.045066364109516144, + "loss_depth": 0.03409406542778015, + "loss_point": 0.043846286833286285, + "loss_action": 0.008644891902804375, + "video_timestep_mean": 800.71875, + "action_timestep_mean": 461.4375, + "prediction_rgb_norm": 1707.3424072265625, + "prediction_depth_norm": 2257.48876953125, + "prediction_point_norm": 1777.875732421875, + "prediction_action_norm": 107.14266967773438, + "grad_norm": 0.3622361421585083 + }, + "last100_mean_loss_action": 0.008668523519299925, + "weights": "/workspace/group_share/rc-wbc/shouqx@xiaopeng.com/real_robot_training/stack_cups_new67_clean40_refitnorm_30ep_16h200_r2_20261114/runs/stack_cups/final_weights.pt", + "weights_bytes": 11030029192, + "normalizer_sha256": "10f598cd5148e9ac85fa2b7b8c3b2f2c503127fcb307f33ad3e5277f8a9ff6f8", + "checks": [ + { + "buffer": "state_q01", + "dtype": "torch.bfloat16", + "matches_exact_expected_after_dtype_conversion": true, + "max_joint_rounding_rad": 0.0037899017333984375, + "max_gripper_rounding_raw": 0.023340225219726562 + }, + { + "buffer": "state_q99", + "dtype": "torch.bfloat16", + "matches_exact_expected_after_dtype_conversion": true, + "max_joint_rounding_rad": 0.005093097686767578, + "max_gripper_rounding_raw": 0.19965362548828125 + }, + { + "buffer": "action_q01", + "dtype": "torch.bfloat16", + "matches_exact_expected_after_dtype_conversion": true, + "max_joint_rounding_rad": 0.0017759203910827637, + "max_gripper_rounding_raw": 0.0 + }, + { + "buffer": "action_q99", + "dtype": "torch.bfloat16", + "matches_exact_expected_after_dtype_conversion": true, + "max_joint_rounding_rad": 0.00553131103515625, + "max_gripper_rounding_raw": 0.19965362548828125 + } + ], + "root_cause": "Export verifier converted quantized BF16 buffers back to FP32 and required bitwise equality to original FP32 statistics. All4 buffers exactly match correctly quantized selected40 statistics. Data pipeline normalizes from the original JSON, not these stored buffers.", + "inference_note": "Use accompanying reference_norm_stats.json for exact FP32 normalization. No new GPU inference acceptance is claimed by this metadata/buffer audit.", + "checked_utc": "2026-11-15T06:13:04Z" +} diff --git a/spacedreamer_stack_cups_clean40_30ep/reference_norm_stats.json b/spacedreamer_stack_cups_clean40_30ep/reference_norm_stats.json new file mode 100644 index 0000000000000000000000000000000000000000..e8d176575222b8650b1272c9020c3dbede88bc3e --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/reference_norm_stats.json @@ -0,0 +1,136 @@ +{ + "norm_stats": { + "state": { + "mean": [ + -0.13387035604186223, + 0.9388066769062815, + -0.299460975359783, + -0.312899377843042, + 0.022158233386334263, + -0.013364365686538105, + 87.67583665493922, + 0.12845319557789786, + 1.0475177045852482, + -0.3682720079020069, + -0.4035744942019698, + 0.09048382540451419, + 0.05871849529736292, + 88.37490834410208 + ], + "std": [ + 0.23867227111243974, + 1.0532982660446137, + 0.4497836674554094, + 0.521777337868213, + 0.0756900378886052, + 0.06476722031329171, + 26.71132088139677, + 0.1772370447709506, + 1.0844598923871354, + 0.49809485962894984, + 0.5703715013499049, + 0.1773866395459158, + 0.07316617681887644, + 23.618158731215697 + ], + "q01": [ + -0.7902127504348755, + 0.21595744788646698, + -1.4180850982666016, + -1.5263829231262207, + -0.26163404583930966, + -0.22297872602939606, + 12.665559768676758, + -0.04957446828484535, + 0.19914893805980682, + -1.4697531938552857, + -1.5238298177719116, + -0.19659574329853058, + -0.07021276652812958, + 18.101660156250002 + ], + "q99": [ + 0.0274468082934618, + 2.8855319023132324, + -0.004680851008743048, + 0.15954042673110888, + 0.2176595777273178, + 0.07531914860010147, + 100.07527160644531, + 0.5876595973968506, + 2.826382875442505, + -0.003617021255195141, + 0.0663829818367958, + 0.8282978534698486, + 0.2704255282878876, + 101.19965362548828 + ] + }, + "actions": { + "mean": [ + -0.14292097344549323, + 0.7355806430838492, + -0.29974345955300896, + -0.3301010056178284, + -0.0026388765341016007, + -0.03810476927142731, + 87.42233316834316, + 0.11110324934059856, + 0.8414215713411128, + -0.3732627181630525, + -0.42123973672921317, + 0.06530478815310313, + 0.03338203608640254, + 83.2737688514841 + ], + "std": [ + 0.2332689717409583, + 1.0724814487217418, + 0.4633568221251692, + 0.5291241592842331, + 0.07604966254227942, + 0.06502434652450503, + 32.40528197209235, + 0.17417586584814845, + 1.10393937977348, + 0.514695628729752, + 0.5803439554075464, + 0.17785820538290803, + 0.07406519902851658, + 36.22980289874895 + ], + "q01": [ + -0.7869321703910828, + -0.0, + -1.4373400211334229, + -1.5707999467849731, + -0.2991262674331665, + -0.24697090685367584, + 0.0, + -0.07516505569219589, + -0.0, + -1.4986991882324219, + -1.5707999467849731, + -0.22396118938922882, + -0.09664078801870346, + 0.0 + ], + "q99": [ + 0.012271846644580364, + 2.723245506286619, + 0.0, + 0.15800002217292786, + 0.19174760580062866, + 0.05368932709097862, + 100.0, + 0.5599029660224915, + 2.6507186889648438, + 0.0, + 0.06442718952894211, + 0.8053399324417114, + 0.24697090685367584, + 101.19965362548828 + ] + } + } +} diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/requirements.txt b/spacedreamer_stack_cups_clean40_30ep/runtime/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f7fda83a32011ccc38211735d62b0f429192ef72 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/requirements.txt @@ -0,0 +1,22 @@ +# CPU packaging validation versions; choose the matching PyTorch CUDA wheel index. +torch==2.9.0+cu128 +torchvision==0.24.0+cu128 +numpy==1.26.4 +opencv-python-headless==4.11.0.86 +Pillow==12.2.0 +accelerate==1.13.0 +einops==0.8.1 +safetensors==0.7.0 +transformers==4.57.3 +ftfy==6.3.1 +sentencepiece==0.2.1 +PyYAML==6.0.3 +h5py==3.14.0 +zarr==2.18.7 +numcodecs==0.15.1 +fasteners==0.20 +asciitree==0.3.3 +zstandard==0.25.0 +lz4==4.4.5 +easydict==1.13 +huggingface-hub==0.36.2 diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/run_inference.py b/spacedreamer_stack_cups_clean40_30ep/runtime/run_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..6b3fc8d1a3c35019cda6f486b1ac46e7080b78a6 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/run_inference.py @@ -0,0 +1,29 @@ +"""Predict and save an action chunk from one NPZ observation; no robot I/O.""" +from pathlib import Path +import argparse +import numpy as np +from spacedreamer_galexea import SpaceDreamerGalaxea + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--task-dir', type=Path, required=True) + p.add_argument('--assets-dir', type=Path) + p.add_argument('--observation', type=Path, required=True, + help='NPZ containing head_rgb uint8[512,640,3] or letterboxed[224,224,3], state[14], optional depth_m[224,224]') + p.add_argument('--output', type=Path, required=True) + p.add_argument('--steps', type=int, default=10) + p.add_argument('--seed', type=int, default=0) + p.add_argument('--device', default='cuda:0') + a = p.parse_args() + with np.load(a.observation, allow_pickle=False) as obs: + rgb, state = obs['head_rgb'], obs['state'] + depth = obs['depth_m'] if 'depth_m' in obs else None + policy = SpaceDreamerGalaxea(a.task_dir, a.assets_dir, device=a.device) + actions = policy.infer_action(rgb, state, depth_m=depth, + num_inference_steps=a.steps, seed=a.seed) + a.output.parent.mkdir(parents=True, exist_ok=True) + np.save(a.output, actions, allow_pickle=False) + print(f'Saved {actions.shape} {actions.dtype} absolute commands to {a.output}; no robot commands sent.') + +if __name__ == '__main__': + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/runtime_packages.json b/spacedreamer_stack_cups_clean40_30ep/runtime/runtime_packages.json new file mode 100644 index 0000000000000000000000000000000000000000..419717d549baf62299c3b908b8912947827e4384 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/runtime_packages.json @@ -0,0 +1,161 @@ +{ + "python": "3.12.14", + "context": "Local CPU packaging validation runtime; remote training pod package dump is not claimed.", + "packages": { + "absl-py": "2.4.0", + "accelerate": "1.13.0", + "aiohappyeyeballs": "2.6.1", + "aiohttp": "3.14.0", + "aiosignal": "1.4.0", + "albucore": "0.0.17", + "albumentations": "1.4.18", + "annotated-types": "0.7.0", + "antlr4-python3-runtime": "4.9.3", + "asciitree": "0.3.3", + "attrs": "25.4.0", + "av": "15.1.0", + "boto3": "1.35.99", + "botocore": "1.35.99", + "certifi": "2026.7.22", + "cffi": "2.0.0", + "charset-normalizer": "3.5.2", + "click": "8.1.8", + "cloudpickle": "3.1.2", + "contourpy": "1.3.2", + "cryptography": "48.0.0", + "cycler": "0.12.1", + "datasets": "3.6.0", + "deepspeed": "0.17.6", + "diffusers": "0.35.1", + "dill": "0.3.8", + "dm-tree": "0.1.9", + "docstring-parser": "0.17.0", + "easydict": "1.13", + "einops": "0.8.1", + "eval-type-backport": "0.3.1", + "farama-notifications": "0.0.4", + "fasteners": "0.20", + "fastwam": "0.1.0", + "filelock": "4.0.9", + "flash-attn": "2.8.3", + "fonttools": "4.62.1", + "frozenlist": "1.8.0", + "fsspec": "2025.3.0", + "ftfy": "6.3.1", + "gitdb": "4.0.12", + "gitpython": "3.1.50", + "gr00t": "0.1.0", + "gymnasium": "1.2.2", + "h5py": "3.14.0", + "hf-xet": "1.4.2", + "hjson": "3.1.0", + "huggingface-hub": "0.36.2", + "hydra-core": "1.3.2", + "idna": "3.20", + "imageio": "2.37.3", + "imageio-ffmpeg": "0.6.0", + "importlib-metadata": "8.7.1", + "inquirerpy": "0.3.4", + "jinja2": "3.1.6", + "jmespath": "1.1.0", + "jsonlines": "4.0.0", + "kiwisolver": "1.5.0", + "lazy-loader": "0.5", + "lmdb": "1.7.5", + "lz4": "4.4.5", + "markdown-it-py": "4.0.0", + "markupsafe": "3.0.3", + "matplotlib": "3.10.1", + "mdurl": "0.1.2", + "ml-dtypes": "0.5.4", + "modelscope": "1.34.0", + "mpmath": "1.3.0", + "msgpack": "1.1.0", + "msgpack-numpy": "0.4.8", + "multidict": "6.7.1", + "multiprocess": "0.70.16", + "networkx": "3.4.2", + "ninja": "1.13.0", + "numcodecs": "0.15.1", + "numpy": "1.26.4", + "nvidia-cublas-cu12": "12.8.4.1", + "nvidia-cuda-cupti-cu12": "12.8.90", + "nvidia-cuda-nvrtc-cu12": "12.8.93", + "nvidia-cuda-runtime-cu12": "12.8.90", + "nvidia-cudnn-cu12": "9.10.2.21", + "nvidia-cufft-cu12": "11.3.3.83", + "nvidia-cufile-cu12": "1.13.1.3", + "nvidia-curand-cu12": "10.3.9.90", + "nvidia-cusolver-cu12": "11.7.3.90", + "nvidia-cusparse-cu12": "12.5.8.93", + "nvidia-cusparselt-cu12": "0.7.1", + "nvidia-nccl-cu12": "2.27.5", + "nvidia-nvjitlink-cu12": "12.8.93", + "nvidia-nvshmem-cu12": "3.3.20", + "nvidia-nvtx-cu12": "12.8.90", + "omegaconf": "2.3.1", + "onnx": "1.20.1", + "onnx-ir": "0.2.1", + "onnxscript": "0.7.0", + "opencv-python-headless": "4.11.0.86", + "packaging": "26.3", + "pandas": "2.2.3", + "peft": "0.17.1", + "pfzy": "0.3.4", + "pillow": "12.2.0", + "pip": "26.2.1", + "platformdirs": "4.9.4", + "prompt-toolkit": "3.0.52", + "propcache": "0.4.1", + "protobuf": "6.33.5", + "psutil": "7.2.2", + "py-cpuinfo": "9.0.0", + "pyarrow": "23.0.1", + "pycparser": "3.0", + "pydantic": "2.12.5", + "pydantic-core": "2.41.5", + "pygments": "2.20.0", + "pyparsing": "3.3.2", + "python-dateutil": "2.9.0.post0", + "pytz": "2026.1.post1", + "pyyaml": "6.0.3", + "pyzmq": "27.0.1", + "regex": "2026.2.28", + "requests": "2.34.2", + "rich": "14.3.3", + "s3transfer": "0.10.4", + "safetensors": "0.7.0", + "scikit-image": "0.25.2", + "scipy": "1.15.3", + "sentencepiece": "0.2.1", + "sentry-sdk": "2.55.0", + "setuptools": "84.0.0", + "shtab": "1.8.0", + "six": "1.17.0", + "smmap": "5.0.3", + "sympy": "1.14.0", + "termcolor": "3.2.0", + "tifffile": "2025.5.10", + "tokenizers": "0.22.2", + "torch": "2.9.0+cu128", + "torchcodec": "0.8.0", + "torchvision": "0.24.0+cu128", + "tqdm": "4.70.1", + "transformers": "4.57.3", + "triton": "3.5.0", + "typeguard": "4.5.1", + "typing-extensions": "4.15.0", + "typing-inspection": "0.4.2", + "tyro": "0.9.17", + "tzdata": "2025.3", + "urllib3": "2.8.0", + "wandb": "0.23.0", + "wcwidth": "0.6.0", + "wrapt": "2.1.2", + "xxhash": "3.6.0", + "yarl": "1.23.0", + "zarr": "2.18.7", + "zipp": "3.23.0", + "zstandard": "0.25.0" + } +} diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/__init__.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f40dcb36ffb7cb5999600c8d9520cd72e3f60af0 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/__init__.py @@ -0,0 +1 @@ +"""FastWAM package.""" diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/__init__.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/dataset_utils.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/dataset_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e7d5830119dd42c492ec3ba5c01081c26b836d71 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/dataset_utils.py @@ -0,0 +1,109 @@ + +from typing import Optional + +import torch +import torchvision.transforms.functional as transforms_F +from PIL import Image + + +def obtain_image_size(data: torch.Tensor | Image.Image) -> tuple[int, int]: + r"""Return spatial size from a PIL image or image/video tensor. + + Args: + data (torch.Tensor | Image.Image): Input image or video tensor. + Returns: + width (int): Input width. + height (int): Input height. + """ + + if isinstance(data, Image.Image): + width, height = data.size + elif isinstance(data, torch.Tensor): + height, width = data.size()[-2:] + else: + raise ValueError("data to random crop should be PIL Image or tensor") + + return width, height + +class ResizeSmallestSideAspectPreserving: + def __init__(self, args: Optional[dict] = None) -> None: + self.args = args + + def __call__(self, video: torch.Tensor | Image.Image) -> torch.Tensor | Image.Image: + r"""Resize while preserving aspect ratio. + + The output is scaled so both spatial dimensions are at least the + requested target size. + + Args: + video (torch.Tensor | Image.Image): Input image or video tensor. + Returns: + torch.Tensor | Image.Image: Resized image or video tensor. + """ + + assert self.args is not None, "Please specify args in augmentations" + + img_w, img_h = self.args["img_w"], self.args["img_h"] + + orig_w, orig_h = obtain_image_size(video) + scaling_ratio = max((img_w / orig_w), (img_h / orig_h)) + target_size = (int(scaling_ratio * orig_h + 0.5), int(scaling_ratio * orig_w + 0.5)) + + assert ( + target_size[0] >= img_h and target_size[1] >= img_w + ), f"Resize error. orig {(orig_w, orig_h)} desire {(img_w, img_h)} compute {target_size}" + + return transforms_F.resize( + video, + size=target_size, # type: ignore + interpolation=self.args.get("interpolation", transforms_F.InterpolationMode.BICUBIC), + antialias=True, + ) + + +class CenterCrop: + def __init__(self, args: Optional[dict] = None) -> None: + self.args = args + + def __call__(self, video: torch.Tensor | Image.Image) -> torch.Tensor | Image.Image: + r"""Center crop to the requested spatial size. + + Args: + video (torch.Tensor | Image.Image): Input image or video tensor. + Returns: + torch.Tensor | Image.Image: Center cropped image or video tensor. + """ + assert ( + (self.args is not None) and ("img_w" in self.args) and ("img_h" in self.args) + ), "Please specify size in args" + + img_w, img_h = self.args["img_w"], self.args["img_h"] + return transforms_F.center_crop(video, [img_h, img_w]) + + +class Normalize: + def __init__(self, args: Optional[dict] = None) -> None: + self.args = args + + def __call__(self, video: torch.Tensor | Image.Image) -> torch.Tensor: + r"""Convert to tensor if needed and normalize by mean/std. + + Args: + video (torch.Tensor | Image.Image): Input image or video tensor. + Returns: + torch.Tensor: Normalized image or video tensor. + """ + assert self.args is not None, "Please specify args" + + mean = self.args["mean"] + std = self.args["std"] + + if isinstance(video, torch.Tensor): + data = video.to(dtype=torch.float32) + if video.dtype == torch.uint8: + data = data / 255.0 + data = data.to(dtype=torch.get_default_dtype()) + else: + data = transforms_F.to_tensor(video) # division by 255 is applied in to_tensor() + + return transforms_F.normalize(tensor=data, mean=mean, std=std) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/base_lerobot_dataset.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/base_lerobot_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..60451d6263bae5a761b8279a1a789ff2ca8ffacf --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/base_lerobot_dataset.py @@ -0,0 +1,412 @@ +import torch +import numpy as np +from pathlib import Path +from typing import List, Literal, Dict, Optional, Any, DefaultDict +from tqdm import tqdm +from .lerobot.lerobot_dataset import LeRobotDatasetMetadata, MultiLeRobotDataset + +from concurrent.futures import ThreadPoolExecutor, as_completed +import traceback +from fastwam.utils.logging_config import get_logger +from .processors.base_processor import BaseProcessor + +logger = get_logger(__name__) + +MAX_GETITEM_ATTEMPT = 5 + +class BaseLerobotDataset(torch.utils.data.Dataset): + def __init__( + self, + dataset_dirs: List[str], + + # shapes + shape_meta: Dict[str, Any], + action_size: int = 1, + past_action_size: int = 0, # Excludes the current frame + obs_size: int = 1, # should be + past_obs_size: int = 0, + + # train vs val + val_set_proportion: float = 0.05, + is_training_set: bool = False, + seed: int = 42, + + # sampling + global_sample_stride: int = 1, + ): + assert len(dataset_dirs) > 0, "At least one dataset directory is required" + assert past_action_size == 0 + assert past_obs_size == 0 + assert action_size == obs_size - 1, "In this dataset, action_size should be obs_size - 1" + + self.dataset_dirs = dataset_dirs + self.shape_meta = shape_meta + self.action_size = action_size + self.past_action_size = past_action_size + self.obs_size = obs_size + self.processor = None # Will be set externally + metas = [] + for ds_dir in dataset_dirs: + ds_root = Path(ds_dir) + repo_id = ds_dir + meta = LeRobotDatasetMetadata(repo_id=repo_id, root=ds_root) + metas.append(meta) + + fps_list = [m.fps for m in metas] + assert len(set(fps_list)) == 1, f"All dataset_dirs must have the same fps, got {fps_list}" + fps = fps_list[0] + + self.global_sample_stride = global_sample_stride + + self.val_set_proportion = val_set_proportion + self.is_training_set = is_training_set + + self.image_meta = shape_meta["images"] + self.state_meta = shape_meta["state"] + self.action_meta = shape_meta["action"] + + delta_timestamps = {} + for meta in self.image_meta: + key = meta["key"] + meta["lerobot_key"] = f"observation.images.{key}" if key != "default" else "observation.images" + delta_timestamps[meta["lerobot_key"]] = [ + (t * global_sample_stride) / fps for t in range(-past_obs_size, -past_obs_size + obs_size) + ] + + for meta in self.state_meta: + key = meta["key"] + meta["lerobot_key"] = f"observation.state.{key}" if key != "default" else "observation.state" + delta_timestamps[meta["lerobot_key"]] = [ + (t * global_sample_stride) / fps for t in range(-past_obs_size, -past_obs_size + obs_size) + ] + + for meta in self.action_meta: + key = meta["key"] + meta["lerobot_key"] = f"action.{key}" if key != "default" else "action" + delta_timestamps[meta["lerobot_key"]] = [(t * global_sample_stride) / fps for t in range(-past_action_size, -past_action_size + action_size)] + + episodes = {} + if val_set_proportion < 1e-6: + for meta in metas: + episodes.update({meta.repo_id: list(range(meta.total_episodes))}) + else: + for meta in metas: + split_idx = int(meta.total_episodes * (1 - val_set_proportion)) + # random shuffle episode indices before splitting + episode_indices = list(range(meta.total_episodes)) + rng = np.random.default_rng(seed) + rng.shuffle(episode_indices) + if self.is_training_set: + episodes.update({meta.repo_id: [episode_indices[i] for i in range(split_idx)]}) + else: + episodes.update({meta.repo_id: [episode_indices[i] for i in range(split_idx, meta.total_episodes)]}) + + self.multi_dataset = MultiLeRobotDataset( + dataset_dirs=self.dataset_dirs, + episodes=episodes, + delta_timestamps=delta_timestamps, + ) + + # HACK: lerobot 3.0 will fix this + episode_data_index = [] + end_index = 0 + for dataset in self.multi_dataset._datasets: + multi_episode_data_index = { + "from": dataset.episode_data_index["from"] + end_index, + "to": dataset.episode_data_index["to"] + end_index, + } + episode_data_index.append(multi_episode_data_index) + end_index = multi_episode_data_index["to"][-1] + + self.episode_data_index = { + "from": torch.cat([dataset["from"] for dataset in episode_data_index]), + "to": torch.cat([dataset["to"] for dataset in episode_data_index]), + } + + def _get_action(self, meta, lerobot_sample) -> torch.Tensor: + key, lerobot_key, raw_shape = meta["key"], meta["lerobot_key"], meta["raw_shape"] + action: torch.Tensor = lerobot_sample[lerobot_key] # [T, action_dim] + if action.ndim == 1: # for shape of 1, like gripper + action = action.unsqueeze(-1) + assert action.shape[-1] == raw_shape, f"Action '{key}' shape {action.shape[-1]} mismatch with meta {raw_shape}." + return action + + def _get_state(self, meta, lerobot_sample) -> torch.Tensor: + key, lerobot_key, raw_shape = meta["key"], meta["lerobot_key"], meta["raw_shape"] + state: torch.Tensor = lerobot_sample[lerobot_key] + if state.ndim == 1: # for shape of 1, like gripper + state = state.unsqueeze(-1) + # state = state[..., :-1, :] # use state_{t} as observation_t + assert state.shape[-1] == raw_shape, f"State '{key}' shape {state.shape[-1]} mismatch with meta {raw_shape}." + return state + + def _get_image(self, meta, lerobot_sample) -> torch.Tensor: + key, lerobot_key, raw_shape = meta["key"], meta["lerobot_key"], meta["raw_shape"] + image: torch.Tensor = lerobot_sample[lerobot_key] + if image.ndim == 3: # time dim will lost when obs_size is 1 + image = image.unsqueeze(0) + image = (image * 255).to(torch.uint8) # (1, 3, H, W) + # For config simplication + # assert image.shape[1:] == raw_shape, f"Image '{key}' shape {image.shape[1:]} mismatch with {raw_shape}." + return image + + def _split_lerobot_sample(self, lerobot_sample) -> Dict[str, Any]: + return lerobot_sample + + def _get_episode_data(self, episode_idx): + lerobot_sample = self.multi_dataset.get_episode_data(episode_idx) + lerobot_sample = self._split_lerobot_sample(lerobot_sample) + state, action = {}, {} + for meta in self.state_meta: + s = self._get_state(meta, lerobot_sample) + state[meta["key"]] = s.unsqueeze(1).float() + for meta in self.action_meta: + a = self._get_action(meta, lerobot_sample) + a = sliding_window_with_replication(a, self.action_size) + action[meta["key"]] = a.float() + return {"action": action, "state": state} + + def _set_return_images(self, flag: bool): + self.return_images = flag + self.multi_dataset.set_during_training(flag) + + def __len__(self): + return self.multi_dataset.num_frames + + def _get_additional_data(self, sample, lerobot_sample): + return sample + + def __getitem__(self, idx): + if idx >= len(self): + raise IndexError(f"Index {idx} out of bounds {len(self)}.") + + # Retry with random indices until we successfully load a frame. + sample_idx = idx + attempt = 0 + last_exception: Optional[Exception] = None + while attempt < MAX_GETITEM_ATTEMPT: + try: + lerobot_sample = self.multi_dataset[sample_idx] + lerobot_sample = self._split_lerobot_sample(lerobot_sample) + break + except Exception as err: + attempt += 1 + last_exception = err + logger.warning( + f"Error loading sample {sample_idx} (attempt {attempt}). " + "Retrying with a random index. " + f"Error: {err}" + ) + sample_idx = np.random.randint(len(self)) + print(traceback.format_exc()) + else: + raise RuntimeError( + f"Failed to load a valid sample after {MAX_GETITEM_ATTEMPT} attempts " + f"for index {idx}." + ) from last_exception + + # Get data from lerobot, organized in nested dict + sample = { + "idx": sample_idx, + "task": lerobot_sample["task"], + "action": {}, + "state": {}, + "images": {}, + } + for meta in self.state_meta: + sample["state"][meta["key"]] = self._get_state(meta, lerobot_sample) + + for meta in self.action_meta: + sample["action"][meta["key"]] = self._get_action(meta, lerobot_sample) + + for meta in self.image_meta: + sample["images"][meta["key"]] = self._get_image(meta, lerobot_sample) + + sample["action_is_pad"] = lerobot_sample[f"{self.action_meta[0]['lerobot_key']}_is_pad"] + sample["state_is_pad"] = lerobot_sample[f"{self.state_meta[0]['lerobot_key']}_is_pad"] + sample["image_is_pad"] = lerobot_sample[f"{self.image_meta[0]['lerobot_key']}_is_pad"] + + sample = self._get_additional_data(sample, lerobot_sample) + + for key in lerobot_sample: + if key not in sample and "observation" not in key and "action" not in key: + sample[key] = lerobot_sample[key] + + # Preprocess the sample using the processor + # for quick data loading + if self.processor is not None: + sample = self.processor.preprocess(sample) + + return sample + + def set_processor(self, processor: BaseProcessor): + """Set processor instance from external initialization.""" + self.processor = processor + if self.is_training_set: + self.processor.train() + else: + self.processor.eval() + return self + + def get_dataset_stats(self, preprocessor: BaseProcessor): + state_min = DefaultDict(list) + state_max = DefaultDict(list) + state_mean = DefaultDict(list) + state_var = DefaultDict(list) + state_q01 = DefaultDict(list) + state_q99 = DefaultDict(list) + + action_min = DefaultDict(list) + action_max = DefaultDict(list) + action_mean = DefaultDict(list) + action_var = DefaultDict(list) + action_q01 = DefaultDict(list) + action_q99 = DefaultDict(list) + + episodes_num = self.multi_dataset.num_episodes + + def process_episode(episode_idx): + batch = self._get_episode_data(episode_idx) + batch = preprocessor.action_state_transform(batch) + return batch + + multi_thread = True + if not multi_thread: + for episode_idx in tqdm(range(episodes_num), desc="Iterating dataset to get normalization"): + batch = process_episode(episode_idx) + for meta in self.state_meta: + key = meta["key"] + cur_state: torch.Tensor = batch["state"][key] # (B, T, dim) + state_min[key].append(cur_state.amin(0)) + state_max[key].append(cur_state.amax(0)) + state_mean[key].append(cur_state.mean(0)) + state_var[key].append(cur_state.var(0)) + state_q01[key].append(torch.quantile(cur_state, 0.01, dim=0, keepdim=False)) + state_q99[key].append(torch.quantile(cur_state, 0.99, dim=0, keepdim=False)) + for meta in self.action_meta: + key = meta["key"] + cur_action: torch.Tensor = batch["action"][key] # (B, T, dim) + action_min[key].append(cur_action.amin(0)) + action_max[key].append(cur_action.amax(0)) + action_mean[key].append(cur_action.mean(0)) + action_var[key].append(cur_action.var(0)) + action_q01[key].append(torch.quantile(cur_action, 0.01, dim=0, keepdim=False)) + action_q99[key].append(torch.quantile(cur_action, 0.99, dim=0, keepdim=False)) + + else: + with ThreadPoolExecutor() as executor: + futures = [executor.submit(process_episode, num) for num in range(episodes_num)] + + for future in tqdm(as_completed(futures), total=episodes_num, desc="Iterating dataset to get normalization"): + try: + batch = future.result() + for meta in self.state_meta: + key = meta["key"] + cur_state: torch.Tensor = batch["state"][key] # (B, T, dim) + state_min[key].append(cur_state.amin(0)) + state_max[key].append(cur_state.amax(0)) + state_mean[key].append(cur_state.mean(0)) + state_var[key].append(cur_state.var(0)) + state_q01[key].append(torch.quantile(cur_state, 0.01, dim=0, keepdim=False)) + state_q99[key].append(torch.quantile(cur_state, 0.99, dim=0, keepdim=False)) + + for meta in self.action_meta: + key = meta["key"] + cur_action: torch.Tensor = batch["action"][key] # (B, T, dim) + action_min[key].append(cur_action.amin(0)) + action_max[key].append(cur_action.amax(0)) + action_mean[key].append(cur_action.mean(0)) + action_var[key].append(cur_action.var(0)) + action_q01[key].append(torch.quantile(cur_action, 0.01, dim=0, keepdim=False)) + action_q99[key].append(torch.quantile(cur_action, 0.99, dim=0, keepdim=False)) + + except Exception as e: + logger.error(f"Error processing episode: {e}") + print(traceback.format_exc()) + raise e + + # assume that each minibatch has equal number of samples + def get_mean_std(means, vars): + means = torch.stack(means) + vars = torch.stack(vars) + stepwise_mean = means.mean(0) + stepwise_std = (vars + (means - stepwise_mean) ** 2).mean(0).sqrt() + global_mean = means.mean((0, 1)) + global_std = (vars + (means - global_mean) ** 2).mean((0, 1)).sqrt() + return stepwise_mean, stepwise_std, global_mean, global_std + + stats = {"state": DefaultDict(dict), "action": DefaultDict(dict), "num_episodes": episodes_num, "num_transition": self.multi_dataset.num_frames} + for meta in self.state_meta: + key = meta["key"] + stats["state"][key]["stepwise_min"] = torch.stack(state_min[key]).amin(0) + stats["state"][key]["stepwise_max"] = torch.stack(state_max[key]).amax(0) + stats["state"][key]["global_min"] = stats["state"][key]["stepwise_min"].amin(0) + stats["state"][key]["global_max"] = stats["state"][key]["stepwise_max"].amax(0) + stats["state"][key]["stepwise_q01"] = torch.stack(state_q01[key]).amin(0) + stats["state"][key]["stepwise_q99"] = torch.stack(state_q99[key]).amax(0) + stats["state"][key]["global_q01"] = stats["state"][key]["stepwise_q01"].amin(0) + stats["state"][key]["global_q99"] = stats["state"][key]["stepwise_q99"].amax(0) + ( + stats["state"][key]["stepwise_mean"], + stats["state"][key]["stepwise_std"], + stats["state"][key]["global_mean"], + stats["state"][key]["global_std"], + ) = get_mean_std(state_mean[key], state_var[key]) + + for meta in self.action_meta: + key = meta["key"] + stats["action"][key]["stepwise_min"] = torch.stack(action_min[key]).amin(0) + stats["action"][key]["stepwise_max"] = torch.stack(action_max[key]).amax(0) + stats["action"][key]["global_min"] = stats["action"][key]["stepwise_min"].amin(0) + stats["action"][key]["global_max"] = stats["action"][key]["stepwise_max"].amax(0) + stats["action"][key]["stepwise_q01"] = torch.stack(action_q01[key]).amin(0) + stats["action"][key]["stepwise_q99"] = torch.stack(action_q99[key]).amax(0) + stats["action"][key]["global_q01"] = stats["action"][key]["stepwise_q01"].amin(0) + stats["action"][key]["global_q99"] = stats["action"][key]["stepwise_q99"].amax(0) + ( + stats["action"][key]["stepwise_mean"], + stats["action"][key]["stepwise_std"], + stats["action"][key]["global_mean"], + stats["action"][key]["global_std"], + ) = get_mean_std(action_mean[key], action_var[key]) + + return stats + + +def sliding_window_with_replication(x: torch.Tensor, window_size: int) -> torch.Tensor: + """ + Construct a sliding-window tensor from the input tensor x (shape: [N, D]). + The output shape is [N, window_size, D]. + + For each starting index i: + out[i, j, :] = + x[i + j, :] if i + j < N + x[-1, :] otherwise (replicate the last row when out of bounds) + + Args: + x (torch.Tensor): Input tensor of shape [N, D] + window_size (int): Size of the sliding window + + Returns: + torch.Tensor: Tensor of shape [N, window_size, D] + """ + assert x.dim() == 2 + assert window_size > 0 + + N, D = x.shape + + # shape [N, window_size] + # indices[i, j] = i + j + i_indices = torch.arange(N).unsqueeze(1) # [N, 1] + j_indices = torch.arange(window_size).unsqueeze(0) # [1, window_size] + indices = i_indices + j_indices # [N, window_size] + + # N-1 + # torch.clamp [0, N-1] + clamped_indices = torch.clamp(indices, min=0, max=N - 1) + + # clamped_indices [N, window_size],x [N, D] + # out[i, j, :] = x[clamped_indices[i, j], :] + out = x[clamped_indices] # [N, window_size, D] + + return out diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/constants.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..30777239efdfa096bfefbc0f2d729e8f36da90ca --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/constants.py @@ -0,0 +1,54 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# keys +import os +from pathlib import Path + +from huggingface_hub.constants import HF_HOME + +OBS_ENV_STATE = "observation.environment_state" +OBS_STATE = "observation.state" +OBS_IMAGE = "observation.image" +OBS_IMAGES = "observation.images" +ACTION = "action" +REWARD = "next.reward" + +ROBOTS = "robots" +ROBOT_TYPE = "robot_type" +TELEOPERATORS = "teleoperators" + +# files & directories +CHECKPOINTS_DIR = "checkpoints" +LAST_CHECKPOINT_LINK = "last" +PRETRAINED_MODEL_DIR = "pretrained_model" +TRAINING_STATE_DIR = "training_state" +RNG_STATE = "rng_state.safetensors" +TRAINING_STEP = "training_step.json" +OPTIMIZER_STATE = "optimizer_state.safetensors" +OPTIMIZER_PARAM_GROUPS = "optimizer_param_groups.json" +SCHEDULER_STATE = "scheduler_state.json" + +if "LEROBOT_HOME" in os.environ: + raise ValueError( + f"You have a 'LEROBOT_HOME' environment variable set to '{os.getenv('LEROBOT_HOME')}'.\n" + "'LEROBOT_HOME' is deprecated, please use 'HF_LEROBOT_HOME' instead." + ) + +# cache dir +default_cache_path = Path(HF_HOME) / "lerobot" +HF_LEROBOT_HOME = Path(os.getenv("HF_LEROBOT_HOME", default_cache_path)).expanduser() + +# calibration dir +default_calibration_path = HF_LEROBOT_HOME / "calibration" +HF_LEROBOT_CALIBRATION = Path(os.getenv("HF_LEROBOT_CALIBRATION", default_calibration_path)).expanduser() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py new file mode 100644 index 0000000000000000000000000000000000000000..df26818f654de1410ccdab2306a85ca4070b761b --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python + +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +from .utils import load_image_as_numpy + + +def estimate_num_samples( + dataset_len: int, min_num_samples: int = 100, max_num_samples: int = 10_000, power: float = 0.75 +) -> int: + """Heuristic to estimate the number of samples based on dataset size. + The power controls the sample growth relative to dataset size. + Lower the power for less number of samples. + + For default arguments, we have: + - from 1 to ~500, num_samples=100 + - at 1000, num_samples=177 + - at 2000, num_samples=299 + - at 5000, num_samples=594 + - at 10000, num_samples=1000 + - at 20000, num_samples=1681 + """ + if dataset_len < min_num_samples: + min_num_samples = dataset_len + return max(min_num_samples, min(int(dataset_len**power), max_num_samples)) + + +def sample_indices(data_len: int) -> list[int]: + num_samples = estimate_num_samples(data_len) + return np.round(np.linspace(0, data_len - 1, num_samples)).astype(int).tolist() + + +def auto_downsample_height_width(img: np.ndarray, target_size: int = 150, max_size_threshold: int = 300): + _, height, width = img.shape + + if max(width, height) < max_size_threshold: + # no downsampling needed + return img + + downsample_factor = int(width / target_size) if width > height else int(height / target_size) + return img[:, ::downsample_factor, ::downsample_factor] + + +def sample_images(image_paths: list[str]) -> np.ndarray: + sampled_indices = sample_indices(len(image_paths)) + + images = None + for i, idx in enumerate(sampled_indices): + path = image_paths[idx] + # we load as uint8 to reduce memory usage + img = load_image_as_numpy(path, dtype=np.uint8, channel_first=True) + img = auto_downsample_height_width(img) + + if images is None: + images = np.empty((len(sampled_indices), *img.shape), dtype=np.uint8) + + images[i] = img + + return images + + +def get_feature_stats(array: np.ndarray, axis: tuple, keepdims: bool) -> dict[str, np.ndarray]: + return { + "min": np.min(array, axis=axis, keepdims=keepdims), + "max": np.max(array, axis=axis, keepdims=keepdims), + "mean": np.mean(array, axis=axis, keepdims=keepdims), + "std": np.std(array, axis=axis, keepdims=keepdims), + "count": np.array([len(array)]), + } + + +def compute_episode_stats(episode_data: dict[str, list[str] | np.ndarray], features: dict, is_compute_episode_stats_image : bool) -> dict: + ep_stats = {} + for key, data in episode_data.items(): + if features[key]["dtype"] == "string": + continue # HACK: we should receive np.arrays of strings + elif features[key]["dtype"] in ["image", "video"]: + if is_compute_episode_stats_image: + ep_ft_array = sample_images(data) # data is a list of image paths + axes_to_reduce = (0, 2, 3) # keep channel dim + keepdims = True + else: + continue + else: + ep_ft_array = data # data is already a np.ndarray + axes_to_reduce = 0 # compute stats over the first axis + keepdims = data.ndim == 1 # keep as np.array + + ep_stats[key] = get_feature_stats(ep_ft_array, axis=axes_to_reduce, keepdims=keepdims) + + # finally, we normalize and remove batch dim for images + if features[key]["dtype"] in ["image", "video"]: + ep_stats[key] = { + k: v if k == "count" else np.squeeze(v / 255.0, axis=0) for k, v in ep_stats[key].items() + } + + return ep_stats + + +def _assert_type_and_shape(stats_list: list[dict[str, dict]]): + for i in range(len(stats_list)): + for fkey in stats_list[i]: + for k, v in stats_list[i][fkey].items(): + if not isinstance(v, np.ndarray): + raise ValueError( + f"Stats must be composed of numpy array, but key '{k}' of feature '{fkey}' is of type '{type(v)}' instead." + ) + if v.ndim == 0: + raise ValueError("Number of dimensions must be at least 1, and is 0 instead.") + if k == "count" and v.shape != (1,): + raise ValueError(f"Shape of 'count' must be (1), but is {v.shape} instead.") + if "image" in fkey and k != "count" and v.shape != (3, 1, 1): + raise ValueError(f"Shape of '{k}' must be (3,1,1), but is {v.shape} instead.") + + +def aggregate_feature_stats(stats_ft_list: list[dict[str, dict]]) -> dict[str, dict[str, np.ndarray]]: + """Aggregates stats for a single feature.""" + means = np.stack([s["mean"] for s in stats_ft_list]) + variances = np.stack([s["std"] ** 2 for s in stats_ft_list]) + counts = np.stack([s["count"] for s in stats_ft_list]) + total_count = counts.sum(axis=0) + + # Prepare weighted mean by matching number of dimensions + while counts.ndim < means.ndim: + counts = np.expand_dims(counts, axis=-1) + + # Compute the weighted mean + weighted_means = means * counts + total_mean = weighted_means.sum(axis=0) / total_count + + # Compute the variance using the parallel algorithm + delta_means = means - total_mean + weighted_variances = (variances + delta_means**2) * counts + total_variance = weighted_variances.sum(axis=0) / total_count + + return { + "min": np.min(np.stack([s["min"] for s in stats_ft_list]), axis=0), + "max": np.max(np.stack([s["max"] for s in stats_ft_list]), axis=0), + "mean": total_mean, + "std": np.sqrt(total_variance), + "count": total_count, + } + + +def aggregate_stats(stats_list: list[dict[str, dict]]) -> dict[str, dict[str, np.ndarray]]: + """Aggregate stats from multiple compute_stats outputs into a single set of stats. + + The final stats will have the union of all data keys from each of the stats dicts. + + For instance: + - new_min = min(min_dataset_0, min_dataset_1, ...) + - new_max = max(max_dataset_0, max_dataset_1, ...) + - new_mean = (mean of all data, weighted by counts) + - new_std = (std of all data) + """ + + _assert_type_and_shape(stats_list) + + data_keys = {key for stats in stats_list for key in stats} + aggregated_stats = {key: {} for key in data_keys} + + for key in data_keys: + stats_with_key = [stats[key] for stats in stats_list if key in stats] + aggregated_stats[key] = aggregate_feature_stats(stats_with_key) + + return aggregated_stats diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6644945af39d90525013ef6f998bec1a35d5a7a6 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py @@ -0,0 +1,903 @@ +#!/usr/bin/env python + +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import contextlib +import importlib.resources +import io +import json +import logging +from collections.abc import Iterator +from itertools import accumulate +from pathlib import Path +from pprint import pformat +from types import SimpleNamespace +from typing import Any +import array + +import datasets +import jsonlines +import numpy as np +import packaging.version +import torch +from datasets.table import embed_table_storage +from huggingface_hub import DatasetCard, DatasetCardData, HfApi +from huggingface_hub.errors import RevisionNotFoundError +from PIL import Image as PILImage +from torchvision import transforms + +from functools import partial + +# from lerobot.configs.types import DictLike, FeatureType, PolicyFeature +# from lerobot.datasets.backward_compatibility import ( +# V21_MESSAGE, +# BackwardCompatibilityError, +# ForwardCompatibilityError, +# ) +# from lerobot.robots import Robot +# from lerobot.utils.utils import is_valid_numpy_dtype_string + +DEFAULT_CHUNK_SIZE = 1000 # Max number of episodes per chunk + +INFO_PATH = "meta/info.json" +EPISODES_PATH = "meta/episodes.jsonl" +STATS_PATH = "meta/stats.json" +EPISODES_STATS_PATH = "meta/episodes_stats.jsonl" +TASKS_PATH = "meta/tasks.jsonl" + +ANNOTATION_PATHS = { + "subtask": "annotations/subtask_annotations.jsonl", + "scene": "annotations/scene_annotations.jsonl", + "gripper_mode": "annotations/gripper_mode_annotation.jsonl", + "gripper_activity": "annotations/gripper_activity_annotation.jsonl", + "eef_velocity": "annotations/eef_velocity_annotation.jsonl", + "eef_acc_mag": "annotations/eef_acc_mag_annotation.jsonl", + "eef_direction": "annotations/eef_direction_annotation.jsonl", +} + +DEFAULT_VIDEO_PATH = "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4" +DEFAULT_PARQUET_PATH = "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet" +DEFAULT_IMAGE_PATH = "images/{image_key}/episode_{episode_index:06d}/frame_{frame_index:06d}.jpeg" + +DATASET_CARD_TEMPLATE = """ +--- +# Metadata will go there +--- +This dataset was created using [LeRobot](https://github.com/huggingface/lerobot). + +## {} + +""" + +DEFAULT_FEATURES = { + "timestamp": {"dtype": "float32", "shape": (1,), "names": None}, + "frame_index": {"dtype": "int64", "shape": (1,), "names": None}, + "episode_index": {"dtype": "int64", "shape": (1,), "names": None}, + "index": {"dtype": "int64", "shape": (1,), "names": None}, + "coarse_task_index": {"dtype": "int64", "shape": (1,), "names": None}, + "task_index": {"dtype": "int64", "shape": (1,), "names": None}, + "coarse_quality_index": {"dtype": "int64", "shape": (1,), "names": None}, + "quality_index": {"dtype": "int64", "shape": (1,), "names": None}, +} + + +def flatten_dict(d: dict, parent_key: str = "", sep: str = "/") -> dict: + """Flatten a nested dictionary structure by collapsing nested keys into one key with a separator. + + For example: + ``` + >>> dct = {"a": {"b": 1, "c": {"d": 2}}, "e": 3}` + >>> print(flatten_dict(dct)) + {"a/b": 1, "a/c/d": 2, "e": 3} + """ + items = [] + for k, v in d.items(): + new_key = f"{parent_key}{sep}{k}" if parent_key else k + if isinstance(v, dict): + items.extend(flatten_dict(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + +def unflatten_dict(d: dict, sep: str = "/") -> dict: + outdict = {} + for key, value in d.items(): + parts = key.split(sep) + d = outdict + for part in parts[:-1]: + if part not in d: + d[part] = {} + d = d[part] + d[parts[-1]] = value + return outdict + + +# def get_nested_item(obj: DictLike, flattened_key: str, sep: str = "/") -> Any: +# split_keys = flattened_key.split(sep) +# getter = obj[split_keys[0]] +# if len(split_keys) == 1: +# return getter + +# for key in split_keys[1:]: +# getter = getter[key] + +# return getter + + +def serialize_dict(stats: dict[str, torch.Tensor | np.ndarray | dict]) -> dict: + serialized_dict = {} + for key, value in flatten_dict(stats).items(): + if isinstance(value, (torch.Tensor, np.ndarray)): + serialized_dict[key] = value.tolist() + elif isinstance(value, np.generic): + serialized_dict[key] = value.item() + elif isinstance(value, (int, float)): + serialized_dict[key] = value + else: + raise NotImplementedError(f"The value '{value}' of type '{type(value)}' is not supported.") + return unflatten_dict(serialized_dict) + + +def embed_images(dataset: datasets.Dataset) -> datasets.Dataset: + # Embed image bytes into the table before saving to parquet + format = dataset.format + dataset = dataset.with_format("arrow") + dataset = dataset.map(embed_table_storage, batched=False) + dataset = dataset.with_format(**format) + return dataset + + +def load_json(fpath: Path) -> Any: + with open(fpath) as f: + return json.load(f) + + +def write_json(data: dict, fpath: Path) -> None: + fpath.parent.mkdir(exist_ok=True, parents=True) + with open(fpath, "w") as f: + json.dump(data, f, indent=4, ensure_ascii=False) + + +def load_jsonlines(fpath: Path) -> list[Any]: + # allow \t and \n + loose_loader = partial(json.loads, strict=False) + with jsonlines.open(fpath, "r", loads=loose_loader) as reader: + return list(reader) + + +def write_jsonlines(data: dict, fpath: Path) -> None: + fpath.parent.mkdir(exist_ok=True, parents=True) + with jsonlines.open(fpath, "w") as writer: + writer.write_all(data) + + +def append_jsonlines(data: dict, fpath: Path) -> None: + fpath.parent.mkdir(exist_ok=True, parents=True) + with jsonlines.open(fpath, "a") as writer: + writer.write(data) + + +def write_info(info: dict, local_dir: Path): + write_json(info, local_dir / INFO_PATH) + + +def load_info(local_dir: Path) -> dict: + info = load_json(local_dir / INFO_PATH) + for ft in info["features"].values(): + ft["shape"] = tuple(ft["shape"]) + return info + + +def write_stats(stats: dict, local_dir: Path): + serialized_stats = serialize_dict(stats) + write_json(serialized_stats, local_dir / STATS_PATH) + + +def cast_stats_to_numpy(stats) -> dict[str, dict[str, np.ndarray]]: + stats = {key: np.array(value) for key, value in flatten_dict(stats).items()} + return unflatten_dict(stats) + + +def load_stats(local_dir: Path) -> dict[str, dict[str, np.ndarray]]: + if not (local_dir / STATS_PATH).exists(): + return None + stats = load_json(local_dir / STATS_PATH) + return cast_stats_to_numpy(stats) + + +def write_task(task_index: int, task: dict, local_dir: Path): + task_dict = { + "task_index": task_index, + "task": task, + } + append_jsonlines(task_dict, local_dir / TASKS_PATH) + + +def load_tasks(local_dir: Path) -> tuple[dict, dict]: + tasks = load_jsonlines(local_dir / TASKS_PATH) + tasks = {item["task_index"]: item["task"] for item in sorted(tasks, key=lambda x: x["task_index"])} + task_to_task_index = {task: task_index for task_index, task in tasks.items()} + return tasks, task_to_task_index + +def load_annotations(local_dir: Path) -> dict[str, dict[int, str]]: + annotations = {} + for key, path in ANNOTATION_PATHS.items(): + anno = load_jsonlines(local_dir / path) + anno = {item[f"{key}_index"]: item[key] for item in sorted(anno, key=lambda x: x[f"{key}_index"])} + annotations[key] = anno + return annotations + +def write_episode(episode: dict, local_dir: Path): + append_jsonlines(episode, local_dir / EPISODES_PATH) + + +def load_episodes(local_dir: Path) -> dict: + episodes = load_jsonlines(local_dir / EPISODES_PATH) + return {item["episode_index"]: item for item in sorted(episodes, key=lambda x: x["episode_index"])} + + +def write_episode_stats(episode_index: int, episode_stats: dict, local_dir: Path): + # We wrap episode_stats in a dictionary since `episode_stats["episode_index"]` + # is a dictionary of stats and not an integer. + episode_stats = {"episode_index": episode_index, "stats": serialize_dict(episode_stats)} + append_jsonlines(episode_stats, local_dir / EPISODES_STATS_PATH) + + +def load_episodes_stats(local_dir: Path) -> dict: + episodes_stats = load_jsonlines(local_dir / EPISODES_STATS_PATH) + return { + item["episode_index"]: cast_stats_to_numpy(item["stats"]) + for item in sorted(episodes_stats, key=lambda x: x["episode_index"]) + } + + +def backward_compatible_episodes_stats( + stats: dict[str, dict[str, np.ndarray]], episodes: list[int] +) -> dict[str, dict[str, np.ndarray]]: + return dict.fromkeys(episodes, stats) + + +def load_image_as_numpy( + fpath: str | Path, dtype: np.dtype = np.float32, channel_first: bool = True +) -> np.ndarray: + img = PILImage.open(fpath).convert("RGB") + img_array = np.array(img, dtype=dtype) + if channel_first: # (H, W, C) -> (C, H, W) + img_array = np.transpose(img_array, (2, 0, 1)) + if np.issubdtype(dtype, np.floating): + img_array /= 255.0 + return img_array + + +def hf_transform_to_torch(items_dict: dict[torch.Tensor | None]): + """Get a transform function that convert items from Hugging Face dataset (pyarrow) + to torch tensors. Importantly, images are converted from PIL, which corresponds to + a channel last representation (h w c) of uint8 type, to a torch image representation + with channel first (c h w) of float32 type in range [0,1]. + """ + for key in items_dict: + first_item = items_dict[key][0] + if type(first_item) == dict and 'bytes' in first_item and first_item['bytes'] is not None: + to_pil = lambda img: PILImage.open(io.BytesIO(img['bytes'])) + items_dict[key] = [to_pil(img) for img in items_dict[key]] + first_item = items_dict[key][0] + if isinstance(first_item, PILImage.Image): + to_tensor = transforms.ToTensor() + items_dict[key] = [to_tensor(img) for img in items_dict[key]] + elif first_item is None: + pass + else: + items_dict[key] = [x if isinstance(x, str) else torch.tensor(x) for x in items_dict[key]] + return items_dict + + +def is_valid_version(version: str) -> bool: + try: + packaging.version.parse(version) + return True + except packaging.version.InvalidVersion: + return False + + +# def check_version_compatibility( +# repo_id: str, +# version_to_check: str | packaging.version.Version, +# current_version: str | packaging.version.Version, +# enforce_breaking_major: bool = True, +# ) -> None: +# v_check = ( +# packaging.version.parse(version_to_check) +# if not isinstance(version_to_check, packaging.version.Version) +# else version_to_check +# ) +# v_current = ( +# packaging.version.parse(current_version) +# if not isinstance(current_version, packaging.version.Version) +# else current_version +# ) +# if v_check.major < v_current.major and enforce_breaking_major: +# raise BackwardCompatibilityError(repo_id, v_check) +# elif v_check.minor < v_current.minor: +# logging.warning(V21_MESSAGE.format(repo_id=repo_id, version=v_check)) + + +def get_repo_versions(repo_id: str) -> list[packaging.version.Version]: + """Returns available valid versions (branches and tags) on given repo.""" + api = HfApi() + repo_refs = api.list_repo_refs(repo_id, repo_type="dataset") + repo_refs = [b.name for b in repo_refs.branches + repo_refs.tags] + repo_versions = [] + for ref in repo_refs: + with contextlib.suppress(packaging.version.InvalidVersion): + repo_versions.append(packaging.version.parse(ref)) + + return repo_versions + + +# def get_safe_version(repo_id: str, version: str | packaging.version.Version) -> str: +# """ +# Returns the version if available on repo or the latest compatible one. +# Otherwise, will throw a `CompatibilityError`. +# """ +# target_version = ( +# packaging.version.parse(version) if not isinstance(version, packaging.version.Version) else version +# ) +# hub_versions = get_repo_versions(repo_id) + +# if not hub_versions: +# raise RevisionNotFoundError( +# f"""Your dataset must be tagged with a codebase version. +# Assuming _version_ is the codebase_version value in the info.json, you can run this: +# ```python +# from huggingface_hub import HfApi + +# hub_api = HfApi() +# hub_api.create_tag("{repo_id}", tag="_version_", repo_type="dataset") +# ``` +# """ +# ) + +# if target_version in hub_versions: +# return f"v{target_version}" + +# compatibles = [ +# v for v in hub_versions if v.major == target_version.major and v.minor <= target_version.minor +# ] +# if compatibles: +# return_version = max(compatibles) +# if return_version < target_version: +# logging.warning(f"Revision {version} for {repo_id} not found, using version v{return_version}") +# return f"v{return_version}" + +# lower_major = [v for v in hub_versions if v.major < target_version.major] +# if lower_major: +# raise BackwardCompatibilityError(repo_id, max(lower_major)) + +# upper_versions = [v for v in hub_versions if v > target_version] +# assert len(upper_versions) > 0 +# raise ForwardCompatibilityError(repo_id, min(upper_versions)) + + +def get_hf_features_from_features(features: dict) -> datasets.Features: + hf_features = {} + for key, ft in features.items(): + if ft["dtype"] == "video": + continue + elif ft["dtype"] == "image": + hf_features[key] = datasets.Image() + elif ft["shape"] == (1,): + hf_features[key] = datasets.Value(dtype=ft["dtype"]) + elif len(ft["shape"]) == 1: + hf_features[key] = datasets.Sequence( + length=ft["shape"][0], feature=datasets.Value(dtype=ft["dtype"]) + ) + elif len(ft["shape"]) == 2: + hf_features[key] = datasets.Array2D(shape=ft["shape"], dtype=ft["dtype"]) + elif len(ft["shape"]) == 3: + hf_features[key] = datasets.Array3D(shape=ft["shape"], dtype=ft["dtype"]) + elif len(ft["shape"]) == 4: + hf_features[key] = datasets.Array4D(shape=ft["shape"], dtype=ft["dtype"]) + elif len(ft["shape"]) == 5: + hf_features[key] = datasets.Array5D(shape=ft["shape"], dtype=ft["dtype"]) + else: + raise ValueError(f"Corresponding feature is not valid: {ft}") + + return datasets.Features(hf_features) + + +def _validate_feature_names(features: dict[str, dict]) -> None: + invalid_features = {name: ft for name, ft in features.items() if "/" in name} + if invalid_features: + raise ValueError(f"Feature names should not contain '/'. Found '/' in '{invalid_features}'.") + + +def hw_to_dataset_features( + hw_features: dict[str, type | tuple], prefix: str, use_video: bool = True +) -> dict[str, dict]: + features = {} + joint_fts = {key: ftype for key, ftype in hw_features.items() if ftype is float} + cam_fts = {key: shape for key, shape in hw_features.items() if isinstance(shape, tuple)} + + if joint_fts and prefix == "action": + features[prefix] = { + "dtype": "float32", + "shape": (len(joint_fts),), + "names": list(joint_fts), + } + + if joint_fts and prefix == "observation": + features[f"{prefix}.state"] = { + "dtype": "float32", + "shape": (len(joint_fts),), + "names": list(joint_fts), + } + + for key, shape in cam_fts.items(): + features[f"{prefix}.images.{key}"] = { + "dtype": "video" if use_video else "image", + "shape": shape, + "names": ["height", "width", "channels"], + } + + _validate_feature_names(features) + return features + + +def build_dataset_frame( + ds_features: dict[str, dict], values: dict[str, Any], prefix: str +) -> dict[str, np.ndarray]: + frame = {} + for key, ft in ds_features.items(): + if key in DEFAULT_FEATURES or not key.startswith(prefix): + continue + elif ft["dtype"] == "float32" and len(ft["shape"]) == 1: + frame[key] = np.array([values[name] for name in ft["names"]], dtype=np.float32) + elif ft["dtype"] in ["image", "video"]: + frame[key] = values[key.removeprefix(f"{prefix}.images.")] + + return frame + + +# def get_features_from_robot(robot: Robot, use_videos: bool = True) -> dict: +# camera_ft = {} +# if robot.cameras: +# camera_ft = { +# key: {"dtype": "video" if use_videos else "image", **ft} +# for key, ft in robot.camera_features.items() +# } +# return {**robot.motor_features, **camera_ft, **DEFAULT_FEATURES} + + +# def dataset_to_policy_features(features: dict[str, dict]) -> dict[str, PolicyFeature]: +# # TODO(aliberts): Implement "type" in dataset features and simplify this +# policy_features = {} +# for key, ft in features.items(): +# shape = ft["shape"] +# if ft["dtype"] in ["image", "video"]: +# type = FeatureType.VISUAL +# if len(shape) != 3: +# raise ValueError(f"Number of dimensions of {key} != 3 (shape={shape})") + +# names = ft["names"] +# # Backward compatibility for "channel" which is an error introduced in LeRobotDataset v2.0 for ported datasets. +# if names[2] in ["channel", "channels"]: # (h, w, c) -> (c, h, w) +# shape = (shape[2], shape[0], shape[1]) +# elif key == "observation.environment_state": +# type = FeatureType.ENV +# elif key.startswith("observation"): +# type = FeatureType.STATE +# elif key.startswith("action"): +# type = FeatureType.ACTION +# else: +# continue + +# policy_features[key] = PolicyFeature( +# type=type, +# shape=shape, +# ) + +# return policy_features + + +def create_empty_dataset_info( + codebase_version: str, + fps: int, + features: dict, + use_videos: bool, + robot_type: str | None = None, +) -> dict: + return { + "codebase_version": codebase_version, + "robot_type": robot_type, + "total_episodes": 0, + "total_frames": 0, + "total_tasks": 0, + "total_videos": 0, + "total_chunks": 0, + "chunks_size": DEFAULT_CHUNK_SIZE, + "fps": fps, + "splits": {}, + "data_path": DEFAULT_PARQUET_PATH, + "video_path": DEFAULT_VIDEO_PATH if use_videos else None, + "features": features, + } + + +def get_episode_data_index( + episode_dicts: dict[dict], episodes: list[int] | None = None +) -> dict[str, torch.Tensor]: + episode_lengths = {ep_idx: ep_dict["length"] for ep_idx, ep_dict in episode_dicts.items()} + if episodes is not None: + episode_lengths = {ep_idx: episode_lengths[ep_idx] for ep_idx in episodes} + + cumulative_lengths = list(accumulate(episode_lengths.values())) + return { + "from": torch.LongTensor([0] + cumulative_lengths[:-1]), + "to": torch.LongTensor(cumulative_lengths), + } + + +def check_timestamps_sync( + timestamps: np.ndarray, + episode_indices: np.ndarray, + episode_data_index: dict[str, np.ndarray], + fps: int, + tolerance_s: float, + raise_value_error: bool = True, +) -> bool: + """ + This check is to make sure that each timestamp is separated from the next by (1/fps) +/- tolerance + to account for possible numerical error. + + Args: + timestamps (np.ndarray): Array of timestamps in seconds. + episode_indices (np.ndarray): Array indicating the episode index for each timestamp. + episode_data_index (dict[str, np.ndarray]): A dictionary that includes 'to', + which identifies indices for the end of each episode. + fps (int): Frames per second. Used to check the expected difference between consecutive timestamps. + tolerance_s (float): Allowed deviation from the expected (1/fps) difference. + raise_value_error (bool): Whether to raise a ValueError if the check fails. + + Returns: + bool: True if all checked timestamp differences lie within tolerance, False otherwise. + + Raises: + ValueError: If the check fails and `raise_value_error` is True. + """ + if timestamps.shape != episode_indices.shape: + raise ValueError( + "timestamps and episode_indices should have the same shape. " + f"Found {timestamps.shape=} and {episode_indices.shape=}." + ) + + # Consecutive differences + diffs = np.diff(timestamps) + within_tolerance = np.abs(diffs - (1.0 / fps)) <= tolerance_s + + # Mask to ignore differences at the boundaries between episodes + mask = np.ones(len(diffs), dtype=bool) + ignored_diffs = episode_data_index["to"][:-1] - 1 # indices at the end of each episode + mask[ignored_diffs] = False + filtered_within_tolerance = within_tolerance[mask] + + # Check if all remaining diffs are within tolerance + if not np.all(filtered_within_tolerance): + # Track original indices before masking + original_indices = np.arange(len(diffs)) + filtered_indices = original_indices[mask] + outside_tolerance_filtered_indices = np.nonzero(~filtered_within_tolerance)[0] + outside_tolerance_indices = filtered_indices[outside_tolerance_filtered_indices] + + outside_tolerances = [] + for idx in outside_tolerance_indices: + entry = { + "timestamps": [timestamps[idx], timestamps[idx + 1]], + "diff": diffs[idx], + "episode_index": episode_indices[idx].item() + if hasattr(episode_indices[idx], "item") + else episode_indices[idx], + } + outside_tolerances.append(entry) + + if raise_value_error: + raise ValueError( + f"""One or several timestamps unexpectedly violate the tolerance inside episode range. + This might be due to synchronization issues during data collection. + \n{pformat(outside_tolerances)}""" + ) + return False + + return True + + +def check_delta_timestamps( + delta_timestamps: dict[str, list[float]], fps: int, tolerance_s: float, raise_value_error: bool = True +) -> bool: + """This will check if all the values in delta_timestamps are multiples of 1/fps +/- tolerance. + This is to ensure that these delta_timestamps added to any timestamp from a dataset will themselves be + actual timestamps from the dataset. + """ + outside_tolerance = {} + for key, delta_ts in delta_timestamps.items(): + within_tolerance = [abs(ts * fps - round(ts * fps)) / fps <= tolerance_s for ts in delta_ts] + if not all(within_tolerance): + outside_tolerance[key] = [ + ts for ts, is_within in zip(delta_ts, within_tolerance, strict=True) if not is_within + ] + + if len(outside_tolerance) > 0: + if raise_value_error: + raise ValueError( + f""" + The following delta_timestamps are found outside of tolerance range. + Please make sure they are multiples of 1/{fps} +/- tolerance and adjust + their values accordingly. + \n{pformat(outside_tolerance)} + """ + ) + return False + + return True + + +def get_delta_indices(delta_timestamps: dict[str, list[float]], fps: int) -> dict[str, list[int]]: + delta_indices = {} + for key, delta_ts in delta_timestamps.items(): + delta_indices[key] = [round(d * fps) for d in delta_ts] + + return delta_indices + + +def cycle(iterable): + """The equivalent of itertools.cycle, but safe for Pytorch dataloaders. + + See https://github.com/pytorch/pytorch/issues/23900 for information on why itertools.cycle is not safe. + """ + iterator = iter(iterable) + while True: + try: + yield next(iterator) + except StopIteration: + iterator = iter(iterable) + + +def create_branch(repo_id, *, branch: str, repo_type: str | None = None) -> None: + """Create a branch on a existing Hugging Face repo. Delete the branch if it already + exists before creating it. + """ + api = HfApi() + + branches = api.list_repo_refs(repo_id, repo_type=repo_type).branches + refs = [branch.ref for branch in branches] + ref = f"refs/heads/{branch}" + if ref in refs: + api.delete_branch(repo_id, repo_type=repo_type, branch=branch) + + api.create_branch(repo_id, repo_type=repo_type, branch=branch) + + +def create_lerobot_dataset_card( + tags: list | None = None, + dataset_info: dict | None = None, + **kwargs, +) -> DatasetCard: + """ + Keyword arguments will be used to replace values in src/lerobot/datasets/card_template.md. + Note: If specified, license must be one of https://huggingface.co/docs/hub/repositories-licenses. + """ + card_tags = ["LeRobot"] + + if tags: + card_tags += tags + if dataset_info: + dataset_structure = "[meta/info.json](meta/info.json):\n" + dataset_structure += f"```json\n{json.dumps(dataset_info, indent=4)}\n```\n" + kwargs = {**kwargs, "dataset_structure": dataset_structure} + card_data = DatasetCardData( + license=kwargs.get("license"), + tags=card_tags, + task_categories=["robotics"], + configs=[ + { + "config_name": "default", + "data_files": "data/*/*.parquet", + } + ], + ) + + card_template = (importlib.resources.files("lerobot.datasets") / "card_template.md").read_text() + + return DatasetCard.from_template( + card_data=card_data, + template_str=card_template, + **kwargs, + ) + + +class IterableNamespace(SimpleNamespace): + """ + A namespace object that supports both dictionary-like iteration and dot notation access. + Automatically converts nested dictionaries into IterableNamespaces. + + This class extends SimpleNamespace to provide: + - Dictionary-style iteration over keys + - Access to items via both dot notation (obj.key) and brackets (obj["key"]) + - Dictionary-like methods: items(), keys(), values() + - Recursive conversion of nested dictionaries + + Args: + dictionary: Optional dictionary to initialize the namespace + **kwargs: Additional keyword arguments passed to SimpleNamespace + + Examples: + >>> data = {"name": "Alice", "details": {"age": 25}} + >>> ns = IterableNamespace(data) + >>> ns.name + 'Alice' + >>> ns.details.age + 25 + >>> list(ns.keys()) + ['name', 'details'] + >>> for key, value in ns.items(): + ... print(f"{key}: {value}") + name: Alice + details: IterableNamespace(age=25) + """ + + def __init__(self, dictionary: dict[str, Any] = None, **kwargs): + super().__init__(**kwargs) + if dictionary is not None: + for key, value in dictionary.items(): + if isinstance(value, dict): + setattr(self, key, IterableNamespace(value)) + else: + setattr(self, key, value) + + def __iter__(self) -> Iterator[str]: + return iter(vars(self)) + + def __getitem__(self, key: str) -> Any: + return vars(self)[key] + + def items(self): + return vars(self).items() + + def values(self): + return vars(self).values() + + def keys(self): + return vars(self).keys() + + +def validate_frame(frame: dict, features: dict): + expected_features = set(features) - set(DEFAULT_FEATURES) + actual_features = set(frame) + + error_message = validate_features_presence(actual_features, expected_features) + + common_features = actual_features & expected_features + for name in common_features - {"task"}: + error_message += validate_feature_dtype_and_shape(name, features[name], frame[name]) + + if error_message: + raise ValueError(error_message) + + +def validate_features_presence(actual_features: set[str], expected_features: set[str]): + error_message = "" + missing_features = expected_features - actual_features + extra_features = actual_features - expected_features + + if missing_features or extra_features: + error_message += "Feature mismatch in `frame` dictionary:\n" + if missing_features: + error_message += f"Missing features: {missing_features}\n" + if extra_features: + error_message += f"Extra features: {extra_features}\n" + + return error_message + +def is_valid_numpy_dtype_string(dtype_str: str) -> bool: + """ + Return True if a given string can be converted to a numpy dtype. + """ + try: + # Attempt to convert the string to a numpy dtype + np.dtype(dtype_str) + return True + except TypeError: + # If a TypeError is raised, the string is not a valid dtype + return False +def validate_feature_dtype_and_shape(name: str, feature: dict, value: np.ndarray | PILImage.Image | str | bytes): + if isinstance(value, bytes) or isinstance(value, array.array): # ROS 1 and 2 + #TODO fix bytes + return "" + expected_dtype = feature["dtype"] + expected_shape = feature["shape"] + if is_valid_numpy_dtype_string(expected_dtype): + return validate_feature_numpy_array(name, expected_dtype, expected_shape, value) + elif expected_dtype in ["image", "video"]: + return validate_feature_image_or_video(name, expected_shape, value) + elif expected_dtype == "string": + return validate_feature_string(name, value) + else: + raise NotImplementedError(f"The feature dtype '{expected_dtype}' is not implemented yet.") + + +def validate_feature_numpy_array( + name: str, expected_dtype: str, expected_shape: list[int], value: np.ndarray +): + error_message = "" + if isinstance(value, np.ndarray): + actual_dtype = value.dtype + actual_shape = value.shape + + if actual_dtype != np.dtype(expected_dtype): + error_message += f"The feature '{name}' of dtype '{actual_dtype}' is not of the expected dtype '{expected_dtype}'.\n" + + if actual_shape != expected_shape: + error_message += f"The feature '{name}' of shape '{actual_shape}' does not have the expected shape '{expected_shape}'.\n" + else: + error_message += f"The feature '{name}' is not a 'np.ndarray'. Expected type is '{expected_dtype}', but type '{type(value)}' provided instead.\n" + + return error_message + + +def validate_feature_image_or_video(name: str, expected_shape: list[str], value: np.ndarray | PILImage.Image): + # Note: The check of pixels range ([0,1] for float and [0,255] for uint8) is done by the image writer threads. + error_message = "" + if isinstance(value, np.ndarray): + actual_shape = value.shape + c, h, w = expected_shape + if len(actual_shape) != 3 or (actual_shape != (c, h, w) and actual_shape != (h, w, c)): + error_message += f"The feature '{name}' of shape '{actual_shape}' does not have the expected shape '{(c, h, w)}' or '{(h, w, c)}'.\n" + elif isinstance(value, PILImage.Image): + pass + else: + error_message += f"The feature '{name}' is expected to be of type 'PIL.Image' or 'np.ndarray' channel first or channel last, but type '{type(value)}' provided instead.\n" + + return error_message + + +def validate_feature_string(name: str, value: str): + if not isinstance(value, str): + return f"The feature '{name}' is expected to be of type 'str', but type '{type(value)}' provided instead.\n" + return "" + + +def validate_episode_buffer(episode_buffer: dict, total_episodes: int, features: dict): + if "size" not in episode_buffer: + raise ValueError("size key not found in episode_buffer") + + if "task" not in episode_buffer: + raise ValueError("task key not found in episode_buffer") + + if episode_buffer["episode_index"] != total_episodes: + # TODO(aliberts): Add option to use existing episode_index + raise NotImplementedError( + "You might have manually provided the episode_buffer with an episode_index that doesn't " + "match the total number of episodes already in the dataset. This is not supported for now." + ) + + if episode_buffer["size"] == 0: + raise ValueError("You must add one or several frames with `add_frame` before calling `add_episode`.") + + buffer_keys = set(episode_buffer.keys()) - {"task", "size"} + if not buffer_keys == set(features): + raise ValueError( + f"Features from `episode_buffer` don't match the ones in `features`." + f"In episode_buffer not in features: {buffer_keys - set(features)}" + f"In features not in episode_buffer: {set(features) - buffer_keys}" + ) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9dd6fee91c9d867d5a0f4540e5ae0713b7f753dc --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py @@ -0,0 +1,527 @@ +#!/usr/bin/env python + +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import glob +import importlib +import logging +import warnings +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, ClassVar + +import av +import pyarrow as pa +import torch +import torchvision +from datasets.features.features import register_feature +from PIL import Image + + +def get_safe_default_codec(): + if importlib.util.find_spec("torchcodec"): + return "torchcodec" + else: + logging.warning( + "'torchcodec' is not available in your platform, falling back to 'pyav' as a default decoder" + ) + return "pyav" + + +def decode_video_frames( + video_path: Path | str, + timestamps: list[float], + tolerance_s: float, + backend: str | None = None, +) -> torch.Tensor: + """ + Decodes video frames using the specified backend. + + Args: + video_path (Path): Path to the video file. + timestamps (list[float]): List of timestamps to extract frames. + tolerance_s (float): Allowed deviation in seconds for frame retrieval. + backend (str, optional): Backend to use for decoding. Defaults to "torchcodec" when available in the platform; otherwise, defaults to "pyav".. + + Returns: + torch.Tensor: Decoded frames. + + Currently supports torchcodec on cpu and pyav. + """ + if backend is None: + backend = get_safe_default_codec() + if backend == "torchcodec": + try: + return decode_video_frames_torchcodec(video_path, timestamps, tolerance_s) + except Exception as err: + warnings.warn( + f"torchcodec video decode failed ({type(err).__name__}: {err}); falling back to torchvision/pyav." + ) + return decode_video_frames_torchvision(video_path, timestamps, tolerance_s, backend="pyav") + elif backend in ["pyav", "video_reader"]: + return decode_video_frames_torchvision(video_path, timestamps, tolerance_s, backend) + else: + raise ValueError(f"Unsupported video backend: {backend}") + + +def decode_video_frames_torchvision( + video_path: Path | str, + timestamps: list[float], + tolerance_s: float, + backend: str = "pyav", + log_loaded_timestamps: bool = False, +) -> torch.Tensor: + """Loads frames associated to the requested timestamps of a video + + The backend can be either "pyav" (default) or "video_reader". + "video_reader" requires installing torchvision from source, see: + https://github.com/pytorch/vision/blob/main/torchvision/csrc/io/decoder/gpu/README.rst + (note that you need to compile against ffmpeg<4.3) + + While both use cpu, "video_reader" is supposedly faster than "pyav" but requires additional setup. + For more info on video decoding, see `benchmark/video/README.md` + + See torchvision doc for more info on these two backends: + https://pytorch.org/vision/0.18/index.html?highlight=backend#torchvision.set_video_backend + + Note: Video benefits from inter-frame compression. Instead of storing every frame individually, + the encoder stores a reference frame (or a key frame) and subsequent frames as differences relative to + that key frame. As a consequence, to access a requested frame, we need to load the preceding key frame, + and all subsequent frames until reaching the requested frame. The number of key frames in a video + can be adjusted during encoding to take into account decoding time and video size in bytes. + """ + video_path = str(video_path) + + # set backend + keyframes_only = False + torchvision.set_video_backend(backend) + if backend == "pyav": + keyframes_only = True # pyav doesn't support accurate seek + + # set a video stream reader + # TODO(rcadene): also load audio stream at the same time + reader = torchvision.io.VideoReader(video_path, "video") + + # set the first and last requested timestamps + # Note: previous timestamps are usually loaded, since we need to access the previous key frame + first_ts = min(timestamps) + last_ts = max(timestamps) + + # access closest key frame of the first requested frame + # Note: closest key frame timestamp is usually smaller than `first_ts` (e.g. key frame can be the first frame of the video) + # for details on what `seek` is doing see: https://pyav.basswood-io.com/docs/stable/api/container.html?highlight=inputcontainer#av.container.InputContainer.seek + reader.seek(first_ts, keyframes_only=keyframes_only) + + # load all frames until last requested frame + loaded_frames = [] + loaded_ts = [] + for frame in reader: + current_ts = frame["pts"] + if log_loaded_timestamps: + logging.info(f"frame loaded at timestamp={current_ts:.4f}") + loaded_frames.append(frame["data"]) + loaded_ts.append(current_ts) + if current_ts >= last_ts: + break + + if backend == "pyav": + reader.container.close() + + reader = None + + # Use float32 for timestamp distance computation (torch.cdist doesn't support bfloat16). + query_ts = torch.tensor(timestamps, dtype=torch.float32) + loaded_ts = torch.tensor(loaded_ts, dtype=torch.float32) + + # compute distances between each query timestamp and timestamps of all loaded frames + dist = torch.cdist(query_ts[:, None], loaded_ts[:, None], p=1) + min_, argmin_ = dist.min(1) + + is_within_tol = min_ < tolerance_s + assert is_within_tol.all(), ( + f"One or several query timestamps unexpectedly violate the tolerance ({min_[~is_within_tol]} > {tolerance_s=})." + "It means that the closest frame that can be loaded from the video is too far away in time." + "This might be due to synchronization issues with timestamps during data collection." + "To be safe, we advise to ignore this item during training." + f"\nqueried timestamps: {query_ts}" + f"\nloaded timestamps: {loaded_ts}" + f"\nvideo: {video_path}" + f"\nbackend: {backend}" + ) + + # get closest frames to the query timestamps + closest_frames = torch.stack([loaded_frames[idx] for idx in argmin_]) + closest_ts = loaded_ts[argmin_] + + if log_loaded_timestamps: + logging.info(f"{closest_ts=}") + + # convert to the pytorch format which is float32 in [0,1] range (and channel first) + closest_frames = closest_frames.type(torch.float32) / 255 + + assert len(timestamps) == len(closest_frames) + return closest_frames + + +def decode_video_frames_torchcodec( + video_path: Path | str, + timestamps: list[float], + tolerance_s: float, + device: str = "cpu", + log_loaded_timestamps: bool = False, +) -> torch.Tensor: + """Loads frames associated with the requested timestamps of a video using torchcodec. + + Note: Setting device="cuda" outside the main process, e.g. in data loader workers, will lead to CUDA initialization errors. + + Note: Video benefits from inter-frame compression. Instead of storing every frame individually, + the encoder stores a reference frame (or a key frame) and subsequent frames as differences relative to + that key frame. As a consequence, to access a requested frame, we need to load the preceding key frame, + and all subsequent frames until reaching the requested frame. The number of key frames in a video + can be adjusted during encoding to take into account decoding time and video size in bytes. + """ + + if importlib.util.find_spec("torchcodec"): + from torchcodec.decoders import VideoDecoder + else: + raise ImportError("torchcodec is required but not available.") + + # initialize video decoder + decoder = VideoDecoder(video_path, device=device, seek_mode="approximate") + loaded_frames = [] + loaded_ts = [] + # get metadata for frame information + metadata = decoder.metadata + average_fps = metadata.average_fps + + # convert timestamps to frame indices + frame_indices = [round(ts * average_fps) for ts in timestamps] + + # retrieve frames based on indices + frames_batch = decoder.get_frames_at(indices=frame_indices) + + for frame, pts in zip(frames_batch.data, frames_batch.pts_seconds, strict=False): + loaded_frames.append(frame) + loaded_ts.append(pts.item()) + if log_loaded_timestamps: + logging.info(f"Frame loaded at timestamp={pts:.4f}") + + # Use float32 for timestamp distance computation (torch.cdist doesn't support bfloat16). + query_ts = torch.tensor(timestamps, dtype=torch.float32) + loaded_ts = torch.tensor(loaded_ts, dtype=torch.float32) + + # compute distances between each query timestamp and loaded timestamps + dist = torch.cdist(query_ts[:, None], loaded_ts[:, None], p=1) + min_, argmin_ = dist.min(1) + + is_within_tol = min_ < tolerance_s + assert is_within_tol.all(), ( + f"One or several query timestamps unexpectedly violate the tolerance ({min_[~is_within_tol]} > {tolerance_s=})." + "It means that the closest frame that can be loaded from the video is too far away in time." + "This might be due to synchronization issues with timestamps during data collection." + "To be safe, we advise to ignore this item during training." + f"\nqueried timestamps: {query_ts}" + f"\nloaded timestamps: {loaded_ts}" + f"\nvideo: {video_path}" + ) + + # get closest frames to the query timestamps + closest_frames = torch.stack([loaded_frames[idx] for idx in argmin_]) + closest_ts = loaded_ts[argmin_] + + if log_loaded_timestamps: + logging.info(f"{closest_ts=}") + + # convert to float32 in [0,1] range (channel first) + closest_frames = closest_frames.type(torch.float32) / 255 + + assert len(timestamps) == len(closest_frames) + return closest_frames + +def encode_video_frames( + imgs_dir: Path | str, + video_path: Path | str, + fps: int, + vcodec: str = "libsvtav1", + pix_fmt: str = "yuv420p", + g: int | None = 2, + crf: int | None = 30, + fast_decode: int = 0, + log_level: int | None = av.logging.ERROR, + overwrite: bool = False, +) -> None: + """More info on ffmpeg arguments tuning on `benchmark/video/README.md`""" + # Check encoder availability + if vcodec == "h264_nvenc" : + return encode_video_frames_ffmpeg( + imgs_dir, video_path, fps, pix_fmt = pix_fmt, overwrite=overwrite + ) + + if vcodec not in ["h264", "hevc", "libsvtav1"]: + raise ValueError(f"Unsupported video codec: {vcodec}. Supported codecs are: h264, hevc, libsvtav1.") + + video_path = Path(video_path) + imgs_dir = Path(imgs_dir) + + video_path.parent.mkdir(parents=True, exist_ok=overwrite) + + # Encoders/pixel formats incompatibility check + if (vcodec == "libsvtav1" or vcodec == "hevc") and pix_fmt == "yuv444p": + logging.warning( + f"Incompatible pixel format 'yuv444p' for codec {vcodec}, auto-selecting format 'yuv420p'" + ) + pix_fmt = "yuv420p" + + # Get input frames + template = "frame_" + ("[0-9]" * 6) + ".jpeg" + input_list = sorted( + glob.glob(str(imgs_dir / template)), key=lambda x: int(x.split("_")[-1].split(".")[0]) + ) + + # Define video output frame size (assuming all input frames are the same size) + if len(input_list) == 0: + raise FileNotFoundError(f"No images found in {imgs_dir}.") + dummy_image = Image.open(input_list[0]) + width, height = dummy_image.size + + # Define video codec options + video_options = {} + + if g is not None: + video_options["g"] = str(g) + + if crf is not None: + video_options["crf"] = str(crf) + + if fast_decode: + key = "svtav1-params" if vcodec == "libsvtav1" else "tune" + value = f"fast-decode={fast_decode}" if vcodec == "libsvtav1" else "fastdecode" + video_options[key] = value + + # Set logging level + if log_level is not None: + # "While less efficient, it is generally preferable to modify logging with Python’s logging" + logging.getLogger("libav").setLevel(log_level) + + # Create and open output file (overwrite by default) + with av.open(str(video_path), "w") as output: + output_stream = output.add_stream(vcodec, fps, options=video_options) + output_stream.pix_fmt = pix_fmt + output_stream.width = width + output_stream.height = height + + # Loop through input frames and encode them + for input_data in input_list: + input_image = Image.open(input_data).convert("RGB") + input_frame = av.VideoFrame.from_image(input_image) + packet = output_stream.encode(input_frame) + if packet: + output.mux(packet) + + # Flush the encoder + packet = output_stream.encode() + if packet: + output.mux(packet) + + # Reset logging level + if log_level is not None: + av.logging.restore_default_callback() + + if not video_path.exists(): + raise OSError(f"Video encoding did not work. File not found: {video_path}.") + +# GPU speed up video encoding +def encode_video_frames_ffmpeg( + imgs_dir: Path | str, + video_path: Path | str, + fps: int, + vcodec: str = "h264_nvenc", + pix_fmt: str = "yuv420p", + g: int | None = 4, + crf: int | None = 30, + overwrite: bool = False, +) -> None: + import subprocess + import shutil + + video_path = Path(video_path) + imgs_dir = Path(imgs_dir) + + video_path.parent.mkdir(parents=True, exist_ok=True) + + # Check if ffmpeg is available + ffmpeg_path = shutil.which("ffmpeg") + if ffmpeg_path is None: + raise RuntimeError("ffmpeg is not installed or not in system PATH") + + # Build ffmpeg command + cmd = [ + ffmpeg_path, + "-y" if overwrite else "-n", + "-framerate", str(fps), + "-pattern_type", "sequence", + "-i", str(imgs_dir / "frame_%06d.jpeg"), + "-vcodec", vcodec, + "-pix_fmt", pix_fmt, + ] + + if g is not None: + cmd.extend(["-g", str(g)]) + + if crf is not None: + cmd.extend(["-crf", str(crf)]) + + cmd.append(str(video_path)) + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + check=True + ) + except subprocess.CalledProcessError as e: + error_msg = f"FFmpeg failed:\nCommand: {' '.join(cmd)}\nStdout: {e.stdout}\nStderr: {e.stderr}" + logging.error(error_msg) + logging.error(video_path) + logging.error(vcodec) + print(f"ffmpeg {' '.join(cmd)}") + raise RuntimeError(error_msg) + except Exception as e: + error_msg = f"Failed to run FFmpeg command: {' '.join(cmd)}\nError: {str(e)}" + logging.error(error_msg) + raise RuntimeError(error_msg) + + +@dataclass +class VideoFrame: + # TODO(rcadene, lhoestq): move to Hugging Face `datasets` repo + """ + Provides a type for a dataset containing video frames. + + Example: + + ```python + data_dict = [{"image": {"path": "videos/episode_0.mp4", "timestamp": 0.3}}] + features = {"image": VideoFrame()} + Dataset.from_dict(data_dict, features=Features(features)) + ``` + """ + + pa_type: ClassVar[Any] = pa.struct({"path": pa.string(), "timestamp": pa.float32()}) + _type: str = field(default="VideoFrame", init=False, repr=False) + + def __call__(self): + return self.pa_type + + +with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + "'register_feature' is experimental and might be subject to breaking changes in the future.", + category=UserWarning, + ) + # to make VideoFrame available in HuggingFace `datasets` + register_feature(VideoFrame, "VideoFrame") + + +def get_audio_info(video_path: Path | str) -> dict: + # Set logging level + logging.getLogger("libav").setLevel(av.logging.ERROR) + + # Getting audio stream information + audio_info = {} + with av.open(str(video_path), "r") as audio_file: + try: + audio_stream = audio_file.streams.audio[0] + except IndexError: + # Reset logging level + av.logging.restore_default_callback() + return {"has_audio": False} + + audio_info["audio.channels"] = audio_stream.channels + audio_info["audio.codec"] = audio_stream.codec.canonical_name + # In an ideal loseless case : bit depth x sample rate x channels = bit rate. + # In an actual compressed case, the bit rate is set according to the compression level : the lower the bit rate, the more compression is applied. + audio_info["audio.bit_rate"] = audio_stream.bit_rate + audio_info["audio.sample_rate"] = audio_stream.sample_rate # Number of samples per second + # In an ideal loseless case : fixed number of bits per sample. + # In an actual compressed case : variable number of bits per sample (often reduced to match a given depth rate). + audio_info["audio.bit_depth"] = audio_stream.format.bits + audio_info["audio.channel_layout"] = audio_stream.layout.name + audio_info["has_audio"] = True + + # Reset logging level + av.logging.restore_default_callback() + + return audio_info + + +def get_video_info(video_path: Path | str) -> dict: + # Set logging level + logging.getLogger("libav").setLevel(av.logging.ERROR) + + # Getting video stream information + video_info = {} + with av.open(str(video_path), "r") as video_file: + try: + video_stream = video_file.streams.video[0] + except IndexError: + # Reset logging level + av.logging.restore_default_callback() + return {} + + video_info["video.height"] = video_stream.height + video_info["video.width"] = video_stream.width + video_info["video.codec"] = video_stream.codec.canonical_name + video_info["video.pix_fmt"] = video_stream.pix_fmt + video_info["video.is_depth_map"] = False + + # Calculate fps from r_frame_rate + video_info["video.fps"] = int(video_stream.base_rate) + + pixel_channels = get_video_pixel_channels(video_stream.pix_fmt) + video_info["video.channels"] = pixel_channels + + # Reset logging level + av.logging.restore_default_callback() + + # Adding audio stream information + video_info.update(**get_audio_info(video_path)) + + return video_info + + +def get_video_pixel_channels(pix_fmt: str) -> int: + if "gray" in pix_fmt or "depth" in pix_fmt or "monochrome" in pix_fmt: + return 1 + elif "rgba" in pix_fmt or "yuva" in pix_fmt: + return 4 + elif "rgb" in pix_fmt or "yuv" in pix_fmt: + return 3 + else: + raise ValueError("Unknown format") + + +def get_image_pixel_channels(image: Image): + if image.mode == "L": + return 1 # Grayscale + elif image.mode == "LA": + return 2 # Grayscale + Alpha + elif image.mode == "RGB": + return 3 # RGB + elif image.mode == "RGBA": + return 4 # RGBA + else: + raise ValueError("Unknown format") diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..f7f2c2babe6bad4ce9925f1971f539792dd03146 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py @@ -0,0 +1,1304 @@ +#!/usr/bin/env python + +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import contextlib +import logging +import shutil +from pathlib import Path +from typing import Callable, List, Literal +import warnings + +import datasets +import numpy as np +import packaging.version +import PIL.Image +import torch +import torch.utils +import pyarrow.parquet as pq +from datasets import concatenate_datasets, load_dataset +from huggingface_hub import HfApi, snapshot_download +from huggingface_hub.constants import REPOCARD_NAME +from huggingface_hub.errors import RevisionNotFoundError + +# adapte to own path +from ..constants import HF_LEROBOT_HOME +from .datasets.compute_stats import aggregate_stats, compute_episode_stats +from .datasets.utils import ( + DEFAULT_FEATURES, + DEFAULT_IMAGE_PATH, + INFO_PATH, + TASKS_PATH, + _validate_feature_names, + append_jsonlines, + backward_compatible_episodes_stats, + check_delta_timestamps, + check_timestamps_sync, + # check_version_compatibility, + create_empty_dataset_info, + create_lerobot_dataset_card, + embed_images, + get_delta_indices, + get_episode_data_index, + get_hf_features_from_features, + # get_safe_version, + hf_transform_to_torch, + is_valid_version, + load_episodes, + load_episodes_stats, + load_info, + load_stats, + load_tasks, + load_annotations, + validate_episode_buffer, + validate_frame, + write_episode, + write_episode_stats, + write_info, + write_json, +) +from .datasets.video_utils import ( + VideoFrame, + decode_video_frames, + encode_video_frames, + get_safe_default_codec, + get_video_info, +) +import traceback + +CODEBASE_VERSION = "v2.1" + + +class LeRobotDatasetMetadata: + def __init__( + self, + repo_id: str, + root: str | Path | None = None, + revision: str | None = None, + force_cache_sync: bool = False, + ): + self.repo_id = repo_id + self.revision = revision if revision else CODEBASE_VERSION + self.root = Path(root) if root is not None else HF_LEROBOT_HOME / repo_id + + try: + if force_cache_sync: + raise FileNotFoundError + self.load_metadata() + except (FileNotFoundError, NotADirectoryError): + # if is_valid_version(self.revision): + # self.revision = get_safe_version(self.repo_id, self.revision) + + (self.root / "meta").mkdir(exist_ok=True, parents=True) + self.pull_from_repo(allow_patterns="meta/") + self.load_metadata() + + def load_metadata(self): + self.info = load_info(self.root) + # TODO add new check + # check_version_compatibility(self.repo_id, self._version, CODEBASE_VERSION) + self.tasks, self.task_to_task_index = load_tasks(self.root) + if (self.root / "annotations").exists(): + self.annotations = load_annotations(self.root) + self.episodes = load_episodes(self.root) + if self._version < packaging.version.parse("v2.1"): + self.stats = load_stats(self.root) + self.episodes_stats = backward_compatible_episodes_stats(self.stats, self.episodes) + else: + self.episodes_stats = load_episodes_stats(self.root) + self.stats = aggregate_stats(list(self.episodes_stats.values())) + + def pull_from_repo( + self, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + ) -> None: + snapshot_download( + self.repo_id, + repo_type="dataset", + revision=self.revision, + local_dir=self.root, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + ) + + @property + def _version(self) -> packaging.version.Version: + """Codebase version used to create this dataset.""" + return packaging.version.parse(self.info["codebase_version"]) + + def get_data_file_path(self, ep_index: int) -> Path: + ep_chunk = self.get_episode_chunk(ep_index) + fpath = self.data_path.format(episode_chunk=ep_chunk, episode_index=ep_index) + return Path(fpath) + + def get_video_file_path(self, ep_index: int, vid_key: str) -> Path: + ep_chunk = self.get_episode_chunk(ep_index) + fpath = self.video_path.format(episode_chunk=ep_chunk, video_key=vid_key, episode_index=ep_index) + return Path(fpath) + + def get_episode_chunk(self, ep_index: int) -> int: + return ep_index // self.chunks_size + + @property + def data_path(self) -> str: + """Formattable string for the parquet files.""" + return self.info["data_path"] + + @property + def video_path(self) -> str | None: + """Formattable string for the video files.""" + return self.info["video_path"] + + @property + def robot_type(self) -> str | None: + """Robot type used in recording this dataset.""" + return self.info["robot_type"] + + @property + def fps(self) -> int: + """Frames per second used during data collection.""" + return self.info["fps"] + + @property + def features(self) -> dict[str, dict]: + """All features contained in the dataset.""" + return self.info["features"] + + @property + def image_keys(self) -> list[str]: + """Keys to access visual modalities stored as images.""" + return [key for key, ft in self.features.items() if ft["dtype"] == "image"] + + @property + def video_keys(self) -> list[str]: + """Keys to access visual modalities stored as videos.""" + return [key for key, ft in self.features.items() if ft["dtype"] == "video"] + + @property + def camera_keys(self) -> list[str]: + """Keys to access visual modalities (regardless of their storage method).""" + return [key for key, ft in self.features.items() if ft["dtype"] in ["video", "image"]] + + @property + def names(self) -> dict[str, list | dict]: + """Names of the various dimensions of vector modalities.""" + return {key: ft["names"] for key, ft in self.features.items()} + + @property + def shapes(self) -> dict: + """Shapes for the different features.""" + return {key: tuple(ft["shape"]) for key, ft in self.features.items()} + + @property + def total_episodes(self) -> int: + """Total number of episodes available.""" + return self.info["total_episodes"] + + @property + def total_frames(self) -> int: + """Total number of frames saved in this dataset.""" + return self.info["total_frames"] + + @property + def total_tasks(self) -> int: + """Total number of different tasks performed in this dataset.""" + return self.info["total_tasks"] + + @property + def total_chunks(self) -> int: + """Total number of chunks (groups of episodes).""" + return self.info["total_chunks"] + + @property + def chunks_size(self) -> int: + """Max number of episodes per chunk.""" + return self.info["chunks_size"] + + def get_task_index(self, task: str) -> int | None: + """ + Given a task in natural language, returns its task_index if the task already exists in the dataset, + otherwise return None. + """ + return self.task_to_task_index.get(task, None) + + def add_task(self, task: str): + """ + Given a task in natural language, add it to the dictionary of tasks. + """ + if task in self.task_to_task_index: + raise ValueError(f"The task '{task}' already exists and can't be added twice.") + + task_index = self.info["total_tasks"] + self.task_to_task_index[task] = task_index + self.tasks[task_index] = task + self.info["total_tasks"] += 1 + + task_dict = { + "task_index": task_index, + "task": task, + } + append_jsonlines(task_dict, self.root / TASKS_PATH) + + def save_episode( + self, + episode_index: int, + episode_length: int, + episode_tasks: list[str], + episode_stats: dict[str, dict], + raw_file_name: str | None = None, + ) -> None: + self.info["total_episodes"] += 1 + self.info["total_frames"] += episode_length + + chunk = self.get_episode_chunk(episode_index) + if chunk >= self.total_chunks: + self.info["total_chunks"] += 1 + + self.info["splits"] = {"train": f"0:{self.info['total_episodes']}"} + self.info["total_videos"] += len(self.video_keys) + if len(self.video_keys) > 0: + self.update_video_info() + + write_info(self.info, self.root) + + episode_dict = { + "episode_index": episode_index, + "tasks": episode_tasks, + "length": episode_length, + } + if raw_file_name is not None: + episode_dict["raw_file_name"] = raw_file_name + + self.episodes[episode_index] = episode_dict + write_episode(episode_dict, self.root) + + self.episodes_stats[episode_index] = episode_stats + self.stats = aggregate_stats([self.stats, episode_stats]) if self.stats else episode_stats + write_episode_stats(episode_index, episode_stats, self.root) + + def update_video_info(self) -> None: + """ + Warning: this function writes info from first episode videos, implicitly assuming that all videos have + been encoded the same way. Also, this means it assumes the first episode exists. + """ + for key in self.video_keys: + if not self.features[key].get("info", None): + video_path = self.root / self.get_video_file_path(ep_index=0, vid_key=key) + self.info["features"][key]["info"] = get_video_info(video_path) + + def __repr__(self): + feature_keys = list(self.features) + return ( + f"{self.__class__.__name__}({{\n" + f" Repository ID: '{self.repo_id}',\n" + f" Total episodes: '{self.total_episodes}',\n" + f" Total frames: '{self.total_frames}',\n" + f" Features: '{feature_keys}',\n" + "})',\n" + ) + + @classmethod + def create( + cls, + repo_id: str, + fps: int, + features: dict, + robot_type: str | None = None, + root: str | Path | None = None, + use_videos: bool = True, + ) -> "LeRobotDatasetMetadata": + """Creates metadata for a LeRobotDataset.""" + obj = cls.__new__(cls) + obj.repo_id = repo_id + obj.root = Path(root) if root is not None else HF_LEROBOT_HOME / repo_id + + obj.root.mkdir(parents=True, exist_ok=False) + + # TODO(aliberts, rcadene): implement sanity check for features + features = {**features, **DEFAULT_FEATURES} + _validate_feature_names(features) + + obj.tasks, obj.task_to_task_index = {}, {} + obj.episodes_stats, obj.stats, obj.episodes = {}, {}, {} + obj.info = create_empty_dataset_info(CODEBASE_VERSION, fps, features, use_videos, robot_type) + if len(obj.video_keys) > 0 and not use_videos: + raise ValueError() + write_json(obj.info, obj.root / INFO_PATH) + obj.revision = None + return obj + + +class LeRobotDataset(torch.utils.data.Dataset): + def __init__( + self, + repo_id: str, + root: str | Path | None = None, + episodes: list[int] | None = None, + image_transforms: Callable | None = None, + delta_timestamps: dict[list[float]] | None = None, + tolerance_s: float = 1e-4, + revision: str | None = None, + force_cache_sync: bool = False, + download_videos: bool = True, + video_backend: str | None = None, + video_codec: Literal["h264", "hevc", "libsvtav1", "h264_nvenc"] = "libsvtav1", + is_compute_episode_stats_image: bool = True, + ): + """ + 2 modes are available for instantiating this class, depending on 2 different use cases: + + 1. Your dataset already exists: + - On your local disk in the 'root' folder. This is typically the case when you recorded your + dataset locally and you may or may not have pushed it to the hub yet. Instantiating this class + with 'root' will load your dataset directly from disk. This can happen while you're offline (no + internet connection). + + - On the Hugging Face Hub at the address https://huggingface.co/datasets/{repo_id} and not on + your local disk in the 'root' folder. Instantiating this class with this 'repo_id' will download + the dataset from that address and load it, pending your dataset is compliant with + codebase_version v2.0. If your dataset has been created before this new format, you will be + prompted to convert it using our conversion script from v1.6 to v2.0, which you can find at + lerobot/datasets/v2/convert_dataset_v1_to_v2.py. + + + 2. Your dataset doesn't already exists (either on local disk or on the Hub): you can create an empty + LeRobotDataset with the 'create' classmethod. This can be used for recording a dataset or port an + existing dataset to the LeRobotDataset format. + + + In terms of files, LeRobotDataset encapsulates 3 main things: + - metadata: + - info contains various information about the dataset like shapes, keys, fps etc. + - stats stores the dataset statistics of the different modalities for normalization + - tasks contains the prompts for each task of the dataset, which can be used for + task-conditioned training. + - hf_dataset (from datasets.Dataset), which will read any values from parquet files. + - videos (optional) from which frames are loaded to be synchronous with data from parquet files. + + A typical LeRobotDataset looks like this from its root path: + . + ├── data + │ ├── chunk-000 + │ │ ├── episode_000000.parquet + │ │ ├── episode_000001.parquet + │ │ ├── episode_000002.parquet + │ │ └── ... + │ ├── chunk-001 + │ │ ├── episode_001000.parquet + │ │ ├── episode_001001.parquet + │ │ ├── episode_001002.parquet + │ │ └── ... + │ └── ... + ├── meta + │ ├── episodes.jsonl + │ ├── info.json + │ ├── stats.json + │ └── tasks.jsonl + └── videos + ├── chunk-000 + │ ├── observation.images.laptop + │ │ ├── episode_000000.mp4 + │ │ ├── episode_000001.mp4 + │ │ ├── episode_000002.mp4 + │ │ └── ... + │ ├── observation.images.phone + │ │ ├── episode_000000.mp4 + │ │ ├── episode_000001.mp4 + │ │ ├── episode_000002.mp4 + │ │ └── ... + ├── chunk-001 + └── ... + + Note that this file-based structure is designed to be as versatile as possible. The files are split by + episodes which allows a more granular control over which episodes one wants to use and download. The + structure of the dataset is entirely described in the info.json file, which can be easily downloaded + or viewed directly on the hub before downloading any actual data. The type of files used are very + simple and do not need complex tools to be read, it only uses .parquet, .json and .mp4 files (and .md + for the README). + + Args: + repo_id (str): This is the repo id that will be used to fetch the dataset. Locally, the dataset + will be stored under root/repo_id. + root (Path | None, optional): Local directory to use for downloading/writing files. You can also + set the LEROBOT_HOME environment variable to point to a different location. Defaults to + '~/.cache/huggingface/lerobot'. + episodes (list[int] | None, optional): If specified, this will only load episodes specified by + their episode_index in this list. Defaults to None. + image_transforms (Callable | None, optional): You can pass standard v2 image transforms from + torchvision.transforms.v2 here which will be applied to visual modalities (whether they come + from videos or images). Defaults to None. + delta_timestamps (dict[list[float]] | None, optional): _description_. Defaults to None. + tolerance_s (float, optional): Tolerance in seconds used to ensure data timestamps are actually in + sync with the fps value. It is used at the init of the dataset to make sure that each + timestamps is separated to the next by 1/fps +/- tolerance_s. This also applies to frames + decoded from video files. It is also used to check that `delta_timestamps` (when provided) are + multiples of 1/fps. Defaults to 1e-4. + revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a + commit hash. Defaults to current codebase version tag. + sync_cache_first (bool, optional): Flag to sync and refresh local files first. If True and files + are already present in the local cache, this will be faster. However, files loaded might not + be in sync with the version on the hub, especially if you specified 'revision'. Defaults to + False. + download_videos (bool, optional): Flag to download the videos. Note that when set to True but the + video files are already present on local disk, they won't be downloaded again. Defaults to + True. + video_backend (str | None, optional): Video backend to use for decoding videos. Defaults to torchcodec when available int the platform; otherwise, defaults to 'pyav'. + You can also use the 'pyav' decoder used by Torchvision, which used to be the default option, or 'video_reader' which is another decoder of Torchvision. + """ + super().__init__() + self.repo_id = repo_id + self.root = Path(root) if root else HF_LEROBOT_HOME / repo_id + self.image_transforms = image_transforms + self.delta_timestamps = delta_timestamps + self.episodes = episodes + self.tolerance_s = tolerance_s + self.revision = revision if revision else CODEBASE_VERSION + self.video_backend = video_backend if video_backend else get_safe_default_codec() + self.video_codec = video_codec + self.is_compute_episode_stats_image = is_compute_episode_stats_image + self.delta_indices = None + self.during_training = True + + # Unused attributes + self.image_writer = None + self.episode_buffer = None + + self.root.mkdir(exist_ok=True, parents=True) + + # Load metadata + self.meta = LeRobotDatasetMetadata( + self.repo_id, self.root, self.revision, force_cache_sync=force_cache_sync + ) + if self.episodes is not None and self.meta._version >= packaging.version.parse("v2.1"): + episodes_stats = [self.meta.episodes_stats[ep_idx] for ep_idx in self.episodes] + self.stats = aggregate_stats(episodes_stats) + + # Load actual data + try: + if force_cache_sync: + raise FileNotFoundError + assert all((self.root / fpath).is_file() for fpath in self.get_episodes_file_paths()) + self.hf_dataset = self.load_hf_dataset() + except (AssertionError, FileNotFoundError, NotADirectoryError): + # self.revision = get_safe_version(self.repo_id, self.revision) + self.revision = CODEBASE_VERSION + self.download_episodes(download_videos) + self.hf_dataset = self.load_hf_dataset() + + self.episode_data_index = get_episode_data_index(self.meta.episodes, self.episodes) + + # Check timestamps + timestamps = torch.stack(self.hf_dataset["timestamp"]).numpy() + episode_indices = torch.stack(self.hf_dataset["episode_index"]).numpy() + ep_data_index_np = {k: t.numpy() for k, t in self.episode_data_index.items()} + # check_timestamps_sync(timestamps, episode_indices, ep_data_index_np, self.fps, self.tolerance_s) + + # Setup delta_indices + if self.delta_timestamps is not None: + # check_delta_timestamps(self.delta_timestamps, self.fps, self.tolerance_s) + self.delta_indices = get_delta_indices(self.delta_timestamps, self.fps) + + def push_to_hub( + self, + branch: str | None = None, + tags: list | None = None, + license: str | None = "apache-2.0", + tag_version: bool = True, + push_videos: bool = True, + private: bool = False, + allow_patterns: list[str] | str | None = None, + upload_large_folder: bool = False, + **card_kwargs, + ) -> None: + ignore_patterns = ["images/"] + if not push_videos: + ignore_patterns.append("videos/") + + hub_api = HfApi() + hub_api.create_repo( + repo_id=self.repo_id, + private=private, + repo_type="dataset", + exist_ok=True, + ) + if branch: + hub_api.create_branch( + repo_id=self.repo_id, + branch=branch, + revision=self.revision, + repo_type="dataset", + exist_ok=True, + ) + + upload_kwargs = { + "repo_id": self.repo_id, + "folder_path": self.root, + "repo_type": "dataset", + "revision": branch, + "allow_patterns": allow_patterns, + "ignore_patterns": ignore_patterns, + } + if upload_large_folder: + hub_api.upload_large_folder(**upload_kwargs) + else: + hub_api.upload_folder(**upload_kwargs) + + if not hub_api.file_exists(self.repo_id, REPOCARD_NAME, repo_type="dataset", revision=branch): + card = create_lerobot_dataset_card( + tags=tags, dataset_info=self.meta.info, license=license, **card_kwargs + ) + card.push_to_hub(repo_id=self.repo_id, repo_type="dataset", revision=branch) + + if tag_version: + with contextlib.suppress(RevisionNotFoundError): + hub_api.delete_tag(self.repo_id, tag=CODEBASE_VERSION, repo_type="dataset") + hub_api.create_tag(self.repo_id, tag=CODEBASE_VERSION, revision=branch, repo_type="dataset") + + def pull_from_repo( + self, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + ) -> None: + snapshot_download( + self.repo_id, + repo_type="dataset", + revision=self.revision, + local_dir=self.root, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + ) + + def download_episodes(self, download_videos: bool = True) -> None: + """Downloads the dataset from the given 'repo_id' at the provided version. If 'episodes' is given, this + will only download those episodes (selected by their episode_index). If 'episodes' is None, the whole + dataset will be downloaded. Thanks to the behavior of snapshot_download, if the files are already present + in 'local_dir', they won't be downloaded again. + """ + # TODO(rcadene, aliberts): implement faster transfer + # https://huggingface.co/docs/huggingface_hub/en/guides/download#faster-downloads + files = None + ignore_patterns = None if download_videos else "videos/" + if self.episodes is not None: + files = self.get_episodes_file_paths() + + self.pull_from_repo(allow_patterns=files, ignore_patterns=ignore_patterns) + + def get_episodes_file_paths(self) -> list[Path]: + episodes = self.episodes if self.episodes is not None else list(range(self.meta.total_episodes)) + fpaths = [str(self.meta.get_data_file_path(ep_idx)) for ep_idx in episodes] + if len(self.meta.video_keys) > 0: + video_files = [ + str(self.meta.get_video_file_path(ep_idx, vid_key)) + for vid_key in self.meta.video_keys + for ep_idx in episodes + ] + fpaths += video_files + + return fpaths + + def load_hf_dataset(self) -> datasets.Dataset: + """hf_dataset contains all the observations, states, actions, rewards, etc.""" + if self.episodes is None: + path = str(self.root / "data") + hf_dataset = load_dataset("parquet", data_dir=path, split="train") + else: + files = [str(self.root / self.meta.get_data_file_path(ep_idx)) for ep_idx in self.episodes] + hf_dataset = load_dataset("parquet", data_files=files, split="train") + + # TODO(aliberts): hf_dataset.set_format("torch") + hf_dataset.set_transform(hf_transform_to_torch) + return hf_dataset + + def create_hf_dataset(self) -> datasets.Dataset: + features = get_hf_features_from_features(self.features) + ft_dict = {col: [] for col in features} + hf_dataset = datasets.Dataset.from_dict(ft_dict, features=features, split="train") + + # TODO(aliberts): hf_dataset.set_format("torch") + hf_dataset.set_transform(hf_transform_to_torch) + return hf_dataset + + @property + def fps(self) -> int: + """Frames per second used during data collection.""" + return self.meta.fps + + @property + def num_frames(self) -> int: + """Number of frames in selected episodes.""" + return len(self.hf_dataset) if self.hf_dataset is not None else self.meta.total_frames + + @property + def num_episodes(self) -> int: + """Number of episodes selected.""" + return len(self.episodes) if self.episodes is not None else self.meta.total_episodes + + @property + def features(self) -> dict[str, dict]: + return self.meta.features + + @property + def hf_features(self) -> datasets.Features: + """Features of the hf_dataset.""" + if self.hf_dataset is not None: + return self.hf_dataset.features + else: + return get_hf_features_from_features(self.features) + + def _get_query_indices(self, idx: int, ep_idx: int) -> tuple[dict[str, list[int | bool]]]: + ep_start = self.episode_data_index["from"][ep_idx] + ep_end = self.episode_data_index["to"][ep_idx] + query_indices = { + key: [max(ep_start.item(), min(ep_end.item() - 1, idx + delta)) for delta in delta_idx] + for key, delta_idx in self.delta_indices.items() + } + padding = { # Pad values outside of current episode range + f"{key}_is_pad": torch.BoolTensor( + [(idx + delta < ep_start.item()) | (idx + delta >= ep_end.item()) for delta in delta_idx] + ) + for key, delta_idx in self.delta_indices.items() + } + return query_indices, padding + + def _get_query_timestamps( + self, + current_ts: float, + query_indices: dict[str, list[int]] | None = None, + ) -> dict[str, list[float]]: + query_timestamps = {} + for key in self.meta.video_keys: + if query_indices is not None and key in query_indices: + timestamps = self.hf_dataset.select(query_indices[key])["timestamp"] + query_timestamps[key] = torch.stack(timestamps).tolist() + else: + query_timestamps[key] = [current_ts] + + return query_timestamps + + def _query_hf_dataset(self, query_indices: dict[str, list[int]]) -> dict: + return { + key: torch.stack(self.hf_dataset.select(q_idx)[key]) + for key, q_idx in query_indices.items() + if key not in self.meta.video_keys + } + + def _query_hf_dataset_fast(self, query_indices: dict[str, list[int]]) -> dict: + result = {} + processed_indices = set() + index_to_selected = {} + for key, q_idx in query_indices.items(): + if key not in self.meta.video_keys : + if 'images' in key and not self.during_training: + continue + q_idx_tuple = tuple(q_idx) + if q_idx_tuple not in processed_indices: + selected_data = self.hf_dataset.select(q_idx) + index_to_selected[q_idx_tuple] = selected_data + processed_indices.add(q_idx_tuple) + else: + selected_data = index_to_selected[q_idx_tuple] + result[key] = torch.stack(selected_data[key]) + return result + + # no videos + def get_episode_data(self, episode_id: int) -> dict: + ep_start = self.episode_data_index["from"][episode_id].item() + ep_end = self.episode_data_index["to"][episode_id].item() + q_idx = list(range(ep_start, ep_end)) + # selected_data = self.hf_dataset.select(q_idx) + selected_data = self.hf_dataset[q_idx] + res_keys = self.meta.features.keys() - set(self.meta.video_keys) + res = {key : torch.stack(selected_data[key]) for key in res_keys} + return res + + def _query_videos(self, query_timestamps: dict[str, list[float]], ep_idx: int) -> dict[str, torch.Tensor]: + """Note: When using data workers (e.g. DataLoader with num_workers>0), do not call this function + in the main process (e.g. by using a second Dataloader with num_workers=0). It will result in a + Segmentation Fault. This probably happens because a memory reference to the video loader is created in + the main process and a subprocess fails to access it. + """ + item = {} + for vid_key, query_ts in query_timestamps.items(): + video_path = self.root / self.meta.get_video_file_path(ep_idx, vid_key) + frames = decode_video_frames(video_path, query_ts, self.tolerance_s, self.video_backend) + item[vid_key] = frames.squeeze(0) + + return item + + def _add_padding_keys(self, item: dict, padding: dict[str, list[bool]]) -> dict: + for key, val in padding.items(): + item[key] = torch.BoolTensor(val) + return item + + def __len__(self): + return self.num_frames + + def __getitem__(self, idx) -> dict: + item = self.hf_dataset[idx] + ep_idx = item["episode_index"].item() + + query_indices = None + if self.delta_indices is not None: + current_ep_idx = self.episodes.index(ep_idx) if self.episodes is not None else ep_idx + query_indices, padding = self._get_query_indices(idx, current_ep_idx) + query_result = self._query_hf_dataset_fast(query_indices) + item = {**item, **padding} + for key, val in query_result.items(): + item[key] = val + + if len(self.meta.video_keys) > 0 and self.during_training: + current_ts = item["timestamp"].item() + query_timestamps = self._get_query_timestamps(current_ts, query_indices) + video_frames = self._query_videos(query_timestamps, ep_idx) + item = {**video_frames, **item} + + if self.image_transforms is not None: + image_keys = self.meta.camera_keys + for cam in image_keys: + item[cam] = self.image_transforms(item[cam]) + + # Add task as a string + task_idx = item["task_index"].item() + item["task"] = self.meta.tasks[task_idx] + if "coarse_task_index" in item: + coarse_task_index = item["coarse_task_index"].item() + item["coarse_task"] = self.meta.tasks[coarse_task_index] + + if "operating_hand_index" in item: + operating_hand_index = item["operating_hand_index"].item() + item["operating_hand"] = self.meta.tasks[operating_hand_index] + + if "subtask_annotation" in item and hasattr(self.meta, "annotations"): + index = item["subtask_annotation"][0].item() + item["subtask"] = self.meta.annotations["subtask"][index] + + if "atomic_task_index" in item and item["atomic_task_index"] is not None: + atomic_task_index = item["atomic_task_index"].item() + item["subtask"] = self.meta.tasks[int(atomic_task_index)] + + return item + + def __repr__(self): + feature_keys = list(self.features) + return ( + f"{self.__class__.__name__}({{\n" + f" Repository ID: '{self.repo_id}',\n" + f" Number of selected episodes: '{self.num_episodes}',\n" + f" Number of selected samples: '{self.num_frames}',\n" + f" Features: '{feature_keys}',\n" + "})',\n" + ) + + def create_episode_buffer(self, episode_index: int | None = None) -> dict: + current_ep_idx = self.meta.total_episodes if episode_index is None else episode_index + ep_buffer = {} + # size and task are special cases that are not in self.features + ep_buffer["size"] = 0 + ep_buffer["task"] = [] + for key in self.features: + ep_buffer[key] = current_ep_idx if key == "episode_index" else [] + return ep_buffer + + def _get_image_file_path(self, episode_index: int, image_key: str, frame_index: int) -> Path: + fpath = DEFAULT_IMAGE_PATH.format( + image_key=image_key, episode_index=episode_index, frame_index=frame_index + ) + return self.root / fpath + + # def _save_image(self, image: torch.Tensor | np.ndarray | PIL.Image.Image | bytes, fpath: Path) -> None: + # if self.image_writer is None: + # if isinstance(image, torch.Tensor): + # image = image.cpu().numpy() + # write_image(image, fpath) + # else: + # self.image_writer.save_image(image=image, fpath=fpath) + + def add_frame(self, frame: dict, task: List[str], timestamp: float | None = None) -> None: + """ + This function only adds the frame to the episode_buffer. Apart from images — which are written in a + temporary directory — nothing is written to disk. To save those frames, the 'save_episode()' method + then needs to be called. + """ + # Convert torch to numpy if needed + assert len(task) == 4, "Task frame must be of two elements" + for name in frame: + if isinstance(frame[name], torch.Tensor): + frame[name] = frame[name].numpy() + + validate_frame(frame, self.features) + + if self.episode_buffer is None: + self.episode_buffer = self.create_episode_buffer() + + # Automatically add frame_index and timestamp to episode buffer + frame_index = self.episode_buffer["size"] + if timestamp is None: + timestamp = frame_index / self.fps + self.episode_buffer["frame_index"].append(frame_index) + self.episode_buffer["timestamp"].append(timestamp) + self.episode_buffer["task"].append(task) + + # Add frame features to episode_buffer + for key in frame: + if key not in self.features: + raise ValueError( + f"An element of the frame is not in the features. '{key}' not in '{self.features.keys()}'." + ) + + if self.features[key]["dtype"] in ["image", "video"]: + img_path = self._get_image_file_path( + episode_index=self.episode_buffer["episode_index"], image_key=key, frame_index=frame_index + ) + if frame_index == 0: + img_path.parent.mkdir(parents=True, exist_ok=True) + # self._save_image(frame[key], img_path) + self.episode_buffer[key].append(str(img_path)) + else: + self.episode_buffer[key].append(frame[key]) + + self.episode_buffer["size"] += 1 + + def save_episode(self, episode_data: dict | None = None, raw_file_name: str | None = None) -> None: + """ + This will save to disk the current episode in self.episode_buffer. + + Args: + episode_data (dict | None, optional): Dict containing the episode data to save. If None, this will + save the current episode in self.episode_buffer, which is filled with 'add_frame'. Defaults to + None. + """ + if not episode_data: + episode_buffer = self.episode_buffer + + validate_episode_buffer(episode_buffer, self.meta.total_episodes, self.features) + + # size and task are special cases that won't be added to hf_dataset + episode_length = episode_buffer.pop("size") + tasks = episode_buffer.pop("task") + episode_tasks = list(set([item for sublist in tasks for item in sublist])) + episode_index = episode_buffer["episode_index"] + + episode_buffer["index"] = np.arange(self.meta.total_frames, self.meta.total_frames + episode_length) + episode_buffer["episode_index"] = np.full((episode_length,), episode_index) + + # Add new tasks to the tasks dictionary + for task in episode_tasks: + task_index = self.meta.get_task_index(task) + if task_index is None: + self.meta.add_task(task) + + # Given tasks in natural language, find their corresponding task indices + episode_buffer["coarse_task_index"] = np.array([self.meta.get_task_index(task[0]) for task in tasks]) + episode_buffer["task_index"] = np.array([self.meta.get_task_index(task[1]) for task in tasks]) + episode_buffer["coarse_quality_index"] = np.array([self.meta.get_task_index(task[2]) for task in tasks]) + episode_buffer["quality_index"] = np.array([self.meta.get_task_index(task[3]) for task in tasks]) + + for key, ft in self.features.items(): + # index, episode_index, task_index are already processed above, and image and video + # are processed separately by storing image path and frame info as meta data + if key in ["index", "episode_index", "coarse_task_index", "task_index", "coarse_quality_index", "quality_index"] or ft["dtype"] in ["image", "video"]: + continue + episode_buffer[key] = np.stack(episode_buffer[key]) + + self._wait_image_writer() + self._save_episode_table(episode_buffer, episode_index) + ep_stats = compute_episode_stats(episode_buffer, self.features, self.is_compute_episode_stats_image) + + if len(self.meta.video_keys) > 0: + video_paths = self.encode_episode_videos(episode_index) + for key in self.meta.video_keys: + episode_buffer[key] = video_paths[key] + + # `meta.save_episode` be executed after encoding the videos + self.meta.save_episode(episode_index, episode_length, episode_tasks, ep_stats, raw_file_name) + + ep_data_index = get_episode_data_index(self.meta.episodes, [episode_index]) + ep_data_index_np = {k: t.numpy() for k, t in ep_data_index.items()} + check_timestamps_sync( + episode_buffer["timestamp"], + episode_buffer["episode_index"], + ep_data_index_np, + self.fps, + self.tolerance_s, + ) + + video_files = list(self.root.rglob("*.mp4")) + assert len(video_files) == self.num_episodes * len(self.meta.video_keys) + + parquet_files = list(self.root.rglob("*.parquet")) + assert len(parquet_files) == self.num_episodes + + # delete images + img_dir = self.root / "images" + if img_dir.is_dir(): + shutil.rmtree(self.root / "images") + + if not episode_data: # Reset the buffer + self.episode_buffer = self.create_episode_buffer() + + def _save_episode_table(self, episode_buffer: dict, episode_index: int) -> None: + episode_dict = {key: episode_buffer[key] for key in self.hf_features} + ep_dataset = datasets.Dataset.from_dict(episode_dict, features=self.hf_features, split="train") + ep_dataset = embed_images(ep_dataset) + self.hf_dataset = concatenate_datasets([self.hf_dataset, ep_dataset]) + self.hf_dataset.set_transform(hf_transform_to_torch) + ep_data_path = self.root / self.meta.get_data_file_path(ep_index=episode_index) + ep_data_path.parent.mkdir(parents=True, exist_ok=True) + ep_dataset.to_parquet(ep_data_path) + + def clear_episode_buffer(self) -> None: + episode_index = self.episode_buffer["episode_index"] + if self.image_writer is not None: + for cam_key in self.meta.camera_keys: + img_dir = self._get_image_file_path( + episode_index=episode_index, image_key=cam_key, frame_index=0 + ).parent + if img_dir.is_dir(): + shutil.rmtree(img_dir) + + # Reset the buffer + self.episode_buffer = self.create_episode_buffer() + + # def start_image_writer(self, num_processes: int = 0, num_threads: int = 4) -> None: + # if isinstance(self.image_writer, AsyncImageWriter): + # logging.warning( + # "You are starting a new AsyncImageWriter that is replacing an already existing one in the dataset." + # ) + + # self.image_writer = AsyncImageWriter( + # num_processes=num_processes, + # num_threads=num_threads, + # ) + + def stop_image_writer(self) -> None: + """ + Whenever wrapping this dataset inside a parallelized DataLoader, this needs to be called first to + remove the image_writer in order for the LeRobotDataset object to be picklable and parallelized. + """ + if self.image_writer is not None: + self.image_writer.stop() + self.image_writer = None + + def _wait_image_writer(self) -> None: + """Wait for asynchronous image writer to finish.""" + if self.image_writer is not None: + self.image_writer.wait_until_done() + + def encode_videos(self) -> None: + """ + Use ffmpeg to convert frames stored as png into mp4 videos. + Note: `encode_video_frames` is a blocking call. Making it asynchronous shouldn't speedup encoding, + since video encoding with ffmpeg is already using multithreading. + """ + for ep_idx in range(self.meta.total_episodes): + self.encode_episode_videos(ep_idx) + + def encode_episode_videos(self, episode_index: int) -> dict: + """ + Use ffmpeg to convert frames stored as png into mp4 videos. + Note: `encode_video_frames` is a blocking call. Making it asynchronous shouldn't speedup encoding, + since video encoding with ffmpeg is already using multithreading. + """ + video_paths = {} + for key in self.meta.video_keys: + video_path = self.root / self.meta.get_video_file_path(episode_index, key) + video_paths[key] = str(video_path) + if video_path.is_file(): + # Skip if video is already encoded. Could be the case when resuming data recording. + continue + img_dir = self._get_image_file_path( + episode_index=episode_index, image_key=key, frame_index=0 + ).parent + encode_video_frames(img_dir, video_path, self.fps, overwrite=True, vcodec=self.video_codec) + + return video_paths + + @classmethod + def create( + cls, + repo_id: str, + fps: int, + features: dict, + root: str | Path | None = None, + robot_type: str | None = None, + use_videos: bool = True, + tolerance_s: float = 1e-4, + image_writer_processes: int = 0, + image_writer_threads: int = 0, + video_backend: str | None = None, + video_codec: Literal["h264", "hevc", "libsvtav1", "h264_nvenc"] = "libsvtav1", + is_compute_episode_stats_image = True, + ) -> "LeRobotDataset": + """Create a LeRobot Dataset from scratch in order to record data.""" + obj = cls.__new__(cls) + obj.meta = LeRobotDatasetMetadata.create( + repo_id=repo_id, + fps=fps, + robot_type=robot_type, + features=features, + root=root, + use_videos=use_videos, + ) + obj.repo_id = obj.meta.repo_id + obj.root = obj.meta.root + obj.revision = None + obj.tolerance_s = tolerance_s + obj.image_writer = None + + # if image_writer_processes or image_writer_threads: + # obj.start_image_writer(image_writer_processes, image_writer_threads) + + # TODO(aliberts, rcadene, alexander-soare): Merge this with OnlineBuffer/DataBuffer + obj.episode_buffer = obj.create_episode_buffer() + + obj.episodes = None + obj.hf_dataset = obj.create_hf_dataset() + obj.image_transforms = None + obj.delta_timestamps = None + obj.delta_indices = None + obj.episode_data_index = None + obj.video_backend = video_backend if video_backend is not None else get_safe_default_codec() + obj.video_codec = video_codec + obj.is_compute_episode_stats_image = is_compute_episode_stats_image + return obj + + +class MultiLeRobotDataset(torch.utils.data.Dataset): + """A dataset consisting of multiple underlying `LeRobotDataset`s. + + The underlying `LeRobotDataset`s are effectively concatenated, and this class adopts much of the API + structure of `LeRobotDataset`. + """ + + def __init__( + self, + dataset_dirs: list[str], + episodes: dict | None = None, + image_transforms: Callable | None = None, + delta_timestamps: dict[list[float]] | None = None, + tolerances_s: dict | None = None, + download_videos: bool = True, + video_backend: str | None = None, + ): + super().__init__() + self.dataset_dirs = dataset_dirs + ds_roots = [Path(ds_dir) for ds_dir in dataset_dirs] + ds_names = [ds_dir for ds_dir in dataset_dirs] + self.ds_names = ds_names + self.ds_roots = ds_roots + self.tolerances_s = tolerances_s if tolerances_s else dict.fromkeys(ds_names, 0.0001) + # Construct the underlying datasets passing everything but `transform` and `delta_timestamps` which + # are handled by this class. + self._datasets = [] + for ds_root, ds_name in zip(ds_roots, ds_names, strict=True): + try: + _dataset = LeRobotDataset( + ds_name, + root=ds_root, + episodes=episodes[ds_name] if episodes else None, + image_transforms=image_transforms, + delta_timestamps=delta_timestamps, + tolerance_s=self.tolerances_s[ds_name], + download_videos=download_videos, + video_backend=video_backend, + ) + self._datasets.append(_dataset) + except Exception as e: + # logging.error(e) + logging.error(f"Exception while process ds_root: {ds_root}, ds_name: {ds_name}") + traceback.print_exc() + raise e + + # Disable any data keys that are not common across all of the datasets. Note: we may relax this + # restriction in future iterations of this class. For now, this is necessary at least for being able + # to use PyTorch's default DataLoader collate function. + self.disabled_features = set() + intersection_features = set(self._datasets[0].features) + for ds in self._datasets: + intersection_features.intersection_update(ds.features) + if len(intersection_features) == 0: + raise RuntimeError( + "Multiple datasets were provided but they had no keys common to all of them. " + "The multi-dataset functionality currently only keeps common keys." + ) + for ds_name, ds in zip(self.ds_names, self._datasets, strict=True): + # Disable warning of empty extra keys do + extra_keys = set(ds.features).difference(intersection_features) + if extra_keys: + logging.warning( + f"keys {extra_keys} of {ds_name} were disabled as they are not contained in all the " + "other datasets." + ) + self.disabled_features.update(extra_keys) + + self.image_transforms = image_transforms + self.delta_timestamps = delta_timestamps + # TODO(rcadene, aliberts): We should not perform this aggregation for datasets + # with multiple robots of different ranges. Instead we should have one normalization + # per robot. + self.stats = aggregate_stats([dataset.meta.stats for dataset in self._datasets]) + + def set_during_training(self, during_training: bool): + for dataset in self._datasets: + dataset.during_training = during_training + + @property + def repo_id_to_index(self): + """Return a mapping from dataset repo_id to a dataset index automatically created by this class. + + This index is incorporated as a data key in the dictionary returned by `__getitem__`. + """ + return {repo_id: i for i, repo_id in enumerate(self.ds_names)} + + @property + def repo_index_to_id(self): + """Return the inverse mapping if repo_id_to_index.""" + return {v: k for k, v in self.repo_id_to_index} + + @property + def fps(self) -> int: + """Frames per second used during data collection. + + NOTE: Fow now, this relies on a check in __init__ to make sure all sub-datasets have the same info. + """ + return self._datasets[0].meta.info["fps"] + + @property + def video(self) -> bool: + """Returns True if this dataset loads video frames from mp4 files. + + Returns False if it only loads images from png files. + + NOTE: Fow now, this relies on a check in __init__ to make sure all sub-datasets have the same info. + """ + return self._datasets[0].meta.info.get("video", False) + + @property + def features(self) -> datasets.Features: + features = {} + for dataset in self._datasets: + features.update({k: v for k, v in dataset.hf_features.items() if k not in self.disabled_features}) + return features + + @property + def camera_keys(self) -> list[str]: + """Keys to access image and video stream from cameras.""" + keys = [] + for key, feats in self.features.items(): + if isinstance(feats, (datasets.Image, VideoFrame)): + keys.append(key) + return keys + + @property + def video_frame_keys(self) -> list[str]: + """Keys to access video frames that requires to be decoded into images. + + Note: It is empty if the dataset contains images only, + or equal to `self.cameras` if the dataset contains videos only, + or can even be a subset of `self.cameras` in a case of a mixed image/video dataset. + """ + video_frame_keys = [] + for key, feats in self.features.items(): + if isinstance(feats, VideoFrame): + video_frame_keys.append(key) + return video_frame_keys + + @property + def num_frames(self) -> int: + """Number of samples/frames.""" + return sum(d.num_frames for d in self._datasets) + + @property + def num_episodes(self) -> int: + """Number of episodes.""" + return sum(d.num_episodes for d in self._datasets) + + @property + def tolerance_s(self) -> float: + """Tolerance in seconds used to discard loaded frames when their timestamps + are not close enough from the requested frames. It is only used when `delta_timestamps` + is provided or when loading video frames from mp4 files. + """ + # 1e-4 to account for possible numerical error + return 1 / self.fps - 1e-4 + + def get_episode_data(self, episode_idx: int) -> dict: + for dataset in self._datasets: + if episode_idx < dataset.num_episodes: + file = str(dataset.root / dataset.meta.get_data_file_path(dataset.episodes[episode_idx])) + table = pq.read_table(str(file)) + + result_dict = {} + for col_name in table.column_names: + col = table[col_name] + try: + np_arr = col.to_numpy(zero_copy_only=True) + except Exception: + raw = col.to_numpy() + np_arr = np.stack(raw) if raw.dtype == object else raw + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="The given NumPy array is not writable", + category=UserWarning, + ) + # deal with string in parquet file + if np_arr.dtype == 'O': + result_dict[col_name] = np_arr + else: + result_dict[col_name] = torch.from_numpy(np_arr) + return result_dict + else: + episode_idx -= dataset.num_episodes + raise IndexError(f"Episode index {episode_idx} out of bounds.") + + def __len__(self): + return self.num_frames + + def __getitem__(self, idx: int) -> dict[str, torch.Tensor]: + if idx >= len(self): + raise IndexError(f"Index {idx} out of bounds.") + # Determine which dataset to get an item from based on the index. + start_idx = 0 + dataset_idx = 0 + for dataset in self._datasets: + if idx >= start_idx + dataset.num_frames: + start_idx += dataset.num_frames + dataset_idx += 1 + continue + break + else: + raise AssertionError("We expect the loop to break out as long as the index is within bounds.") + item = self._datasets[dataset_idx][idx - start_idx] + item["dataset_index"] = torch.tensor(dataset_idx) + for data_key in self.disabled_features: + if data_key in item: + del item[data_key] + + return item + + def __repr__(self): + return ( + f"{self.__class__.__name__}(\n" + f" Dataset Names: '{self.ds_names}',\n" + f" Number of Samples: {self.num_frames},\n" + f" Number of Episodes: {self.num_episodes},\n" + f" Type: {'video (.mp4)' if self.video else 'image (.png)'},\n" + f" Recorded Frames per Second: {self.fps},\n" + f" Camera Keys: {self.camera_keys},\n" + f" Video Frame Keys: {self.video_frame_keys if self.video else 'N/A'},\n" + f" Transformations: {self.image_transforms},\n" + f")" + ) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/base_processor.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/base_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..39d1665b6f0da68060fad7a120241577301a6139 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/base_processor.py @@ -0,0 +1,263 @@ +from abc import ABC, abstractmethod +from typing import Dict, Any, Optional, List, Literal + +import torch +import numpy as np +from copy import deepcopy +from ..utils.normalizer import LinearNormalizer, NormMode +from fastwam.utils.pytorch_utils import dict_apply + + +class BaseProcessor(ABC): + def __init__( + self, + # keys + shape_meta: Dict[str, Any], + num_obs_steps: int, + num_output_cameras: int, + action_output_dim: int, + proprio_output_dim: int, + + action_state_transforms: Optional[List[Any]], + + # action & state normalization + use_stepwise_action_norm: bool, + norm_default_mode: NormMode, + norm_exception_mode: Dict[str, Dict[str, NormMode]], + + action_state_merger, + + # image transform + train_transforms: Dict[str, List[Any]] | None, + val_transforms: Dict[str, List[Any]] | None, + + # instruction transform + drop_high_level_prob: float, + use_zh_instruction: bool, + + tokenizer: Any + ): + self.shape_meta = shape_meta + self.num_obs_steps = num_obs_steps + self.num_output_cameras = num_output_cameras + self.action_output_dim = action_output_dim + self.proprio_output_dim = proprio_output_dim + + self.drop_high_level_prob = drop_high_level_prob + self.use_zh_instruction = use_zh_instruction + + # image + self.train_transforms = train_transforms + self.val_transforms = val_transforms + + self._is_train = None + + self.action_state_transforms = action_state_transforms + self.action_state_merger = action_state_merger + self.action_state_merger.set_shape_meta(self.shape_meta) + + self.use_stepwise_action_norm = use_stepwise_action_norm + self.norm_default_mode = norm_default_mode + self.norm_exception_mode = norm_exception_mode + self._normalizer = None + + self.tokenizer = tokenizer + + @property + def is_train(self): + if self._is_train is None: + raise ValueError("is_train has not been set. Please call train() and eval() first.") + return self._is_train + + @property + def normalizer(self) -> LinearNormalizer: + if self._normalizer is None: + raise ValueError("normalizer has not been set. Please call set_normalizer_from_stats() first.") + return self._normalizer + + def train(self): + self._is_train = True + return self + + def eval(self): + self._is_train = False + return self + + def set_normalizer_from_stats(self, dataset_stats: Dict[str, Any] = None): + self._normalizer = LinearNormalizer( + use_stepwise_action_norm=self.use_stepwise_action_norm, + shape_meta=self.shape_meta, + default_mode=self.norm_default_mode, + exception_mode=self.norm_exception_mode, + stats=dataset_stats, + ) + + def augment_instruction(self, data: Dict[str, str] | List[str]) -> List[str]: + """ + Args: + data: Dict[str, str] | List[str], lerobot sample in raw mcap + + Returns: + List[str], processed instructions + """ + # if single instruction, convert to list + if "coarse_task" in data: + high_level_instruction = data["coarse_task"] + else: + high_level_instruction = "" + if "task" not in data: + return f"[high] {high_level_instruction}" + + low_level_instruction = data["task"] + # Galaxea lerobot use @ to split Chinese and English instruction + if "@" in low_level_instruction: + zh, eng = low_level_instruction.split("@") + low_level_instruction = zh if self.use_zh_instruction else eng + + if np.random.rand() < self.drop_high_level_prob: + instruction = f"[Low]: {low_level_instruction}" + else: + instruction = f"[High]: {high_level_instruction}, [Low]: {low_level_instruction}" + + return instruction + + def action_state_transform(self, batch): + if "action" in batch: + for meta in self.shape_meta["action"]: + k, meta_shape = meta["key"], meta["raw_shape"] + actual_shape = batch["action"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"Action key {k} actual raw shape {actual_shape} mismatch with meta raw shape {meta_shape}." + + for meta in self.shape_meta["state"]: + k, meta_shape = meta["key"], meta["raw_shape"] + actual_shape = batch["state"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"State key {k} actual raw shape {actual_shape} mismatch with meta raw shape{meta_shape}." + + if self.action_state_transforms is not None: + for trans in self.action_state_transforms: + batch = trans.forward(batch) + + if "action" in batch: + for meta in self.shape_meta["action"]: + k, meta_shape = meta["key"], meta["shape"] + actual_shape = batch["action"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"Action key {k} actual transformed shape {actual_shape} mismatch with meta shape {meta_shape}." + + for meta in self.shape_meta["state"]: + k, meta_shape = meta["key"], meta["shape"] + actual_shape = batch["state"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"State key {k} actual transformed shape {actual_shape} mismatch with meta raw shape {meta_shape}." + + return batch + + def preprocess(self, data: Dict[str, Any]) -> Dict[str, Any]: + """ + Preprocess the data for the policy model. + + Args: + Data: Dict[str, Any], lerobot sample in raw mcap obtained from dataset __getitem__: + - "action": Optional, Dict[str, torch.Tensor] -> [action_horizon, action_dim] + - "state": Dict[str, torch.Tensor] -> [num_obs_steps, state_dim] + - "images": Dict[str, torch.Tensor] -> [num_obs_steps, C, H, W] + - "action_is_pad": Optional, torch.Tensor -> [action_horizon,] + - "state_is_pad": torch.Tensor -> [num_obs_steps,] + - "image_is_pad": torch.Tensor -> [num_obs_steps,] + - "idx": int, sample index + + Returns: + Sample: Dict[str, Any], which can collated: + - "input_ids": torch.Tensor -> [max_image_text_tokens,] + - "attention_mask": torch.Tensor -> [max_image_text_tokens,] + - "pixel_values": torch.Tensor -> [num_input_cameras, C, H, W] + - "image_is_pad": torch.Tensor -> [num_obs_steps,] + - "proprio": torch.Tensor -> [num_obs_steps, proprio_dim] + - "state_is_pad": torch.Tensor -> [num_obs_steps,] + - "action": Optional, torch.Tensor -> [action_horizon, action_dim] + - "action_is_pad": Optional, torch.Tensor -> [action_horizon,] + - "gt_action: Optional, deepcopy of input action for open loop eval, which is left untouched + - "idx": int, sample index + """ + sample = {} + # 1. instruction + sample["instruction"] = self.augment_instruction(data) + sample["image_is_pad"] = data["image_is_pad"] + + # 2. image + processed_images = [] + for meta in self.shape_meta["images"]: + key, shape = meta["key"], meta["shape"] + image = data["images"][key] # [num_obs_steps, C, H, W] + assert image.ndim == 4, f"Expected 4 dimensions (num_obs_steps, C, H, W), got shape {image.shape}" + + # Apply transforms efficiently on the merged batch + transforms = self.train_transforms if self.is_train else self.val_transforms + for trans in transforms[key]: + image = trans(image) + + meta_shape = [self.num_obs_steps] + shape + assert image.shape == meta_shape, \ + f"Expected shape {meta_shape}, got {image.shape} after transforms for key {key}" + + processed_images.append(image) + + pixel_values = torch.cat(processed_images, dim=0) # [num_input_cameras, C, H, W] + if self.num_output_cameras > pixel_values.shape[0]: + out = torch.zeros((self.num_output_cameras,) + pixel_values.shape[1:], device=pixel_values.device, dtype=pixel_values.dtype) + out[0: pixel_values.shape[0]] = pixel_values + sample["pixel_values"] = out + else: + sample["pixel_values"] = pixel_values + + # Copy action before transform for open-loop evaluation, + # disabled for training dataset as it may cause collating key problem. + if not self.is_train and "action" in data: + sample["gt_action"] = deepcopy(data["action"]) + + # 3. action & state + data = self.action_state_transform(data) + data = self.normalizer.forward(data) + data = self.action_state_merger.forward(data) + + if "action" in data: + sample["action"] = data["action"] # [action_horizon, action_dim] + sample["action_is_pad"] = data["action_is_pad"] # [action_horizon,] + sample["action_dim_is_pad"] = data["action_dim_is_pad"] # [action_dim,] + assert sample["action"].shape[-1] == self.action_output_dim + + # TODO: rename all "state" into "proprio" + sample["proprio"] = data["state"] # [num_obs_steps, proprio_dim] + sample["proprio_is_pad"] = data["state_is_pad"] # [num_obs_steps,] + sample["proprio_dim_is_pad"] = data["state_dim_is_pad"] # [proprio_dim,] + assert sample["proprio"].shape[-1] == self.proprio_output_dim + + sample["idx"] = data["idx"] + + sample = self.tokenizer(sample) + + return sample + + def postprocess(self, data: Dict[str, Any]) -> Dict[str, Any]: + """ + Postprocess the data for the policy model. + + Args: + data: Dict[str, Any], lerobot sample in raw mcap + + Returns: + data: Dict[str, Any], processed data including unnormalized action + """ + assert "action" in data, "Action is required in postprocess" + data["state"] = data.pop("proprio") + data = self.action_state_merger.backward(data) + data = self.normalizer.backward(data) + if self.action_state_transforms is not None: + for trans in reversed(self.action_state_transforms): + data = trans.backward(data) + + start_obs_step = self.num_obs_steps - 1 + data["action"] = dict_apply(data["action"], lambda x: x[:, start_obs_step:, :]) + return data diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/fastwam_processor.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/fastwam_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..881ee1d4bf23b40ca9ebc8453d54ecf119a72bf7 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/fastwam_processor.py @@ -0,0 +1,304 @@ +from abc import ABC, abstractmethod +from typing import Dict, Any, Optional, List, Literal + +import torch +import numpy as np +from copy import deepcopy +from ..utils.normalizer import LinearNormalizer, NormMode +from fastwam.utils.pytorch_utils import dict_apply +from fastwam.utils.logging_config import get_logger +from .base_processor import BaseProcessor + +logger = get_logger(__name__) + +class FastWAMProcessor(BaseProcessor): + def __init__( + self, + # keys + shape_meta: Dict[str, Any], + num_obs_steps: int, + num_output_cameras: int, + action_output_dim: int, + proprio_output_dim: int, + + action_state_transforms: Optional[List[Any]], + + # action & state normalization + use_stepwise_action_norm: bool, + norm_default_mode: NormMode, + norm_exception_mode: Dict[str, Dict[str, NormMode]], + + action_state_merger, + + # image transform + train_transforms: Dict[str, List[Any]] | None, + val_transforms: Dict[str, List[Any]] | None, + + # instruction transform + drop_high_level_prob: float = 1.0, + use_zh_instruction: bool = False, + + tokenizer: Optional[Any] = None, + delta_action_dim_mask: Optional[Dict[str, List[bool]]] = None, + ): + self.shape_meta = shape_meta + self.num_obs_steps = num_obs_steps + self.num_output_cameras = num_output_cameras + self.action_output_dim = action_output_dim + self.proprio_output_dim = proprio_output_dim + + self.drop_high_level_prob = drop_high_level_prob + self.use_zh_instruction = use_zh_instruction + + # image + self.train_transforms = train_transforms + self.val_transforms = val_transforms + + self._is_train = None + + self.action_state_transforms = action_state_transforms + self.action_state_merger = action_state_merger + self.action_state_merger.set_shape_meta(self.shape_meta) + + self.use_stepwise_action_norm = use_stepwise_action_norm + self.norm_default_mode = norm_default_mode + self.norm_exception_mode = norm_exception_mode + self._normalizer = None + + self.tokenizer = tokenizer + if delta_action_dim_mask is None: + self.delta_action_dim_mask = None + else: + action_meta = self.shape_meta["action"] + expected_keys = [m["key"] for m in action_meta] + provided_keys = list(delta_action_dim_mask.keys()) + if set(provided_keys) != set(expected_keys): + raise ValueError( + f"`delta_action_dim_mask` keys mismatch. Expected {expected_keys}, got {provided_keys}." + ) + + self.delta_action_dim_mask = {} + for meta in action_meta: + key = meta["key"] + expected_dim = meta["shape"] + mask = delta_action_dim_mask[key] + if len(mask) != expected_dim: + raise ValueError( + f"`delta_action_dim_mask[{key}]` length must be {expected_dim}, got {len(mask)}." + ) + self.delta_action_dim_mask[key] = torch.as_tensor(mask, dtype=torch.bool) + + @property + def is_train(self): + if self._is_train is None: + raise ValueError("is_train has not been set. Please call train() and eval() first.") + return self._is_train + + @property + def normalizer(self) -> LinearNormalizer: + if self._normalizer is None: + raise ValueError("normalizer has not been set. Please call set_normalizer_from_stats() first.") + return self._normalizer + + def train(self): + self._is_train = True + return self + + def eval(self): + self._is_train = False + return self + + def set_normalizer_from_stats(self, dataset_stats: Dict[str, Any] = None): + self._normalizer = LinearNormalizer( + use_stepwise_action_norm=self.use_stepwise_action_norm, + shape_meta=self.shape_meta, + default_mode=self.norm_default_mode, + exception_mode=self.norm_exception_mode, + stats=dataset_stats, + ) + + def augment_instruction(self, data: Dict[str, str] | List[str]) -> List[str]: + """ + Args: + data: Dict[str, str] | List[str], lerobot sample in raw mcap + + Returns: + List[str], processed instructions + """ + # if single instruction, convert to list + if "coarse_task" in data: + high_level_instruction = data["coarse_task"] + else: + high_level_instruction = "" + if "task" not in data: + return f"[high] {high_level_instruction}" + + low_level_instruction = data["task"] + # Galaxea lerobot use @ to split Chinese and English instruction + if "@" in low_level_instruction: + zh, eng = low_level_instruction.split("@") + low_level_instruction = zh if self.use_zh_instruction else eng + + if np.random.rand() < self.drop_high_level_prob: + instruction = f"{low_level_instruction}" + else: + instruction = f"[High]: {high_level_instruction}, [Low]: {low_level_instruction}" + + return instruction + + def action_state_transform(self, batch): + if "action" in batch: + for meta in self.shape_meta["action"]: + k, meta_shape = meta["key"], meta["raw_shape"] + actual_shape = batch["action"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"Action key {k} actual raw shape {actual_shape} mismatch with meta raw shape {meta_shape}." + + for meta in self.shape_meta["state"]: + k, meta_shape = meta["key"], meta["raw_shape"] + actual_shape = batch["state"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"State key {k} actual raw shape {actual_shape} mismatch with meta raw shape {meta_shape}." + + if self.action_state_transforms is not None: + for trans in self.action_state_transforms: + batch = trans.forward(batch) + + if "action" in batch: + for meta in self.shape_meta["action"]: + k, meta_shape = meta["key"], meta["shape"] + actual_shape = batch["action"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"Action key {k} actual transformed shape {actual_shape} mismatch with meta shape {meta_shape}." + + for meta in self.shape_meta["state"]: + k, meta_shape = meta["key"], meta["shape"] + actual_shape = batch["state"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"State key {k} actual transformed shape {actual_shape} mismatch with meta raw shape {meta_shape}." + + return batch + + def preprocess(self, data: Dict[str, Any]) -> Dict[str, Any]: + """ + Preprocess the data for the policy model. + + Args: + Data: Dict[str, Any], lerobot sample in raw mcap obtained from dataset __getitem__: + - "action": Optional, Dict[str, torch.Tensor] -> [action_horizon, action_dim] + - "state": Dict[str, torch.Tensor] -> [num_obs_steps, state_dim] + - "images": Dict[str, torch.Tensor] -> [num_obs_steps, C, H, W] + - "action_is_pad": Optional, torch.Tensor -> [action_horizon,] + - "state_is_pad": torch.Tensor -> [num_obs_steps,] + - "image_is_pad": torch.Tensor -> [num_obs_steps,] + - "idx": int, sample index + + Returns: + Sample: Dict[str, Any], which can collated: + - "input_ids": torch.Tensor -> [max_image_text_tokens,] + - "attention_mask": torch.Tensor -> [max_image_text_tokens,] + - "pixel_values": torch.Tensor -> [num_input_cameras, C, H, W] + - "image_is_pad": torch.Tensor -> [num_obs_steps,] + - "proprio": torch.Tensor -> [num_obs_steps, proprio_dim] + - "state_is_pad": torch.Tensor -> [num_obs_steps,] + - "action": Optional, torch.Tensor -> [action_horizon, action_dim] + - "action_is_pad": Optional, torch.Tensor -> [action_horizon,] + - "gt_action: Optional, deepcopy of input action for open loop eval, which is left untouched + - "idx": int, sample index + """ + sample = {} + # 1. instruction + sample["instruction"] = self.augment_instruction(data) + sample["image_is_pad"] = data["image_is_pad"] + + # 2. image + processed_images = [] + for meta in self.shape_meta["images"]: + key, shape = meta["key"], meta["shape"] + image = data["images"][key] # [num_obs_steps, C, H, W] + assert image.ndim == 4, f"Expected 4 dimensions (num_obs_steps, C, H, W), got shape {image.shape}" + + # Apply transforms efficiently on the merged batch + transforms = self.train_transforms if self.is_train else self.val_transforms + current_transforms = transforms[key] if isinstance(transforms, dict) else transforms + for trans in current_transforms: + image = trans(image) + + meta_shape = [self.num_obs_steps] + shape + assert image.shape == meta_shape, \ + f"Expected shape {meta_shape}, got {image.shape} after transforms for key {key}" + + processed_images.append(image) + pixel_values = torch.stack(processed_images, dim=0) # [num_input_cameras, T, C, H, W] + + if self.num_output_cameras > pixel_values.shape[0]: + out = torch.zeros((self.num_output_cameras,) + pixel_values.shape[1:], device=pixel_values.device, dtype=pixel_values.dtype) + out[0: pixel_values.shape[0]] = pixel_values + sample["pixel_values"] = out + elif self.num_output_cameras < pixel_values.shape[0]: + logger.warning(f"num_output_cameras {self.num_output_cameras} is less than the number of cameras in data {pixel_values.shape[0]}, " + f"truncating the input to the first {self.num_output_cameras} cameras.") + sample["pixel_values"] = pixel_values[:self.num_output_cameras] + else: + sample["pixel_values"] = pixel_values + + # Copy action before transform for open-loop evaluation, + # disabled for training dataset as it may cause collating key problem. + if not self.is_train and "action" in data: + sample["gt_action"] = deepcopy(data["action"]) + + # 3. action & state + if "action" in data and self.delta_action_dim_mask is not None: + action_is_pad = torch.as_tensor(data["action_is_pad"], dtype=torch.bool) + if bool(action_is_pad.any().item()): + for key, dim_mask in self.delta_action_dim_mask.items(): + cur_action = data["action"][key] + cur_action_is_pad = action_is_pad.to(device=cur_action.device) + cur_dim_mask = dim_mask.to(device=cur_action.device) + pad_delta_mask = cur_action_is_pad.unsqueeze(1) & cur_dim_mask.unsqueeze(0) + cur_action[pad_delta_mask] = 0.0 + data = self.action_state_transform(data) + data = self.normalizer.forward(data) + data = self.action_state_merger.forward(data) + + if "action" in data: + sample["action"] = data["action"] # [action_horizon, action_dim] + sample["action_is_pad"] = data["action_is_pad"] # [action_horizon,] + sample["action_dim_is_pad"] = data["action_dim_is_pad"] # [action_dim,] + assert sample["action"].shape[-1] == self.action_output_dim + # sample["action"][sample["action_is_pad"], :-1] = 0.0 # NOTE: we assume use delta_eef_pose + gripper, so pad action is 0 + + + # TODO: rename all "state" into "proprio" + sample["proprio"] = data["state"] # [num_obs_steps, proprio_dim] + sample["proprio_is_pad"] = data["state_is_pad"] # [num_obs_steps,] + sample["proprio_dim_is_pad"] = data["state_dim_is_pad"] # [proprio_dim,] + assert sample["proprio"].shape[-1] == self.proprio_output_dim + + sample["idx"] = data["idx"] + + # sample = self.tokenizer(sample) + + return sample + + def postprocess(self, data: Dict[str, Any]) -> Dict[str, Any]: + """ + Postprocess the data for the policy model. + + Args: + data: Dict[str, Any], lerobot sample in raw mcap + + Returns: + data: Dict[str, Any], processed data including unnormalized action + """ + assert "action" in data, "Action is required in postprocess" + data["state"] = data.pop("proprio") + data = self.action_state_merger.backward(data) + data = self.normalizer.backward(data) + if self.action_state_transforms is not None: + for trans in reversed(self.action_state_transforms): + data = trans.backward(data) + + start_obs_step = self.num_obs_steps - 1 + data["action"] = dict_apply(data["action"], lambda x: x[:, start_obs_step:, :]) + return data diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/robot_video_dataset.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/robot_video_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..fb2096841ff73b1870ea6fd58d828460c2918509 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/robot_video_dataset.py @@ -0,0 +1,279 @@ +import hashlib +import os +from typing import Optional +import time +import numpy as np +import traceback +import torch +import torchvision.transforms.functional as transforms_F +from contextlib import contextmanager + +from omegaconf import DictConfig, OmegaConf + +from hydra.utils import instantiate +from .base_lerobot_dataset import BaseLerobotDataset +from .utils.normalizer import save_dataset_stats_to_json, load_dataset_stats_from_json +from ..dataset_utils import ResizeSmallestSideAspectPreserving, CenterCrop, Normalize +from fastwam.utils.logging_config import get_logger +from fastwam.utils import misc, pytorch_utils +from accelerate import PartialState +logger = get_logger(__name__) + + +DEFAULT_PROMPT = "A video recorded from a robot's point of view executing the following instruction: {task}" + +class RobotVideoDataset(torch.utils.data.Dataset): + def __init__( + self, + dataset_dirs, + shape_meta, + num_frames=33, + video_size=[384, 640], + camera_key=None, + processor=None, + text_embedding_cache_dir=None, + context_len=128, + pretrained_norm_stats=None, + val_set_proportion=0.05, + is_training_set=False, + global_sample_stride=1, + action_video_freq_ratio: int = 1, + skip_padding_as_possible: bool = False, + max_padding_retry: int = 3, + concat_multi_camera: str = "horizontal", # "horizontal", "vertical", "robotwin", or None + override_instruction: Optional[str] = None, # whether to hardcode a specific instruction for all samples, for debugging + ): + self.lerobot_dataset = BaseLerobotDataset( + dataset_dirs=dataset_dirs, + shape_meta=OmegaConf.to_container(shape_meta, resolve=True), + obs_size=num_frames, + action_size=num_frames - 1, + val_set_proportion=val_set_proportion, + is_training_set=is_training_set, + global_sample_stride=global_sample_stride, + ) + + self.num_frames = num_frames + self.action_video_freq_ratio = action_video_freq_ratio + + assert (num_frames - 1) % self.action_video_freq_ratio == 0, \ + f"num_frames-1 must be divisible by action_video_freq_ratio, got {num_frames - 1} and {self.action_video_freq_ratio}" + assert ((num_frames - 1) // self.action_video_freq_ratio) % 4 == 0, \ + f"video frames must be divisible by 4 for tokenization, got {(num_frames - 1) // self.action_video_freq_ratio}" + self.video_sample_indices = list(range(0, num_frames, self.action_video_freq_ratio)) + + self.camera_key = camera_key + self.lerobot_dataset._set_return_images(True) + + self.video_size = video_size + self.text_embedding_cache_dir = text_embedding_cache_dir + self.context_len = context_len + self.skip_padding_as_possible = skip_padding_as_possible + self.max_padding_retry = max_padding_retry + self.concat_multi_camera = concat_multi_camera + self.override_instruction = override_instruction + + self.resize_transform = ResizeSmallestSideAspectPreserving( + args={"img_w": self.video_size[1], "img_h": self.video_size[0]}, + ) + self.crop_transform = CenterCrop( + args={"img_w": self.video_size[1], "img_h": self.video_size[0]}, + ) + self.normalize_transform = Normalize( + args={"mean": 0.5, "std": 0.5}, + ) + if processor is not None: + if isinstance(processor, DictConfig): + processor = instantiate(processor) + if not pretrained_norm_stats: + if not is_training_set: + raise ValueError("pretrained_norm_stats must be provided for validation/test sets since we don't want to calculate stats on them.") + if PartialState().is_main_process: + logger.info("Calculating dataset stats for normalization...") + dataset_stats = self.lerobot_dataset.get_dataset_stats(processor) + work_dir = misc.get_work_dir() + save_dataset_stats_to_json(dataset_stats, os.path.join(work_dir, "dataset_stats.json")) + else: + dataset_stats = None + if torch.distributed.is_available() and torch.distributed.is_initialized(): + obj_list = [dataset_stats] + torch.distributed.broadcast_object_list(obj_list, src=0) + dataset_stats = obj_list[0] + else: + dataset_stats = load_dataset_stats_from_json(pretrained_norm_stats) + logger.info(f"Using dataset stats: {pretrained_norm_stats}") + if PartialState().is_main_process: + work_dir = misc.get_work_dir() + save_dataset_stats_to_json(dataset_stats, os.path.join(work_dir, "dataset_stats.json")) + + processor.set_normalizer_from_stats(dataset_stats) + self.lerobot_dataset.set_processor(processor) + + def __len__(self): + return len(self.lerobot_dataset) + + def _get(self, idx): + sample_idx = idx + sample = None + for attempt in range(self.max_padding_retry + 1): + sample = self.lerobot_dataset[sample_idx] + + if not self.skip_padding_as_possible: + break + + action_is_pad = sample["action_is_pad"] + image_is_pad = sample["image_is_pad"] + proprio_is_pad = sample["proprio_is_pad"] + has_pad = False + if bool(action_is_pad.any().item()): + has_pad = True + if bool(image_is_pad.any().item()): + has_pad = True + if bool(proprio_is_pad.any().item()): + has_pad = True + + if not has_pad or attempt >= self.max_padding_retry: + break + + sample_idx = np.random.randint(len(self.lerobot_dataset)) + + image_is_pad = sample["image_is_pad"] + + video = sample["pixel_values"] # [T, C, H, W] or [num_cameras, T, C, H, W] + num_cameras = 1 + if video.ndim == 5: + video = video[:, self.video_sample_indices, :, :, :] # [num_cameras, T_video, C, H, W] + num_cameras, T_video, C, H, W = video.shape + else: + assert video.ndim == 4, f"Expected video to have shape [T, C, H, W], but got {video.shape}" + video = video[self.video_sample_indices, :, :, :] # [T_video, C, H, W] + T_video, C, H, W = video.shape + image_is_pad = image_is_pad[self.video_sample_indices] + + video = video.view(num_cameras, T_video, C, H, W) # [num_cameras, T_video, C, H, W] + if self.concat_multi_camera == "robotwin": + if num_cameras != 3: + raise ValueError( + f"`concat_multi_camera='robotwin'` requires exactly 3 cameras, got {num_cameras}" + ) + cam_top = transforms_F.resize( + video[0], + size=[256, 320], + interpolation=transforms_F.InterpolationMode.BILINEAR, + antialias=True, + ) # [T_video, C, 256, 320] + cam_left = transforms_F.resize( + video[1], + size=[128, 160], + interpolation=transforms_F.InterpolationMode.BILINEAR, + antialias=True, + ) # [T_video, C, 128, 160] + cam_right = transforms_F.resize( + video[2], + size=[128, 160], + interpolation=transforms_F.InterpolationMode.BILINEAR, + antialias=True, + ) # [T_video, C, 128, 160] + bottom = torch.cat([cam_left, cam_right], dim=-1) # [T_video, C, 128, 320] + video = torch.cat([cam_top, bottom], dim=-2) # [T_video, C, 384, 320] + elif num_cameras > 1: + if self.concat_multi_camera == "horizontal": + video = torch.cat([video[i] for i in range(num_cameras)], dim=-1) # [T_video, C, H, num_cameras*W] + elif self.concat_multi_camera == "vertical": + video = torch.cat([video[i] for i in range(num_cameras)], dim=-2) # [T_video, C, num_cameras*H, W] + else: + raise ValueError( + f"Invalid concat_multi_camera: {self.concat_multi_camera}. " + "Expected one of: horizontal, vertical, robotwin." + ) + else: + video = video.squeeze(0) # [T_video, C, H, W] + + # final resize and normalization + video = self.resize_transform(video) + video = self.crop_transform(video) + video = self.normalize_transform(video) # [T_video, C, H, W] + + video = video.permute(1, 0, 2, 3) # [C, T_video, H, W], range [-1, 1] + + # Proxy (from lerobot): + # action: [num_frames-1, action_dim] # start from t0, except the last frame + # proprio: [num_frames, proprio_dim] # start from t0 to the last frame, aligned with video frames + action = sample["action"] # [T-1, action_dim] + proprio = sample["proprio"][:-1, :] # [T-1, state_dim], to align with action + if video.shape[1] <= 1: + raise ValueError(f"`video` must have at least 2 frames, got shape {tuple(video.shape)}") + if action.shape[0] % (video.shape[1] - 1) != 0: + raise ValueError( + f"`action` horizon must be divisible by `video` transitions, got {action.shape[0]} and {video.shape[1] - 1}" + ) + + task = sample["instruction"] + + # FIXME + if self.override_instruction is not None: + task = self.override_instruction + instruction = DEFAULT_PROMPT.format(task=task) + + context, context_mask = self._get_cached_text_context(instruction) + # NOTE: to keep consistent with wan2.2's behavior + context[~context_mask] = 0.0 + context_mask = torch.ones_like(context_mask) + + data = { + "video": video, + "action": action, + "proprio": proprio, + "prompt": instruction, + "context": context, + "context_mask": context_mask, + "image_is_pad": image_is_pad, + "action_is_pad": sample["action_is_pad"], + "proprio_is_pad": sample["proprio_is_pad"], + } + return data + + def _get_cached_text_context(self, prompt: str): + if self.text_embedding_cache_dir is None: + raise ValueError("text_embedding_cache_dir is not set.") + cache_dir = self.text_embedding_cache_dir + os.makedirs(cache_dir, exist_ok=True) + hashed = hashlib.sha256(prompt.encode("utf-8")).hexdigest() + cache_path = os.path.join(cache_dir, f"{hashed}.t5_len{self.context_len}.wan22ti2v5b.pt") + if not os.path.exists(cache_path): + raise FileNotFoundError( + f"Missing text embedding cache: {cache_path}. " + "Run scripts/precompute_text_embeds.py first." + ) + payload = torch.load(cache_path, map_location="cpu") + context = payload["context"] + context_mask = payload["mask"].bool() + if context.ndim != 2: + raise ValueError( + f"Cached `context` must be 2D [L, D], got shape {tuple(context.shape)} in {cache_path}" + ) + if context_mask.ndim != 1: + raise ValueError( + f"Cached `mask` must be 1D [L], got shape {tuple(context_mask.shape)} in {cache_path}" + ) + if context.shape[0] != self.context_len: + raise ValueError( + f"Cached context_len mismatch: expected {self.context_len}, got {context.shape[0]} in {cache_path}" + ) + if context_mask.shape[0] != self.context_len: + raise ValueError( + f"Cached mask_len mismatch: expected {self.context_len}, got {context_mask.shape[0]} in {cache_path}" + ) + + return context, context_mask + + def __getitem__(self, idx): + try: + data = self._get(idx) + except Exception as e: + print(f"Error processing sample idx {idx}: {e}. Returning a random sample instead.") + # trace back + print(traceback.format_exc()) + random_idx = np.random.randint(len(self)) + data = self._get(random_idx) + return data diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/action_state_merger.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/action_state_merger.py new file mode 100644 index 0000000000000000000000000000000000000000..0a35644a4fbc65a5cb6abe3f642f4af35a4e438b --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/action_state_merger.py @@ -0,0 +1,78 @@ +from typing import Dict + +import torch +from torch.nn.functional import pad + + +class ConcatLeftAlign: + def __init__( + self, + action_target_dim: int | None = None, + state_target_dim: int | None = None + ): + self.action_target_dim = action_target_dim + self.state_target_dim = state_target_dim + + def set_shape_meta(self, shape_meta): + self.action_meta = shape_meta["action"] + self.state_meta = shape_meta["state"] + + def forward(self, batch): + if "action" in batch: + batch["action"] = self._concat(batch["action"], self.action_meta) + batch["action"], batch["action_dim_is_pad"] = self._pad(batch["action"], self.action_target_dim) + + batch["state"] = self._concat(batch["state"], self.state_meta) + batch["state"], batch["state_dim_is_pad"] = self._pad(batch["state"], self.state_target_dim) + + return batch + + def backward(self, batch): + if self.state_target_dim is not None: + assert batch["state"].shape[-1] == self.state_target_dim + batch["state"] = self._crop(batch["state"], self.state_meta) + batch["state"] = self._split(batch["state"], self.state_meta) + + if self.action_target_dim is not None: + assert batch["action"].shape[-1] == self.action_target_dim + batch["action"] = self._crop(batch["action"], self.action_meta) + batch["action"] = self._split(batch["action"], self.action_meta) + + return batch + + @staticmethod + def _pad(x: torch.Tensor, dim: int): + if dim is None: + dim = x.shape[-1] + + assert x.ndim == 2 and x.shape[-1] <= dim + pad_dim = dim - x.shape[-1] + x_padded = pad(x, (0, pad_dim)) + mask = torch.zeros_like(x[0]).bool() + mask = pad(mask, (0, pad_dim), value=True) + return x_padded, mask + + @staticmethod + def _crop(x: torch.Tensor, meta: int): + assert x.ndim == 3 + dim = sum([m["shape"] for m in meta]) + x = x[:, :, :dim] + return x + + @staticmethod + def _concat(x: Dict[str, torch.Tensor], meta: Dict[str, Dict]): + x = torch.cat([x[m["key"]] for m in meta], dim=-1) + assert x.ndim == 2 + return x + + @staticmethod + def _split(x: torch.Tensor, meta: Dict[str, Dict]): + assert x.ndim == 3 + y = {} + idx = 0 + for m in meta: + key, dim = m["key"], m["shape"] + y[key] = x[:, :, idx: idx + dim] + idx += dim + + return y \ No newline at end of file diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/image.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/image.py new file mode 100644 index 0000000000000000000000000000000000000000..8dbeda01c09dd57bc772ad4d53cf8b49586a2d06 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/image.py @@ -0,0 +1,25 @@ +import torch +import torch.nn as nn +import torchvision.transforms as TF + + +class ToTensor(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x: torch.Tensor): + assert x.dtype == torch.uint8 + x = x.to(torch.float32) / 255.0 + return x + +class Pad(nn.Module): + def __init__(self, padding, fill=0, padding_mode='constant'): + super().__init__() + self.padding = padding + self.fill = fill + self.padding_mode = padding_mode + self.pad = TF.Pad(padding=tuple(padding), fill=fill, padding_mode=padding_mode) + + def forward(self, x: torch.Tensor): + assert x.ndim == 4, "Can only pad tensor of 4 dims." + return self.pad(x) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/misc.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..49279d6888f565078aafbcc3a830584bb34515a5 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/misc.py @@ -0,0 +1,20 @@ +from typing import List + +import torch + + +class WrapStateAngle: + def __init__(self, keys: List[str]): + self.keys = keys + + @staticmethod + def _wrap(x): + return torch.atan2(torch.sin(x), torch.cos(x)) + + def forward(self, batch): + for k in self.keys: + batch["state"][k] = self._wrap(batch["state"][k]) + return batch + + def backward(self, batch): + return batch \ No newline at end of file diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/relative_action.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/relative_action.py new file mode 100644 index 0000000000000000000000000000000000000000..b4bd7b695066cc2f92472704ccbea1c3045d1e39 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/relative_action.py @@ -0,0 +1,104 @@ +from typing import List, Dict + +import torch +from .rotation import quaternion_to_matrix, matrix_to_quaternion + +# pose: position and quaternion in x, y, z, i, j, k, r +# mat: homogeneous transformation matrix in 4×4 +# pos: position in x, y, z +# quat: quaternion in i, j, k, r + + +class RelativePoseTransform: + def __init__(self, keys: List[str]): + self.keys = keys + + def forward(self, batch: Dict): + # for close-loop eval, "action" may not be in batch + if "action" not in batch: + return batch + + for k in self.keys: + action = batch["action"][k] + state = batch["state"][k] + batch["action"][k] = self._forward(action, state[..., -1:, :]) + + return batch + + def backward(self, batch: Dict): + for k in self.keys: + action = batch["action"][k] + state = batch["state"][k] + batch["action"][k] = self._backward(action, state[..., -1:, :]) + + return batch + + def _forward(self, pose: torch.Tensor, base_pose: torch.Tensor): + # pose & base_pose: position and quaternion in x, y, z, i, j, k, r + assert pose.shape[-1] == 7, f"Pose shape must be (..., 7), but got {pose.shape}" + assert base_pose.shape[-1] == 7, f"Base pose shape must be (..., 7), but got {base_pose.shape}" + pose_matrix = self._pose_to_matrix(pose) + base_pose_matrix = self._pose_to_matrix(base_pose) + pose_matrix = self._absolute_to_relative(pose_matrix, base_pose_matrix) + pose = self._matrix_to_pose(pose_matrix) + return pose + + def _backward(self, pose: torch.Tensor, base_pose: torch.Tensor): + # pose & base_pose: position and quaternion in x, y, z, i, j, k, r + assert pose.shape[-1] == 7, f"Pose shape must be (..., 7), but got {pose.shape}" + assert base_pose.shape[-1] == 7, f"Base pose shape must be (..., 7), but got {base_pose.shape}" + pose_matrix = self._pose_to_matrix(pose) + base_pose_matrix = self._pose_to_matrix(base_pose) + pose_matrix = self._relative_to_absolute(pose_matrix, base_pose_matrix) + pose = self._matrix_to_pose(pose_matrix) + return pose + + @staticmethod + def _pose_to_matrix(pose: torch.Tensor): + position = pose[..., 0: 3] + quaternion = pose[..., [6, 3, 4, 5]] # (i j k r) to (r i j k) + rotation = quaternion_to_matrix(quaternion) + matrix = torch.zeros(pose.shape[:-1] + (4, 4), dtype=pose.dtype, device=pose.device) + matrix[..., 0: 3, 0: 3] = rotation + matrix[..., 0: 3, 3] = position + matrix[..., 3, 3] = 1 + return matrix + + @staticmethod + def _matrix_to_pose(matrix: torch.Tensor): + position = matrix[..., 0: 3, 3] / matrix[..., 3, 3][..., None] + rotation = matrix[..., 0: 3, 0: 3] + quaternion = matrix_to_quaternion(rotation) + quaternion = quaternion[..., [1, 2, 3, 0]] # (r i j k) to (i j k r) + pose = torch.cat([position, quaternion], dim=-1) + return pose + @staticmethod + def _absolute_to_relative(pose_matrix: torch.Tensor, base_pose_matrix: torch.Tensor): + return torch.linalg.inv(base_pose_matrix) @ pose_matrix + + @staticmethod + def _relative_to_absolute(pose_matrix: torch.Tensor, base_pose_matrix: torch.Tensor): + return base_pose_matrix @ pose_matrix + + +class RelativeJointTransform: + def __init__(self, keys: List[str]): + self.keys = keys + + def forward(self, batch: Dict): + # for close-loop eval, "action" may not be in batch + if "action" not in batch: + return batch + + for k in self.keys: + # NOTE: fixed to the first frame + batch["action"][k] = batch["action"][k] - batch["state"][k][..., :1, :] + + return batch + + def backward(self, batch: Dict): + for k in self.keys: + # NOTE: fixed to the first frame + batch["action"][k] = batch["action"][k] + batch["state"][k][..., :1, :] + + return batch diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/rotation.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/rotation.py new file mode 100644 index 0000000000000000000000000000000000000000..4f19924703f4274401f01537180aae11f9bc84c7 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/rotation.py @@ -0,0 +1,133 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Dict, List, Tuple +import torch + +from ..utils.rotation import ( + quaternion_to_matrix, + matrix_to_quaternion, + matrix_to_rotation_6d, + rotation_6d_to_matrix, + matrix_to_rotation_9d, + rotation_9d_to_matrix, + quaternion_to_axis_angle, + axis_angle_to_quaternion, + axis_angle_to_matrix, + matrix_to_axis_angle, + euler_angles_to_matrix, + matrix_to_euler_angles, +) + +class PoseRotationTransform: + def __init__(self, rotation_type: str, category_keys: Dict[str, List[str]]): + self.rotation_type = rotation_type + self.category_keys = category_keys + self.src_type, self.dst_type = self._parse_rotation_type(rotation_type) + + def forward(self, batch): + for cat, ks in self.category_keys.items(): + if cat == "action" and "action" not in batch: + continue + + for k in ks: + batch[cat][k] = self._forward(batch[cat][k]) + + return batch + + def backward(self, batch): + for cat, ks in self.category_keys.items(): + for k in ks: + batch[cat][k] = self._backward(batch[cat][k]) + + return batch + + + def _forward(self, pose): + assert pose.shape[-1] >= 6 + if self.src_type == self.dst_type: + return pose + position, rotation, others = self._split_pose(pose, self.src_type) + matrix = self._rotation_to_matrix(rotation, self.src_type) + rotation_out = self._matrix_to_rotation(matrix, self.dst_type) + return torch.cat([position, rotation_out, others], axis=-1) + + def _backward(self, pose: torch.Tensor): + if self.src_type == self.dst_type: + return pose + position, rotation, others = self._split_pose(pose, self.dst_type) + matrix = self._rotation_to_matrix(rotation, self.dst_type) + rotation_out = self._matrix_to_rotation(matrix, self.src_type) + return torch.cat([position, rotation_out, others], axis=-1) + + def add_noise(self, pose: torch.Tensor, std_position=0.05, std_angle=0.05): + position, rotation, others = self._split_pose(pose, self.src_type) + matrix = self._rotation_to_matrix(rotation, self.src_type) + axis_angles = matrix_to_axis_angle(matrix) + position = position + std_position * torch.randn_like(position) + axis_angles = axis_angles + std_angle * torch.randn_like(axis_angles) + matrix = axis_angle_to_matrix(axis_angles) + rotation_out = self._matrix_to_rotation(matrix, self.src_type) + return torch.cat([position, rotation_out, others], axis=-1) + + def _parse_rotation_type(self, rotation_type: str) -> Tuple[str, str]: + if "_to_" in rotation_type: + src, dst = rotation_type.split("_to_", 1) + else: + src, dst = "quaternion", rotation_type + return src, dst + + def _split_pose(self, pose: torch.Tensor, rotation_type: str) -> Tuple[torch.Tensor, torch.Tensor]: + rotation_dim = self._rotation_dim(rotation_type) + position = pose[..., 0:3] + rotation = pose[..., 3: 3 + rotation_dim] + others = pose[..., 3 + rotation_dim:] if pose.shape[-1] > 3 + rotation_dim else torch.empty_like(position[..., :0]) + assert rotation.shape[-1] == rotation_dim, f"Expected {rotation_dim} dims for {rotation_type}" + return position, rotation, others + + def _rotation_dim(self, rotation_type: str) -> int: + if rotation_type == "quaternion": + return 4 + if rotation_type == "axis_angle": + return 3 + if rotation_type in ("rotation_6d",): + return 6 + if rotation_type in ("rotation_9d", "matrix"): + return 9 + if rotation_type.startswith("euler_"): + return 3 + raise ValueError(f"Unsupported rotation type '{rotation_type}'") + + def _rotation_to_matrix(self, rotation: torch.Tensor, rotation_type: str) -> torch.Tensor: + if rotation_type == "quaternion": + quaternion = rotation[..., [3, 0, 1, 2]] + return quaternion_to_matrix(quaternion) + if rotation_type == "axis_angle": + return axis_angle_to_matrix(rotation) + if rotation_type == "rotation_6d": + return rotation_6d_to_matrix(rotation) + if rotation_type in ("rotation_9d", "matrix"): + return rotation_9d_to_matrix(rotation) + if rotation_type.startswith("euler_"): + convention = rotation_type.split("_", 1)[1].upper() + return euler_angles_to_matrix(rotation, convention) + raise ValueError(f"Unsupported rotation type '{rotation_type}'") + + def _matrix_to_rotation(self, matrix: torch.Tensor, rotation_type: str) -> torch.Tensor: + if rotation_type == "quaternion": + quaternion = matrix_to_quaternion(matrix) + return quaternion[..., [1, 2, 3, 0]] + if rotation_type == "axis_angle": + return matrix_to_axis_angle(matrix) + if rotation_type == "rotation_6d": + return matrix_to_rotation_6d(matrix) + if rotation_type in ("rotation_9d", "matrix"): + return matrix_to_rotation_9d(matrix) + if rotation_type.startswith("euler_"): + convention = rotation_type.split("_", 1)[1].upper() + return matrix_to_euler_angles(matrix, convention) + raise ValueError(f"Unsupported rotation type '{rotation_type}'") + diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/normalizer.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/normalizer.py new file mode 100644 index 0000000000000000000000000000000000000000..1c674b0234bcf3cf1a1bef0f8fc0523020449b4a --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/normalizer.py @@ -0,0 +1,291 @@ +from typing import Literal, Dict, Annotated, Union, Any, List, Tuple, Optional +import torch +import json +from collections import defaultdict +import numpy as np +from omegaconf import DictConfig, OmegaConf +import hashlib +from pathlib import Path +from git import Repo +from fastwam.utils.logging_config import get_logger + +from fastwam.utils.pytorch_utils import dict_apply + +logger = get_logger(__name__) + +ConstConstStr = Annotated[str, "format: 'const_min/const_max', where const_min and const_max give the constant range"] +NormMode = Union[Literal["min/max", "q01/q99", "z-score"], ConstConstStr] + +class LinearNormalizer: + def __init__( + self, + shape_meta, + use_stepwise_action_norm, + default_mode: NormMode, + exception_mode: Dict[str, Dict[str, NormMode]], + stats: Dict[str, Dict[str, Dict[str, torch.Tensor]]] + ): + super().__init__() + self.normalizers = {"action": {}, "state": {}} + self.stats = stats + + for meta in shape_meta["action"]: + key = meta["key"] + + if use_stepwise_action_norm: + cur_stats = {k.removeprefix("stepwise_"): v for k, v in stats["action"][key].items() if k.startswith("stepwise_")} + else: + cur_stats = {k.removeprefix("global_"): v for k, v in stats["action"][key].items() if k.startswith("global_")} + + if exception_mode is not None and "action" in exception_mode and key in exception_mode["action"]: + cur_mode = exception_mode["action"][key] + else: + cur_mode = default_mode + + self.normalizers["action"][key] = SingleFieldLinearNormalizer( + stats=cur_stats, + mode=cur_mode, + ) + + for meta in shape_meta["state"]: + key = meta["key"] + cur_stats = {k.removeprefix("global_"): v for k, v in stats["state"][key].items() if k.startswith("global_")} + + if exception_mode is not None and "state" in exception_mode and key in exception_mode["state"]: + cur_mode = exception_mode["state"][key] + else: + cur_mode = default_mode + + self.normalizers["state"][key] = SingleFieldLinearNormalizer( + stats=cur_stats, + mode=cur_mode, + ) + + def get_stats(self): + stats = { + "action": {key: norm.get_stats() for key, norm in self.normalizers["action"].items()}, + "state": {key: norm.get_stats() for key, norm in self.normalizers["state"].items()} + } + return stats + + def forward(self, batch: Dict[str, Dict[str, torch.Tensor]]) -> torch.Tensor: + if "action" in batch: + for key, norm in self.normalizers["action"].items(): + batch["action"][key] = norm.forward(batch["action"][key]) + + for key, norm in self.normalizers["state"].items(): + batch["state"][key] = norm.forward(batch["state"][key]) + + return batch + + def backward(self, batch: Dict[str, Dict[str, torch.Tensor]]) -> torch.Tensor: + for key, norm in self.normalizers["action"].items(): + batch["action"][key] = norm.backward(batch["action"][key]) + + for key, norm in self.normalizers["state"].items(): + batch["state"][key] = norm.backward(batch["state"][key]) + + return batch + + +class SingleFieldLinearNormalizer: + std_reg = 1e-8 + range_tol = 1e-4 + output_max = 1.0 + output_min = -1.0 + def __init__(self, stats, mode: NormMode="min/max"): + self.stats = stats + self.mode = mode + + if mode == "z-score": + input_mean, input_std = stats["mean"], stats["std"] + scale = 1.0 / (input_std + self.std_reg) + offset = - input_mean / (input_std + self.std_reg) + else: + if mode == "min/max": + input_min, input_max = stats["min"], stats["max"] + elif mode == "q01/q99": + input_min, input_max = stats["q01"], stats["q99"] + else: + # parse const_min/const_max + input_min, input_max = map(float, mode.split("/")) + input_min = torch.full_like(stats["min"], input_min) + input_max = torch.full_like(stats["max"], input_max) + + input_range = input_max - input_min + ignore_dim = input_range < self.range_tol + input_range[ignore_dim] = self.output_max - self.output_min + scale = (self.output_max - self.output_min) / input_range + offset = self.output_min - scale * input_min + offset[ignore_dim] = (self.output_max + self.output_min) / 2 - input_min[ignore_dim] + + self.scale = scale + self.offset = offset + def get_stats(self): + return self.stats + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x * self.scale + self.offset + x = torch.clamp(x, -5.0, 5.0) + return x + def backward(self, x: torch.Tensor) -> torch.Tensor: + x = (x - self.offset) / self.scale + return x + +def save_dataset_stats_to_json(dataset_stats: dict, file_path: str): + + def convert_tensor(obj): + if isinstance(obj, torch.Tensor): + return obj.detach().cpu().numpy().tolist() + elif isinstance(obj, (defaultdict, dict)): + return {k: convert_tensor(v) for k, v in dict(obj).items()} + elif isinstance(obj, (list, tuple)): + return [convert_tensor(item) for item in obj] + elif isinstance(obj, (int, float, str, bool, type(None))): + return obj + else: + return str(obj) + + serializable_stats = convert_tensor(dataset_stats) + + with open(file_path, 'w', encoding='utf-8') as f: + json.dump(serializable_stats, f, ensure_ascii=False, indent=2) + +def load_dataset_stats_from_json(file_path: str, + try_convert_tensor: bool = True) -> Dict[str, Any]: + + def is_numeric_list(obj): + if isinstance(obj, list): + if not obj: + return True + first = obj[0] + if isinstance(first, (int, float)): + return all(isinstance(x, (int, float)) for x in obj) + elif isinstance(first, list): + return all(is_numeric_list(item) for item in obj) + else: + return False + return False + + def convert_back_to_tensor(obj): + if isinstance(obj, dict): + return {k: convert_back_to_tensor(v) for k, v in obj.items()} + elif isinstance(obj, list): + if is_numeric_list(obj): + try: + arr = np.array(obj) + return torch.from_numpy(arr) + except Exception: + return [convert_back_to_tensor(item) for item in obj] + else: + return [convert_back_to_tensor(item) for item in obj] + else: + return obj + + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + if try_convert_tensor: + data = convert_back_to_tensor(data) + + data = dict_apply( + data, + lambda x: x.to(torch.float32) if isinstance(x, torch.Tensor) else x, + ) + + return data + + +def search_dataset_stats_cache_json(cache_dir: str | Path, data_config: DictConfig) -> Tuple[bool, str | None]: + if isinstance(cache_dir, str): + cache_dir = Path(cache_dir) + cache_dir.mkdir(parents=True, exist_ok=True) + + def get_git_hash() -> Optional[str]: + repo = Repo(__file__, search_parent_directories=True) + return repo.head.commit.hexsha + + def to_plain(value: Any) -> Any: + if OmegaConf.is_config(value): + return OmegaConf.to_container(value, resolve=True) + return value + + def normalize_str_list(value: Any) -> List[str]: + value = to_plain(value) + if value is None: + return [] + if isinstance(value, str): + value = [value] + return [str(item) for item in value if item is not None] + + def normalize_transforms(value: Any) -> Any: + value = to_plain(value) + if isinstance(value, dict): + return [value] + return value + + def normalize_dataset_dirs(cfg: DictConfig) -> Any: + dataset_cfg = cfg.get("dataset") + if dataset_cfg is None: + return None + embodiment_datasets = dataset_cfg.get("embodiment_datasets") + if embodiment_datasets is not None: + emb_dirs: Dict[str, List[str]] = {} + for emb, emb_cfg in embodiment_datasets.items(): + dataset_groups = emb_cfg.get("dataset_groups") + if dataset_groups is None: + emb_dirs[emb] = [] + continue + dirs: List[str] = [] + for group in dataset_groups: + group_dirs = group.get("dataset_dirs") + if group_dirs is None: + continue + dirs.extend(normalize_str_list(group_dirs)) + emb_dirs[emb] = sorted(dirs) + return emb_dirs + + dataset_dirs = dataset_cfg.get("dataset_dirs") + return sorted(normalize_str_list(dataset_dirs)) + + def normalize_action_state_transforms(cfg: DictConfig) -> Any: + processor_cfg = cfg.get("processor") + if processor_cfg is None: + return None + embodiment_processors = processor_cfg.get("embodiment_processors") + if embodiment_processors is not None: + emb_transforms: Dict[str, Any] = {} + for emb, emb_cfg in embodiment_processors.items(): + transforms = emb_cfg.get("action_state_transforms") + emb_transforms[emb] = normalize_transforms(transforms) + return emb_transforms + + transforms = processor_cfg.get("action_state_transforms") + return normalize_transforms(transforms) + + signature = { + "action_size": data_config.dataset.action_size, + "dataset_dirs": normalize_dataset_dirs(data_config), + "action_state_transforms": normalize_action_state_transforms(data_config), + } + signature_json = json.dumps(signature, sort_keys=True, separators=(",", ":")) + dataset_hash = hashlib.sha256(signature_json.encode("utf-8"), usedforsecurity=False).hexdigest() + + git_hash = get_git_hash() + precise_name = f"dataset_stats_{dataset_hash}_{git_hash}.json" + precise = cache_dir / precise_name + if precise.exists(): + logger.info(f"Found dataset stats cache with precisely matching dataset and git hash: {precise_name}.") + return True, str(precise) + + candidates = sorted(cache_dir.glob(f"dataset_stats_{dataset_hash}_*.json")) + if not candidates: + logger.info(f"No dataset stats cache found for dataset hash {dataset_hash}") + return False, str(precise) # return precise cache path for saving cache + + picked = candidates[0] + prefix = f"dataset_stats_{dataset_hash}_" + picked_git_hash = picked.name[len(prefix):-5] + assert picked_git_hash != git_hash + logger.warning(f"Found substitute dataset stats cache {picked.name} which mismatch current git hash {git_hash}.") + return True, str(picked) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/rotation.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/rotation.py new file mode 100644 index 0000000000000000000000000000000000000000..f468cd4f24092bfe8a601a85e2cb8e5e6b84cbbb --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/rotation.py @@ -0,0 +1,621 @@ + +from typing import Optional, Union, Literal, List +import torch +import torch.nn.functional as F + +Device = Union[str, torch.device] + + +# [Important] Below is adapted from pytorch3d, which uses (r, i, j, k) quat order + +""" +The transformation matrices returned from the functions in this file assume +the points on which the transformation will be applied are column vectors. +i.e. the R matrix is structured as + + R = [ + [Rxx, Rxy, Rxz], + [Ryx, Ryy, Ryz], + [Rzx, Rzy, Rzz], + ] # (3, 3) + +This matrix can be applied to column vectors by post multiplication +by the points e.g. + + points = [[0], [1], [2]] # (3 x 1) xyz coordinates of a point + transformed_points = R * points + +To apply the same matrix to points which are row vectors, the R matrix +can be transposed and pre multiplied by the points: + +e.g. + points = [[0, 1, 2]] # (1 x 3) xyz coordinates of a point + transformed_points = points * R.transpose(1, 0) +""" + + +def quaternion_to_matrix(quaternions: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as quaternions to rotation matrices. + + Args: + quaternions: quaternions with real part first, + as tensor of shape (..., 4). + + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + r, i, j, k = torch.unbind(quaternions, -1) + # pyre-fixme[58]: `/` is not supported for operand types `float` and `Tensor`. + two_s = 2.0 / (quaternions * quaternions).sum(-1) + + o = torch.stack( + ( + 1 - two_s * (j * j + k * k), + two_s * (i * j - k * r), + two_s * (i * k + j * r), + two_s * (i * j + k * r), + 1 - two_s * (i * i + k * k), + two_s * (j * k - i * r), + two_s * (i * k - j * r), + two_s * (j * k + i * r), + 1 - two_s * (i * i + j * j), + ), + -1, + ) + return o.reshape(quaternions.shape[:-1] + (3, 3)) + + +def _copysign(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + """ + Return a tensor where each element has the absolute value taken from the, + corresponding element of a, with sign taken from the corresponding + element of b. This is like the standard copysign floating-point operation, + but is not careful about negative 0 and NaN. + + Args: + a: source tensor. + b: tensor whose signs will be used, of the same shape as a. + + Returns: + Tensor of the same shape as a with the signs of b. + """ + signs_differ = (a < 0) != (b < 0) + return torch.where(signs_differ, -a, a) + +@torch.jit.script +def _sqrt_positive_part(x: torch.Tensor) -> torch.Tensor: + """ + Returns torch.sqrt(torch.max(0, x)) + but with a zero subgradient where x is 0. + """ + ret = torch.zeros_like(x) + positive_mask = x > 0 + ret[positive_mask] = torch.sqrt(x[positive_mask]) + return ret + + +def matrix_to_quaternion(matrix: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as rotation matrices to quaternions. + + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + + Returns: + quaternions with real part first, as tensor of shape (..., 4). + """ + if matrix.size(-1) != 3 or matrix.size(-2) != 3: + raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.") + + batch_dim = matrix.shape[:-2] + m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.unbind( + matrix.reshape(batch_dim + (9,)), dim=-1 + ) + + q_abs = _sqrt_positive_part( + torch.stack( + [ + 1.0 + m00 + m11 + m22, + 1.0 + m00 - m11 - m22, + 1.0 - m00 + m11 - m22, + 1.0 - m00 - m11 + m22, + ], + dim=-1, + ) + ) + + # we produce the desired quaternion multiplied by each of r, i, j, k + quat_by_rijk = torch.stack( + [ + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + torch.stack([q_abs[..., 0] ** 2, m21 - m12, m02 - m20, m10 - m01], dim=-1), + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + torch.stack([m21 - m12, q_abs[..., 1] ** 2, m10 + m01, m02 + m20], dim=-1), + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + torch.stack([m02 - m20, m10 + m01, q_abs[..., 2] ** 2, m12 + m21], dim=-1), + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + torch.stack([m10 - m01, m20 + m02, m21 + m12, q_abs[..., 3] ** 2], dim=-1), + ], + dim=-2, + ) + + # We floor here at 0.1 but the exact level is not important; if q_abs is small, + # the candidate won't be picked. + flr = torch.tensor(0.1).to(dtype=q_abs.dtype, device=q_abs.device) + quat_candidates = quat_by_rijk / (2.0 * q_abs[..., None].max(flr)) + + # if not for numerical problems, quat_candidates[i] should be same (up to a sign), + # forall i; we pick the best-conditioned one (with the largest denominator) + out = quat_candidates[ + F.one_hot(q_abs.argmax(dim=-1), num_classes=4) > 0.5, : + ].reshape(batch_dim + (4,)) + return standardize_quaternion(out) + + +def _axis_angle_rotation(axis: str, angle: torch.Tensor) -> torch.Tensor: + """ + Return the rotation matrices for one of the rotations about an axis + of which Euler angles describe, for each value of the angle given. + + Args: + axis: Axis label "X" or "Y or "Z". + angle: any shape tensor of Euler angles in radians + + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + + cos = torch.cos(angle) + sin = torch.sin(angle) + one = torch.ones_like(angle) + zero = torch.zeros_like(angle) + + if axis == "X": + R_flat = (one, zero, zero, zero, cos, -sin, zero, sin, cos) + elif axis == "Y": + R_flat = (cos, zero, sin, zero, one, zero, -sin, zero, cos) + elif axis == "Z": + R_flat = (cos, -sin, zero, sin, cos, zero, zero, zero, one) + else: + raise ValueError("letter must be either X, Y or Z.") + + return torch.stack(R_flat, -1).reshape(angle.shape + (3, 3)) + + +def euler_angles_to_matrix(euler_angles: torch.Tensor, convention: str) -> torch.Tensor: + """ + Convert rotations given as Euler angles in radians to rotation matrices. + + Args: + euler_angles: Euler angles in radians as tensor of shape (..., 3). + convention: Convention string of three uppercase letters from + {"X", "Y", and "Z"}. + + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + if euler_angles.dim() == 0 or euler_angles.shape[-1] != 3: + raise ValueError("Invalid input euler angles.") + if len(convention) != 3: + raise ValueError("Convention must have 3 letters.") + if convention[1] in (convention[0], convention[2]): + raise ValueError(f"Invalid convention {convention}.") + for letter in convention: + if letter not in ("X", "Y", "Z"): + raise ValueError(f"Invalid letter {letter} in convention string.") + matrices = [ + _axis_angle_rotation(c, e) + for c, e in zip(convention, torch.unbind(euler_angles, -1)) + ] + # return functools.reduce(torch.matmul, matrices) + return torch.matmul(torch.matmul(matrices[0], matrices[1]), matrices[2]) + + +def _angle_from_tan( + axis: str, other_axis: str, data, horizontal: bool, tait_bryan: bool +) -> torch.Tensor: + """ + Extract the first or third Euler angle from the two members of + the matrix which are positive constant times its sine and cosine. + + Args: + axis: Axis label "X" or "Y or "Z" for the angle we are finding. + other_axis: Axis label "X" or "Y or "Z" for the middle axis in the + convention. + data: Rotation matrices as tensor of shape (..., 3, 3). + horizontal: Whether we are looking for the angle for the third axis, + which means the relevant entries are in the same row of the + rotation matrix. If not, they are in the same column. + tait_bryan: Whether the first and third axes in the convention differ. + + Returns: + Euler Angles in radians for each matrix in data as a tensor + of shape (...). + """ + + i1, i2 = {"X": (2, 1), "Y": (0, 2), "Z": (1, 0)}[axis] + if horizontal: + i2, i1 = i1, i2 + even = (axis + other_axis) in ["XY", "YZ", "ZX"] + if horizontal == even: + return torch.atan2(data[..., i1], data[..., i2]) + if tait_bryan: + return torch.atan2(-data[..., i2], data[..., i1]) + return torch.atan2(data[..., i2], -data[..., i1]) + + +def _index_from_letter(letter: str) -> int: + if letter == "X": + return 0 + if letter == "Y": + return 1 + if letter == "Z": + return 2 + raise ValueError("letter must be either X, Y or Z.") + + +def matrix_to_euler_angles(matrix: torch.Tensor, convention: str) -> torch.Tensor: + """ + Convert rotations given as rotation matrices to Euler angles in radians. + + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + convention: Convention string of three uppercase letters. + + Returns: + Euler angles in radians as tensor of shape (..., 3). + """ + if len(convention) != 3: + raise ValueError("Convention must have 3 letters.") + if convention[1] in (convention[0], convention[2]): + raise ValueError(f"Invalid convention {convention}.") + for letter in convention: + if letter not in ("X", "Y", "Z"): + raise ValueError(f"Invalid letter {letter} in convention string.") + if matrix.size(-1) != 3 or matrix.size(-2) != 3: + raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.") + i0 = _index_from_letter(convention[0]) + i2 = _index_from_letter(convention[2]) + tait_bryan = i0 != i2 + if tait_bryan: + central_angle = torch.asin( + matrix[..., i0, i2] * (-1.0 if i0 - i2 in [-1, 2] else 1.0) + ) + else: + central_angle = torch.acos(matrix[..., i0, i0]) + + o = ( + _angle_from_tan( + convention[0], convention[1], matrix[..., i2], False, tait_bryan + ), + central_angle, + _angle_from_tan( + convention[2], convention[1], matrix[..., i0, :], True, tait_bryan + ), + ) + return torch.stack(o, -1) + + +def random_quaternions( + n: int, dtype: Optional[torch.dtype] = None, device: Optional[Device] = None +) -> torch.Tensor: + """ + Generate random quaternions representing rotations, + i.e. versors with nonnegative real part. + + Args: + n: Number of quaternions in a batch to return. + dtype: Type to return. + device: Desired device of returned tensor. Default: + uses the current device for the default tensor type. + + Returns: + Quaternions as tensor of shape (N, 4). + """ + if isinstance(device, str): + device = torch.device(device) + o = torch.randn((n, 4), dtype=dtype, device=device) + s = (o * o).sum(1) + o = o / _copysign(torch.sqrt(s), o[:, 0])[:, None] + return o + + +def random_rotations( + n: int, dtype: Optional[torch.dtype] = None, device: Optional[Device] = None +) -> torch.Tensor: + """ + Generate random rotations as 3x3 rotation matrices. + + Args: + n: Number of rotation matrices in a batch to return. + dtype: Type to return. + device: Device of returned tensor. Default: if None, + uses the current device for the default tensor type. + + Returns: + Rotation matrices as tensor of shape (n, 3, 3). + """ + quaternions = random_quaternions(n, dtype=dtype, device=device) + return quaternion_to_matrix(quaternions) + + +def random_rotation( + dtype: Optional[torch.dtype] = None, device: Optional[Device] = None +) -> torch.Tensor: + """ + Generate a single random 3x3 rotation matrix. + + Args: + dtype: Type to return + device: Device of returned tensor. Default: if None, + uses the current device for the default tensor type + + Returns: + Rotation matrix as tensor of shape (3, 3). + """ + return random_rotations(1, dtype, device)[0] + + +def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor: + """ + Convert a unit quaternion to a standard form: one in which the real + part is non negative. + + Args: + quaternions: Quaternions with real part first, + as tensor of shape (..., 4). + + Returns: + Standardized quaternions as tensor of shape (..., 4). + """ + return torch.where(quaternions[..., 0:1] < 0, -quaternions, quaternions) + + +def quaternion_raw_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + """ + Multiply two quaternions. + Usual torch rules for broadcasting apply. + + Args: + a: Quaternions as tensor of shape (..., 4), real part first. + b: Quaternions as tensor of shape (..., 4), real part first. + + Returns: + The product of a and b, a tensor of quaternions shape (..., 4). + """ + aw, ax, ay, az = torch.unbind(a, -1) + bw, bx, by, bz = torch.unbind(b, -1) + ow = aw * bw - ax * bx - ay * by - az * bz + ox = aw * bx + ax * bw + ay * bz - az * by + oy = aw * by - ax * bz + ay * bw + az * bx + oz = aw * bz + ax * by - ay * bx + az * bw + return torch.stack((ow, ox, oy, oz), -1) + + +def quaternion_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + """ + Multiply two quaternions representing rotations, returning the quaternion + representing their composition, i.e. the versor with nonnegative real part. + Usual torch rules for broadcasting apply. + + Args: + a: Quaternions as tensor of shape (..., 4), real part first. + b: Quaternions as tensor of shape (..., 4), real part first. + + Returns: + The product of a and b, a tensor of quaternions of shape (..., 4). + """ + ab = quaternion_raw_multiply(a, b) + return standardize_quaternion(ab) + + +def quaternion_invert(quaternion: torch.Tensor) -> torch.Tensor: + """ + Given a quaternion representing rotation, get the quaternion representing + its inverse. + + Args: + quaternion: Quaternions as tensor of shape (..., 4), with real part + first, which must be versors (unit quaternions). + + Returns: + The inverse, a tensor of quaternions of shape (..., 4). + """ + + scaling = torch.tensor([1, -1, -1, -1], device=quaternion.device) + return quaternion * scaling + + +def quaternion_apply(quaternion: torch.Tensor, point: torch.Tensor) -> torch.Tensor: + """ + Apply the rotation given by a quaternion to a 3D point. + Usual torch rules for broadcasting apply. + + Args: + quaternion: Tensor of quaternions, real part first, of shape (..., 4). + point: Tensor of 3D points of shape (..., 3). + + Returns: + Tensor of rotated points of shape (..., 3). + """ + if point.size(-1) != 3: + raise ValueError(f"Points are not in 3D, {point.shape}.") + real_parts = point.new_zeros(point.shape[:-1] + (1,)) + point_as_quaternion = torch.cat((real_parts, point), -1) + out = quaternion_raw_multiply( + quaternion_raw_multiply(quaternion, point_as_quaternion), + quaternion_invert(quaternion), + ) + return out[..., 1:] + + +def axis_angle_to_matrix(axis_angle: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as axis/angle to rotation matrices. + + Args: + axis_angle: Rotations given as a vector in axis angle form, + as a tensor of shape (..., 3), where the magnitude is + the angle turned anticlockwise in radians around the + vector's direction. + + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + return quaternion_to_matrix(axis_angle_to_quaternion(axis_angle)) + + +def matrix_to_axis_angle(matrix: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as rotation matrices to axis/angle. + + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + + Returns: + Rotations given as a vector in axis angle form, as a tensor + of shape (..., 3), where the magnitude is the angle + turned anticlockwise in radians around the vector's + direction. + """ + return quaternion_to_axis_angle(matrix_to_quaternion(matrix)) + + +def axis_angle_to_quaternion(axis_angle: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as axis/angle to quaternions. + + Args: + axis_angle: Rotations given as a vector in axis angle form, + as a tensor of shape (..., 3), where the magnitude is + the angle turned anticlockwise in radians around the + vector's direction. + + Returns: + quaternions with real part first, as tensor of shape (..., 4). + """ + angles = torch.norm(axis_angle, p=2, dim=-1, keepdim=True) + half_angles = angles * 0.5 + eps = 1e-6 + small_angles = angles.abs() < eps + sin_half_angles_over_angles = torch.empty_like(angles) + sin_half_angles_over_angles[~small_angles] = ( + torch.sin(half_angles[~small_angles]) / angles[~small_angles] + ) + # for x small, sin(x/2) is about x/2 - (x/2)^3/6 + # so sin(x/2)/x is about 1/2 - (x*x)/48 + sin_half_angles_over_angles[small_angles] = ( + 0.5 - (angles[small_angles] * angles[small_angles]) / 48 + ) + quaternions = torch.cat( + [torch.cos(half_angles), axis_angle * sin_half_angles_over_angles], dim=-1 + ) + return quaternions + + +def quaternion_to_axis_angle(quaternions: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as quaternions to axis/angle. + + Args: + quaternions: quaternions with real part first, + as tensor of shape (..., 4). + + Returns: + Rotations given as a vector in axis angle form, as a tensor + of shape (..., 3), where the magnitude is the angle + turned anticlockwise in radians around the vector's + direction. + """ + norms = torch.norm(quaternions[..., 1:], p=2, dim=-1, keepdim=True) + half_angles = torch.atan2(norms, quaternions[..., :1]) + angles = 2 * half_angles + eps = 1e-6 + small_angles = angles.abs() < eps + sin_half_angles_over_angles = torch.empty_like(angles) + sin_half_angles_over_angles[~small_angles] = ( + torch.sin(half_angles[~small_angles]) / angles[~small_angles] + ) + # for x small, sin(x/2) is about x/2 - (x/2)^3/6 + # so sin(x/2)/x is about 1/2 - (x*x)/48 + sin_half_angles_over_angles[small_angles] = ( + 0.5 - (angles[small_angles] * angles[small_angles]) / 48 + ) + return quaternions[..., 1:] / sin_half_angles_over_angles + + +def rotation_6d_to_matrix(d6: torch.Tensor) -> torch.Tensor: + """ + Converts 6D rotation representation by Zhou et al. [1] to rotation matrix + using Gram--Schmidt orthogonalization per Section B of [1]. + Args: + d6: 6D rotation representation, of size (*, 6) + + Returns: + batch of rotation matrices of size (*, 3, 3) + + [1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H. + On the Continuity of Rotation Representations in Neural Networks. + IEEE Conference on Computer Vision and Pattern Recognition, 2019. + Retrieved from http://arxiv.org/abs/1812.07035 + """ + + a1, a2 = d6[..., :3], d6[..., 3:] + b1 = F.normalize(a1, dim=-1) + b2 = a2 - (b1 * a2).sum(-1, keepdim=True) * b1 + b2 = F.normalize(b2, dim=-1) + b3 = torch.cross(b1, b2, dim=-1) + return torch.stack((b1, b2, b3), dim=-2) + + +def matrix_to_rotation_6d(matrix: torch.Tensor) -> torch.Tensor: + """ + Converts rotation matrices to 6D rotation representation by Zhou et al. [1] + by dropping the last row. Note that 6D representation is not unique. + Args: + matrix: batch of rotation matrices of size (*, 3, 3) + + Returns: + 6D rotation representation, of size (*, 6) + + [1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H. + On the Continuity of Rotation Representations in Neural Networks. + IEEE Conference on Computer Vision and Pattern Recognition, 2019. + Retrieved from http://arxiv.org/abs/1812.07035 + """ + batch_dim = matrix.size()[:-2] + return matrix[..., :2, :].clone().reshape(batch_dim + (6,)) + + +def rotation_9d_to_matrix(d9: torch.Tensor) -> torch.Tensor: + """" + Args: + d9: 9D rotaion representation of size (..., 9) + + Returns: + batch of rotation matrices of size (..., 3, 3) + """ + assert d9.shape[-1] == 9 + matrices = d9.reshape(-1, 3, 3).permute(0, 2, 1) + u, s, vt = torch.linalg.svd(matrices) # torch.svd returns v instead of vt + det = torch.det(torch.matmul(u, vt)) + vt[:, :, -1] *= det[:, None] + rot_matrices = torch.matmul(u, vt) + rot_matrices = rot_matrices.unflatten(0, d9.shape[:-1]) + return rot_matrices + + +def matrix_to_rotation_9d(matrix: torch.Tensor) -> torch.Tensor: + """" + Args: + matrix: batch of rotation matrices of size (..., 3, 3) + Returns: + 9D rotaion representation of size (..., 9) + """ + assert matrix.shape[-2:] == (3, 3) + return matrix.transpose(-2, -1).flatten(-2, -1) \ No newline at end of file diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/__init__.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f45b7d25cd81883bbc40b37fd6366225d1e84105 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/__init__.py @@ -0,0 +1 @@ +"""Model namespace for fastwam.""" diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/__init__.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/action_dit.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/action_dit.py new file mode 100644 index 0000000000000000000000000000000000000000..f6716e69a84e891e83796fc32995918f83df2746 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/action_dit.py @@ -0,0 +1,337 @@ +import os +import torch +import torch.nn as nn +from typing import Any, Dict, Optional + +from fastwam.utils.logging_config import get_logger + +from .helpers.gradient import gradient_checkpoint_forward +from .wan_video_dit import ( + DiTBlock, + sinusoidal_embedding_1d, + precompute_freqs_cis, +) + +logger = get_logger(__name__) + + +class ActionHead(nn.Module): + def __init__(self, hidden_dim: int, out_dim: int, eps: float): + super().__init__() + self.norm = nn.LayerNorm(hidden_dim, eps=eps, elementwise_affine=False) + self.proj = nn.Linear(hidden_dim, out_dim) + self.modulation = nn.Parameter(torch.randn(1, 2, hidden_dim) / hidden_dim**0.5) + + def forward(self, x: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + shift, scale = (self.modulation.to(dtype=t.dtype, device=t.device) + t.unsqueeze(1)).chunk(2, dim=1) + shift = shift.squeeze(1) + scale = scale.squeeze(1) + return self.proj(self.norm(x) * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)) + + +class ActionDiT(nn.Module): + ACTION_BACKBONE_SKIP_PREFIXES = ("action_encoder.", "head.") + ACTION_BACKBONE_META_KEYS = ( + "hidden_dim", + "ffn_dim", + "num_layers", + "num_heads", + "attn_head_dim", + "text_dim", + "freq_dim", + "eps", + ) + + def __init__( + self, + hidden_dim: int, + action_dim: int, + ffn_dim: int, + text_dim: int, + freq_dim: int, + eps: float, + num_heads: int, + attn_head_dim: int, + num_layers: int, + use_gradient_checkpointing: bool = False, + ): + super().__init__() + self.hidden_dim = hidden_dim + self.action_dim = action_dim + self.ffn_dim = ffn_dim + self.text_dim = text_dim + self.freq_dim = freq_dim + self.num_heads = num_heads + self.attn_head_dim = attn_head_dim + + if num_heads <= 0: + raise ValueError(f"`num_heads` must be > 0, got {num_heads}") + if attn_head_dim <= 0: + raise ValueError(f"`attn_head_dim` must be > 0, got {attn_head_dim}") + if attn_head_dim % 2 != 0: + raise ValueError(f"`attn_head_dim` must be even for RoPE, got {attn_head_dim}") + + self.action_encoder = nn.Linear(action_dim, hidden_dim) + self.text_embedding = nn.Sequential( + nn.Linear(text_dim, hidden_dim), + nn.GELU(approximate="tanh"), + nn.Linear(hidden_dim, hidden_dim), + ) + self.time_embedding = nn.Sequential( + nn.Linear(freq_dim, hidden_dim), + nn.SiLU(), + nn.Linear(hidden_dim, hidden_dim), + ) + self.time_projection = nn.Sequential(nn.SiLU(), nn.Linear(hidden_dim, hidden_dim * 6)) + self.blocks = nn.ModuleList( + [ + DiTBlock( + hidden_dim=hidden_dim, + attn_head_dim=attn_head_dim, + num_heads=num_heads, + ffn_dim=ffn_dim, + eps=eps, + ) + for _ in range(num_layers) + ] + ) + self.head = nn.Linear(hidden_dim, action_dim) + self.freqs = precompute_freqs_cis(attn_head_dim, end=1024) + + self.use_gradient_checkpointing = use_gradient_checkpointing + + @classmethod + def backbone_key_set(cls, keys) -> set[str]: + return { + key + for key in keys + if not any(key.startswith(prefix) for prefix in cls.ACTION_BACKBONE_SKIP_PREFIXES) + } + + @classmethod + def from_pretrained( + cls, + action_dit_config: dict[str, Any], + action_dit_pretrained_path: str | None = None, + skip_dit_load_from_pretrain: bool = False, + device: str = "cuda", + torch_dtype: torch.dtype = torch.bfloat16, + ) -> "ActionDiT": + if action_dit_config is None: + raise ValueError("`action_dit_config` is required for ActionDiT.from_pretrained().") + if skip_dit_load_from_pretrain: + logger.info( + "Skipping ActionDiT pretrained load (`skip_dit_load_from_pretrain=True`); " + "initializing action expert randomly and expecting checkpoint override." + ) + return cls(**action_dit_config).to(device=device, dtype=torch_dtype) + if not action_dit_pretrained_path: + logger.info("No `action_dit_pretrained_path` provided, initializing ActionDiT with random weights.") + return cls(**action_dit_config).to(device=device, dtype=torch_dtype) + from pathlib import Path + p = Path(action_dit_pretrained_path) + if not p.is_absolute(): + p = Path(__file__).resolve().parents[4] / p + action_dit_pretrained_path = str(p) + if not os.path.isfile(action_dit_pretrained_path): + raise FileNotFoundError( + f"`action_dit_pretrained_path` does not exist: {action_dit_pretrained_path}" + ) + + action_cfg = dict(action_dit_config) + action_expert = cls(**action_cfg).to(device=device, dtype=torch_dtype) + action_state = action_expert.state_dict() + expected_backbone_keys = cls.backbone_key_set(action_state.keys()) + + payload = torch.load(action_dit_pretrained_path, map_location="cpu") + if not isinstance(payload, dict): + raise ValueError( + f"Invalid action backbone payload type from {action_dit_pretrained_path}: {type(payload)}" + ) + + policy = payload.get("policy", {}) + if policy: + logger.info(f"ActionDiT backbone payload policy: {policy}") + + meta = payload.get("meta") + expected_meta = { + "hidden_dim": int(action_cfg["hidden_dim"]), + "ffn_dim": int(action_cfg["ffn_dim"]), + "num_layers": int(action_cfg["num_layers"]), + "num_heads": int(action_cfg["num_heads"]), + "attn_head_dim": int(action_cfg["attn_head_dim"]), + "text_dim": int(action_cfg["text_dim"]), + "freq_dim": int(action_cfg["freq_dim"]), + "eps": float(action_cfg["eps"]), + } + for key in cls.ACTION_BACKBONE_META_KEYS: + if key not in meta: + raise ValueError(f"`meta.{key}` missing in {action_dit_pretrained_path}") + expected_value = expected_meta[key] + got_value = meta[key] + if key == "eps": + if abs(float(got_value) - float(expected_value)) > 1e-12: + raise ValueError( + f"`meta.{key}` mismatch in {action_dit_pretrained_path}: " + f"expected {expected_value}, got {got_value}" + ) + elif int(got_value) != int(expected_value): + raise ValueError( + f"`meta.{key}` mismatch in {action_dit_pretrained_path}: " + f"expected {expected_value}, got {got_value}" + ) + + backbone_state_dict = payload.get("backbone_state_dict") + if not isinstance(backbone_state_dict, dict): + raise ValueError( + f"`backbone_state_dict` must be a dict in {action_dit_pretrained_path}, " + f"got {type(backbone_state_dict)}" + ) + + provided_keys = set(backbone_state_dict.keys()) + missing_keys = sorted(expected_backbone_keys - provided_keys) + unexpected_keys = sorted(provided_keys - expected_backbone_keys) + if missing_keys or unexpected_keys: + raise ValueError( + "Action backbone key mismatch in preprocessed payload. " + f"missing={missing_keys[:10]}{'...' if len(missing_keys) > 10 else ''}, " + f"unexpected={unexpected_keys[:10]}{'...' if len(unexpected_keys) > 10 else ''}" + ) + + merged_state = dict(action_state) + for key in expected_backbone_keys: + value = backbone_state_dict[key] + if not isinstance(value, torch.Tensor): + raise ValueError( + f"`backbone_state_dict[{key}]` must be torch.Tensor in {action_dit_pretrained_path}, " + f"got {type(value)}" + ) + target = merged_state[key] + if tuple(value.shape) != tuple(target.shape): + raise ValueError( + f"Shape mismatch for `{key}` in {action_dit_pretrained_path}: " + f"expected {tuple(target.shape)}, got {tuple(value.shape)}" + ) + merged_state[key] = value.to(device=target.device, dtype=target.dtype) + + action_expert.load_state_dict(merged_state, strict=True) + logger.info( + "Loaded ActionDiT backbone from %s (keys=%d; random_kept_prefixes=%s).", + action_dit_pretrained_path, + len(expected_backbone_keys), + list(cls.ACTION_BACKBONE_SKIP_PREFIXES), + ) + return action_expert.to(device=device, dtype=torch_dtype) + + def pre_dit( + self, + action_tokens: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: Optional[torch.Tensor] = None, + ) -> Dict[str, Any]: + if action_tokens.ndim != 3: + raise ValueError( + f"`action_tokens` must be 3D [B, T, action_dim], got shape {tuple(action_tokens.shape)}" + ) + if action_tokens.shape[2] != self.action_dim: + raise ValueError( + f"`action_tokens` last dim must be {self.action_dim}, got {action_tokens.shape[2]}" + ) + if timestep.ndim != 1: + raise ValueError(f"`timestep` must be 1D [B] or [1], got shape {tuple(timestep.shape)}") + if context.ndim != 3: + raise ValueError( + f"`context` must be 3D [B, L, D], got shape {tuple(context.shape)}" + ) + + batch_size = action_tokens.shape[0] + if context.shape[0] != batch_size: + raise ValueError( + f"Batch mismatch between action tokens and text context: {batch_size} vs {context.shape[0]}" + ) + if timestep.shape[0] not in (1, batch_size): + raise ValueError( + f"`timestep` length must be 1 or batch_size({batch_size}), got {timestep.shape[0]}" + ) + if timestep.shape[0] == 1 and batch_size > 1: + if self.training: + raise ValueError("During training, action timestep length must match batch_size.") + timestep = timestep.expand(batch_size) + + if context_mask is None: + context_mask = torch.ones( + (batch_size, context.shape[1]), dtype=torch.bool, device=context.device + ) + else: + if context_mask.ndim != 2: + raise ValueError(f"`context_mask` must be 2D [B, L], got shape {tuple(context_mask.shape)}") + if context_mask.shape[0] != batch_size or context_mask.shape[1] != context.shape[1]: + raise ValueError( + f"`context_mask` shape must match `context` shape [B, L], got {tuple(context_mask.shape)} vs {tuple(context.shape)}" + ) + + seq_len = action_tokens.shape[1] + if seq_len > self.freqs.shape[0]: + raise ValueError( + f"Action token length {seq_len} exceeds RoPE cache {self.freqs.shape[0]}." + ) + + t = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timestep)) + t_mod = self.time_projection(t).unflatten(1, (6, self.hidden_dim)) + + tokens = self.action_encoder(action_tokens) + context_emb = self.text_embedding(context) + context_attn_mask = context_mask.unsqueeze(1).expand(-1, seq_len, -1) + freqs = self.freqs[:seq_len].view(seq_len, 1, -1).to(tokens.device) + + return { + "tokens": tokens, + "freqs": freqs, + "t": t, + "t_mod": t_mod, + "context": context_emb, + "context_mask": context_attn_mask, + "meta": { + "batch_size": batch_size, + "seq_len": seq_len, + }, + } + + def post_dit(self, tokens: torch.Tensor, pre_state: Dict[str, Any]) -> torch.Tensor: + return self.head(tokens) + + def forward( + self, + action_tokens: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + pre_state = self.pre_dit( + action_tokens=action_tokens, + timestep=timestep, + context=context, + context_mask=context_mask, + ) + x = pre_state["tokens"] + context = pre_state["context"] + t_mod = pre_state["t_mod"] + freqs = pre_state["freqs"] + context_mask = pre_state["context_mask"] + + for block in self.blocks: + if self.use_gradient_checkpointing: + x = gradient_checkpoint_forward( + block, + self.use_gradient_checkpointing, + x, + context, + t_mod, + freqs, + context_mask=context_mask, + ) + else: + x = block(x, context, t_mod, freqs, context_mask=context_mask) + + return self.post_dit(x, pre_state) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/attention_masks.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/attention_masks.py new file mode 100644 index 0000000000000000000000000000000000000000..774e537c1559c183b0f3bab137ec875311bdaaf1 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/attention_masks.py @@ -0,0 +1,82 @@ +"""Structured attention masks for video/action joint denoising. + +Boolean masks follow PyTorch SDPA semantics: ``True`` means that the query is +allowed to attend to the key. Rows are queries and columns are keys. +""" + +from __future__ import annotations + +import torch + + +def build_frame_causal_video_mask( + video_seq_len: int, + video_tokens_per_frame: int, + *, + device: torch.device | str | None = None, +) -> torch.Tensor: + """Build block-causal video attention with bidirectional intra-frame attention. + + Every token in frame ``i`` can see every token in frames ``0..i``. Thus + spatial tokens inside a frame are bidirectional, while information cannot + travel directly from a future frame to a past frame through this sub-mask. + """ + if video_seq_len <= 0: + raise ValueError(f"`video_seq_len` must be > 0, got {video_seq_len}") + if video_tokens_per_frame <= 0: + raise ValueError( + "`video_tokens_per_frame` must be > 0, " + f"got {video_tokens_per_frame}" + ) + if video_seq_len % video_tokens_per_frame != 0: + raise ValueError( + "`video_seq_len` must be divisible by `video_tokens_per_frame`, " + f"got {video_seq_len} and {video_tokens_per_frame}" + ) + + num_frames = video_seq_len // video_tokens_per_frame + frame_ids = torch.arange(num_frames, device=device).repeat_interleave( + video_tokens_per_frame + ) + query_frame = frame_ids[:, None] + key_frame = frame_ids[None, :] + return key_frame <= query_frame + + +def build_joint_denoise_attention_mask( + video_seq_len: int, + action_seq_len: int, + video_tokens_per_frame: int, + *, + device: torch.device | str | None = None, +) -> torch.Tensor: + """Build shared-self-attention visibility for joint video/action denoising. + + The four query/key quadrants are: + + * video -> video: frame causal, intra-frame bidirectional; + * video -> action: fully visible; + * action -> video: fully visible; + * action -> action: bidirectional. + + This differs from upstream ``FastWAMJoint``, whose released mask permits + action -> video but blocks video -> action. + + Full cross-modal visibility can indirectly carry future-video information + through action tokens in deeper layers. Consequently only the direct + video/video sub-mask is causal; the whole multimodal graph is not strictly + causal. + """ + if action_seq_len <= 0: + raise ValueError(f"`action_seq_len` must be > 0, got {action_seq_len}") + + total_seq_len = video_seq_len + action_seq_len + mask = torch.ones( + (total_seq_len, total_seq_len), dtype=torch.bool, device=device + ) + mask[:video_seq_len, :video_seq_len] = build_frame_causal_video_mask( + video_seq_len, + video_tokens_per_frame, + device=device, + ) + return mask diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam.py new file mode 100644 index 0000000000000000000000000000000000000000..106beb3d83b143a161825b7be0a26f03002dbd6b --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam.py @@ -0,0 +1,1122 @@ +from typing import Any, Optional, Sequence, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from PIL import Image + +from fastwam.utils.logging_config import get_logger + +from .action_dit import ActionDiT +from .helpers.loader import load_wan22_ti2v_5b_components +from .mot import MoT +from .schedulers.scheduler_continuous import WanContinuousFlowMatchScheduler + +logger = get_logger(__name__) + + +class FastWAM(torch.nn.Module): + """MoT world model with video/action experts.""" + + def __init__( + self, + video_expert, + action_expert: ActionDiT, + mot: MoT, + vae, + text_encoder=None, + tokenizer=None, + text_dim: Optional[int] = None, + proprio_dim: Optional[int] = None, + device: str = "cpu", + torch_dtype: torch.dtype = torch.float32, + video_train_shift: float = 5.0, + video_infer_shift: float = 5.0, + video_num_train_timesteps: int = 1000, + action_train_shift: float = 5.0, + action_infer_shift: float = 5.0, + action_num_train_timesteps: int = 1000, + loss_lambda_video: float = 1.0, + loss_lambda_action: float = 1.0, + ): + super().__init__() + self.video_expert = video_expert + self.action_expert = action_expert + self.mot = mot + # Keep trainer compatibility: optimizer and freeze logic use `model.dit`. + self.dit = self.mot + + self.vae = vae + self.text_encoder = text_encoder + self.tokenizer = tokenizer + if text_dim is None: + if self.text_encoder is None: + raise ValueError("`text_dim` is required when `text_encoder` is not loaded.") + text_dim = int(self.text_encoder.dim) + self.text_dim = int(text_dim) + self.proprio_dim = None if proprio_dim is None else int(proprio_dim) + if self.proprio_dim is not None: + self.proprio_encoder = nn.Linear(self.proprio_dim, self.text_dim).to(torch_dtype) + else: + self.proprio_encoder = None + + self.train_video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=video_num_train_timesteps, + shift=video_train_shift, + ) + self.infer_video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=video_num_train_timesteps, + shift=video_infer_shift, + ) + self.train_action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=action_num_train_timesteps, + shift=action_train_shift, + ) + self.infer_action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=action_num_train_timesteps, + shift=action_infer_shift, + ) + # Optional aliases for consistency with Wan22Core naming. + self.train_scheduler = self.train_video_scheduler + self.infer_scheduler = self.infer_video_scheduler + + self.device = torch.device(device) + self.torch_dtype = torch_dtype + self.loss_lambda_video = float(loss_lambda_video) + self.loss_lambda_action = float(loss_lambda_action) + + self.to(self.device) + + @classmethod + def from_wan22_pretrained( + cls, + device: str = "cuda", + torch_dtype: torch.dtype = torch.bfloat16, + model_id: str = "Wan-AI/Wan2.2-TI2V-5B", + tokenizer_model_id: str = "Wan-AI/Wan2.1-T2V-1.3B", + tokenizer_max_len: int = 512, + load_text_encoder: bool = True, + proprio_dim: Optional[int] = None, + redirect_common_files: bool = True, + video_dit_config: dict[str, Any] | None = None, + action_dit_config: dict[str, Any] | None = None, + action_dit_pretrained_path: str | None = None, + skip_dit_load_from_pretrain: bool = False, + mot_checkpoint_mixed_attn: bool = True, + video_train_shift: float = 5.0, + video_infer_shift: float = 5.0, + video_num_train_timesteps: int = 1000, + action_train_shift: float = 5.0, + action_infer_shift: float = 5.0, + action_num_train_timesteps: int = 1000, + loss_lambda_video: float = 1.0, + loss_lambda_action: float = 1.0, + ): + if video_dit_config is None: + raise ValueError("`video_dit_config` is required for FastWAM.from_wan22_pretrained().") + if "text_dim" not in video_dit_config: + raise ValueError("`video_dit_config['text_dim']` is required for FastWAM.") + + components = load_wan22_ti2v_5b_components( + device=device, + torch_dtype=torch_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + tokenizer_max_len=tokenizer_max_len, + redirect_common_files=redirect_common_files, + dit_config=video_dit_config, + skip_dit_load_from_pretrain=skip_dit_load_from_pretrain, + load_text_encoder=load_text_encoder, + ) + + video_expert = components.dit + action_expert = ActionDiT.from_pretrained( + action_dit_config=action_dit_config, + action_dit_pretrained_path=action_dit_pretrained_path, + skip_dit_load_from_pretrain=skip_dit_load_from_pretrain, + device=device, + torch_dtype=torch_dtype, + ) + if int(action_expert.num_heads) != int(video_expert.num_heads): + raise ValueError("ActionDiT `num_heads` must match video expert for MoT mixed attention.") + if int(action_expert.attn_head_dim) != int(video_expert.attn_head_dim): + raise ValueError("ActionDiT `attn_head_dim` must match video expert for MoT mixed attention.") + if int(len(action_expert.blocks)) != int(len(video_expert.blocks)): + raise ValueError("ActionDiT `num_layers` must match video expert.") + + mot = MoT( + mixtures={"video": video_expert, "action": action_expert}, + mot_checkpoint_mixed_attn=mot_checkpoint_mixed_attn, + ) + + model = cls( + video_expert=video_expert, + action_expert=action_expert, + mot=mot, + vae=components.vae, + text_encoder=components.text_encoder, + tokenizer=components.tokenizer, + text_dim=int(video_dit_config["text_dim"]), + proprio_dim=proprio_dim, + device=device, + torch_dtype=torch_dtype, + video_train_shift=video_train_shift, + video_infer_shift=video_infer_shift, + video_num_train_timesteps=video_num_train_timesteps, + action_train_shift=action_train_shift, + action_infer_shift=action_infer_shift, + action_num_train_timesteps=action_num_train_timesteps, + loss_lambda_video=loss_lambda_video, + loss_lambda_action=loss_lambda_action, + ) + model.model_paths = { + "video_dit": components.dit_path, + "vae": components.vae_path, + "text_encoder": components.text_encoder_path, + "tokenizer": components.tokenizer_path, + "action_dit_backbone": ( + "SKIPPED_PRETRAIN" if skip_dit_load_from_pretrain else action_dit_pretrained_path + ), + } + return model + + def to(self, *args, **kwargs): + super().to(*args, **kwargs) + self.mot.to(*args, **kwargs) + if self.text_encoder is not None: + self.text_encoder.to(*args, **kwargs) + self.vae.to(*args, **kwargs) + return self + + @staticmethod + def _check_resize_height_width(height, width, num_frames): + if height % 16 != 0: + height = (height + 15) // 16 * 16 + if width % 16 != 0: + width = (width + 15) // 16 * 16 + if num_frames % 4 != 1: + num_frames = (num_frames + 3) // 4 * 4 + 1 + return height, width, num_frames + + @torch.no_grad() + def encode_prompt(self, prompt: Union[str, Sequence[str]]): + if self.text_encoder is None or self.tokenizer is None: + raise ValueError( + "Prompt encoding requires loaded text encoder/tokenizer. " + "Set `load_text_encoder=true` or provide precomputed `context/context_mask`." + ) + ids, mask = self.tokenizer(prompt, return_mask=True, add_special_tokens=True) + ids = ids.to(self.device) + mask = mask.to(self.device, dtype=torch.bool) + prompt_emb = self.text_encoder(ids, mask) + # FIXME: original implementation's zero padding is visible in cross-attn. + seq_lens = mask.gt(0).sum(dim=1).long() + for i, v in enumerate(seq_lens): + prompt_emb[i, v:] = 0 + mask = torch.ones_like(mask) + return prompt_emb.to(device=self.device), mask + + def _append_proprio_to_context( + self, + context: torch.Tensor, + context_mask: torch.Tensor, + proprio: Optional[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor]: + if self.proprio_encoder is None or proprio is None: + return context, context_mask + if proprio.ndim != 2: + raise ValueError(f"`proprio` must be 2D [B, D], got shape {tuple(proprio.shape)}") + if self.proprio_dim is None or proprio.shape[1] != self.proprio_dim: + raise ValueError( + f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}" + ) + proprio_token = self.proprio_encoder( + proprio.to(device=self.device, dtype=context.dtype).unsqueeze(1) + ).to(dtype=context.dtype) # [B, 1, D] + proprio_mask = torch.ones((context_mask.shape[0], 1), dtype=torch.bool, device=context_mask.device) + return ( + torch.cat([context, proprio_token], dim=1), + torch.cat([context_mask, proprio_mask], dim=1), + ) + + @torch.no_grad() + def _encode_video_latents(self, video_tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)): + z = self.vae.encode( + video_tensor, + device=self.device, + tiled=tiled, + tile_size=tile_size, + tile_stride=tile_stride, + ) + return z + + @torch.no_grad() + def _encode_input_image_latents_tensor(self, input_image: torch.Tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)): + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + image = input_image.to(device=self.device)[0].unsqueeze(1) + z = self.vae.encode([image], device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride) + if isinstance(z, list): + z = z[0].unsqueeze(0) + return z + + def _decode_latents(self, latents, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)): + video_tensor = self.vae.decode(latents, device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride) + video_tensor = video_tensor.squeeze(0).detach().float().clamp(-1, 1) + video_tensor = ((video_tensor + 1.0) * 127.5).to(torch.uint8).cpu() + frames = [] + for t in range(video_tensor.shape[1]): + frame = video_tensor[:, t].permute(1, 2, 0).numpy() + frames.append(Image.fromarray(frame)) + return frames + + def build_inputs(self, sample, tiled: bool = False): + video = sample["video"] + if "context" not in sample or "context_mask" not in sample: + raise ValueError( + "FastWAM training requires `sample['context']` and `sample['context_mask']`." + ) + context = sample["context"] + context_mask = sample["context_mask"] + proprio = sample.get("proprio", None) + if video.ndim != 5: + raise ValueError(f"`sample['video']` must be 5D [B, 3, T, H, W], got shape {tuple(video.shape)}") + if video.shape[1] != 3: + raise ValueError(f"`sample['video']` channel dimension must be 3, got shape {tuple(video.shape)}") + + batch_size, _, num_frames, height, width = video.shape + if height % 16 != 0 or width % 16 != 0: + raise ValueError( + f"Video spatial dims must be multiples of 16, got H={height}, W={width}" + ) + if num_frames % 4 != 1: + raise ValueError(f"Video T must satisfy T % 4 == 1, got T={num_frames}") + if num_frames <= 1: + raise ValueError(f"Video T must be > 1 for action-conditioned training, got T={num_frames}") + + if "action" not in sample: + raise ValueError("`sample['action']` is required for FastWAM training.") + + action = sample["action"] + if action.ndim != 3: + raise ValueError(f"`sample['action']` must be 3D [B, T, a_dim], got shape {tuple(action.shape)}") + action_horizon = int(action.shape[1]) + if action_horizon % (num_frames - 1) != 0: + raise ValueError( + f"`sample['action']` temporal dimension must be divisible by video transitions ({num_frames - 1}), got {action_horizon}" + ) + + action_is_pad = sample.get("action_is_pad", None) + if action_is_pad is not None: + if action_is_pad.ndim != 2: + raise ValueError( + f"`sample['action_is_pad']` must be 2D [B, T], got shape {tuple(action_is_pad.shape)}" + ) + if action_is_pad.shape[0] != batch_size or action_is_pad.shape[1] != action_horizon: + raise ValueError( + "`sample['action_is_pad']` shape mismatch: " + f"got {tuple(action_is_pad.shape)} vs expected ({batch_size}, {action_horizon})" + ) + + image_is_pad = sample.get("image_is_pad", None) + if image_is_pad is not None: + if image_is_pad.ndim != 2: + raise ValueError( + f"`sample['image_is_pad']` must be 2D [B, T], got shape {tuple(image_is_pad.shape)}" + ) + if image_is_pad.shape[0] != batch_size or image_is_pad.shape[1] != num_frames: + raise ValueError( + "`sample['image_is_pad']` shape mismatch: " + f"got {tuple(image_is_pad.shape)} vs expected ({batch_size}, {num_frames})" + ) + + input_video = video.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + input_latents = self._encode_video_latents(input_video, tiled=tiled) + + first_frame_latents = None + fuse_flag = False + if getattr(self.video_expert, "fuse_vae_embedding_in_latents", False): + first_frame_latents = input_latents[:, :, 0:1] + fuse_flag = True + + if context.ndim != 3 or context_mask.ndim != 2: + raise ValueError( + f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}" + ) + context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True) + if self.proprio_encoder is not None: + if proprio is None: + raise ValueError("`sample['proprio']` is required when `proprio_dim` is enabled.") + if proprio.ndim != 3: + raise ValueError(f"`sample['proprio']` must be 3D [B, T, d], got shape {tuple(proprio.shape)}") + if proprio.shape[2] != self.proprio_dim: + raise ValueError( + f"`sample['proprio']` last dim must be {self.proprio_dim}, got {proprio.shape[2]}" + ) + proprio = proprio[:, 0, :] # [B, D] + context, context_mask = self._append_proprio_to_context( + context=context, + context_mask=context_mask, + proprio=proprio.to(device=self.device, dtype=self.torch_dtype), + ) + action = action.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + + if action_is_pad is not None: + action_is_pad = action_is_pad.to(device=self.device, dtype=torch.bool, non_blocking=True) + if image_is_pad is not None: + image_is_pad = image_is_pad.to(device=self.device, dtype=torch.bool, non_blocking=True) + + return { + "context": context, + "context_mask": context_mask, + "input_latents": input_latents, + "first_frame_latents": first_frame_latents, + "fuse_vae_embedding_in_latents": fuse_flag, + "action": action, + "action_is_pad": action_is_pad, + "image_is_pad": image_is_pad, + } + + @torch.no_grad() + def _build_mot_attention_mask( + self, + video_seq_len: int, + action_seq_len: int, + video_tokens_per_frame: int, + device: torch.device, + ) -> torch.Tensor: + total_seq_len = video_seq_len + action_seq_len + mask = torch.zeros((total_seq_len, total_seq_len), dtype=torch.bool, device=device) + + # video -> video + mask[:video_seq_len, :video_seq_len] = self.video_expert.build_video_to_video_mask( + video_seq_len=video_seq_len, + video_tokens_per_frame=video_tokens_per_frame, + device=device, + ) + # action -> action + mask[video_seq_len:, video_seq_len:] = True + # action -> first-frame video only + first_frame_tokens = min(video_tokens_per_frame, video_seq_len) + mask[video_seq_len:, :first_frame_tokens] = True + return mask + + def _compute_video_loss_per_sample( + self, + pred_video: torch.Tensor, + target_video: torch.Tensor, + image_is_pad: Optional[torch.Tensor], + include_initial_video_step: bool, + ) -> torch.Tensor: + video_loss_token = F.mse_loss(pred_video.float(), target_video.float(), reduction="none").mean(dim=(1, 3, 4)) + if image_is_pad is None: + return video_loss_token.mean(dim=1) + + temporal_factor = int(self.vae.temporal_downsample_factor) + if temporal_factor <= 0: + raise ValueError(f"`vae.temporal_downsample_factor` must be positive, got {temporal_factor}.") + if image_is_pad.shape[1] < 1: + raise ValueError("`image_is_pad` must contain at least one frame.") + if (image_is_pad.shape[1] - 1) % temporal_factor != 0: + raise ValueError( + "Cannot align `image_is_pad` with video latent steps: " + f"num_frames={image_is_pad.shape[1]}, temporal_downsample_factor={temporal_factor}." + ) + + tail_is_pad = image_is_pad[:, 1:] + latent_tail_is_pad = tail_is_pad.view(image_is_pad.shape[0], -1, temporal_factor).all(dim=2) + if include_initial_video_step: + video_is_pad = torch.cat([image_is_pad[:, :1], latent_tail_is_pad], dim=1) + else: + video_is_pad = latent_tail_is_pad + + if video_is_pad.shape[1] != video_loss_token.shape[1]: + raise ValueError( + "Video-loss mask shape mismatch: " + f"mask steps={video_is_pad.shape[1]}, loss steps={video_loss_token.shape[1]}." + ) + + valid = (~video_is_pad).to(device=video_loss_token.device, dtype=video_loss_token.dtype) + valid_sum = valid.sum(dim=1).clamp(min=1.0) + return (video_loss_token * valid).sum(dim=1) / valid_sum + + def training_loss(self, sample, tiled: bool = False): + inputs = self.build_inputs(sample, tiled=tiled) + input_latents = inputs["input_latents"] + batch_size = input_latents.shape[0] + context = inputs["context"] + context_mask = inputs["context_mask"] + action = inputs["action"] + action_is_pad = inputs["action_is_pad"] + image_is_pad = inputs["image_is_pad"] + + noise_video = torch.randn_like(input_latents) + timestep_video = self.train_video_scheduler.sample_training_t( + batch_size=batch_size, + device=self.device, + dtype=input_latents.dtype, + ) + latents = self.train_video_scheduler.add_noise(input_latents, noise_video, timestep_video) + target_video = self.train_video_scheduler.training_target(input_latents, noise_video, timestep_video) + + if inputs["first_frame_latents"] is not None: + latents[:, :, 0:1] = inputs["first_frame_latents"] + + noise_action = torch.randn_like(action) + timestep_action = self.train_action_scheduler.sample_training_t( + batch_size=batch_size, + device=self.device, + dtype=action.dtype, + ) + noisy_action = self.train_action_scheduler.add_noise(action, noise_action, timestep_action) + target_action = self.train_action_scheduler.training_target(action, noise_action, timestep_action) + + video_pre = self.video_expert.pre_dit( + x=latents, + timestep=timestep_video, + context=context, + context_mask=context_mask, + action=action, + fuse_vae_embedding_in_latents=inputs["fuse_vae_embedding_in_latents"], + ) + + action_pre = self.action_expert.pre_dit( + action_tokens=noisy_action, + timestep=timestep_action, + context=context, + context_mask=context_mask, + ) + + video_tokens = video_pre["tokens"] + action_tokens = action_pre["tokens"] + + attention_mask = self._build_mot_attention_mask( + video_seq_len=video_tokens.shape[1], + action_seq_len=action_tokens.shape[1], + video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]), + device=video_tokens.device, + ) + tokens_out = self.mot( + embeds_all={ + "video": video_tokens, + "action": action_tokens, + }, + attention_mask=attention_mask, + freqs_all={ + "video": video_pre["freqs"], + "action": action_pre["freqs"], + }, + context_all={ + "video": { + "context": video_pre["context"], + "mask": video_pre["context_mask"], + }, + "action": { + "context": action_pre["context"], + "mask": action_pre["context_mask"], + }, + }, + t_mod_all={ + "video": video_pre["t_mod"], + "action": action_pre["t_mod"], + }, + ) + + pred_video = self.video_expert.post_dit(tokens_out["video"], video_pre) + + pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre) + + include_initial_video_step = inputs["first_frame_latents"] is None + if inputs["first_frame_latents"] is not None: + pred_video = pred_video[:, :, 1:] + target_video = target_video[:, :, 1:] + + loss_video_per_sample = self._compute_video_loss_per_sample( + pred_video=pred_video, + target_video=target_video, + image_is_pad=image_is_pad, + include_initial_video_step=include_initial_video_step, + ) + video_weight = self.train_video_scheduler.training_weight(timestep_video).to( + loss_video_per_sample.device, dtype=loss_video_per_sample.dtype + ) + loss_video = (loss_video_per_sample * video_weight).mean() + + action_loss_token = F.mse_loss(pred_action.float(), target_action.float(), reduction="none").mean(dim=2) # [B, T] + if action_is_pad is not None: + valid = (~action_is_pad).to(device=action_loss_token.device, dtype=action_loss_token.dtype) + valid_sum = valid.sum(dim=1).clamp(min=1.0) + action_loss_per_sample = (action_loss_token * valid).sum(dim=1) / valid_sum + else: + action_loss_per_sample = action_loss_token.mean(dim=1) + + action_weight = self.train_action_scheduler.training_weight(timestep_action).to( + action_loss_per_sample.device, dtype=action_loss_per_sample.dtype + ) + loss_action = (action_loss_per_sample * action_weight).mean() + + loss_total = self.loss_lambda_video * loss_video + self.loss_lambda_action * loss_action + loss_dict = { + "loss_video": self.loss_lambda_video * float(loss_video.detach().item()), + "loss_action": self.loss_lambda_action * float(loss_action.detach().item()), + } + return loss_total, loss_dict + + @torch.no_grad() + def _predict_joint_noise( + self, + latents_video: torch.Tensor, + latents_action: torch.Tensor, + timestep_video: torch.Tensor, + timestep_action: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + fuse_vae_embedding_in_latents: bool, + gt_action: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + video_pre = self.video_expert.pre_dit( + x=latents_video, + timestep=timestep_video, + context=context, + context_mask=context_mask, + action=gt_action, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + action_pre = self.action_expert.pre_dit( + action_tokens=latents_action, + timestep=timestep_action, + context=context, + context_mask=context_mask, + ) + + attention_mask = self._build_mot_attention_mask( + video_seq_len=video_pre["tokens"].shape[1], + action_seq_len=action_pre["tokens"].shape[1], + video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]), + device=video_pre["tokens"].device, + ) + + tokens_out = self.mot( + embeds_all={ + "video": video_pre["tokens"], + "action": action_pre["tokens"], + }, + attention_mask=attention_mask, + freqs_all={ + "video": video_pre["freqs"], + "action": action_pre["freqs"], + }, + context_all={ + "video": { + "context": video_pre["context"], + "mask": video_pre["context_mask"], + }, + "action": { + "context": action_pre["context"], + "mask": action_pre["context_mask"], + }, + }, + t_mod_all={ + "video": video_pre["t_mod"], + "action": action_pre["t_mod"], + }, + ) + + pred_video = self.video_expert.post_dit(tokens_out["video"], video_pre) + pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre) + return pred_video, pred_action + + @torch.no_grad() + def _predict_action_noise( + self, + first_frame_latents: torch.Tensor, + latents_action: torch.Tensor, + timestep_action: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + fuse_vae_embedding_in_latents: bool, + ) -> torch.Tensor: + timestep_video = torch.zeros_like(timestep_action, dtype=first_frame_latents.dtype, device=self.device) + video_pre = self.video_expert.pre_dit( + x=first_frame_latents, + timestep=timestep_video, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + action_pre = self.action_expert.pre_dit( + action_tokens=latents_action, + timestep=timestep_action, + context=context, + context_mask=context_mask, + ) + + attention_mask = self._build_mot_attention_mask( + video_seq_len=video_pre["tokens"].shape[1], + action_seq_len=action_pre["tokens"].shape[1], + video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]), + device=video_pre["tokens"].device, + ) + tokens_out = self.mot( + embeds_all={ + "video": video_pre["tokens"], + "action": action_pre["tokens"], + }, + attention_mask=attention_mask, + freqs_all={ + "video": video_pre["freqs"], + "action": action_pre["freqs"], + }, + context_all={ + "video": { + "context": video_pre["context"], + "mask": video_pre["context_mask"], + }, + "action": { + "context": action_pre["context"], + "mask": action_pre["context_mask"], + }, + }, + t_mod_all={ + "video": video_pre["t_mod"], + "action": action_pre["t_mod"], + }, + ) + pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre) + return pred_action + + @torch.no_grad() + def _predict_action_noise_with_cache( + self, + latents_action: torch.Tensor, + timestep_action: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + video_kv_cache: list[dict[str, torch.Tensor]], + attention_mask: torch.Tensor, + video_seq_len: int, + ) -> torch.Tensor: + action_pre = self.action_expert.pre_dit( + action_tokens=latents_action, + timestep=timestep_action, + context=context, + context_mask=context_mask, + ) + action_tokens = self.mot.forward_action_with_video_cache( + action_tokens=action_pre["tokens"], + action_freqs=action_pre["freqs"], + action_t_mod=action_pre["t_mod"], + action_context_payload={ + "context": action_pre["context"], + "mask": action_pre["context_mask"], + }, + video_kv_cache=video_kv_cache, + attention_mask=attention_mask, + video_seq_len=video_seq_len, + ) + return self.action_expert.post_dit(action_tokens, action_pre) + + @torch.no_grad() + def infer_joint( + self, + prompt: Optional[str], + input_image: torch.Tensor, + num_video_frames: int, + action_horizon: int, + action: Optional[torch.Tensor] = None, # NOTE: this is gt action for conditioning videos, not for action expert + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + test_action_with_infer_action: bool = True, + ) -> dict[str, Any]: + self.eval() + if test_action_with_infer_action: + if seed is None: + raise ValueError("`test_action_with_infer_action=True` requires non-null `seed`.") + action_only_out = self.infer_action( + prompt=prompt, + input_image=input_image.clone(), + action_horizon=action_horizon, + context=context.clone() if context is not None else None, + context_mask=context_mask.clone() if context_mask is not None else None, + num_inference_steps=num_inference_steps, + sigma_shift=sigma_shift, + seed=seed, + rand_device=rand_device, + tiled=tiled, + proprio=proprio.clone() if proprio is not None else None, + )["action"] + + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + _, _, height, width = input_image.shape + checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_video_frames) + if (checked_h, checked_w) != (height, width): + raise ValueError( + f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})" + ) + if checked_t != num_video_frames: + raise ValueError( + f"`num_video_frames` must satisfy T % 4 == 1, got {num_video_frames}" + ) + if action is not None: + if action.ndim == 2: + action = action.unsqueeze(0) + if action.ndim != 3 or action.shape[0] != 1 or action.shape[1] != action_horizon: + # NOTE: This enforces action condition to have the same shape as action horizon to predict, which may be unnecessary + raise ValueError( + f"`action` must have shape [1, T, a_dim] or [T, a_dim], got {tuple(action.shape)} with action_horizon={action_horizon}" + ) + action = action.to(device=self.device, dtype=self.torch_dtype) + if proprio is not None: + if self.proprio_dim is None: + raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.") + if proprio.ndim == 1: + proprio = proprio.unsqueeze(0) + elif proprio.ndim == 2 and proprio.shape[0] == 1: + pass + else: + raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}") + if proprio.shape[1] != self.proprio_dim: + raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}") + proprio = proprio.to(device=self.device, dtype=self.torch_dtype) + + latent_t = (num_video_frames - 1) // self.vae.temporal_downsample_factor + 1 + latent_h = height // self.vae.upsampling_factor + latent_w = width // self.vae.upsampling_factor + + video_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + action_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + latents_video = torch.randn( + (1, self.vae.model.z_dim, latent_t, latent_h, latent_w), + generator=video_generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + latents_action = torch.randn( + (1, action_horizon, self.action_expert.action_dim), + generator=action_generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + + input_image = input_image.to(device=self.device, dtype=self.torch_dtype) + first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled) + latents_video[:, :, 0:1] = first_frame_latents.clone() + fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False)) + + use_prompt = prompt is not None + use_context = context is not None or context_mask is not None + if use_prompt and use_context: + raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.") + if not use_prompt and not use_context: + raise ValueError("Either `prompt` or both `context/context_mask` must be provided.") + + if use_prompt: + context, context_mask = self.encode_prompt(prompt) + else: + if context is None or context_mask is None: + raise ValueError("`context` and `context_mask` must be both provided together.") + if context.ndim == 2: + context = context.unsqueeze(0) + if context_mask.ndim == 1: + context_mask = context_mask.unsqueeze(0) + if context.ndim != 3 or context_mask.ndim != 2: + raise ValueError( + f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}" + ) + context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True) + if proprio is not None: + context, context_mask = self._append_proprio_to_context( + context=context, + context_mask=context_mask, + proprio=proprio, + ) + + infer_timesteps_video, infer_deltas_video = self.infer_video_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_video.dtype, + shift_override=sigma_shift, + ) + infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_action.dtype, + shift_override=sigma_shift, + ) + for step_t_video, step_delta_video, step_t_action, step_delta_action in zip( + infer_timesteps_video, + infer_deltas_video, + infer_timesteps_action, + infer_deltas_action, + ): + timestep_video = step_t_video.unsqueeze(0).to(dtype=latents_video.dtype, device=self.device) + timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device) + + pred_video_posi, pred_action_posi = self._predict_joint_noise( + latents_video=latents_video, + latents_action=latents_action, + timestep_video=timestep_video, + timestep_action=timestep_action, + context=context, + context_mask=context_mask, + fuse_vae_embedding_in_latents=fuse_flag, + gt_action=action, + ) + pred_video = pred_video_posi + pred_action = pred_action_posi + + latents_video = self.infer_video_scheduler.step(pred_video, step_delta_video, latents_video) + latents_action = self.infer_action_scheduler.step(pred_action, step_delta_action, latents_action) + latents_video[:, :, 0:1] = first_frame_latents.clone() + + action_out = latents_action[0].detach().to(device="cpu", dtype=torch.float32) + if test_action_with_infer_action: + if not torch.allclose(action_out, action_only_out, atol=1e-2, rtol=1e-2): + max_abs_diff = (action_out - action_only_out).abs().max().item() + logger.warning( + f"Action from infer_joint and infer_action differ with max abs diff {max_abs_diff:.6f}. " + ) + + return { + "video": self._decode_latents(latents_video, tiled=tiled), + "action": action_out, + } + + @torch.no_grad() + def infer_action( + self, + prompt: Optional[str], + input_image: torch.Tensor, + action_horizon: int, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + ) -> dict[str, Any]: + self.eval() + if str(getattr(self.video_expert, "video_attention_mask_mode", "")) != "first_frame_causal": + raise ValueError( + "`infer_action` requires `video_attention_mask_mode='first_frame_causal'`." + ) + + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + _, _, height, width = input_image.shape + if height % 16 != 0 or width % 16 != 0: + raise ValueError( + f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})" + ) + if proprio is not None: + if self.proprio_dim is None: + raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.") + if proprio.ndim == 1: + proprio = proprio.unsqueeze(0) + elif proprio.ndim == 2 and proprio.shape[0] == 1: + pass + else: + raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}") + if proprio.shape[1] != self.proprio_dim: + raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}") + proprio = proprio.to(device=self.device, dtype=self.torch_dtype) + + generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + latents_action = torch.randn( + (1, action_horizon, self.action_expert.action_dim), + generator=generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + + input_image = input_image.to(device=self.device, dtype=self.torch_dtype) + first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled) + fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False)) + + use_prompt = prompt is not None + use_context = context is not None or context_mask is not None + if use_prompt and use_context: + raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.") + if not use_prompt and not use_context: + raise ValueError("Either `prompt` or both `context/context_mask` must be provided.") + + if use_prompt: + context, context_mask = self.encode_prompt(prompt) + else: + if context is None or context_mask is None: + raise ValueError("`context` and `context_mask` must be both provided together.") + if context.ndim == 2: + context = context.unsqueeze(0) + if context_mask.ndim == 1: + context_mask = context_mask.unsqueeze(0) + if context.ndim != 3 or context_mask.ndim != 2: + raise ValueError( + f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}" + ) + context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True) + if proprio is not None: + context, context_mask = self._append_proprio_to_context( + context=context, + context_mask=context_mask, + proprio=proprio, + ) + + timestep_video = torch.zeros( + (first_frame_latents.shape[0],), + dtype=first_frame_latents.dtype, + device=self.device, + ) + video_pre = self.video_expert.pre_dit( + x=first_frame_latents, + timestep=timestep_video, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_flag, + ) + video_seq_len = int(video_pre["tokens"].shape[1]) + attention_mask = self._build_mot_attention_mask( + video_seq_len=video_seq_len, + action_seq_len=latents_action.shape[1], + video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]), + device=video_pre["tokens"].device, + ) + video_kv_cache = self.mot.prefill_video_cache( + video_tokens=video_pre["tokens"], + video_freqs=video_pre["freqs"], + video_t_mod=video_pre["t_mod"], + video_context_payload={ + "context": video_pre["context"], + "mask": video_pre["context_mask"], + }, + video_attention_mask=attention_mask[:video_seq_len, :video_seq_len], + ) + + infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_action.dtype, + shift_override=sigma_shift, + ) + for step_t_action, step_delta_action in zip(infer_timesteps_action, infer_deltas_action): + timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device) + + pred_action_posi = self._predict_action_noise_with_cache( + latents_action=latents_action, + timestep_action=timestep_action, + context=context, + context_mask=context_mask, + video_kv_cache=video_kv_cache, + attention_mask=attention_mask, + video_seq_len=video_seq_len, + ) + pred_action = pred_action_posi + + latents_action = self.infer_action_scheduler.step(pred_action, step_delta_action, latents_action) + + return { + "action": latents_action[0].detach().to(device="cpu", dtype=torch.float32), + } + + @torch.no_grad() + def infer( + self, + prompt: Optional[str], + input_image: torch.Tensor, + num_frames: int, + action: Optional[torch.Tensor] = None, + action_horizon: Optional[int] = None, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 5.0, + action_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + ): + return self.infer_joint( + prompt=prompt, + input_image=input_image, + num_video_frames=num_frames, + action_horizon=action_horizon, + action=action, + proprio=proprio, + context=context, + context_mask=context_mask, + negative_prompt=negative_prompt, + text_cfg_scale=text_cfg_scale, + num_inference_steps=num_inference_steps, + sigma_shift=sigma_shift, + seed=seed, + rand_device=rand_device, + tiled=tiled, + ) + + def save_checkpoint(self, path, optimizer=None, step=None): + payload = { + "mot": self.mot.state_dict(), + "step": step, + "torch_dtype": str(self.torch_dtype), + } + if self.proprio_encoder is not None: + payload["proprio_encoder"] = self.proprio_encoder.state_dict() + if optimizer is not None: + payload["optimizer"] = optimizer.state_dict() + torch.save(payload, path) + + def load_checkpoint(self, path, optimizer=None): + payload = torch.load(path, map_location="cpu") + if "mot" in payload: + self.mot.load_state_dict(payload["mot"], strict=False) + elif "dit" in payload: + logger.warning("Loading legacy `dit` checkpoint into video expert only.") + self.video_expert.load_state_dict(payload["dit"], strict=False) + else: + raise ValueError(f"Checkpoint missing both `mot` and `dit` keys: {path}") + if self.proprio_encoder is not None: + if "proprio_encoder" in payload: + self.proprio_encoder.load_state_dict(payload["proprio_encoder"], strict=True) + else: + logger.warning("Checkpoint has no `proprio_encoder` weights; keeping current `proprio_encoder` params.") + elif "proprio_encoder" in payload: + logger.warning("Checkpoint contains `proprio_encoder` weights but current model has `proprio_dim=None`; ignoring.") + + if optimizer is not None and "optimizer" in payload: + optimizer.load_state_dict(payload["optimizer"]) + return payload + + def forward(self, *args, **kwargs): + return self.training_loss(*args, **kwargs) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_idm.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_idm.py new file mode 100644 index 0000000000000000000000000000000000000000..9bced1b291b17f47337bcb11f58b8fa48cf3ff5e --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_idm.py @@ -0,0 +1,452 @@ +from typing import Any, Optional + +import torch +import torch.nn.functional as F + +from fastwam.utils.logging_config import get_logger + +from .fastwam_joint import FastWAMJoint + +logger = get_logger(__name__) + + +class FastWAMIDM(FastWAMJoint): + """IDM variant with teacher-forcing video conditioning for action denoising.""" + + # Hardcoded probability: during training, cond-video is noised with this chance. + video_cond_noise_prob = 0.5 + + @torch.no_grad() + def _build_teacher_forcing_attention_mask( + self, + noisy_video_seq_len: int, + cond_video_seq_len: int, + action_seq_len: int, + noisy_video_tokens_per_frame: int, + cond_video_tokens_per_frame: int, + device: torch.device, + ) -> torch.Tensor: + if noisy_video_tokens_per_frame != cond_video_tokens_per_frame: + raise ValueError( + "Teacher-forcing requires identical `tokens_per_frame` for noisy and cond video branches, " + f"got {noisy_video_tokens_per_frame} and {cond_video_tokens_per_frame}." + ) + + noisy_end = noisy_video_seq_len + cond_end = noisy_video_seq_len + cond_video_seq_len + total_seq_len = cond_end + action_seq_len + mask = torch.zeros((total_seq_len, total_seq_len), dtype=torch.bool, device=device) + + # noisy_video -> noisy_video + mask[:noisy_end, :noisy_end] = self.video_expert.build_video_to_video_mask( + video_seq_len=noisy_video_seq_len, + video_tokens_per_frame=noisy_video_tokens_per_frame, + device=device, + ) + # cond_video -> cond_video + mask[noisy_end:cond_end, noisy_end:cond_end] = self.video_expert.build_video_to_video_mask( + video_seq_len=cond_video_seq_len, + video_tokens_per_frame=cond_video_tokens_per_frame, + device=device, + ) + # action -> action + mask[cond_end:, cond_end:] = True + # action -> cond_video only + mask[cond_end:, noisy_end:cond_end] = True + return mask + + def training_loss(self, sample, tiled: bool = False): + inputs = self.build_inputs(sample, tiled=tiled) + input_latents = inputs["input_latents"] + batch_size = input_latents.shape[0] + context = inputs["context"] + context_mask = inputs["context_mask"] + action = inputs["action"] + action_is_pad = inputs["action_is_pad"] + image_is_pad = inputs["image_is_pad"] + fuse_flag = inputs["fuse_vae_embedding_in_latents"] + + # Branch A: noisy video (for video denoising target). + noise_video = torch.randn_like(input_latents) + timestep_video = self.train_video_scheduler.sample_training_t( + batch_size=batch_size, + device=self.device, + dtype=input_latents.dtype, + ) + latents_noisy = self.train_video_scheduler.add_noise(input_latents, noise_video, timestep_video) + target_video = self.train_video_scheduler.training_target(input_latents, noise_video, timestep_video) + if inputs["first_frame_latents"] is not None: + latents_noisy[:, :, 0:1] = inputs["first_frame_latents"] + + # Branch B: noisy action. + noise_action = torch.randn_like(action) + timestep_action = self.train_action_scheduler.sample_training_t( + batch_size=batch_size, + device=self.device, + dtype=action.dtype, + ) + noisy_action = self.train_action_scheduler.add_noise(action, noise_action, timestep_action) + target_action = self.train_action_scheduler.training_target(action, noise_action, timestep_action) + + # Branch C: teacher-forcing cond-video. + # Each sample is independently noised with probability `video_cond_noise_prob`. + cond_noise_mask = torch.rand((batch_size,), device=self.device) < float(self.video_cond_noise_prob) + timestep_video_cond = torch.zeros_like(timestep_video, dtype=input_latents.dtype, device=self.device) + latents_cond = input_latents + if bool(cond_noise_mask.any()): + timestep_video_cond_sampled = self.train_video_scheduler.sample_training_t( + batch_size=batch_size, + device=self.device, + dtype=input_latents.dtype, + ) + timestep_video_cond = torch.where(cond_noise_mask, timestep_video_cond_sampled, timestep_video_cond) + noise_video_cond = torch.randn_like(input_latents) + latents_cond_noisy = self.train_video_scheduler.add_noise( + input_latents, noise_video_cond, timestep_video_cond_sampled + ) + cond_noise_selector = cond_noise_mask.view(batch_size, 1, 1, 1, 1) + latents_cond = torch.where(cond_noise_selector, latents_cond_noisy, input_latents) + if inputs["first_frame_latents"] is not None: + latents_cond = latents_cond.clone() + latents_cond[:, :, 0:1] = inputs["first_frame_latents"] + + video_pre_noisy = self.video_expert.pre_dit( + x=latents_noisy, + timestep=timestep_video, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_flag, + ) + video_pre_cond = self.video_expert.pre_dit( + x=latents_cond, + timestep=timestep_video_cond, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_flag, + ) + if video_pre_noisy["t_mod"].ndim != 4 or video_pre_cond["t_mod"].ndim != 4: + raise ValueError( + "Teacher-forcing requires token-wise `t_mod`; " + "ensure `seperated_timestep=true` and `fuse_vae_embedding_in_latents=true`." + ) + + action_pre = self.action_expert.pre_dit( + action_tokens=noisy_action, + timestep=timestep_action, + context=context, + context_mask=context_mask, + ) + + noisy_video_seq_len = int(video_pre_noisy["tokens"].shape[1]) + cond_video_seq_len = int(video_pre_cond["tokens"].shape[1]) + noisy_video_tokens_per_frame = int(video_pre_noisy["meta"]["tokens_per_frame"]) + cond_video_tokens_per_frame = int(video_pre_cond["meta"]["tokens_per_frame"]) + + # Concatenate [noisy_video, cond_video] as the video expert sequence. + merged_video_tokens = torch.cat([video_pre_noisy["tokens"], video_pre_cond["tokens"]], dim=1) + merged_video_freqs = torch.cat([video_pre_noisy["freqs"], video_pre_cond["freqs"]], dim=0) + merged_video_t_mod = torch.cat([video_pre_noisy["t_mod"], video_pre_cond["t_mod"]], dim=1) + merged_video_context_mask = torch.cat([video_pre_noisy["context_mask"], video_pre_cond["context_mask"]], dim=1) + + attention_mask = self._build_teacher_forcing_attention_mask( + noisy_video_seq_len=noisy_video_seq_len, + cond_video_seq_len=cond_video_seq_len, + action_seq_len=action_pre["tokens"].shape[1], + noisy_video_tokens_per_frame=noisy_video_tokens_per_frame, + cond_video_tokens_per_frame=cond_video_tokens_per_frame, + device=merged_video_tokens.device, + ) + + tokens_out = self.mot( + embeds_all={ + "video": merged_video_tokens, + "action": action_pre["tokens"], + }, + attention_mask=attention_mask, + freqs_all={ + "video": merged_video_freqs, + "action": action_pre["freqs"], + }, + context_all={ + "video": { + "context": video_pre_noisy["context"], + "mask": merged_video_context_mask, + }, + "action": { + "context": action_pre["context"], + "mask": action_pre["context_mask"], + }, + }, + t_mod_all={ + "video": merged_video_t_mod, + "action": action_pre["t_mod"], + }, + ) + + # Only the noisy-video half contributes to video denoising loss. + pred_video_tokens = tokens_out["video"][:, :noisy_video_seq_len] + pred_video = self.video_expert.post_dit(pred_video_tokens, video_pre_noisy) + pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre) + + include_initial_video_step = inputs["first_frame_latents"] is None + if inputs["first_frame_latents"] is not None: + pred_video = pred_video[:, :, 1:] + target_video = target_video[:, :, 1:] + + loss_video_per_sample = self._compute_video_loss_per_sample( + pred_video=pred_video, + target_video=target_video, + image_is_pad=image_is_pad, + include_initial_video_step=include_initial_video_step, + ) + video_weight = self.train_video_scheduler.training_weight(timestep_video).to( + loss_video_per_sample.device, dtype=loss_video_per_sample.dtype + ) + loss_video = (loss_video_per_sample * video_weight).mean() + + action_loss_token = F.mse_loss(pred_action.float(), target_action.float(), reduction="none").mean(dim=2) + if action_is_pad is not None: + valid = (~action_is_pad).to(device=action_loss_token.device, dtype=action_loss_token.dtype) + valid_sum = valid.sum(dim=1).clamp(min=1.0) + action_loss_per_sample = (action_loss_token * valid).sum(dim=1) / valid_sum + else: + action_loss_per_sample = action_loss_token.mean(dim=1) + + action_weight = self.train_action_scheduler.training_weight(timestep_action).to( + action_loss_per_sample.device, dtype=action_loss_per_sample.dtype + ) + loss_action = (action_loss_per_sample * action_weight).mean() + + loss_total = self.loss_lambda_video * loss_video + self.loss_lambda_action * loss_action + loss_dict = { + "loss_video": self.loss_lambda_video * float(loss_video.detach().item()), + "loss_action": self.loss_lambda_action * float(loss_action.detach().item()), + } + return loss_total, loss_dict + + @torch.no_grad() + def infer_action( + self, + prompt: Optional[str], + input_image: torch.Tensor, + action_horizon: int, + num_video_frames: int, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + ) -> dict[str, Any]: + # Reuse infer_joint pipeline and keep infer_action output contract. + out = self.infer_joint( + prompt=prompt, + input_image=input_image, + num_video_frames=num_video_frames, + action_horizon=action_horizon, + action=None, + proprio=proprio, + context=context, + context_mask=context_mask, + negative_prompt=negative_prompt, + text_cfg_scale=text_cfg_scale, + num_inference_steps=num_inference_steps, + sigma_shift=sigma_shift, + seed=seed, + rand_device=rand_device, + tiled=tiled, + test_action_with_infer_action=False, + ) + return {"action": out["action"]} + + @torch.no_grad() + def infer_joint( + self, + prompt: Optional[str], + input_image: torch.Tensor, + num_video_frames: int, + action_horizon: int, + action: Optional[torch.Tensor] = None, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + test_action_with_infer_action: bool = True, + ) -> dict[str, Any]: + del negative_prompt, text_cfg_scale, test_action_with_infer_action + self.eval() + + if action is not None: + logger.warning( + "`FastWAMIDM.infer_joint` ignores `action` input; " + "video is denoised in a standalone first stage." + ) + + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + _, _, height, width = input_image.shape + checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_video_frames) + if (checked_h, checked_w) != (height, width): + raise ValueError( + f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})" + ) + if checked_t != num_video_frames: + raise ValueError( + f"`num_video_frames` must satisfy T % 4 == 1, got {num_video_frames}" + ) + + if proprio is not None: + if self.proprio_dim is None: + raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.") + if proprio.ndim == 1: + proprio = proprio.unsqueeze(0) + elif proprio.ndim == 2 and proprio.shape[0] == 1: + pass + else: + raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}") + if proprio.shape[1] != self.proprio_dim: + raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}") + proprio = proprio.to(device=self.device, dtype=self.torch_dtype) + + latent_t = (num_video_frames - 1) // self.vae.temporal_downsample_factor + 1 + latent_h = height // self.vae.upsampling_factor + latent_w = width // self.vae.upsampling_factor + + video_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + action_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + latents_video = torch.randn( + (1, self.vae.model.z_dim, latent_t, latent_h, latent_w), + generator=video_generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + latents_action = torch.randn( + (1, action_horizon, self.action_expert.action_dim), + generator=action_generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + + input_image = input_image.to(device=self.device, dtype=self.torch_dtype) + first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled) + latents_video[:, :, 0:1] = first_frame_latents.clone() + fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False)) + + use_prompt = prompt is not None + use_context = context is not None or context_mask is not None + if use_prompt and use_context: + raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.") + if not use_prompt and not use_context: + raise ValueError("Either `prompt` or both `context/context_mask` must be provided.") + + if use_prompt: + context, context_mask = self.encode_prompt(prompt) + else: + if context is None or context_mask is None: + raise ValueError("`context` and `context_mask` must be both provided together.") + if context.ndim == 2: + context = context.unsqueeze(0) + if context_mask.ndim == 1: + context_mask = context_mask.unsqueeze(0) + if context.ndim != 3 or context_mask.ndim != 2: + raise ValueError( + f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}" + ) + context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True) + if proprio is not None: + context, context_mask = self._append_proprio_to_context( + context=context, + context_mask=context_mask, + proprio=proprio, + ) + + # Stage 1: denoise video only. + infer_timesteps_video, infer_deltas_video = self.infer_video_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_video.dtype, + shift_override=sigma_shift, + ) + for step_t_video, step_delta_video in zip(infer_timesteps_video, infer_deltas_video): + timestep_video = step_t_video.unsqueeze(0).to(dtype=latents_video.dtype, device=self.device) + pred_video = self.video_expert( + x=latents_video, + timestep=timestep_video, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_flag, + ) + latents_video = self.infer_video_scheduler.step(pred_video, step_delta_video, latents_video) + latents_video[:, :, 0:1] = first_frame_latents.clone() + + # Stage 2: freeze denoised video as cond and denoise action via video K/V cache. + timestep_video_cond = torch.zeros( + (latents_video.shape[0],), dtype=latents_video.dtype, device=self.device + ) + video_pre_cond = self.video_expert.pre_dit( + x=latents_video, + timestep=timestep_video_cond, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_flag, + ) + video_seq_len = int(video_pre_cond["tokens"].shape[1]) + attention_mask = self._build_mot_attention_mask( + video_seq_len=video_seq_len, + action_seq_len=latents_action.shape[1], + video_tokens_per_frame=int(video_pre_cond["meta"]["tokens_per_frame"]), + device=video_pre_cond["tokens"].device, + ) + video_kv_cache = self.mot.prefill_video_cache( + video_tokens=video_pre_cond["tokens"], + video_freqs=video_pre_cond["freqs"], + video_t_mod=video_pre_cond["t_mod"], + video_context_payload={ + "context": video_pre_cond["context"], + "mask": video_pre_cond["context_mask"], + }, + video_attention_mask=attention_mask[:video_seq_len, :video_seq_len], + ) + + infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_action.dtype, + shift_override=sigma_shift, + ) + for step_t_action, step_delta_action in zip(infer_timesteps_action, infer_deltas_action): + timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device) + pred_action = self._predict_action_noise_with_cache( + latents_action=latents_action, + timestep_action=timestep_action, + context=context, + context_mask=context_mask, + video_kv_cache=video_kv_cache, + attention_mask=attention_mask, + video_seq_len=video_seq_len, + ) + latents_action = self.infer_action_scheduler.step(pred_action, step_delta_action, latents_action) + + return { + "video": self._decode_latents(latents_video, tiled=tiled), + "action": latents_action[0].detach().to(device="cpu", dtype=torch.float32), + } diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_joint.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_joint.py new file mode 100644 index 0000000000000000000000000000000000000000..ef44268a216aaf4896bd6d808c8b6aef06108b2a --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_joint.py @@ -0,0 +1,229 @@ +from typing import Any, Optional + +import torch + +from fastwam.utils.logging_config import get_logger + +from .attention_masks import build_joint_denoise_attention_mask +from .fastwam import FastWAM + +logger = get_logger(__name__) + + +class FastWAMJoint(FastWAM): + """MoT variant with bidirectional video/action joint denoising.""" + + @classmethod + def from_wan22_pretrained(cls, **kwargs): + video_dit_config = kwargs.get("video_dit_config", None) + if not isinstance(video_dit_config, dict): + raise ValueError( + "`video_dit_config` must be provided as dict for FastWAMJoint." + ) + if bool(video_dit_config.get("action_conditioned", False)): + raise ValueError( + "FastWAMJoint requires `video_dit_config['action_conditioned']=false`." + ) + return super().from_wan22_pretrained(**kwargs) + + @torch.no_grad() + def _build_mot_attention_mask( + self, + video_seq_len: int, + action_seq_len: int, + video_tokens_per_frame: int, + device: torch.device, + ) -> torch.Tensor: + return build_joint_denoise_attention_mask( + video_seq_len=video_seq_len, + action_seq_len=action_seq_len, + video_tokens_per_frame=video_tokens_per_frame, + device=device, + ) + + @torch.no_grad() + def infer_joint( + self, + prompt: Optional[str], + input_image: torch.Tensor, + num_video_frames: int, + action_horizon: int, + action: Optional[torch.Tensor] = None, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + test_action_with_infer_action: bool = True, + ) -> dict[str, Any]: + if test_action_with_infer_action: + logger.warning( + "`FastWAMJoint.infer_joint` ignores `test_action_with_infer_action=True` " + "and always runs with `test_action_with_infer_action=False`." + ) + return super().infer_joint( + prompt=prompt, + input_image=input_image, + num_video_frames=num_video_frames, + action_horizon=action_horizon, + action=action, + proprio=proprio, + context=context, + context_mask=context_mask, + negative_prompt=negative_prompt, + text_cfg_scale=text_cfg_scale, + num_inference_steps=num_inference_steps, + sigma_shift=sigma_shift, + seed=seed, + rand_device=rand_device, + tiled=tiled, + test_action_with_infer_action=False, + ) + + @torch.no_grad() + def infer_action( + self, + prompt: Optional[str], + input_image: torch.Tensor, + action_horizon: int, + num_video_frames: int, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + ) -> dict[str, Any]: + self.eval() + + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + _, _, height, width = input_image.shape + checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_video_frames) + if (checked_h, checked_w) != (height, width): + raise ValueError( + f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})" + ) + if checked_t != num_video_frames: + raise ValueError( + f"`num_video_frames` must satisfy T % 4 == 1, got {num_video_frames}" + ) + + if proprio is not None: + if self.proprio_dim is None: + raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.") + if proprio.ndim == 1: + proprio = proprio.unsqueeze(0) + elif proprio.ndim == 2 and proprio.shape[0] == 1: + pass + else: + raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}") + if proprio.shape[1] != self.proprio_dim: + raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}") + proprio = proprio.to(device=self.device, dtype=self.torch_dtype) + + latent_t = (num_video_frames - 1) // self.vae.temporal_downsample_factor + 1 + latent_h = height // self.vae.upsampling_factor + latent_w = width // self.vae.upsampling_factor + + video_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + action_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + latents_video = torch.randn( + (1, self.vae.model.z_dim, latent_t, latent_h, latent_w), + generator=video_generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + latents_action = torch.randn( + (1, action_horizon, self.action_expert.action_dim), + generator=action_generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + + input_image = input_image.to(device=self.device, dtype=self.torch_dtype) + first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled) + latents_video[:, :, 0:1] = first_frame_latents.clone() + fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False)) + + use_prompt = prompt is not None + use_context = context is not None or context_mask is not None + if use_prompt and use_context: + raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.") + if not use_prompt and not use_context: + raise ValueError("Either `prompt` or both `context/context_mask` must be provided.") + + if use_prompt: + context, context_mask = self.encode_prompt(prompt) + else: + if context is None or context_mask is None: + raise ValueError("`context` and `context_mask` must be both provided together.") + if context.ndim == 2: + context = context.unsqueeze(0) + if context_mask.ndim == 1: + context_mask = context_mask.unsqueeze(0) + if context.ndim != 3 or context_mask.ndim != 2: + raise ValueError( + f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}" + ) + context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True) + if proprio is not None: + context, context_mask = self._append_proprio_to_context( + context=context, + context_mask=context_mask, + proprio=proprio, + ) + + infer_timesteps_video, infer_deltas_video = self.infer_video_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_video.dtype, + shift_override=sigma_shift, + ) + infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_action.dtype, + shift_override=sigma_shift, + ) + for step_t_video, step_delta_video, step_t_action, step_delta_action in zip( + infer_timesteps_video, + infer_deltas_video, + infer_timesteps_action, + infer_deltas_action, + ): + timestep_video = step_t_video.unsqueeze(0).to(dtype=latents_video.dtype, device=self.device) + timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device) + + pred_video_posi, pred_action_posi = self._predict_joint_noise( + latents_video=latents_video, + latents_action=latents_action, + timestep_video=timestep_video, + timestep_action=timestep_action, + context=context, + context_mask=context_mask, + fuse_vae_embedding_in_latents=fuse_flag, + gt_action=None, + ) + + latents_video = self.infer_video_scheduler.step(pred_video_posi, step_delta_video, latents_video) + latents_action = self.infer_action_scheduler.step(pred_action_posi, step_delta_action, latents_action) + latents_video[:, :, 0:1] = first_frame_latents.clone() + + return { + "action": latents_action[0].detach().to(device="cpu", dtype=torch.float32), + } diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/__init__.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..35bac73c6b329a83d34b02e39a12012305879347 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/__init__.py @@ -0,0 +1,15 @@ +from .io import ModelConfig, hash_model_file, load_state_dict +from .state_dict_converters import ( + wan_video_dit_from_diffusers, + wan_video_dit_state_dict_converter, + wan_video_vae_state_dict_converter, +) + +__all__ = [ + "ModelConfig", + "hash_model_file", + "load_state_dict", + "wan_video_dit_from_diffusers", + "wan_video_dit_state_dict_converter", + "wan_video_vae_state_dict_converter", +] diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/gradient.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/gradient.py new file mode 100644 index 0000000000000000000000000000000000000000..636ef5c2e29a96553bfc6d21e47c70506a3831f4 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/gradient.py @@ -0,0 +1,25 @@ +import torch + + +def create_custom_forward(module): + def custom_forward(*inputs, **kwargs): + return module(*inputs, **kwargs) + return custom_forward + + +def gradient_checkpoint_forward( + model, + use_gradient_checkpointing, + *args, + **kwargs, +): + if use_gradient_checkpointing: + model_output = torch.utils.checkpoint.checkpoint( + create_custom_forward(model), + *args, + **kwargs, + use_reentrant=False, + ) + else: + model_output = model(*args, **kwargs) + return model_output diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/io.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/io.py new file mode 100644 index 0000000000000000000000000000000000000000..26d65d08cfa52a40c6e41d4e6b1d9fd4b65843b6 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/io.py @@ -0,0 +1,207 @@ +import glob +import hashlib +import os +from dataclasses import dataclass +from typing import Dict, Optional, Union + +import torch +from safetensors import safe_open + + +@dataclass +class ModelConfig: + path: Union[str, list[str], None] = None + model_id: Optional[str] = None + origin_file_pattern: Union[str, list[str], None] = None + download_source: Optional[str] = None + local_model_path: Optional[str] = None + skip_download: Optional[bool] = None + state_dict: Optional[Dict[str, torch.Tensor]] = None + + def check_input(self): + if self.path is None and self.model_id is None: + raise ValueError("ModelConfig requires either `path` or (`model_id`, `origin_file_pattern`).") + + def parse_original_file_pattern(self): + if self.origin_file_pattern in [None, "", "./"]: + return "*" + if isinstance(self.origin_file_pattern, list): + return self.origin_file_pattern + if self.origin_file_pattern.endswith("/"): + return self.origin_file_pattern + "*" + return self.origin_file_pattern + + def parse_download_source(self): + if self.download_source is not None: + return self.download_source + env = os.environ.get("DIFFSYNTH_DOWNLOAD_SOURCE") + return env if env is not None else "modelscope" + + def parse_skip_download(self): + if self.skip_download is not None: + return self.skip_download + env = os.environ.get("DIFFSYNTH_SKIP_DOWNLOAD") + if env is None: + return False + return env.lower() == "true" + + def reset_local_model_path(self): + if os.environ.get("DIFFSYNTH_MODEL_BASE_PATH") is not None: + self.local_model_path = os.environ.get("DIFFSYNTH_MODEL_BASE_PATH") + elif self.local_model_path is None: + self.local_model_path = "./checkpoints/" + + def require_downloading(self): + if self.path is not None: + return False + origin_file_pattern = self.parse_original_file_pattern() + local_root = os.path.join(self.local_model_path, self.model_id) + if isinstance(origin_file_pattern, list): + all_exist = True + for pattern in origin_file_pattern: + matches = glob.glob(os.path.join(local_root, pattern)) + if len(matches) == 0: + all_exist = False + break + if all_exist: + return False + else: + if len(glob.glob(os.path.join(local_root, origin_file_pattern))) > 0: + return False + return not self.parse_skip_download() + + def download(self): + origin_file_pattern = self.parse_original_file_pattern() + root = os.path.join(self.local_model_path, self.model_id) + downloaded_files = glob.glob(origin_file_pattern, root_dir=root) + download_source = self.parse_download_source().lower() + if download_source == "modelscope": + from modelscope import snapshot_download + + snapshot_download( + self.model_id, + local_dir=root, + allow_file_pattern=origin_file_pattern, + ignore_file_pattern=downloaded_files, + local_files_only=False, + ) + elif download_source == "huggingface": + from huggingface_hub import snapshot_download as hf_snapshot_download + + hf_snapshot_download( + self.model_id, + local_dir=root, + allow_patterns=origin_file_pattern, + ignore_patterns=downloaded_files, + local_files_only=False, + ) + else: + raise ValueError("`download_source` should be `modelscope` or `huggingface`.") + + def download_if_necessary(self): + self.check_input() + self.reset_local_model_path() + if self.require_downloading(): + self.download() + if self.path is None: + if self.origin_file_pattern in [None, "", "./"]: + self.path = os.path.join(self.local_model_path, self.model_id) + else: + matches = glob.glob(os.path.join(self.local_model_path, self.model_id, self.origin_file_pattern)) + matches.sort() + self.path = matches + if isinstance(self.path, list) and len(self.path) == 1: + self.path = self.path[0] + + +def load_state_dict(file_path, torch_dtype=None, device="cpu"): + if isinstance(file_path, list): + state_dict = {} + for file_path_ in file_path: + state_dict.update(load_state_dict(file_path_, torch_dtype=torch_dtype, device=device)) + return state_dict + if file_path.endswith(".safetensors"): + return load_state_dict_from_safetensors(file_path, torch_dtype=torch_dtype, device=device) + return load_state_dict_from_bin(file_path, torch_dtype=torch_dtype, device=device) + + +def load_state_dict_from_safetensors(file_path, torch_dtype=None, device="cpu"): + state_dict = {} + with safe_open(file_path, framework="pt", device=str(device)) as f: + for key in f.keys(): + value = f.get_tensor(key) + if torch_dtype is not None: + value = value.to(torch_dtype) + state_dict[key] = value + return state_dict + + +def load_state_dict_from_bin(file_path, torch_dtype=None, device="cpu"): + state_dict = torch.load(file_path, map_location=device, weights_only=True) + if len(state_dict) == 1: + if "state_dict" in state_dict: + state_dict = state_dict["state_dict"] + elif "module" in state_dict: + state_dict = state_dict["module"] + elif "model_state" in state_dict: + state_dict = state_dict["model_state"] + if torch_dtype is not None: + for key in state_dict: + if isinstance(state_dict[key], torch.Tensor): + state_dict[key] = state_dict[key].to(torch_dtype) + return state_dict + + +def _load_keys_dict_from_safetensors(file_path): + keys_dict = {} + with safe_open(file_path, framework="pt", device="cpu") as f: + for key in f.keys(): + keys_dict[key] = f.get_slice(key).get_shape() + return keys_dict + + +def _convert_state_dict_to_keys_dict(state_dict): + keys_dict = {} + for key, value in state_dict.items(): + if isinstance(value, torch.Tensor): + keys_dict[key] = list(value.shape) + else: + keys_dict[key] = _convert_state_dict_to_keys_dict(value) + return keys_dict + + +def _load_keys_dict_from_bin(file_path): + state_dict = load_state_dict_from_bin(file_path) + return _convert_state_dict_to_keys_dict(state_dict) + + +def _load_keys_dict(file_path): + if isinstance(file_path, list): + merged = {} + for path in file_path: + merged.update(_load_keys_dict(path)) + return merged + if file_path.endswith(".safetensors"): + return _load_keys_dict_from_safetensors(file_path) + return _load_keys_dict_from_bin(file_path) + + +def _convert_keys_dict_to_single_str(keys_dict, with_shape=True): + keys = [] + for key, value in keys_dict.items(): + if isinstance(key, str): + if isinstance(value, dict): + keys.append(key + "|" + _convert_keys_dict_to_single_str(value, with_shape=with_shape)) + else: + if with_shape: + shape = "_".join(map(str, list(value))) + keys.append(key + ":" + shape) + keys.append(key) + keys.sort() + return ",".join(keys) + + +def hash_model_file(path, with_shape=True): + keys_dict = _load_keys_dict(path) + keys_str = _convert_keys_dict_to_single_str(keys_dict, with_shape=with_shape).encode("UTF-8") + return hashlib.md5(keys_str).hexdigest() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/loader.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/loader.py new file mode 100644 index 0000000000000000000000000000000000000000..99d92ff409fb104c6ebf8cc29905baa12d80e28f --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/loader.py @@ -0,0 +1,221 @@ +from dataclasses import dataclass +import inspect +from typing import Any + +import torch +import time + +from .io import ModelConfig, hash_model_file, load_state_dict +from .state_dict_converters import ( + wan_video_vae_state_dict_converter, +) +from ..wan_video_dit import WanVideoDiT +from ..wan_video_text_encoder import HuggingfaceTokenizer, WanTextEncoder +from ..wan_video_vae import WanVideoVAE38 +from fastwam.utils.logging_config import get_logger + +logger = get_logger(__name__) +SKIPPED_PRETRAIN_SENTINEL = "SKIPPED_PRETRAIN" + + +@dataclass +class Wan22LoadedComponents: + dit: WanVideoDiT + vae: WanVideoVAE38 + text_encoder: WanTextEncoder | None + tokenizer: HuggingfaceTokenizer | None + dit_path: str + vae_path: str + text_encoder_path: str | None + tokenizer_path: str | None + + +WAN22_MODEL_REGISTRY = [ + { + # Example: ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth") + "model_hash": "9c8818c2cbea55eca56c7b447df170da", + "model_name": "wan_video_text_encoder", + "model_class": WanTextEncoder, + }, + { + # Example: ModelConfig(model_id="Wan-AI/Wan2.2-TI2V-5B", origin_file_pattern="diffusion_pytorch_model*.safetensors") + "model_hash": "1f5ab7703c6fc803fdded85ff040c316", + "model_name": "wan_video_dit", + "model_class": WanVideoDiT, + }, + { + # Example: ModelConfig(model_id="Wan-AI/Wan2.2-TI2V-5B", origin_file_pattern="Wan2.2_VAE.pth") + "model_hash": "e1de6c02cdac79f8b739f4d3698cd216", + "model_name": "wan_video_vae", + "model_class": WanVideoVAE38, + "state_dict_converter": wan_video_vae_state_dict_converter, + }, +] + + +def _validate_dit_config(dit_config: dict[str, Any]) -> dict[str, Any]: + if not isinstance(dit_config, dict): + raise ValueError(f"`dit_config` must be a dict, got {type(dit_config)}") + + validated = dict(dit_config) + + signature = inspect.signature(WanVideoDiT.__init__) + allowed_keys = set() + required_keys = set() + for name, param in signature.parameters.items(): + if name == "self": + continue + allowed_keys.add(name) + if param.default is inspect.Signature.empty: + required_keys.add(name) + + unknown_keys = sorted(set(validated) - allowed_keys) + if unknown_keys: + raise ValueError( + f"Unknown keys in `dit_config`: {unknown_keys}. " + f"Allowed keys: {sorted(allowed_keys)}" + ) + + missing_keys = sorted(required_keys - set(validated)) + if missing_keys: + raise ValueError( + f"Missing required keys in `dit_config`: {missing_keys}. " + "Please specify all required WanVideoDiT constructor args." + ) + + return validated + + +def _load_registered_model( + path, + model_name: str, + torch_dtype: torch.dtype, + device: str, + model_kwargs_override: dict[str, Any] | None = None, +): + model_hash = hash_model_file(path) + + matched_config = None + for config in WAN22_MODEL_REGISTRY: + if config["model_hash"] == model_hash and config["model_name"] == model_name: + matched_config = config + break + if matched_config is None: + raise ValueError( + f"Cannot detect model type for {model_name}. File: {path}. " + f"Model hash: {model_hash}. This standalone package follows DiffSynth hash-based loading." + ) + + model_class = matched_config["model_class"] + model_kwargs = dict(matched_config.get("extra_kwargs", {})) + if model_kwargs_override is not None: + model_kwargs.update(model_kwargs_override) + state_dict_converter = matched_config.get("state_dict_converter") + + model = model_class(**model_kwargs) + state_dict = load_state_dict(path, torch_dtype=torch_dtype, device="cpu") + if state_dict_converter is not None: + state_dict = state_dict_converter(state_dict) + + model.load_state_dict(state_dict, strict=False) + model = model.to(device=device, dtype=torch_dtype) + return model + + +def _resolve_configs(model_id: str, tokenizer_model_id: str, redirect_common_files: bool = True): + dit_config = ModelConfig(model_id=model_id, origin_file_pattern="diffusion_pytorch_model*.safetensors") + text_config = ModelConfig(model_id=model_id, origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth") + vae_config = ModelConfig(model_id=model_id, origin_file_pattern="Wan2.2_VAE.pth") + tokenizer_config = ModelConfig(model_id=tokenizer_model_id, origin_file_pattern="google/umt5-xxl/") + + if redirect_common_files: + redirect_dict = { + "models_t5_umt5-xxl-enc-bf16.pth": ("DiffSynth-Studio/Wan-Series-Converted-Safetensors", "models_t5_umt5-xxl-enc-bf16.safetensors"), + "Wan2.2_VAE.pth": ("DiffSynth-Studio/Wan-Series-Converted-Safetensors", "Wan2.2_VAE.safetensors"), + } + text_config.model_id, text_config.origin_file_pattern = redirect_dict[text_config.origin_file_pattern] + vae_config.model_id, vae_config.origin_file_pattern = redirect_dict[vae_config.origin_file_pattern] + return dit_config, text_config, vae_config, tokenizer_config + + +def load_wan22_ti2v_5b_components( + device: str = "cuda", + torch_dtype: torch.dtype = torch.bfloat16, + model_id: str = "Wan-AI/Wan2.2-TI2V-5B", + tokenizer_model_id: str = "Wan-AI/Wan2.1-T2V-1.3B", + tokenizer_max_len: int = 512, + redirect_common_files: bool = True, + dit_config: dict[str, Any] | None = None, + skip_dit_load_from_pretrain: bool = False, + load_text_encoder: bool = True, +): + logger.info("Loading Wan2.2-TI2V-5B components...") + start = time.time() + + if dit_config is None: + raise ValueError("`dit_config` is required for Wan2.2-TI2V-5B loading.") + validated_dit_config = _validate_dit_config(dit_config) + + dit_model_config, text_config, vae_config, tokenizer_config = _resolve_configs( + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + redirect_common_files=redirect_common_files, + ) + + vae_config.download_if_necessary() + if load_text_encoder: + text_config.download_if_necessary() + tokenizer_config.download_if_necessary() + + if skip_dit_load_from_pretrain: + logger.info( + "Skipping pretrained video DiT load (`skip_dit_load_from_pretrain=True`); " + "initializing video expert randomly and expecting checkpoint override." + ) + dit: WanVideoDiT = WanVideoDiT(**validated_dit_config).to(device=device, dtype=torch_dtype) + dit_path = SKIPPED_PRETRAIN_SENTINEL + else: + dit_model_config.download_if_necessary() + dit = _load_registered_model( + dit_model_config.path, + "wan_video_dit", + torch_dtype=torch_dtype, + device=device, + model_kwargs_override=validated_dit_config, + ) + dit_path = str(dit_model_config.path) + text_encoder: WanTextEncoder | None = None + tokenizer: HuggingfaceTokenizer | None = None + text_encoder_path: str | None = None + tokenizer_path: str | None = None + if load_text_encoder: + text_encoder = _load_registered_model( + text_config.path, + "wan_video_text_encoder", + torch_dtype=torch_dtype, + device=device, + ) + tokenizer = HuggingfaceTokenizer( + name=tokenizer_config.path, + seq_len=int(tokenizer_max_len), + clean="whitespace", + ) + text_encoder_path = str(text_config.path) + tokenizer_path = str(tokenizer_config.path) + else: + logger.info( + "Skipping pretrained text encoder/tokenizer load (`load_text_encoder=False`); " + "training must provide cached `context/context_mask`." + ) + vae: WanVideoVAE38 = _load_registered_model(vae_config.path, "wan_video_vae", torch_dtype=torch_dtype, device=device) + logger.info("Finished loading Wan2.2-TI2V-5B components in %.2f seconds.", time.time() - start) + return Wan22LoadedComponents( + dit=dit, + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + dit_path=dit_path, + vae_path=str(vae_config.path), + text_encoder_path=text_encoder_path, + tokenizer_path=tokenizer_path, + ) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/state_dict_converters.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/state_dict_converters.py new file mode 100644 index 0000000000000000000000000000000000000000..3a83b5c554d79278c3b73859eea092a78f3ec528 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/state_dict_converters.py @@ -0,0 +1,90 @@ +def wan_video_vae_state_dict_converter(state_dict): + converted = {} + if "model_state" in state_dict: + state_dict = state_dict["model_state"] + for name, value in state_dict.items(): + converted[f"model.{name}"] = value + return converted + + +def wan_video_dit_from_diffusers(state_dict): + rename_dict = { + "blocks.0.attn1.norm_k.weight": "blocks.0.self_attn.norm_k.weight", + "blocks.0.attn1.norm_q.weight": "blocks.0.self_attn.norm_q.weight", + "blocks.0.attn1.to_k.bias": "blocks.0.self_attn.k.bias", + "blocks.0.attn1.to_k.weight": "blocks.0.self_attn.k.weight", + "blocks.0.attn1.to_out.0.bias": "blocks.0.self_attn.o.bias", + "blocks.0.attn1.to_out.0.weight": "blocks.0.self_attn.o.weight", + "blocks.0.attn1.to_q.bias": "blocks.0.self_attn.q.bias", + "blocks.0.attn1.to_q.weight": "blocks.0.self_attn.q.weight", + "blocks.0.attn1.to_v.bias": "blocks.0.self_attn.v.bias", + "blocks.0.attn1.to_v.weight": "blocks.0.self_attn.v.weight", + "blocks.0.attn2.norm_k.weight": "blocks.0.cross_attn.norm_k.weight", + "blocks.0.attn2.norm_q.weight": "blocks.0.cross_attn.norm_q.weight", + "blocks.0.attn2.to_k.bias": "blocks.0.cross_attn.k.bias", + "blocks.0.attn2.to_k.weight": "blocks.0.cross_attn.k.weight", + "blocks.0.attn2.to_out.0.bias": "blocks.0.cross_attn.o.bias", + "blocks.0.attn2.to_out.0.weight": "blocks.0.cross_attn.o.weight", + "blocks.0.attn2.to_q.bias": "blocks.0.cross_attn.q.bias", + "blocks.0.attn2.to_q.weight": "blocks.0.cross_attn.q.weight", + "blocks.0.attn2.to_v.bias": "blocks.0.cross_attn.v.bias", + "blocks.0.attn2.to_v.weight": "blocks.0.cross_attn.v.weight", + "blocks.0.attn2.add_k_proj.bias": "blocks.0.cross_attn.k_img.bias", + "blocks.0.attn2.add_k_proj.weight": "blocks.0.cross_attn.k_img.weight", + "blocks.0.attn2.add_v_proj.bias": "blocks.0.cross_attn.v_img.bias", + "blocks.0.attn2.add_v_proj.weight": "blocks.0.cross_attn.v_img.weight", + "blocks.0.attn2.norm_added_k.weight": "blocks.0.cross_attn.norm_k_img.weight", + "blocks.0.ffn.net.0.proj.bias": "blocks.0.ffn.0.bias", + "blocks.0.ffn.net.0.proj.weight": "blocks.0.ffn.0.weight", + "blocks.0.ffn.net.2.bias": "blocks.0.ffn.2.bias", + "blocks.0.ffn.net.2.weight": "blocks.0.ffn.2.weight", + "blocks.0.norm2.bias": "blocks.0.norm3.bias", + "blocks.0.norm2.weight": "blocks.0.norm3.weight", + "blocks.0.scale_shift_table": "blocks.0.modulation", + "condition_embedder.text_embedder.linear_1.bias": "text_embedding.0.bias", + "condition_embedder.text_embedder.linear_1.weight": "text_embedding.0.weight", + "condition_embedder.text_embedder.linear_2.bias": "text_embedding.2.bias", + "condition_embedder.text_embedder.linear_2.weight": "text_embedding.2.weight", + "condition_embedder.time_embedder.linear_1.bias": "time_embedding.0.bias", + "condition_embedder.time_embedder.linear_1.weight": "time_embedding.0.weight", + "condition_embedder.time_embedder.linear_2.bias": "time_embedding.2.bias", + "condition_embedder.time_embedder.linear_2.weight": "time_embedding.2.weight", + "condition_embedder.time_proj.bias": "time_projection.1.bias", + "condition_embedder.time_proj.weight": "time_projection.1.weight", + "condition_embedder.image_embedder.ff.net.0.proj.bias": "img_emb.proj.1.bias", + "condition_embedder.image_embedder.ff.net.0.proj.weight": "img_emb.proj.1.weight", + "condition_embedder.image_embedder.ff.net.2.bias": "img_emb.proj.3.bias", + "condition_embedder.image_embedder.ff.net.2.weight": "img_emb.proj.3.weight", + "condition_embedder.image_embedder.norm1.bias": "img_emb.proj.0.bias", + "condition_embedder.image_embedder.norm1.weight": "img_emb.proj.0.weight", + "condition_embedder.image_embedder.norm2.bias": "img_emb.proj.4.bias", + "condition_embedder.image_embedder.norm2.weight": "img_emb.proj.4.weight", + "patch_embedding.bias": "patch_embedding.bias", + "patch_embedding.weight": "patch_embedding.weight", + "scale_shift_table": "head.modulation", + "proj_out.bias": "head.head.bias", + "proj_out.weight": "head.head.weight", + } + converted = {} + for name, value in state_dict.items(): + if name in rename_dict: + converted[rename_dict[name]] = value + else: + probe_name = ".".join(name.split(".")[:1] + ["0"] + name.split(".")[2:]) + if probe_name in rename_dict: + mapped = rename_dict[probe_name] + mapped = ".".join(mapped.split(".")[:1] + [name.split(".")[1]] + mapped.split(".")[2:]) + converted[mapped] = value + return converted + + +def wan_video_dit_state_dict_converter(state_dict): + converted = {} + for name, value in state_dict.items(): + if name.startswith("vace"): + continue + if name.split(".")[0] in ["pose_patch_embedding", "face_adapter", "face_encoder", "motion_encoder"]: + continue + key = name[6:] if name.startswith("model.") else name + converted[key] = value + return converted diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/mot.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/mot.py new file mode 100644 index 0000000000000000000000000000000000000000..1935d6af001a2976bb3a550f3bd34ce013733a3e --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/mot.py @@ -0,0 +1,563 @@ +from __future__ import annotations + +from typing import Dict, Optional + +import torch +import torch.nn as nn + +from .wan_video_dit import flash_attention, modulate, rope_apply +from fastwam.utils.logging_config import get_logger + +logger = get_logger(__name__) + + +class MoT(nn.Module): + def __init__( + self, + mixtures: Dict[str, nn.Module], + mot_checkpoint_mixed_attn: bool = True, + ): + super().__init__() + if not mixtures: + raise ValueError("`mixtures` cannot be empty.") + if "video" not in mixtures or "action" not in mixtures: + raise ValueError("`mixtures` must include both 'video' and 'action' experts.") + + self.mixtures = nn.ModuleDict(mixtures) + self.expert_order = list(self.mixtures.keys()) + self.mot_checkpoint_mixed_attn = mot_checkpoint_mixed_attn + # Opt-in performance switches. Defaults preserve the historical path. + self.omit_all_true_attention_mask = False + self.rope_compute_dtype = torch.float64 + if mot_checkpoint_mixed_attn: + logger.info("Using gradient checkpointing for mixture attention. This will save memory but use more computation.") + + first_expert = self.mixtures[self.expert_order[0]] + self.num_layers = len(first_expert.blocks) + self.num_heads = first_expert.num_heads + self.attn_head_dim = first_expert.attn_head_dim + + for name in self.expert_order[1:]: + expert = self.mixtures[name] + if len(expert.blocks) != self.num_layers: + raise ValueError( + f"All experts must have same number of layers; got {self.num_layers} and {len(expert.blocks)}" + ) + if expert.num_heads != self.num_heads: + raise ValueError( + f"All experts must have same num_heads; got {self.num_heads} and {expert.num_heads}" + ) + if expert.attn_head_dim != self.attn_head_dim: + raise ValueError( + "All experts must have same attn_head_dim; " + f"got {self.attn_head_dim} and {expert.attn_head_dim}" + ) + + logger.info(f"Initialized MoT with experts: {self.expert_order}, num_layers={self.num_layers}") + for name in self.expert_order: + expert = self.mixtures[name] + logger.info(f" Expert '{name}': num_params={sum(p.numel() for p in expert.parameters()) / 1e9:.2f} B") + + @staticmethod + def _split_modulation(block, t_mod: torch.Tensor): + has_seq = len(t_mod.shape) == 4 + chunk_dim = 2 if has_seq else 1 + + base_mod = block.modulation.to(dtype=t_mod.dtype, device=t_mod.device) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (base_mod + t_mod).chunk(6, dim=chunk_dim) + if has_seq: + # means t_mod has separate modulation for each token, otherwise same modulation for all tokens in the block + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + shift_msa.squeeze(2), + scale_msa.squeeze(2), + gate_msa.squeeze(2), + shift_mlp.squeeze(2), + scale_mlp.squeeze(2), + gate_mlp.squeeze(2), + ) + return shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp + + def _mixed_attention( + self, + q_cat: torch.Tensor, + k_cat: torch.Tensor, + v_cat: torch.Tensor, + attention_mask: Optional[torch.Tensor], + ) -> torch.Tensor: + attn_mask = ( + None + if attention_mask is None or self.omit_all_true_attention_mask + else attention_mask.to(device=q_cat.device) + ) + + def _forward(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor: + return flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=attn_mask) + + if self.mot_checkpoint_mixed_attn and self.training: + return torch.utils.checkpoint.checkpoint( + _forward, + q_cat, + k_cat, + v_cat, + use_reentrant=False, + ) + return _forward(q_cat, k_cat, v_cat) + + @staticmethod + def _apply_expert_post_block( + block, + residual_x: torch.Tensor, + mixed_attn_out: torch.Tensor, + gate_msa: torch.Tensor, + shift_mlp: torch.Tensor, + scale_mlp: torch.Tensor, + gate_mlp: torch.Tensor, + context_payload: Optional[dict], + ) -> torch.Tensor: + x = block.gate(residual_x, gate_msa, block.self_attn.o(mixed_attn_out)) + + if context_payload is not None: + context = context_payload.get("context") + if context is not None: + context_mask = context_payload.get("mask") + if context_mask is not None and context_mask.dim() == 3: + context_mask = context_mask.unsqueeze(1) + x = x + block.cross_attn(block.norm3(x), context, ctx_mask=context_mask) + + mlp_input = modulate(block.norm2(x), shift_mlp, scale_mlp) + x = block.gate(x, gate_mlp, block.ffn(mlp_input)) + return x + + def _build_expert_attention_io( + self, + expert, + block, + x: torch.Tensor, + freqs: torch.Tensor, + t_mod: torch.Tensor, + ) -> tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + bool, + ]: + """Build per-expert attention tensors and post-block states. + + Args: + expert: Expert module that owns this `block`; only used to read + `use_gradient_checkpointing`. + block: Transformer block for current layer (`expert.blocks[layer_idx]`). + x: Current expert tokens, shape [B, S, D]. + freqs: RoPE frequencies aligned with token sequence, shape [S, 1, rope_dim]. + t_mod: Time modulation tensor for this expert/layer. + + Returns: + q: Query after q-proj, RMSNorm, and RoPE, shape [B, S, H*Dh]. + k: Key after k-proj, RMSNorm, and RoPE, shape [B, S, H*Dh]. + v: Value after v-proj, shape [B, S, H*Dh]. + residual_x: Original input `x` for residual path in post block. + gate_msa: Gating tensor for self-attention residual branch. + shift_mlp: Shift tensor for MLP modulation. + scale_mlp: Scale tensor for MLP modulation. + gate_mlp: Gating tensor for MLP residual branch. + use_gradient_checkpointing: Whether this expert enables checkpointing. + """ + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self._split_modulation(block, t_mod) + attn_input = modulate(block.norm1(x), shift_msa, scale_msa) + + q = block.self_attn.norm_q(block.self_attn.q(attn_input)) + k = block.self_attn.norm_k(block.self_attn.k(attn_input)) + v = block.self_attn.v(attn_input) + + q = rope_apply(q, freqs, block.num_heads, self.rope_compute_dtype) + k = rope_apply(k, freqs, block.num_heads, self.rope_compute_dtype) + + use_gradient_checkpointing = bool(getattr(expert, "use_gradient_checkpointing", False)) + return ( + q, + k, + v, + x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + use_gradient_checkpointing, + ) + + def _apply_post_with_optional_checkpoint( + self, + block, + residual_x: torch.Tensor, + gate_msa: torch.Tensor, + shift_mlp: torch.Tensor, + scale_mlp: torch.Tensor, + gate_mlp: torch.Tensor, + use_gradient_checkpointing: bool, + mixed_slice: torch.Tensor, + context_payload: Optional[dict], + ) -> torch.Tensor: + """Apply post-attention computations, with optional checkpointing. + + Args: + block: Transformer block for current layer. + residual_x: Residual input tokens before attention update, shape [B, S, D]. + gate_msa: Gating tensor used after mixed self-attention. + shift_mlp: Shift tensor for MLP input modulation. + scale_mlp: Scale tensor for MLP input modulation. + gate_mlp: Gating tensor used after MLP. + use_gradient_checkpointing: If True and training, checkpoint this post block. + mixed_slice: Mixed-attention output for this expert, shape [B, S, H*Dh]. + context_payload: Optional dict for cross-attention. + - `context`: encoder states [B, L, D] + - `mask`: attention mask [B, S, L] or [B, 1, S, L] + + Returns: + Updated expert tokens after self-attn residual, optional cross-attn, and MLP. + """ + def _post_fn( + _mixed_slice: torch.Tensor, + _x: torch.Tensor, + _gate_msa: torch.Tensor, + _shift_mlp: torch.Tensor, + _scale_mlp: torch.Tensor, + _gate_mlp: torch.Tensor, + _block=block, + _context_payload=context_payload, + ) -> torch.Tensor: + return self._apply_expert_post_block( + block=_block, + residual_x=_x, + mixed_attn_out=_mixed_slice, + gate_msa=_gate_msa, + shift_mlp=_shift_mlp, + scale_mlp=_scale_mlp, + gate_mlp=_gate_mlp, + context_payload=_context_payload, + ) + + if use_gradient_checkpointing and self.training: + return torch.utils.checkpoint.checkpoint( + _post_fn, + mixed_slice, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + use_reentrant=False, + ) + return _post_fn( + mixed_slice, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + ) + + def prefill_video_cache( + self, + video_tokens: torch.Tensor, + video_freqs: torch.Tensor, + video_t_mod: torch.Tensor, + video_context_payload: Optional[dict], + video_attention_mask: torch.Tensor, + ) -> list[dict[str, torch.Tensor]]: + """Prefill video branch once and cache per-layer K/V for action denoising. + + Args: + video_tokens: Video tokens before layer 0, shape [B, Sv, D]. + video_freqs: Video RoPE frequencies, shape [Sv, 1, rope_dim]. + video_t_mod: Video time modulation tensor. + video_context_payload: Optional dict for video cross-attention. + - `context`: encoder states [B, L, D] + - `mask`: attention mask [B, Sv, L] or [B, 1, Sv, L] + video_attention_mask: Video self-attention mask, shape [Sv, Sv]. + + Returns: + Layer-wise cache list with length `num_layers`. + Each entry contains: + - `k`: video key tensor [B, Sv, H*Dh] + - `v`: video value tensor [B, Sv, H*Dh] + """ + if "video" not in self.mixtures: + raise ValueError("MoT requires `video` expert for `prefill_video_cache`.") + if video_attention_mask.ndim != 2: + raise ValueError( + f"`video_attention_mask` must be 2D [S,S], got shape {tuple(video_attention_mask.shape)}" + ) + if video_attention_mask.shape[0] != video_attention_mask.shape[1]: + raise ValueError( + f"`video_attention_mask` must be square, got shape {tuple(video_attention_mask.shape)}" + ) + if video_attention_mask.shape[0] != video_tokens.shape[1]: + raise ValueError( + "`video_attention_mask` seq length mismatch: " + f"mask={video_attention_mask.shape[0]} vs tokens={video_tokens.shape[1]}" + ) + + expert = self.mixtures["video"] + x = video_tokens + kv_cache: list[dict[str, torch.Tensor]] = [] + for layer_idx in range(self.num_layers): + block = expert.blocks[layer_idx] + # Build video Q/K/V from current layer input tokens. + ( + q, + k, + v, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + use_gradient_checkpointing, + ) = self._build_expert_attention_io( + expert=expert, + block=block, + x=x, + freqs=video_freqs, + t_mod=video_t_mod, + ) + # Video prefill uses only video self-attention mask. + mixed = self._mixed_attention( + q_cat=q, + k_cat=k, + v_cat=v, + attention_mask=video_attention_mask, + ) + # Update video tokens for the next layer and persist current layer K/V. + x = self._apply_post_with_optional_checkpoint( + block=block, + residual_x=residual_x, + gate_msa=gate_msa, + shift_mlp=shift_mlp, + scale_mlp=scale_mlp, + gate_mlp=gate_mlp, + use_gradient_checkpointing=use_gradient_checkpointing, + mixed_slice=mixed, + context_payload=video_context_payload, + ) + kv_cache.append({"k": k, "v": v}) + return kv_cache + + def forward_action_with_video_cache( + self, + action_tokens: torch.Tensor, + action_freqs: torch.Tensor, + action_t_mod: torch.Tensor, + action_context_payload: Optional[dict], + video_kv_cache: list[dict[str, torch.Tensor]], + attention_mask: torch.Tensor, + video_seq_len: int, + ) -> torch.Tensor: + """Run action branch with cached video K/V instead of recomputing video tokens. + + Args: + action_tokens: Action tokens before layer 0, shape [B, Sa, D]. + action_freqs: Action RoPE frequencies, shape [Sa, 1, rope_dim]. + action_t_mod: Action time modulation tensor. + action_context_payload: Optional dict for action cross-attention. + - `context`: encoder states [B, L, D] + - `mask`: attention mask [B, Sa, L] or [B, 1, Sa, L] + video_kv_cache: Layer-wise cached video K/V from `prefill_video_cache`. + attention_mask: Joint [video+action] mask, shape [Sv+Sa, Sv+Sa]. + video_seq_len: Video token count `Sv` in the joint sequence prefix. + + Returns: + Updated action tokens after all layers, shape [B, Sa, D]. + """ + if "action" not in self.mixtures: + raise ValueError("MoT requires `action` expert for `forward_action_with_video_cache`.") + if len(video_kv_cache) != self.num_layers: + raise ValueError( + f"`video_kv_cache` must contain {self.num_layers} layers, got {len(video_kv_cache)}." + ) + if attention_mask.ndim != 2: + raise ValueError(f"`attention_mask` must be 2D [S,S], got shape {tuple(attention_mask.shape)}") + if attention_mask.shape[0] != attention_mask.shape[1]: + raise ValueError(f"`attention_mask` must be square, got shape {tuple(attention_mask.shape)}") + + action_seq_len = int(action_tokens.shape[1]) + total_seq_len = int(video_seq_len) + action_seq_len + if attention_mask.shape[0] != total_seq_len: + raise ValueError( + "`attention_mask` seq length mismatch: " + f"mask={attention_mask.shape[0]} vs expected_total={total_seq_len}" + ) + # Use the action query rows from the joint [video+action] mask. + action_attention_mask = attention_mask[video_seq_len:total_seq_len, :total_seq_len] + + expert = self.mixtures["action"] + x = action_tokens + for layer_idx in range(self.num_layers): + block = expert.blocks[layer_idx] + # Action query/key/value are still step-dependent and must be recomputed each step. + ( + q_action, + k_action, + v_action, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + use_gradient_checkpointing, + ) = self._build_expert_attention_io( + expert=expert, + block=block, + x=x, + freqs=action_freqs, + t_mod=action_t_mod, + ) + layer_cache = video_kv_cache[layer_idx] + if "k" not in layer_cache or "v" not in layer_cache: + raise ValueError( + f"`video_kv_cache[{layer_idx}]` must contain `k` and `v`." + ) + + k_video = layer_cache["k"] + v_video = layer_cache["v"] + if k_video.shape[1] != video_seq_len or v_video.shape[1] != video_seq_len: + raise ValueError( + f"`video_kv_cache[{layer_idx}]` seq len mismatch, expected {video_seq_len}." + ) + + # Mixed attention: action queries attend to cached video K/V plus current action K/V. + k_cat = torch.cat([k_video, k_action], dim=1) + v_cat = torch.cat([v_video, v_action], dim=1) + mixed = self._mixed_attention( + q_cat=q_action, + k_cat=k_cat, + v_cat=v_cat, + attention_mask=action_attention_mask, + ) + x = self._apply_post_with_optional_checkpoint( + block=block, + residual_x=residual_x, + gate_msa=gate_msa, + shift_mlp=shift_mlp, + scale_mlp=scale_mlp, + gate_mlp=gate_mlp, + use_gradient_checkpointing=use_gradient_checkpointing, + mixed_slice=mixed, + context_payload=action_context_payload, + ) + return x + + def forward( + self, + embeds_all: Dict[str, torch.Tensor], + attention_mask: torch.Tensor, + freqs_all: Dict[str, torch.Tensor], + context_all: Dict[str, Optional[dict]], + t_mod_all: Dict[str, torch.Tensor], + ): + missing = [k for k in self.expert_order if k not in embeds_all] + if missing: + raise ValueError(f"Missing expert tokens for {missing}") + missing = [k for k in self.expert_order if k not in freqs_all] + if missing: + raise ValueError(f"Missing expert freqs for {missing}") + missing = [k for k in self.expert_order if k not in t_mod_all] + if missing: + raise ValueError(f"Missing expert t_mod for {missing}") + + if attention_mask.ndim != 2: + raise ValueError(f"`attention_mask` must be 2D [S, S], got shape {tuple(attention_mask.shape)}") + if attention_mask.shape[0] != attention_mask.shape[1]: + raise ValueError(f"`attention_mask` must be square, got shape {tuple(attention_mask.shape)}") + + tokens_all = {k: v for k, v in embeds_all.items()} + + for layer_idx in range(self.num_layers): + q_chunks = [] + k_chunks = [] + v_chunks = [] + cached = {} + seq_lens = [] + + for name in self.expert_order: + expert = self.mixtures[name] + block = expert.blocks[layer_idx] + x = tokens_all[name] + freqs = freqs_all[name] + t_mod = t_mod_all[name] + + ( + q, + k, + v, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + use_gradient_checkpointing, + ) = self._build_expert_attention_io( + expert=expert, + block=block, + x=x, + freqs=freqs, + t_mod=t_mod, + ) + + q_chunks.append(q) + k_chunks.append(k) + v_chunks.append(v) + seq_lens.append(x.shape[1]) + cached[name] = { + "block": block, + "residual_x": residual_x, + "gate_msa": gate_msa, + "shift_mlp": shift_mlp, + "scale_mlp": scale_mlp, + "gate_mlp": gate_mlp, + "use_gradient_checkpointing": use_gradient_checkpointing, + } + + # 3. concat all tokens for mixed attention + q_cat = torch.cat(q_chunks, dim=1) + k_cat = torch.cat(k_chunks, dim=1) + v_cat = torch.cat(v_chunks, dim=1) + + total_seq = q_cat.shape[1] + if attention_mask.shape[0] != total_seq: + raise ValueError( + "Attention mask seq length mismatch: " + f"mask={attention_mask.shape[0]} vs tokens={total_seq}" + ) + + mixed = self._mixed_attention(q_cat=q_cat, k_cat=k_cat, v_cat=v_cat, attention_mask=attention_mask) + + start = 0 + for name, seq_len in zip(self.expert_order, seq_lens): + # 4. split mixed attention output and apply post-attention blocks for each expert + end = start + seq_len + mixed_slice = mixed[:, start:end, :] + cached_expert = cached[name] + block = cached_expert["block"] + context_payload = context_all.get(name) + + updated_tokens = self._apply_post_with_optional_checkpoint( + block=block, + residual_x=cached_expert["residual_x"], + gate_msa=cached_expert["gate_msa"], + shift_mlp=cached_expert["shift_mlp"], + scale_mlp=cached_expert["scale_mlp"], + gate_mlp=cached_expert["gate_mlp"], + use_gradient_checkpointing=cached_expert["use_gradient_checkpointing"], + mixed_slice=mixed_slice, + context_payload=context_payload, + ) + + tokens_all[name] = updated_tokens + start = end + + return tokens_all diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/schedulers/scheduler_continuous.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/schedulers/scheduler_continuous.py new file mode 100644 index 0000000000000000000000000000000000000000..9028f6c86e2e35bc9efd75dda26d9cf88ff898bb --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/schedulers/scheduler_continuous.py @@ -0,0 +1,88 @@ +import torch + + +class WanContinuousFlowMatchScheduler: + """Continuous-time Flow-Matching scheduler with shift-based sampling.""" + + def __init__(self, num_train_timesteps: int = 1000, shift: float = 5.0, eps: float = 1e-10): + if num_train_timesteps <= 0: + raise ValueError(f"`num_train_timesteps` must be positive, got {num_train_timesteps}") + if shift <= 0: + raise ValueError(f"`shift` must be positive, got {shift}") + self.num_train_timesteps = int(num_train_timesteps) + self.shift = float(shift) + self.eps = float(eps) + self._y_min, self._weight_norm_const = self._precompute_training_weight_stats() + + @staticmethod + def _phi(u: torch.Tensor, shift: float) -> torch.Tensor: + return shift * u / (1.0 + (shift - 1.0) * u) + + def _precompute_training_weight_stats(self) -> tuple[float, float]: + steps = self.num_train_timesteps + u_grid = torch.linspace(1.0, 0.0, steps + 1, dtype=torch.float64)[:-1] + t_grid = self._phi(u_grid, self.shift) * float(steps) + y_grid = torch.exp(-2.0 * ((t_grid - (steps / 2.0)) / steps) ** 2) + y_min = float(y_grid.min().item()) + y_shifted_grid = y_grid - y_min + norm_const = float(y_shifted_grid.mean().item()) + return y_min, norm_const + + def sample_training_t(self, batch_size: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor: + if batch_size <= 0: + raise ValueError(f"`batch_size` must be positive, got {batch_size}") + u = torch.rand((batch_size,), device=device, dtype=torch.float32) + sigma = self._phi(u, self.shift) + timestep = sigma * float(self.num_train_timesteps) + return timestep.to(dtype=dtype) + + def training_weight(self, timestep: torch.Tensor) -> torch.Tensor: + t = timestep.to(dtype=torch.float32) + steps = float(self.num_train_timesteps) + y = torch.exp(-2.0 * ((t - (steps / 2.0)) / steps) ** 2) + y_shifted = y - self._y_min + weight = y_shifted / (self._weight_norm_const + self.eps) + if weight.numel() == 1: + return weight.reshape(()) + return weight + + def add_noise(self, original_samples: torch.Tensor, noise: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor: + sigma = (timestep / float(self.num_train_timesteps)).to( + original_samples.device, dtype=original_samples.dtype + ) + if sigma.ndim == 0: + return (1 - sigma) * original_samples + sigma * noise + sigma = sigma.view(-1, *([1] * (original_samples.ndim - 1))) + return (1 - sigma) * original_samples + sigma * noise + + @staticmethod + def training_target(sample: torch.Tensor, noise: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor: + del timestep + return noise - sample + + def build_inference_schedule( + self, + num_inference_steps: int, + device: torch.device, + dtype: torch.dtype, + shift_override: float | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if num_inference_steps <= 0: + raise ValueError(f"`num_inference_steps` must be positive, got {num_inference_steps}") + shift = self.shift if shift_override is None else float(shift_override) + if shift <= 0: + raise ValueError(f"`shift` must be positive, got {shift}") + + u_steps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + sigma_steps = self._phi(u_steps, shift) + timesteps = sigma_steps[:-1] * float(self.num_train_timesteps) + deltas = sigma_steps[1:] - sigma_steps[:-1] + return timesteps.to(dtype=dtype), deltas.to(dtype=dtype) + + @staticmethod + def step(model_output: torch.Tensor, delta: torch.Tensor, sample: torch.Tensor) -> torch.Tensor: + delta = delta.to(sample.device, dtype=sample.dtype) + if delta.ndim == 0: + return sample + model_output * delta + delta = delta.view(-1, *([1] * (sample.ndim - 1))) + return sample + model_output * delta diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan22.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan22.py new file mode 100644 index 0000000000000000000000000000000000000000..0a7f5a6bccb07193e2fcaf0329a27b28ddda20f0 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan22.py @@ -0,0 +1,405 @@ +import numpy as np +import os +import torch +import torch.nn.functional as F +from PIL import Image +from typing import Any, Optional, Sequence, Union + +from .helpers.loader import load_wan22_ti2v_5b_components +from .schedulers.scheduler_continuous import WanContinuousFlowMatchScheduler +from .wan_video_dit import WanVideoDiT + + +class Wan22Core(torch.nn.Module): + """Standalone Wan2.2-TI2V-5B core without pipeline unit graph.""" + + def __init__( + self, + dit: WanVideoDiT, + vae, + text_encoder, + tokenizer, + device="cpu", + torch_dtype=torch.float32, + train_shift: float = 5.0, + infer_shift: float = 5.0, + num_train_timesteps: int = 1000, + ): + super().__init__() + self.dit = dit + self.vae = vae + self.text_encoder = text_encoder + self.tokenizer = tokenizer + self.train_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=num_train_timesteps, + shift=train_shift, + ) + self.infer_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=num_train_timesteps, + shift=infer_shift, + ) + self.device = torch.device(device) + self.torch_dtype = torch_dtype + self.to(self.device) + + @classmethod + def from_wan22_pretrained( + cls, + device="cuda", + torch_dtype=torch.bfloat16, + model_id="Wan-AI/Wan2.2-TI2V-5B", + tokenizer_model_id="Wan-AI/Wan2.1-T2V-1.3B", + tokenizer_max_len: int = 512, + redirect_common_files=True, + dit_config: dict[str, Any] | None = None, + train_shift: float = 5.0, + infer_shift: float = 5.0, + num_train_timesteps: int = 1000, + ): + if dit_config is None: + raise ValueError("`dit_config` is required for Wan22Core.from_wan22_pretrained().") + components = load_wan22_ti2v_5b_components( + device=device, + torch_dtype=torch_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + tokenizer_max_len=tokenizer_max_len, + redirect_common_files=redirect_common_files, + dit_config=dit_config, + ) + model = cls( + dit=components.dit, + vae=components.vae, + text_encoder=components.text_encoder, + tokenizer=components.tokenizer, + device=device, + torch_dtype=torch_dtype, + train_shift=train_shift, + infer_shift=infer_shift, + num_train_timesteps=num_train_timesteps, + ) + model.model_paths = { + "dit": components.dit_path, + "vae": components.vae_path, + "text_encoder": components.text_encoder_path, + "tokenizer": components.tokenizer_path, + } + return model + + def to(self, *args, **kwargs): + super().to(*args, **kwargs) + self.dit.to(*args, **kwargs) + self.text_encoder.to(*args, **kwargs) + self.vae.to(*args, **kwargs) + return self + + @staticmethod + def _check_resize_height_width(height, width, num_frames): + if height % 16 != 0: + height = (height + 15) // 16 * 16 + if width % 16 != 0: + width = (width + 15) // 16 * 16 + if num_frames % 4 != 1: + num_frames = (num_frames + 3) // 4 * 4 + 1 + return height, width, num_frames + + def encode_prompt(self, prompt: Union[str, Sequence[str]]): + ids, mask = self.tokenizer(prompt, return_mask=True, add_special_tokens=True) + ids = ids.to(self.device) + mask = mask.to(self.device, dtype=torch.bool) + prompt_emb = self.text_encoder(ids, mask) + return prompt_emb.to(device=self.device), mask + + def _encode_video_latents(self, video_tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)): + z = self.vae.encode( + video_tensor, + device=self.device, + tiled=tiled, + tile_size=tile_size, + tile_stride=tile_stride, + ) + return z + + def _encode_input_image_latents_tensor(self, input_image: torch.Tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)): + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + image = input_image.to(device=self.device)[0].unsqueeze(1) + z = self.vae.encode([image], device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride) + if isinstance(z, list): + z = z[0].unsqueeze(0) + return z + + def _decode_latents(self, latents, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)): + video_tensor = self.vae.decode(latents, device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride) + video_tensor = video_tensor.squeeze(0).detach().float().clamp(-1, 1) + video_tensor = ((video_tensor + 1.0) * 127.5).to(torch.uint8).cpu() + frames = [] + for t in range(video_tensor.shape[1]): + frame = video_tensor[:, t].permute(1, 2, 0).numpy() + frames.append(Image.fromarray(frame)) + return frames + + def _model_fn(self, + latents: torch.Tensor, # [B, C, T, H, W] + timestep: torch.Tensor, # [B] or [1] (inference mode) + context: torch.Tensor, # [B, L, D] + context_mask: Optional[torch.Tensor] = None, # [B, L] + action: Optional[torch.Tensor] = None, # [B, T-1, a_dim] + fuse_vae_embedding_in_latents=False): + return self.dit( + x=latents, + timestep=timestep, + context=context, + context_mask=context_mask, + action=action, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + + def build_inputs(self, sample, tiled=False): + video = sample["video"] + prompt = sample["prompt"] + if not isinstance(video, torch.Tensor): + raise TypeError( + f"`sample['video']` must be a torch.Tensor with shape [B, 3, T, H, W], got {type(video)}" + ) + if video.ndim != 5: + raise ValueError(f"`sample['video']` must be 5D [B, 3, T, H, W], got shape {tuple(video.shape)}") + if video.shape[1] != 3: + raise ValueError(f"`sample['video']` channel dimension must be 3, got shape {tuple(video.shape)}") + + if isinstance(prompt, str): + prompt_list = [prompt] + elif isinstance(prompt, Sequence): + prompt_list = list(prompt) + else: + raise TypeError(f"`sample['prompt']` must be str or list[str], got {type(prompt)}") + + batch_size, _, num_frames, height, width = video.shape + if len(prompt_list) != batch_size: + raise ValueError( + f"Prompt batch mismatch: got len(prompt)={len(prompt_list)} and video batch={batch_size}" + ) + if height % 16 != 0 or width % 16 != 0: + raise ValueError( + f"Video spatial dims must be multiples of 16, got H={height}, W={width}" + ) + if num_frames % 4 != 1: + raise ValueError(f"Video T must satisfy T % 4 == 1, got T={num_frames}") + + input_video = video.to(device=self.device, dtype=self.torch_dtype) + input_latents = self._encode_video_latents(input_video, tiled=tiled) # [B, C, Latent_T, H', W'] + + first_frame_latents = None + fuse_flag = False + if getattr(self.dit, "fuse_vae_embedding_in_latents", False): + first_frame_latents = input_latents[:, :, 0:1] + fuse_flag = True + context, context_mask = self.encode_prompt(prompt_list) + + action = None + if "action" in sample: + action = sample["action"] + if not isinstance(action, torch.Tensor): + raise TypeError( + f"`sample['action']` must be a torch.Tensor with shape [B, T, a_dim], got {type(action)}" + ) + if action.ndim != 3: + raise ValueError(f"`sample['action']` must be 3D [B, T, a_dim], got shape {tuple(action.shape)}") + if action.shape[1] <= 0: + raise ValueError(f"`sample['action']` temporal dimension must be positive, got {action.shape[1]}") + if action.shape[1] % (num_frames - 1) != 0: + raise ValueError( + "`sample['action']` temporal dimension must be divisible by video transitions " + f"({num_frames - 1}), got {action.shape[1]}" + ) + action = action.to(device=self.device, dtype=self.torch_dtype) + + return { + "context": context, + "context_mask": context_mask, + "input_latents": input_latents, + "first_frame_latents": first_frame_latents, + "fuse_vae_embedding_in_latents": fuse_flag, + "action": action, + } + + def training_loss(self, sample, tiled=False): + inputs = self.build_inputs(sample, tiled=tiled) + input_latents = inputs["input_latents"] + batch_size = input_latents.shape[0] + action = inputs["action"] + context = inputs["context"] + context_mask = inputs["context_mask"] + + # 1. Continuous timestep sampling and noise injection. + noise = torch.randn_like(input_latents) + timestep = self.train_scheduler.sample_training_t( + batch_size=batch_size, + device=self.device, + dtype=input_latents.dtype, + ) + latents = self.train_scheduler.add_noise(input_latents, noise, timestep) + target = self.train_scheduler.training_target(input_latents, noise, timestep) + + # 2. fix first latent + if inputs["first_frame_latents"] is not None: + latents[:, :, 0: 1] = inputs["first_frame_latents"] + + pred = self._model_fn( + latents=latents, # [B, C, Latent_T, H', W'] + timestep=timestep, + context=context, + context_mask=context_mask, + action=action, + fuse_vae_embedding_in_latents=inputs["fuse_vae_embedding_in_latents"], + ) + if inputs["first_frame_latents"] is not None: + pred = pred[:, :, 1:] + target = target[:, :, 1:] + loss_per_sample = F.mse_loss(pred.float(), target.float(), reduction="none").mean(dim=(1, 2, 3, 4)) + sample_weight = self.train_scheduler.training_weight(timestep).to( + loss_per_sample.device, dtype=loss_per_sample.dtype + ) + loss_total = (loss_per_sample * sample_weight).mean() + loss_dict = { + "loss_video": float(loss_total.detach().item()), + } + return loss_total, loss_dict + + @torch.no_grad() + def infer( + self, + prompt: str, + input_image: torch.Tensor, + num_frames: int, + action: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 5.0, + action_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + **kwargs + ): + self.eval() + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + _, _, height, width = input_image.shape + checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_frames) + if (checked_h, checked_w) != (height, width): + raise ValueError( + f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})" + ) + if checked_t != num_frames: + raise ValueError( + f"`num_frames` must satisfy T % 4 == 1, got {num_frames}" + ) + + latent_t = (num_frames - 1) // self.vae.temporal_downsample_factor + 1 + latent_h = height // self.vae.upsampling_factor + latent_w = width // self.vae.upsampling_factor + + generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + latents = torch.randn( + (1, self.vae.model.z_dim, latent_t, latent_h, latent_w), + generator=generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + input_image = input_image.to(device=self.device, dtype=self.torch_dtype) + if action is not None: + action = action.to(device=self.device, dtype=latents.dtype) + if action.ndim != 2: + raise ValueError(f"`action` must be 2D [T, a_dim], got shape {tuple(action.shape)}") + action = action.unsqueeze(0) # [1, T, a_dim] + if action.shape[1] % (num_frames - 1) != 0: + raise ValueError( + "`action` temporal dimension must be divisible by `num_frames - 1`, " + f"got {action.shape[1]} vs {num_frames - 1}" + ) + if action_cfg_scale != 1.0 and action is None: + raise ValueError("`action_cfg_scale` != 1.0 requires non-null `action` input.") + + z = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled) + latents[:, :, 0:1] = z + first_frame_latents = z + fuse_flag = True + + context_posi, context_posi_mask = self.encode_prompt(prompt) + context_nega = None + context_nega_mask = None + if text_cfg_scale != 1.0: + context_nega, context_nega_mask = self.encode_prompt("" if negative_prompt is None else negative_prompt) + action_nega = torch.zeros_like(action) if (action is not None and action_cfg_scale != 1.0) else None + + infer_timesteps, infer_deltas = self.infer_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents.dtype, + shift_override=sigma_shift, + ) + for step_t, step_delta in zip(infer_timesteps, infer_deltas): + timestep = step_t.unsqueeze(0).to(dtype=latents.dtype, device=self.device) + noise_pred_posi = self._model_fn( + latents=latents, + timestep=timestep, + context=context_posi, + context_mask=context_posi_mask, + action=action, + fuse_vae_embedding_in_latents=fuse_flag, + ) + noise_pred = noise_pred_posi + if context_nega is not None: + noise_pred_text_nega = self._model_fn( + latents=latents, + timestep=timestep, + context=context_nega, + context_mask=context_nega_mask, + action=action, + fuse_vae_embedding_in_latents=fuse_flag, + ) + noise_pred = noise_pred + (text_cfg_scale - 1.0) * (noise_pred_posi - noise_pred_text_nega) + if action_nega is not None: + noise_pred_action_nega = self._model_fn( + latents=latents, + timestep=timestep, + context=context_posi, + context_mask=context_posi_mask, + action=action_nega, + fuse_vae_embedding_in_latents=fuse_flag, + ) + noise_pred = noise_pred + (action_cfg_scale - 1.0) * (noise_pred_posi - noise_pred_action_nega) + latents = self.infer_scheduler.step(noise_pred, step_delta, latents) + latents[:, :, 0:1] = first_frame_latents + + return {"video": self._decode_latents(latents, tiled=tiled)} + + def save_checkpoint(self, path, optimizer=None, step=None): + payload = { + "dit": self.dit.state_dict(), + "step": step, + "torch_dtype": str(self.torch_dtype), + } + if optimizer is not None: + payload["optimizer"] = optimizer.state_dict() + torch.save(payload, path) + + def load_checkpoint(self, path, optimizer=None): + payload = torch.load(path, map_location="cpu") + self.dit.load_state_dict(payload["dit"], strict=False) + if optimizer is not None and "optimizer" in payload: + optimizer.load_state_dict(payload["optimizer"]) + return payload + + def forward(self, *args, **kwargs): + return self.training_loss(*args, **kwargs) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_dit.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_dit.py new file mode 100644 index 0000000000000000000000000000000000000000..26802ddc9e7f4004b85894696f1db7b18ddca087 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_dit.py @@ -0,0 +1,673 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import math +from typing import Any, Dict, Tuple, Optional +from einops import rearrange +from .helpers.gradient import gradient_checkpoint_forward + +from fastwam.utils.logging_config import get_logger + +logger = get_logger(__name__) + + +def flash_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, num_heads: int, ctx_mask: Optional[torch.Tensor] = None, compatibility_mode=True): + if compatibility_mode: + q = rearrange(q, "b s (n d) -> b n s d", n=num_heads) + k = rearrange(k, "b s (n d) -> b n s d", n=num_heads) + v = rearrange(v, "b s (n d) -> b n s d", n=num_heads) + x = F.scaled_dot_product_attention(q, k, v, attn_mask=ctx_mask) + x = rearrange(x, "b n s d -> b s (n d)", n=num_heads) + return x + else: + raise NotImplementedError("Only compatibility mode is implemented for flash attention. Please set compatibility_mode=True.") + + + +def modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor): + return (x * (1 + scale) + shift) + + +def sinusoidal_embedding_1d(dim, position): + sinusoid = torch.outer(position.type(torch.float64), torch.pow( + 10000, -torch.arange(dim//2, dtype=torch.float64, device=position.device).div(dim//2))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + return x.to(position.dtype) + + +def precompute_freqs_cis_3d(dim: int, end: int = 1024, theta: float = 10000.0): + # 3d rope precompute + f_freqs_cis = precompute_freqs_cis(dim - 2 * (dim // 3), end, theta) + h_freqs_cis = precompute_freqs_cis(dim // 3, end, theta) + w_freqs_cis = precompute_freqs_cis(dim // 3, end, theta) + return f_freqs_cis, h_freqs_cis, w_freqs_cis + + +def precompute_freqs_cis(dim: int, end: int = 1024, theta: float = 10000.0): + # 1d rope precompute + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2) + [: (dim // 2)].double() / dim)) + freqs = torch.outer(torch.arange(end, device=freqs.device), freqs) + freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex dtype follows freqs + return freqs_cis + + +def rope_apply(x, freqs, num_heads, compute_dtype=torch.float64): + x = rearrange(x, "b s (n d) -> b s n d", n=num_heads) + if compute_dtype not in (torch.float32, torch.float64): + raise ValueError(f"RoPE compute_dtype must be float32 or float64, got {compute_dtype}") + complex_dtype = torch.complex64 if compute_dtype == torch.float32 else torch.complex128 + x_out = torch.view_as_complex(x.to(compute_dtype).reshape( + x.shape[0], x.shape[1], x.shape[2], -1, 2)) + if compute_dtype == torch.float64 and freqs.device.type == "npu": + # Preserve the historical NPU compatibility path. + freqs = freqs.to(torch.complex64) + else: + freqs = freqs.to(complex_dtype) + x_out = torch.view_as_real(x_out * freqs).flatten(2) + return x_out.to(x.dtype) + + +def create_group_causal_attn_mask( + num_temporal_groups: int, num_query_per_group: int, num_key_per_group: int, mode: str = "causal" +) -> torch.Tensor: + """ + Creates a group-based attention mask for scaled dot-product attention with two modes: + 'causal' and 'group_diagonal'. + + Parameters: + - num_temporal_groups (int): The number of temporal groups (e.g., frames in a video sequence). + - num_query_per_group (int): The number of query tokens per temporal group. (e.g., latent tokens in a frame, H x W). + - num_key_per_group (int): The number of key tokens per temporal group. (e.g., action tokens per frame). + - mode (str): The mode of the attention mask. Options are: + - 'causal': Query tokens can attend to key tokens from the same or previous temporal groups. + - 'group_diagonal': Query tokens can attend only to key tokens from the same temporal group. + + Returns: + - attn_mask (torch.Tensor): A boolean tensor of shape (L, S), where: + - L = num_temporal_groups * num_query_per_group (total number of query tokens) + - S = num_temporal_groups * num_key_per_group (total number of key tokens) + The mask indicates where attention is allowed (True) and disallowed (False). + + Example: + Input: + num_temporal_groups = 3 + num_query_per_group = 4 + num_key_per_group = 2 + Output: + Causal Mask Shape: torch.Size([12, 6]) + Group Diagonal Mask Shape: torch.Size([12, 6]) + if mode='causal': + tensor([[ True, True, False, False, False, False], + [ True, True, False, False, False, False], + [ True, True, False, False, False, False], + [ True, True, False, False, False, False], + [ True, True, True, True, False, False], + [ True, True, True, True, False, False], + [ True, True, True, True, False, False], + [ True, True, True, True, False, False], + [ True, True, True, True, True, True], + [ True, True, True, True, True, True], + [ True, True, True, True, True, True], + [ True, True, True, True, True, True]]) + + if mode='group_diagonal': + tensor([[ True, True, False, False, False, False], + [ True, True, False, False, False, False], + [ True, True, False, False, False, False], + [ True, True, False, False, False, False], + [False, False, True, True, False, False], + [False, False, True, True, False, False], + [False, False, True, True, False, False], + [False, False, True, True, False, False], + [False, False, False, False, True, True], + [False, False, False, False, True, True], + [False, False, False, False, True, True], + [False, False, False, False, True, True]]) + + """ + assert mode in ["causal", "group_diagonal"], f"Mode {mode} must be 'causal' or 'group_diagonal'" + + # Total number of query and key tokens + total_num_query_tokens = num_temporal_groups * num_query_per_group # Total number of query tokens (L) + total_num_key_tokens = num_temporal_groups * num_key_per_group # Total number of key tokens (S) + + # Generate time indices for query and key tokens (shape: [L] and [S]) + query_time_indices = torch.arange(num_temporal_groups).repeat_interleave(num_query_per_group) # Shape: [L] + key_time_indices = torch.arange(num_temporal_groups).repeat_interleave(num_key_per_group) # Shape: [S] + + # Expand dimensions to compute outer comparison + query_time_indices = query_time_indices.unsqueeze(1) # Shape: [L, 1] + key_time_indices = key_time_indices.unsqueeze(0) # Shape: [1, S] + + if mode == "causal": + # Causal Mode: Query can attend to keys where key_time <= query_time + attn_mask = query_time_indices >= key_time_indices # Shape: [L, S] + elif mode == "group_diagonal": + # Group Diagonal Mode: Query can attend only to keys where key_time == query_time + attn_mask = query_time_indices == key_time_indices # Shape: [L, S] + + assert attn_mask.shape == (total_num_query_tokens, total_num_key_tokens), "Attention mask shape mismatch" + return attn_mask + + +class RMSNorm(nn.Module): + def __init__(self, dim, eps=1e-5): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps) + + def forward(self, x): + dtype = x.dtype + return self.norm(x.float()).to(dtype) * self.weight + + +class AttentionModule(nn.Module): + def __init__(self, num_heads): + super().__init__() + self.num_heads = num_heads + + def forward(self, q, k, v, ctx_mask=None): + x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=ctx_mask) + return x + + +class SelfAttention(nn.Module): + def __init__(self, hidden_dim: int, attn_head_dim: int, num_heads: int, eps: float = 1e-6): + super().__init__() + self.hidden_dim = hidden_dim + self.num_heads = num_heads + self.attn_head_dim = attn_head_dim + self.attn_hidden_dim = self.num_heads * self.attn_head_dim + + self.q = nn.Linear(hidden_dim, self.attn_hidden_dim) + self.k = nn.Linear(hidden_dim, self.attn_hidden_dim) + self.v = nn.Linear(hidden_dim, self.attn_hidden_dim) + self.o = nn.Linear(self.attn_hidden_dim, hidden_dim) + self.norm_q = RMSNorm(self.attn_hidden_dim, eps=eps) + self.norm_k = RMSNorm(self.attn_hidden_dim, eps=eps) + + # self.attn = AttentionModule(self.num_heads) + + def forward(self, x, freqs, self_attn_mask: Optional[torch.Tensor] = None): + q = self.norm_q(self.q(x)) + k = self.norm_k(self.k(x)) + v = self.v(x) + q = rope_apply(q, freqs, self.num_heads) + k = rope_apply(k, freqs, self.num_heads) + x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=self_attn_mask) + return self.o(x) + + +class CrossAttention(nn.Module): + def __init__(self, hidden_dim: int, attn_head_dim: int, num_heads: int, eps: float = 1e-6,): + super().__init__() + self.hidden_dim = hidden_dim + self.num_heads = num_heads + self.attn_head_dim = attn_head_dim + self.attn_hidden_dim = self.num_heads * self.attn_head_dim + + self.q = nn.Linear(hidden_dim, self.attn_hidden_dim) + self.k = nn.Linear(hidden_dim, self.attn_hidden_dim) + self.v = nn.Linear(hidden_dim, self.attn_hidden_dim) + self.o = nn.Linear(self.attn_hidden_dim, hidden_dim) + self.norm_q = RMSNorm(self.attn_hidden_dim, eps=eps) + self.norm_k = RMSNorm(self.attn_hidden_dim, eps=eps) + + # self.attn = AttentionModule(self.num_heads) + + def forward(self, x: torch.Tensor, ctx: torch.Tensor, ctx_mask: Optional[torch.Tensor] = None): + q = self.norm_q(self.q(x)) + k = self.norm_k(self.k(ctx)) + v = self.v(ctx) + x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=ctx_mask) + return self.o(x) + + +class GateModule(nn.Module): + def __init__(self,): + super().__init__() + + def forward(self, x, gate, residual): + return x + gate * residual + +class DiTBlock(nn.Module): + def __init__(self, hidden_dim: int, attn_head_dim: int, num_heads: int, ffn_dim: int, eps: float = 1e-6): + super().__init__() + self.hidden_dim = hidden_dim + self.attn_head_dim = attn_head_dim + self.num_heads = num_heads + self.ffn_dim = ffn_dim + + self.self_attn = SelfAttention(hidden_dim, attn_head_dim, num_heads, eps) + self.cross_attn = CrossAttention( + hidden_dim, attn_head_dim, num_heads, eps) + self.norm1 = nn.LayerNorm(hidden_dim, eps=eps, elementwise_affine=False) + self.norm2 = nn.LayerNorm(hidden_dim, eps=eps, elementwise_affine=False) + self.norm3 = nn.LayerNorm(hidden_dim, eps=eps) + self.ffn = nn.Sequential(nn.Linear(hidden_dim, ffn_dim), nn.GELU( + approximate='tanh'), nn.Linear(ffn_dim, hidden_dim)) + self.modulation = nn.Parameter(torch.randn(1, 6, hidden_dim) / hidden_dim**0.5) + self.gate = GateModule() + + def forward(self, x, context, t_mod, freqs, context_mask=None, self_attn_mask: Optional[torch.Tensor] = None): + if context_mask is not None and context_mask.dim() == 3: + context_mask = context_mask.unsqueeze(1) # (B, 1, seq_len, context_len), 1 for heads + has_seq = len(t_mod.shape) == 4 + chunk_dim = 2 if has_seq else 1 + # msa: multi-head self-attention mlp: multi-layer perceptron + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + self.modulation.to(dtype=t_mod.dtype, device=t_mod.device) + t_mod).chunk(6, dim=chunk_dim) + if has_seq: + # means t_mod has separate modulation for each token, otherwise same modulation for all tokens in the block + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + shift_msa.squeeze(2), scale_msa.squeeze(2), gate_msa.squeeze(2), + shift_mlp.squeeze(2), scale_mlp.squeeze(2), gate_mlp.squeeze(2), + ) + input_x = modulate(self.norm1(x), shift_msa, scale_msa) + x = self.gate(x, gate_msa, self.self_attn(input_x, freqs, self_attn_mask=self_attn_mask)) + x = x + self.cross_attn(self.norm3(x), context, ctx_mask=context_mask) + input_x = modulate(self.norm2(x), shift_mlp, scale_mlp) + x = self.gate(x, gate_mlp, self.ffn(input_x)) + return x + + +class MLP(torch.nn.Module): + def __init__(self, in_dim, out_dim, has_pos_emb=False): + super().__init__() + self.proj = torch.nn.Sequential( + nn.LayerNorm(in_dim), + nn.Linear(in_dim, in_dim), + nn.GELU(), + nn.Linear(in_dim, out_dim), + nn.LayerNorm(out_dim) + ) + self.has_pos_emb = has_pos_emb + if has_pos_emb: + self.emb_pos = torch.nn.Parameter(torch.zeros((1, 514, 1280))) + + def forward(self, x): + if self.has_pos_emb: + x = x + self.emb_pos.to(dtype=x.dtype, device=x.device) + return self.proj(x) + + +class Head(nn.Module): + def __init__(self, dim: int, out_dim: int, patch_size: Tuple[int, int, int], eps: float): + super().__init__() + self.dim = dim + self.patch_size = patch_size + self.norm = nn.LayerNorm(dim, eps=eps, elementwise_affine=False) + self.head = nn.Linear(dim, out_dim * math.prod(patch_size)) + self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5) + + def forward(self, x, t_mod): + if len(t_mod.shape) == 3: + shift, scale = (self.modulation.unsqueeze(0).to(dtype=t_mod.dtype, device=t_mod.device) + t_mod.unsqueeze(2)).chunk(2, dim=2) + x = (self.head(self.norm(x) * (1 + scale.squeeze(2)) + shift.squeeze(2))) + else: + shift, scale = (self.modulation.to(dtype=t_mod.dtype, device=t_mod.device) + t_mod).chunk(2, dim=1) + x = (self.head(self.norm(x) * (1 + scale) + shift)) + return x + + +class WanVideoDiT(torch.nn.Module): + def __init__( + self, + hidden_dim: int, + in_dim: int, + ffn_dim: int, + out_dim: int, + text_dim: int, + freq_dim: int, + eps: float, + patch_size: Tuple[int, int, int], + num_heads: int, + attn_head_dim: int, + num_layers: int, + has_image_input: bool, + has_image_pos_emb: bool = False, + has_ref_conv: bool = False, + add_control_adapter: bool = False, + in_dim_control_adapter: int = 24, + seperated_timestep: bool = False, + require_vae_embedding: bool = False, + require_clip_embedding: bool = False, + fuse_vae_embedding_in_latents: bool = True, + action_conditioned: bool = False, + action_dim: int = 7, + action_group_causal_mask_mode = "causal", + video_attention_mask_mode: str = "bidirectional", + use_gradient_checkpointing: bool = False, + ): + super().__init__() + self.hidden_dim = hidden_dim + self.in_dim = in_dim + self.freq_dim = freq_dim + self.patch_size = patch_size + self.num_heads = num_heads + self.attn_head_dim = attn_head_dim + self.seperated_timestep = seperated_timestep + self.require_vae_embedding = require_vae_embedding + self.require_clip_embedding = require_clip_embedding + self.fuse_vae_embedding_in_latents = fuse_vae_embedding_in_latents + self.video_attention_mask_mode = str(video_attention_mask_mode) + + if num_heads <= 0: + raise ValueError(f"`num_heads` must be > 0, got {num_heads}") + if attn_head_dim <= 0: + raise ValueError(f"`attn_head_dim` must be > 0, got {attn_head_dim}") + if attn_head_dim % 2 != 0: + raise ValueError( + f"`attn_head_dim` must be even for RoPE, got {attn_head_dim}" + ) + + self.action_conditioned = action_conditioned + self.action_dim = action_dim + assert has_image_input == False + assert require_clip_embedding == False + assert require_vae_embedding == False and fuse_vae_embedding_in_latents == True, "Only support fusing vae embedding in latents" + + self.patch_embedding = nn.Conv3d( + in_dim, hidden_dim, kernel_size=patch_size, stride=patch_size) + self.text_embedding = nn.Sequential( + nn.Linear(text_dim, hidden_dim), + nn.GELU(approximate='tanh'), + nn.Linear(hidden_dim, hidden_dim) + ) + self.time_embedding = nn.Sequential( + nn.Linear(freq_dim, hidden_dim), + nn.SiLU(), + nn.Linear(hidden_dim, hidden_dim) + ) + self.time_projection = nn.Sequential( + nn.SiLU(), nn.Linear(hidden_dim, hidden_dim * 6)) + self.blocks = nn.ModuleList([ + DiTBlock(hidden_dim, attn_head_dim, num_heads, ffn_dim, eps) + for _ in range(num_layers) + ]) + self.head = Head(hidden_dim, out_dim, patch_size, eps) + self.freqs = precompute_freqs_cis_3d(attn_head_dim) + if has_ref_conv: + self.ref_conv = nn.Conv2d(16, hidden_dim, kernel_size=(2, 2), stride=(2, 2)) + self.has_image_pos_emb = has_image_pos_emb + self.has_ref_conv = has_ref_conv + self.control_adapter = None + + if self.action_conditioned: + self.action_embedding = nn.Linear(action_dim, hidden_dim) + self.action_group_causal_mask_mode = action_group_causal_mask_mode + + self.use_gradient_checkpointing = use_gradient_checkpointing + if self.use_gradient_checkpointing: + logger.info("Using gradient checkpointing for DiT blocks. This will save memory but use more computation.") + + + def patchify(self, x: torch.Tensor, control_camera_latents_input: Optional[torch.Tensor] = None): + x = self.patch_embedding(x) + if self.control_adapter is not None and control_camera_latents_input is not None: + y_camera = self.control_adapter(control_camera_latents_input) + x = [u + v for u, v in zip(x, y_camera)] + x = x[0].unsqueeze(0) + return x + + def unpatchify(self, x: torch.Tensor, grid_size: torch.Tensor): + return rearrange( + x, 'b (f h w) (x y z c) -> b c (f x) (h y) (w z)', + f=grid_size[0], h=grid_size[1], w=grid_size[2], + x=self.patch_size[0], y=self.patch_size[1], z=self.patch_size[2] + ) + + def _validate_forward_inputs( + self, + x: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: Optional[torch.Tensor], + action: Optional[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if x.ndim != 5: + raise ValueError(f"`latents` must be 5D [B, C, T, H, W], got shape {tuple(x.shape)}") + num_latent_frames = x.shape[2] + if context.ndim != 3: + raise ValueError(f"`context` must be 3D [B, L, D], got shape {tuple(context.shape)}") + if timestep.ndim != 1: + raise ValueError(f"`timestep` must be 1D [B] or [1], got shape {tuple(timestep.shape)}") + if self.action_conditioned: + allow_text_only_single_frame = (num_latent_frames == 1 and action is None) + if not allow_text_only_single_frame: + assert action is not None, "Action input is required for action-conditioned model." + if action.ndim != 3: + raise ValueError(f"`action` must be 3D [B, action_horizon, action_dim], got shape {tuple(action.shape)}") + if action.shape[2] != self.action_dim: + raise ValueError(f"`action` last dimension must be {self.action_dim}, got {action.shape[2]}") + if num_latent_frames <= 1: + raise ValueError(f"video length must be > 1 for action-conditioned model, got {num_latent_frames}") + if action.shape[1] % (num_latent_frames - 1) != 0: + raise ValueError( + f"action horizon must be divisible by (num_latent_frames - 1), got action_horizon={action.shape[1]}" + ) + if context_mask is None: + context_mask = torch.ones((context.shape[0], context.shape[1]), dtype=torch.bool, device=context.device) + else: + if context_mask.ndim != 2: + raise ValueError(f"`context_mask` must be 2D [B, L], got shape {tuple(context_mask.shape)}") + if context_mask.shape[0] != context.shape[0] or context_mask.shape[1] != context.shape[1]: + raise ValueError(f"`context_mask` shape must match `context` shape [B, L], got {tuple(context_mask.shape)} vs {tuple(context.shape)}") + + batch_size = x.shape[0] + if batch_size != context.shape[0]: + if not self.training and batch_size == 1: + x = x.expand(context.shape[0], -1, -1, -1, -1) + batch_size = context.shape[0] + else: + raise ValueError( + f"Batch mismatch between latents and context: {batch_size} vs {context.shape[0]}." + ) + + if timestep.shape[0] not in (1, batch_size): + raise ValueError( + f"`timestep` length must be 1 or batch_size({batch_size}), got {timestep.shape[0]}" + ) + if timestep.shape[0] == 1 and batch_size > 1: + assert not self.training, "During training, timestep length must match batch_size." + timestep = timestep.expand(batch_size) + return x, timestep, context_mask + + def build_video_to_video_mask( + self, + video_seq_len: int, + video_tokens_per_frame: int, + device: torch.device, + ) -> torch.Tensor: + if video_seq_len <= 0: + raise ValueError(f"`video_seq_len` must be positive, got {video_seq_len}") + if video_tokens_per_frame <= 0: + raise ValueError(f"`video_tokens_per_frame` must be positive, got {video_tokens_per_frame}") + + if self.video_attention_mask_mode == "bidirectional": + return torch.ones((video_seq_len, video_seq_len), dtype=torch.bool, device=device) + + if self.video_attention_mask_mode == "per_frame_causal": + if video_seq_len % video_tokens_per_frame != 0: + raise ValueError( + "`video_seq_len` must be divisible by `video_tokens_per_frame` in `per_frame_causal` mode, " + f"got {video_seq_len} and {video_tokens_per_frame}" + ) + num_video_frames = video_seq_len // video_tokens_per_frame + frame_causal = torch.tril( + torch.ones((num_video_frames, num_video_frames), dtype=torch.bool, device=device) + ) + return frame_causal.repeat_interleave(video_tokens_per_frame, dim=0).repeat_interleave( + video_tokens_per_frame, dim=1 + ) + + if self.video_attention_mask_mode == "first_frame_causal": + video_mask = torch.ones((video_seq_len, video_seq_len), dtype=torch.bool, device=device) + first_frame_tokens = min(video_tokens_per_frame, video_seq_len) + video_mask[:first_frame_tokens, first_frame_tokens:] = False + return video_mask + + raise ValueError(f"Unsupported video attention mask mode: {self.video_attention_mask_mode}") + + def pre_dit( + self, + x: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: Optional[torch.Tensor] = None, + action: Optional[torch.Tensor] = None, + fuse_vae_embedding_in_latents: bool = False, + control_camera_latents_input: Optional[torch.Tensor] = None, + ) -> Dict[str, Any]: + x, timestep, context_mask = self._validate_forward_inputs( + x=x, + timestep=timestep, + context=context, + context_mask=context_mask, + action=action, + ) + + batch_size = x.shape[0] + patch_h = int(self.patch_size[1]) + patch_w = int(self.patch_size[2]) + if x.shape[3] % patch_h != 0 or x.shape[4] % patch_w != 0: + raise ValueError( + "Latent spatial shape must be divisible by DiT patch size, " + f"got HxW=({x.shape[3]}, {x.shape[4]}), patch=({patch_h}, {patch_w})" + ) + tokens_per_frame = (x.shape[3] // patch_h) * (x.shape[4] // patch_w) + + if self.seperated_timestep and fuse_vae_embedding_in_latents: + if not hasattr(self, "patch_size") or len(self.patch_size) < 3: + raise ValueError(f"Invalid dit.patch_size: {getattr(self, 'patch_size', None)}") + + token_timesteps = torch.ones( + (batch_size, x.shape[2], tokens_per_frame), + dtype=timestep.dtype, + device=timestep.device, + ) * timestep.view(batch_size, 1, 1) + token_timesteps[:, 0, :] = 0 + token_timesteps = token_timesteps.reshape(batch_size, -1) + token_t_emb = sinusoidal_embedding_1d(self.freq_dim, token_timesteps.reshape(-1)) + t = self.time_embedding(token_t_emb).reshape(batch_size, -1, self.hidden_dim) + t_mod = self.time_projection(t).unflatten(2, (6, self.hidden_dim)) + else: + raise NotImplementedError("Only support seperated_timestep with fuse_vae_embedding_in_latents for now.") + t = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timestep)) + t_mod = self.time_projection(t).unflatten(1, (6, self.hidden_dim)) + x = self.patchify(x, control_camera_latents_input=control_camera_latents_input) + f, h, w = x.shape[2:] + + context = self.text_embedding(context) # (B, L, dim) + context_len = context.shape[1] + if self.action_conditioned and action is not None: + action_len = action.shape[1] + action_emb = self.action_embedding(action) # (B, action_len, dim) + action_pos_embed = sinusoidal_embedding_1d(self.hidden_dim, + torch.arange(action_len, device=action_emb.device)) # (action_len, dim) + action_emb = action_emb + action_pos_embed.unsqueeze(0) # (B, action_len, dim) + context = torch.cat([context, action_emb], dim=1) # (B, context_len + action_len, dim) + + # new mask + num_temporal_groups = f - 1 # first latent frame do not attend to actions + if num_temporal_groups <= 0: + raise ValueError( + "Action-conditioned context mask requires at least 2 latent frames when `action` is provided." + ) + assert action_emb.shape[1] % num_temporal_groups == 0, \ + f"Action embedding length {action_emb.shape[1]} must be divisible by number of temporal groups {num_temporal_groups}" + # Each latent frame (from the 2nd one) attends to the corresponding group of action tokens + action_group_mask = create_group_causal_attn_mask( + num_temporal_groups=num_temporal_groups, + num_query_per_group=tokens_per_frame, + num_key_per_group=action_len // num_temporal_groups, + mode=self.action_group_causal_mask_mode, + ).to(context.device) # ((f-1)*tokens_per_frame, action_len) + + seq_len = f * h * w # query length + final_context_mask = torch.zeros((batch_size, seq_len, context.shape[1]), dtype=torch.bool, device=context.device) # (B, seq_len, L + action_len) + # all latent frames attend to text tokens + final_context_mask[:, :, :context_len] = context_mask.unsqueeze(1).expand(-1, seq_len, -1) # (B, seq_len, L) + # latent frames from the 2nd one attend to action tokens + final_context_mask[:, tokens_per_frame:, context_len:] = action_group_mask.unsqueeze(0).expand(batch_size, -1, -1) # (B, seq_len, action_len) + context_mask = final_context_mask + elif self.action_conditioned and action is None: + if f != 1: + raise ValueError( + "Action-conditioned model requires `action` unless running single-frame text-only mode with num_latent_frames=1." + ) + context_mask = context_mask.unsqueeze(1).expand(-1, f * h * w, -1) # (B, seq_len, L) + else: + context_mask = context_mask.unsqueeze(1).expand(-1, f * h * w, -1) # (B, seq_len, L) + + x_tokens = rearrange(x, "b c f h w -> b (f h w) c").contiguous() + + freqs = torch.cat([ + self.freqs[0][:f].view(f, 1, 1, -1).expand(f, h, w, -1), + self.freqs[1][:h].view(1, h, 1, -1).expand(f, h, w, -1), + self.freqs[2][:w].view(1, 1, w, -1).expand(f, h, w, -1) + ], dim=-1).reshape(f * h * w, 1, -1).to(x_tokens.device) + + return { + "tokens": x_tokens, + "freqs": freqs, + "t": t, + "t_mod": t_mod, + "context": context, + "context_mask": context_mask, + "meta": { + "grid_size": (f, h, w), + "tokens_per_frame": tokens_per_frame, + "batch_size": batch_size, + }, + } + + def post_dit(self, x_tokens: torch.Tensor, pre_state: Dict[str, Any]) -> torch.Tensor: + f, h, w = pre_state["meta"]["grid_size"] + x = self.head(x_tokens, pre_state["t"]) + x = self.unpatchify(x, (f, h, w)) + return x + + def forward( + self, + x: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: Optional[torch.Tensor] = None, + action: Optional[torch.Tensor] = None, + fuse_vae_embedding_in_latents: bool = False, + ): + pre_state = self.pre_dit( + x=x, + timestep=timestep, + context=context, + context_mask=context_mask, + action=action, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + x_tokens = pre_state["tokens"] + context_emb = pre_state["context"] + t_mod = pre_state["t_mod"] + freqs = pre_state["freqs"] + context_attn_mask = pre_state["context_mask"] + self_attn_mask = self.build_video_to_video_mask( + video_seq_len=x_tokens.shape[1], + video_tokens_per_frame=int(pre_state["meta"]["tokens_per_frame"]), + device=x_tokens.device, + ) if self.video_attention_mask_mode != "bidirectional" else None # special rule for faster speed + + for block in self.blocks: + if self.use_gradient_checkpointing: + x_tokens = gradient_checkpoint_forward( + block, + self.use_gradient_checkpointing, + x_tokens, context_emb, t_mod, freqs, context_mask=context_attn_mask, self_attn_mask=self_attn_mask + ) + else: + x_tokens = block(x_tokens, context_emb, t_mod, freqs, context_mask=context_attn_mask, self_attn_mask=self_attn_mask) + + return self.post_dit(x_tokens, pre_state) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_text_encoder.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_text_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..24f89f49c971d3798a2eed8e20faa98ec01cba20 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_text_encoder.py @@ -0,0 +1,342 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import AutoTokenizer +import html +import string +try: + import ftfy +except ModuleNotFoundError: + class _FTFY: + @staticmethod + def fix_text(text): + return text + + ftfy = _FTFY() +try: + import regex as re +except ModuleNotFoundError: + import re + +def fp16_clamp(x): + if x.dtype == torch.float16 and torch.isinf(x).any(): + clamp = torch.finfo(x.dtype).max - 1000 + x = torch.clamp(x, min=-clamp, max=clamp) + return x + + +class GELU(nn.Module): + + def forward(self, x): + return 0.5 * x * (1.0 + torch.tanh( + math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0)))) + + +class T5LayerNorm(nn.Module): + + def __init__(self, dim, eps=1e-6): + super(T5LayerNorm, self).__init__() + self.dim = dim + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def forward(self, x): + x = x * torch.rsqrt(x.float().pow(2).mean(dim=-1, keepdim=True) + + self.eps) + if self.weight.dtype in [torch.float16, torch.bfloat16]: + x = x.type_as(self.weight) + return self.weight * x + + +class T5Attention(nn.Module): + + def __init__(self, dim, dim_attn, num_heads, dropout=0.1): + assert dim_attn % num_heads == 0 + super(T5Attention, self).__init__() + self.dim = dim + self.dim_attn = dim_attn + self.num_heads = num_heads + self.head_dim = dim_attn // num_heads + + # layers + self.q = nn.Linear(dim, dim_attn, bias=False) + self.k = nn.Linear(dim, dim_attn, bias=False) + self.v = nn.Linear(dim, dim_attn, bias=False) + self.o = nn.Linear(dim_attn, dim, bias=False) + self.dropout = nn.Dropout(dropout) + + def forward(self, x, context=None, mask=None, pos_bias=None): + """ + x: [B, L1, C]. + context: [B, L2, C] or None. + mask: [B, L2] or [B, L1, L2] or None. + """ + # check inputs + context = x if context is None else context + b, n, c = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.q(x).view(b, -1, n, c) + k = self.k(context).view(b, -1, n, c) + v = self.v(context).view(b, -1, n, c) + + # attention bias + attn_bias = x.new_zeros(b, n, q.size(1), k.size(1)) + if pos_bias is not None: + attn_bias += pos_bias + if mask is not None: + assert mask.ndim in [2, 3] + mask = mask.view(b, 1, 1, + -1) if mask.ndim == 2 else mask.unsqueeze(1) + attn_bias.masked_fill_(mask == 0, torch.finfo(x.dtype).min) + + # compute attention (T5 does not use scaling) + attn = torch.einsum('binc,bjnc->bnij', q, k) + attn_bias + attn = F.softmax(attn.float(), dim=-1).type_as(attn) + x = torch.einsum('bnij,bjnc->binc', attn, v) + + # output + x = x.reshape(b, -1, n * c) + x = self.o(x) + x = self.dropout(x) + return x + + +class T5FeedForward(nn.Module): + + def __init__(self, dim, dim_ffn, dropout=0.1): + super(T5FeedForward, self).__init__() + self.dim = dim + self.dim_ffn = dim_ffn + + # layers + self.gate = nn.Sequential(nn.Linear(dim, dim_ffn, bias=False), GELU()) + self.fc1 = nn.Linear(dim, dim_ffn, bias=False) + self.fc2 = nn.Linear(dim_ffn, dim, bias=False) + self.dropout = nn.Dropout(dropout) + + def forward(self, x): + x = self.fc1(x) * self.gate(x) + x = self.dropout(x) + x = self.fc2(x) + x = self.dropout(x) + return x + + +class T5SelfAttention(nn.Module): + + def __init__(self, + dim, + dim_attn, + dim_ffn, + num_heads, + num_buckets, + shared_pos=True, + dropout=0.1): + super(T5SelfAttention, self).__init__() + self.dim = dim + self.dim_attn = dim_attn + self.dim_ffn = dim_ffn + self.num_heads = num_heads + self.num_buckets = num_buckets + self.shared_pos = shared_pos + + # layers + self.norm1 = T5LayerNorm(dim) + self.attn = T5Attention(dim, dim_attn, num_heads, dropout) + self.norm2 = T5LayerNorm(dim) + self.ffn = T5FeedForward(dim, dim_ffn, dropout) + self.pos_embedding = None if shared_pos else T5RelativeEmbedding( + num_buckets, num_heads, bidirectional=True) + + def forward(self, x, mask=None, pos_bias=None): + e = pos_bias if self.shared_pos else self.pos_embedding( + x.size(1), x.size(1)) + x = fp16_clamp(x + self.attn(self.norm1(x), mask=mask, pos_bias=e)) + x = fp16_clamp(x + self.ffn(self.norm2(x))) + return x + + +class T5RelativeEmbedding(nn.Module): + + def __init__(self, num_buckets, num_heads, bidirectional, max_dist=128): + super(T5RelativeEmbedding, self).__init__() + self.num_buckets = num_buckets + self.num_heads = num_heads + self.bidirectional = bidirectional + self.max_dist = max_dist + + # layers + self.embedding = nn.Embedding(num_buckets, num_heads) + + def forward(self, lq, lk): + device = self.embedding.weight.device + # rel_pos = torch.arange(lk).unsqueeze(0).to(device) - \ + # torch.arange(lq).unsqueeze(1).to(device) + rel_pos = torch.arange(lk, device=device).unsqueeze(0) - \ + torch.arange(lq, device=device).unsqueeze(1) + rel_pos = self._relative_position_bucket(rel_pos) + rel_pos_embeds = self.embedding(rel_pos) + rel_pos_embeds = rel_pos_embeds.permute(2, 0, 1).unsqueeze( + 0) # [1, N, Lq, Lk] + return rel_pos_embeds.contiguous() + + def _relative_position_bucket(self, rel_pos): + # preprocess + if self.bidirectional: + num_buckets = self.num_buckets // 2 + rel_buckets = (rel_pos > 0).long() * num_buckets + rel_pos = torch.abs(rel_pos) + else: + num_buckets = self.num_buckets + rel_buckets = 0 + rel_pos = -torch.min(rel_pos, torch.zeros_like(rel_pos)) + + # embeddings for small and large positions + max_exact = num_buckets // 2 + rel_pos_large = max_exact + (torch.log(rel_pos.float() / max_exact) / + math.log(self.max_dist / max_exact) * + (num_buckets - max_exact)).long() + rel_pos_large = torch.min( + rel_pos_large, torch.full_like(rel_pos_large, num_buckets - 1)) + rel_buckets += torch.where(rel_pos < max_exact, rel_pos, rel_pos_large) + return rel_buckets + +def init_weights(m): + if isinstance(m, T5LayerNorm): + nn.init.ones_(m.weight) + elif isinstance(m, T5FeedForward): + nn.init.normal_(m.gate[0].weight, std=m.dim**-0.5) + nn.init.normal_(m.fc1.weight, std=m.dim**-0.5) + nn.init.normal_(m.fc2.weight, std=m.dim_ffn**-0.5) + elif isinstance(m, T5Attention): + nn.init.normal_(m.q.weight, std=(m.dim * m.dim_attn)**-0.5) + nn.init.normal_(m.k.weight, std=m.dim**-0.5) + nn.init.normal_(m.v.weight, std=m.dim**-0.5) + nn.init.normal_(m.o.weight, std=(m.num_heads * m.dim_attn)**-0.5) + elif isinstance(m, T5RelativeEmbedding): + nn.init.normal_( + m.embedding.weight, std=(2 * m.num_buckets * m.num_heads)**-0.5) + + +class WanTextEncoder(torch.nn.Module): + + def __init__(self, + vocab=256384, + dim=4096, + dim_attn=4096, + dim_ffn=10240, + num_heads=64, + num_layers=24, + num_buckets=32, + shared_pos=False, + dropout=0.1): + super(WanTextEncoder, self).__init__() + self.dim = dim + self.dim_attn = dim_attn + self.dim_ffn = dim_ffn + self.num_heads = num_heads + self.num_layers = num_layers + self.num_buckets = num_buckets + self.shared_pos = shared_pos + + # layers + self.token_embedding = vocab if isinstance(vocab, nn.Embedding) \ + else nn.Embedding(vocab, dim) + self.pos_embedding = T5RelativeEmbedding( + num_buckets, num_heads, bidirectional=True) if shared_pos else None + self.dropout = nn.Dropout(dropout) + self.blocks = nn.ModuleList([ + T5SelfAttention(dim, dim_attn, dim_ffn, num_heads, num_buckets, + shared_pos, dropout) for _ in range(num_layers) + ]) + self.norm = T5LayerNorm(dim) + + # Skip costly random init when loading pretrained checkpoints. + # Verified: checkpoint fully covers this module (missing/unexpected keys are both 0). + # self.apply(init_weights) + + def forward(self, ids, mask=None): + x = self.token_embedding(ids) + x = self.dropout(x) + e = self.pos_embedding(x.size(1), + x.size(1)) if self.shared_pos else None + for block in self.blocks: + x = block(x, mask, pos_bias=e) + x = self.norm(x) + x = self.dropout(x) + return x + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +def canonicalize(text, keep_punctuation_exact_string=None): + text = text.replace('_', ' ') + if keep_punctuation_exact_string: + text = keep_punctuation_exact_string.join( + part.translate(str.maketrans('', '', string.punctuation)) + for part in text.split(keep_punctuation_exact_string)) + else: + text = text.translate(str.maketrans('', '', string.punctuation)) + text = text.lower() + text = re.sub(r'\s+', ' ', text) + return text.strip() + + +class HuggingfaceTokenizer: + + def __init__(self, name, seq_len=None, clean=None, **kwargs): + assert clean in (None, 'whitespace', 'lower', 'canonicalize') + self.name = name + self.seq_len = seq_len + self.clean = clean + + # init tokenizer + self.tokenizer = AutoTokenizer.from_pretrained(name, **kwargs) + self.vocab_size = self.tokenizer.vocab_size + + def __call__(self, sequence, **kwargs): + return_mask = kwargs.pop('return_mask', False) + + # arguments + _kwargs = {'return_tensors': 'pt'} + if self.seq_len is not None: + _kwargs.update({ + 'padding': 'max_length', + 'truncation': True, + 'max_length': self.seq_len + }) + _kwargs.update(**kwargs) + + # tokenization + if isinstance(sequence, str): + sequence = [sequence] + if self.clean: + sequence = [self._clean(u) for u in sequence] + ids = self.tokenizer(sequence, **_kwargs) + + # output + if return_mask: + return ids.input_ids, ids.attention_mask + else: + return ids.input_ids + + def _clean(self, text): + if self.clean == 'whitespace': + text = whitespace_clean(basic_clean(text)) + elif self.clean == 'lower': + text = whitespace_clean(basic_clean(text)).lower() + elif self.clean == 'canonicalize': + text = canonicalize(basic_clean(text)) + return text diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_vae.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_vae.py new file mode 100644 index 0000000000000000000000000000000000000000..0386bdedb6a532781eb81509e5f574011113a706 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_vae.py @@ -0,0 +1,1384 @@ +from einops import rearrange, repeat + +import torch +import torch.nn as nn +import torch.nn.functional as F +from tqdm import tqdm + +CACHE_T = 2 + + +def check_is_instance(model, module_class): + if isinstance(model, module_class): + return True + if hasattr(model, "module") and isinstance(model.module, module_class): + return True + return False + + +def block_causal_mask(x, block_size): + # params + b, n, s, _, device = *x.size(), x.device + assert s % block_size == 0 + num_blocks = s // block_size + + # build mask + mask = torch.zeros(b, n, s, s, dtype=torch.bool, device=device) + for i in range(num_blocks): + mask[:, :, + i * block_size:(i + 1) * block_size, :(i + 1) * block_size] = 1 + return mask + + +class CausalConv3d(nn.Conv3d): + """ + Causal 3d convolusion. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._padding = (self.padding[2], self.padding[2], self.padding[1], + self.padding[1], 2 * self.padding[0], 0) + self.padding = (0, 0, 0) + + def forward(self, x, cache_x=None): + padding = list(self._padding) + if cache_x is not None and self._padding[4] > 0: + cache_x = cache_x.to(x.device) + x = torch.cat([cache_x, x], dim=2) + padding[4] -= cache_x.shape[2] + x = F.pad(x, padding) + + return super().forward(x) + + +class RMS_norm(nn.Module): + + def __init__(self, dim, channel_first=True, images=True, bias=False): + super().__init__() + broadcastable_dims = (1, 1, 1) if not images else (1, 1) + shape = (dim, *broadcastable_dims) if channel_first else (dim,) + + self.channel_first = channel_first + self.scale = dim**0.5 + self.gamma = nn.Parameter(torch.ones(shape)) + self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0. + + def forward(self, x): + return F.normalize( + x, dim=(1 if self.channel_first else + -1)) * self.scale * self.gamma + self.bias + + +class Upsample(nn.Upsample): + + def forward(self, x): + """ + Fix bfloat16 support for nearest neighbor interpolation. + """ + return super().forward(x.float()).type_as(x) + + +class Resample(nn.Module): + + def __init__(self, dim, mode): + assert mode in ('none', 'upsample2d', 'upsample3d', 'downsample2d', + 'downsample3d') + super().__init__() + self.dim = dim + self.mode = mode + + # layers + if mode == 'upsample2d': + self.resample = nn.Sequential( + Upsample(scale_factor=(2., 2.), mode='nearest-exact'), + nn.Conv2d(dim, dim // 2, 3, padding=1)) + elif mode == 'upsample3d': + self.resample = nn.Sequential( + Upsample(scale_factor=(2., 2.), mode='nearest-exact'), + nn.Conv2d(dim, dim // 2, 3, padding=1)) + self.time_conv = CausalConv3d(dim, + dim * 2, (3, 1, 1), + padding=(1, 0, 0)) + + elif mode == 'downsample2d': + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), + nn.Conv2d(dim, dim, 3, stride=(2, 2))) + elif mode == 'downsample3d': + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), + nn.Conv2d(dim, dim, 3, stride=(2, 2))) + self.time_conv = CausalConv3d(dim, + dim, (3, 1, 1), + stride=(2, 1, 1), + padding=(0, 0, 0)) + + else: + self.resample = nn.Identity() + + def forward(self, x, feat_cache=None, feat_idx=[0]): + b, c, t, h, w = x.size() + if self.mode == 'upsample3d': + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = 'Rep' + feat_idx[0] += 1 + else: + + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[ + idx] is not None and feat_cache[idx] != 'Rep': + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + if cache_x.shape[2] < 2 and feat_cache[ + idx] is not None and feat_cache[idx] == 'Rep': + cache_x = torch.cat([ + torch.zeros_like(cache_x).to(cache_x.device), + cache_x + ], + dim=2) + if feat_cache[idx] == 'Rep': + x = self.time_conv(x) + else: + x = self.time_conv(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + + x = x.reshape(b, 2, c, t, h, w) + x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]), + 3) + x = x.reshape(b, c, t * 2, h, w) + t = x.shape[2] + x = rearrange(x, 'b c t h w -> (b t) c h w') + x = self.resample(x) + x = rearrange(x, '(b t) c h w -> b c t h w', t=t) + + if self.mode == 'downsample3d': + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = x.clone() + feat_idx[0] += 1 + else: + cache_x = x[:, :, -1:, :, :].clone() + x = self.time_conv( + torch.cat([feat_cache[idx][:, :, -1:, :, :], x], 2)) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + return x, feat_cache, feat_idx + + def init_weight(self, conv): + conv_weight = conv.weight + nn.init.zeros_(conv_weight) + c1, c2, t, h, w = conv_weight.size() + one_matrix = torch.eye(c1, c2) + init_matrix = one_matrix + nn.init.zeros_(conv_weight) + conv_weight.data[:, :, 1, 0, 0] = init_matrix + conv.weight.data.copy_(conv_weight) + nn.init.zeros_(conv.bias.data) + + def init_weight2(self, conv): + conv_weight = conv.weight.data + nn.init.zeros_(conv_weight) + c1, c2, t, h, w = conv_weight.size() + init_matrix = torch.eye(c1 // 2, c2) + conv_weight[:c1 // 2, :, -1, 0, 0] = init_matrix + conv_weight[c1 // 2:, :, -1, 0, 0] = init_matrix + conv.weight.data.copy_(conv_weight) + nn.init.zeros_(conv.bias.data) + + + +def patchify(x, patch_size): + if patch_size == 1: + return x + if x.dim() == 4: + x = rearrange(x, "b c (h q) (w r) -> b (c r q) h w", q=patch_size, r=patch_size) + elif x.dim() == 5: + x = rearrange(x, + "b c f (h q) (w r) -> b (c r q) f h w", + q=patch_size, + r=patch_size) + else: + raise ValueError(f"Invalid input shape: {x.shape}") + return x + + +def unpatchify(x, patch_size): + if patch_size == 1: + return x + if x.dim() == 4: + x = rearrange(x, "b (c r q) h w -> b c (h q) (w r)", q=patch_size, r=patch_size) + elif x.dim() == 5: + x = rearrange(x, + "b (c r q) f h w -> b c f (h q) (w r)", + q=patch_size, + r=patch_size) + return x + + +class Resample38(Resample): + + def __init__(self, dim, mode): + assert mode in ( + "none", + "upsample2d", + "upsample3d", + "downsample2d", + "downsample3d", + ) + super(Resample, self).__init__() + self.dim = dim + self.mode = mode + + # layers + if mode == "upsample2d": + self.resample = nn.Sequential( + Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, dim, 3, padding=1), + ) + elif mode == "upsample3d": + self.resample = nn.Sequential( + Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, dim, 3, padding=1), + ) + self.time_conv = CausalConv3d(dim, dim * 2, (3, 1, 1), padding=(1, 0, 0)) + elif mode == "downsample2d": + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2)) + ) + elif mode == "downsample3d": + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2)) + ) + self.time_conv = CausalConv3d( + dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0) + ) + else: + self.resample = nn.Identity() + +class ResidualBlock(nn.Module): + + def __init__(self, in_dim, out_dim, dropout=0.0): + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + + # layers + self.residual = nn.Sequential( + RMS_norm(in_dim, images=False), nn.SiLU(), + CausalConv3d(in_dim, out_dim, 3, padding=1), + RMS_norm(out_dim, images=False), nn.SiLU(), nn.Dropout(dropout), + CausalConv3d(out_dim, out_dim, 3, padding=1)) + self.shortcut = CausalConv3d(in_dim, out_dim, 1) \ + if in_dim != out_dim else nn.Identity() + + def forward(self, x, feat_cache=None, feat_idx=[0]): + h = self.shortcut(x) + for layer in self.residual: + if check_is_instance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x + h, feat_cache, feat_idx + + +class AttentionBlock(nn.Module): + """ + Causal self-attention with a single head. + """ + + def __init__(self, dim): + super().__init__() + self.dim = dim + + # layers + self.norm = RMS_norm(dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x): + identity = x + b, c, t, h, w = x.size() + x = rearrange(x, 'b c t h w -> (b t) c h w') + x = self.norm(x) + # compute query, key, value + q, k, v = self.to_qkv(x).reshape(b * t, 1, c * 3, -1).permute( + 0, 1, 3, 2).contiguous().chunk(3, dim=-1) + + # apply attention + x = F.scaled_dot_product_attention( + q, + k, + v, + #attn_mask=block_causal_mask(q, block_size=h * w) + ) + x = x.squeeze(1).permute(0, 2, 1).reshape(b * t, c, h, w) + + # output + x = self.proj(x) + x = rearrange(x, '(b t) c h w-> b c t h w', t=t) + return x + identity + + +class AvgDown3D(nn.Module): + def __init__( + self, + in_channels, + out_channels, + factor_t, + factor_s=1, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.factor_t = factor_t + self.factor_s = factor_s + self.factor = self.factor_t * self.factor_s * self.factor_s + + assert in_channels * self.factor % out_channels == 0 + self.group_size = in_channels * self.factor // out_channels + + def forward(self, x: torch.Tensor) -> torch.Tensor: + pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t + pad = (0, 0, 0, 0, pad_t, 0) + x = F.pad(x, pad) + B, C, T, H, W = x.shape + x = x.view( + B, + C, + T // self.factor_t, + self.factor_t, + H // self.factor_s, + self.factor_s, + W // self.factor_s, + self.factor_s, + ) + x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous() + x = x.view( + B, + C * self.factor, + T // self.factor_t, + H // self.factor_s, + W // self.factor_s, + ) + x = x.view( + B, + self.out_channels, + self.group_size, + T // self.factor_t, + H // self.factor_s, + W // self.factor_s, + ) + x = x.mean(dim=2) + return x + + +class DupUp3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + factor_t, + factor_s=1, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + + self.factor_t = factor_t + self.factor_s = factor_s + self.factor = self.factor_t * self.factor_s * self.factor_s + + assert out_channels * self.factor % in_channels == 0 + self.repeats = out_channels * self.factor // in_channels + + def forward(self, x: torch.Tensor, first_chunk=False) -> torch.Tensor: + x = x.repeat_interleave(self.repeats, dim=1) + x = x.view( + x.size(0), + self.out_channels, + self.factor_t, + self.factor_s, + self.factor_s, + x.size(2), + x.size(3), + x.size(4), + ) + x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous() + x = x.view( + x.size(0), + self.out_channels, + x.size(2) * self.factor_t, + x.size(4) * self.factor_s, + x.size(6) * self.factor_s, + ) + if first_chunk: + x = x[:, :, self.factor_t - 1 :, :, :] + return x + + +class Down_ResidualBlock(nn.Module): + def __init__( + self, in_dim, out_dim, dropout, mult, temperal_downsample=False, down_flag=False + ): + super().__init__() + + # Shortcut path with downsample + self.avg_shortcut = AvgDown3D( + in_dim, + out_dim, + factor_t=2 if temperal_downsample else 1, + factor_s=2 if down_flag else 1, + ) + + # Main path with residual blocks and downsample + downsamples = [] + for _ in range(mult): + downsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + in_dim = out_dim + + # Add the final downsample block + if down_flag: + mode = "downsample3d" if temperal_downsample else "downsample2d" + downsamples.append(Resample38(out_dim, mode=mode)) + + self.downsamples = nn.Sequential(*downsamples) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + x_copy = x.clone() + for module in self.downsamples: + x, feat_cache, feat_idx = module(x, feat_cache, feat_idx) + + return x + self.avg_shortcut(x_copy), feat_cache, feat_idx + + +class Up_ResidualBlock(nn.Module): + def __init__( + self, in_dim, out_dim, dropout, mult, temperal_upsample=False, up_flag=False + ): + super().__init__() + # Shortcut path with upsample + if up_flag: + self.avg_shortcut = DupUp3D( + in_dim, + out_dim, + factor_t=2 if temperal_upsample else 1, + factor_s=2 if up_flag else 1, + ) + else: + self.avg_shortcut = None + + # Main path with residual blocks and upsample + upsamples = [] + for _ in range(mult): + upsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + in_dim = out_dim + + # Add the final upsample block + if up_flag: + mode = "upsample3d" if temperal_upsample else "upsample2d" + upsamples.append(Resample38(out_dim, mode=mode)) + + self.upsamples = nn.Sequential(*upsamples) + + def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False): + x_main = x.clone() + for module in self.upsamples: + x_main, feat_cache, feat_idx = module(x_main, feat_cache, feat_idx) + if self.avg_shortcut is not None: + x_shortcut = self.avg_shortcut(x, first_chunk) + x_main = x_main + x_shortcut + return x_main, feat_cache, feat_idx + + +class Encoder3d(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[True, True, False], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + + # dimensions + dims = [dim * u for u in [1] + dim_mult] + scale = 1.0 + + # init block + self.conv1 = CausalConv3d(3, dims[0], 3, padding=1) + + # downsample blocks + downsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + for _ in range(num_res_blocks): + downsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + if scale in attn_scales: + downsamples.append(AttentionBlock(out_dim)) + in_dim = out_dim + + # downsample block + if i != len(dim_mult) - 1: + mode = 'downsample3d' if temperal_downsample[ + i] else 'downsample2d' + downsamples.append(Resample(out_dim, mode=mode)) + scale /= 2.0 + self.downsamples = nn.Sequential(*downsamples) + + # middle blocks + self.middle = nn.Sequential(ResidualBlock(out_dim, out_dim, dropout), + AttentionBlock(out_dim), + ResidualBlock(out_dim, out_dim, dropout)) + + # output blocks + self.head = nn.Sequential(RMS_norm(out_dim, images=False), nn.SiLU(), + CausalConv3d(out_dim, z_dim, 3, padding=1)) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + ## downsamples + for layer in self.downsamples: + if feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## middle + for layer in self.middle: + if check_is_instance(layer, ResidualBlock) and feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## head + for layer in self.head: + if check_is_instance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x, feat_cache, feat_idx + + +class Encoder3d_38(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[False, True, True], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + + # dimensions + dims = [dim * u for u in [1] + dim_mult] + scale = 1.0 + + # init block + self.conv1 = CausalConv3d(12, dims[0], 3, padding=1) + + # downsample blocks + downsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + t_down_flag = ( + temperal_downsample[i] if i < len(temperal_downsample) else False + ) + downsamples.append( + Down_ResidualBlock( + in_dim=in_dim, + out_dim=out_dim, + dropout=dropout, + mult=num_res_blocks, + temperal_downsample=t_down_flag, + down_flag=i != len(dim_mult) - 1, + ) + ) + scale /= 2.0 + self.downsamples = nn.Sequential(*downsamples) + + # middle blocks + self.middle = nn.Sequential( + ResidualBlock(out_dim, out_dim, dropout), + AttentionBlock(out_dim), + ResidualBlock(out_dim, out_dim, dropout), + ) + + # # output blocks + self.head = nn.Sequential( + RMS_norm(out_dim, images=False), + nn.SiLU(), + CausalConv3d(out_dim, z_dim, 3, padding=1), + ) + + + def forward(self, x, feat_cache=None, feat_idx=[0]): + + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + ## downsamples + for layer in self.downsamples: + if feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## middle + for layer in self.middle: + if isinstance(layer, ResidualBlock) and feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## head + for layer in self.head: + if isinstance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :] + .unsqueeze(2) + .to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + + return x, feat_cache, feat_idx + + +class Decoder3d(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_upsample=[False, True, True], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_upsample = temperal_upsample + + # dimensions + dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + scale = 1.0 / 2**(len(dim_mult) - 2) + + # init block + self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1) + + # middle blocks + self.middle = nn.Sequential(ResidualBlock(dims[0], dims[0], dropout), + AttentionBlock(dims[0]), + ResidualBlock(dims[0], dims[0], dropout)) + + # upsample blocks + upsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + if i == 1 or i == 2 or i == 3: + in_dim = in_dim // 2 + for _ in range(num_res_blocks + 1): + upsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + if scale in attn_scales: + upsamples.append(AttentionBlock(out_dim)) + in_dim = out_dim + + # upsample block + if i != len(dim_mult) - 1: + mode = 'upsample3d' if temperal_upsample[i] else 'upsample2d' + upsamples.append(Resample(out_dim, mode=mode)) + scale *= 2.0 + self.upsamples = nn.Sequential(*upsamples) + + # output blocks + self.head = nn.Sequential(RMS_norm(out_dim, images=False), nn.SiLU(), + CausalConv3d(out_dim, 3, 3, padding=1)) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + ## conv1 + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + ## middle + for layer in self.middle: + if check_is_instance(layer, ResidualBlock) and feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## upsamples + for layer in self.upsamples: + if feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## head + for layer in self.head: + if check_is_instance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x, feat_cache, feat_idx + + + +class Decoder3d_38(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_upsample=[False, True, True], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_upsample = temperal_upsample + + # dimensions + dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + scale = 1.0 / 2 ** (len(dim_mult) - 2) + # init block + self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1) + + # middle blocks + self.middle = nn.Sequential(ResidualBlock(dims[0], dims[0], dropout), + AttentionBlock(dims[0]), + ResidualBlock(dims[0], dims[0], dropout)) + + # upsample blocks + upsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + t_up_flag = temperal_upsample[i] if i < len(temperal_upsample) else False + upsamples.append( + Up_ResidualBlock(in_dim=in_dim, + out_dim=out_dim, + dropout=dropout, + mult=num_res_blocks + 1, + temperal_upsample=t_up_flag, + up_flag=i != len(dim_mult) - 1)) + self.upsamples = nn.Sequential(*upsamples) + + # output blocks + self.head = nn.Sequential(RMS_norm(out_dim, images=False), nn.SiLU(), + CausalConv3d(out_dim, 12, 3, padding=1)) + + + def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False): + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + for layer in self.middle: + if check_is_instance(layer, ResidualBlock) and feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## upsamples + for layer in self.upsamples: + if feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx, first_chunk) + else: + x = layer(x) + + ## head + for layer in self.head: + if check_is_instance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :] + .unsqueeze(2) + .to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x, feat_cache, feat_idx + + +def count_conv3d(model): + count = 0 + for m in model.modules(): + if isinstance(m, CausalConv3d): + count += 1 + return count + + +class VideoVAE_(nn.Module): + + def __init__(self, + dim=96, + z_dim=16, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[False, True, True], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + self.temperal_upsample = temperal_downsample[::-1] + + # modules + self.encoder = Encoder3d(dim, z_dim * 2, dim_mult, num_res_blocks, + attn_scales, self.temperal_downsample, dropout) + self.conv1 = CausalConv3d(z_dim * 2, z_dim * 2, 1) + self.conv2 = CausalConv3d(z_dim, z_dim, 1) + self.decoder = Decoder3d(dim, z_dim, dim_mult, num_res_blocks, + attn_scales, self.temperal_upsample, dropout) + + def forward(self, x): + mu, log_var = self.encode(x) + z = self.reparameterize(mu, log_var) + x_recon = self.decode(z) + return x_recon, mu, log_var + + def encode(self, x, scale): + self.clear_cache() + ## cache + t = x.shape[2] + iter_ = 1 + (t - 1) // 4 + + for i in range(iter_): + self._enc_conv_idx = [0] + if i == 0: + out, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, :1, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx) + else: + out_, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, 1 + 4 * (i - 1):1 + 4 * i, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx) + out = torch.cat([out, out_], 2) + mu, log_var = self.conv1(out).chunk(2, dim=1) + if isinstance(scale[0], torch.Tensor): + scale = [s.to(dtype=mu.dtype, device=mu.device) for s in scale] + mu = (mu - scale[0].view(1, self.z_dim, 1, 1, 1)) * scale[1].view( + 1, self.z_dim, 1, 1, 1) + else: + scale = scale.to(dtype=mu.dtype, device=mu.device) + mu = (mu - scale[0]) * scale[1] + return mu + + def decode(self, z, scale): + self.clear_cache() + # z: [b,c,t,h,w] + if isinstance(scale[0], torch.Tensor): + scale = [s.to(dtype=z.dtype, device=z.device) for s in scale] + z = z / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view( + 1, self.z_dim, 1, 1, 1) + else: + scale = scale.to(dtype=z.dtype, device=z.device) + z = z / scale[1] + scale[0] + iter_ = z.shape[2] + x = self.conv2(z) + for i in range(iter_): + self._conv_idx = [0] + if i == 0: + out, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx) + else: + out_, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx) + out = torch.cat([out, out_], 2) # may add tensor offload + return out + + def reparameterize(self, mu, log_var): + std = torch.exp(0.5 * log_var) + eps = torch.randn_like(std) + return eps * std + mu + + def sample(self, imgs, deterministic=False): + mu, log_var = self.encode(imgs) + if deterministic: + return mu + std = torch.exp(0.5 * log_var.clamp(-30.0, 20.0)) + return mu + std * torch.randn_like(std) + + def clear_cache(self): + self._conv_num = count_conv3d(self.decoder) + self._conv_idx = [0] + self._feat_map = [None] * self._conv_num + # cache encode + self._enc_conv_num = count_conv3d(self.encoder) + self._enc_conv_idx = [0] + self._enc_feat_map = [None] * self._enc_conv_num + + +class WanVideoVAE(nn.Module): + + def __init__(self, z_dim=16): + super().__init__() + + mean = [ + -0.7571, -0.7089, -0.9113, 0.1075, -0.1745, 0.9653, -0.1517, 1.5508, + 0.4134, -0.0715, 0.5517, -0.3632, -0.1922, -0.9497, 0.2503, -0.2921 + ] + std = [ + 2.8184, 1.4541, 2.3275, 2.6558, 1.2196, 1.7708, 2.6052, 2.0743, + 3.2687, 2.1526, 2.8652, 1.5579, 1.6382, 1.1253, 2.8251, 1.9160 + ] + self.mean = torch.tensor(mean) + self.std = torch.tensor(std) + self.scale = [self.mean, 1.0 / self.std] + + # init model + self.model = VideoVAE_(z_dim=z_dim).eval().requires_grad_(False) + self.upsampling_factor = 8 + self.temporal_downsample_factor = 4 + self.z_dim = z_dim + + + def build_1d_mask(self, length, left_bound, right_bound, border_width): + x = torch.ones((length,)) + if not left_bound: + x[:border_width] = (torch.arange(border_width) + 1) / border_width + if not right_bound: + x[-border_width:] = torch.flip((torch.arange(border_width) + 1) / border_width, dims=(0,)) + return x + + + def build_mask(self, data, is_bound, border_width): + _, _, _, H, W = data.shape + h = self.build_1d_mask(H, is_bound[0], is_bound[1], border_width[0]) + w = self.build_1d_mask(W, is_bound[2], is_bound[3], border_width[1]) + + h = repeat(h, "H -> H W", H=H, W=W) + w = repeat(w, "W -> H W", H=H, W=W) + + mask = torch.stack([h, w]).min(dim=0).values + mask = rearrange(mask, "H W -> 1 1 1 H W") + return mask + + + def tiled_decode(self, hidden_states, device, tile_size, tile_stride): + _, _, T, H, W = hidden_states.shape + size_h, size_w = tile_size + stride_h, stride_w = tile_stride + + # Split tasks + tasks = [] + for h in range(0, H, stride_h): + if (h-stride_h >= 0 and h-stride_h+size_h >= H): continue + for w in range(0, W, stride_w): + if (w-stride_w >= 0 and w-stride_w+size_w >= W): continue + h_, w_ = h + size_h, w + size_w + tasks.append((h, h_, w, w_)) + + data_device = "cpu" + computation_device = device + + out_T = T * 4 - 3 + weight = torch.zeros((1, 1, out_T, H * self.upsampling_factor, W * self.upsampling_factor), dtype=hidden_states.dtype, device=data_device) + values = torch.zeros((1, 3, out_T, H * self.upsampling_factor, W * self.upsampling_factor), dtype=hidden_states.dtype, device=data_device) + + for h, h_, w, w_ in tqdm(tasks, desc="VAE decoding"): + hidden_states_batch = hidden_states[:, :, :, h:h_, w:w_].to(computation_device) + hidden_states_batch = self.model.decode(hidden_states_batch, self.scale).to(data_device) + + mask = self.build_mask( + hidden_states_batch, + is_bound=(h==0, h_>=H, w==0, w_>=W), + border_width=((size_h - stride_h) * self.upsampling_factor, (size_w - stride_w) * self.upsampling_factor) + ).to(dtype=hidden_states.dtype, device=data_device) + + target_h = h * self.upsampling_factor + target_w = w * self.upsampling_factor + values[ + :, + :, + :, + target_h:target_h + hidden_states_batch.shape[3], + target_w:target_w + hidden_states_batch.shape[4], + ] += hidden_states_batch * mask + weight[ + :, + :, + :, + target_h: target_h + hidden_states_batch.shape[3], + target_w: target_w + hidden_states_batch.shape[4], + ] += mask + values = values / weight + values = values.clamp_(-1, 1) + return values + + + def tiled_encode(self, video, device, tile_size, tile_stride): + _, _, T, H, W = video.shape + size_h, size_w = tile_size + stride_h, stride_w = tile_stride + + # Split tasks + tasks = [] + for h in range(0, H, stride_h): + if (h-stride_h >= 0 and h-stride_h+size_h >= H): continue + for w in range(0, W, stride_w): + if (w-stride_w >= 0 and w-stride_w+size_w >= W): continue + h_, w_ = h + size_h, w + size_w + tasks.append((h, h_, w, w_)) + + data_device = "cpu" + computation_device = device + + out_T = (T + 3) // 4 + weight = torch.zeros((1, 1, out_T, H // self.upsampling_factor, W // self.upsampling_factor), dtype=video.dtype, device=data_device) + values = torch.zeros((1, self.z_dim, out_T, H // self.upsampling_factor, W // self.upsampling_factor), dtype=video.dtype, device=data_device) + + for h, h_, w, w_ in tqdm(tasks, desc="VAE encoding"): + hidden_states_batch = video[:, :, :, h:h_, w:w_].to(computation_device) + hidden_states_batch = self.model.encode(hidden_states_batch, self.scale).to(data_device) + + mask = self.build_mask( + hidden_states_batch, + is_bound=(h==0, h_>=H, w==0, w_>=W), + border_width=((size_h - stride_h) // self.upsampling_factor, (size_w - stride_w) // self.upsampling_factor) + ).to(dtype=video.dtype, device=data_device) + + target_h = h // self.upsampling_factor + target_w = w // self.upsampling_factor + values[ + :, + :, + :, + target_h:target_h + hidden_states_batch.shape[3], + target_w:target_w + hidden_states_batch.shape[4], + ] += hidden_states_batch * mask + weight[ + :, + :, + :, + target_h: target_h + hidden_states_batch.shape[3], + target_w: target_w + hidden_states_batch.shape[4], + ] += mask + values = values / weight + return values + + + def single_encode(self, video, device): + video = video.to(device) + x = self.model.encode(video, self.scale) + return x + + + def single_decode(self, hidden_state, device): + hidden_state = hidden_state.to(device) + video = self.model.decode(hidden_state, self.scale) + return video.clamp_(-1, 1) + + + def encode(self, videos, device, tiled=False, tile_size=(34, 34), tile_stride=(18, 16)): + # videos = [video.to("cpu") for video in videos] + hidden_states = [] + for video in videos: + video = video.unsqueeze(0) + if tiled: + raise NotImplementedError("Tiled encoding is not allowed yet.") + tile_size = (tile_size[0] * self.upsampling_factor, tile_size[1] * self.upsampling_factor) + tile_stride = (tile_stride[0] * self.upsampling_factor, tile_stride[1] * self.upsampling_factor) + hidden_state = self.tiled_encode(video, device, tile_size, tile_stride) + else: + hidden_state = self.single_encode(video, device) + hidden_state = hidden_state.squeeze(0) + hidden_states.append(hidden_state) + hidden_states = torch.stack(hidden_states) + return hidden_states + + + def decode(self, hidden_states, device, tiled=False, tile_size=(34, 34), tile_stride=(18, 16)): + hidden_states = [hidden_state.to("cpu") for hidden_state in hidden_states] + videos = [] + for hidden_state in hidden_states: + hidden_state = hidden_state.unsqueeze(0) + if tiled: + video = self.tiled_decode(hidden_state, device, tile_size, tile_stride) + else: + video = self.single_decode(hidden_state, device) + video = video.squeeze(0) + videos.append(video) + videos = torch.stack(videos) + return videos + + + @staticmethod + def state_dict_converter(): + return WanVideoVAEStateDictConverter() + + +class WanVideoVAEStateDictConverter: + + def __init__(self): + pass + + def from_civitai(self, state_dict): + state_dict_ = {} + if 'model_state' in state_dict: + state_dict = state_dict['model_state'] + for name in state_dict: + state_dict_['model.' + name] = state_dict[name] + return state_dict_ + + +class VideoVAE38_(VideoVAE_): + + def __init__(self, + dim=160, + z_dim=48, + dec_dim=256, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[False, True, True], + dropout=0.0): + super(VideoVAE_, self).__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + self.temperal_upsample = temperal_downsample[::-1] + + # modules + self.encoder = Encoder3d_38(dim, z_dim * 2, dim_mult, num_res_blocks, + attn_scales, self.temperal_downsample, dropout) + self.conv1 = CausalConv3d(z_dim * 2, z_dim * 2, 1) + self.conv2 = CausalConv3d(z_dim, z_dim, 1) + self.decoder = Decoder3d_38(dec_dim, z_dim, dim_mult, num_res_blocks, + attn_scales, self.temperal_upsample, dropout) + + + def encode(self, x, scale): + self.clear_cache() + x = patchify(x, patch_size=2) + t = x.shape[2] + iter_ = 1 + (t - 1) // 4 + for i in range(iter_): + self._enc_conv_idx = [0] + if i == 0: + out, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, :1, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx) + else: + out_, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, 1 + 4 * (i - 1):1 + 4 * i, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx) + out = torch.cat([out, out_], 2) + mu, log_var = self.conv1(out).chunk(2, dim=1) + if isinstance(scale[0], torch.Tensor): + scale = [s.to(dtype=mu.dtype, device=mu.device) for s in scale] + mu = (mu - scale[0].view(1, self.z_dim, 1, 1, 1)) * scale[1].view( + 1, self.z_dim, 1, 1, 1) + else: + scale = scale.to(dtype=mu.dtype, device=mu.device) + mu = (mu - scale[0]) * scale[1] + self.clear_cache() + return mu + + + def decode(self, z, scale): + self.clear_cache() + if isinstance(scale[0], torch.Tensor): + scale = [s.to(dtype=z.dtype, device=z.device) for s in scale] + z = z / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view( + 1, self.z_dim, 1, 1, 1) + else: + scale = scale.to(dtype=z.dtype, device=z.device) + z = z / scale[1] + scale[0] + iter_ = z.shape[2] + x = self.conv2(z) + for i in range(iter_): + self._conv_idx = [0] + if i == 0: + out, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx, + first_chunk=True) + else: + out_, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx) + out = torch.cat([out, out_], 2) + out = unpatchify(out, patch_size=2) + self.clear_cache() + return out + + +class WanVideoVAE38(WanVideoVAE): + + def __init__(self, z_dim=48, dim=160): + super(WanVideoVAE, self).__init__() + + mean = [ + -0.2289, -0.0052, -0.1323, -0.2339, -0.2799, 0.0174, 0.1838, 0.1557, + -0.1382, 0.0542, 0.2813, 0.0891, 0.1570, -0.0098, 0.0375, -0.1825, + -0.2246, -0.1207, -0.0698, 0.5109, 0.2665, -0.2108, -0.2158, 0.2502, + -0.2055, -0.0322, 0.1109, 0.1567, -0.0729, 0.0899, -0.2799, -0.1230, + -0.0313, -0.1649, 0.0117, 0.0723, -0.2839, -0.2083, -0.0520, 0.3748, + 0.0152, 0.1957, 0.1433, -0.2944, 0.3573, -0.0548, -0.1681, -0.0667 + ] + std = [ + 0.4765, 1.0364, 0.4514, 1.1677, 0.5313, 0.4990, 0.4818, 0.5013, + 0.8158, 1.0344, 0.5894, 1.0901, 0.6885, 0.6165, 0.8454, 0.4978, + 0.5759, 0.3523, 0.7135, 0.6804, 0.5833, 1.4146, 0.8986, 0.5659, + 0.7069, 0.5338, 0.4889, 0.4917, 0.4069, 0.4999, 0.6866, 0.4093, + 0.5709, 0.6065, 0.6415, 0.4944, 0.5726, 1.2042, 0.5458, 1.6887, + 0.3971, 1.0600, 0.3943, 0.5537, 0.5444, 0.4089, 0.7468, 0.7744 + ] + self.mean = torch.tensor(mean) + self.std = torch.tensor(std) + self.scale = [self.mean, 1.0 / self.std] + + # init model + self.model = VideoVAE38_(z_dim=z_dim, dim=dim).eval().requires_grad_(False) + self.upsampling_factor = 16 + self.temporal_downsample_factor = 4 + self.z_dim = z_dim diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/runtime.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..bb8ae13f005bb181988b532278fe9056926c5321 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/runtime.py @@ -0,0 +1,436 @@ +import logging +import os +import inspect +from pathlib import Path + +import torch +from hydra.utils import instantiate +from omegaconf import DictConfig +from PIL import Image +import numpy as np +from einops import repeat +from omegaconf import OmegaConf + +from .trainer import Wan22Trainer +from .utils.logging_config import get_logger, setup_logging +from .utils.video_io import save_mp4 +from .utils import misc + +logger = get_logger(__name__) + + +def _normalize_mixed_precision(mixed_precision: str) -> str: + if not isinstance(mixed_precision, str): + raise ValueError(f"`mixed_precision` must be str, got {type(mixed_precision)}") + key = mixed_precision.strip().lower() + if key not in {"no", "fp16", "bf16"}: + raise ValueError( + f"Unsupported mixed_precision: {mixed_precision}. " + "Expected one of: ['no', 'fp16', 'bf16']." + ) + return key + + +def _mixed_precision_to_model_dtype(mixed_precision: str) -> torch.dtype: + precision = _normalize_mixed_precision(mixed_precision) + if precision == "no": + return torch.float32 + if precision == "fp16": + return torch.float16 + return torch.bfloat16 + + +def create_wan22_model( + model_id: str, + tokenizer_model_id: str, + dit_config, + tokenizer_max_len: int = 512, + train_shift: float = 5.0, + infer_shift: float = 5.0, + num_train_timesteps: int = 1000, + redirect_common_files: bool = True, + model_dtype: torch.dtype = torch.bfloat16, + device: str = "cuda", +): + from .models.wan22.wan22 import Wan22Core + + if isinstance(dit_config, DictConfig): + dit_config = OmegaConf.to_container(dit_config, resolve=True) + if not isinstance(dit_config, dict): + raise ValueError(f"`dit_config` must resolve to a dict, got {type(dit_config)}") + + return Wan22Core.from_wan22_pretrained( + device=device, + torch_dtype=model_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + tokenizer_max_len=int(tokenizer_max_len), + redirect_common_files=bool(redirect_common_files), + dit_config=dit_config, + train_shift=float(train_shift), + infer_shift=float(infer_shift), + num_train_timesteps=int(num_train_timesteps), + ) + + +def create_fastwam( + model_id: str, + tokenizer_model_id: str, + video_dit_config, + tokenizer_max_len: int = 512, + load_text_encoder: bool = True, + proprio_dim: int | None = None, + action_dit_config=None, + action_dit_pretrained_path: str | None = None, + skip_dit_load_from_pretrain: bool = False, + video_scheduler=None, + action_scheduler=None, + loss=None, + mot_checkpoint_mixed_attn: bool = True, + redirect_common_files: bool = True, + model_dtype: torch.dtype = torch.bfloat16, + device: str = "cuda", +): + from .models.wan22.fastwam import FastWAM + + if isinstance(video_dit_config, DictConfig): + video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True) + if not isinstance(video_dit_config, dict): + raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}") + + if isinstance(action_dit_config, DictConfig): + action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True) + if action_dit_config is None: + action_dit_config = {} + if not isinstance(action_dit_config, dict): + raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}") + + if isinstance(video_scheduler, DictConfig): + video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True) + if video_scheduler is None: + video_scheduler = {} + if not isinstance(video_scheduler, dict): + raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}") + + if isinstance(action_scheduler, DictConfig): + action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True) + if action_scheduler is None: + raise ValueError("`action_scheduler` is required for FastWAM.") + if not isinstance(action_scheduler, dict): + raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}") + required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"} + missing_keys = required_action_scheduler_keys - set(action_scheduler.keys()) + if missing_keys: + raise ValueError( + f"`action_scheduler` missing required keys: {sorted(missing_keys)}. " + "Expected keys: train_shift, infer_shift, num_train_timesteps." + ) + + if isinstance(loss, DictConfig): + loss = OmegaConf.to_container(loss, resolve=True) + if loss is None: + loss = {} + if not isinstance(loss, dict): + raise ValueError(f"`loss` must be dict-like, got {type(loss)}") + + return FastWAM.from_wan22_pretrained( + device=device, + torch_dtype=model_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + tokenizer_max_len=int(tokenizer_max_len), + load_text_encoder=bool(load_text_encoder), + proprio_dim=(None if proprio_dim is None else int(proprio_dim)), + redirect_common_files=bool(redirect_common_files), + video_dit_config=video_dit_config, + action_dit_config=action_dit_config, + action_dit_pretrained_path=action_dit_pretrained_path, + skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain), + mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn), + video_train_shift=float(video_scheduler.get("train_shift", 5.0)), + video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)), + video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)), + action_train_shift=float(action_scheduler["train_shift"]), + action_infer_shift=float(action_scheduler["infer_shift"]), + action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]), + loss_lambda_video=float(loss.get("lambda_video", 1.0)), + loss_lambda_action=float(loss.get("lambda_action", 1.0)), + ) + + +def create_fastwam_joint( + model_id: str, + tokenizer_model_id: str, + video_dit_config, + tokenizer_max_len: int = 512, + load_text_encoder: bool = True, + proprio_dim: int | None = None, + action_dit_config=None, + action_dit_pretrained_path: str | None = None, + skip_dit_load_from_pretrain: bool = False, + video_scheduler=None, + action_scheduler=None, + loss=None, + mot_checkpoint_mixed_attn: bool = True, + redirect_common_files: bool = True, + model_dtype: torch.dtype = torch.bfloat16, + device: str = "cuda", +): + from .models.wan22.fastwam_joint import FastWAMJoint + + if isinstance(video_dit_config, DictConfig): + video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True) + if not isinstance(video_dit_config, dict): + raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}") + + if isinstance(action_dit_config, DictConfig): + action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True) + if action_dit_config is None: + action_dit_config = {} + if not isinstance(action_dit_config, dict): + raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}") + + if isinstance(video_scheduler, DictConfig): + video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True) + if video_scheduler is None: + video_scheduler = {} + if not isinstance(video_scheduler, dict): + raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}") + + if isinstance(action_scheduler, DictConfig): + action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True) + if action_scheduler is None: + raise ValueError("`action_scheduler` is required for FastWAM.") + if not isinstance(action_scheduler, dict): + raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}") + required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"} + missing_keys = required_action_scheduler_keys - set(action_scheduler.keys()) + if missing_keys: + raise ValueError( + f"`action_scheduler` missing required keys: {sorted(missing_keys)}. " + "Expected keys: train_shift, infer_shift, num_train_timesteps." + ) + + if isinstance(loss, DictConfig): + loss = OmegaConf.to_container(loss, resolve=True) + if loss is None: + loss = {} + if not isinstance(loss, dict): + raise ValueError(f"`loss` must be dict-like, got {type(loss)}") + + return FastWAMJoint.from_wan22_pretrained( + device=device, + torch_dtype=model_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + tokenizer_max_len=int(tokenizer_max_len), + load_text_encoder=bool(load_text_encoder), + proprio_dim=(None if proprio_dim is None else int(proprio_dim)), + redirect_common_files=bool(redirect_common_files), + video_dit_config=video_dit_config, + action_dit_config=action_dit_config, + action_dit_pretrained_path=action_dit_pretrained_path, + skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain), + mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn), + video_train_shift=float(video_scheduler.get("train_shift", 5.0)), + video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)), + video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)), + action_train_shift=float(action_scheduler["train_shift"]), + action_infer_shift=float(action_scheduler["infer_shift"]), + action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]), + loss_lambda_video=float(loss.get("lambda_video", 1.0)), + loss_lambda_action=float(loss.get("lambda_action", 1.0)), + ) + + +def create_fastwam_idm( + model_id: str, + tokenizer_model_id: str, + video_dit_config, + tokenizer_max_len: int = 512, + load_text_encoder: bool = True, + proprio_dim: int | None = None, + action_dit_config=None, + action_dit_pretrained_path: str | None = None, + skip_dit_load_from_pretrain: bool = False, + video_scheduler=None, + action_scheduler=None, + loss=None, + mot_checkpoint_mixed_attn: bool = True, + redirect_common_files: bool = True, + model_dtype: torch.dtype = torch.bfloat16, + device: str = "cuda", +): + from .models.wan22.fastwam_idm import ( + FastWAMIDM, + ) + + if isinstance(video_dit_config, DictConfig): + video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True) + if not isinstance(video_dit_config, dict): + raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}") + + if isinstance(action_dit_config, DictConfig): + action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True) + if action_dit_config is None: + action_dit_config = {} + if not isinstance(action_dit_config, dict): + raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}") + + if isinstance(video_scheduler, DictConfig): + video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True) + if video_scheduler is None: + video_scheduler = {} + if not isinstance(video_scheduler, dict): + raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}") + + if isinstance(action_scheduler, DictConfig): + action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True) + if action_scheduler is None: + raise ValueError("`action_scheduler` is required for FastWAM.") + if not isinstance(action_scheduler, dict): + raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}") + required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"} + missing_keys = required_action_scheduler_keys - set(action_scheduler.keys()) + if missing_keys: + raise ValueError( + f"`action_scheduler` missing required keys: {sorted(missing_keys)}. " + "Expected keys: train_shift, infer_shift, num_train_timesteps." + ) + + if isinstance(loss, DictConfig): + loss = OmegaConf.to_container(loss, resolve=True) + if loss is None: + loss = {} + if not isinstance(loss, dict): + raise ValueError(f"`loss` must be dict-like, got {type(loss)}") + + return FastWAMIDM.from_wan22_pretrained( + device=device, + torch_dtype=model_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + tokenizer_max_len=int(tokenizer_max_len), + load_text_encoder=bool(load_text_encoder), + proprio_dim=(None if proprio_dim is None else int(proprio_dim)), + redirect_common_files=bool(redirect_common_files), + video_dit_config=video_dit_config, + action_dit_config=action_dit_config, + action_dit_pretrained_path=action_dit_pretrained_path, + skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain), + mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn), + video_train_shift=float(video_scheduler.get("train_shift", 5.0)), + video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)), + video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)), + action_train_shift=float(action_scheduler["train_shift"]), + action_infer_shift=float(action_scheduler["infer_shift"]), + action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]), + loss_lambda_video=float(loss.get("lambda_video", 1.0)), + loss_lambda_action=float(loss.get("lambda_action", 1.0)), + ) + + +def build_datasets(data_cfg: DictConfig): + train_ds = instantiate(data_cfg.train) + if data_cfg.get("val") is None: + val_ds = train_ds + else: + train_stats_path = data_cfg.train.get("pretrained_norm_stats") + default_stats_path = os.path.join(misc.get_work_dir(), "dataset_stats.json") + val_stats_path = data_cfg.val.get("pretrained_norm_stats") + pretrained_norm_stats = val_stats_path or train_stats_path or default_stats_path + logger.info("Building val dataset with pretrained_norm_stats: %s", pretrained_norm_stats) + val_ds = instantiate(data_cfg.val, pretrained_norm_stats=pretrained_norm_stats) + return train_ds, val_ds + + +def _resolve_train_device() -> str: + if not torch.cuda.is_available(): + return "cpu" + device_count = torch.cuda.device_count() + if device_count <= 1: + return "cuda:0" + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + if local_rank < 0 or local_rank >= device_count: + return "cuda:0" + return f"cuda:{local_rank}" + + +def run_training(cfg: DictConfig): + setup_logging( + log_level=logging.INFO, + is_main_process=torch.distributed.get_rank() == 0 if torch.distributed.is_initialized() else True, + ) + misc.register_work_dir(cfg.output_dir) + config_payload = OmegaConf.to_container(cfg, resolve=True) + with open(Path(cfg.output_dir) / "config.yaml", "w") as f: + OmegaConf.save(config_payload, f) + + model_device = _resolve_train_device() + mixed_precision = _normalize_mixed_precision(cfg.mixed_precision) + model_dtype = _mixed_precision_to_model_dtype(mixed_precision) + model = instantiate(cfg.model, model_dtype=model_dtype, device=model_device) + train_ds, val_ds = build_datasets(cfg.data) + + trainer = Wan22Trainer( + cfg=cfg, + model=model, + train_dataset=train_ds, + val_dataset=val_ds, + ) + trainer.train() + +def run_inference(cfg: DictConfig): + setup_logging(log_level=logging.INFO) + inference_cfg = cfg.inference + mixed_precision = _normalize_mixed_precision(cfg.mixed_precision) + model_dtype = _mixed_precision_to_model_dtype(mixed_precision) + + model = instantiate(cfg.model, model_dtype=model_dtype, device=str(inference_cfg.device)) + checkpoint_path = inference_cfg.get("checkpoint_path") + if checkpoint_path: + ckpt = Path(checkpoint_path) + if ckpt.exists(): + logger.info("Loading finetuned checkpoint: %s", checkpoint_path) + model.load_checkpoint(checkpoint_path) + else: + logger.warning("Checkpoint not found, skipping load: %s", checkpoint_path) + model.eval() + + def center_crop_resize(img: Image, width: int, height: int) -> Image.Image: + src_w, src_h = img.size + scale = max(width / src_w, height / src_h) + resized = img.resize((round(src_w * scale), round(src_h * scale)), resample=Image.BILINEAR) + rw, rh = resized.size + left = max((rw - width) // 2, 0) + top = max((rh - height) // 2, 0) + return resized.crop((left, top, left + width, top + height)) + + input_image = Image.open(str(inference_cfg.input_image_path)).convert("RGB") + input_image = center_crop_resize(input_image, width=inference_cfg.width, height=inference_cfg.height) + arr = np.array(input_image, dtype=np.float32) + x = torch.from_numpy(arr) + x = x.to(device=model.device, dtype=model.torch_dtype) + x = x * (2.0 / 255.0) - 1.0 + x = repeat(x, "H W C -> B C H W", B=1) + output_mp4 = str(inference_cfg.output_mp4) + + infer_kwargs = { + "prompt": str(inference_cfg.prompt), + "negative_prompt": str(inference_cfg.negative_prompt), + "text_cfg_scale": float(inference_cfg.text_cfg_scale), + "action_cfg_scale": float(inference_cfg.action_cfg_scale), + "input_image": x, + "num_frames": int(inference_cfg.num_frames), + "num_inference_steps": int(inference_cfg.num_inference_steps), + "sigma_shift": None if inference_cfg.get("sigma_shift") is None else float(inference_cfg.sigma_shift), + "seed": int(inference_cfg.seed), + "rand_device": str(inference_cfg.rand_device), + "tiled": bool(inference_cfg.tiled), + } + + infer_out = model.infer(**infer_kwargs) + video = infer_out["video"] + save_mp4(video, output_mp4, fps=15) + logger.info("Saved inference video to %s", output_mp4) + return output_mp4 diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/trainer.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..75e0f2e06f6a0d46188eb12c3d2a58eb654dd13c --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/trainer.py @@ -0,0 +1,791 @@ +import logging +import json +import inspect +import os +import re +from math import ceil +from pathlib import Path +import time + +import numpy as np +import torch +from accelerate import Accelerator +from omegaconf import DictConfig +from PIL import Image +from torch.optim.lr_scheduler import ConstantLR, CosineAnnealingLR, LinearLR, SequentialLR +from torch.utils.data import DataLoader + +from .utils.fs import ensure_dir +from .utils.logging_config import get_logger, setup_logging +from .utils.pytorch_utils import set_global_seed +from .utils.samplers import ResumableEpochSampler +from .utils.video_io import save_mp4 +from .utils.video_metrics import pil_frames_to_video_tensor, video_psnr, video_ssim + +logger = get_logger(__name__) + + +class Wan22Trainer: + def __init__(self, model, train_dataset, val_dataset=None, *, cfg: DictConfig): + self.model = model + self.train_dataset = train_dataset + self.val_dataset = val_dataset + self.cfg = cfg + self.output_dir = str(cfg.output_dir) + self.learning_rate = float(cfg.learning_rate) + self.weight_decay = float(cfg.weight_decay) + self.batch_size = int(cfg.batch_size) + self.num_workers = int(cfg.num_workers) + self.num_epochs = int(cfg.num_epochs) + max_steps = cfg.max_steps + self.max_steps = int(max_steps) if max_steps is not None else None + self.log_every = int(cfg.log_every) + self.save_every = int(cfg.save_every) + self.eval_every = int(cfg.eval_every) + self.eval_num_inference_steps = int(cfg.eval_num_inference_steps) + self.gradient_accumulation_steps = int(cfg.gradient_accumulation_steps) + self.max_grad_norm = float(cfg.max_grad_norm) + self.seed = int(cfg.seed) + + self.resume = cfg.resume + self.mixed_precision = str(cfg.mixed_precision).strip().lower() + if self.mixed_precision not in {"no", "fp16", "bf16"}: + raise ValueError( + f"Unsupported mixed_precision: {cfg.mixed_precision}. " + "Expected one of: ['no', 'fp16', 'bf16']." + ) + self.wandb_enabled = bool(cfg.wandb.enabled) + + self.accelerator = Accelerator( + gradient_accumulation_steps=self.gradient_accumulation_steps, + mixed_precision=self.mixed_precision, + step_scheduler_with_optimizer=False, + ) + + logger.info( + "Accelerate training: distributed_type=%s zero_stage=%s world_size=%d process_index=%d cfg_mixed_precision=%s accelerator_mixed_precision=%s grad_accum=%d grad_clip=%.4f", + self.accelerator.distributed_type, + self.accelerator.state.deepspeed_plugin.deepspeed_config.get("zero_optimization", {}).get("stage", "unknown"), + self.accelerator.num_processes, + self.accelerator.process_index, + self.mixed_precision, + self.accelerator.mixed_precision, + self.gradient_accumulation_steps, + self.max_grad_norm, + ) + logger.info("using accelerator.device=%s", self.accelerator.device) + worker_init_fn = set_global_seed(self.seed, get_worker_init_fn=True) + self._assert_dataset_length_consistent(self.train_dataset, "train_dataset") + if self.val_dataset is not None: + self._assert_dataset_length_consistent(self.val_dataset, "val_dataset") + + # Freeze non-trainable modules before optimizer/deepspeed initialization. + # This keeps DiT (+ optional proprio encoder) as trainable when ZeRO builds optimizer state. + self._apply_dit_only_train_mode(self.model) + trainable_params = list(self.model.dit.parameters()) + proprio_encoder = getattr(self.model, "proprio_encoder", None) + if proprio_encoder is not None: + trainable_params.extend(list(proprio_encoder.parameters())) + self.optimizer = torch.optim.AdamW( + trainable_params, + lr=self.learning_rate, + weight_decay=self.weight_decay, + betas=(0.9, 0.95), + ) + + self.train_loader = self._build_loader(self.train_dataset, worker_init_fn=worker_init_fn) + total_train_steps = self._estimate_total_train_steps() + self.max_steps = total_train_steps + warmup_steps = int(total_train_steps * 0.05) + self.scheduler = self._build_scheduler( + scheduler_type=cfg.lr_scheduler_type, + total_train_steps=total_train_steps, + warmup_steps=warmup_steps, + ) + self.global_step = 0 + self.epoch = 0 + self.batch_in_epoch = 0 + + self.checkpoint_root = os.path.join(self.output_dir, "checkpoints") + self.weights_dir = os.path.join(self.checkpoint_root, "weights") + self.state_dir = os.path.join(self.checkpoint_root, "state") + self.eval_dir = os.path.join(self.output_dir, "eval") + + ensure_dir(self.output_dir) + ensure_dir(self.checkpoint_root) + ensure_dir(self.weights_dir) + ensure_dir(self.state_dir) + ensure_dir(self.eval_dir) + + self.model, self.optimizer, self.train_loader, self.scheduler = self.accelerator.prepare( + self.model, self.optimizer, self.train_loader, self.scheduler + ) + self.optimizer.zero_grad(set_to_none=True) + self.wandb_run = None + self._init_wandb() + self._resume_or_load_checkpoint() + + val_size = len(self.val_dataset) if self.val_dataset is not None else len(self.train_dataset) + logger.info("Train/val dataset size: %d/%d", len(self.train_dataset), val_size) + + def _init_wandb(self): + if not self.wandb_enabled or not self.accelerator.is_main_process: + return + try: + import wandb + except ImportError as e: + raise ImportError( + "wandb logging is enabled in config (`wandb.enabled=true`) but wandb is not installed." + ) from e + + self.wandb_run = wandb.init( + entity=self.cfg.wandb.workspace, + project=self.cfg.wandb.project, + name=self.cfg.wandb.name, + group=None if self.cfg.wandb.group in (None, "null", "") else str(self.cfg.wandb.group), + mode=self.cfg.wandb.mode, + dir=self.output_dir, + ) + logger.info( + "Initialized wandb run: workspace=%s project=%s name=%s", + self.cfg.wandb.workspace, + self.cfg.wandb.project, + self.cfg.wandb.name, + ) + + def _wandb_log(self, payload: dict): + if self.wandb_run is None: + return + self.wandb_run.log(payload, step=self.global_step) + + def _finish_wandb(self): + if self.wandb_run is None: + return + self.wandb_run.finish() + self.wandb_run = None + + def _build_loader(self, dataset, worker_init_fn=None): + self.train_sampler = ResumableEpochSampler( + dataset=dataset, + seed=self.seed, + batch_size=self.batch_size, + num_processes=self.accelerator.num_processes, + ) + return DataLoader( + dataset, + batch_size=self.batch_size, + shuffle=False, + sampler=self.train_sampler, + num_workers=self.num_workers, + pin_memory=torch.cuda.is_available(), + worker_init_fn=worker_init_fn, + ) + + def _assert_dataset_length_consistent(self, dataset, dataset_name: str): + if not hasattr(dataset, "__len__"): + raise TypeError(f"`{dataset_name}` must implement __len__ for rank consistency checks.") + + local_length = len(dataset) + gathered_lengths = self.accelerator.gather( + torch.tensor([local_length], device=self.accelerator.device, dtype=torch.int64) + ).reshape(-1) + if torch.all(gathered_lengths == gathered_lengths[0]): + return + + if self.accelerator.is_main_process: + print(f"[dataset-check] {dataset_name} length mismatch across ranks after initialization:") + for rank, rank_length in enumerate(gathered_lengths.cpu().tolist()): + print(f"rank {rank}: {rank_length}") + self.accelerator.wait_for_everyone() + raise RuntimeError( + f"{dataset_name} length mismatch across ranks: {gathered_lengths.cpu().tolist()}" + ) + + def _estimate_total_train_steps(self) -> int: + if self.max_steps is not None: + return max(int(self.max_steps), 1) + + if not hasattr(self.train_dataset, "__len__"): + raise TypeError("`train_dataset` must implement __len__ when `max_steps` is None.") + + num_processes = max(int(self.accelerator.num_processes), 1) + global_batch_size = max(self.batch_size * num_processes, 1) + micro_steps_per_epoch = max(ceil(len(self.train_dataset) / global_batch_size), 1) + opt_steps_per_epoch = max( + ceil(micro_steps_per_epoch / self.gradient_accumulation_steps), + 1, + ) + return max(opt_steps_per_epoch * self.num_epochs, 1) + + def _build_scheduler(self, scheduler_type, total_train_steps: int, warmup_steps: int = 0): + scheduler_type = str(scheduler_type).strip().lower() + total_train_steps = max(int(total_train_steps), 1) + warmup_steps = min(max(int(warmup_steps), 0), total_train_steps - 1) + + remaining_steps = max(total_train_steps - warmup_steps, 1) + if scheduler_type == "cosine": + main_scheduler = CosineAnnealingLR( + self.optimizer, + T_max=remaining_steps, + eta_min=self.learning_rate * 0.01, + ) + elif scheduler_type == "constant": + main_scheduler = ConstantLR(self.optimizer, factor=1.0, total_iters=remaining_steps) + else: + raise ValueError( + f"Unsupported lr_scheduler_type: {scheduler_type}. " + "Expected one of: ['cosine', 'constant']." + ) + + if warmup_steps <= 0: + return main_scheduler + + warmup_scheduler = LinearLR( + self.optimizer, + start_factor=1.0 / warmup_steps, + end_factor=1.0, + total_iters=warmup_steps, + ) + return SequentialLR( + self.optimizer, + schedulers=[warmup_scheduler, main_scheduler], + milestones=[warmup_steps], + ) + + def _estimate_eta(self): + elapsed = max(time.perf_counter() - self.run_start_time, 1e-6) + done_steps = max(self.global_step - self.run_start_step, 1) + steps_per_sec = done_steps / elapsed + remaining_steps = max(self.max_steps - self.global_step, 0) + eta_seconds = int(remaining_steps / max(steps_per_sec, 1e-9)) + eta_h, eta_rem = divmod(eta_seconds, 3600) + eta_m, eta_s = divmod(eta_rem, 60) + return f"{eta_h:02d}:{eta_m:02d}:{eta_s:02d}", steps_per_sec + + def _resume_or_load_checkpoint(self): + resume = self.resume + if not resume: + return + resume_path = Path(str(resume)) + if resume_path.is_dir(): + logger.info("Resuming full training state from directory: %s", resume) + self.load_training_state(str(resume_path)) + return + if not resume_path.exists(): + raise FileNotFoundError(f"Resume checkpoint not found: {resume}") + logger.info("Loading weight checkpoint only: %s", resume) + self.accelerator.unwrap_model(self.model).load_checkpoint(str(resume_path), optimizer=None) + logger.warning("Loaded .pt weights only; optimizer/scheduler/step were not restored under ZeRO2.") + + def _set_dit_only_train_mode(self): + # Match DiffSynth's freeze_except("dit"): only DiT stays trainable/in-train-mode. + logger.info("Setting DiT to train mode and freezing other model components.") + model = self.accelerator.unwrap_model(self.model) + self._apply_dit_only_train_mode(model) + + @staticmethod + def _apply_dit_only_train_mode(model): + model.eval() + model.requires_grad_(False) + model.dit.train() + model.dit.requires_grad_(True) + proprio_encoder = getattr(model, "proprio_encoder", None) + if proprio_encoder is not None: + proprio_encoder.train() + proprio_encoder.requires_grad_(True) + + @staticmethod + def _to_batched_eval_sample(sample): + video = sample["video"] + prompt = sample["prompt"] + action = sample.get("action", None) + proprio = sample.get("proprio", None) + context = sample.get("context", None) + context_mask = sample.get("context_mask", None) + + if not isinstance(video, torch.Tensor): + raise TypeError( + f"Expected tensor video for evaluation, got {type(video)}. " + "Evaluation now expects `video` with shape [3,T,H,W] or [B,3,T,H,W]." + ) + if video.ndim == 4: + video = video.unsqueeze(0) + if video.ndim != 5: + raise ValueError(f"Expected video shape [3,T,H,W] or [B,3,T,H,W], got {tuple(video.shape)}") + num_video_frames = video.shape[2] + if num_video_frames <= 1: + raise ValueError(f"`sample['video']` must have at least 2 frames for action evaluation, got {num_video_frames}") + + if isinstance(prompt, str): + prompt = [prompt] + elif isinstance(prompt, tuple): + prompt = list(prompt) + elif not isinstance(prompt, list): + raise TypeError(f"Expected prompt type str/list[str], got {type(prompt)}") + if len(prompt) != video.shape[0]: + raise ValueError(f"Prompt batch mismatch: len(prompt)={len(prompt)} vs video batch={video.shape[0]}") + + action_horizon = None + action = None + if "action" in sample: + action = sample["action"] + if not isinstance(action, torch.Tensor): + raise TypeError( + f"`sample['action']` must be a torch.Tensor, got {type(action)}" + ) + if action.ndim == 2: + action = action.unsqueeze(0) + if action.ndim != 3: + raise ValueError(f"`sample['action']` must be 3D [B, T, a_dim], got shape {tuple(action.shape)}") + if action.shape[1] % (num_video_frames - 1) != 0: + raise ValueError(f"`sample['action']` temporal dimension must be divisible by video frames-1={num_video_frames - 1}, got {action.shape[1]}") + action_horizon = int(action.shape[1]) + + proprio = None + if "proprio" in sample: + proprio = sample["proprio"] + if not isinstance(proprio, torch.Tensor): + raise TypeError(f"`sample['proprio']` must be a torch.Tensor, got {type(proprio)}") + if proprio.ndim == 2: + proprio = proprio.unsqueeze(0) + if proprio.ndim != 3: + raise ValueError(f"`sample['proprio']` must be 3D [B, T, d], got shape {tuple(proprio.shape)}") + + if context is not None or context_mask is not None: + if context is None or context_mask is None: + raise ValueError("`context` and `context_mask` must both exist in eval sample.") + if context.ndim == 2: + context = context.unsqueeze(0) + if context_mask.ndim == 1: + context_mask = context_mask.unsqueeze(0) + if context.ndim != 3 or context_mask.ndim != 2: + raise ValueError( + f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}" + ) + + return { + "video": video, + "prompt": prompt, + "action": action, + "proprio": proprio, + "context": context, + "context_mask": context_mask, + "action_horizon": action_horizon, + } + + @torch.no_grad() + def evaluate(self): + if self.val_dataset is None: + return None + + model = self.accelerator.unwrap_model(self.model) + was_dit_training = model.dit.training + model.eval() + + # eval_index = (self.global_step + self.accelerator.process_index) % len(self.val_dataset) + rng = torch.Generator(device="cpu").manual_seed(self.global_step + self.accelerator.process_index) + eval_index = torch.randint(0, len(self.val_dataset), (1,), generator=rng).item() + sample = self._to_batched_eval_sample(self.val_dataset[eval_index]) + + # 1. training loss + with self.accelerator.autocast(): + val_loss, _ = model.training_loss(sample) + val_loss = val_loss.float().item() + + prompt = sample["prompt"][0] + video0 = sample["video"][0] # Tensor [3, T, H, W] in (-1, 1) + action = sample["action"][0] if "action" in sample and sample["action"] is not None else None + proprio = sample["proprio"][0, 0] if "proprio" in sample and sample["proprio"] is not None else None # from [1, T, d] to [d] + input_image = video0[:, 0].unsqueeze(0) + _, num_frames, _, _ = video0.shape + + # 2. inference and video saving + infer_kwargs = { + "input_image": input_image, + "num_frames": num_frames, + "action": action, + "action_horizon": sample['action_horizon'], + "proprio": proprio, + "text_cfg_scale": 1.0, + "action_cfg_scale": 1.0, + "num_inference_steps": self.eval_num_inference_steps, + "seed": 42, + "tiled": False, + } + if sample["context"] is not None: + infer_kwargs["prompt"] = None + infer_kwargs["context"] = sample["context"][0] + infer_kwargs["context_mask"] = sample["context_mask"][0] + else: + infer_kwargs["prompt"] = prompt + + pred = model.infer( + **infer_kwargs, + ) + + pred_video = pred["video"] + pred_action = pred.get("action", None) + + # 3. inference metrics against GT video + pred_video_tensor = pil_frames_to_video_tensor(pred_video) + gt_video_tensor = ((video0.detach().float().cpu().clamp(-1.0, 1.0) + 1.0) * 0.5).contiguous() + + assert pred_video_tensor.shape == gt_video_tensor.shape, ( + "Eval infer prediction/GT shape mismatch: " + f"pred={tuple(pred_video_tensor.shape)} vs gt={tuple(gt_video_tensor.shape)}" + ) + + psnr_rollout_vs_gt = video_psnr(pred=pred_video_tensor, target=gt_video_tensor) + ssim_rollout_vs_gt = video_ssim(pred=pred_video_tensor, target=gt_video_tensor) + + action_l1 = None + action_l2 = None + if action is not None and pred_action is not None: + if sample["proprio"] is None: + raise ValueError("Eval sample must contain `proprio` for action denormalization.") + proprio = sample["proprio"].detach().to(device="cpu", dtype=torch.float32) + + processor = self.val_dataset.lerobot_dataset.processor + + denorm_actions = {} + action_meta = processor.shape_meta["action"] + state_meta = processor.shape_meta["state"] + for action_name, raw_action in (("pred", pred_action), ("gt", action)): + if not isinstance(raw_action, torch.Tensor): + raise TypeError(f"{action_name} action must be a torch.Tensor, got {type(raw_action)}") + if raw_action.ndim == 2: + action_btd = raw_action.unsqueeze(0) + elif raw_action.ndim == 3 and raw_action.shape[0] == 1: + action_btd = raw_action + else: + raise ValueError( + f"{action_name} action must have shape [T, D] or [1, T, D], got {tuple(raw_action.shape)}" + ) + action_btd = action_btd.detach().to(device="cpu", dtype=torch.float32) + + batch = { + "action": action_btd, + "state": proprio, + } + batch = processor.action_state_merger.backward(batch) + batch = processor.normalizer.backward(batch) + merged_batch = { + "action": {meta["key"]: batch["action"][meta["key"]].squeeze(0) for meta in action_meta}, + "state": {meta["key"]: batch["state"][meta["key"]].squeeze(0) for meta in state_meta}, + } + merged_batch = processor.action_state_merger.forward(merged_batch) + denorm_action = merged_batch["action"].unsqueeze(0) + if denorm_action.ndim != 3 or denorm_action.shape[0] != 1: + raise ValueError( + f"Denormalized {action_name} action must have shape [1, T, D], got {tuple(denorm_action.shape)}" + ) + denorm_actions[action_name] = denorm_action + + pred_action_denorm = denorm_actions["pred"] + gt_action_denorm = denorm_actions["gt"] + + if pred_action_denorm.shape != gt_action_denorm.shape: + raise ValueError( + "Predicted action/GT action shape mismatch after denormalization: " + f"pred={tuple(pred_action_denorm.shape)} vs gt={tuple(gt_action_denorm.shape)}" + ) + action_diff = pred_action_denorm - gt_action_denorm + action_l1 = action_diff.abs().mean().item() + action_l2 = action_diff.pow(2).mean().item() + + # 4. VAE reconstruction metrics against GT video + gt_video_batch = video0.unsqueeze(0).to(device=model.device, dtype=model.torch_dtype) + vae_latents = model._encode_video_latents(gt_video_batch, tiled=False) + vae_recon_video = model._decode_latents(vae_latents, tiled=False) + vae_video_tensor = pil_frames_to_video_tensor(vae_recon_video) + + assert vae_video_tensor.shape == gt_video_tensor.shape, ( + "Eval VAE reconstruction/GT shape mismatch: " + f"vae={tuple(vae_video_tensor.shape)} vs gt={tuple(gt_video_tensor.shape)}" + ) + + psnr_decode_vs_gt = video_psnr(pred=vae_video_tensor, target=gt_video_tensor) + ssim_decode_vs_gt = video_ssim(pred=vae_video_tensor, target=gt_video_tensor) + + psnr_rollout_vs_decode = video_psnr(pred=pred_video_tensor, target=vae_video_tensor) + ssim_rollout_vs_decode = video_ssim(pred=pred_video_tensor, target=vae_video_tensor) + + stitched_video_tensor = torch.cat( + [pred_video_tensor, vae_video_tensor, gt_video_tensor], + dim=2, + ).contiguous() + stitched_frames = [] + for t in range(stitched_video_tensor.shape[1]): + frame = (stitched_video_tensor[:, t].permute(1, 2, 0).clamp(0.0, 1.0).numpy() * 255.0).astype(np.uint8) + stitched_frames.append(Image.fromarray(frame)) + + video_path = os.path.join( + self.eval_dir, + f"step_{self.global_step:06d}_rank_{self.accelerator.process_index:03d}.mp4", + ) + save_mp4(stitched_frames, video_path, fps=8) + + local_metrics = torch.tensor( + [ + float(val_loss), + float(psnr_rollout_vs_gt), + float(ssim_rollout_vs_gt), + float(psnr_rollout_vs_decode), + float(ssim_rollout_vs_decode), + float(psnr_decode_vs_gt), + float(ssim_decode_vs_gt), + float(action_l2) if action_l2 is not None else -1.0, + float(action_l1) if action_l1 is not None else -1.0, + ], + device=self.accelerator.device, + dtype=torch.float32, + ).unsqueeze(0) + gathered_metrics = self.accelerator.gather_for_metrics(local_metrics) + mean_metrics = gathered_metrics[:, :7].mean(dim=0) + action_l2_mean = gathered_metrics[:, 7].mean().item() if action_l2 is not None else None + action_l1_mean = gathered_metrics[:, 8].mean().item() if action_l1 is not None else None + + if was_dit_training: + self._set_dit_only_train_mode() + + result = { + "val_loss": float(mean_metrics[0].item()), + "psnr_rg": float(mean_metrics[1].item()), + "ssim_rg": float(mean_metrics[2].item()), + "psnr_rd": float(mean_metrics[3].item()), + "ssim_rd": float(mean_metrics[4].item()), + "psnr_dg": float(mean_metrics[5].item()), + "ssim_dg": float(mean_metrics[6].item()), + "video_path": video_path, + } + if action_l2_mean is not None: + result["action_l2"] = float(action_l2_mean) + if action_l1_mean is not None: + result["action_l1"] = float(action_l1_mean) + return result + + def _save_weights_checkpoint(self, step_tag: str): + model = self.accelerator.unwrap_model(self.model) + ckpt_path = os.path.join(self.weights_dir, f"{step_tag}.pt") + model.save_checkpoint(ckpt_path, optimizer=None, step=self.global_step) + return ckpt_path + + def _save_trainer_state(self, state_path: str): + state_file = os.path.join(state_path, "trainer_state.json") + payload = { + "global_step": int(self.global_step), + "epoch": int(self.epoch), + "batch_in_epoch": int(self.batch_in_epoch), + } + with open(state_file, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=True, indent=2) + + def save_checkpoint(self): + step_tag = f"step_{self.global_step:06d}" + + self.accelerator.wait_for_everyone() + ckpt_path = None + if self.accelerator.is_main_process: + ckpt_path = self._save_weights_checkpoint(step_tag=step_tag) + self.accelerator.wait_for_everyone() + + state_path = os.path.join(self.state_dir, step_tag) + ensure_dir(state_path) + self.accelerator.save_state(output_dir=state_path) + if self.accelerator.is_main_process: + self._save_trainer_state(state_path) + self.accelerator.wait_for_everyone() + + return {"weights_path": ckpt_path, "state_path": state_path} + + def load_training_state(self, state_dir: str): + self.accelerator.load_state(input_dir=state_dir) + state_file = Path(state_dir) / "trainer_state.json" + if state_file.exists(): + with open(state_file, "r", encoding="utf-8") as f: + payload = json.load(f) + self.global_step = int(payload["global_step"]) + + if "epoch" in payload and "batch_in_epoch" in payload: + self.epoch = int(payload["epoch"]) + self.batch_in_epoch = int(payload["batch_in_epoch"]) + self.train_sampler.set_epoch_offset(self.epoch) + self.train_sampler.set_resume_batch_offset(self.batch_in_epoch) + logger.info( + "Restored dataloader progress: epoch=%d batch_in_epoch=%d sample_offset=%d", + self.epoch, + self.batch_in_epoch, + self.batch_in_epoch * self.batch_size * self.accelerator.num_processes, + ) + else: + self.epoch = 0 + self.batch_in_epoch = 0 + self.train_sampler.clear_resume_batch_offset() + logger.warning( + "State file does not contain `epoch`/`batch_in_epoch`; " + "optimizer/scheduler were restored, but dataloader progress resume is skipped." + ) + self.accelerator.wait_for_everyone() + return + + match = re.search(r"step[_-](\d+)$", str(state_dir).rstrip("/")) + if match: + self.global_step = int(match.group(1)) + else: + self.global_step = 0 + self.epoch = 0 + self.batch_in_epoch = 0 + self.train_sampler.clear_resume_batch_offset() + self.accelerator.wait_for_everyone() + logger.info("Loaded accelerate training state from %s at step=%d", state_dir, self.global_step) + logger.warning( + "State file `%s` is missing; dataloader progress resume is skipped.", + state_file, + ) + + def train(self): + self._set_dit_only_train_mode() + + unwrapped_model = self.accelerator.unwrap_model(self.model) + + if self.max_steps is None: + raise ValueError("`max_steps` must be set before entering the while-step training loop.") + + logger.info("Starting training with max_steps=%d.", self.max_steps) + data_iter = iter(self.train_loader) + self.run_start_step = self.global_step + self.run_start_time = time.perf_counter() + + while self.global_step < self.max_steps: + try: + sample = next(data_iter) + self.batch_in_epoch += 1 + except StopIteration: + self.epoch += 1 + self.batch_in_epoch = 0 + self.train_sampler.clear_resume_batch_offset() + data_iter = iter(self.train_loader) + continue + + with self.accelerator.accumulate(self.model): + train_model = self.model if hasattr(self.model, "training_loss") else self.accelerator.unwrap_model(self.model) + + with self.accelerator.autocast(): + loss, loss_dict = train_model.training_loss(sample) + self.accelerator.backward(loss) + + if self.accelerator.sync_gradients: + grad_norm = self.accelerator.clip_grad_norm_(self.model.parameters(), self.max_grad_norm) + self.optimizer.step() + if not self.accelerator.optimizer_step_was_skipped: + self.scheduler.step() + self.optimizer.zero_grad(set_to_none=True) + self.global_step += 1 + global_loss = float( + self.accelerator.gather(loss.detach().float().reshape(1)).mean().item() + ) + global_loss_metrics = {} + for key, value in loss_dict.items(): + metric_tensor = torch.tensor(float(value), device=loss.device, dtype=torch.float32).reshape(1) + global_loss_metrics[key] = float( + self.accelerator.gather(metric_tensor).mean().item() + ) + grad_norm_tensor = torch.tensor(grad_norm, device=loss.device, dtype=torch.float32) + global_grad_norm = float(self.accelerator.gather(grad_norm_tensor).mean().item()) + + current_lr = float(self.optimizer.param_groups[0]["lr"]) + + if self.log_every > 0 and self.global_step % self.log_every == 0 and self.accelerator.is_main_process: + eta_str, steps_per_sec = self._estimate_eta() + description = "[train] epoch=%d step=%d/%d loss=%.4f " % ( + self.epoch, + self.global_step, + self.max_steps, + global_loss, + ) + if global_loss_metrics: + detail_str = " ".join([f"{k}={v:.4f}" for k, v in sorted(global_loss_metrics.items())]) + description += detail_str + " " + description += "lr=%.2e speed=%.2f step/s, %.2f samples/s eta=%s" % ( + current_lr, + steps_per_sec, + steps_per_sec * self.batch_size * self.accelerator.num_processes, + eta_str, + ) + logger.info(description) + + wandb_payload = { + "train/loss": global_loss, + "train/grad_norm": global_grad_norm, + "train/lr": current_lr, + "performance/steps_per_sec": steps_per_sec, + "performance/samples_per_sec": steps_per_sec * self.batch_size * self.accelerator.num_processes, + } + for key, value in global_loss_metrics.items(): + wandb_payload[f"train/{key}"] = value + self._wandb_log(wandb_payload) + + if ( + self.eval_every > 0 + and self.val_dataset is not None + and self.global_step % self.eval_every == 0 + ): + metrics = self.evaluate() + self.accelerator.wait_for_everyone() + if metrics is not None and self.accelerator.is_main_process: + description = "[eval] step=%d val_loss=%.4f infer_psnr=%.4f infer_ssim=%.4f" % ( + self.global_step, + metrics["val_loss"], + metrics["psnr_rd"], + metrics["ssim_rd"], + ) + if "action_l2" in metrics: + description += " action_l2=%.4f" % metrics["action_l2"] + if "action_l1" in metrics: + description += " action_l1=%.4f" % metrics["action_l1"] + logger.info(description) + eval_payload = { + "eval/val_loss": float(metrics["val_loss"]), + "eval/psnr_rg": float(metrics["psnr_rg"]), + "eval/ssim_rg": float(metrics["ssim_rg"]), + "eval/psnr_rd": float(metrics["psnr_rd"]), + "eval/ssim_rd": float(metrics["ssim_rd"]), + "eval/psnr_dg": float(metrics["psnr_dg"]), + "eval/ssim_dg": float(metrics["ssim_dg"]), + } + if "action_l2" in metrics: + eval_payload["eval/action_l2"] = float(metrics["action_l2"]) + if "action_l1" in metrics: + eval_payload["eval/action_l1"] = float(metrics["action_l1"]) + self._wandb_log(eval_payload) + + if self.save_every > 0 and self.global_step % self.save_every == 0: + ckpt_info = self.save_checkpoint() + if self.accelerator.is_main_process: + logger.info( + "[ckpt] step=%d weights=%s state=%s", + self.global_step, + ckpt_info["weights_path"], + ckpt_info["state_path"], + ) + + if self.global_step >= self.max_steps: + ckpt_info = self.save_checkpoint() + if self.accelerator.is_main_process: + logger.info( + "[done] max_steps reached step=%d weights=%s state=%s", + self.global_step, + ckpt_info["weights_path"], + ckpt_info["state_path"], + ) + return + + ckpt_info = self.save_checkpoint() + if self.accelerator.is_main_process: + logger.info( + "[done] training finished step=%d weights=%s state=%s", + self.global_step, + ckpt_info["weights_path"], + ckpt_info["state_path"], + ) + diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/__init__.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3d0eb927e25fab27c8dae7daa3d52a31c2d532b2 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/__init__.py @@ -0,0 +1,4 @@ +from .fs import ensure_dir +from .video_io import save_mp4 + +__all__ = ["ensure_dir", "save_mp4"] diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/config_resolvers.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/config_resolvers.py new file mode 100644 index 0000000000000000000000000000000000000000..4b1273b96b716ffa1028dfce42474376930a19f1 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/config_resolvers.py @@ -0,0 +1,70 @@ +import math +from pathlib import Path +from typing import Any, List, Callable, Optional +from omegaconf import OmegaConf + + +def _register(name: str, func: Callable) -> None: + """Idempotently register a resolver, replacing any existing one.""" + OmegaConf.register_new_resolver(name, func, replace=True) + +def _oc_load(path: str, key: Optional[str] = None) -> Any: + """ + Load a YAML/JSON config and optionally select a key. + Uses Hydra's to_absolute_path to honor original working dir. + """ + try: + from hydra.utils import to_absolute_path + except ImportError: + to_absolute_path = None # should not happen in normal Hydra runs + load_path = Path(path) + if not load_path.is_absolute() and to_absolute_path is not None: + load_path = Path(to_absolute_path(path)) + cfg = OmegaConf.load(load_path) + if key is None or key == "": + return cfg + return OmegaConf.select(cfg, key) + +def sum_shapes(shape_meta_list): + if not shape_meta_list: + return 0 + total = sum(int(item['shape']) for item in shape_meta_list if item["key"] is not None) + return total + +def max_action_dim(embodiment_datasets_cfg): + max_dim = 0 + for dataset_name, dataset_cfg in embodiment_datasets_cfg.items(): + if "shape_meta" in dataset_cfg: + action_cfg = dataset_cfg.shape_meta.action + current_dim = sum_shapes(action_cfg) + + if current_dim > max_dim: + max_dim = current_dim + + return max_dim + +def max_state_dim(embodiment_datasets_cfg): + max_dim = 0 + for dataset_name, dataset_cfg in embodiment_datasets_cfg.items(): + if "shape_meta" in dataset_cfg: + state_cfg = dataset_cfg.shape_meta.state + current_dim = sum_shapes(state_cfg) + if current_dim > max_dim: + max_dim = current_dim + + return max_dim + +def register_default_resolvers() -> None: + """ + Register all resolvers commonly used across entrypoints. + Safe to call multiple times. + """ + _register("oc.load", _oc_load) + _register("eval", eval) # allows arbitrary python code execution in configs using the ${eval:''} resolver + _register("split", lambda s, idx: s.split('/')[int(idx)]) # split string + _register("max", lambda x: max(x)) + _register("round_up", math.ceil) + _register("round_down", math.floor) + _register("sum_shapes", sum_shapes) + _register("max_action_dim", max_action_dim) + _register("max_state_dim", max_state_dim) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/fs.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/fs.py new file mode 100644 index 0000000000000000000000000000000000000000..b210bde5a81689be2969db6059ab71833958f43a --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/fs.py @@ -0,0 +1,5 @@ +import os + + +def ensure_dir(path: str): + os.makedirs(path, exist_ok=True) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/logging_config.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/logging_config.py new file mode 100644 index 0000000000000000000000000000000000000000..eba39b2fcff02c4d8488cf099d414eca19fbbc47 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/logging_config.py @@ -0,0 +1,140 @@ +import logging +from typing import Optional +import os + +import torch.distributed as dist + +from rich.logging import RichHandler + + +def setup_logging( + log_level: int = logging.INFO, + is_main_process: Optional[bool] = None, + rich_handler_kwargs: Optional[dict] = None, + formatter_kwargs: Optional[dict] = None, + preserve_hydra_handlers: bool = True +) -> None: + """ + Configure the logging system for the entire codebase with Rich formatting. + + In distributed training, only the main process outputs logs while other processes are silenced. + This function configures the root logger so all child loggers inherit the same configuration. + + Args: + log_level: Logging level (default INFO), only applies to main process + is_main_process: Whether this is the main process. If None, infer automatically. + rich_handler_kwargs: Additional kwargs to pass to RichHandler + formatter_kwargs: Additional kwargs to pass to Formatter (fmt and datefmt) + preserve_hydra_handlers: Keep existing FileHandlers from Hydra (default True) + + Example: + ```python + # In a single-machine script + from fastwam.utils.logging_config import setup_logging + setup_logging() + + # In a distributed training script + from accelerate import PartialState + from fastwam.utils.logging_config import setup_logging + + distributed_state = PartialState() + setup_logging( + log_level=logging.INFO, + is_main_process=distributed_state.is_main_process + ) + ``` + """ + if is_main_process is None: + is_main_process = _is_main_process() + + root_logger = logging.getLogger() + + if is_main_process: + # Save existing FileHandlers (e.g., from Hydra) if requested + existing_file_handlers = [] + if preserve_hydra_handlers: + existing_file_handlers = [ + h for h in root_logger.handlers + if isinstance(h, logging.FileHandler) + ] + + # Clear all default handlers on the root logger + root_logger.handlers.clear() + + # Configure RichHandler parameters + default_rich_kwargs = { + "markup": True, + "rich_tracebacks": True, + "show_level": True, + "show_path": True, + "show_time": True, + } + if rich_handler_kwargs: + default_rich_kwargs.update(rich_handler_kwargs) + + # Create RichHandler + rich_handler = RichHandler(**default_rich_kwargs) + + # Configure Formatter + default_formatter_kwargs = { + "fmt": "| >> %(message)s", + "datefmt": "%m/%d [%H:%M:%S]", + } + if formatter_kwargs: + default_formatter_kwargs.update(formatter_kwargs) + + formatter = logging.Formatter(**default_formatter_kwargs) + rich_handler.setFormatter(formatter) + + # Add handler and set logging level + root_logger.addHandler(rich_handler) + + # Restore existing FileHandlers from Hydra + for handler in existing_file_handlers: + root_logger.addHandler(handler) + + root_logger.setLevel(log_level) + + else: + # In non-main processes, set root logger level to ERROR to silence all logs + root_logger.setLevel(logging.ERROR) + + +def _is_main_process() -> bool: + """ + Best-effort check for main process without any synchronization. + """ + # Prefer torch.distributed state if initialized. + if dist is not None and dist.is_available() and dist.is_initialized(): + return dist.get_rank() == 0 + + # Fallback to environment variables commonly set by launchers. + for key in ("RANK", "SLURM_PROCID", "LOCAL_RANK"): + if key in os.environ: + return os.environ.get(key, "0") in ("0", "0\n", "") + + return True + +def get_logger(name: str = __name__, level: int = logging.INFO) -> logging.Logger: + """ + Drop-in replacement for accelerate.logging.get_logger: + - No implicit barriers. + - Only the main process emits log records by default. + """ + logger = logging.getLogger(name) + logger.setLevel(level) + + # if not logger.handlers and _is_main_process(): + # handler = logging.StreamHandler() + # formatter = logging.Formatter( + # fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + # datefmt="%Y-%m-%d %H:%M:%S", + # ) + # handler.setFormatter(formatter) + # logger.addHandler(handler) + + if not _is_main_process(): + logger.propagate = False + logger.disabled = True + + return logger diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/misc.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..9981aa8913c2b565c3b6cf5e317eed12d6a7222c --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/misc.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +from __future__ import annotations + +import collections +import collections.abc +import functools +import json +import os +import random +import time +from contextlib import ContextDecorator +from pathlib import Path +from typing import Any, Callable, List, Optional, Tuple, TypeVar +from urllib.parse import urlparse + +import boto3 +import numpy as np +import termcolor +import torch +from torch import nn +from torch.distributed._functional_collectives import AsyncCollectiveTensor +from torch.distributed._tensor.api import DTensor + +_WORK_DIR: str | None = None +_DEFAULT_WORK_DIR = "./runs/" + + +def register_work_dir(path: str | os.PathLike | None) -> None: + global _WORK_DIR + _WORK_DIR = str(path) if path is not None else None + os.makedirs(path, exist_ok=True) + + +def get_work_dir() -> str | None: + return _WORK_DIR if _WORK_DIR is not None else _DEFAULT_WORK_DIR diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/pytorch_utils.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/pytorch_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..87180a30f9edbba85cc1c94a3fd604b79fd15b1a --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/pytorch_utils.py @@ -0,0 +1,137 @@ +from typing import Dict, List, Callable, Optional +import os +import random +import collections + +import torch +import torch.distributed as dist +import numpy as np + + +def _resolve_global_rank() -> int: + if torch.distributed.is_available() and torch.distributed.is_initialized(): + return int(torch.distributed.get_rank()) + return int(os.environ.get("RANK", os.environ.get("SLURM_PROCID", os.environ.get("LOCAL_RANK", "0")))) + + +def set_global_seed(seed: int, get_worker_init_fn: bool = False) -> Optional[Callable[[int], None]]: + """Sets seed for all randomness libraries (mostly random, numpy, torch) and produces a `worker_init_fn`""" + assert np.iinfo(np.uint32).min < seed < np.iinfo(np.uint32).max, "Seed outside the np.uint32 bounds!" + + # Set Seed as an Environment Variable + os.environ["EXPERIMENT_GLOBAL_SEED"] = str(seed) + + # Process-specific seeding: offset by global rank so each process gets a different seed + global_rank = _resolve_global_rank() + process_seed = seed + global_rank + + random.seed(process_seed) + np.random.seed(process_seed) + torch.manual_seed(process_seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(process_seed) + + return worker_init_function if get_worker_init_fn else None + + +def worker_init_function(worker_id: int) -> None: + """ + Borrowed directly from PyTorch-Lightning; inspired by this issue comment in the PyTorch repo: + > Ref: https://github.com/pytorch/pytorch/issues/5059#issuecomment-817392562 + + Intuition: You can think of the seed sequence spawn function as a "janky" torch.Generator() or jax.PRNGKey that + you can run iterative splitting on to get new (predictable) randomness. + + :param worker_id: Identifier for the given worker [0, num_workers) for the Dataloader in question. + """ + # Get current global `rank` (if running distributed) and `process_seed` + process_seed = torch.initial_seed() + global_rank = _resolve_global_rank() + + # Back out the "base" (original) seed - the per-worker seed is set in PyTorch: + # > https://pytorch.org/docs/stable/data.html#data-loading-randomness + base_seed = process_seed - worker_id + + # "Magic" code --> basically creates a seed sequence that mixes different "sources" and seeds every library... + seed_seq = np.random.SeedSequence([base_seed, worker_id, global_rank]) + + # Use 128 bits (4 x 32-bit words) to represent seed --> generate_state(k) produces a `k` element array! + np.random.seed(seed_seq.generate_state(4)) + + # Spawn distinct child sequences for PyTorch (reseed) and stdlib random + torch_seed_seq, random_seed_seq, tf_seed_seq = seed_seq.spawn(3) + + # Torch Manual seed takes 64 bits (so just specify a dtype of uint64 + torch.manual_seed(torch_seed_seq.generate_state(1, dtype=np.uint64)[0]) + + # Use 128 Bits for `random`, but express as integer instead of as an array + random_seed = (random_seed_seq.generate_state(2, dtype=np.uint64).astype(list) * [1 << 64, 1]).sum() + random.seed(random_seed) + + +def dict_apply( + x: Dict[str, torch.Tensor], + func: Callable[[torch.Tensor], torch.Tensor] + ) -> Dict[str, torch.Tensor]: + result = dict() + for key, value in x.items(): + if isinstance(value, dict): + result[key] = dict_apply(value, func) + else: + result[key] = func(value) + return result + + +def dict_to_array(x): + data = np.concatenate([item for _, item in x.items()], axis=-1) + return data + + +def pad_remaining_dims(x, target): + assert x.shape == target.shape[:len(x.shape)] + return x.reshape(x.shape + (1,)*(len(target.shape) - len(x.shape))) + + +def dict_apply_split( + x: Dict[str, torch.Tensor], + split_func: Callable[[torch.Tensor], Dict[str, torch.Tensor]] + ) -> Dict[str, torch.Tensor]: + results = collections.defaultdict(dict) + for key, value in x.items(): + result = split_func(value) + for k, v in result.items(): + results[k][key] = v + return results + + +def dict_apply_reduce( + x: List[Dict[str, torch.Tensor]], + reduce_func: Callable[[List[torch.Tensor]], torch.Tensor] + ) -> Dict[str, torch.Tensor]: + result = dict() + for key in x[0].keys(): + result[key] = reduce_func([x_[key] for x_ in x]) + return result + + +def optimizer_to(optimizer, device): + for state in optimizer.state.values(): + for k, v in state.items(): + if isinstance(v, torch.Tensor): + state[k] = v.to(device=device) + return optimizer + +def is_rank0() -> bool: + """ + Best-effort check for main process without any synchronization. + """ + # Prefer torch.distributed state if initialized. + if dist is not None and dist.is_available() and dist.is_initialized(): + return dist.get_rank() == 0 + + # Fallback to environment variables commonly set by launchers. + for key in ("RANK", "SLURM_PROCID", "LOCAL_RANK"): + if key in os.environ: + return os.environ.get(key, "0") in ("0", "0\n", "") + + return True \ No newline at end of file diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/samplers.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/samplers.py new file mode 100644 index 0000000000000000000000000000000000000000..8b4f52a26a749fadc78a5c4ae7fa9068570be000 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/samplers.py @@ -0,0 +1,39 @@ +from typing import Iterator, Sized + +import torch +from torch.utils.data import Sampler + + +class ResumableEpochSampler(Sampler[int]): + def __init__(self, dataset: Sized, seed: int, batch_size: int, num_processes: int): + self.dataset = dataset + self.seed = int(seed) + self.batch_size = int(batch_size) + self.num_processes = int(num_processes) + self.epoch = 0 + self.epoch_offset = 0 + self.resume_batch_offset = 0 + + def set_epoch(self, epoch: int): + self.epoch = int(epoch) + + def set_epoch_offset(self, epoch_offset: int): + self.epoch_offset = int(epoch_offset) + + def set_resume_batch_offset(self, batch_in_epoch: int): + self.resume_batch_offset = int(batch_in_epoch) + + def clear_resume_batch_offset(self): + self.resume_batch_offset = 0 + + def __iter__(self) -> Iterator[int]: + g = torch.Generator(device="cpu") + g.manual_seed(self.seed + self.epoch + self.epoch_offset) + indices = torch.randperm(len(self.dataset), generator=g).tolist() + if self.epoch == 0 and self.resume_batch_offset > 0: + sample_offset = self.resume_batch_offset * self.batch_size * self.num_processes + indices = indices[sample_offset:] + return iter(indices) + + def __len__(self) -> int: + return len(self.dataset) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/video_io.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/video_io.py new file mode 100644 index 0000000000000000000000000000000000000000..74898b9c02a79c6b34b36af215a32743a8541d5d --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/video_io.py @@ -0,0 +1,34 @@ +import os +from typing import Iterable + +import imageio +import numpy as np +from PIL import Image + +from .fs import ensure_dir + + +def _to_even_frame(frame: np.ndarray) -> np.ndarray: + h, w = frame.shape[:2] + pad_h = h % 2 + pad_w = w % 2 + if pad_h == 0 and pad_w == 0: + return frame + return np.pad(frame, ((0, pad_h), (0, pad_w), (0, 0)), mode="edge") + + +def save_mp4(frames: Iterable[Image.Image], path: str, fps: int = 8): + ensure_dir(os.path.dirname(path) or ".") + writer = imageio.get_writer( + path, + fps=max(fps, 1), + codec="libx264", + format="FFMPEG", + pixelformat="yuv420p", + ) + try: + for frame in frames: + arr = np.array(frame.convert("RGB")) + writer.append_data(_to_even_frame(arr)) + finally: + writer.close() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/video_metrics.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/video_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..6b2837fa1d7b3c7c2532e99121ee46d68eb6be6e --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/utils/video_metrics.py @@ -0,0 +1,95 @@ +from typing import Sequence + +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image + + +def pil_frames_to_video_tensor(frames: Sequence[Image.Image]) -> torch.Tensor: + if len(frames) == 0: + raise ValueError("`frames` must be non-empty.") + + frame_tensors = [] + for frame in frames: + arr = np.array(frame.convert("RGB"), dtype=np.float32) / 255.0 + x = torch.from_numpy(arr).permute(2, 0, 1).contiguous() # [3, H, W] + frame_tensors.append(x) + return torch.stack(frame_tensors, dim=1) # [3, T, H, W] + + +def _gaussian_kernel_2d(kernel_size: int, sigma: float, channels: int, device: torch.device, dtype: torch.dtype): + coords = torch.arange(kernel_size, device=device, dtype=dtype) - (kernel_size - 1) / 2.0 + g = torch.exp(-(coords**2) / (2.0 * sigma * sigma)) + g = g / g.sum() + kernel_2d = torch.outer(g, g) + kernel_2d = kernel_2d / kernel_2d.sum() + kernel_2d = kernel_2d.view(1, 1, kernel_size, kernel_size) + return kernel_2d.repeat(channels, 1, 1, 1) + + +def video_psnr(pred: torch.Tensor, target: torch.Tensor, data_range: float = 1.0, eps: float = 1e-8) -> float: + """ + Compute average PSNR over all frames. + Expects `pred` and `target` in shape [3, T, H, W] with values in [0, 1]. + """ + if pred.shape != target.shape: + raise ValueError(f"Shape mismatch: pred={tuple(pred.shape)} target={tuple(target.shape)}") + + pred = pred.float() + target = target.float() + mse = (pred - target).pow(2).mean(dim=(0, 2, 3)) # [T] + psnr = 10.0 * torch.log10((data_range * data_range) / (mse + eps)) + return float(psnr.mean().item()) + + +def video_ssim( + pred: torch.Tensor, + target: torch.Tensor, + data_range: float = 1.0, + kernel_size: int = 11, + sigma: float = 1.5, + k1: float = 0.01, + k2: float = 0.03, +) -> float: + """ + Compute average SSIM over all frames. + Expects `pred` and `target` in shape [3, T, H, W] with values in [0, 1]. + """ + if pred.shape != target.shape: + raise ValueError(f"Shape mismatch: pred={tuple(pred.shape)} target={tuple(target.shape)}") + if pred.ndim != 4 or pred.shape[0] != 3: + raise ValueError(f"Expected [3, T, H, W], got {tuple(pred.shape)}") + if kernel_size % 2 == 0: + raise ValueError("`kernel_size` must be odd.") + + pred = pred.float().permute(1, 0, 2, 3).contiguous() # [T, 3, H, W] + target = target.float().permute(1, 0, 2, 3).contiguous() # [T, 3, H, W] + channels = pred.shape[1] + kernel = _gaussian_kernel_2d( + kernel_size=kernel_size, + sigma=sigma, + channels=channels, + device=pred.device, + dtype=pred.dtype, + ) + + pad = kernel_size // 2 + mu_x = F.conv2d(pred, kernel, padding=pad, groups=channels) + mu_y = F.conv2d(target, kernel, padding=pad, groups=channels) + + mu_x2 = mu_x * mu_x + mu_y2 = mu_y * mu_y + mu_xy = mu_x * mu_y + + sigma_x2 = F.conv2d(pred * pred, kernel, padding=pad, groups=channels) - mu_x2 + sigma_y2 = F.conv2d(target * target, kernel, padding=pad, groups=channels) - mu_y2 + sigma_xy = F.conv2d(pred * target, kernel, padding=pad, groups=channels) - mu_xy + + c1 = (k1 * data_range) ** 2 + c2 = (k2 * data_range) ** 2 + + numerator = (2.0 * mu_xy + c1) * (2.0 * sigma_xy + c2) + denominator = (mu_x2 + mu_y2 + c1) * (sigma_x2 + sigma_y2 + c2) + ssim_map = numerator / (denominator + 1e-12) + return float(ssim_map.mean().item()) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/audit_robodojo_warmup_reproduction.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/audit_robodojo_warmup_reproduction.py new file mode 100644 index 0000000000000000000000000000000000000000..aa2edc186076b7cd4ff1d929904a71a28b12fbe7 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/audit_robodojo_warmup_reproduction.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +"""Materialize fail-closed evidence for the warmup-only RoboDojo run.""" + +from __future__ import annotations + +import hashlib +from importlib import metadata as importlib_metadata +import inspect +import json +from pathlib import Path +import platform +from typing import Any + +import accelerate +import torch +import yaml + +import train_robodojo_rgb_state_joint_action_4way as fourway +import train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way as quantile + + +ROOT = Path(__file__).resolve().parent +REFERENCE_EXPERIMENT = ROOT / "ablation_test" / ( + "robodojo_mini_absolute_xyz_so3_q01q99_full_bidir_nostopgrad_fastwam_" + "tailpad_wanflow_init_space023037_fp32master_gbs256_8gpu_20epoch" +) +NEW_EXPERIMENT = ROOT / "ablation_test" / ( + "robodojo_mini_absolute_xyz_so3_q01q99_full_bidir_nostopgrad_fastwam_" + "tailpad_wanflow_init_space023037_fp32master_gbs256_8gpu_20epoch_" + "warmup1000optstep" +) +EVAL_RESULTS = Path( + "/workspace-wl/shouqx@xiaopeng.com/RoboDojo/eval_result/" + "spacedreamer-ego-v1-va-vln-h200/results" +) +LOCAL_INITIALIZATION = Path( + "/dev/shm/robodojo_init_space023037/joint_action_mot_full_dit_023037.pt" +) +SOURCE_INITIALIZATION = Path( + "/workspace-wl/shouqx@xiaopeng.com/wam/" + "model_wan_rgb_depth_point_cloud_action_mot_bidirectional_stopgrad_first_frame/" + "runs/SpaceDreamer-ego/v0_16gpu_gbs256_3ep_restart_20260815_r1/checkpoints/" + "joint_action_mot_full_dit_023037.pt" +) +LOCAL_INITIALIZATION_SHA256 = ( + "8e1520f9bea48c4b621f16b5872049185802c3e62d24392c35e7515ccaf35c98" +) +REFERENCE_WANDB_FILES = ( + REFERENCE_EXPERIMENT + / "run" + / "wandb" + / "run-20260825_052115-104klwse" + / "files" +) +REFERENCE_RUNTIME_PACKAGES = ( + "torch", + "torchvision", + "torchaudio", + "accelerate", + "transformers", + "huggingface_hub", + "safetensors", + "numpy", + "opencv-python-headless", + "av", + "polars", + "rosbags", + "diffusers", + "wandb", + "tokenizers", +) + + +def write_json(path: Path, payload: Any) -> None: + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def nested_diff(old: Any, new: Any, path: str = "") -> list[dict[str, Any]]: + if isinstance(old, dict) and isinstance(new, dict): + result: list[dict[str, Any]] = [] + for key in sorted(set(old) | set(new)): + child = f"{path}.{key}" if path else str(key) + if key not in old: + result.append({"path": child, "old": "", "new": new[key]}) + elif key not in new: + result.append({"path": child, "old": old[key], "new": ""}) + else: + result.extend(nested_diff(old[key], new[key], child)) + return result + if old != new: + return [{"path": path, "old": old, "new": new}] + return [] + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def action_and_video_keys(state: dict[str, Any]) -> tuple[list[str], list[str]]: + action = sorted( + key + for key in state + if key.startswith("mot.mixtures.action.") or key.startswith("action_") + ) + video = sorted(key for key in state if key.startswith("mot.mixtures.video.")) + return action, video + + +def main() -> None: + preflight = NEW_EXPERIMENT / "preflight" + preflight.mkdir(parents=True, exist_ok=True) + + reference_config_path = REFERENCE_EXPERIMENT / "run" / "resolved_config.yaml" + reference = yaml.safe_load(reference_config_path.read_text(encoding="utf-8")) + new_config_path = NEW_EXPERIMENT / "config.yaml" + resolved = quantile.absolute.load_absolute_ee_config(new_config_path) + quantile.validate_quantile_absolute_ee(resolved) + (preflight / "resolved_config.yaml").write_text( + yaml.safe_dump(resolved, sort_keys=False), encoding="utf-8" + ) + + differences = nested_diff(reference, resolved) + expected_paths = { + "experiment.name", + "experiment.output_dir", + "tracking.run_name", + "training.scheduler_step_unit", + } + actual_paths = {item["path"] for item in differences} + if actual_paths != expected_paths: + raise RuntimeError( + f"Effective config is not single-variable safe: {sorted(actual_paths)}" + ) + write_json( + preflight / "config_diff.json", + { + "allowed_paths": sorted(expected_paths), + "effective_diff": differences, + "single_variable_config_diff_passed": True, + }, + ) + + run_config = json.loads((EVAL_RESULTS / "run_config.json").read_text(encoding="utf-8")) + checkpoint_sha = json.loads( + (EVAL_RESULTS / "checkpoint.sha256.json").read_text(encoding="utf-8") + ) + lineage = { + "evaluation_results": str(EVAL_RESULTS.parent), + "training_checkpoint": run_config["checkpoint"], + "training_checkpoint_sha256": checkpoint_sha["sha256"], + "training_resolved_config": run_config["policy_config"], + "training_directory": str(REFERENCE_EXPERIMENT), + "initialization_checkpoint": str(SOURCE_INITIALIZATION), + "initialization_local_copy": str(LOCAL_INITIALIZATION), + "initialization_local_copy_sha256": LOCAL_INITIALIZATION_SHA256, + "initialization_workspace_sha256_status": ( + "not_completed: CPFS full-file read was stopped after 13.7/22.1 GB; " + "the equal-size tmpfs staging copy hash is recorded" + ), + } + write_json(preflight / "reference_lineage.json", lineage) + + metrics: dict[int, dict[str, Any]] = {} + for line in (REFERENCE_EXPERIMENT / "pipeline.log").read_text( + encoding="utf-8", errors="replace" + ).splitlines(): + if not line.startswith('{"global_step"'): + continue + record = json.loads(line) + step = int(record["global_step"]) + if step in {1, 10, 100, 240, 250, 260, 1000}: + metrics[step] = record + warmup_evidence = { + "configured_gradient_accumulation_steps": 4, + "configured_warmup_steps": 1000, + "configured_video_lr": 1.0e-5, + "configured_action_lr": 1.0e-4, + "old_loop_scheduler_call": "once_per_micro_step", + "observed_reference_metrics": metrics, + "conclusion": ( + "step 100 has 0.401x peak LR and step 250 has peak LR, matching " + "four scheduler advances per optimizer update" + ), + } + write_json(preflight / "old_warmup_evidence.json", warmup_evidence) + + video_lr = float(resolved["training"]["video_lr"]) + action_lr = float(resolved["training"]["action_lr"]) + warmup_steps = int(resolved["training"]["warmup_steps"]) + milestones: dict[str, Any] = {} + for step in (1, 10, 100, 250, 500, 999, 1000, 1001): + used_scale = min(step, warmup_steps) / warmup_steps + after_scale = min(step + 1, warmup_steps) / warmup_steps + milestones[str(step)] = { + "used_for_update": [video_lr * used_scale, action_lr * used_scale], + "after_scheduler_step": [video_lr * after_scale, action_lr * after_scale], + } + write_json( + preflight / "expected_lr_milestones.json", + { + "status": "analytical expectation; formal observations are recorded separately", + "semantics": { + "used_for_update": "LR immediately before this optimizer.step", + "after_scheduler_step": "LR for the next optimizer update", + }, + "video_base_lr": video_lr, + "action_base_lr": action_lr, + "action_to_video_ratio": action_lr / video_lr, + "milestones": milestones, + }, + ) + + key_configuration = { + "dataset_root": resolved["paths"]["dataset_root"], + "dataset_format": resolved["robodojo"]["format"], + "split": resolved["data"]["episode_split_per_task"], + "input": ["RGB frame 0", "initial action-state token"], + "output": ["RGB video (33 frames)", "action chunk (32 actions)"], + "action_representation": resolved["action"]["representation"], + "action_coordinate_frame": resolved["action"]["coordinate_frame"], + "action_normalization": resolved["action"]["normalization"], + "initialization": resolved["initialization"], + "per_gpu_batch_size": resolved["data"]["batch_size"], + "global_batch_size": resolved["training"]["global_batch_size"], + "gradient_accumulation_steps": resolved["training"][ + "gradient_accumulation_steps" + ], + "optimizer": { + "type": "AdamW", + "betas": [0.9, 0.95], + "eps": 1.0e-8, + "weight_decay": resolved["training"]["weight_decay"], + }, + "video_lr": video_lr, + "action_lr": action_lr, + "warmup_optimizer_steps": warmup_steps, + "lr_schedule": resolved["training"]["lr_schedule"], + "seed": resolved["experiment"]["seed"], + "epochs": resolved["training"]["exact_epochs"], + "optimizer_steps": resolved["training"]["max_steps"], + } + write_json(preflight / "key_configuration.json", key_configuration) + + reference_checkpoint = Path(run_config["checkpoint"]) + reference_payload = torch.load( + reference_checkpoint, map_location="cpu", weights_only=True, mmap=True + ) + source_payload = torch.load( + LOCAL_INITIALIZATION, map_location="cpu", weights_only=True, mmap=True + ) + reference_action, reference_video = action_and_video_keys( + reference_payload["state_dict"] + ) + source_action, source_video = action_and_video_keys(source_payload["state_dict"]) + loader_source = inspect.getsource(fourway._load_joint_checkpoint_rgb_slice) + copies_every_target_key = ( + 'for key in sorted(target_keys):' in loader_source + and 'target_tensor.copy_' in loader_source + ) + action_key_overlap = sorted(set(reference_action) & set(source_action)) + loader_audit = { + "configured_mode": resolved["initialization"]["mode"], + "reference_target_total_keys": len(reference_payload["state_dict"]), + "reference_target_video_keys": len(reference_video), + "reference_target_action_keys": len(reference_action), + "source_video_keys": len(source_video), + "source_action_keys": len(source_action), + "source_target_action_key_overlap": len(action_key_overlap), + "action_key_examples": action_key_overlap[:20], + "loader_iterates_all_target_keys": copies_every_target_key, + "old_log_statement": "strict-loaded 1657 RGB+Action keys", + "loads_only_rgb_video_slice": False, + "initialization_contract": "reference_exact_full_action_branch_copy", + "user_authorized_full_action_branch_copy": True, + "launch_safe_under_reference_loader_requirement": True, + "resolution": ( + "The user explicitly selected the reference loader's full 832-key " + "action-branch copy, so warmup counting remains the only training variable." + ), + } + write_json(preflight / "loader_audit.json", loader_audit) + + cache_normalizer = Path(resolved["paths"]["cache_dir"]) / "action_normalizer.pt" + cache_payload = torch.load(cache_normalizer, map_location="cpu", weights_only=True) + reference_state = reference_payload["state_dict"] + mean_expected = cache_payload["mean"].to(torch.bfloat16).to(torch.float32) + std_expected = cache_payload["std"].to(torch.bfloat16).to(torch.float32) + normalizer_audit = { + "cache": str(cache_normalizer), + "cache_sha256": sha256(cache_normalizer), + "normalization_type": resolved["action"]["normalization"]["type"], + "count_equal": bool( + torch.equal(cache_payload["count"], reference_state["action_normalizer.count"]) + ), + "fitted_equal": bool( + torch.equal(cache_payload["fitted"], reference_state["action_normalizer.fitted"]) + ), + "mean_equal_after_reference_bf16_roundtrip": bool( + torch.equal(mean_expected, reference_state["action_normalizer.mean"]) + ), + "std_equal_after_reference_bf16_roundtrip": bool( + torch.equal(std_expected, reference_state["action_normalizer.std"]) + ), + "reference_mean": reference_state["action_normalizer.mean"].tolist(), + "reference_std": reference_state["action_normalizer.std"].tolist(), + } + write_json(preflight / "normalizer_audit.json", normalizer_audit) + + wandb_metadata = json.loads( + (REFERENCE_WANDB_FILES / "wandb-metadata.json").read_text(encoding="utf-8") + ) + reference_requirements: dict[str, str] = {} + for requirement in (REFERENCE_WANDB_FILES / "requirements.txt").read_text( + encoding="utf-8" + ).splitlines(): + if "==" not in requirement: + continue + package, version = requirement.split("==", 1) + reference_requirements[package.lower().replace("-", "_")] = version + installed_packages = { + package: importlib_metadata.version(package) + for package in REFERENCE_RUNTIME_PACKAGES + } + expected_packages = { + package: reference_requirements[package.lower().replace("-", "_")] + for package in REFERENCE_RUNTIME_PACKAGES + } + reference_python = wandb_metadata["python"].removeprefix("CPython ") + package_match = installed_packages == expected_packages + exact_runtime_match = ( + platform.python_version() == reference_python + and package_match + and torch.version.cuda == str(wandb_metadata["cudaVersion"]) + ) + environment = { + "hostname": platform.node(), + "python": platform.python_version(), + "torch": torch.__version__, + "torch_cuda": torch.version.cuda, + "accelerate": accelerate.__version__, + "installed_packages": installed_packages, + "reference_wandb_metadata": str( + REFERENCE_WANDB_FILES / "wandb-metadata.json" + ), + "reference_requirements": str(REFERENCE_WANDB_FILES / "requirements.txt"), + "reference_python": reference_python, + "reference_cuda": str(wandb_metadata["cudaVersion"]), + "reference_packages": expected_packages, + "package_match": package_match, + "exact_runtime_match": exact_runtime_match, + } + write_json(preflight / "environment_audit.json", environment) + if not exact_runtime_match: + raise RuntimeError( + "Runtime does not match authoritative reference W&B metadata/requirements: " + f"{environment}" + ) + print( + json.dumps( + { + "config_diff_passed": True, + "loader_contract_passed": True, + "preflight": str(preflight), + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/build_piper_hdf5_index.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/build_piper_hdf5_index.py new file mode 100644 index 0000000000000000000000000000000000000000..b3ba0851c8b98f32d14ef1c53bc036c8c39f9e9f --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/build_piper_hdf5_index.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Build or verify the validated Piper HDF5 episode index.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import yaml + +import piper_hdf5 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--workers", type=int, default=16) + parser.add_argument("--verify-only", action="store_true") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + config = yaml.safe_load(args.config.read_text(encoding="utf-8")) + if not isinstance(config, dict): + raise ValueError("Piper config must be a YAML mapping") + if "base_config" in config: + raise ValueError("The lightweight Piper indexer requires a standalone config") + if args.verify_only: + payload = piper_hdf5.load_episode_index(config, verify_source_stats=True) + print( + f"[piper-index] verified episodes={payload['episode_count']} " + f"frames={payload['total_frames']} path={config['paths']['piper_index_file']}", + flush=True, + ) + return + piper_hdf5.write_episode_index(config, workers=args.workers) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/canonical_spacedreamer_data.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/canonical_spacedreamer_data.py new file mode 100644 index 0000000000000000000000000000000000000000..96a1462cbaf5b430a229e39c9a285c19691b1c89 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/canonical_spacedreamer_data.py @@ -0,0 +1,849 @@ +"""Canonical, source-neutral SpaceDreamer RGB/depth/motion dataset. + +Source-specific conversion belongs in the offline builder. Training consumes +only the schema defined here: + +* rectified 30-fps 256x256 RGB MP4; +* uint16 millimetre depth in ``depth.npz``; +* absolute action14 and camera6 in episode-frame-0 camera coordinates; +* an indexed, uncompressed tar container. + +Dense point maps intentionally remain deterministic derived data. They are +reconstructed from depth, K and camera6 by the same code for every source. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +import hashlib +import io +import json +import math +import os +from pathlib import Path +import stat +import subprocess +import tempfile +from typing import Any, Iterable + +import cv2 +import numpy as np +import torch +from torch.utils.data import Dataset + +from depth_preprocessing import DepthPreprocessingConfig, preprocess_depth_grayscale +from online_egoverse_action import so3_exp +from world_point_cloud import depth_video_to_world_grid, normalize_world_point_video + + +SCHEMA_NAME = "spacedreamer-canonical" +SCHEMA_VERSION = 1 +WINDOW_SIZE = 33 +ACTION_DIM = 14 +CAMERA_DIM = 6 +VALID_ACTION = np.uint8(1) +VALID_CAMERA = np.uint8(2) + + +@dataclass(frozen=True) +class BlobRange: + path: Path + offset: int + size: int + sha256: str + + +@dataclass(frozen=True) +class CanonicalEpisode: + episode_id: str + source: str + length: int + prompt: str + recording_group_id: str + rgb: BlobRange + depth: BlobRange + motion: BlobRange + metadata: BlobRange + + +@dataclass(frozen=True) +class IndexedCanonicalEpisode: + episode_id: str + source: str + length: int + prompt: str + recording_group_id: str + rgb: BlobRange + depth_path: Path + output_width: int + output_height: int + k_input: tuple[tuple[float, ...], ...] + distortion: tuple[float, ...] + k_rectified: tuple[tuple[float, ...], ...] + action_path: Path + camera_path: Path + validity_path: Path + motion_offset: int + coordinate_frame: str + endpoint_type: str + grip_encoding: str + + +@dataclass(frozen=True) +class CanonicalWindow: + episode_slot: int + start: int + sample_weight: float + is_padding: bool = False + + +def stable_u64(text: str, seed: int) -> int: + digest = hashlib.sha256(f"{seed}:{text}".encode("utf-8")).digest() + return int.from_bytes(digest[:8], "big", signed=False) + + +def _blob_from_json(value: dict[str, Any], root: Path) -> BlobRange: + path = Path(value["path"]) + if not path.is_absolute(): + path = root / path + return BlobRange( + path=path, + offset=int(value["offset"]), + size=int(value["size"]), + sha256=str(value["sha256"]), + ) + + +def load_canonical_manifest(path: Path, *, verify_blobs: bool = True) -> list[CanonicalEpisode]: + root = path.parent.parent + episodes: list[CanonicalEpisode] = [] + with path.open("r", encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + if not line.strip(): + continue + row = json.loads(line) + if row.get("schema_name") != SCHEMA_NAME or int(row.get("schema_version", -1)) != SCHEMA_VERSION: + raise ValueError(f"Unsupported canonical schema at {path}:{line_number}") + if row.get("geometry") != "rectified_pinhole" or row.get("distortion") != []: + raise ValueError(f"Non-canonical geometry at {path}:{line_number}") + if int(row["fps"]) != 30 or list(row["resolution"]) != [256, 256]: + raise ValueError(f"Non-canonical timing/resolution at {path}:{line_number}") + length = int(row["frames"]) + if length < WINDOW_SIZE: + raise ValueError(f"Short canonical episode at {path}:{line_number}") + members = row["members"] + episode = CanonicalEpisode( + episode_id=str(row["episode_id"]), + source=str(row["source"]), + length=length, + prompt=str(row["prompt"]), + recording_group_id=str(row["recording_group_id"]), + rgb=_blob_from_json(members["rgb"], root), + depth=_blob_from_json(members["depth"], root), + motion=_blob_from_json(members["motion"], root), + metadata=_blob_from_json(members["metadata"], root), + ) + if not episode.prompt.strip(): + raise ValueError(f"Empty canonical prompt at {path}:{line_number}") + episodes.append(episode) + if not episodes: + raise RuntimeError(f"No canonical episodes in {path}") + ids = [episode.episode_id for episode in episodes] + if len(ids) != len(set(ids)): + raise RuntimeError(f"Duplicate canonical episode IDs in {path}") + if verify_blobs: + paths = {blob.path for episode in episodes for blob in (episode.rgb, episode.depth, episode.motion, episode.metadata)} + missing = [str(path) for path in paths if not path.is_file()] + if missing: + raise FileNotFoundError(f"Missing canonical shards: {missing[:8]}") + for episode in episodes: + for blob in (episode.rgb, episode.depth, episode.motion, episode.metadata): + if blob.offset < 0 or blob.size <= 0 or blob.offset + blob.size > blob.path.stat().st_size: + raise ValueError(f"Invalid byte range for {episode.episode_id}: {blob}") + return episodes + + +def load_indexed_canonical_manifest( + path: Path, *, verify_paths: bool = True +) -> list[IndexedCanonicalEpisode]: + root = path.parent.parent + + def resolve(value: str) -> Path: + result = Path(value) + return result if result.is_absolute() else root / result + + episodes: list[IndexedCanonicalEpisode] = [] + with path.open("r", encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + if not line.strip(): + continue + row = json.loads(line) + if row.get("schema_name") != "spacedreamer-canonical-indexed" or int(row.get("schema_version", -1)) != 1: + raise ValueError(f"Unsupported indexed canonical schema at {path}:{line_number}") + if int(row.get("fps", -1)) != 30: + raise ValueError(f"Indexed canonical FPS is not 30 at {path}:{line_number}") + rgb = row["rgb"] + motion = row["motion"] + if int(motion.get("length", -1)) != int(row["frames"]): + raise ValueError(f"Indexed motion length mismatch at {path}:{line_number}") + k_input = np.asarray(row["K_input"], dtype=np.float64) + k_rectified = np.asarray(row["K_rectified"], dtype=np.float64) + distortion = np.asarray(row["distortion"], dtype=np.float64) + if ( + k_input.shape != (3, 3) or k_rectified.shape != (3, 3) + or not np.isfinite(k_input).all() or not np.isfinite(k_rectified).all() + or k_input[0, 0] <= 0 or k_input[1, 1] <= 0 + or k_rectified[0, 0] <= 0 or k_rectified[1, 1] <= 0 + or distortion.ndim != 1 or distortion.size not in {0, 4, 5, 8, 12, 14} + or not np.isfinite(distortion).all() + ): + raise ValueError(f"Invalid indexed camera calibration at {path}:{line_number}") + episode = IndexedCanonicalEpisode( + episode_id=str(row["episode_id"]), source=str(row["source"]), + length=int(row["frames"]), prompt=str(row["prompt"]), + recording_group_id=str(row["recording_group_id"]), + rgb=BlobRange(resolve(rgb["path"]), int(rgb["offset"]), int(rgb["size"]), str(rgb.get("sha256", ""))), + depth_path=resolve(row["depth_path"]), + output_width=int(row["output_width"]), output_height=int(row["output_height"]), + k_input=tuple(tuple(float(v) for v in values) for values in row["K_input"]), + distortion=tuple(float(v) for v in row["distortion"]), + k_rectified=tuple(tuple(float(v) for v in values) for values in row["K_rectified"]), + action_path=resolve(motion["action_path"]), + camera_path=resolve(motion["camera_path"]), + validity_path=resolve(motion["validity_path"]), + motion_offset=int(motion["offset"]), + coordinate_frame=str(row["coordinate_frame"]), + endpoint_type=str(row["endpoint_type"]), + grip_encoding=str(row["grip_encoding"]), + ) + if ( + episode.length < WINDOW_SIZE or not episode.prompt.strip() + or episode.output_width <= 0 or episode.output_height <= 0 + or episode.motion_offset < 0 + or episode.coordinate_frame != "episode_first_camera" + or not episode.endpoint_type or not episode.grip_encoding + ): + raise ValueError(f"Invalid indexed canonical episode at {path}:{line_number}") + episodes.append(episode) + ids = [episode.episode_id for episode in episodes] + if not episodes or len(ids) != len(set(ids)): + raise RuntimeError(f"Empty or duplicate indexed canonical manifest: {path}") + if verify_paths: + paths = { + value for episode in episodes for value in ( + episode.rgb.path, episode.depth_path, episode.action_path, + episode.camera_path, episode.validity_path, + ) + } + def inspect_path(value: Path) -> tuple[Path, int | None]: + try: + status = value.stat() + except FileNotFoundError: + return value, None + if not stat.S_ISREG(status.st_mode): + return value, None + return value, status.st_size + + missing: list[str] = [] + sizes: dict[Path, int] = {} + ordered_paths = sorted(paths, key=str) + with ThreadPoolExecutor(max_workers=min(32, len(ordered_paths))) as executor: + for value, size in executor.map(inspect_path, ordered_paths): + if size is None: + missing.append(str(value)) + else: + sizes[value] = size + if missing: + raise FileNotFoundError(f"Missing indexed canonical assets: {missing[:8]}") + for episode in episodes: + if episode.rgb.offset < 0 or episode.rgb.size <= 0 or episode.rgb.offset + episode.rgb.size > sizes[episode.rgb.path]: + raise ValueError(f"Invalid RGB byte range: {episode.episode_id}") + action_bytes = sizes[episode.action_path] + camera_bytes = sizes[episode.camera_path] + if action_bytes % (14 * 4) or camera_bytes % (6 * 4): + raise ValueError(f"Misaligned motion cache file: {episode.episode_id}") + action_rows = action_bytes // (14 * 4) + camera_rows = camera_bytes // (6 * 4) + validity_rows = sizes[episode.validity_path] + motion_end = episode.motion_offset + episode.length + if motion_end > min(action_rows, camera_rows, validity_rows): + raise ValueError(f"Invalid motion byte range: {episode.episode_id}") + return episodes + + +def split_canonical_by_group( + episodes: list[CanonicalEpisode], *, validation_fraction: float, seed: int +) -> dict[str, list[CanonicalEpisode]]: + """Split each source independently, keeping recording groups intact.""" + if not 0.0 < validation_fraction < 0.5: + raise ValueError("validation_fraction must be in (0, 0.5)") + result = {"train": [], "validation": []} + for source in sorted({episode.source for episode in episodes}): + source_episodes = [episode for episode in episodes if episode.source == source] + groups: dict[str, list[CanonicalEpisode]] = {} + for episode in source_episodes: + groups.setdefault(episode.recording_group_id, []).append(episode) + ordered = sorted(groups, key=lambda group: (stable_u64(f"{source}:{group}", seed), group)) + target = max(1, int(round(len(source_episodes) * validation_fraction))) + validation_groups: set[str] = set() + count = 0 + for group in ordered: + if count >= target: + break + validation_groups.add(group) + count += len(groups[group]) + for episode in source_episodes: + key = "validation" if episode.recording_group_id in validation_groups else "train" + result[key].append(episode) + if not result["train"] or not result["validation"]: + raise RuntimeError("Canonical split produced an empty partition") + train_groups = {(episode.source, episode.recording_group_id) for episode in result["train"]} + val_groups = {(episode.source, episode.recording_group_id) for episode in result["validation"]} + if train_groups & val_groups: + raise RuntimeError("Recording-group leakage detected") + return result + + +def coverage_starts(length: int, *, seed: int) -> list[int]: + """Construct randomized 33-frame windows covering every episode frame.""" + if length < WINDOW_SIZE: + raise ValueError(f"Episode length {length} is shorter than {WINDOW_SIZE}") + if length == WINDOW_SIZE: + return [0] + count = int(math.ceil(length / WINDOW_SIZE)) + gaps = np.ones(count - 1, dtype=np.int64) + remaining = length - WINDOW_SIZE - len(gaps) + rng = np.random.default_rng(seed) + while remaining > 0: + progressed = False + for slot in rng.permutation(len(gaps)): + capacity = WINDOW_SIZE - int(gaps[slot]) + if capacity <= 0: + continue + amount = min(capacity, remaining) + if amount > 1: + amount = int(rng.integers(1, amount + 1)) + gaps[slot] += amount + remaining -= amount + progressed = True + if remaining == 0: + break + if not progressed: + raise RuntimeError("Could not construct canonical coverage schedule") + starts = [0, *np.cumsum(gaps).astype(np.int64).tolist()] + if starts[-1] != length - WINDOW_SIZE: + raise RuntimeError("Canonical coverage does not reach the episode tail") + return starts + + +def _source_rank_map(world_size: int, sources: tuple[str, str], epoch: int) -> dict[int, str]: + if world_size % 2: + raise ValueError("Exact 1:1 source sampling requires an even world size") + half = world_size // 2 + swap = epoch % 2 + return { + rank: sources[((0 if rank < half else 1) + swap) % 2] + for rank in range(world_size) + } + + +def build_balanced_rank_schedules( + episodes: list[CanonicalEpisode], *, world_size: int, epoch: int, seed: int, + pad_to_multiple: int = 1, +) -> list[list[CanonicalWindow]]: + """Build exact 1:1 schedules by assigning half of DDP ranks per source.""" + sources = tuple(sorted({episode.source for episode in episodes})) + if len(sources) != 2: + raise ValueError(f"Exact mixer requires exactly two sources, found {sources}") + rank_source = _source_rank_map(world_size, sources, epoch) + ranks_by_source = { + source: [rank for rank in range(world_size) if rank_source[rank] == source] + for source in sources + } + schedules: list[list[CanonicalWindow]] = [[] for _ in range(world_size)] + for source in sources: + source_slots = [(slot, episode) for slot, episode in enumerate(episodes) if episode.source == source] + groups: list[tuple[int, list[int]]] = [] + for slot, episode in source_slots: + starts = coverage_starts(episode.length, seed=stable_u64(episode.episode_id, seed + epoch)) + groups.append((slot, starts)) + ranks = ranks_by_source[source] + loads = {rank: 0 for rank in ranks} + tie_rng = np.random.default_rng(seed + epoch + stable_u64(source, seed)) + ties = tie_rng.random(len(groups)) + order = sorted(range(len(groups)), key=lambda index: (-len(groups[index][1]), ties[index])) + bins: dict[int, list[tuple[int, list[int]]]] = {rank: [] for rank in ranks} + for index in order: + rank = min(ranks, key=lambda candidate: (loads[candidate], candidate)) + bins[rank].append(groups[index]) + loads[rank] += len(groups[index][1]) + for rank in ranks: + rng = np.random.default_rng(seed + epoch * 1_000_003 + rank) + rng.shuffle(bins[rank]) + for slot, starts in bins[rank]: + shuffled = list(starts) + rng.shuffle(shuffled) + episode = episodes[slot] + weight = 1.0 / float(episode.length) + schedules[rank].extend(CanonicalWindow(slot, int(start), weight) for start in shuffled) + common = max(len(schedule) for schedule in schedules) + common = int(math.ceil(common / pad_to_multiple) * pad_to_multiple) + # Shorter-source ranks repeat real samples with non-zero weight only after a + # complete pass. Synchronization imbalance within a source remains zero-weight. + per_source_max = { + source: max(len(schedules[rank]) for rank in ranks_by_source[source]) + for source in sources + } + global_real_max = max(per_source_max.values()) + for source in sources: + for rank in ranks_by_source[source]: + schedule = schedules[rank] + if not schedule: + raise RuntimeError(f"Rank {rank} received no {source} samples") + original = list(schedule) + while len(schedule) < common: + record = original[len(schedule) % len(original)] + source_padding = ( + len(schedule) < per_source_max[source] + or per_source_max[source] == global_real_max + ) + schedule.append(CanonicalWindow( + record.episode_slot, record.start, + 0.0 if source_padding else record.sample_weight, + source_padding, + )) + # Repetition must not overweight whichever shorter-source episodes happened + # to occur at the front of a shuffled rank. Give each episode the same total + # mass it had in one coverage pass, then normalize each source over all + # synchronous slots. Equal rank allocation makes source loss mass exactly 1:1. + for source in sources: + ranks = ranks_by_source[source] + occurrences: dict[int, int] = {} + for rank in ranks: + for record in schedules[rank]: + if record.sample_weight > 0: + occurrences[record.episode_slot] = occurrences.get(record.episode_slot, 0) + 1 + for rank in ranks: + schedules[rank] = [ + CanonicalWindow( + record.episode_slot, + record.start, + ( + (math.ceil(episodes[record.episode_slot].length / WINDOW_SIZE) + / float(episodes[record.episode_slot].length)) + / float(occurrences[record.episode_slot]) + ) if record.sample_weight > 0 else 0.0, + record.is_padding, + ) + for record in schedules[rank] + ] + total = sum(record.sample_weight for rank in ranks for record in schedules[rank]) + mean = total / float(len(ranks) * common) + for rank in ranks: + schedules[rank] = [ + CanonicalWindow( + record.episode_slot, record.start, + record.sample_weight / mean, + record.is_padding, + ) + for record in schedules[rank] + ] + return schedules + + +def _read_blob(blob: BlobRange, *, verify_sha256: bool = False) -> bytes: + with blob.path.open("rb", buffering=0) as handle: + handle.seek(blob.offset) + payload = handle.read(blob.size) + if len(payload) != blob.size: + raise IOError(f"Short read from {blob.path}: {len(payload)} != {blob.size}") + if verify_sha256 and hashlib.sha256(payload).hexdigest() != blob.sha256: + raise IOError(f"SHA256 mismatch for {blob.path}@{blob.offset}+{blob.size}") + return payload + + +def _decode_rgb_mp4(payload: bytes, expected_frames: int, cache_dir: Path | None) -> np.ndarray: + digest = hashlib.sha256(payload).hexdigest() + if cache_dir is None: + cache_dir = Path(tempfile.gettempdir()) / "spacedreamer_canonical_rgb" + cache_dir.mkdir(parents=True, exist_ok=True) + path = cache_dir / f"{digest}.mp4" + if not path.is_file() or path.stat().st_size != len(payload): + temporary = path.with_suffix(f".tmp.{os.getpid()}.mp4") + temporary.write_bytes(payload) + os.replace(temporary, path) + capture = cv2.VideoCapture(str(path)) + if not capture.isOpened(): + raise RuntimeError(f"Could not decode canonical RGB: {path}") + frames: list[np.ndarray] = [] + try: + while len(frames) < expected_frames: + ok, bgr = capture.read() + if not ok: + break + if bgr.shape[:2] != (256, 256): + raise ValueError(f"Canonical RGB is not 256x256: {bgr.shape}") + frames.append(cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)) + finally: + capture.release() + if len(frames) != expected_frames: + raise RuntimeError(f"Canonical RGB length mismatch: {len(frames)} != {expected_frames}") + return np.stack(frames) + + +def camera6_to_matrices(camera: np.ndarray) -> np.ndarray: + camera = np.asarray(camera, dtype=np.float32) + matrices = np.repeat(np.eye(4, dtype=np.float32)[None], len(camera), axis=0) + matrices[:, :3, 3] = camera[:, :3] + for index, vector in enumerate(camera[:, 3:6]): + matrices[index, :3, :3] = so3_exp(vector).astype(np.float32) + return matrices + + +class CanonicalSpaceDreamerDataset(Dataset): + """Training dataset with no source-dependent behavior.""" + + def __init__( + self, episodes: list[CanonicalEpisode], schedule: list[CanonicalWindow], *, + prompt_cache_dir: Path, action_mean: torch.Tensor, action_std: torch.Tensor, + camera_mean: torch.Tensor, camera_std: torch.Tensor, + normalization_clip: float | None, depth_config: dict[str, Any], + point_config: dict[str, Any], rgb_cache_dir: Path | None = None, + verify_member_hashes: bool = False, resolution: int = 256, + cache_derived_modalities: bool = False, prefetch_next_episode: bool = False, + ) -> None: + if int(resolution) != 256: + raise ValueError("Canonical v1 is fixed to 256x256") + self.episodes = episodes + self.schedule = schedule + self.prompt_cache_dir = prompt_cache_dir + self.action_mean = action_mean.float().cpu() + self.action_std = action_std.float().cpu() + self.camera_mean = camera_mean.float().cpu() + self.camera_std = camera_std.float().cpu() + self.normalization_clip = normalization_clip + self.depth_config = depth_config + self.point_config = point_config + self.rgb_cache_dir = rgb_cache_dir + self.verify_member_hashes = verify_member_hashes + # Accepted for a common training constructor. Canonical v1 already + # caches all expensive source-specific transforms offline. + self.cache_derived_modalities = bool(cache_derived_modalities) + self.prefetch_next_episode = bool(prefetch_next_episode) + self._cached_slot: int | None = None + self._cached: dict[str, Any] | None = None + + def __len__(self) -> int: + return len(self.schedule) + + def set_schedule(self, schedule: list[CanonicalWindow]) -> None: + self.schedule = schedule + self._cached_slot = None + self._cached = None + + def _load_episode(self, slot: int) -> dict[str, Any]: + if slot == self._cached_slot and self._cached is not None: + return self._cached + episode = self.episodes[slot] + depth_bytes = _read_blob(episode.depth, verify_sha256=self.verify_member_hashes) + motion_bytes = _read_blob(episode.motion, verify_sha256=self.verify_member_hashes) + with np.load(io.BytesIO(depth_bytes), allow_pickle=False) as bundle: + depth = bundle["depth_mm"].astype(np.uint16, copy=False) + intrinsics = bundle["K_256"].astype(np.float32) + with np.load(io.BytesIO(motion_bytes), allow_pickle=False) as bundle: + action = bundle["action_abs"].astype(np.float32) + camera = bundle["camera_abs"].astype(np.float32) + validity = bundle["validity"].astype(np.uint8) + rgb = _decode_rgb_mp4( + _read_blob(episode.rgb, verify_sha256=self.verify_member_hashes), + episode.length, self.rgb_cache_dir, + ) + if not (len(rgb) == len(depth) == len(action) == len(camera) == len(validity) == episode.length): + raise RuntimeError(f"Canonical modality length mismatch for {episode.episode_id}") + prompt_key = hashlib.sha256(episode.prompt.encode("utf-8")).hexdigest() + prompt_path = self.prompt_cache_dir / f"{prompt_key}.pt" + if not prompt_path.is_file(): + raise FileNotFoundError(prompt_path) + result = { + "rgb": rgb, "depth": depth, "K": intrinsics, + "action": action, "camera": camera, "validity": validity, + "prompt": torch.load(prompt_path, map_location="cpu", weights_only=True), + } + self._cached_slot, self._cached = slot, result + return result + + def __getitem__(self, index: int) -> dict[str, Any]: + record = self.schedule[index] + episode = self.episodes[record.episode_slot] + payload = self._load_episode(record.episode_slot) + start, stop = record.start, record.start + WINDOW_SIZE + validity = payload["validity"][start:stop] + if not bool(validity[0] & VALID_ACTION): + raise ValueError(f"Invalid initial action at {episode.episode_id}:{start}") + action_valid = (validity[1:] & VALID_ACTION) != 0 + camera_valid = (validity[1:] & VALID_CAMERA) != 0 + if not bool(camera_valid.all()): + raise ValueError(f"Invalid camera target at {episode.episode_id}:{start}") + action_raw = torch.from_numpy(payload["action"][start + 1:stop].copy()) + camera_raw = torch.from_numpy(payload["camera"][start + 1:stop].copy()) + camera_window = np.concatenate([ + payload["camera"][start:start + 1], payload["camera"][start + 1:stop] + ]) + camera_to_reference = camera6_to_matrices(camera_window) + processed_depth = preprocess_depth_grayscale( + payload["depth"][start:stop], + config=DepthPreprocessingConfig( + depth_min=float(self.depth_config["min"]), + depth_max=float(self.depth_config["max"]), + depth_invalid_value=float(self.depth_config["invalid_value"]), + depth_invalid_fill=str(self.depth_config["invalid_fill"]), + ), input_unit=str(self.depth_config["input_unit"]), + ) + depth_m = payload["depth"][start:stop].astype(np.float32) / np.float32(1000.0) + points, point_valid = depth_video_to_world_grid( + depth_m, payload["K"], camera_to_reference, + stride=1, min_depth_m=float(self.point_config["min_depth_m"]), + max_depth_m=float(self.point_config["max_depth_m"]), + valid_mask=processed_depth.valid_mask[:, 0], + ) + point, clipped = normalize_world_point_video( + points, point_valid, self.point_config["xyz_min_m"], + self.point_config["xyz_max_m"], + invalid_fill_reference=self.point_config["invalid_fill_reference_m"], + ) + action = (action_raw - self.action_mean) / self.action_std + camera = (camera_raw - self.camera_mean) / self.camera_std + if self.normalization_clip is not None: + action = action.clamp(-float(self.normalization_clip), float(self.normalization_clip)) + camera = camera.clamp(-float(self.normalization_clip), float(self.normalization_clip)) + action_is_pad = torch.from_numpy(~action_valid) + action = action.masked_fill(action_is_pad[:, None], 0) + rgb = torch.from_numpy( + payload["rgb"][start:stop].astype(np.float32) / np.float32(127.5) - np.float32(1.0) + ).permute(3, 0, 1, 2) + prompt = payload["prompt"] + return { + "sample_id": f"{episode.episode_id}_start_{start:06d}", + "episode_id": episode.episode_id, + "source": episode.source, + "endpoint_type": getattr(episode, "endpoint_type", "canonical_effector"), + "window_start": torch.tensor(start), + "rgb": rgb, + "depth": torch.from_numpy(processed_depth.vae_video).permute(1, 0, 2, 3), + "point": torch.from_numpy(point), + "action": action, + "camera": camera, + "initial_state": torch.from_numpy(payload["action"][start].copy()), + "action_is_pad": action_is_pad, + "camera_is_pad": torch.from_numpy(~camera_valid), + "sample_weight": torch.tensor(record.sample_weight, dtype=torch.float32), + "is_padding": torch.tensor(record.is_padding), + "point_clipped_fraction": torch.tensor(clipped, dtype=torch.float32), + "context": prompt["context"], + "context_mask": prompt["context_mask"].bool(), + "prompt": episode.prompt, + } + + +def _materialize_indexed_rgb(blob: BlobRange, cache_dir: Path | None) -> Path: + if blob.offset == 0 and blob.size == blob.path.stat().st_size: + return blob.path + if cache_dir is None: + cache_dir = Path(tempfile.gettempdir()) / "spacedreamer_indexed_rgb" + cache_dir.mkdir(parents=True, exist_ok=True) + signature = f"{blob.path}:{blob.path.stat().st_mtime_ns}:{blob.offset}:{blob.size}" + # A ranged member is only needed while one worker decodes its episode. + # Include the PID so workers cannot unlink each other's scratch file. + target = cache_dir / f"{hashlib.sha256(signature.encode()).hexdigest()}.{os.getpid()}.mp4" + if target.is_file() and target.stat().st_size == blob.size: + return target + temporary = target.with_suffix(f".tmp.{os.getpid()}.mp4") + with blob.path.open("rb", buffering=0) as source, temporary.open("wb") as output: + source.seek(blob.offset) + remaining = blob.size + while remaining: + block = source.read(min(16 * 1024 * 1024, remaining)) + if not block: + raise IOError(f"Short indexed RGB range: {blob.path}") + output.write(block) + remaining -= len(block) + os.replace(temporary, target) + return target + + +def _ffmpeg_decode_indexed_rgb( + path: Path, frame_index: np.ndarray, *, output_width: int, output_height: int, + k_input: np.ndarray, distortion: np.ndarray, k_rectified: np.ndarray, +) -> np.ndarray: + command = [ + "ffmpeg", "-v", "error", "-hwaccel", "none", "-i", str(path), + "-vf", f"scale={output_width}:{output_height}", "-pix_fmt", "bgr24", + "-f", "rawvideo", "pipe:1", + ] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert process.stdout is not None + frame_bytes = output_width * output_height * 3 + map_x = map_y = None + if distortion.size and np.any(np.abs(distortion) > 0): + map_x, map_y = cv2.initUndistortRectifyMap( + k_input.astype(np.float64), distortion.astype(np.float64), None, + k_rectified.astype(np.float64), (output_width, output_height), cv2.CV_32FC1, + ) + + def read_exact() -> bytes: + chunks: list[bytes] = [] + remaining = frame_bytes + while remaining: + block = process.stdout.read(remaining) + if not block: + break + chunks.append(block) + remaining -= len(block) + return b"".join(chunks) + + frames: list[np.ndarray] = [] + next_source = 0 + try: + for target_value in frame_index.astype(np.int64): + target = int(target_value) + if target < next_source: + raise ValueError("Indexed source frame_index must be increasing") + while next_source <= target: + payload = read_exact() + if len(payload) != frame_bytes: + stderr = process.stderr.read().decode("utf-8", errors="replace") if process.stderr else "" + raise RuntimeError(f"ffmpeg decode failed at {path}:{target}: {stderr[-2000:]}") + if next_source == target: + bgr = np.frombuffer(payload, np.uint8).reshape(output_height, output_width, 3) + if map_x is not None and map_y is not None: + bgr = cv2.remap(bgr, map_x, map_y, interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT) + crop = min(output_height, output_width) + top, left = (output_height - crop) // 2, (output_width - crop) // 2 + bgr = cv2.resize(bgr[top:top + crop, left:left + crop], (256, 256), interpolation=cv2.INTER_AREA) + frames.append(cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)) + next_source += 1 + finally: + process.stdout.close() + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill(); process.wait(timeout=5) + return np.stack(frames) + + +class IndexedCanonicalSpaceDreamerDataset(CanonicalSpaceDreamerDataset): + """One canonical loader for direct-file and indexed-tar RGB assets.""" + + episodes: list[IndexedCanonicalEpisode] + + def _load_episode(self, slot: int) -> dict[str, Any]: + if slot == self._cached_slot and self._cached is not None: + return self._cached + episode = self.episodes[slot] + with np.load(episode.depth_path, allow_pickle=False) as bundle: + depth = bundle["depth_mm"][:episode.length] + frame_index = bundle["frame_index"][:episode.length].astype(np.int64) + source_k = bundle["K"].astype(np.float32) + if ( + depth.dtype != np.uint16 + or depth.shape != (episode.length, episode.output_height, episode.output_width) + or frame_index.shape != (episode.length,) + or frame_index[0] < 0 + or np.any(np.diff(frame_index) <= 0) + ): + raise ValueError( + f"Indexed depth/frame geometry mismatch for {episode.episode_id}: " + f"depth={depth.shape} frame_index={frame_index.shape}" + ) + k_rectified = np.asarray(episode.k_rectified, dtype=np.float32) + if not np.allclose(source_k, k_rectified, atol=1e-3): + raise ValueError(f"Indexed K mismatch for {episode.episode_id}") + crop = min(depth.shape[1], depth.shape[2]) + top, left = (depth.shape[1] - crop) // 2, (depth.shape[2] - crop) // 2 + depth = depth[:, top:top + crop, left:left + crop] + depth = np.stack([ + cv2.resize(value, (256, 256), interpolation=cv2.INTER_NEAREST) + for value in depth + ]) + from world_point_cloud import scale_intrinsics_after_center_crop + intrinsics = scale_intrinsics_after_center_crop( + k_rectified, episode.output_height, episode.output_width, 256 + ) + rgb_path = _materialize_indexed_rgb(episode.rgb, self.rgb_cache_dir) + ranged_rgb = not ( + episode.rgb.offset == 0 + and episode.rgb.size == episode.rgb.path.stat().st_size + ) + try: + rgb = _ffmpeg_decode_indexed_rgb( + rgb_path, frame_index, output_width=episode.output_width, + output_height=episode.output_height, + k_input=np.asarray(episode.k_input, dtype=np.float32), + distortion=np.asarray(episode.distortion, dtype=np.float64), + k_rectified=k_rectified, + ) + finally: + if ranged_rgb: + rgb_path.unlink(missing_ok=True) + action_rows = episode.action_path.stat().st_size // (14 * 4) + camera_rows = episode.camera_path.stat().st_size // (6 * 4) + action_map = np.memmap(episode.action_path, dtype=" dict[str, Any]: + if not samples: + raise ValueError("Cannot collate an empty batch") + tensors = ( + "rgb", "depth", "point", "action", "camera", "initial_state", + "action_is_pad", "camera_is_pad", "sample_weight", "is_padding", + "point_clipped_fraction", "window_start", + ) + maximum = max(int(sample["context"].shape[0]) for sample in samples) + contexts, masks = [], [] + for sample in samples: + padding = maximum - int(sample["context"].shape[0]) + contexts.append(torch.nn.functional.pad(sample["context"], (0, 0, 0, padding))) + masks.append(torch.nn.functional.pad(sample["context_mask"], (0, padding), value=False)) + return { + **{key: torch.stack([sample[key] for sample in samples]) for key in tensors}, + "context": torch.stack(contexts), "context_mask": torch.stack(masks), + "sample_id": [sample["sample_id"] for sample in samples], + "episode_id": [sample["episode_id"] for sample in samples], + "source": [sample["source"] for sample in samples], + "endpoint_type": [sample["endpoint_type"] for sample in samples], + "prompt": [sample["prompt"] for sample in samples], + } + + +def iter_motion_rows(episodes: Iterable[CanonicalEpisode]) -> Iterable[tuple[str, np.ndarray, np.ndarray, np.ndarray]]: + for episode in episodes: + with np.load(io.BytesIO(_read_blob(episode.motion)), allow_pickle=False) as bundle: + yield ( + episode.source, + bundle["action_abs"].astype(np.float64), + bundle["camera_abs"].astype(np.float64), + bundle["validity"].astype(np.uint8), + ) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/controlled_directional_mot.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/controlled_directional_mot.py new file mode 100644 index 0000000000000000000000000000000000000000..a90e4a14c95cd546810a9b3b8b72277c7b6ad846 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/controlled_directional_mot.py @@ -0,0 +1,187 @@ +"""Two-call MoT used for controlled cross-modal direction ablations. + +Both ablations execute exactly two attention calls per transformer layer: +one for video queries and one for Action queries. They differ only in +whether video queries can read Action K/V and whether Action queries see +differentiable or detached video K/V. +""" + +from __future__ import annotations + +from typing import Dict, Optional + +import torch +from torch import nn + +from fastwam.models.wan22.mot import MoT + + +class ControlledDirectionalTwoCallMoT(MoT): + """Base implementation with explicit forward/backward coupling switches.""" + + video_reads_action: bool + detach_video_for_action: bool + + def __init__( + self, + mixtures: Dict[str, nn.Module], + mot_checkpoint_mixed_attn: bool = True, + *, + video_reads_action: bool, + detach_video_for_action: bool, + ) -> None: + super().__init__(mixtures, mot_checkpoint_mixed_attn) + if self.expert_order != ["video", "action"]: + raise ValueError("Expected expert order ['video', 'action']") + self.video_reads_action = bool(video_reads_action) + self.detach_video_for_action = bool(detach_video_for_action) + + def forward( + self, + embeds_all: Dict[str, torch.Tensor], + attention_mask: Optional[torch.Tensor], + freqs_all: Dict[str, torch.Tensor], + context_all: Dict[str, Optional[dict]], + t_mod_all: Dict[str, torch.Tensor], + ) -> Dict[str, torch.Tensor]: + if attention_mask is not None and attention_mask.ndim != 2: + raise ValueError( + f"attention_mask must be 2D [S, S], got {attention_mask.shape}" + ) + if attention_mask is not None and attention_mask.shape[0] != attention_mask.shape[1]: + raise ValueError( + f"attention_mask must be square, got {attention_mask.shape}" + ) + + tokens_all = dict(embeds_all) + for layer_idx in range(self.num_layers): + attention_io: dict[str, tuple[torch.Tensor, ...]] = {} + cached: dict[str, tuple] = {} + for name in self.expert_order: + expert = self.mixtures[name] + block = expert.blocks[layer_idx] + values = self._build_expert_attention_io( + expert=expert, + block=block, + x=tokens_all[name], + freqs=freqs_all[name], + t_mod=t_mod_all[name], + ) + attention_io[name] = values[:3] + cached[name] = (block, *values[3:]) + + video_q, video_k, video_v = attention_io["video"] + action_q, action_k, action_v = attention_io["action"] + video_len = int(video_q.shape[1]) + action_len = int(action_q.shape[1]) + if attention_mask is not None and attention_mask.shape[0] != video_len + action_len: + raise ValueError( + "Attention mask/token length mismatch: " + f"{attention_mask.shape[0]} != {video_len + action_len}" + ) + + if self.video_reads_action: + video_attention = self._mixed_attention( + q_cat=video_q, + k_cat=torch.cat((video_k, action_k), dim=1), + v_cat=torch.cat((video_v, action_v), dim=1), + attention_mask=( + None if attention_mask is None else attention_mask[:video_len, :] + ), + ) + else: + video_attention = self._mixed_attention( + q_cat=video_q, + k_cat=video_k, + v_cat=video_v, + attention_mask=( + None + if attention_mask is None + else attention_mask[:video_len, :video_len] + ), + ) + + video_k_for_action = ( + video_k.detach() + if self.detach_video_for_action + else video_k + ) + video_v_for_action = ( + video_v.detach() + if self.detach_video_for_action + else video_v + ) + action_attention = self._mixed_attention( + q_cat=action_q, + k_cat=torch.cat((video_k_for_action, action_k), dim=1), + v_cat=torch.cat((video_v_for_action, action_v), dim=1), + attention_mask=( + None if attention_mask is None else attention_mask[video_len:, :] + ), + ) + + for name, mixed_attention in ( + ("video", video_attention), + ("action", action_attention), + ): + ( + block, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + use_gradient_checkpointing, + ) = cached[name] + tokens_all[name] = self._apply_post_with_optional_checkpoint( + block=block, + residual_x=residual_x, + gate_msa=gate_msa, + shift_mlp=shift_mlp, + scale_mlp=scale_mlp, + gate_mlp=gate_mlp, + use_gradient_checkpointing=use_gradient_checkpointing, + mixed_slice=mixed_attention, + context_payload=context_all.get(name), + ) + return tokens_all + + +class ForwardOffBackwardOnMoT(ControlledDirectionalTwoCallMoT): + """Video cannot read Action; Action loss can update video through K/V.""" + + def __init__(self, mixtures, mot_checkpoint_mixed_attn=True) -> None: + super().__init__( + mixtures, + mot_checkpoint_mixed_attn, + video_reads_action=False, + detach_video_for_action=False, + ) + + +class ForwardOnBackwardOffMoT(ControlledDirectionalTwoCallMoT): + """Video reads Action; Action reads detached video K/V.""" + + def __init__(self, mixtures, mot_checkpoint_mixed_attn=True) -> None: + super().__init__( + mixtures, + mot_checkpoint_mixed_attn, + video_reads_action=True, + detach_video_for_action=True, + ) + + +class ForwardOnBackwardOnTwoCallMoT(ControlledDirectionalTwoCallMoT): + """Full bidirectional MoT with gradients enabled across both modalities. + + The two attention calls are numerically equivalent to one concatenated + all-to-all attention call, while retaining explicit per-expert query paths. + """ + + def __init__(self, mixtures, mot_checkpoint_mixed_attn=True) -> None: + super().__init__( + mixtures, + mot_checkpoint_mixed_attn, + video_reads_action=True, + detach_video_for_action=False, + ) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/depth_preprocessing.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/depth_preprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..4c8f8656f67c602157b1e667d7097d47241249cd --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/depth_preprocessing.py @@ -0,0 +1,91 @@ +"""Depth-to-grayscale preprocessing shared by tests and visualizations.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + + +@dataclass(frozen=True) +class DepthPreprocessingConfig: + """Configuration for a dataset-global linear depth representation.""" + + depth_min: float = 0.0 + depth_max: float = 5.0 + depth_invalid_value: float | None = 0.0 + depth_invalid_fill: str = "far" + + def __post_init__(self) -> None: + if not np.isfinite(self.depth_min) or not np.isfinite(self.depth_max): + raise ValueError("depth_min and depth_max must be finite") + if self.depth_max <= self.depth_min: + raise ValueError("depth_max must be greater than depth_min") + if self.depth_invalid_fill not in {"near", "far"}: + raise ValueError("depth_invalid_fill must be 'near' or 'far'") + + +@dataclass(frozen=True) +class DepthGrayscale: + """Preprocessed depth tensors in the dataset and Wan-VAE layouts.""" + + grayscale: np.ndarray + vae_video: np.ndarray + valid_mask: np.ndarray + + +def preprocess_depth_grayscale( + depth: np.ndarray, + *, + config: DepthPreprocessingConfig = DepthPreprocessingConfig(), + input_unit: str = "millimeter", +) -> DepthGrayscale: + """Convert depth to fixed-range grayscale without per-frame normalization. + + Args: + depth: ``[T,H,W]`` or ``[T,1,H,W]`` depth values. + config: Dataset-global metric range and invalid-value behavior. + input_unit: ``millimeter``, ``meter``, ``normalized``, or ``uint8``. + + Returns: + ``grayscale`` is float32 ``[T,1,H,W]`` in ``[0,1]``. + ``vae_video`` is float32 ``[T,3,H,W]`` in ``[-1,1]``. + ``valid_mask`` is bool ``[T,1,H,W]``. + """ + + raw = np.asarray(depth) + if raw.ndim == 4: + if raw.shape[1] != 1: + raise ValueError(f"Expected a singleton depth channel, got {raw.shape}") + raw = raw[:, 0] + if raw.ndim != 3: + raise ValueError(f"Expected depth shape [T,H,W] or [T,1,H,W], got {raw.shape}") + + values = raw.astype(np.float32, copy=False) + valid = np.isfinite(values) + if config.depth_invalid_value is not None: + valid &= values != np.float32(config.depth_invalid_value) + + if input_unit == "millimeter": + metric = values / np.float32(1000.0) + normalized = (metric - config.depth_min) / (config.depth_max - config.depth_min) + elif input_unit == "meter": + normalized = (values - config.depth_min) / (config.depth_max - config.depth_min) + elif input_unit == "normalized": + normalized = values + elif input_unit == "uint8": + normalized = values / np.float32(255.0) + else: + raise ValueError(f"Unsupported input_unit: {input_unit}") + + normalized = np.clip(normalized, 0.0, 1.0) + invalid_fill = np.float32(1.0 if config.depth_invalid_fill == "far" else 0.0) + normalized = np.where(valid, normalized, invalid_fill).astype(np.float32, copy=False) + + grayscale = normalized[:, None] + vae_video = np.repeat(grayscale, 3, axis=1) * np.float32(2.0) - np.float32(1.0) + return DepthGrayscale( + grayscale=grayscale, + vae_video=vae_video.astype(np.float32, copy=False), + valid_mask=valid[:, None], + ) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/evaluate_spacedreamer_validation.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/evaluate_spacedreamer_validation.py new file mode 100644 index 0000000000000000000000000000000000000000..153eb10903da1bce8ef4b07e4277f69e5a8b6044 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/evaluate_spacedreamer_validation.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Run the audited SpaceDreamer absolute-action validation evaluator. + +The stop-gradient and no-stop-gradient experiments must use exactly the same +evaluation implementation. Keep a single audited implementation in the +baseline sibling and execute it with this repository first on ``sys.path`` so +that model construction and checkpoint metadata come from the no-stop-gradient +sibling. A pinned hash prevents silent evaluator drift between experiments. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +import runpy +import sys + + +ROOT = Path(__file__).resolve().parent +REFERENCE_EVALUATOR = ( + ROOT.parent + / "model_wan_rgb_depth_point_cloud_action_mot_bidirectional_stopgrad_first_frame" + / "evaluate_spacedreamer_validation.py" +) +EXPECTED_SHA256 = "79ed3b59b8242b4d0a8ca85db24c34a4d52d7fe0a8b5d440932dbb3dfbe0b4a6" + + +def validated_reference_evaluator() -> Path: + """Return the shared evaluator only when its exact audited bytes match.""" + + if not REFERENCE_EVALUATOR.is_file(): + raise FileNotFoundError( + "The audited baseline SpaceDreamer evaluator is missing: " + f"{REFERENCE_EVALUATOR}" + ) + digest = hashlib.sha256(REFERENCE_EVALUATOR.read_bytes()).hexdigest() + if digest != EXPECTED_SHA256: + raise RuntimeError( + "The shared SpaceDreamer evaluator changed after this no-stop-gradient " + "wrapper was reviewed: " + f"expected_sha256={EXPECTED_SHA256} actual_sha256={digest}. " + "Review the evaluator diff and update both experiments deliberately." + ) + return REFERENCE_EVALUATOR + + +def main() -> None: + evaluator = validated_reference_evaluator() + # The shared script must import this sibling's model factory and runtime + # architecture assertion; otherwise a no-stop-gradient checkpoint would be + # interpreted using the baseline stop-gradient architecture. + root = str(ROOT) + if root in sys.path: + sys.path.remove(root) + sys.path.insert(0, root) + runpy.run_path(str(evaluator), run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/finetune_galexea_rgb_action.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/finetune_galexea_rgb_action.py new file mode 100644 index 0000000000000000000000000000000000000000..afb5213970b0dd3cb860a82cb85e9ff6462cf7a6 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/finetune_galexea_rgb_action.py @@ -0,0 +1,722 @@ +#!/usr/bin/env python3 +"""Fine-tune the RGB/depth/point Action-MoT on Galexea RGB + bimanual actions. + +The Galexea bags do not contain head depth. The adapter therefore supervises +only RGB and action, while zero depth/point latents keep the pretrained joint +144-channel interface intact. Depth/point losses and their dedicated +projection/adapter gradients are disabled. ``--train-mode action-only`` also +freezes the complete video expert. +""" + +from __future__ import annotations + +import argparse +from copy import deepcopy +import hashlib +import json +from pathlib import Path +from typing import Any + +from accelerate import Accelerator +from accelerate.utils import set_seed +import torch +from torch.utils.data import DataLoader, Dataset +import yaml + +from galexea_mcap import ( + GalexeaClipConfig, + GalexeaTopics, + discover_episodes, + extract_clip, + scan_episode, + select_clip_image_indices, +) +from joint_rgb_depth_point_action_diffusion import joint_flow_matching_loss +from online_egoverse_action import OnlineActionNormalizer, OnlineCameraNormalizer +from rgb_depth_point_action_model_factory import build_joint_action_mot +from train_joint_rgb_depth import ( + WanContinuousFlowMatchScheduler, + append_jsonl, + build_lr_scheduler, + dtype_from_name, + load_text_encoder_and_tokenizer, + load_vae, +) +from train_joint_rgb_depth_point import ( + FixedSeedPermutationSampler, + collate_rgb_depth_point_batch, + configure_trainable_parameters as configure_video_trainable_parameters, +) +from train_joint_rgb_depth_point_action import ( + enable_fp32_master_weights, + export_action_mot, +) + +TRAIN_MODES = {"rgb-action", "action-only"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--checkpoint", type=Path) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--train-mode", choices=sorted(TRAIN_MODES)) + parser.add_argument("--max-steps", type=int) + parser.add_argument("--prepare-cache-only", action="store_true") + parser.add_argument("--dataset-smoke-test", action="store_true") + parser.add_argument("--overwrite-cache", action="store_true") + return parser.parse_args() + + +def load_config(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("Config must be a YAML mapping") + return payload + + +def topics_from_config(config: dict[str, Any]) -> GalexeaTopics: + values = config["galexea"].get("topics", {}) + defaults = GalexeaTopics() + return GalexeaTopics( + **{ + name: values.get(name, getattr(defaults, name)) + for name in defaults.__dataclass_fields__ + } + ) + + +def clip_config_from_config(config: dict[str, Any]) -> GalexeaClipConfig: + data, galexea = config["data"], config["galexea"] + gripper = galexea.get("gripper", {}) + configured_fps = data.get("target_fps", "native") + target_fps = None if configured_fps == "native" else float(configured_fps) + return GalexeaClipConfig( + num_frames=int(data["num_frames"]), + target_fps=target_fps, + resolution=int(data["resolution"]), + clips_per_episode=int(data["clips_per_episode"]), + max_alignment_error_s=float(galexea["max_alignment_error_s"]), + gripper_min=float(gripper.get("min", 0.0)), + gripper_max=float(gripper.get("max", 100.0)), + gripper_invert=bool(gripper.get("invert", False)), + ) + + +def validate_config(config: dict[str, Any]) -> dict[str, Any]: + for section in ( + "experiment", + "paths", + "model", + "data", + "conditioning", + "diffusion", + "action", + "camera", + "galexea", + "cache", + "training", + ): + if section not in config: + raise KeyError(f"Missing config section {section}") + data, diffusion, training = config["data"], config["diffusion"], config["training"] + if int(data["num_frames"]) != 33: + raise ValueError( + "Native-FPS Galexea fine-tuning requires 17 * 2 - 1 = 33 frames" + ) + if int(data["resolution"]) != 256: + raise ValueError("Galexea fine-tuning is configured for 256x256 RGB") + target_fps = data.get("target_fps", "native") + if target_fps != "native" and float(target_fps) <= 0: + raise ValueError("data.target_fps must be native or a positive number") + if int(data["clips_per_episode"]) <= 0 or int(data["batch_size"]) <= 0: + raise ValueError("clips_per_episode and batch_size must be positive") + if config["conditioning"].get("fix_rgb_first_latent") is not True: + raise ValueError("RGB frame-zero conditioning must remain enabled") + if config["conditioning"].get("exclude_rgb_condition_from_loss") is not True: + raise ValueError("RGB frame zero must remain excluded from the prediction loss") + if float(diffusion.get("lambda_action", 0)) <= 0: + raise ValueError("diffusion.lambda_action must be positive") + mode = str(training.get("mode", "rgb-action")) + if mode not in TRAIN_MODES: + raise ValueError(f"training.mode must be one of {sorted(TRAIN_MODES)}") + if int(training["max_steps"]) <= 0 or float(training["action_lr"]) <= 0: + raise ValueError("training.max_steps and action_lr must be positive") + if mode == "rgb-action" and float(training["video_lr"]) <= 0: + raise ValueError("training.video_lr must be positive for rgb-action") + root = Path(config["paths"]["galexea_root"]) + checkpoint = Path(config["paths"]["checkpoint"]) + if not root.is_dir(): + raise FileNotFoundError(root) + if not checkpoint.is_file(): + raise FileNotFoundError(checkpoint) + cache_dir = Path(config["paths"]["cache_dir"]).resolve() + output_dir = Path(config["experiment"]["output_dir"]).resolve() + checkpoint = checkpoint.resolve() + if cache_dir == checkpoint.parent or checkpoint.is_relative_to(cache_dir): + raise ValueError( + "Galexea cache_dir must be isolated from the source checkpoint" + ) + if output_dir == checkpoint.parent or checkpoint.is_relative_to(output_dir): + raise ValueError("Galexea output_dir must not contain the source checkpoint") + if ( + cache_dir == output_dir + or cache_dir.is_relative_to(output_dir) + or output_dir.is_relative_to(cache_dir) + ): + raise ValueError("Galexea cache_dir and output_dir must be separate trees") + return config + + +def cache_fingerprint(config: dict[str, Any], episodes: list[Path]) -> str: + selected = { + "format_version": 1, + "episodes": [str(path) for path in episodes], + "data": { + key: config["data"][key] + for key in ("num_frames", "target_fps", "resolution", "clips_per_episode") + }, + "galexea": config["galexea"], + "prompt": config["data"]["prompt"], + "pretrained_model_dir": config["paths"]["pretrained_model_dir"], + } + return hashlib.sha256(json.dumps(selected, sort_keys=True).encode()).hexdigest() + + +def resolved_episodes(config: dict[str, Any]) -> list[Path]: + episodes = discover_episodes(Path(config["paths"]["galexea_root"])) + count = int(config["data"].get("num_episodes", len(episodes))) + if count <= 0 or count > len(episodes): + raise ValueError(f"data.num_episodes={count} outside [1,{len(episodes)}]") + return episodes[:count] + + +def dataset_smoke_test(config: dict[str, Any]) -> None: + topics, clip_config = topics_from_config(config), clip_config_from_config(config) + mcap = resolved_episodes(config)[0] + index = scan_episode(mcap, topics) + selected = select_clip_image_indices(index.image_time_ns, clip_config)[0] + sample = extract_clip(index, topics, clip_config, selected) + action = sample["action_raw"] + print( + json.dumps( + { + "status": "galexea_dataset_smoke_ok", + "episode": index.episode_id, + "rgb_shape": list(sample["rgb"].shape), + "action_shape": list(action.shape), + "camera_shape": list(sample["camera_raw"].shape), + "initial_state_shape": list(sample["initial_state"].shape), + "image_indices": selected.tolist(), + "action_translation_abs_max_m": float( + action[:, [0, 1, 2, 7, 8, 9]].abs().max() + ), + "action_rotation_abs_max_rad": float( + action[:, [3, 4, 5, 10, 11, 12]].abs().max() + ), + "grip_range": [ + float(action[:, [6, 13]].min()), + float(action[:, [6, 13]].max()), + ], + "effective_fps": float(sample["effective_fps"]), + }, + indent=2, + ), + flush=True, + ) + + +@torch.no_grad() +def prepare_cache( + config: dict[str, Any], + *, + device: torch.device, + overwrite: bool, +) -> None: + cache_dir = Path(config["paths"]["cache_dir"]) + episodes = resolved_episodes(config) + fingerprint = cache_fingerprint(config, episodes) + manifest_path = cache_dir / "manifest.json" + if manifest_path.is_file() and not overwrite: + existing = json.loads(manifest_path.read_text(encoding="utf-8")) + if existing.get("fingerprint") != fingerprint: + raise RuntimeError( + f"Galexea cache fingerprint mismatch: {manifest_path}; use a new cache_dir" + ) + print(f"[cache] reuse {manifest_path}", flush=True) + return + cache_dir.mkdir(parents=True, exist_ok=True) + dtype = dtype_from_name(str(config["model"]["dtype"])) + model_dir = Path(config["paths"]["pretrained_model_dir"]) + text_encoder, tokenizer = load_text_encoder_and_tokenizer( + model_dir, + dtype, + device, + int(config["cache"].get("tokenizer_max_length", 512)), + ) + prompt = str(config["data"]["prompt"]) + ids, mask = tokenizer([prompt], return_mask=True, add_special_tokens=True) + context = text_encoder(ids.to(device), mask.to(device=device, dtype=torch.bool))[ + 0 + ].cpu() + context_mask = mask[0].to(dtype=torch.bool).cpu() + torch.save( + { + "entries": [ + {"prompt": prompt, "context": context, "context_mask": context_mask} + ] + }, + cache_dir / "prompt_contexts.pt", + ) + text_encoder.to("cpu") + del text_encoder, tokenizer, ids, mask, context, context_mask + if device.type == "cuda": + torch.cuda.empty_cache() + vae = load_vae(model_dir, dtype, device) + topics, clip_config = topics_from_config(config), clip_config_from_config(config) + entries: list[dict[str, Any]] = [] + sample_number = 0 + expected_t = (int(config["data"]["num_frames"]) - 1) // 4 + 1 + expected_hw = int(config["data"]["resolution"]) // 16 + expected_shape = (1, 48, expected_t, expected_hw, expected_hw) + for episode_number, mcap in enumerate(episodes): + print( + f"[cache] scan {episode_number + 1}/{len(episodes)} {mcap.parent.name}", + flush=True, + ) + index = scan_episode(mcap, topics) + windows = select_clip_image_indices(index.image_time_ns, clip_config) + for clip_number, image_indices in enumerate(windows): + sample = extract_clip(index, topics, clip_config, image_indices) + rgb = sample.pop("rgb").unsqueeze(0).to(device=device, dtype=dtype) + z_rgb = vae.encode(rgb, device=device) + if tuple(z_rgb.shape) != expected_shape: + raise RuntimeError( + f"Unexpected Galexea latent shape {tuple(z_rgb.shape)} != {expected_shape}" + ) + sample_id = f"{index.episode_id}_clip_{clip_number:02d}" + filename = f"{sample_number:04d}_{sample_id}.pt" + torch.save( + { + "sample_id": sample_id, + "z_rgb": z_rgb[0].cpu(), + # Galexea head depth is absent. Zero placeholders preserve the + # pretrained 144-channel interface and receive no loss/gradient. + "z_depth": torch.zeros_like(z_rgb[0].cpu()), + "z_point": torch.zeros_like(z_rgb[0].cpu()), + **sample, + }, + cache_dir / filename, + ) + entries.append( + { + "sample_id": sample_id, + "file": filename, + "prompt_index": 0, + "episode_index": episode_number, + "clip_index": clip_number, + } + ) + sample_number += 1 + print(f"[cache] wrote {sample_id}", flush=True) + manifest_path.write_text( + json.dumps( + { + "format_version": 1, + "fingerprint": fingerprint, + "source": "galexea_ros2_mcap_0728", + "modalities": ["rgb", "action", "camera"], + "missing_modalities": ["depth", "world_point"], + "samples": entries, + }, + indent=2, + ), + encoding="utf-8", + ) + vae.to("cpu") + del vae + if device.type == "cuda": + torch.cuda.empty_cache() + + +class GalexeaLatentActionDataset(Dataset): + def __init__( + self, + cache_dir: Path, + expected_fingerprint: str, + action_normalizer: OnlineActionNormalizer, + camera_normalizer: OnlineCameraNormalizer, + ) -> None: + manifest = json.loads((cache_dir / "manifest.json").read_text(encoding="utf-8")) + if manifest.get("fingerprint") != expected_fingerprint: + raise RuntimeError("Galexea latent cache does not match config") + if not bool(action_normalizer.fitted) or not bool(camera_normalizer.fitted): + raise RuntimeError( + "Checkpoint action/camera normalizers must be fitted; Galexea must not refit them" + ) + self.cache_dir = cache_dir + self.entries = list(manifest["samples"]) + prompt = torch.load( + cache_dir / "prompt_contexts.pt", map_location="cpu", weights_only=True + )["entries"][0] + self.context = prompt["context"] + self.context_mask = prompt["context_mask"].to(dtype=torch.bool) + self.prompt = prompt["prompt"] + self.action_mean = action_normalizer.mean.detach().cpu() + self.action_std = action_normalizer.std.detach().cpu() + self.action_clip = action_normalizer.clip + self.camera_mean = camera_normalizer.mean.detach().cpu() + self.camera_std = camera_normalizer.std.detach().cpu() + self.camera_clip = camera_normalizer.clip + + def __len__(self) -> int: + return len(self.entries) + + def __getitem__(self, index: int) -> dict[str, Any]: + payload = torch.load( + self.cache_dir / self.entries[index]["file"], + map_location="cpu", + weights_only=True, + ) + action = (payload["action_raw"] - self.action_mean) / self.action_std + camera = (payload["camera_raw"] - self.camera_mean) / self.camera_std + if self.action_clip is not None: + action = action.clamp(-self.action_clip, self.action_clip) + if self.camera_clip is not None: + camera = camera.clamp(-self.camera_clip, self.camera_clip) + return { + "sample_id": payload["sample_id"], + "z_rgb": payload["z_rgb"], + "z_depth": payload["z_depth"], + "z_point": payload["z_point"], + "action": action, + "action_is_pad": payload["action_is_pad"], + "camera": camera, + "camera_is_pad": payload["camera_is_pad"], + "initial_state": payload["initial_state"], + "left_initial_pose_reference": payload["left_initial_pose_reference"], + "right_initial_pose_reference": payload["right_initial_pose_reference"], + "context": self.context, + "context_mask": self.context_mask, + "prompt": self.prompt, + } + + +class RgbProjectionGradientMask: + """Update only RGB input/output projection slices, preserving depth/point.""" + + def __init__(self, video_expert: torch.nn.Module) -> None: + rgb = int(video_expert.rgb_latent_channels) + joint = ( + rgb + + int(video_expert.depth_latent_channels) + + int(video_expert.point_latent_channels) + ) + patch_mask = torch.zeros_like(video_expert.patch_embedding.weight) + patch_mask[:, :rgb].fill_(1) + patch_volume = int(torch.tensor(video_expert.patch_size).prod()) + head_weight_mask = torch.zeros_like(video_expert.head.head.weight) + head_weight_mask.view(patch_volume, joint, video_expert.head.head.in_features)[ + :, :rgb + ].fill_(1) + head_bias_mask = torch.zeros_like(video_expert.head.head.bias) + head_bias_mask.view(patch_volume, joint)[:, :rgb].fill_(1) + self.handles = [ + video_expert.patch_embedding.weight.register_hook( + lambda grad: grad * patch_mask.to(grad) + ), + video_expert.head.head.weight.register_hook( + lambda grad: grad * head_weight_mask.to(grad) + ), + video_expert.head.head.bias.register_hook( + lambda grad: grad * head_bias_mask.to(grad) + ), + ] + + def close(self) -> None: + for handle in self.handles: + handle.remove() + + +def configure_trainable( + model: torch.nn.Module, mode: str +) -> RgbProjectionGradientMask | None: + model.requires_grad_(False) + mask = None + if mode == "rgb-action": + configure_video_trainable_parameters(model.video_expert, "full_dit") + model.video_expert.point_first_frame_timestep_adapter.requires_grad_(False) + mask = RgbProjectionGradientMask(model.video_expert) + model.action_expert.requires_grad_(True) + model.action_state_encoder.requires_grad_(True) + model.structured_type_embedding.requires_grad_(True) + # Galexea camera is provided as context but is not a requested target. + model.camera_encoder.requires_grad_(False) + model.camera_head.requires_grad_(False) + return mask + + +def build_optimizer( + model: torch.nn.Module, config: dict[str, Any], mode: str +) -> torch.optim.Optimizer: + groups = [] + if mode == "rgb-action": + groups.append( + { + "name": "rgb_video_expert", + "params": [ + p for p in model.video_expert.parameters() if p.requires_grad + ], + "lr": float(config["training"]["video_lr"]), + } + ) + action_parameters = [ + p + for module in (model.action_expert, model.action_state_encoder) + for p in module.parameters() + if p.requires_grad + ] + [model.structured_type_embedding] + groups.append( + { + "name": "action_expert", + "params": action_parameters, + "lr": float(config["training"]["action_lr"]), + } + ) + return torch.optim.AdamW( + groups, + betas=(0.9, 0.95), + eps=1.0e-8, + weight_decay=float(config["training"].get("weight_decay", 0.0)), + ) + + +def compute_loss( + model: torch.nn.Module, + batch: dict[str, Any], + video_scheduler: WanContinuousFlowMatchScheduler, + action_scheduler: WanContinuousFlowMatchScheduler, + config: dict[str, Any], + mode: str, + device: torch.device, + dtype: torch.dtype, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + moved = { + key: batch[key].to(device=device, dtype=dtype, non_blocking=True) + for key in ( + "z_rgb", + "z_depth", + "z_point", + "action", + "camera", + "initial_state", + "context", + ) + } + context_mask = batch["context_mask"].to(device=device, dtype=torch.bool) + action_is_pad = batch["action_is_pad"].to(device=device, dtype=torch.bool) + camera_is_pad = batch["camera_is_pad"].to(device=device, dtype=torch.bool) + conditioning, diffusion = config["conditioning"], config["diffusion"] + return joint_flow_matching_loss( + model, + video_scheduler, + action_scheduler, + z_rgb=moved["z_rgb"], + z_depth=moved["z_depth"], + z_point=moved["z_point"], + action=moved["action"], + camera=moved["camera"], + initial_state=moved["initial_state"], + context=moved["context"], + context_mask=context_mask, + action_is_pad=action_is_pad, + camera_is_pad=camera_is_pad, + fix_rgb_first_latent=True, + fix_depth_first_latent=False, + exclude_rgb_condition_from_loss=True, + exclude_depth_condition_from_loss=True, + lambda_rgb=float(diffusion["lambda_rgb"]) if mode == "rgb-action" else 0.0, + lambda_depth=0.0, + lambda_point=0.0, + lambda_action=float(diffusion["lambda_action"]), + lambda_camera_translation=0.0, + lambda_camera_rotation=0.0, + ) + + +def train(config: dict[str, Any], mode: str, output_dir: Path) -> None: + accelerator = Accelerator( + gradient_accumulation_steps=int( + config["training"].get("gradient_accumulation_steps", 1) + ), + mixed_precision="bf16" if config["model"]["dtype"] == "bfloat16" else "no", + ) + set_seed(int(config["experiment"]["seed"]), device_specific=True) + if accelerator.is_main_process: + output_dir.mkdir(parents=True, exist_ok=True) + resolved = dict(config) + resolved["training"] = {**config["training"], "mode": mode} + (output_dir / "resolved_config.yaml").write_text( + yaml.safe_dump(resolved, sort_keys=False), encoding="utf-8" + ) + accelerator.wait_for_everyone() + checkpoint = Path(config["paths"]["checkpoint"]) + # Keep the shared production factory and original Egoverse pipeline untouched. + # The source label is provenance-only and is absent from checkpoint metadata. + architecture_config = deepcopy(config) + architecture_config["action"]["source"] = "online_egoverse_pose_keypoints" + model = build_joint_action_mot( + architecture_config, checkpoint, load_pretrained=False + ) + if bool(config["training"].get("fp32_master_weights", True)): + enable_fp32_master_weights(model) + episodes = resolved_episodes(config) + dataset = GalexeaLatentActionDataset( + Path(config["paths"]["cache_dir"]), + cache_fingerprint(config, episodes), + model.action_normalizer, + model.camera_normalizer, + ) + dataloader = DataLoader( + dataset, + batch_size=int(config["data"]["batch_size"]), + sampler=FixedSeedPermutationSampler( + dataset, int(config["data"]["sample_order_seed"]) + ), + num_workers=int(config["data"].get("num_workers", 0)), + pin_memory=True, + drop_last=False, + collate_fn=collate_rgb_depth_point_batch, + ) + projection_mask = configure_trainable(model, mode) + optimizer = build_optimizer(model, config, mode) + max_steps = int(config["training"]["max_steps"]) + scheduler = build_lr_scheduler( + optimizer, + max_steps, + int(config["training"].get("warmup_steps", 0)), + schedule_type=str(config["training"].get("lr_schedule", "cosine")), + ) + model, optimizer, dataloader, scheduler = accelerator.prepare( + model, optimizer, dataloader, scheduler + ) + video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=float(config["diffusion"]["train_shift"]), + ) + action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["action_num_train_timesteps"]), + shift=float(config["diffusion"]["action_train_shift"]), + ) + dtype = dtype_from_name(str(config["model"]["dtype"])) + log_path = output_dir / "metrics.jsonl" + global_step = 0 + optimizer.zero_grad(set_to_none=True) + model.train() + while global_step < max_steps: + for batch in dataloader: + if global_step >= max_steps: + break + with accelerator.accumulate(model): + loss, metrics = compute_loss( + model, + batch, + video_scheduler, + action_scheduler, + config, + mode, + accelerator.device, + dtype, + ) + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_( + [p for p in model.parameters() if p.requires_grad], + float(config["training"].get("max_grad_norm", 1.0)), + ) + optimizer.step() + scheduler.step() + optimizer.zero_grad(set_to_none=True) + if not accelerator.sync_gradients: + continue + global_step += 1 + if ( + global_step == 1 + or global_step % int(config["training"]["log_every"]) == 0 + ): + reduced = { + key: accelerator.reduce(value.float(), reduction="mean") + for key, value in metrics.items() + if key.startswith("loss_") + } + record = { + "train_mode": mode, + "global_step": global_step, + "loss_total": float( + accelerator.reduce( + loss.detach().float(), reduction="mean" + ).item() + ), + **{key: float(value.item()) for key, value in reduced.items()}, + "learning_rates": [ + float(group["lr"]) for group in optimizer.param_groups + ], + } + if accelerator.is_main_process: + append_jsonl(log_path, record) + print(json.dumps(record, sort_keys=True), flush=True) + if global_step % int(config["training"]["save_every"]) == 0: + export_action_mot( + model, + accelerator, + output_dir + / "checkpoints" + / f"joint_action_mot_galexea_{mode}_{global_step:06d}.pt", + ) + export_action_mot( + model, + accelerator, + output_dir + / "checkpoints" + / f"joint_action_mot_galexea_{mode}_{global_step:06d}.pt", + ) + if projection_mask is not None: + projection_mask.close() + accelerator.end_training() + + +def main() -> None: + args = parse_args() + config = load_config(args.config) + if args.checkpoint is not None: + config["paths"]["checkpoint"] = str(args.checkpoint) + if args.output_dir is not None: + config["experiment"]["output_dir"] = str(args.output_dir) + if args.train_mode is not None: + config["training"]["mode"] = args.train_mode + if args.max_steps is not None: + config["training"]["max_steps"] = args.max_steps + config = validate_config(config) + if args.dataset_smoke_test: + dataset_smoke_test(config) + return + accelerator = Accelerator() + if accelerator.is_main_process: + prepare_cache( + config, + device=torch.device(str(config["cache"].get("device", accelerator.device))), + overwrite=args.overwrite_cache, + ) + accelerator.wait_for_everyone() + accelerator.end_training() + if args.prepare_cache_only: + return + train( + config, + str(config["training"]["mode"]), + Path(config["experiment"]["output_dir"]), + ) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/galexea_mcap.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/galexea_mcap.py new file mode 100644 index 0000000000000000000000000000000000000000..79268623e973bd810982291b633dc23efc534125 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/galexea_mcap.py @@ -0,0 +1,463 @@ +"""Galexea ROS2 MCAP reader aligned to the Egoverse Action-MoT contract. + +The adapter reads head-left RGB, bimanual end-effector poses, grippers, the +floating torso pose, and the static torso-to-camera calibration. It samples 33 consecutive acquisition frames using their native header timestamps +and produces the same raw targets used by the original training pipeline: + +* RGB video in ``[-1, 1]`` with shape ``[3,T,H,W]``; +* 14D actions: left SE(3) body-log + grip, right SE(3) body-log + grip; +* 6D camera poses relative to the first camera frame; +* clean 14D initial state in the first-camera reference frame. + +No derived files are written by this module. Cache creation is handled by the +fine-tuning entrypoint. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import cv2 +import numpy as np +import torch +from rosbags.highlevel import AnyReader + +from online_egoverse_action import ACTION_DIM, CAMERA_DIM, _rigid_inverse, se3_log + + +@dataclass(frozen=True) +class GalexeaTopics: + rgb: str = "/hdas/camera_head/left_raw/image_raw_color/compressed" + left_ee: str = "/motion_control/pose_ee_arm_left" + right_ee: str = "/motion_control/pose_ee_arm_right" + left_gripper: str = "/hdas/feedback_gripper_left" + right_gripper: str = "/hdas/feedback_gripper_right" + floating_base: str = "/motion_control/pose_floating_base" + tf_static: str = "/tf_static" + camera_parent_frame: str = "torso_link3" + camera_child_frame: str = "camera_head_left" + + +@dataclass(frozen=True) +class GalexeaClipConfig: + num_frames: int = 33 + # None means native acquisition timing: use consecutive RGB messages and + # preserve their header timestamps instead of resampling to a fixed FPS. + target_fps: float | None = None + resolution: int = 256 + clips_per_episode: int = 2 + max_alignment_error_s: float = 0.05 + gripper_min: float = 0.0 + gripper_max: float = 100.0 + gripper_invert: bool = False + + +@dataclass +class TimedArray: + time_ns: np.ndarray + value: np.ndarray + + +@dataclass +class GalexeaEpisodeIndex: + episode_id: str + mcap_path: Path + image_time_ns: np.ndarray + left_ee: TimedArray + right_ee: TimedArray + left_gripper: TimedArray + right_gripper: TimedArray + floating_base: TimedArray + torso_from_camera: np.ndarray + + +def _stamp_ns(message: Any, fallback_ns: int) -> int: + header = getattr(message, "header", None) + stamp = getattr(header, "stamp", None) + if stamp is None: + return int(fallback_ns) + value = int(stamp.sec) * 1_000_000_000 + int(stamp.nanosec) + return value if value > 0 else int(fallback_ns) + + +def quaternion_xyzw_to_matrix(quaternion: np.ndarray) -> np.ndarray: + quaternion = np.asarray(quaternion, dtype=np.float64) + if quaternion.shape != (4,): + raise ValueError(f"Expected quaternion [x,y,z,w], got {quaternion.shape}") + norm = float(np.linalg.norm(quaternion)) + if not np.isfinite(norm) or norm < 1.0e-8: + raise ValueError("Quaternion must be finite and nonzero") + x, y, z, w = quaternion / norm + return np.array( + [ + [1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w)], + [2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w)], + [2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)], + ], + dtype=np.float64, + ) + + +def pose_to_matrix(position: np.ndarray, quaternion_xyzw: np.ndarray) -> np.ndarray: + output = np.eye(4, dtype=np.float64) + output[:3, :3] = quaternion_xyzw_to_matrix(quaternion_xyzw) + output[:3, 3] = np.asarray(position, dtype=np.float64) + if not np.isfinite(output).all(): + raise ValueError("Pose contains non-finite values") + return output + + +def _pose_message_array(message: Any, expected_frame: str | None = None) -> np.ndarray: + if expected_frame is not None and message.header.frame_id != expected_frame: + raise ValueError( + f"Expected pose frame {expected_frame!r}, got {message.header.frame_id!r}" + ) + p, q = message.pose.position, message.pose.orientation + return np.array([p.x, p.y, p.z, q.x, q.y, q.z, q.w], dtype=np.float64) + + +def _transform_message_matrix(message: Any) -> np.ndarray: + p, q = message.transform.translation, message.transform.rotation + return pose_to_matrix(np.array([p.x, p.y, p.z]), np.array([q.x, q.y, q.z, q.w])) + + +def _sorted_timed(times: list[int], values: list[np.ndarray], name: str) -> TimedArray: + if not times: + raise RuntimeError(f"Galexea topic {name} contains no messages") + order = np.argsort(np.asarray(times, dtype=np.int64), kind="stable") + return TimedArray( + time_ns=np.asarray(times, dtype=np.int64)[order], + value=np.asarray(values)[order], + ) + + +def discover_episodes(root: Path) -> list[Path]: + episodes = [] + for directory in sorted(root.glob("*_RAW")): + if not directory.is_dir(): + continue + mcap = directory / f"{directory.name}.mcap" + if mcap.is_file(): + episodes.append(mcap) + if not episodes: + raise FileNotFoundError(f"No *_RAW/.mcap episodes found under {root}") + return episodes + + +def scan_episode(mcap_path: Path, topics: GalexeaTopics) -> GalexeaEpisodeIndex: + wanted = { + topics.rgb, + topics.left_ee, + topics.right_ee, + topics.left_gripper, + topics.right_gripper, + topics.floating_base, + topics.tf_static, + } + image_times: list[int] = [] + times: dict[str, list[int]] = {name: [] for name in wanted if name != topics.rgb} + values: dict[str, list[np.ndarray]] = { + name: [] for name in wanted if name not in {topics.rgb, topics.tf_static} + } + torso_from_camera: np.ndarray | None = None + with AnyReader([mcap_path]) as reader: + connections = [c for c in reader.connections if c.topic in wanted] + present = {c.topic for c in connections} + missing = wanted - present + if missing: + raise RuntimeError( + f"Missing Galexea topics in {mcap_path}: {sorted(missing)}" + ) + for connection, bag_time, raw in reader.messages(connections=connections): + message = reader.deserialize(raw, connection.msgtype) + topic = connection.topic + if topic == topics.rgb: + image_times.append(_stamp_ns(message, bag_time)) + elif topic == topics.tf_static: + for transform in message.transforms: + if ( + transform.header.frame_id == topics.camera_parent_frame + and transform.child_frame_id == topics.camera_child_frame + ): + candidate = _transform_message_matrix(transform) + if torso_from_camera is not None and not np.allclose( + torso_from_camera, candidate, rtol=0, atol=1.0e-7 + ): + raise RuntimeError("Inconsistent torso-to-camera tf_static") + torso_from_camera = candidate + elif topic in {topics.left_ee, topics.right_ee, topics.floating_base}: + times[topic].append(_stamp_ns(message, bag_time)) + values[topic].append(_pose_message_array(message, "base_link")) + elif topic in {topics.left_gripper, topics.right_gripper}: + if len(message.position) < 1: + continue + times[topic].append(_stamp_ns(message, bag_time)) + values[topic].append(np.asarray(message.position[0], dtype=np.float64)) + if torso_from_camera is None: + raise RuntimeError( + f"Missing tf_static {topics.camera_parent_frame}->{topics.camera_child_frame}" + ) + image_time_ns = np.asarray(image_times, dtype=np.int64) + if len(image_time_ns) < 2 or np.any(np.diff(image_time_ns) <= 0): + raise RuntimeError("RGB timestamps must be strictly increasing") + return GalexeaEpisodeIndex( + episode_id=mcap_path.parent.name, + mcap_path=mcap_path, + image_time_ns=image_time_ns, + left_ee=_sorted_timed( + times[topics.left_ee], values[topics.left_ee], topics.left_ee + ), + right_ee=_sorted_timed( + times[topics.right_ee], values[topics.right_ee], topics.right_ee + ), + left_gripper=_sorted_timed( + times[topics.left_gripper], values[topics.left_gripper], topics.left_gripper + ), + right_gripper=_sorted_timed( + times[topics.right_gripper], + values[topics.right_gripper], + topics.right_gripper, + ), + floating_base=_sorted_timed( + times[topics.floating_base], + values[topics.floating_base], + topics.floating_base, + ), + torso_from_camera=torso_from_camera, + ) + + +def nearest_indices( + source_time_ns: np.ndarray, + target_time_ns: np.ndarray, + max_error_s: float, + *, + name: str, +) -> np.ndarray: + source = np.asarray(source_time_ns, dtype=np.int64) + target = np.asarray(target_time_ns, dtype=np.int64) + right = np.searchsorted(source, target, side="left").clip(0, len(source) - 1) + left = (right - 1).clip(0, len(source) - 1) + choose_left = np.abs(source[left] - target) <= np.abs(source[right] - target) + indices = np.where(choose_left, left, right) + error_s = np.abs(source[indices] - target) / 1.0e9 + if np.any(error_s > float(max_error_s)): + worst = float(error_s.max()) + raise RuntimeError( + f"{name} alignment error {worst:.6f}s exceeds {max_error_s}s" + ) + return indices.astype(np.int64) + + +def select_clip_image_indices( + image_time_ns: np.ndarray, + config: GalexeaClipConfig, +) -> list[np.ndarray]: + if config.num_frames < 2 or config.clips_per_episode <= 0: + raise ValueError("Invalid Galexea clip geometry") + if config.target_fps is None: + span = config.num_frames + available_start = len(image_time_ns) - span + if available_start < 0: + raise RuntimeError("Episode is shorter than one requested native-FPS clip") + starts = [ + (clip_index + 1) * available_start // (config.clips_per_episode + 1) + for clip_index in range(config.clips_per_episode) + ] + if any( + right - left < span for left, right in zip(starts, starts[1:], strict=False) + ): + raise RuntimeError("Requested native-FPS Galexea clips overlap") + return [ + start + np.arange(config.num_frames, dtype=np.int64) for start in starts + ] + if config.target_fps <= 0: + raise ValueError("target_fps must be positive or None for native timing") + span_ns = int(round((config.num_frames - 1) * 1.0e9 / config.target_fps)) + available_ns = int(image_time_ns[-1] - image_time_ns[0] - span_ns) + if available_ns < 0: + raise RuntimeError("Episode is shorter than one requested clip") + outputs: list[np.ndarray] = [] + used: set[int] = set() + for clip_index in range(config.clips_per_episode): + start_ns = int(image_time_ns[0]) + ( + (clip_index + 1) * available_ns // (config.clips_per_episode + 1) + ) + target = start_ns + np.rint( + np.arange(config.num_frames) * 1.0e9 / config.target_fps + ).astype(np.int64) + indices = nearest_indices( + image_time_ns, + target, + max_error_s=0.5 / config.target_fps, + name="RGB", + ) + if np.any(np.diff(indices) <= 0): + raise RuntimeError("Selected RGB frames are not strictly increasing") + overlap = used.intersection(indices.tolist()) + if overlap: + raise RuntimeError( + f"Uniform Galexea clips overlap at frames {sorted(overlap)}" + ) + used.update(indices.tolist()) + outputs.append(indices) + return outputs + + +def _decode_selected_images( + index: GalexeaEpisodeIndex, + topics: GalexeaTopics, + selected_indices: np.ndarray, + resolution: int, +) -> torch.Tensor: + selected = { + int(value): slot for slot, value in enumerate(selected_indices.tolist()) + } + frames: list[np.ndarray | None] = [None] * len(selected_indices) + ordinal = 0 + with AnyReader([index.mcap_path]) as reader: + connections = [c for c in reader.connections if c.topic == topics.rgb] + for connection, _, raw in reader.messages(connections=connections): + if ordinal in selected: + message = reader.deserialize(raw, connection.msgtype) + frame = cv2.imdecode( + np.asarray(message.data, dtype=np.uint8), cv2.IMREAD_COLOR + ) + if frame is None: + raise RuntimeError(f"Failed to decode RGB frame {ordinal}") + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + height, width = frame.shape[:2] + crop = min(height, width) + top, left = (height - crop) // 2, (width - crop) // 2 + frame = frame[top : top + crop, left : left + crop] + frames[selected[ordinal]] = cv2.resize( + frame, (resolution, resolution), interpolation=cv2.INTER_AREA + ) + ordinal += 1 + if any(frame is None for frame in frames): + raise RuntimeError("Not all selected RGB frames were decoded") + rgb = np.stack(frames).astype(np.float32) / np.float32(127.5) - np.float32(1.0) + return torch.from_numpy(rgb).permute(3, 0, 1, 2).contiguous() + + +def _matrices_from_pose_rows(rows: np.ndarray) -> np.ndarray: + return np.stack([pose_to_matrix(row[:3], row[3:7]) for row in rows]) + + +def build_action_targets( + *, + left_base: np.ndarray, + right_base: np.ndarray, + base_from_torso: np.ndarray, + torso_from_camera: np.ndarray, + left_grip: np.ndarray, + right_grip: np.ndarray, +) -> dict[str, torch.Tensor]: + frame_count = int(left_base.shape[0]) + expected_pose_shape = (frame_count, 4, 4) + for name, value in ( + ("right_base", right_base), + ("base_from_torso", base_from_torso), + ): + if tuple(value.shape) != expected_pose_shape: + raise ValueError(f"{name} must have shape {expected_pose_shape}") + if left_grip.shape != (frame_count,) or right_grip.shape != (frame_count,): + raise ValueError("Grip arrays must be [T]") + base_from_camera = base_from_torso @ torso_from_camera[None] + reference_from_base = _rigid_inverse(base_from_camera[0]) + left_reference = reference_from_base[None] @ left_base + right_reference = reference_from_base[None] @ right_base + camera_reference = reference_from_base[None] @ base_from_camera + action = np.zeros((frame_count - 1, ACTION_DIM), dtype=np.float32) + camera = np.zeros((frame_count - 1, CAMERA_DIM), dtype=np.float32) + for i in range(frame_count - 1): + action[i, 0:6] = se3_log( + _rigid_inverse(left_reference[i]) @ left_reference[i + 1] + ) + action[i, 6] = left_grip[i + 1] + action[i, 7:13] = se3_log( + _rigid_inverse(right_reference[i]) @ right_reference[i + 1] + ) + action[i, 13] = right_grip[i + 1] + camera[i, :3] = camera_reference[i + 1, :3, 3] + camera[i, 3:] = se3_log(camera_reference[i + 1])[3:] + initial_state = np.concatenate( + [ + se3_log(left_reference[0]), + [left_grip[0]], + se3_log(right_reference[0]), + [right_grip[0]], + ] + ).astype(np.float32) + if not all(np.isfinite(value).all() for value in (action, camera, initial_state)): + raise ValueError("Galexea action/camera targets contain non-finite values") + return { + "action_raw": torch.from_numpy(action), + "action_is_pad": torch.zeros(frame_count - 1, dtype=torch.bool), + "camera_raw": torch.from_numpy(camera), + "camera_is_pad": torch.zeros(frame_count - 1, dtype=torch.bool), + "initial_state": torch.from_numpy(initial_state), + "left_initial_pose_reference": torch.from_numpy( + left_reference[0].astype(np.float32) + ), + "right_initial_pose_reference": torch.from_numpy( + right_reference[0].astype(np.float32) + ), + } + + +def extract_clip( + index: GalexeaEpisodeIndex, + topics: GalexeaTopics, + config: GalexeaClipConfig, + image_indices: np.ndarray, +) -> dict[str, Any]: + target_times = index.image_time_ns[image_indices] + + def aligned(stream: TimedArray, name: str) -> np.ndarray: + rows = nearest_indices( + stream.time_ns, + target_times, + config.max_alignment_error_s, + name=name, + ) + return stream.value[rows] + + left_pose = _matrices_from_pose_rows(aligned(index.left_ee, "left_ee")) + right_pose = _matrices_from_pose_rows(aligned(index.right_ee, "right_ee")) + floating = _matrices_from_pose_rows(aligned(index.floating_base, "floating_base")) + scale = float(config.gripper_max - config.gripper_min) + if scale <= 0: + raise ValueError("gripper_max must be greater than gripper_min") + + def grip(stream: TimedArray, name: str) -> np.ndarray: + value = (aligned(stream, name).astype(np.float64) - config.gripper_min) / scale + value = np.clip(value, 0.0, 1.0) + return 1.0 - value if config.gripper_invert else value + + output = build_action_targets( + left_base=left_pose, + right_base=right_pose, + base_from_torso=floating, + torso_from_camera=index.torso_from_camera, + left_grip=grip(index.left_gripper, "left_gripper"), + right_grip=grip(index.right_gripper, "right_gripper"), + ) + output.update( + { + "rgb": _decode_selected_images( + index, topics, image_indices, config.resolution + ), + "image_indices": torch.from_numpy(image_indices.copy()), + "image_time_ns": torch.from_numpy(target_times.copy()), + "effective_fps": torch.tensor( + (len(target_times) - 1) + * 1.0e9 + / float(target_times[-1] - target_times[0]), + dtype=torch.float64, + ), + } + ) + return output diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_action_diffusion.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_action_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..ddeafaa82a178a05b5132d475b49410b18c0efa4 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_action_diffusion.py @@ -0,0 +1,209 @@ +"""Flow-matching helpers for isolated joint RGB-video and action denoising.""" + +from __future__ import annotations + +from typing import Any + +import torch +import torch.nn.functional as F + + +def _video_mse( + prediction: torch.Tensor, + target: torch.Tensor, + image_is_pad: torch.Tensor | None, + *, + exclude_rgb_condition_from_loss: bool, +) -> torch.Tensor: + token_loss = F.mse_loss( + prediction.float(), target.float(), reduction="none" + ).mean(dim=(1, 3, 4)) + if image_is_pad is None: + return token_loss.mean(dim=1) + if image_is_pad.ndim != 2 or image_is_pad.shape[0] != token_loss.shape[0]: + raise ValueError("image_is_pad must be [B,T_rgb]") + raw_tail = image_is_pad[:, 1:].to( + device=token_loss.device, dtype=torch.bool + ) + latent_tail_steps = ( + token_loss.shape[1] + if exclude_rgb_condition_from_loss + else token_loss.shape[1] - 1 + ) + if latent_tail_steps <= 0 or raw_tail.shape[1] % latent_tail_steps != 0: + raise ValueError("image_is_pad cannot be aligned with video latent steps") + temporal_factor = raw_tail.shape[1] // latent_tail_steps + latent_tail_is_pad = raw_tail.reshape( + raw_tail.shape[0], latent_tail_steps, temporal_factor + ).all(dim=2) + video_is_pad = ( + latent_tail_is_pad + if exclude_rgb_condition_from_loss + else torch.cat( + [image_is_pad[:, :1].to(device=token_loss.device), latent_tail_is_pad], + dim=1, + ) + ) + if tuple(video_is_pad.shape) != tuple(token_loss.shape): + raise ValueError("Video pad mask shape does not match latent loss tokens") + valid = (~video_is_pad).to(dtype=token_loss.dtype) + return (token_loss * valid).sum(dim=1) / valid.sum(dim=1).clamp(min=1) + + +def _action_mse( + prediction: torch.Tensor, + target: torch.Tensor, + action_is_pad: torch.Tensor | None, +) -> torch.Tensor: + token_loss = F.mse_loss( + prediction.float(), target.float(), reduction="none" + ).mean(dim=2) + if action_is_pad is None: + return token_loss.mean(dim=1) + if tuple(action_is_pad.shape) != tuple(token_loss.shape): + raise ValueError("action_is_pad shape does not match action tokens") + valid = (~action_is_pad.to(device=token_loss.device, dtype=torch.bool)).to( + token_loss.dtype + ) + return (token_loss * valid).sum(dim=1) / valid.sum(dim=1).clamp(min=1) + + +def joint_rgb_action_flow_matching_loss( + model: torch.nn.Module, + video_scheduler: Any, + action_scheduler: Any, + *, + z_rgb: torch.Tensor, + action: torch.Tensor, + initial_state: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor | None, + action_is_pad: torch.Tensor | None = None, + image_is_pad: torch.Tensor | None = None, + fix_rgb_first_latent: bool = True, + exclude_rgb_condition_from_loss: bool = True, + lambda_rgb: float = 1.0, + lambda_action: float = 1.0, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + """Use independent training timesteps and a single two-expert MoT forward.""" + + if z_rgb.ndim != 5: + raise ValueError("z_rgb must be [B,C,T,H,W]") + if action.ndim != 3 or action.shape[0] != z_rgb.shape[0]: + raise ValueError("action must be [B,T_action,action_dim]") + batch = int(z_rgb.shape[0]) + video_timestep = video_scheduler.sample_training_t( + batch, z_rgb.device, z_rgb.dtype + ) + action_timestep = action_scheduler.sample_training_t( + batch, action.device, action.dtype + ) + video_noise = torch.randn_like(z_rgb) + noisy_video = video_scheduler.add_noise(z_rgb, video_noise, video_timestep) + target_video = video_scheduler.training_target( + z_rgb, video_noise, video_timestep + ) + if fix_rgb_first_latent: + noisy_video[:, :, 0:1] = z_rgb[:, :, 0:1] + action_noise = torch.randn_like(action) + noisy_action = action_scheduler.add_noise(action, action_noise, action_timestep) + target_action = action_scheduler.training_target( + action, action_noise, action_timestep + ) + output = model( + noisy_video, + video_timestep, + context, + action_tokens=noisy_action, + initial_state=initial_state, + action_timestep=action_timestep, + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + rgb_start = 1 if exclude_rgb_condition_from_loss else 0 + video_per_sample = _video_mse( + output["video"][:, :, rgb_start:], + target_video[:, :, rgb_start:], + image_is_pad, + exclude_rgb_condition_from_loss=exclude_rgb_condition_from_loss, + ) + action_per_sample = _action_mse( + output["action"], target_action, action_is_pad + ) + video_weight = video_scheduler.training_weight(video_timestep).to( + device=z_rgb.device, dtype=torch.float32 + ) + action_weight = action_scheduler.training_weight(action_timestep).to( + device=action.device, dtype=torch.float32 + ) + loss_rgb = (video_per_sample * video_weight).mean() + loss_action = (action_per_sample * action_weight).mean() + total = float(lambda_rgb) * loss_rgb + float(lambda_action) * loss_action + return total, { + "loss_rgb": loss_rgb.detach(), + "loss_action": loss_action.detach(), + "video_timestep_mean": video_timestep.float().mean().detach(), + "action_timestep_mean": action_timestep.float().mean().detach(), + "prediction_rgb_norm": output["video"].float().norm().detach(), + "prediction_action_norm": output["action"].float().norm().detach(), + } + + +@torch.no_grad() +def sample_joint_rgb_and_action( + model: torch.nn.Module, + video_scheduler: Any, + action_scheduler: Any, + *, + video_shape: tuple[int, int, int, int, int], + action_shape: tuple[int, int, int], + context: torch.Tensor, + context_mask: torch.Tensor | None, + initial_state: torch.Tensor, + rgb_condition_latent: torch.Tensor, + num_inference_steps: int, + generator: torch.Generator | None = None, + dtype: torch.dtype | None = None, + device: torch.device | str | None = None, +) -> dict[str, torch.Tensor]: + """Jointly sample one RGB latent video and one action trajectory.""" + + batch, channels, _, height, width = video_shape + if channels != 48: + raise ValueError(f"Expected 48 RGB latent channels, got {channels}") + if action_shape[0] != batch: + raise ValueError("Video and action batch sizes must match") + if tuple(rgb_condition_latent.shape) != (batch, 48, 1, height, width): + raise ValueError("rgb_condition_latent has the wrong shape") + device = context.device if device is None else torch.device(device) + dtype = context.dtype if dtype is None else dtype + video = torch.randn( + video_shape, generator=generator, device=device, dtype=dtype + ) + action = torch.randn( + action_shape, generator=generator, device=device, dtype=dtype + ) + video[:, :, 0:1] = rgb_condition_latent.to(video) + video_ts, video_deltas = video_scheduler.build_inference_schedule( + num_inference_steps, device, dtype + ) + action_ts, action_deltas = action_scheduler.build_inference_schedule( + num_inference_steps, device, dtype + ) + for video_t, video_delta, action_t, action_delta in zip( + video_ts, video_deltas, action_ts, action_deltas, strict=True + ): + output = model( + video, + video_t.expand(batch), + context, + action_tokens=action, + initial_state=initial_state, + action_timestep=action_t.expand(batch), + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + video = video_scheduler.step(output["video"], video_delta, video) + action = action_scheduler.step(output["action"], action_delta, action) + video[:, :, 0:1] = rgb_condition_latent.to(video) + return {"rgb": video, "action": action} diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_action_diffusion_shared_timestep.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_action_diffusion_shared_timestep.py new file mode 100644 index 0000000000000000000000000000000000000000..246b3c4fa1dafc54e1b73781003da283c59cc36f --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_action_diffusion_shared_timestep.py @@ -0,0 +1,140 @@ +"""Shared-timestep training loss for joint RGB-video and action denoising. + +This sibling changes only the training-time timestep coupling. Video and +action still receive independent Gaussian noise and keep their own scheduler +operations, losses, weights, output heads, and gradient paths. +""" + +from __future__ import annotations + +import math +from typing import Any + +import torch + +from joint_rgb_action_diffusion import _action_mse, _video_mse + + +def _validate_matching_training_schedulers( + video_scheduler: Any, action_scheduler: Any +) -> None: + """Fail closed unless one sampled timestep means the same noise level.""" + + video_steps = int(video_scheduler.num_train_timesteps) + action_steps = int(action_scheduler.num_train_timesteps) + if video_steps != action_steps: + raise ValueError( + "Shared timestep requires equal scheduler lengths: " + f"video={video_steps}, action={action_steps}" + ) + video_shift = float(video_scheduler.shift) + action_shift = float(action_scheduler.shift) + if not math.isclose(video_shift, action_shift, rel_tol=0.0, abs_tol=0.0): + raise ValueError( + "Shared timestep requires equal training shifts: " + f"video={video_shift}, action={action_shift}" + ) + + +def joint_rgb_action_shared_timestep_flow_matching_loss( + model: torch.nn.Module, + video_scheduler: Any, + action_scheduler: Any, + *, + z_rgb: torch.Tensor, + action: torch.Tensor, + initial_state: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor | None, + action_is_pad: torch.Tensor | None = None, + image_is_pad: torch.Tensor | None = None, + fix_rgb_first_latent: bool = True, + exclude_rgb_condition_from_loss: bool = True, + lambda_rgb: float = 1.0, + lambda_action: float = 1.0, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + """Denoise video and action at one shared training timestep. + + The joint timestep is sampled exactly once from the video scheduler. The + action scheduler has the same length and shift by contract, so the value + represents the same sigma for both modalities. Noise remains independent. + """ + + if z_rgb.ndim != 5: + raise ValueError("z_rgb must be [B,C,T,H,W]") + if action.ndim != 3 or action.shape[0] != z_rgb.shape[0]: + raise ValueError("action must be [B,T_action,action_dim]") + if z_rgb.device != action.device: + raise ValueError("Shared-timestep video and action must be on one device") + _validate_matching_training_schedulers(video_scheduler, action_scheduler) + + batch = int(z_rgb.shape[0]) + joint_timestep = video_scheduler.sample_training_t( + batch, z_rgb.device, z_rgb.dtype + ) + video_timestep = joint_timestep + action_timestep = joint_timestep.to(device=action.device, dtype=action.dtype) + if not torch.equal(video_timestep, action_timestep): + raise RuntimeError("Shared video/action timestep diverged after device/dtype cast") + + video_noise = torch.randn_like(z_rgb) + noisy_video = video_scheduler.add_noise(z_rgb, video_noise, video_timestep) + target_video = video_scheduler.training_target( + z_rgb, video_noise, video_timestep + ) + if fix_rgb_first_latent: + noisy_video[:, :, 0:1] = z_rgb[:, :, 0:1] + + action_noise = torch.randn_like(action) + noisy_action = action_scheduler.add_noise( + action, action_noise, action_timestep + ) + target_action = action_scheduler.training_target( + action, action_noise, action_timestep + ) + + output = model( + noisy_video, + video_timestep, + context, + action_tokens=noisy_action, + initial_state=initial_state, + action_timestep=action_timestep, + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + rgb_start = 1 if exclude_rgb_condition_from_loss else 0 + video_per_sample = _video_mse( + output["video"][:, :, rgb_start:], + target_video[:, :, rgb_start:], + image_is_pad, + exclude_rgb_condition_from_loss=exclude_rgb_condition_from_loss, + ) + action_per_sample = _action_mse( + output["action"], target_action, action_is_pad + ) + video_weight = video_scheduler.training_weight(video_timestep).to( + device=z_rgb.device, dtype=torch.float32 + ) + action_weight = action_scheduler.training_weight(action_timestep).to( + device=action.device, dtype=torch.float32 + ) + loss_rgb = (video_per_sample * video_weight).mean() + loss_action = (action_per_sample * action_weight).mean() + total = float(lambda_rgb) * loss_rgb + float(lambda_action) * loss_action + timestep_max_abs_diff = ( + video_timestep.float() - action_timestep.float() + ).abs().max() + return total, { + "loss_rgb": loss_rgb.detach(), + "loss_action": loss_action.detach(), + "joint_timestep_mean": joint_timestep.float().mean().detach(), + "video_timestep_mean": video_timestep.float().mean().detach(), + "action_timestep_mean": action_timestep.float().mean().detach(), + "timestep_max_abs_diff": timestep_max_abs_diff.detach(), + # The existing trainer logs metrics whose names begin with ``loss_``. + # This is telemetry only and is not added to ``total`` above. + "loss_timestep_alignment_error": timestep_max_abs_diff.detach(), + "prediction_rgb_norm": output["video"].float().norm().detach(), + "prediction_action_norm": output["action"].float().norm().detach(), + } diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_depth_action_fastwam_diffusion.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_depth_action_fastwam_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..8321bd38be6e78d921d2c3c739177b8b3156928a --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_depth_action_fastwam_diffusion.py @@ -0,0 +1,117 @@ +"""Joint RGB/depth/action flow matching with FastWAM tail masking.""" + +from __future__ import annotations + +from typing import Any + +import torch + +from joint_rgb_action_diffusion import _action_mse, _video_mse + + +def joint_rgb_depth_action_fastwam_loss( + model: torch.nn.Module, + video_scheduler: Any, + action_scheduler: Any, + *, + z_rgb: torch.Tensor, + z_depth: torch.Tensor, + action: torch.Tensor, + initial_state: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor | None, + image_is_pad: torch.Tensor, + action_is_pad: torch.Tensor, + lambda_rgb: float, + lambda_depth: float, + lambda_action: float, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + """Use one video timestep, independent RGB/depth noise, and one action timestep.""" + + if z_rgb.ndim != 5 or tuple(z_rgb.shape) != tuple(z_depth.shape): + raise ValueError("RGB and depth latents must have matching [B,48,T,H,W] shapes") + if z_rgb.shape[1] != 48: + raise ValueError(f"Expected 48 latent channels per video modality, got {z_rgb.shape[1]}") + if action.ndim != 3 or action.shape[0] != z_rgb.shape[0]: + raise ValueError("action must be [B,T,14]") + if image_is_pad.shape != (z_rgb.shape[0], 33): + raise ValueError(f"image_is_pad must be [B,33], got {tuple(image_is_pad.shape)}") + + batch = int(z_rgb.shape[0]) + video_timestep = video_scheduler.sample_training_t(batch, z_rgb.device, z_rgb.dtype) + action_timestep = action_scheduler.sample_training_t(batch, action.device, action.dtype) + + # SpaceDreamer contract: modality noise is independent while both video + # modalities use the same sampled video timestep. + rgb_noise = torch.randn_like(z_rgb) + depth_noise = torch.randn_like(z_depth) + noisy_rgb = video_scheduler.add_noise(z_rgb, rgb_noise, video_timestep) + noisy_depth = video_scheduler.add_noise(z_depth, depth_noise, video_timestep) + target_rgb = video_scheduler.training_target(z_rgb, rgb_noise, video_timestep) + target_depth = video_scheduler.training_target(z_depth, depth_noise, video_timestep) + + # Both oracle first-frame conditions use Wan t=0 and are fixed at every + # training forward. They are excluded from their respective losses below. + noisy_rgb[:, :, 0:1] = z_rgb[:, :, 0:1] + noisy_depth[:, :, 0:1] = z_depth[:, :, 0:1] + + action_noise = torch.randn_like(action) + noisy_action = action_scheduler.add_noise(action, action_noise, action_timestep) + target_action = action_scheduler.training_target(action, action_noise, action_timestep) + + output = model( + torch.cat((noisy_rgb, noisy_depth), dim=1), + video_timestep, + context, + action_tokens=noisy_action, + initial_state=initial_state, + action_timestep=action_timestep, + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + if tuple(output["video"].shape) != ( + z_rgb.shape[0], 96, *z_rgb.shape[2:] + ): + raise ValueError(f"Unexpected joint RGB/depth output {tuple(output['video'].shape)}") + prediction_rgb, prediction_depth = output["video"].split(48, dim=1) + + # The future 32 raw frames map to 8 temporal latent steps. A latent is + # masked only when all four contributing raw frames are padded. + rgb_per_sample = _video_mse( + prediction_rgb[:, :, 1:], + target_rgb[:, :, 1:], + image_is_pad, + exclude_rgb_condition_from_loss=True, + ) + depth_per_sample = _video_mse( + prediction_depth[:, :, 1:], + target_depth[:, :, 1:], + image_is_pad, + exclude_rgb_condition_from_loss=True, + ) + action_per_sample = _action_mse(output["action"], target_action, action_is_pad) + + video_weight = video_scheduler.training_weight(video_timestep).to( + device=z_rgb.device, dtype=torch.float32 + ) + action_weight = action_scheduler.training_weight(action_timestep).to( + device=action.device, dtype=torch.float32 + ) + loss_rgb = (rgb_per_sample * video_weight).mean() + loss_depth = (depth_per_sample * video_weight).mean() + loss_action = (action_per_sample * action_weight).mean() + total = ( + float(lambda_rgb) * loss_rgb + + float(lambda_depth) * loss_depth + + float(lambda_action) * loss_action + ) + return total, { + "loss_rgb": loss_rgb.detach(), + "loss_depth": loss_depth.detach(), + "loss_action": loss_action.detach(), + "video_timestep_mean": video_timestep.float().mean().detach(), + "action_timestep_mean": action_timestep.float().mean().detach(), + "prediction_rgb_norm": prediction_rgb.float().norm().detach(), + "prediction_depth_norm": prediction_depth.float().norm().detach(), + "prediction_action_norm": output["action"].float().norm().detach(), + } diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_depth_point_action_diffusion.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_depth_point_action_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..1f9f70e9d578a5aff11cf670fd60a3692296857a --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_depth_point_action_diffusion.py @@ -0,0 +1,278 @@ +"""Four-output flow-matching helpers for RGB/depth/point/action MoT.""" + +from __future__ import annotations + +from typing import Any + +import torch +import torch.nn.functional as F + +from wan_rgb_depth_point_initialization import split_joint_rgb_depth_point_prediction + + +def _mse_video_per_sample(prediction: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + return F.mse_loss(prediction.float(), target.float(), reduction="none").mean( + dim=(1, 2, 3, 4) + ) + + +def _mse_action_per_sample( + prediction: torch.Tensor, + target: torch.Tensor, + action_is_pad: torch.Tensor | None, +) -> torch.Tensor: + token_loss = F.mse_loss( + prediction.float(), target.float(), reduction="none" + ).mean(dim=2) + if action_is_pad is None: + return token_loss.mean(dim=1) + if action_is_pad.shape != token_loss.shape: + raise ValueError( + f"action_is_pad must have shape {tuple(token_loss.shape)}, " + f"got {tuple(action_is_pad.shape)}" + ) + valid = (~action_is_pad.to(device=token_loss.device, dtype=torch.bool)).to( + dtype=token_loss.dtype + ) + return (token_loss * valid).sum(dim=1) / valid.sum(dim=1).clamp(min=1.0) + + +def joint_flow_matching_loss( + model: torch.nn.Module, + video_scheduler: Any, + action_scheduler: Any, + *, + z_rgb: torch.Tensor, + z_depth: torch.Tensor, + z_point: torch.Tensor, + action: torch.Tensor, + camera: torch.Tensor, + initial_state: torch.Tensor | None = None, + context: torch.Tensor, + context_mask: torch.Tensor | None, + action_is_pad: torch.Tensor | None = None, + camera_is_pad: torch.Tensor | None = None, + fix_rgb_first_latent: bool = True, + fix_depth_first_latent: bool = True, + exclude_rgb_condition_from_loss: bool = True, + exclude_depth_condition_from_loss: bool = True, + lambda_rgb: float = 1.0, + lambda_depth: float = 1.0, + lambda_point: float = 1.0, + lambda_action: float = 1.0, + lambda_camera_translation: float = 1.0, + lambda_camera_rotation: float = 1.0, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + """Compute independent video/action flow matching with one MoT forward.""" + + video_shapes = {tuple(value.shape) for value in (z_rgb, z_depth, z_point)} + if len(video_shapes) != 1: + raise ValueError(f"RGB/depth/point latent shapes must match, got {video_shapes}") + if action.ndim != 3 or action.shape[0] != z_rgb.shape[0]: + raise ValueError("action must be [B, T_action, action_dim]") + expected_camera = (int(action.shape[0]), int(action.shape[1]), 6) + if tuple(camera.shape) != expected_camera: + raise ValueError( + f"camera must have shape {expected_camera}, got {tuple(camera.shape)}" + ) + batch = int(z_rgb.shape[0]) + video_timestep = video_scheduler.sample_training_t( + batch, z_rgb.device, z_rgb.dtype + ) + action_timestep = action_scheduler.sample_training_t( + batch, action.device, action.dtype + ) + + clean_video = (z_rgb, z_depth, z_point) + noise_video = tuple(torch.randn_like(value) for value in clean_video) + noisy_video = [ + video_scheduler.add_noise(value, noise, video_timestep) + for value, noise in zip(clean_video, noise_video, strict=True) + ] + target_video = tuple( + video_scheduler.training_target(value, noise, video_timestep) + for value, noise in zip(clean_video, noise_video, strict=True) + ) + if fix_rgb_first_latent: + noisy_video[0][:, :, 0:1] = z_rgb[:, :, 0:1] + if fix_depth_first_latent: + noisy_video[1][:, :, 0:1] = z_depth[:, :, 0:1] + + action_noise = torch.randn_like(action) + noisy_action = action_scheduler.add_noise(action, action_noise, action_timestep) + target_action = action_scheduler.training_target( + action, action_noise, action_timestep + ) + camera_noise = torch.randn_like(camera) + noisy_camera = action_scheduler.add_noise( + camera, camera_noise, action_timestep + ) + target_camera = action_scheduler.training_target( + camera, camera_noise, action_timestep + ) + output = model( + torch.cat(noisy_video, dim=1), + video_timestep, + context, + action_tokens=noisy_action, + camera_tokens=noisy_camera, + initial_state=initial_state, + action_timestep=action_timestep, + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + pred_rgb, pred_depth, pred_point = split_joint_rgb_depth_point_prediction( + output["video"], + rgb_channels=int(z_rgb.shape[1]), + depth_channels=int(z_depth.shape[1]), + point_channels=int(z_point.shape[1]), + ) + pred_action = output["action"] + pred_camera = output["camera"] + + rgb_start = 1 if exclude_rgb_condition_from_loss else 0 + depth_start = 1 if exclude_depth_condition_from_loss else 0 + rgb_per_sample = _mse_video_per_sample( + pred_rgb[:, :, rgb_start:], target_video[0][:, :, rgb_start:] + ) + depth_per_sample = _mse_video_per_sample( + pred_depth[:, :, depth_start:], target_video[1][:, :, depth_start:] + ) + point_per_sample = _mse_video_per_sample(pred_point, target_video[2]) + action_per_sample = _mse_action_per_sample( + pred_action, target_action, action_is_pad + ) + camera_translation_per_sample = _mse_action_per_sample( + pred_camera[..., :3], target_camera[..., :3], camera_is_pad + ) + camera_rotation_per_sample = _mse_action_per_sample( + pred_camera[..., 3:], target_camera[..., 3:], camera_is_pad + ) + + video_weight = video_scheduler.training_weight(video_timestep).to( + device=z_rgb.device, dtype=torch.float32 + ) + action_weight = action_scheduler.training_weight(action_timestep).to( + device=action.device, dtype=torch.float32 + ) + loss_rgb = (rgb_per_sample * video_weight).mean() + loss_depth = (depth_per_sample * video_weight).mean() + loss_point = (point_per_sample * video_weight).mean() + loss_action = (action_per_sample * action_weight).mean() + loss_camera_translation = ( + camera_translation_per_sample * action_weight + ).mean() + loss_camera_rotation = ( + camera_rotation_per_sample * action_weight + ).mean() + total = ( + float(lambda_rgb) * loss_rgb + + float(lambda_depth) * loss_depth + + float(lambda_point) * loss_point + + float(lambda_action) * loss_action + + float(lambda_camera_translation) * loss_camera_translation + + float(lambda_camera_rotation) * loss_camera_rotation + ) + return total, { + "loss_rgb": loss_rgb.detach(), + "loss_depth": loss_depth.detach(), + "loss_point": loss_point.detach(), + "loss_action": loss_action.detach(), + "loss_camera_translation": loss_camera_translation.detach(), + "loss_camera_rotation": loss_camera_rotation.detach(), + "video_timestep_mean": video_timestep.float().mean().detach(), + "action_timestep_mean": action_timestep.float().mean().detach(), + "prediction_rgb_norm": pred_rgb.float().norm().detach(), + "prediction_depth_norm": pred_depth.float().norm().detach(), + "prediction_point_norm": pred_point.float().norm().detach(), + "prediction_action_norm": pred_action.float().norm().detach(), + "prediction_camera_norm": pred_camera.float().norm().detach(), + } + + +@torch.no_grad() +def sample_joint_latents_and_actions( + model: torch.nn.Module, + video_scheduler: Any, + action_scheduler: Any, + *, + video_shape: tuple[int, int, int, int, int], + action_shape: tuple[int, int, int], + camera_shape: tuple[int, int, int], + context: torch.Tensor, + context_mask: torch.Tensor | None, + initial_state: torch.Tensor | None = None, + rgb_condition_latent: torch.Tensor, + depth_condition_latent: torch.Tensor, + num_inference_steps: int, + generator: torch.Generator | None = None, + dtype: torch.dtype | None = None, + device: torch.device | str | None = None, +) -> dict[str, torch.Tensor]: + """Jointly sample RGB/depth/point videos and an action trajectory.""" + + batch, channels, frames, height, width = video_shape + if channels != 144: + raise ValueError(f"Expected 144 joint video channels, got {channels}") + if action_shape[0] != batch or camera_shape[0] != batch: + raise ValueError("Video/action/camera batch sizes must match") + if camera_shape != (action_shape[0], action_shape[1], 6): + raise ValueError("Camera shape must be [B,T_action,6]") + device = context.device if device is None else torch.device(device) + dtype = context.dtype if dtype is None else dtype + joint_video = torch.randn( + video_shape, generator=generator, device=device, dtype=dtype + ) + noisy_action = torch.randn( + action_shape, generator=generator, device=device, dtype=dtype + ) + noisy_camera = torch.randn( + camera_shape, generator=generator, device=device, dtype=dtype + ) + if tuple(rgb_condition_latent.shape) != (batch, 48, 1, height, width): + raise ValueError("rgb_condition_latent has the wrong shape") + if tuple(depth_condition_latent.shape) != (batch, 48, 1, height, width): + raise ValueError("depth_condition_latent has the wrong shape") + joint_video[:, :48, 0:1] = rgb_condition_latent.to(joint_video) + joint_video[:, 48:96, 0:1] = depth_condition_latent.to(joint_video) + + video_ts, video_deltas = video_scheduler.build_inference_schedule( + num_inference_steps, device, dtype + ) + action_ts, action_deltas = action_scheduler.build_inference_schedule( + num_inference_steps, device, dtype + ) + for video_t, video_delta, action_t, action_delta in zip( + video_ts, video_deltas, action_ts, action_deltas, strict=True + ): + output = model( + joint_video, + video_t.expand(batch), + context, + action_tokens=noisy_action, + camera_tokens=noisy_camera, + initial_state=initial_state, + action_timestep=action_t.expand(batch), + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + joint_video = video_scheduler.step( + output["video"], video_delta, joint_video + ) + noisy_action = action_scheduler.step( + output["action"], action_delta, noisy_action + ) + noisy_camera = action_scheduler.step( + output["camera"], action_delta, noisy_camera + ) + joint_video[:, :48, 0:1] = rgb_condition_latent.to(joint_video) + joint_video[:, 48:96, 0:1] = depth_condition_latent.to(joint_video) + + rgb, depth, point = split_joint_rgb_depth_point_prediction(joint_video) + return { + "rgb": rgb, + "depth": depth, + "point": point, + "action": noisy_action, + "camera": noisy_camera, + } diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_depth_point_action_fastwam_diffusion.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_depth_point_action_fastwam_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..f6ce6a56088211b88f6f70d8cd64f7a6b563feed --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_depth_point_action_fastwam_diffusion.py @@ -0,0 +1,160 @@ +"""Four-output RGB/depth/point/action flow matching with FastWAM tail masks.""" + +from __future__ import annotations + +from typing import Any + +import torch + +from joint_rgb_action_diffusion import _action_mse, _video_mse +from wan_rgb_depth_point_initialization import split_joint_rgb_depth_point_prediction + + +# Per-process evidence collected directly on the multimodal prediction tensors. +# This survives FSDP parameter flattening/sharding, unlike hooks registered on a +# pre-wrap output-head parameter. +_OUTPUT_GRADIENT_EVIDENCE: dict[str, float] = {} + + +def output_gradient_evidence() -> dict[str, float]: + return dict(_OUTPUT_GRADIENT_EVIDENCE) + + + +def _weighted_action_mse(prediction, target, action_is_pad, weights): + if prediction.shape != target.shape or prediction.ndim != 3 or prediction.shape[-1] != 14: + raise ValueError("Expected matching [B,T,14] action tensors") + w = torch.as_tensor(weights, device=prediction.device, dtype=torch.float32) + if w.shape != (14,) or not torch.isfinite(w).all() or (w <= 0).any(): + raise ValueError("Action weights must be fourteen finite positive values") + token = ((prediction.float() - target.float()).square() * w).sum(dim=-1) / w.sum() + if action_is_pad is None: + return token.mean(dim=1) + if action_is_pad.shape != token.shape: + raise ValueError("action_is_pad shape does not match action tokens") + valid = (~action_is_pad.to(device=token.device, dtype=torch.bool)).to(token.dtype) + return (token * valid).sum(dim=1) / valid.sum(dim=1).clamp(min=1) + +def joint_rgb_depth_point_action_fastwam_loss( + model: torch.nn.Module, + video_scheduler: Any, + action_scheduler: Any, + *, + z_rgb: torch.Tensor, + z_depth: torch.Tensor, + z_point: torch.Tensor, + action: torch.Tensor, + initial_state: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor | None, + image_is_pad: torch.Tensor, + action_is_pad: torch.Tensor, + lambda_rgb: float, + lambda_depth: float, + lambda_point: float, + lambda_action: float, + condition_depth_frame0: bool = True, + action_dimension_loss_weights: list[float] | None = None, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + shapes = {tuple(value.shape) for value in (z_rgb, z_depth, z_point)} + if len(shapes) != 1 or z_rgb.ndim != 5 or z_rgb.shape[1] != 48: + raise ValueError(f"RGB/depth/point latents must match [B,48,T,H,W], got {shapes}") + if image_is_pad.shape != (z_rgb.shape[0], 33): + raise ValueError(f"image_is_pad must be [B,33], got {tuple(image_is_pad.shape)}") + if action.ndim != 3 or action.shape[0] != z_rgb.shape[0]: + raise ValueError("action must be [B,T,14]") + + batch = int(z_rgb.shape[0]) + video_timestep = video_scheduler.sample_training_t(batch, z_rgb.device, z_rgb.dtype) + action_timestep = action_scheduler.sample_training_t(batch, action.device, action.dtype) + clean = (z_rgb, z_depth, z_point) + noises = tuple(torch.randn_like(value) for value in clean) + noisy = [ + video_scheduler.add_noise(value, noise, video_timestep) + for value, noise in zip(clean, noises, strict=True) + ] + targets = tuple( + video_scheduler.training_target(value, noise, video_timestep) + for value, noise in zip(clean, noises, strict=True) + ) + # RGB frame zero is always observed. Depth frame zero is configurable so + # the same output head can train RGB-only -> RGB/depth/point/action models. + # Point frame zero is always a diffusion target. + noisy[0][:, :, 0:1] = z_rgb[:, :, 0:1] + if condition_depth_frame0: + noisy[1][:, :, 0:1] = z_depth[:, :, 0:1] + + action_noise = torch.randn_like(action) + noisy_action = action_scheduler.add_noise(action, action_noise, action_timestep) + target_action = action_scheduler.training_target(action, action_noise, action_timestep) + output = model( + torch.cat(noisy, dim=1), + video_timestep, + context, + action_tokens=noisy_action, + initial_state=initial_state, + action_timestep=action_timestep, + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + if tuple(output["video"].shape) != (z_rgb.shape[0], 144, *z_rgb.shape[2:]): + raise ValueError(f"Unexpected joint RGB/depth/point output {output['video'].shape}") + pred_rgb, pred_depth, pred_point = split_joint_rgb_depth_point_prediction( + output["video"] + ) + if torch.is_grad_enabled() and pred_point.requires_grad: + def capture_point_prediction_gradient(gradient: torch.Tensor) -> torch.Tensor: + _OUTPUT_GRADIENT_EVIDENCE["point_prediction"] = float( + gradient.detach().float().norm().item() + ) + return gradient + + pred_point.register_hook(capture_point_prediction_gradient) + per_sample_rgb = _video_mse( + pred_rgb[:, :, 1:], targets[0][:, :, 1:], image_is_pad, + exclude_rgb_condition_from_loss=True, + ) + if condition_depth_frame0: + per_sample_depth = _video_mse( + pred_depth[:, :, 1:], targets[1][:, :, 1:], image_is_pad, + exclude_rgb_condition_from_loss=True, + ) + else: + per_sample_depth = _video_mse( + pred_depth, targets[1], image_is_pad, + exclude_rgb_condition_from_loss=False, + ) + per_sample_point = _video_mse( + pred_point, targets[2], image_is_pad, + exclude_rgb_condition_from_loss=False, + ) + per_sample_action = (_action_mse(output["action"], target_action, action_is_pad) if action_dimension_loss_weights is None else + _weighted_action_mse(output["action"], target_action, action_is_pad, action_dimension_loss_weights)) + video_weight = video_scheduler.training_weight(video_timestep).to( + device=z_rgb.device, dtype=torch.float32 + ) + action_weight = action_scheduler.training_weight(action_timestep).to( + device=action.device, dtype=torch.float32 + ) + loss_rgb = (per_sample_rgb * video_weight).mean() + loss_depth = (per_sample_depth * video_weight).mean() + loss_point = (per_sample_point * video_weight).mean() + loss_action = (per_sample_action * action_weight).mean() + total = ( + float(lambda_rgb) * loss_rgb + + float(lambda_depth) * loss_depth + + float(lambda_point) * loss_point + + float(lambda_action) * loss_action + ) + return total, { + "loss_rgb": loss_rgb.detach(), + "loss_depth": loss_depth.detach(), + "loss_point": loss_point.detach(), + "loss_action": loss_action.detach(), + "video_timestep_mean": video_timestep.float().mean().detach(), + "action_timestep_mean": action_timestep.float().mean().detach(), + "prediction_rgb_norm": pred_rgb.float().norm().detach(), + "prediction_depth_norm": pred_depth.float().norm().detach(), + "prediction_point_norm": pred_point.float().norm().detach(), + "prediction_action_norm": output["action"].float().norm().detach(), + } diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_geometry_action_diffusion.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_geometry_action_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..e0f7239a2924829ed4089510fea4b3934de0949e --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/joint_rgb_geometry_action_diffusion.py @@ -0,0 +1,233 @@ +"""Flow matching for RoboDojo RGB/depth[/point] video and absolute-EE action.""" + +from __future__ import annotations + +from typing import Any + +import torch +import torch.nn.functional as F + + +def _video_mse(prediction: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + return F.mse_loss(prediction.float(), target.float(), reduction="none").mean( + dim=(1, 2, 3, 4) + ) + + +def _action_mse( + prediction: torch.Tensor, + target: torch.Tensor, + action_is_pad: torch.Tensor | None, +) -> torch.Tensor: + token_loss = F.mse_loss( + prediction.float(), target.float(), reduction="none" + ).mean(dim=2) + if action_is_pad is None: + return token_loss.mean(dim=1) + if tuple(action_is_pad.shape) != tuple(token_loss.shape): + raise ValueError("action_is_pad shape does not match action tokens") + valid = (~action_is_pad.to(device=token_loss.device, dtype=torch.bool)).to( + token_loss.dtype + ) + return (token_loss * valid).sum(dim=1) / valid.sum(dim=1).clamp(min=1) + + +def joint_geometry_action_flow_matching_loss( + model: torch.nn.Module, + video_scheduler: Any, + action_scheduler: Any, + *, + z_rgb: torch.Tensor, + z_depth: torch.Tensor, + z_point: torch.Tensor | None, + action: torch.Tensor, + initial_state: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor | None, + action_is_pad: torch.Tensor | None, + lambda_rgb: float, + lambda_depth: float, + lambda_point: float | None, + lambda_action: float, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + video_latents = [z_rgb, z_depth] + names = ["rgb", "depth"] + if z_point is not None: + video_latents.append(z_point) + names.append("point") + shapes = {tuple(value.shape) for value in video_latents} + if len(shapes) != 1 or z_rgb.ndim != 5: + raise ValueError(f"Geometry latent shapes must match, got {shapes}") + if action.ndim != 3 or action.shape[0] != z_rgb.shape[0]: + raise ValueError("action must be [B,T_action,action_dim]") + + batch = int(z_rgb.shape[0]) + video_timestep = video_scheduler.sample_training_t( + batch, z_rgb.device, z_rgb.dtype + ) + action_timestep = action_scheduler.sample_training_t( + batch, action.device, action.dtype + ) + noises = [torch.randn_like(value) for value in video_latents] + noisy = [ + video_scheduler.add_noise(value, noise, video_timestep) + for value, noise in zip(video_latents, noises, strict=True) + ] + targets = [ + video_scheduler.training_target(value, noise, video_timestep) + for value, noise in zip(video_latents, noises, strict=True) + ] + # The current RGB and current metric depth are observations at inference. + noisy[0][:, :, 0:1] = z_rgb[:, :, 0:1] + noisy[1][:, :, 0:1] = z_depth[:, :, 0:1] + + action_noise = torch.randn_like(action) + noisy_action = action_scheduler.add_noise(action, action_noise, action_timestep) + target_action = action_scheduler.training_target( + action, action_noise, action_timestep + ) + output = model( + torch.cat(noisy, dim=1), + video_timestep, + context, + action_tokens=noisy_action, + initial_state=initial_state, + action_timestep=action_timestep, + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + expected_channels = 48 * len(video_latents) + if tuple(output["video"].shape) != ( + z_rgb.shape[0], expected_channels, *z_rgb.shape[2:] + ): + raise ValueError(f"Unexpected joint video prediction {tuple(output['video'].shape)}") + predictions = list(output["video"].split(48, dim=1)) + weights = [float(lambda_rgb), float(lambda_depth)] + if z_point is not None: + if lambda_point is None: + raise ValueError("Point loss weight is required for the point experiment") + weights.append(float(lambda_point)) + + video_weight = video_scheduler.training_weight(video_timestep).to( + device=z_rgb.device, dtype=torch.float32 + ) + metrics: dict[str, torch.Tensor] = {} + losses = [] + for index, (name, prediction, target, weight) in enumerate( + zip(names, predictions, targets, weights, strict=True) + ): + # RGB and depth frame zero are clean conditions; point frame zero is a target. + start = 1 if index < 2 else 0 + per_sample = _video_mse( + prediction[:, :, start:], target[:, :, start:] + ) + loss = (per_sample * video_weight).mean() + metrics[f"loss_{name}"] = loss.detach() + metrics[f"prediction_{name}_norm"] = prediction.float().norm().detach() + losses.append(weight * loss) + + action_per_sample = _action_mse( + output["action"], target_action, action_is_pad + ) + action_weight = action_scheduler.training_weight(action_timestep).to( + device=action.device, dtype=torch.float32 + ) + loss_action = (action_per_sample * action_weight).mean() + total = sum(losses) + float(lambda_action) * loss_action + metrics.update( + { + "loss_action": loss_action.detach(), + "video_timestep_mean": video_timestep.float().mean().detach(), + "action_timestep_mean": action_timestep.float().mean().detach(), + "prediction_action_norm": output["action"].float().norm().detach(), + } + ) + return total, metrics + + +@torch.no_grad() +def sample_joint_geometry_and_action( + model: torch.nn.Module, + video_scheduler: Any, + action_scheduler: Any, + *, + modalities: tuple[str, ...], + latent_shape: tuple[int, int, int, int, int], + action_shape: tuple[int, int, int], + context: torch.Tensor, + context_mask: torch.Tensor | None, + initial_state: torch.Tensor, + rgb_condition_latent: torch.Tensor, + depth_condition_latent: torch.Tensor | None, + num_inference_steps: int, + generator: torch.Generator | None = None, + action_dim_mask: torch.Tensor | None = None, +) -> dict[str, torch.Tensor]: + batch, channels, _, height, width = latent_shape + if modalities not in {("rgb", "depth"), ("rgb", "depth", "point")}: + raise ValueError(f"Unsupported modalities: {modalities}") + if channels != 48 * len(modalities): + raise ValueError("Joint video latent channel count is inconsistent") + expected_condition = (batch, 48, 1, height, width) + if tuple(rgb_condition_latent.shape) != expected_condition: + raise ValueError("RGB condition latent has the wrong shape") + if depth_condition_latent is not None and tuple(depth_condition_latent.shape) != expected_condition: + raise ValueError("Depth condition latent has the wrong shape") + video = torch.randn( + latent_shape, + generator=generator, + device=context.device, + dtype=context.dtype, + ) + action = torch.randn( + action_shape, + generator=generator, + device=context.device, + dtype=context.dtype, + ) + if action_dim_mask is not None: + action_dim_mask = torch.as_tensor(action_dim_mask, device=action.device, dtype=action.dtype) + if action_dim_mask.ndim == 1: + action_dim_mask = action_dim_mask[None, None, :] + elif action_dim_mask.ndim == 2: + action_dim_mask = action_dim_mask[:, None, :] + if action_dim_mask.shape not in ((1, 1, action_shape[-1]), (batch, 1, action_shape[-1])): + raise ValueError('Action dimension mask does not match batch/action width') + if not torch.all((action_dim_mask == 0) | (action_dim_mask == 1)): + raise ValueError('Action dimension mask must be binary') + action = action * action_dim_mask + video[:, :48, 0:1] = rgb_condition_latent.to(video) + if depth_condition_latent is not None: + video[:, 48:96, 0:1] = depth_condition_latent.to(video) + video_ts, video_deltas = video_scheduler.build_inference_schedule( + num_inference_steps, context.device, context.dtype + ) + action_ts, action_deltas = action_scheduler.build_inference_schedule( + num_inference_steps, context.device, context.dtype + ) + for video_t, video_delta, action_t, action_delta in zip( + video_ts, video_deltas, action_ts, action_deltas, strict=True + ): + output = model( + video, + video_t.expand(batch), + context, + action_tokens=action, + initial_state=initial_state, + action_timestep=action_t.expand(batch), + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + video = video_scheduler.step(output["video"], video_delta, video) + action = action_scheduler.step(output["action"], action_delta, action) + if action_dim_mask is not None: + action = action * action_dim_mask + video[:, :48, 0:1] = rgb_condition_latent.to(video) + if depth_condition_latent is not None: + video[:, 48:96, 0:1] = depth_condition_latent.to(video) + result = { + name: value + for name, value in zip(modalities, video.split(48, dim=1), strict=True) + } + result["action"] = action + return result diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/online_egoverse_action.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/online_egoverse_action.py new file mode 100644 index 0000000000000000000000000000000000000000..d6dff25b699ebcff94cec9af620e9601d3c258a1 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/online_egoverse_action.py @@ -0,0 +1,681 @@ +"""Online Egoverse bimanual action construction without derived-data files. + +For every RGB/depth/point latent sample this module follows its existing +``annotation_path`` and ``source_indices`` back to the Egoverse Zarr episode, +loads only the selected pose/keypoint rows, and constructs a 14D action for +each adjacent frame pair: + + left SE(3) body-log (6), left grip proxy (1), + right SE(3) body-log (6), right grip proxy (1). + +No action sample or statistics file is written. Normalization statistics are +kept as module buffers so they can travel inside the model checkpoint. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from pathlib import Path +from typing import Any, Iterable + +import numpy as np +import torch +from torch import nn +from torch.utils.data import Dataset + +from train_joint_rgb_depth_point import collate_rgb_depth_point_batch +from world_point_cloud import ( + _read_zarr_v3_sharded_numeric_rows, + pose_txyz_qwxyz_to_matrix, +) + + +ACTION_DIM = 14 +CAMERA_DIM = 6 + + +@dataclass(frozen=True) +class OnlineActionConfig: + """Geometry and numerical conventions for online action construction.""" + + camera_to_head_transform: tuple[tuple[float, ...], ...] = ( + (1.0, 0.0, 0.0, 0.0), + (0.0, 1.0, 0.0, 0.0), + (0.0, 0.0, 1.0, 0.0), + (0.0, 0.0, 0.0, 1.0), + ) + quaternion_eps: float = 1.0e-8 + palm_width_eps_m: float = 1.0e-4 + normalization_eps: float = 1.0e-6 + normalization_clip: float | None = 10.0 + + +@dataclass(frozen=True) +class OnlineActionResult: + action: torch.Tensor + action_is_pad: torch.Tensor + camera: torch.Tensor + camera_is_pad: torch.Tensor + initial_state: torch.Tensor + left_initial_pose_reference: torch.Tensor + right_initial_pose_reference: torch.Tensor + grip_open_raw: torch.Tensor + + +def _rigid_inverse(matrix: np.ndarray) -> np.ndarray: + matrix = np.asarray(matrix, dtype=np.float64) + rotation = matrix[..., :3, :3] + translation = matrix[..., :3, 3] + output = np.broadcast_to(np.eye(4), matrix.shape).copy() + output[..., :3, :3] = np.swapaxes(rotation, -1, -2) + output[..., :3, 3] = -np.einsum( + "...ij,...j->...i", output[..., :3, :3], translation + ) + return output + + +def _skew(vector: np.ndarray) -> np.ndarray: + x, y, z = np.asarray(vector, dtype=np.float64) + return np.array([[0.0, -z, y], [z, 0.0, -x], [-y, x, 0.0]]) + + +def so3_log(rotation: np.ndarray) -> np.ndarray: + """Stable logarithm from SO(3) to an axis-angle rotation vector.""" + + rotation = np.asarray(rotation, dtype=np.float64) + cosine = float(np.clip((np.trace(rotation) - 1.0) * 0.5, -1.0, 1.0)) + angle = float(np.arccos(cosine)) + vee = 0.5 * np.array( + [ + rotation[2, 1] - rotation[1, 2], + rotation[0, 2] - rotation[2, 0], + rotation[1, 0] - rotation[0, 1], + ], + dtype=np.float64, + ) + if angle < 1.0e-7: + return vee + sine = float(np.sin(angle)) + if abs(sine) > 1.0e-6: + return vee * (angle / sine) + + # Near pi, the antisymmetric part vanishes. Recover a signed eigen-axis. + diagonal = np.maximum((np.diag(rotation) + 1.0) * 0.5, 0.0) + axis = np.sqrt(diagonal) + largest = int(np.argmax(axis)) + if axis[largest] < 1.0e-7: + raise ValueError("Cannot recover rotation axis near pi") + if largest == 0: + axis[1] = np.copysign(axis[1], rotation[0, 1] + rotation[1, 0]) + axis[2] = np.copysign(axis[2], rotation[0, 2] + rotation[2, 0]) + elif largest == 1: + axis[0] = np.copysign(axis[0], rotation[0, 1] + rotation[1, 0]) + axis[2] = np.copysign(axis[2], rotation[1, 2] + rotation[2, 1]) + else: + axis[0] = np.copysign(axis[0], rotation[0, 2] + rotation[2, 0]) + axis[1] = np.copysign(axis[1], rotation[1, 2] + rotation[2, 1]) + axis /= np.linalg.norm(axis) + return axis * angle + + +def so3_exp(rotation_vector: np.ndarray) -> np.ndarray: + """Exponential from an axis-angle rotation vector to SO(3).""" + + rotation_vector = np.asarray(rotation_vector, dtype=np.float64) + angle = float(np.linalg.norm(rotation_vector)) + omega = _skew(rotation_vector) + identity = np.eye(3, dtype=np.float64) + if angle < 1.0e-7: + return identity + omega + 0.5 * (omega @ omega) + return ( + identity + + (np.sin(angle) / angle) * omega + + ((1.0 - np.cos(angle)) / (angle * angle)) * (omega @ omega) + ) + + +def se3_log(transform: np.ndarray) -> np.ndarray: + """Return the exact 6D twist ``[rho, omega]`` of an SE(3) transform.""" + + transform = np.asarray(transform, dtype=np.float64) + omega_vector = so3_log(transform[:3, :3]) + angle = float(np.linalg.norm(omega_vector)) + omega = _skew(omega_vector) + identity = np.eye(3, dtype=np.float64) + if angle < 1.0e-7: + v_inverse = identity - 0.5 * omega + (omega @ omega) / 12.0 + else: + coefficient = ( + 1.0 - 0.5 * angle / np.tan(0.5 * angle) + ) / (angle * angle) + v_inverse = identity - 0.5 * omega + coefficient * (omega @ omega) + rho = v_inverse @ transform[:3, 3] + return np.concatenate([rho, omega_vector]) + + +def se3_exp(twist: np.ndarray) -> np.ndarray: + """Reconstruct an SE(3) transform from a 6D twist ``[rho, omega]``.""" + + twist = np.asarray(twist, dtype=np.float64) + if twist.shape != (6,): + raise ValueError(f"Expected 6D twist, got {twist.shape}") + rho, omega_vector = twist[:3], twist[3:] + angle = float(np.linalg.norm(omega_vector)) + omega = _skew(omega_vector) + identity = np.eye(3, dtype=np.float64) + if angle < 1.0e-7: + v = identity + 0.5 * omega + (omega @ omega) / 6.0 + else: + v = ( + identity + + ((1.0 - np.cos(angle)) / (angle * angle)) * omega + + ((angle - np.sin(angle)) / (angle**3)) * (omega @ omega) + ) + output = np.eye(4, dtype=np.float64) + output[:3, :3] = so3_exp(omega_vector) + output[:3, 3] = v @ rho + return output + + +def _safe_pose_matrices( + pose: np.ndarray, quaternion_eps: float +) -> tuple[np.ndarray, np.ndarray]: + pose = np.asarray(pose, dtype=np.float64) + if pose.ndim != 2 or pose.shape[1] != 7: + raise ValueError(f"Expected pose [T,7], got {pose.shape}") + valid = np.isfinite(pose).all(axis=1) + quaternion_norm = np.linalg.norm(pose[:, 3:7], axis=1) + valid &= np.isfinite(quaternion_norm) & (quaternion_norm >= quaternion_eps) + safe = pose.copy() + safe[~valid] = np.array([0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0]) + return pose_txyz_qwxyz_to_matrix(safe).astype(np.float64), valid + + +class EgoverseOnlineActionProcessor: + """Load selected Egoverse rows and create raw 14D actions in memory.""" + + def __init__(self, config: OnlineActionConfig | None = None) -> None: + self.config = OnlineActionConfig() if config is None else config + camera_to_head = np.asarray( + self.config.camera_to_head_transform, dtype=np.float64 + ) + if camera_to_head.shape != (4, 4) or not np.isfinite(camera_to_head).all(): + raise ValueError("camera_to_head_transform must be finite [4,4]") + self.camera_to_head = camera_to_head + + @staticmethod + def source_episode_root(annotation_path: Path) -> Path: + with np.load(annotation_path, allow_pickle=False) as payload: + metadata = json.loads(payload["metadata_json"].item()) + source_metadata_path = metadata.get("episode", {}).get( + "source_metadata_path" + ) + if not source_metadata_path: + raise KeyError(f"Missing source_metadata_path in {annotation_path}") + root = Path(source_metadata_path).parent + if not (root / "zarr.json").is_file(): + raise FileNotFoundError(root / "zarr.json") + return root + + def __call__( + self, annotation_path: str | Path, source_indices: torch.Tensor | np.ndarray + ) -> OnlineActionResult: + indices = np.asarray(source_indices, dtype=np.int64).reshape(-1) + if len(indices) < 2 or np.any(np.diff(indices) <= 0): + raise ValueError("source_indices must be strictly increasing with length >= 2") + root = self.source_episode_root(Path(annotation_path)) + arrays = { + key: _read_zarr_v3_sharded_numeric_rows(root / key, indices) + for key in ( + "left.obs_ee_pose", + "right.obs_ee_pose", + "left.obs_keypoints", + "right.obs_keypoints", + "obs_head_pose", + ) + } + return self.process_arrays(**{ + "left_ee_pose": arrays["left.obs_ee_pose"], + "right_ee_pose": arrays["right.obs_ee_pose"], + "left_keypoints": arrays["left.obs_keypoints"], + "right_keypoints": arrays["right.obs_keypoints"], + "head_pose": arrays["obs_head_pose"], + }) + + def process_arrays( + self, + *, + left_ee_pose: np.ndarray, + right_ee_pose: np.ndarray, + left_keypoints: np.ndarray, + right_keypoints: np.ndarray, + head_pose: np.ndarray, + ) -> OnlineActionResult: + frame_count = int(np.asarray(head_pose).shape[0]) + if frame_count < 2: + raise ValueError("At least two frames are required") + left_matrix, left_valid = _safe_pose_matrices( + left_ee_pose, self.config.quaternion_eps + ) + right_matrix, right_valid = _safe_pose_matrices( + right_ee_pose, self.config.quaternion_eps + ) + head_matrix, head_valid = _safe_pose_matrices( + head_pose, self.config.quaternion_eps + ) + if not head_valid[0]: + raise ValueError("The first head pose must be valid for camera-frame alignment") + for name, value in ( + ("left_keypoints", left_keypoints), + ("right_keypoints", right_keypoints), + ): + if np.asarray(value).shape != (frame_count, 63): + raise ValueError(f"{name} must be [T,63]") + + # Source poses are T_W_H and T_W_E. camera_to_head is T_H_C. + world_from_camera0 = head_matrix[0] @ self.camera_to_head + reference_from_world = _rigid_inverse(world_from_camera0) + world_from_camera = head_matrix @ self.camera_to_head + camera_reference = reference_from_world[None] @ world_from_camera + left_reference = reference_from_world[None] @ left_matrix + right_reference = reference_from_world[None] @ right_matrix + + left_points = np.asarray(left_keypoints, dtype=np.float64).reshape( + frame_count, 21, 3 + ) + right_points = np.asarray(right_keypoints, dtype=np.float64).reshape( + frame_count, 21, 3 + ) + rotation = reference_from_world[:3, :3] + translation = reference_from_world[:3, 3] + left_points = np.einsum("ij,tkj->tki", rotation, left_points) + translation + right_points = np.einsum("ij,tkj->tki", rotation, right_points) + translation + + def grip(points: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + required = points[:, [4, 8, 5, 17]] + valid = np.isfinite(required).all(axis=(1, 2)) + pinch = np.linalg.norm(points[:, 4] - points[:, 8], axis=1) + palm = np.linalg.norm(points[:, 5] - points[:, 17], axis=1) + valid &= np.isfinite(pinch) & np.isfinite(palm) + valid &= palm >= self.config.palm_width_eps_m + value = pinch / np.maximum(palm, self.config.palm_width_eps_m) + value[~valid] = 0.0 + return value, valid + + left_grip, left_grip_valid = grip(left_points) + right_grip, right_grip_valid = grip(right_points) + actions = np.zeros((frame_count - 1, ACTION_DIM), dtype=np.float64) + transition_valid = ( + left_valid[:-1] + & left_valid[1:] + & right_valid[:-1] + & right_valid[1:] + & left_grip_valid[1:] + & right_grip_valid[1:] + ) + for index in range(frame_count - 1): + if not transition_valid[index]: + continue + left_delta = _rigid_inverse(left_reference[index]) @ left_reference[index + 1] + right_delta = _rigid_inverse(right_reference[index]) @ right_reference[index + 1] + actions[index, 0:6] = se3_log(left_delta) + actions[index, 6] = left_grip[index + 1] + actions[index, 7:13] = se3_log(right_delta) + actions[index, 13] = right_grip[index + 1] + finite = np.isfinite(actions).all(axis=1) + transition_valid &= finite + actions[~transition_valid] = 0.0 + + # Camera token t predicts source frame t+1 relative to the first + # camera. Use direct translation plus SO(3) log rotation. + camera = np.zeros((frame_count - 1, CAMERA_DIM), dtype=np.float64) + camera_valid = head_valid[1:].copy() + for index in range(frame_count - 1): + if not camera_valid[index]: + continue + transform = camera_reference[index + 1] + camera[index, :3] = transform[:3, 3] + camera[index, 3:] = so3_log(transform[:3, :3]) + camera_valid &= np.isfinite(camera).all(axis=1) + camera[~camera_valid] = 0.0 + + initial_state = np.concatenate( + [ + se3_log(left_reference[0]), + np.asarray([left_grip[0]], dtype=np.float64), + se3_log(right_reference[0]), + np.asarray([right_grip[0]], dtype=np.float64), + ] + ) + if initial_state.shape != (ACTION_DIM,) or not np.isfinite(initial_state).all(): + raise ValueError("Initial bimanual state must be finite 14D") + return OnlineActionResult( + action=torch.from_numpy(actions.astype(np.float32)), + action_is_pad=torch.from_numpy(~transition_valid), + camera=torch.from_numpy(camera.astype(np.float32)), + camera_is_pad=torch.from_numpy(~camera_valid), + initial_state=torch.from_numpy(initial_state.astype(np.float32)), + left_initial_pose_reference=torch.from_numpy( + left_reference[0].astype(np.float32) + ), + right_initial_pose_reference=torch.from_numpy( + right_reference[0].astype(np.float32) + ), + grip_open_raw=torch.from_numpy( + np.stack([left_grip[1:], right_grip[1:]], axis=1).astype(np.float32) + ), + ) + + +class OnlineActionNormalizer(nn.Module): + """Mean/std action normalization stored as checkpoint-compatible buffers.""" + + def __init__( + self, + action_dim: int = ACTION_DIM, + *, + eps: float = 1.0e-6, + clip: float | None = 10.0, + ) -> None: + super().__init__() + self.action_dim = int(action_dim) + self.eps = float(eps) + self.clip = None if clip is None else float(clip) + self.register_buffer("mean", torch.zeros(self.action_dim, dtype=torch.float32)) + self.register_buffer("std", torch.ones(self.action_dim, dtype=torch.float32)) + self.register_buffer("count", torch.zeros((), dtype=torch.long)) + self.register_buffer("fitted", torch.tensor(False, dtype=torch.bool)) + + @torch.no_grad() + def fit(self, results: Iterable[OnlineActionResult]) -> "OnlineActionNormalizer": + total = torch.zeros(self.action_dim, dtype=torch.float64) + squared = torch.zeros(self.action_dim, dtype=torch.float64) + count = 0 + for result in results: + valid = ~result.action_is_pad.to(dtype=torch.bool) + values = result.action[valid].to(dtype=torch.float64) + if values.numel() == 0: + continue + total += values.sum(dim=0) + squared += values.square().sum(dim=0) + count += int(values.shape[0]) + if count < 2: + raise RuntimeError("At least two valid action transitions are required") + mean = total / count + variance = (squared / count - mean.square()).clamp(min=self.eps**2) + self.mean.copy_(mean.float()) + self.std.copy_(variance.sqrt().float()) + self.count.fill_(count) + self.fitted.fill_(True) + return self + + def normalize( + self, action: torch.Tensor, action_is_pad: torch.Tensor | None = None + ) -> torch.Tensor: + if not bool(self.fitted): + raise RuntimeError("OnlineActionNormalizer must be fitted or restored") + output = (action - self.mean.to(action)) / self.std.to(action) + if self.clip is not None: + output = output.clamp(-self.clip, self.clip) + if action_is_pad is not None: + output = output.masked_fill(action_is_pad.to(output.device).unsqueeze(-1), 0) + return output + + def denormalize(self, action: torch.Tensor) -> torch.Tensor: + if not bool(self.fitted): + raise RuntimeError("OnlineActionNormalizer must be fitted or restored") + return action * self.std.to(action) + self.mean.to(action) + + +class OnlineCameraNormalizer(nn.Module): + """Mean/std normalization for first-camera-relative 6D camera tokens.""" + + def __init__( + self, + camera_dim: int = CAMERA_DIM, + *, + eps: float = 1.0e-6, + clip: float | None = 10.0, + ) -> None: + super().__init__() + self.camera_dim = int(camera_dim) + self.eps = float(eps) + self.clip = None if clip is None else float(clip) + self.register_buffer("mean", torch.zeros(self.camera_dim, dtype=torch.float32)) + self.register_buffer("std", torch.ones(self.camera_dim, dtype=torch.float32)) + self.register_buffer("count", torch.zeros((), dtype=torch.long)) + self.register_buffer("fitted", torch.tensor(False, dtype=torch.bool)) + + @torch.no_grad() + def fit(self, results: Iterable[OnlineActionResult]) -> "OnlineCameraNormalizer": + total = torch.zeros(self.camera_dim, dtype=torch.float64) + squared = torch.zeros(self.camera_dim, dtype=torch.float64) + count = 0 + for result in results: + valid = ~result.camera_is_pad.to(dtype=torch.bool) + values = result.camera[valid].to(dtype=torch.float64) + if values.numel() == 0: + continue + total += values.sum(dim=0) + squared += values.square().sum(dim=0) + count += int(values.shape[0]) + if count < 2: + raise RuntimeError("At least two valid camera poses are required") + mean = total / count + variance = (squared / count - mean.square()).clamp(min=self.eps**2) + self.mean.copy_(mean.float()) + self.std.copy_(variance.sqrt().float()) + self.count.fill_(count) + self.fitted.fill_(True) + return self + + def normalize( + self, camera: torch.Tensor, camera_is_pad: torch.Tensor | None = None + ) -> torch.Tensor: + if not bool(self.fitted): + raise RuntimeError("OnlineCameraNormalizer must be fitted or restored") + output = (camera - self.mean.to(camera)) / self.std.to(camera) + if self.clip is not None: + output = output.clamp(-self.clip, self.clip) + if camera_is_pad is not None: + output = output.masked_fill( + camera_is_pad.to(output.device).unsqueeze(-1), 0 + ) + return output + + def denormalize(self, camera: torch.Tensor) -> torch.Tensor: + if not bool(self.fitted): + raise RuntimeError("OnlineCameraNormalizer must be fitted or restored") + return camera * self.std.to(camera) + self.mean.to(camera) + + +class OnlineEgoverseActionDataset(Dataset): + """Add online normalized actions to the existing latent dataset.""" + + def __init__( + self, + video_dataset: Dataset, + processor: EgoverseOnlineActionProcessor, + normalizer: OnlineActionNormalizer, + camera_normalizer: OnlineCameraNormalizer, + ) -> None: + if not bool(normalizer.fitted) or not bool(camera_normalizer.fitted): + raise RuntimeError("Fit action/camera normalizers before constructing the dataset") + self.video_dataset = video_dataset + self.processor = processor + # Snapshot 14D statistics on CPU. The model-owned normalizer may later + # move to GPU/FSDP; DataLoader workers must remain CPU-only. + self.action_mean = normalizer.mean.detach().float().cpu().clone() + self.action_std = normalizer.std.detach().float().cpu().clone() + self.normalization_clip = normalizer.clip + self.camera_mean = camera_normalizer.mean.detach().float().cpu().clone() + self.camera_std = camera_normalizer.std.detach().float().cpu().clone() + self.camera_normalization_clip = camera_normalizer.clip + + def __len__(self) -> int: + return len(self.video_dataset) + + def __getitem__(self, index: int) -> dict[str, Any]: + sample = dict(self.video_dataset[index]) + if "annotation_path" not in sample or "source_indices" not in sample: + raise KeyError("Video sample needs annotation_path and source_indices") + result = self.processor(sample["annotation_path"], sample["source_indices"]) + action = (result.action - self.action_mean) / self.action_std + if self.normalization_clip is not None: + action = action.clamp(-self.normalization_clip, self.normalization_clip) + sample["action"] = action.masked_fill( + result.action_is_pad.unsqueeze(-1), 0 + ) + sample["action_is_pad"] = result.action_is_pad + camera = (result.camera - self.camera_mean) / self.camera_std + if self.camera_normalization_clip is not None: + camera = camera.clamp( + -self.camera_normalization_clip, self.camera_normalization_clip + ) + sample["camera"] = camera.masked_fill( + result.camera_is_pad.unsqueeze(-1), 0 + ) + sample["camera_is_pad"] = result.camera_is_pad + sample["initial_state"] = result.initial_state + sample["left_initial_pose_reference"] = result.left_initial_pose_reference + sample["right_initial_pose_reference"] = result.right_initial_pose_reference + return sample + + +def fit_online_normalizer( + video_dataset: Dataset, + processor: EgoverseOnlineActionProcessor, + normalizer: OnlineActionNormalizer | None = None, +) -> OnlineActionNormalizer: + """One startup pass over metadata; keeps only 14D statistics in memory.""" + + if normalizer is None: + normalizer = OnlineActionNormalizer( + eps=processor.config.normalization_eps, + clip=processor.config.normalization_clip, + ) + + def results() -> Iterable[OnlineActionResult]: + for index in range(len(video_dataset)): + sample = video_dataset[index] + yield processor(sample["annotation_path"], sample["source_indices"]) + + return normalizer.fit(results()) + + +def fit_online_camera_normalizer( + video_dataset: Dataset, + processor: EgoverseOnlineActionProcessor, + normalizer: OnlineCameraNormalizer | None = None, +) -> OnlineCameraNormalizer: + """Fit camera statistics without persisting derived trajectories.""" + + if normalizer is None: + normalizer = OnlineCameraNormalizer() + + def results() -> Iterable[OnlineActionResult]: + for index in range(len(video_dataset)): + sample = video_dataset[index] + yield processor(sample["annotation_path"], sample["source_indices"]) + + return normalizer.fit(results()) + + +def processor_from_config(config: dict[str, Any]) -> EgoverseOnlineActionProcessor: + """Build the online processor from the shared point/action YAML sections.""" + + point = config.get("point_cloud", {}) + normalization = config.get("action", {}).get("normalization", {}) + transform = point.get("camera_to_head_transform", np.eye(4).tolist()) + return EgoverseOnlineActionProcessor( + OnlineActionConfig( + camera_to_head_transform=tuple( + tuple(float(value) for value in row) for row in transform + ), + normalization_eps=float(normalization.get("eps", 1.0e-6)), + normalization_clip=normalization.get("clip", 10.0), + ) + ) + + +def prepare_online_training_dataset( + video_dataset: Dataset, + model: nn.Module, + config: dict[str, Any], +) -> tuple[OnlineEgoverseActionDataset, EgoverseOnlineActionProcessor]: + """Fit checkpointed statistics if needed and wrap the training dataset.""" + + processor = processor_from_config(config) + action_normalizer = getattr(model, "action_normalizer", None) + camera_normalizer = getattr(model, "camera_normalizer", None) + if not isinstance(action_normalizer, OnlineActionNormalizer): + raise TypeError("Model must own an OnlineActionNormalizer module") + if not isinstance(camera_normalizer, OnlineCameraNormalizer): + raise TypeError("Model must own an OnlineCameraNormalizer module") + if not bool(action_normalizer.fitted) or not bool(camera_normalizer.fitted): + results = [ + processor(sample["annotation_path"], sample["source_indices"]) + for sample in ( + video_dataset[index] for index in range(len(video_dataset)) + ) + ] + if not bool(action_normalizer.fitted): + action_normalizer.fit(results) + if not bool(camera_normalizer.fitted): + camera_normalizer.fit(results) + return OnlineEgoverseActionDataset( + video_dataset, processor, action_normalizer, camera_normalizer + ), processor + + +def collate_online_action_batch(samples: list[dict[str, Any]]) -> dict[str, Any]: + """Collate fixed-horizon online actions and the existing text/latent fields.""" + + if not samples: + raise ValueError("Cannot collate an empty online action batch") + horizons = {int(sample["action"].shape[0]) for sample in samples} + if len(horizons) != 1: + raise ValueError(f"Online action horizons must match, got {sorted(horizons)}") + camera_horizons = {int(sample["camera"].shape[0]) for sample in samples} + if camera_horizons != horizons: + raise ValueError( + f"Camera/action horizons must match, got {sorted(camera_horizons)} " + f"and {sorted(horizons)}" + ) + return collate_rgb_depth_point_batch(samples) + + +@torch.no_grad() +def integrate_normalized_actions( + normalized_action: torch.Tensor, + *, + left_initial_pose_reference: torch.Tensor, + right_initial_pose_reference: torch.Tensor, + normalizer: OnlineActionNormalizer, +) -> dict[str, torch.Tensor]: + """Inference-time denormalization and SE(3) integration in memory.""" + + if normalized_action.ndim != 2 or normalized_action.shape[1] != ACTION_DIM: + raise ValueError(f"Expected normalized action [T,{ACTION_DIM}]") + raw = normalizer.denormalize(normalized_action).float().cpu().numpy() + left = np.asarray(left_initial_pose_reference.float().cpu(), dtype=np.float64) + right = np.asarray(right_initial_pose_reference.float().cpu(), dtype=np.float64) + left_trajectory = [left.copy()] + right_trajectory = [right.copy()] + for action in raw: + left = left @ se3_exp(action[0:6]) + right = right @ se3_exp(action[7:13]) + left_trajectory.append(left.copy()) + right_trajectory.append(right.copy()) + return { + "left_pose_reference": torch.from_numpy( + np.stack(left_trajectory).astype(np.float32) + ), + "right_pose_reference": torch.from_numpy( + np.stack(right_trajectory).astype(np.float32) + ), + "grip_open": torch.from_numpy(raw[:, [6, 13]].astype(np.float32)), + "raw_action": torch.from_numpy(raw.astype(np.float32)), + } + diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/optimizer_step_scheduler.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/optimizer_step_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..9f81332e2198a2754692f73646d46eccecc455a8 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/optimizer_step_scheduler.py @@ -0,0 +1,144 @@ +"""Fail-closed LR scheduler stepping in optimizer-update units.""" + +from __future__ import annotations + +from typing import Any +import warnings + +import torch + + +def position_lr_scheduler_at_optimizer_step( + lr_scheduler: torch.optim.lr_scheduler.LRScheduler, + *, + global_step: int, +) -> None: + """Position a fresh scheduler for a model-only training resume. + + Model-only checkpoints do not contain scheduler state. Reconstructing the + closed-form schedule at the persisted optimizer update keeps warmup from + restarting while preserving the same fail-closed update index contract. + """ + + scheduler = getattr(lr_scheduler, "scheduler", lr_scheduler) + global_step = int(global_step) + if global_step < 0: + raise ValueError(f"global_step must be non-negative, got {global_step}") + if int(scheduler.last_epoch) != 0: + raise RuntimeError( + "A model-only resume can only position a fresh scheduler: " + f"last_epoch={int(scheduler.last_epoch)}" + ) + if global_step == 0: + return + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="The epoch parameter in `scheduler.step\(\)` was not necessary", + ) + warnings.filterwarnings( + "ignore", + message="Detected call of `lr_scheduler.step\(\)` before `optimizer.step\(\)`", + ) + scheduler.step(global_step) + if int(scheduler.last_epoch) != global_step: + raise RuntimeError( + "Failed to reconstruct scheduler update index for model-only resume: " + f"scheduler.last_epoch={int(scheduler.last_epoch)} global_step={global_step}" + ) + + +def step_lr_scheduler_on_optimizer_update( + lr_scheduler: torch.optim.lr_scheduler.LRScheduler, + accelerator: Any, + *, + optimizer: torch.optim.Optimizer | None = None, + expected_global_step: int | None = None, +) -> bool: + """Advance an unwrapped scheduler once iff an optimizer update completed.""" + + scheduler = getattr(lr_scheduler, "scheduler", lr_scheduler) + before = getattr(scheduler, "last_epoch", None) + if expected_global_step is not None and before is not None: + if int(before) != int(expected_global_step): + raise RuntimeError( + "LR scheduler was already misaligned before this micro-step: " + f"scheduler.last_epoch={int(before)} " + f"global_step={int(expected_global_step)}" + ) + + if not accelerator.sync_gradients: + after = getattr(scheduler, "last_epoch", None) + if before is not None and after is not None and int(after) != int(before): + raise RuntimeError( + "LR scheduler advanced during a gradient-accumulation micro-step: " + f"before={int(before)} after={int(after)}" + ) + return False + + # AcceleratedOptimizer exposes this after optimizer.step(). A skipped AMP + # update is not an optimizer update and therefore advances neither counter. + if optimizer is not None and bool(getattr(optimizer, "step_was_skipped", False)): + return False + + lr_scheduler.step() + after = getattr(scheduler, "last_epoch", None) + if before is not None and after is not None and int(after) != int(before) + 1: + raise RuntimeError( + "LR scheduler must advance exactly once per optimizer update: " + f"before={int(before)} after={int(after)}" + ) + if expected_global_step is not None and after is not None: + expected_after = int(expected_global_step) + 1 + if int(after) != expected_after: + raise RuntimeError( + "LR scheduler update index disagrees with the completed optimizer update: " + f"scheduler.last_epoch={int(after)} expected={expected_after}" + ) + return True + + +def assert_lr_scheduler_optimizer_step_index( + lr_scheduler: Any, + *, + global_step: int, + config: dict[str, Any], +) -> None: + """Fail immediately if warmup is being counted in any other unit.""" + + if config["training"].get("scheduler_step_unit") != "optimizer_step": + return + scheduler = getattr(lr_scheduler, "scheduler", lr_scheduler) + actual = int(scheduler.last_epoch) + if actual != int(global_step): + raise RuntimeError( + "LR scheduler is not aligned to optimizer steps: " + f"scheduler.last_epoch={actual} global_step={global_step}" + ) + + +def lr_scheduler_audit_record( + lr_scheduler: Any, + optimizer: torch.optim.Optimizer, + *, + micro_step: int, + global_step: int, + learning_rates_used_for_update: list[float] | None = None, +) -> dict[str, Any]: + """Return the persisted, human-readable scheduler/update audit record.""" + + scheduler = getattr(lr_scheduler, "scheduler", lr_scheduler) + return { + "micro_step": int(micro_step), + "optimizer_global_step": int(global_step), + "scheduler_last_epoch": int(scheduler.last_epoch), + "scheduler_update_index": int(scheduler.last_epoch), + "learning_rates_used_for_update": ( + None + if learning_rates_used_for_update is None + else [float(value) for value in learning_rates_used_for_update] + ), + "learning_rates_after_scheduler_step": [ + float(group["lr"]) for group in optimizer.param_groups + ], + } diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/piper_hdf5.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/piper_hdf5.py new file mode 100644 index 0000000000000000000000000000000000000000..e5668c7abf48f3cb1ed80b2fe9dbfd72a915c8cf --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/piper_hdf5.py @@ -0,0 +1,590 @@ +"""Native Piper HDF5 adapter for the RGB + joint-action FastWAM pipeline. + +The source files follow the ALOHA-style layout used by this workspace: + +* ``observations/images/cam_high``: ``[T, 480, 640, 3]`` RGB uint8 +* ``observations/qpos``: ``[T, 14]`` current dual-arm joint/gripper state +* ``action``: ``[T, 14]`` recorded dual-arm joint/gripper command + +No end-effector conversion is performed. ``action[t]`` is supervised directly +and the first selected ``qpos`` is supplied as the clean 14D action-context +token. The adapter deliberately uses an explicit, validated episode index so +eight distributed ranks do not independently crawl 1.7 TB of HDF5 files. +""" + +from __future__ import annotations + +import atexit +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +from typing import Any, Mapping + +import cv2 +import h5py +import numpy as np +import torch + + +INDEX_FORMAT_VERSION = 1 +ACTION_DIM = 14 +_EPISODE_FILE_PATTERN = re.compile(r"episode_(\d+)\.hdf5$") + + +@dataclass(frozen=True) +class PiperClipConfig: + num_frames: int = 33 + resolution: int = 256 + clips_per_episode: int = 2 + fps: float = 25.0 + video_key: str = "observations/images/cam_high" + color_order: str = "rgb" + + +@dataclass(frozen=True) +class PiperEpisode: + """Compatibility record consumed by the existing exhaustive sampler.""" + + episode_index: int + task: str + length: int + data_chunk_index: int + data_file_index: int + video_chunk_index: int + video_file_index: int + video_from_timestamp: float + video_to_timestamp: float + + +_ACTIVE_CONFIG: dict[str, Any] | None = None +_EPISODE_PATH_BY_INDEX: dict[int, Path] = {} +_HDF5_HANDLES: dict[str, h5py.File] = {} + + +def _close_hdf5_handles() -> None: + for handle in list(_HDF5_HANDLES.values()): + try: + handle.close() + except Exception: + pass + _HDF5_HANDLES.clear() + + +atexit.register(_close_hdf5_handles) + + +def _source_contract(config: Mapping[str, Any]) -> dict[str, Any]: + source = config["piper"] + return { + "tasks": source["tasks"], + "qualities": list(source["qualities"]), + "quality_to_split": source["quality_to_split"], + "exclude_files": sorted(str(value) for value in source.get("exclude_files", [])), + "image_key": str(source["image_key"]), + "state_key": str(source["state_key"]), + "action_key": str(source["action_key"]), + "image_color_order": str(source["image_color_order"]), + "nominal_fps": float(config["data"]["fps"]), + } + + +def selected_task_prompts(config: Mapping[str, Any]) -> dict[str, str]: + tasks = config["piper"]["tasks"] + result: dict[str, str] = {} + for task_id, task_config in tasks.items(): + if not isinstance(task_config, Mapping) or not str(task_config.get("prompt", "")).strip(): + raise ValueError(f"Piper task {task_id!r} must define a nonempty prompt") + result[str(task_id)] = str(task_config["prompt"]) + if not result: + raise ValueError("piper.tasks must not be empty") + if len(set(result.values())) != len(result): + raise ValueError("Every Piper task must use a distinct prompt") + return result + + +def clip_config_from_config(config: Mapping[str, Any]) -> PiperClipConfig: + data, source = config["data"], config["piper"] + return PiperClipConfig( + num_frames=int(data["num_frames"]), + resolution=int(data["resolution"]), + clips_per_episode=int(data.get("clips_per_episode", 1)), + fps=float(data["fps"]), + video_key=str(source["image_key"]), + color_order=str(source["image_color_order"]), + ) + + +def _episode_source_path(root: Path, task_id: str, quality: str, source_index: int) -> Path: + return root / task_id / quality / f"episode_{source_index}.hdf5" + + +def _inspect_episode( + path: Path, + *, + root: Path, + task_id: str, + prompt: str, + quality: str, + split: str, + source_index: int, + image_key: str, + state_key: str, + action_key: str, +) -> dict[str, Any]: + stat = path.stat() + with h5py.File(path, "r") as handle: + required = (image_key, state_key, action_key) + missing = [key for key in required if key not in handle] + if missing: + raise KeyError(f"missing HDF5 fields {missing}") + image = handle[image_key] + state = handle[state_key] + action = handle[action_key] + if image.ndim != 4 or tuple(image.shape[1:]) != (480, 640, 3): + raise ValueError(f"unexpected image shape {image.shape}") + if image.dtype != np.dtype(np.uint8): + raise ValueError(f"unexpected image dtype {image.dtype}") + if tuple(state.shape[1:]) != (ACTION_DIM,) or tuple(action.shape[1:]) != (ACTION_DIM,): + raise ValueError(f"unexpected state/action shapes {state.shape}/{action.shape}") + if state.dtype != np.dtype(np.float32) or action.dtype != np.dtype(np.float32): + raise ValueError(f"unexpected state/action dtypes {state.dtype}/{action.dtype}") + lengths = {int(image.shape[0]), int(state.shape[0]), int(action.shape[0])} + if len(lengths) != 1: + raise ValueError( + f"image/state/action lengths differ: {image.shape[0]}/{state.shape[0]}/{action.shape[0]}" + ) + length = lengths.pop() + if length < 2: + raise ValueError(f"episode length must be at least two, got {length}") + sim = bool(handle.attrs.get("sim", False)) + if sim: + raise ValueError("Piper native adapter accepts real-robot episodes only") + return { + "relative_path": str(path.relative_to(root)), + "task_id": task_id, + "prompt": prompt, + "quality": quality, + "split": split, + "source_episode_index": int(source_index), + "length": int(length), + "size": int(stat.st_size), + "mtime_ns": int(stat.st_mtime_ns), + } + + +def build_episode_index(config: Mapping[str, Any], *, workers: int = 16) -> dict[str, Any]: + """Scan and validate every configured source episode.""" + + if workers <= 0: + raise ValueError("workers must be positive") + root = Path(config["paths"]["dataset_root"]).resolve() + if not root.is_dir(): + raise FileNotFoundError(root) + source = config["piper"] + prompts = selected_task_prompts(config) + qualities = [str(value) for value in source["qualities"]] + quality_to_split = {str(key): str(value) for key, value in source["quality_to_split"].items()} + if set(qualities) != set(quality_to_split): + raise ValueError("piper.quality_to_split must exactly cover piper.qualities") + excluded = {str(value) for value in source.get("exclude_files", [])} + discovered: list[tuple[Path, str, str, str, int]] = [] + excluded_records: list[dict[str, Any]] = [] + for task_id in prompts: + for quality in qualities: + directory = root / task_id / quality + if not directory.is_dir(): + raise FileNotFoundError(directory) + for path in sorted(directory.glob("episode_*.hdf5")): + match = _EPISODE_FILE_PATTERN.fullmatch(path.name) + if match is None: + continue + relative = str(path.relative_to(root)) + source_index = int(match.group(1)) + if relative in excluded: + stat = path.stat() + excluded_records.append( + { + "relative_path": relative, + "reason": "configured_exclusion", + "size": int(stat.st_size), + "mtime_ns": int(stat.st_mtime_ns), + } + ) + continue + discovered.append((path, task_id, prompts[task_id], quality, source_index)) + + records: list[dict[str, Any]] = [] + failures: list[str] = [] + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = { + executor.submit( + _inspect_episode, + path, + root=root, + task_id=task_id, + prompt=prompt, + quality=quality, + split=quality_to_split[quality], + source_index=source_index, + image_key=str(source["image_key"]), + state_key=str(source["state_key"]), + action_key=str(source["action_key"]), + ): path + for path, task_id, prompt, quality, source_index in discovered + } + for number, future in enumerate(as_completed(futures), start=1): + path = futures[future] + try: + records.append(future.result()) + except Exception as error: + failures.append(f"{path}: {type(error).__name__}: {error}") + if number == 1 or number % 50 == 0 or number == len(futures): + print(f"[piper-index] scanned {number}/{len(futures)}", flush=True) + if failures: + preview = "\n".join(failures[:20]) + raise RuntimeError( + f"Piper index scan found {len(failures)} invalid non-excluded files:\n{preview}" + ) + records.sort( + key=lambda entry: ( + list(prompts).index(str(entry["task_id"])), + qualities.index(str(entry["quality"])), + int(entry["source_episode_index"]), + ) + ) + for episode_index, record in enumerate(records): + record["episode_index"] = episode_index + + expected_counts = source.get("expected_episode_counts") + if expected_counts is not None: + actual: dict[str, dict[str, int]] = { + task_id: {quality: 0 for quality in qualities} for task_id in prompts + } + for record in records: + actual[str(record["task_id"])][str(record["quality"])] += 1 + normalized_expected = { + str(task_id): {str(quality): int(count) for quality, count in counts.items()} + for task_id, counts in expected_counts.items() + } + if actual != normalized_expected: + raise RuntimeError(f"Piper episode counts mismatch: actual={actual} expected={normalized_expected}") + + split_counts: dict[str, int] = {name: 0 for name in ("train", "validation", "test")} + split_frames: dict[str, int] = {name: 0 for name in split_counts} + for record in records: + split = str(record["split"]) + if split not in split_counts: + raise ValueError(f"Unsupported split {split!r}") + split_counts[split] += 1 + split_frames[split] += int(record["length"]) + return { + "format_version": INDEX_FORMAT_VERSION, + "created_at": datetime.now(timezone.utc).isoformat(), + "dataset_root": str(root), + "source_contract": _source_contract(config), + "episodes": records, + "excluded_files": sorted(excluded_records, key=lambda entry: entry["relative_path"]), + "episode_count": len(records), + "total_frames": sum(int(record["length"]) for record in records), + "split_episode_counts": split_counts, + "split_frame_counts": split_frames, + } + + +def write_episode_index(config: Mapping[str, Any], *, workers: int = 16) -> Path: + payload = build_episode_index(config, workers=workers) + output = Path(config["paths"]["piper_index_file"]) + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_name(f"{output.name}.tmp.{os.getpid()}") + temporary.write_text(json.dumps(payload, indent=2), encoding="utf-8") + os.replace(temporary, output) + print( + f"[piper-index] wrote episodes={payload['episode_count']} " + f"frames={payload['total_frames']} path={output}", + flush=True, + ) + return output + + +def load_episode_index( + config: Mapping[str, Any], *, verify_source_stats: bool = True +) -> dict[str, Any]: + path = Path(config["paths"]["piper_index_file"]) + if not path.is_file(): + raise FileNotFoundError( + f"Piper episode index is missing: {path}. Run build_piper_hdf5_index.py first." + ) + payload = json.loads(path.read_text(encoding="utf-8")) + root = Path(config["paths"]["dataset_root"]).resolve() + if int(payload.get("format_version", -1)) != INDEX_FORMAT_VERSION: + raise RuntimeError("Unsupported Piper episode-index format") + if payload.get("dataset_root") != str(root): + raise RuntimeError("Piper episode-index dataset root mismatch") + if payload.get("source_contract") != _source_contract(config): + raise RuntimeError("Piper episode-index source contract mismatch") + records = payload.get("episodes") + if not isinstance(records, list) or len(records) != int(payload.get("episode_count", -1)): + raise RuntimeError("Piper episode-index record count mismatch") + if verify_source_stats: + mismatches = [] + for record in records: + source_path = root / str(record["relative_path"]) + try: + stat = source_path.stat() + except FileNotFoundError: + mismatches.append(f"missing:{source_path}") + continue + if int(stat.st_size) != int(record["size"]) or int(stat.st_mtime_ns) != int(record["mtime_ns"]): + mismatches.append(f"changed:{source_path}") + if mismatches: + raise RuntimeError( + "Piper sources changed after indexing: " + ", ".join(mismatches[:20]) + ) + return payload + + +def configure_source(config: dict[str, Any], *, verify_source_stats: bool = True) -> dict[str, Any]: + global _ACTIVE_CONFIG, _EPISODE_PATH_BY_INDEX + payload = load_episode_index(config, verify_source_stats=verify_source_stats) + root = Path(config["paths"]["dataset_root"]).resolve() + _ACTIVE_CONFIG = config + _EPISODE_PATH_BY_INDEX = { + int(record["episode_index"]): root / str(record["relative_path"]) + for record in payload["episodes"] + } + return payload + + +def _record_to_episode(record: Mapping[str, Any], fps: float) -> PiperEpisode: + length = int(record["length"]) + source_index = int(record["source_episode_index"]) + return PiperEpisode( + episode_index=int(record["episode_index"]), + task=str(record["prompt"]), + length=length, + data_chunk_index=0, + data_file_index=source_index, + video_chunk_index=0, + video_file_index=source_index, + video_from_timestamp=0.0, + video_to_timestamp=float(max(length - 1, 0)) / float(fps), + ) + + +def resolve_splits(config: dict[str, Any]) -> dict[str, list[PiperEpisode]]: + payload = configure_source(config) + outputs: dict[str, list[PiperEpisode]] = { + "train": [], + "validation": [], + "test": [], + } + fps = float(config["data"]["fps"]) + for record in payload["episodes"]: + outputs[str(record["split"])].append(_record_to_episode(record, fps)) + return outputs + + +def data_hdf5_path(root: Path, episode: PiperEpisode) -> Path: + try: + path = _EPISODE_PATH_BY_INDEX[int(episode.episode_index)] + except KeyError as error: + raise RuntimeError("Piper source is not configured for this process") from error + resolved_root = Path(root).resolve() + if not path.resolve().is_relative_to(resolved_root): + raise RuntimeError(f"Piper episode path escapes dataset root: {path}") + return path + + +def load_data_shard(path: Path) -> Path: + if not Path(path).is_file(): + raise FileNotFoundError(path) + return Path(path) + + +def episode_arrays(path: Path, episode: PiperEpisode) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + if _ACTIVE_CONFIG is None: + raise RuntimeError("Piper source is not configured") + source = _ACTIVE_CONFIG["piper"] + with h5py.File(path, "r") as handle: + state = np.asarray(handle[str(source["state_key"])][:], dtype=np.float32) + action = np.asarray(handle[str(source["action_key"])][:], dtype=np.float32) + expected = (int(episode.length), ACTION_DIM) + if state.shape != expected or action.shape != expected: + raise RuntimeError(f"Piper state/action shape mismatch: {state.shape}/{action.shape} != {expected}") + if not np.isfinite(state).all() or not np.isfinite(action).all(): + raise RuntimeError(f"Piper state/action contains non-finite values: {path}") + timestamps = np.arange(int(episode.length), dtype=np.float64) / float( + _ACTIVE_CONFIG["data"]["fps"] + ) + return state, action, timestamps + + +def build_joint_command_targets( + state: np.ndarray, + joint_command: np.ndarray, + image_indices: np.ndarray, +) -> dict[str, torch.Tensor]: + """Use recorded Piper joint commands directly, without EE conversion.""" + + state = np.asarray(state, dtype=np.float32) + joint_command = np.asarray(joint_command, dtype=np.float32) + indices = np.asarray(image_indices, dtype=np.int64) + if state.shape != joint_command.shape or state.ndim != 2 or state.shape[1] != ACTION_DIM: + raise ValueError("Piper state/action must have matching [T,14] shapes") + if len(indices) < 2: + raise ValueError("Piper windows must contain at least two frames") + steps = np.diff(indices) + if np.any((steps != 0) & (steps != 1)): + raise ValueError("Piper windows must be consecutive with optional terminal repeats") + if int(indices[0]) < 0 or int(indices[-1]) >= len(state): + raise IndexError("Piper window indices are outside the episode") + repeated = steps == 0 + if np.any(repeated) and np.any(indices[:-1][repeated] != len(state) - 1): + raise ValueError("Only the terminal Piper frame may be repeated") + action = joint_command[indices[:-1]].copy() + initial_state = state[int(indices[0])].copy() + if not np.isfinite(action).all() or not np.isfinite(initial_state).all(): + raise RuntimeError("Piper joint targets contain non-finite values") + return { + "action_raw": torch.from_numpy(action), + "action_is_pad": torch.zeros(len(indices) - 1, dtype=torch.bool), + "initial_state": torch.from_numpy(initial_state), + } + + +def dataset_smoke_test(config: dict[str, Any]) -> None: + splits = resolve_splits(config) + episode = splits["train"][0] + root = Path(config["paths"]["dataset_root"]) + path = data_hdf5_path(root, episode) + state, action, timestamps = episode_arrays(path, episode) + window = int(config["data"]["num_frames"]) + indices = np.arange(window, dtype=np.int64) + targets = build_joint_command_targets(state, action, indices) + rgb = decode_rgb_window( + path, + episode_from_timestamp=0.0, + image_indices=indices, + config=clip_config_from_config(config), + ) + current_error = float(np.mean(np.abs(action[:-1] - state[:-1]))) + next_error = float(np.mean(np.abs(action[:-1] - state[1:]))) + print( + json.dumps( + { + "status": "piper_native_hdf5_joint_action_data_ok", + "episode_index": int(episode.episode_index), + "source_path": str(path), + "task": episode.task, + "split_episode_counts": { + name: len(values) for name, values in splits.items() + }, + "nominal_fps": float(config["data"]["fps"]), + "rgb_shape": list(rgb.shape), + "rgb_range": [float(rgb.min()), float(rgb.max())], + "action_shape": list(targets["action_raw"].shape), + "initial_state_shape": list(targets["initial_state"].shape), + "timestamp_delta_s": float(timestamps[1] - timestamps[0]), + "action_t_qpos_t_mean_abs_error": current_error, + "action_t_qpos_t_plus_1_mean_abs_error": next_error, + "action_is_used_without_ee_conversion": True, + "normalization": "train-weighted-q01-q99-all-14-dims-no-clip", + "sampling": { + "replacement": bool(config["sampling"]["replacement"]), + "shuffle_each_epoch": bool( + config["sampling"]["shuffle_each_epoch"] + ), + }, + "model_inputs": [ + "noisy_cam_high_rgb_with_clean_frame0", + "initial_qpos", + ], + "model_outputs": ["rgb_video", "native_joint_command"], + }, + indent=2, + ), + flush=True, + ) + + +def _cached_hdf5(path: Path) -> h5py.File: + key = str(Path(path).resolve()) + handle = _HDF5_HANDLES.get(key) + if handle is None or not handle.id.valid: + handle = h5py.File(key, "r") + _HDF5_HANDLES[key] = handle + return handle + + +def decode_rgb_window( + path: Path, + *, + episode_from_timestamp: float, + image_indices: np.ndarray, + config: PiperClipConfig, +) -> torch.Tensor: + del episode_from_timestamp + requested = np.asarray(image_indices, dtype=np.int64) + if requested.ndim != 1 or requested.size == 0 or np.any(np.diff(requested) < 0): + raise ValueError("image_indices must be a nonempty nondecreasing sequence") + handle = _cached_hdf5(path) + dataset = handle[config.video_key] + if int(requested[0]) < 0 or int(requested[-1]) >= int(dataset.shape[0]): + raise IndexError("Piper image indices are outside the episode") + unique, inverse = np.unique(requested, return_inverse=True) + if np.array_equal(unique, np.arange(int(unique[0]), int(unique[-1]) + 1)): + source = np.asarray(dataset[int(unique[0]) : int(unique[-1]) + 1]) + frames = source[inverse] + else: + frames = np.asarray(dataset[unique.tolist()])[inverse] + if config.color_order == "bgr": + frames = frames[..., ::-1] + elif config.color_order != "rgb": + raise ValueError(f"Unsupported Piper image color order {config.color_order!r}") + processed = [] + for frame in frames: + height, width = frame.shape[:2] + crop = min(height, width) + top, left = (height - crop) // 2, (width - crop) // 2 + square = frame[top : top + crop, left : left + crop] + processed.append( + cv2.resize( + square, + (config.resolution, config.resolution), + interpolation=cv2.INTER_AREA, + ) + ) + array = np.stack(processed).astype(np.float32) / np.float32(127.5) - np.float32(1.0) + return torch.from_numpy(array).permute(3, 0, 1, 2).contiguous() + + +def cache_fingerprint(config: Mapping[str, Any], splits: Mapping[str, list[Any]]) -> str: + index_path = Path(config["paths"]["piper_index_file"]) + index_bytes = index_path.read_bytes() + records = [ + { + "split": split, + "episode_index": int(episode.episode_index), + "task": str(episode.task), + "length": int(episode.length), + } + for split in ("train", "validation", "test") + for episode in splits[split] + ] + payload = { + "format": "piper_native_hdf5_joint_action_fastwam_v1", + "index_sha256": hashlib.sha256(index_bytes).hexdigest(), + "episodes": records, + "data": config["data"], + "piper": config["piper"], + "action": config["action"], + "sampling": config["sampling"], + "pretrained_model_dir": config["paths"]["pretrained_model_dir"], + "modalities": ["rgb", "joint_action", "joint_initial_state"], + } + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/post_training_test.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/post_training_test.py new file mode 100644 index 0000000000000000000000000000000000000000..94caf8ffe1565ff39243885dcacd245d26555b3c --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/post_training_test.py @@ -0,0 +1,746 @@ +#!/usr/bin/env python3 +"""Evaluate a trained joint Wan RGB-depth checkpoint on 20 fixed train samples. + +The script performs full diffusion sampling, decodes RGB and depth with the +frozen shared Wan VAE, writes every GT/generated frame and video, computes +pixel/depth metrics, and optionally uploads the result folder to a Hugging Face +dataset repository. It never uploads source NPZ files or model checkpoints. +""" + +from __future__ import annotations + +import argparse +import csv +import gc +import json +import math +import os +from pathlib import Path +import shutil +from typing import Any + +import cv2 +import numpy as np +import torch +import yaml + +from train_joint_rgb_depth import ( + CachedRgbDepthLatents, + build_joint_dit, + config_fingerprint, + dtype_from_name, + load_config, + load_vae, + sample_joint_latents, + validate_config, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--checkpoint", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--skip-upload", action="store_true") + parser.add_argument("--overwrite", action="store_true") + return parser.parse_args() + + +def prepare_output_directory(path: Path, overwrite: bool) -> None: + if path.exists() and any(path.iterdir()): + if not overwrite: + raise FileExistsError( + f"Output directory is not empty: {path}. Pass --overwrite to replace it." + ) + shutil.rmtree(path) + path.mkdir(parents=True, exist_ok=True) + + +def write_rgb_png(path: Path, image_rgb: np.ndarray) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + image = np.asarray(image_rgb, dtype=np.uint8) + if image.ndim == 2: + image = np.repeat(image[..., None], 3, axis=-1) + if image.ndim != 3 or image.shape[-1] != 3: + raise ValueError(f"Expected RGB image, got {image.shape}") + if not cv2.imwrite(str(path), cv2.cvtColor(image, cv2.COLOR_RGB2BGR)): + raise RuntimeError(f"Failed to write PNG: {path}") + + +def write_gray_png(path: Path, image_gray: np.ndarray) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + image = np.asarray(image_gray, dtype=np.uint8) + if image.ndim != 2: + raise ValueError(f"Expected grayscale image, got {image.shape}") + if not cv2.imwrite(str(path), image): + raise RuntimeError(f"Failed to write PNG: {path}") + + +def write_rgb_video(path: Path, frames_rgb: np.ndarray, fps: int) -> None: + frames = np.asarray(frames_rgb, dtype=np.uint8) + if frames.ndim != 4 or frames.shape[-1] != 3: + raise ValueError(f"Expected [T,H,W,3] frames, got {frames.shape}") + path.parent.mkdir(parents=True, exist_ok=True) + height, width = frames.shape[1:3] + writer = cv2.VideoWriter( + str(path), + cv2.VideoWriter_fourcc(*"mp4v"), + float(fps), + (width, height), + ) + if not writer.isOpened(): + raise RuntimeError(f"Failed to open MP4 writer: {path}") + try: + for frame in frames: + writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)) + finally: + writer.release() + if not path.is_file() or path.stat().st_size == 0: + raise RuntimeError(f"Empty MP4 output: {path}") + + +def depth_gray_to_color(gray_uint8: np.ndarray) -> np.ndarray: + colored_bgr = cv2.applyColorMap(gray_uint8, cv2.COLORMAP_TURBO) + return cv2.cvtColor(colored_bgr, cv2.COLOR_BGR2RGB) + + +def labeled_panel(image_rgb: np.ndarray, label: str, conditional: bool) -> np.ndarray: + height, width = image_rgb.shape[:2] + header_height = 28 + panel = np.zeros((height + header_height, width, 3), dtype=np.uint8) + panel[header_height:] = image_rgb + panel[:header_height] = (115, 30, 30) if conditional else (25, 25, 25) + cv2.putText( + panel, + label, + (5, 19), + cv2.FONT_HERSHEY_SIMPLEX, + 0.42, + (255, 255, 255), + 1, + cv2.LINE_AA, + ) + return panel + + +def make_comparison_frame( + gt_rgb: np.ndarray, + generated_rgb: np.ndarray, + gt_depth_gray: np.ndarray, + generated_depth_gray: np.ndarray, + frame_index: int, + depth_first_frame_conditioned: bool, +) -> np.ndarray: + rgb_conditional = frame_index == 0 + depth_conditional = rgb_conditional and depth_first_frame_conditioned + rgb_suffix = " | CONDITIONAL" if rgb_conditional else " | GENERATED TARGET" + depth_suffix = " | CONDITIONAL" if depth_conditional else " | GENERATED TARGET" + panels = [ + labeled_panel(gt_rgb, f"GT RGB f={frame_index:02d}{rgb_suffix}", rgb_conditional), + labeled_panel( + generated_rgb, + f"GEN RGB f={frame_index:02d}{rgb_suffix}", + rgb_conditional, + ), + labeled_panel( + depth_gray_to_color(gt_depth_gray), + f"GT DEPTH f={frame_index:02d}{depth_suffix}", + depth_conditional, + ), + labeled_panel( + depth_gray_to_color(generated_depth_gray), + f"GEN DEPTH f={frame_index:02d}{depth_suffix}", + depth_conditional, + ), + ] + return np.concatenate(panels, axis=1) + + +def tensor_rgb_to_uint8(decoded: torch.Tensor) -> np.ndarray: + if tuple(decoded.shape[:2]) != (1, 3): + raise ValueError(f"Unexpected decoded RGB shape: {tuple(decoded.shape)}") + return ( + ((decoded[0].float().clamp(-1, 1) + 1.0) * 127.5) + .round() + .to(torch.uint8) + .permute(1, 2, 3, 0) + .cpu() + .numpy() + ) + + +def decoded_depth_to_gray(decoded: torch.Tensor) -> tuple[np.ndarray, float]: + if tuple(decoded.shape[:2]) != (1, 3): + raise ValueError(f"Unexpected decoded depth shape: {tuple(decoded.shape)}") + depth_channels = decoded[0].float().clamp(-1, 1) + disagreement = float((depth_channels.std(dim=0, correction=0) / 2.0).mean()) + gray = ((depth_channels.mean(dim=0) + 1.0) / 2.0).clamp(0, 1) + return gray.cpu().numpy(), disagreement + + +def normalize_depth_valid_mask( + valid_mask: np.ndarray, + expected_shape: tuple[int, ...], +) -> np.ndarray: + """Return a boolean [T,H,W] mask matching decoded depth arrays.""" + + valid = np.asarray(valid_mask, dtype=bool) + if valid.ndim == 4 and valid.shape[1] == 1: + valid = valid[:, 0] + elif valid.ndim == 4 and valid.shape[-1] == 1: + valid = valid[..., 0] + if valid.shape != expected_shape: + raise ValueError( + f"Depth valid mask shape {valid.shape} does not match depth " + f"shape {expected_shape}" + ) + return valid + + +def calculate_metrics( + gt_rgb_uint8: np.ndarray, + generated_rgb_uint8: np.ndarray, + gt_depth_01: np.ndarray, + generated_depth_01: np.ndarray, + valid_mask: np.ndarray, + depth_min: float, + depth_max: float, + three_channel_disagreement: float, + depth_first_frame_conditioned: bool = True, +) -> dict[str, float]: + gt_rgb = gt_rgb_uint8.astype(np.float32) / 255.0 + generated_rgb = generated_rgb_uint8.astype(np.float32) / 255.0 + rgb_error = generated_rgb - gt_rgb + rgb_rmse = float(np.sqrt(np.mean(np.square(rgb_error)))) + future_rgb_error = rgb_error[1:] + future_rgb_rmse = float(np.sqrt(np.mean(np.square(future_rgb_error)))) + + depth_span = depth_max - depth_min + gt_depth_m = gt_depth_01.astype(np.float32) * depth_span + depth_min + generated_depth_m = generated_depth_01.astype(np.float32) * depth_span + depth_min + depth_error = generated_depth_m - gt_depth_m + valid = normalize_depth_valid_mask(valid_mask, depth_error.shape) + if not np.any(valid): + raise ValueError("Depth valid mask is empty") + future_valid = valid[1:] + if not np.any(future_valid): + raise ValueError("Future depth valid mask is empty") + depth_target_error = depth_error[1:] if depth_first_frame_conditioned else depth_error + depth_target_valid = valid[1:] if depth_first_frame_conditioned else valid + if not np.any(depth_target_valid): + raise ValueError("Generated depth target valid mask is empty") + temporal_valid = valid[1:] & valid[:-1] + if not np.any(temporal_valid): + raise ValueError("Temporal depth valid mask is empty") + + depth_delta_error = ( + generated_depth_m[1:] + - generated_depth_m[:-1] + - (gt_depth_m[1:] - gt_depth_m[:-1]) + ) + conditional_rgb_error = rgb_error[0] + depth_frame0_error = depth_error[0][valid[0]] + return { + "rgb_l1": float(np.mean(np.abs(rgb_error))), + "rgb_rmse": rgb_rmse, + "rgb_psnr_db": float(20.0 * math.log10(1.0 / max(rgb_rmse, 1.0e-12))), + "rgb_future_l1": float(np.mean(np.abs(future_rgb_error))), + "rgb_future_psnr_db": float( + 20.0 * math.log10(1.0 / max(future_rgb_rmse, 1.0e-12)) + ), + "rgb_conditional_l1": float(np.mean(np.abs(conditional_rgb_error))), + "depth_l1_m": float(np.mean(np.abs(depth_error[valid]))), + "depth_rmse_m": float(np.sqrt(np.mean(np.square(depth_error[valid])))), + "depth_future_l1_m": float(np.mean(np.abs(depth_error[1:][future_valid]))), + "depth_future_rmse_m": float( + np.sqrt(np.mean(np.square(depth_error[1:][future_valid]))) + ), + "depth_generated_l1_m": float( + np.mean(np.abs(depth_target_error[depth_target_valid])) + ), + "depth_generated_rmse_m": float( + np.sqrt(np.mean(np.square(depth_target_error[depth_target_valid]))) + ), + "depth_frame0_l1_m": float(np.mean(np.abs(depth_frame0_error))), + "depth_frame0_rmse_m": float( + np.sqrt(np.mean(np.square(depth_frame0_error))) + ), + "depth_temporal_difference_l1_m": float( + np.mean(np.abs(depth_delta_error[temporal_valid])) + ), + "depth_three_channel_disagreement": three_channel_disagreement, + "depth_valid_fraction": float(np.mean(valid)), + } + + +def save_sample_visualization( + sample_dir: Path, + sample_id: str, + sample_index: int, + gt_rgb_uint8: np.ndarray, + generated_rgb_uint8: np.ndarray, + gt_depth_01: np.ndarray, + generated_depth_01: np.ndarray, + valid_mask: np.ndarray, + source_indices: np.ndarray, + metrics: dict[str, float], + config: dict[str, Any], +) -> None: + post_config = config["post_training_test"] + fps = int(post_config["fps"]) + frame_count = int(gt_rgb_uint8.shape[0]) + expected_frames = int(config["data"]["num_frames"]) + depth_first_frame_conditioned = ( + config["conditioning"]["depth_condition_mode"] == "first_frame_oracle" + ) + valid_mask = normalize_depth_valid_mask(valid_mask, gt_depth_01.shape) + shapes = { + "gt_rgb": gt_rgb_uint8.shape, + "generated_rgb": generated_rgb_uint8.shape, + "gt_depth": gt_depth_01.shape, + "generated_depth": generated_depth_01.shape, + "valid_mask": valid_mask.shape, + } + if frame_count != expected_frames or any(shape[0] != frame_count for shape in shapes.values()): + raise ValueError(f"Frame-count mismatch for {sample_id}: {shapes}") + + gt_depth_gray = np.rint(np.clip(gt_depth_01, 0, 1) * 255.0).astype(np.uint8) + generated_depth_gray = np.rint( + np.clip(generated_depth_01, 0, 1) * 255.0 + ).astype(np.uint8) + invalid = ~valid_mask.astype(bool) + gt_depth_gray = gt_depth_gray.copy() + generated_depth_gray = generated_depth_gray.copy() + gt_depth_gray[invalid] = 0 + generated_depth_gray[invalid] = 0 + + comparison_frames = np.stack( + [ + make_comparison_frame( + gt_rgb_uint8[index], + generated_rgb_uint8[index], + gt_depth_gray[index], + generated_depth_gray[index], + index, + depth_first_frame_conditioned, + ) + for index in range(frame_count) + ] + ) + + if bool(post_config["save_individual_frames"]): + for index in range(frame_count): + filename = f"frame_{index:03d}.png" + write_rgb_png(sample_dir / "frames" / "gt_rgb" / filename, gt_rgb_uint8[index]) + write_rgb_png( + sample_dir / "frames" / "generated_rgb" / filename, + generated_rgb_uint8[index], + ) + write_gray_png( + sample_dir / "frames" / "gt_depth_gray" / filename, + gt_depth_gray[index], + ) + write_gray_png( + sample_dir / "frames" / "generated_depth_gray" / filename, + generated_depth_gray[index], + ) + write_rgb_png( + sample_dir / "frames" / "comparison" / filename, + comparison_frames[index], + ) + + conditional_dir = sample_dir / "conditional_frame" + write_rgb_png(conditional_dir / "gt_rgb.png", gt_rgb_uint8[0]) + write_rgb_png(conditional_dir / "generated_rgb.png", generated_rgb_uint8[0]) + if depth_first_frame_conditioned: + write_gray_png(conditional_dir / "gt_depth_gray.png", gt_depth_gray[0]) + write_gray_png( + conditional_dir / "generated_depth_gray.png", generated_depth_gray[0] + ) + write_rgb_png(conditional_dir / "comparison.png", comparison_frames[0]) + + if bool(post_config["save_videos"]): + gt_depth_rgb = np.repeat(gt_depth_gray[..., None], 3, axis=-1) + generated_depth_rgb = np.repeat( + generated_depth_gray[..., None], 3, axis=-1 + ) + write_rgb_video(sample_dir / "gt_rgb.mp4", gt_rgb_uint8, fps) + write_rgb_video( + sample_dir / "generated_rgb.mp4", generated_rgb_uint8, fps + ) + write_rgb_video(sample_dir / "gt_depth_gray.mp4", gt_depth_rgb, fps) + write_rgb_video( + sample_dir / "generated_depth_gray.mp4", generated_depth_rgb, fps + ) + write_rgb_video(sample_dir / "comparison.mp4", comparison_frames, fps) + + if bool(post_config["save_depth_numpy"]): + depth_config = config["data"]["depth"] + depth_min = float(depth_config["min"]) + depth_max = float(depth_config["max"]) + np.save(sample_dir / "gt_depth_m.npy", gt_depth_01 * (depth_max - depth_min) + depth_min) + np.save( + sample_dir / "generated_depth_m.npy", + generated_depth_01 * (depth_max - depth_min) + depth_min, + ) + np.save(sample_dir / "depth_valid_mask.npy", valid_mask.astype(bool)) + + frame_manifest = [ + { + "frame_index": index, + "source_rgb_frame_index": int(source_indices[index]), + "rgb_role": "conditional" if index == 0 else "generated_target", + "depth_role": "conditional" if index == 0 and depth_first_frame_conditioned else "generated_target", + "role": "rgb_condition_depth_target" if index == 0 and not depth_first_frame_conditioned else ("conditional" if index == 0 else "generated_target"), + "gt_rgb": f"frames/gt_rgb/frame_{index:03d}.png", + "generated_rgb": f"frames/generated_rgb/frame_{index:03d}.png", + "gt_depth_gray": f"frames/gt_depth_gray/frame_{index:03d}.png", + "generated_depth_gray": f"frames/generated_depth_gray/frame_{index:03d}.png", + "comparison": f"frames/comparison/frame_{index:03d}.png", + } + for index in range(frame_count) + ] + with (sample_dir / "frames.json").open("w", encoding="utf-8") as handle: + json.dump(frame_manifest, handle, indent=2) + with (sample_dir / "metrics.json").open("w", encoding="utf-8") as handle: + json.dump( + { + "sample_index": sample_index, + "sample_id": sample_id, + "rgb_conditional_frame_index": 0, + "depth_conditional_frame_index": 0 if depth_first_frame_conditioned else None, + "rgb_generated_target_frame_indices": list(range(1, frame_count)), + "depth_generated_target_frame_indices": list(range(1 if depth_first_frame_conditioned else 0, frame_count)), + "source_rgb_frame_indices": [int(value) for value in source_indices], + **metrics, + }, + handle, + indent=2, + ) + + +def write_summary( + output_dir: Path, + config: dict[str, Any], + checkpoint: Path, + results: list[dict[str, Any]], +) -> None: + metric_keys = [ + key + for key, value in results[0].items() + if key not in {"sample_index", "sample_id"} and isinstance(value, float) + ] + frame_count = int(config["data"]["num_frames"]) + depth_first_frame_conditioned = ( + config["conditioning"]["depth_condition_mode"] == "first_frame_oracle" + ) + aggregate = { + key: { + "mean": float(np.mean([result[key] for result in results])), + "min": float(np.min([result[key] for result in results])), + "max": float(np.max([result[key] for result in results])), + } + for key in metric_keys + } + summary = { + "format_version": 1, + "experiment": config["experiment"]["name"], + "checkpoint": checkpoint.name, + "num_samples": len(results), + "sample_indices": [result["sample_index"] for result in results], + "rgb_conditional_frame_index": 0, + "depth_conditional_frame_index": 0 if depth_first_frame_conditioned else None, + "rgb_generated_target_frame_indices": list(range(1, frame_count)), + "depth_generated_target_frame_indices": list(range(1 if depth_first_frame_conditioned else 0, frame_count)), + "aggregate": aggregate, + "samples": results, + } + with (output_dir / "summary.json").open("w", encoding="utf-8") as handle: + json.dump(summary, handle, indent=2) + with (output_dir / "metrics.csv").open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=list(results[0])) + writer.writeheader() + writer.writerows(results) + public_config = { + key: value for key, value in config.items() if not key.startswith("_") + } + public_config["paths"] = { + "pretrained_model": Path(config["paths"]["pretrained_model_dir"]).name, + "dataset": "Egoverse_3D/episodes/Egoverse/chunk-000", + "cache": Path(config["paths"]["cache_dir"]).name, + } + with (output_dir / "resolved_config.yaml").open("w", encoding="utf-8") as handle: + yaml.safe_dump(public_config, handle, sort_keys=False) + + rows = [] + for result in results: + sample_id = result["sample_id"] + rows.append( + "| {sample} | [{video}](samples/{sample}/comparison.mp4) | " + "[{condition}](samples/{sample}/conditional_frame/comparison.png) | " + "{rgb:.3f} | {depth:.3f} |".format( + sample=sample_id, + video="video", + condition="frame 0", + rgb=result["rgb_future_psnr_db"], + depth=result["depth_generated_rmse_m"], + ) + ) + readme = f"""--- +pretty_name: Wan RGB-Depth Fixed-20 Post-Training Evaluation +--- + +# Wan RGB-Depth fixed-20 evaluation + +- Experiment: `{config['experiment']['name']}` +- Checkpoint: `{checkpoint.name}` +- Test set: 20 fixed training samples from 10 uniformly spaced episodes +- Frames per sample: {config['data']['num_frames']} +- RGB conditional frame: frame 0 +- Depth conditional frame: {'frame 0' if depth_first_frame_conditioned else 'none'} +- RGB generated target frames: 1-{frame_count - 1} +- Depth generated target frames: {'1-' if depth_first_frame_conditioned else '0-'}{frame_count - 1} +- FPS: {config['post_training_test']['fps']} + +Every sample directory contains every GT frame, every generated frame, explicit +RGB conditional-frame images, RGB/depth videos, metric depth arrays, a frame +manifest, and numerical metrics. Invalid GT depth pixels are excluded from +metrics and displayed as black in depth visualizations. + +| Sample | Comparison video | Frame-0 roles | Future RGB PSNR (dB) | Generated depth RMSE (m) | +| --- | --- | --- | ---: | ---: | +{chr(10).join(rows)} + +Aggregate metrics are available in `summary.json`; per-sample values are in +`metrics.csv` and `samples/*/metrics.json`. +""" + (output_dir / "README.md").write_text(readme, encoding="utf-8") + + +def upload_to_huggingface( + output_dir: Path, + config: dict[str, Any], + checkpoint: Path, +) -> str: + try: + from huggingface_hub import HfApi, get_token + except ImportError as error: + raise RuntimeError( + "Hugging Face upload is enabled but huggingface_hub is not installed" + ) from error + + hf_config = config["post_training_test"]["huggingface"] + repo_id = hf_config.get("repo_id") + if not repo_id: + repo_id = os.environ.get(str(hf_config["repo_id_env"])) + if not repo_id: + raise RuntimeError( + f"Set {hf_config['repo_id_env']} or post_training_test.huggingface.repo_id" + ) + token = os.environ.get(str(hf_config["token_env"])) or get_token() + if not token: + raise RuntimeError( + f"No Hugging Face token found in {hf_config['token_env']} or local login" + ) + + api = HfApi(token=token) + api.create_repo( + repo_id=repo_id, + repo_type=str(hf_config["repo_type"]), + private=bool(hf_config["private"]), + exist_ok=True, + ) + path_in_repo = "/".join( + ( + str(hf_config["path_prefix"]).strip("/"), + str(config["experiment"]["name"]), + checkpoint.stem, + ) + ) + commit = api.upload_folder( + repo_id=repo_id, + repo_type=str(hf_config["repo_type"]), + folder_path=output_dir, + path_in_repo=path_in_repo, + commit_message=str(hf_config["commit_message"]), + token=token, + ) + result = { + "repo_id": repo_id, + "repo_type": hf_config["repo_type"], + "path_in_repo": path_in_repo, + "commit_url": str(commit.commit_url), + } + with (output_dir / "huggingface_upload.json").open("w", encoding="utf-8") as handle: + json.dump(result, handle, indent=2) + return str(commit.commit_url) + + +def run_test(args: argparse.Namespace) -> None: + config = validate_config(load_config(args.config)) + post_config = config.get("post_training_test", {}) + if not bool(post_config.get("enabled", False)): + raise ValueError("post_training_test.enabled must be true") + if not args.checkpoint.is_file(): + raise FileNotFoundError(args.checkpoint) + cache_dir = Path(config["paths"]["cache_dir"]) + if not (cache_dir / "manifest.json").is_file(): + raise FileNotFoundError(f"Missing latent cache: {cache_dir / 'manifest.json'}") + if not torch.cuda.is_available() and str(args.device).startswith("cuda"): + raise RuntimeError("CUDA is required for the real 5B post-training test") + + prepare_output_directory(args.output_dir, args.overwrite) + device = torch.device(args.device) + dtype = dtype_from_name(str(config["model"]["dtype"])) + dataset = CachedRgbDepthLatents(cache_dir, config_fingerprint(config)) + selected_indices = [int(index) for index in post_config["sample_indices"]] + + print(f"[post-test] strict-loading {args.checkpoint}", flush=True) + model = build_joint_dit(config, args.checkpoint).to(device=device, dtype=dtype) + model.use_gradient_checkpointing = False + model.eval().requires_grad_(False) + + latent_dir = args.output_dir / "latents" + latent_dir.mkdir(parents=True, exist_ok=True) + latent_results: list[dict[str, Any]] = [] + with torch.inference_mode(): + for ordinal, sample_index in enumerate(selected_indices): + sample = dataset[sample_index] + z_rgb = sample["z_rgb"].unsqueeze(0).to(device=device, dtype=dtype) + z_depth = sample["z_depth"].unsqueeze(0).to(device=device, dtype=dtype) + context = sample["context"].unsqueeze(0).to(device=device, dtype=dtype) + context_mask = sample["context_mask"].unsqueeze(0).to( + device=device, dtype=torch.bool + ) + generated_rgb, generated_depth = sample_joint_latents( + model, + z_rgb, + z_depth, + context, + context_mask, + config, + seed=int(config["validation"]["fixed_noise_seed"]) + sample_index, + ) + payload = { + "sample_index": sample_index, + "sample_id": sample["sample_id"], + "prompt": sample["prompt"], + "generated_rgb": generated_rgb[0].cpu(), + "generated_depth": generated_depth[0].cpu(), + } + torch.save(payload, latent_dir / f"{ordinal:02d}_{sample['sample_id']}.pt") + latent_results.append(payload) + print( + f"[post-test sample {ordinal + 1:02d}/{len(selected_indices):02d}] " + f"{sample['sample_id']}", + flush=True, + ) + + model.to("cpu") + del model, context, context_mask + gc.collect() + torch.cuda.empty_cache() + + print("[post-test] loading frozen Wan VAE for visualization", flush=True) + vae = load_vae(Path(config["paths"]["pretrained_model_dir"]), dtype, device) + depth_config = config["data"]["depth"] + depth_min = float(depth_config["min"]) + depth_max = float(depth_config["max"]) + results: list[dict[str, Any]] = [] + + with torch.inference_mode(): + for ordinal, latent_payload in enumerate(latent_results): + sample_index = int(latent_payload["sample_index"]) + sample_id = str(latent_payload["sample_id"]) + cache_entry = dataset.samples[sample_index] + cached = torch.load( + cache_dir / cache_entry["file"], + map_location="cpu", + weights_only=True, + ) + generated_rgb_decoded = vae.decode( + latent_payload["generated_rgb"].unsqueeze(0).to(device=device, dtype=dtype), + device=device, + ).float() + generated_depth_decoded = vae.decode( + latent_payload["generated_depth"].unsqueeze(0).to( + device=device, dtype=dtype + ), + device=device, + ).float() + generated_rgb_uint8 = tensor_rgb_to_uint8(generated_rgb_decoded) + generated_depth_01, disagreement = decoded_depth_to_gray( + generated_depth_decoded + ) + gt_rgb_uint8 = ( + cached["rgb_uint8"].permute(1, 2, 3, 0).contiguous().numpy() + ) + gt_depth_01 = cached["depth_gray"][0].float().numpy() + valid_mask = cached["depth_valid_mask"].bool().numpy() + metrics = calculate_metrics( + gt_rgb_uint8, + generated_rgb_uint8, + gt_depth_01, + generated_depth_01, + valid_mask, + depth_min, + depth_max, + disagreement, + depth_first_frame_conditioned=( + config["conditioning"]["depth_condition_mode"] == "first_frame_oracle" + ), + ) + sample_dir = args.output_dir / "samples" / sample_id + sample_dir.mkdir(parents=True, exist_ok=True) + (sample_dir / "prompt.txt").write_text( + str(latent_payload["prompt"]) + "\n", encoding="utf-8" + ) + save_sample_visualization( + sample_dir, + sample_id, + sample_index, + gt_rgb_uint8, + generated_rgb_uint8, + gt_depth_01, + generated_depth_01, + valid_mask, + cached["source_indices"].numpy(), + metrics, + config, + ) + result = { + "sample_index": sample_index, + "sample_id": sample_id, + "prompt": str(latent_payload["prompt"]), + **metrics, + } + results.append(result) + print( + f"[visualize {ordinal + 1:02d}/{len(latent_results):02d}] " + f"{sample_id} depth_rmse={metrics['depth_generated_rmse_m']:.4f}m", + flush=True, + ) + del generated_rgb_decoded, generated_depth_decoded, cached + + vae.to("cpu") + del vae + gc.collect() + torch.cuda.empty_cache() + write_summary(args.output_dir, config, args.checkpoint, results) + + hf_enabled = bool(post_config["huggingface"].get("enabled", False)) + if hf_enabled and not args.skip_upload: + commit_url = upload_to_huggingface(args.output_dir, config, args.checkpoint) + print(f"[huggingface] uploaded: {commit_url}", flush=True) + elif hf_enabled: + print("[huggingface] upload skipped by --skip-upload", flush=True) + print(f"[post-test] complete: {args.output_dir}", flush=True) + + +def main() -> None: + run_test(parse_args()) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/post_training_test_action_mot.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/post_training_test_action_mot.py new file mode 100644 index 0000000000000000000000000000000000000000..4df962b0918ff7e1bc48d44ed92d70f54ca353c9 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/post_training_test_action_mot.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 +"""Fixed-20 RGB/depth/point/action/camera evaluation for Action MoT.""" + +from __future__ import annotations + +import argparse +import gc +import json +from pathlib import Path +from typing import Any + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import torch + +from joint_rgb_depth_point_action_diffusion import sample_joint_latents_and_actions +from online_egoverse_action import ( + integrate_normalized_actions, + prepare_online_training_dataset, + so3_exp, + so3_log, +) +from post_training_test import ( + calculate_metrics, + decoded_depth_to_gray, + prepare_output_directory, + save_sample_visualization, + tensor_rgb_to_uint8, + write_summary, +) +from post_training_test_rgb_depth_point import ( + _point_depth_consistency, + _valid_t_hw, + export_direct_point_video, + write_gt_pred_three_modal_video, +) +from rgb_depth_point_action_model_factory import build_joint_action_mot +from train_joint_rgb_depth import WanContinuousFlowMatchScheduler, dtype_from_name, load_config, load_vae +from train_joint_rgb_depth_point import CachedRgbDepthPointLatents, point_config_fingerprint +from train_joint_rgb_depth_point_action import validate_config +from world_point_cloud import ( + calculate_world_point_cloud_metrics, + denormalize_world_point_video, + depth_video_to_world_grid, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--checkpoint", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--sample-indices", type=int, nargs="+") + parser.add_argument("--overwrite", action="store_true") + return parser.parse_args() + + +def _rotation_error_deg(predicted: np.ndarray, target: np.ndarray) -> np.ndarray: + errors = [] + for pred, gt in zip(predicted, target, strict=True): + relative = so3_exp(pred).T @ so3_exp(gt) + errors.append(np.linalg.norm(so3_log(relative)) * 180.0 / np.pi) + return np.asarray(errors, dtype=np.float64) + + +def _pose_rotation_error_deg(predicted: np.ndarray, target: np.ndarray) -> np.ndarray: + errors = [] + for pred, gt in zip(predicted, target, strict=True): + errors.append( + np.linalg.norm(so3_log(pred[:3, :3].T @ gt[:3, :3])) + * 180.0 + / np.pi + ) + return np.asarray(errors, dtype=np.float64) + + +def _trajectory_metrics( + generated_action: dict[str, torch.Tensor], + target_action: dict[str, torch.Tensor], + generated_camera: np.ndarray, + target_camera: np.ndarray, + action_valid: np.ndarray, + camera_valid: np.ndarray, +) -> dict[str, float]: + pred_left = generated_action["left_pose_reference"].numpy() + pred_right = generated_action["right_pose_reference"].numpy() + gt_left = target_action["left_pose_reference"].numpy() + gt_right = target_action["right_pose_reference"].numpy() + # Transition validity maps to pose indices 1..T. Frame zero is clean context. + pose_valid = np.concatenate(([True], action_valid)) + left_position = np.linalg.norm(pred_left[:, :3, 3] - gt_left[:, :3, 3], axis=1) + right_position = np.linalg.norm(pred_right[:, :3, 3] - gt_right[:, :3, 3], axis=1) + left_rotation = _pose_rotation_error_deg(pred_left, gt_left) + right_rotation = _pose_rotation_error_deg(pred_right, gt_right) + pred_grip = generated_action["grip_open"].numpy() + gt_grip = target_action["grip_open"].numpy() + camera_translation = np.linalg.norm( + generated_camera[:, :3] - target_camera[:, :3], axis=1 + ) + camera_rotation = _rotation_error_deg( + generated_camera[:, 3:], target_camera[:, 3:] + ) + return { + "action_left_position_rmse_m": float(np.sqrt(np.mean(left_position[pose_valid] ** 2))), + "action_right_position_rmse_m": float(np.sqrt(np.mean(right_position[pose_valid] ** 2))), + "action_left_rotation_mean_deg": float(np.mean(left_rotation[pose_valid])), + "action_right_rotation_mean_deg": float(np.mean(right_rotation[pose_valid])), + "action_grip_mae": float(np.mean(np.abs(pred_grip[action_valid] - gt_grip[action_valid]))), + "camera_translation_rmse_m": float( + np.sqrt(np.mean(camera_translation[camera_valid] ** 2)) + ), + "camera_rotation_mean_deg": float(np.mean(camera_rotation[camera_valid])), + } + + +def _set_equal_3d(ax: Any, points: np.ndarray) -> None: + low = np.nanmin(points, axis=0) + high = np.nanmax(points, axis=0) + center = 0.5 * (low + high) + radius = max(float(np.max(high - low)) * 0.55, 0.05) + ax.set_xlim(center[0] - radius, center[0] + radius) + ax.set_ylim(center[1] - radius, center[1] + radius) + ax.set_zlim(center[2] - radius, center[2] + radius) + + +def save_trajectory_visualization( + output_dir: Path, + generated_action: dict[str, torch.Tensor], + target_action: dict[str, torch.Tensor], + generated_camera: np.ndarray, + target_camera: np.ndarray, + generated_action_raw: np.ndarray, + target_action_raw: np.ndarray, + action_is_pad: np.ndarray, + camera_is_pad: np.ndarray, +) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + pred_left = generated_action["left_pose_reference"].numpy() + pred_right = generated_action["right_pose_reference"].numpy() + gt_left = target_action["left_pose_reference"].numpy() + gt_right = target_action["right_pose_reference"].numpy() + pred_grip = generated_action["grip_open"].numpy() + gt_grip = target_action["grip_open"].numpy() + camera_pred_xyz = np.concatenate((np.zeros((1, 3)), generated_camera[:, :3]), axis=0) + camera_gt_xyz = np.concatenate((np.zeros((1, 3)), target_camera[:, :3]), axis=0) + + np.savez_compressed( + output_dir / "trajectories.npz", + generated_action_raw=generated_action_raw, + target_action_raw=target_action_raw, + generated_left_pose_reference=pred_left, + target_left_pose_reference=gt_left, + generated_right_pose_reference=pred_right, + target_right_pose_reference=gt_right, + generated_grip=pred_grip, + target_grip=gt_grip, + generated_camera=generated_camera, + target_camera=target_camera, + action_is_pad=action_is_pad, + camera_is_pad=camera_is_pad, + ) + + figure = plt.figure(figsize=(16, 10), constrained_layout=True) + action_ax = figure.add_subplot(2, 2, 1, projection="3d") + for points, label, color, style in ( + (gt_left[:, :3, 3], "GT left", "tab:blue", "-"), + (pred_left[:, :3, 3], "Pred left", "tab:blue", "--"), + (gt_right[:, :3, 3], "GT right", "tab:orange", "-"), + (pred_right[:, :3, 3], "Pred right", "tab:orange", "--"), + ): + action_ax.plot(*points.T, style, color=color, marker="o", ms=3, label=label) + _set_equal_3d( + action_ax, + np.concatenate((pred_left[:, :3, 3], pred_right[:, :3, 3], gt_left[:, :3, 3], gt_right[:, :3, 3])), + ) + action_ax.set_title("End-effector trajectories (first-camera frame)") + action_ax.set_xlabel("x [m]") + action_ax.set_ylabel("y [m]") + action_ax.set_zlabel("z [m]") + action_ax.legend(fontsize=8) + + camera_ax = figure.add_subplot(2, 2, 2, projection="3d") + camera_ax.plot(*camera_gt_xyz.T, "-o", ms=3, label="GT camera") + camera_ax.plot(*camera_pred_xyz.T, "--o", ms=3, label="Pred camera") + _set_equal_3d(camera_ax, np.concatenate((camera_gt_xyz, camera_pred_xyz))) + camera_ax.set_title("Camera trajectory (frame 0 = identity)") + camera_ax.set_xlabel("x [m]") + camera_ax.set_ylabel("y [m]") + camera_ax.set_zlabel("z [m]") + camera_ax.legend(fontsize=8) + + grip_ax = figure.add_subplot(2, 2, 3) + frames = np.arange(1, len(pred_grip) + 1) + for hand, color in ((0, "tab:blue"), (1, "tab:orange")): + name = "left" if hand == 0 else "right" + grip_ax.plot(frames, gt_grip[:, hand], "-", color=color, label=f"GT {name}") + grip_ax.plot(frames, pred_grip[:, hand], "--", color=color, label=f"Pred {name}") + grip_ax.set_title("Grip proxy") + grip_ax.set_xlabel("target frame") + grip_ax.grid(alpha=0.25) + grip_ax.legend(fontsize=8) + + error_ax = figure.add_subplot(2, 2, 4) + error_ax.plot( + frames, + np.linalg.norm(generated_camera[:, :3] - target_camera[:, :3], axis=1), + label="camera translation [m]", + ) + error_ax.plot( + frames, + _rotation_error_deg(generated_camera[:, 3:], target_camera[:, 3:]), + label="camera rotation [deg]", + ) + error_ax.set_title("Per-frame camera error") + error_ax.set_xlabel("target frame") + error_ax.grid(alpha=0.25) + error_ax.legend(fontsize=8) + figure.savefig(output_dir / "gt_pred_action_camera_trajectories.png", dpi=160) + plt.close(figure) + + +def run_test(args: argparse.Namespace) -> None: + raw_config = load_config(args.config) + target_type = str(raw_config.get("action", {}).get("target_type", "")) + if target_type == "absolute_ee_pose_in_episode_first_camera": + raise ValueError( + "post_training_test_action_mot.py integrates SE(3) delta actions and " + "must not evaluate SpaceDreamer absolute actions. Use " + "evaluate_spacedreamer_validation.py instead." + ) + config = validate_config(raw_config) + post = config["post_training_test"] + if not args.checkpoint.is_file(): + raise FileNotFoundError(args.checkpoint) + if not torch.cuda.is_available() and str(args.device).startswith("cuda"): + raise RuntimeError("CUDA is required for the real Action MoT evaluation") + prepare_output_directory(args.output_dir, args.overwrite) + cache_dir = Path(config["paths"]["cache_dir"]) + base_dataset = CachedRgbDepthPointLatents(cache_dir, point_config_fingerprint(config)) + selected = [ + int(value) + for value in (args.sample_indices if args.sample_indices is not None else post["sample_indices"]) + ] + if not selected or len(selected) != len(set(selected)): + raise ValueError("sample indices must be non-empty and unique") + if min(selected) < 0 or max(selected) >= len(base_dataset): + raise IndexError(f"sample indices outside [0,{len(base_dataset) - 1}]") + + device = torch.device(args.device) + dtype = dtype_from_name(str(config["model"]["dtype"])) + print(f"[post-test] strict-loading {args.checkpoint}", flush=True) + model = build_joint_action_mot(config, args.checkpoint, load_pretrained=False) + dataset, _ = prepare_online_training_dataset(base_dataset, model, config) + model.to(device=device, dtype=dtype).eval().requires_grad_(False) + model.video_expert.use_gradient_checkpointing = False + model.action_expert.use_gradient_checkpointing = False + video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=float(config["diffusion"]["infer_shift"]), + ) + action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["action_num_train_timesteps"]), + shift=float(config["diffusion"]["action_infer_shift"]), + ) + latent_dir = args.output_dir / "latents" + latent_dir.mkdir(parents=True, exist_ok=True) + latent_results: list[dict[str, Any]] = [] + with torch.inference_mode(): + for ordinal, sample_index in enumerate(selected): + sample = dataset[sample_index] + z_rgb, z_depth, z_point = [ + sample[key].unsqueeze(0).to(device=device, dtype=dtype) + for key in ("z_rgb", "z_depth", "z_point") + ] + target_action = sample["action"].unsqueeze(0).to(device=device, dtype=dtype) + target_camera = sample["camera"].unsqueeze(0).to(device=device, dtype=dtype) + initial_state = sample["initial_state"].unsqueeze(0).to(device=device, dtype=dtype) + context = sample["context"].unsqueeze(0).to(device=device, dtype=dtype) + context_mask = sample["context_mask"].unsqueeze(0).to(device=device, dtype=torch.bool) + generator = torch.Generator(device=device).manual_seed( + int(config["validation"]["fixed_noise_seed"]) + sample_index + ) + generated = sample_joint_latents_and_actions( + model, + video_scheduler, + action_scheduler, + video_shape=(1, 144, *z_rgb.shape[2:]), + action_shape=tuple(target_action.shape), + camera_shape=tuple(target_camera.shape), + context=context, + context_mask=context_mask, + initial_state=initial_state, + rgb_condition_latent=z_rgb[:, :, 0:1], + depth_condition_latent=z_depth[:, :, 0:1], + num_inference_steps=int(config["diffusion"]["inference_steps"]), + generator=generator, + dtype=dtype, + device=device, + ) + payload = { + "sample_index": sample_index, + "sample_id": sample["sample_id"], + "prompt": sample["prompt"], + "generated_rgb": generated["rgb"][0].cpu(), + "generated_depth": generated["depth"][0].cpu(), + "generated_point": generated["point"][0].cpu(), + "generated_action_normalized": generated["action"][0].float().cpu(), + "target_action_normalized": target_action[0].float().cpu(), + "generated_camera_normalized": generated["camera"][0].float().cpu(), + "target_camera_normalized": target_camera[0].float().cpu(), + "initial_state": initial_state[0].float().cpu(), + "left_initial_pose_reference": sample["left_initial_pose_reference"], + "right_initial_pose_reference": sample["right_initial_pose_reference"], + "action_is_pad": sample["action_is_pad"], + "camera_is_pad": sample["camera_is_pad"], + } + torch.save(payload, latent_dir / f"{ordinal:02d}_{sample['sample_id']}.pt") + latent_results.append(payload) + print(f"[sample {ordinal + 1:02d}/{len(selected):02d}] {sample['sample_id']}", flush=True) + + action_normalizer = model.action_normalizer.cpu() + camera_normalizer = model.camera_normalizer.cpu() + model.to("cpu") + del model + gc.collect() + torch.cuda.empty_cache() + + print("[post-test] loading frozen Wan VAE", flush=True) + vae = load_vae(Path(config["paths"]["pretrained_model_dir"]), dtype, device) + depth_config = config["data"]["depth"] + depth_min, depth_max = float(depth_config["min"]), float(depth_config["max"]) + point_config = config["point_cloud"] + results: list[dict[str, Any]] = [] + with torch.inference_mode(): + for ordinal, payload in enumerate(latent_results): + sample_index = int(payload["sample_index"]) + sample_id = str(payload["sample_id"]) + entry = base_dataset.samples[sample_index] + cached = torch.load(cache_dir / entry["file"], map_location="cpu", weights_only=True) + decoded = [ + vae.decode(payload[key].unsqueeze(0).to(device=device, dtype=dtype), device=device).float() + for key in ("generated_rgb", "generated_depth", "generated_point") + ] + generated_rgb = tensor_rgb_to_uint8(decoded[0]) + generated_depth_01, disagreement = decoded_depth_to_gray(decoded[1]) + generated_points = denormalize_world_point_video( + decoded[2][0].cpu().numpy(), point_config["xyz_min_m"], point_config["xyz_max_m"] + ) + gt_rgb = cached["rgb_uint8"].permute(1, 2, 3, 0).numpy() + gt_depth_01 = cached["depth_gray"][0].float().numpy() + valid_mask = _valid_t_hw(cached["depth_valid_mask"].numpy()) + gt_depth_m = gt_depth_01 * (depth_max - depth_min) + depth_min + generated_depth_m = generated_depth_01 * (depth_max - depth_min) + depth_min + intrinsics = cached["point_intrinsics"].float().numpy() + camera_to_reference = cached["point_camera_to_reference"].float().numpy() + gt_points, gt_point_valid = depth_video_to_world_grid( + gt_depth_m, intrinsics, camera_to_reference, stride=1, + min_depth_m=float(point_config["min_depth_m"]), max_depth_m=float(point_config["max_depth_m"]), + valid_mask=valid_mask, + ) + depth_points, depth_valid = depth_video_to_world_grid( + generated_depth_m, intrinsics, camera_to_reference, stride=1, + min_depth_m=float(point_config["min_depth_m"]), max_depth_m=float(point_config["max_depth_m"]), + ) + generated_action = integrate_normalized_actions( + payload["generated_action_normalized"], + left_initial_pose_reference=payload["left_initial_pose_reference"], + right_initial_pose_reference=payload["right_initial_pose_reference"], + normalizer=action_normalizer, + ) + target_action = integrate_normalized_actions( + payload["target_action_normalized"], + left_initial_pose_reference=payload["left_initial_pose_reference"], + right_initial_pose_reference=payload["right_initial_pose_reference"], + normalizer=action_normalizer, + ) + generated_camera = camera_normalizer.denormalize(payload["generated_camera_normalized"]).numpy() + target_camera = camera_normalizer.denormalize(payload["target_camera_normalized"]).numpy() + action_valid = ~payload["action_is_pad"].numpy().astype(bool) + camera_valid = ~payload["camera_is_pad"].numpy().astype(bool) + metrics = calculate_metrics( + gt_rgb, generated_rgb, gt_depth_01, generated_depth_01, valid_mask, + depth_min, depth_max, disagreement, depth_first_frame_conditioned=True, + ) + metrics.update(calculate_world_point_cloud_metrics( + gt_points, generated_points, gt_point_valid, depth_first_frame_conditioned=False + )) + metrics.update(_point_depth_consistency(generated_points, depth_points, depth_valid)) + metrics.update(_trajectory_metrics( + generated_action, target_action, generated_camera, target_camera, action_valid, camera_valid + )) + sample_dir = args.output_dir / "samples" / sample_id + sample_dir.mkdir(parents=True, exist_ok=True) + (sample_dir / "prompt.txt").write_text(str(payload["prompt"]) + "\n", encoding="utf-8") + save_sample_visualization( + sample_dir, sample_id, sample_index, gt_rgb, generated_rgb, + gt_depth_01, generated_depth_01, valid_mask, cached["source_indices"].numpy(), metrics, config, + ) + point_frames = export_direct_point_video( + sample_dir / "world_point_cloud", gt_points, generated_points, gt_rgb, generated_rgb, + gt_point_valid, cached["source_indices"].numpy(), config, metrics, fps=int(post["fps"]), + ) + if point_frames is not None: + write_gt_pred_three_modal_video( + sample_dir / "gt_pred_rgb_depth_point_2x3.mp4", gt_rgb, generated_rgb, + gt_depth_01, generated_depth_01, valid_mask, *point_frames, + fps=int(post["fps"]), panel_size=int(point_config.get("render_size", 512)), + ) + save_trajectory_visualization( + sample_dir / "trajectories", generated_action, target_action, + generated_camera, target_camera, generated_action["raw_action"].numpy(), + target_action["raw_action"].numpy(), payload["action_is_pad"].numpy(), payload["camera_is_pad"].numpy(), + ) + result = {"sample_index": sample_index, "sample_id": sample_id, "prompt": str(payload["prompt"]), **metrics} + results.append(result) + print( + f"[visualize {ordinal + 1:02d}/{len(latent_results):02d}] {sample_id} " + f"rgb_psnr={metrics['rgb_future_psnr_db']:.2f} " + f"action_l/r_rmse={metrics['action_left_position_rmse_m']:.3f}/{metrics['action_right_position_rmse_m']:.3f}m " + f"camera_rmse={metrics['camera_translation_rmse_m']:.3f}m", + flush=True, + ) + + vae.to("cpu") + del vae + gc.collect() + torch.cuda.empty_cache() + write_summary(args.output_dir, config, args.checkpoint, results) + with (args.output_dir / "README.md").open("a", encoding="utf-8") as handle: + handle.write( + "\n## Joint Action MoT outputs\n\n" + "This evaluation jointly samples RGB, depth, direct point-map, bimanual action, and camera tokens. " + "Each `samples/*/trajectories/` directory contains a GT/prediction plot and lossless arrays.\n" + ) + print(f"[post-test] complete: {args.output_dir}", flush=True) + + +if __name__ == "__main__": + run_test(parse_args()) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/post_training_test_rgb_depth_point.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/post_training_test_rgb_depth_point.py new file mode 100644 index 0000000000000000000000000000000000000000..731fca2d0506a1cec76bc9ac6b8a36a14994f301 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/post_training_test_rgb_depth_point.py @@ -0,0 +1,583 @@ +#!/usr/bin/env python3 +"""Fixed-20 evaluation for direct RGB+depth+world-point diffusion outputs.""" + +from __future__ import annotations + +import argparse +import gc +import json +from pathlib import Path +from typing import Any + +import cv2 +import numpy as np +import torch + +from post_training_test import ( + calculate_metrics, + decoded_depth_to_gray, + prepare_output_directory, + save_sample_visualization, + tensor_rgb_to_uint8, + upload_to_huggingface, + write_summary, +) +from train_joint_rgb_depth import dtype_from_name, load_config, load_vae +from train_joint_rgb_depth_point import ( + CachedRgbDepthPointLatents, + build_joint_dit, + point_config_fingerprint, + sample_joint_latents, + validate_config, +) +from world_point_cloud import ( + _render_cloud, + _write_rgb_video, + calculate_world_point_cloud_metrics, + denormalize_world_point_video, + depth_video_to_world_grid, + write_binary_ply, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--checkpoint", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--device", default="cuda:0") + parser.add_argument( + "--sample-indices", + type=int, + nargs="+", + help="Override post_training_test.sample_indices from the config.", + ) + parser.add_argument("--skip-upload", action="store_true") + parser.add_argument("--overwrite", action="store_true") + return parser.parse_args() + + +def _valid_t_hw(mask: torch.Tensor | np.ndarray) -> np.ndarray: + valid = np.asarray(mask, dtype=bool) + if valid.ndim == 4 and valid.shape[1] == 1: + valid = valid[:, 0] + if valid.ndim != 3: + raise ValueError(f"Expected valid mask [T,H,W], got {valid.shape}") + return valid + + +def _point_depth_consistency( + direct_world_points: np.ndarray, + depth_world_points: np.ndarray, + depth_valid: np.ndarray, +) -> dict[str, float]: + valid = np.asarray(depth_valid, dtype=bool) + valid &= np.isfinite(direct_world_points).all(axis=-1) + valid &= np.isfinite(depth_world_points).all(axis=-1) + if not np.any(valid): + raise ValueError("No valid generated depth/point correspondences") + error = direct_world_points - depth_world_points + distance = np.linalg.norm(error, axis=-1) + return { + "point_depth_consistency_component_rmse_m": float( + np.sqrt(np.mean(np.square(error[valid]))) + ), + "point_depth_consistency_euclidean_mean_m": float(np.mean(distance[valid])), + "point_depth_consistency_valid_fraction": float(np.mean(valid)), + } + + +def _labeled_panel( + frame: np.ndarray, + *, + size: int, + label: str, + nearest: bool = False, +) -> np.ndarray: + panel = np.asarray(frame) + if panel.ndim == 2: + panel = np.repeat(panel[..., None], 3, axis=-1) + if panel.ndim != 3 or panel.shape[-1] != 3: + raise ValueError(f"Expected RGB/gray panel, got {panel.shape}") + interpolation = cv2.INTER_NEAREST if nearest else cv2.INTER_LINEAR + panel = cv2.resize( + np.asarray(panel, dtype=np.uint8), + (size, size), + interpolation=interpolation, + ) + cv2.rectangle(panel, (0, 0), (size - 1, 34), (20, 20, 20), -1) + cv2.putText( + panel, + label, + (8, 23), + cv2.FONT_HERSHEY_SIMPLEX, + 0.58, + (255, 255, 255), + 1, + cv2.LINE_AA, + ) + return panel + + +def write_gt_pred_three_modal_video( + path: Path, + gt_rgb: np.ndarray, + generated_rgb: np.ndarray, + gt_depth_01: np.ndarray, + generated_depth_01: np.ndarray, + valid_mask: np.ndarray, + gt_point_frames: np.ndarray, + generated_point_frames: np.ndarray, + *, + fps: int, + panel_size: int, +) -> None: + """Write a 2x3 video: GT modalities above their predictions.""" + + gt_rgb = np.asarray(gt_rgb, dtype=np.uint8) + generated_rgb = np.asarray(generated_rgb, dtype=np.uint8) + gt_depth = np.rint(np.clip(gt_depth_01, 0, 1) * 255.0).astype(np.uint8) + generated_depth = np.rint( + np.clip(generated_depth_01, 0, 1) * 255.0 + ).astype(np.uint8) + valid = np.asarray(valid_mask, dtype=bool) + gt_depth = gt_depth.copy() + generated_depth = generated_depth.copy() + gt_depth[~valid] = 0 + generated_depth[~valid] = 0 + arrays = ( + gt_rgb, + generated_rgb, + gt_depth, + generated_depth, + np.asarray(gt_point_frames, dtype=np.uint8), + np.asarray(generated_point_frames, dtype=np.uint8), + ) + frame_count = len(gt_rgb) + if any(len(array) != frame_count for array in arrays): + raise ValueError( + "Frame-count mismatch in 2x3 visualization: " + f"{[len(array) for array in arrays]}" + ) + + frames = [] + for frame_index in range(frame_count): + top = np.concatenate( + ( + _labeled_panel(gt_rgb[frame_index], size=panel_size, label="GT RGB"), + _labeled_panel( + gt_depth[frame_index], + size=panel_size, + label="GT DEPTH", + nearest=True, + ), + _labeled_panel( + gt_point_frames[frame_index], + size=panel_size, + label="GT POINT CLOUD", + ), + ), + axis=1, + ) + bottom = np.concatenate( + ( + _labeled_panel( + generated_rgb[frame_index], + size=panel_size, + label="PRED RGB", + ), + _labeled_panel( + generated_depth[frame_index], + size=panel_size, + label="PRED DEPTH", + nearest=True, + ), + _labeled_panel( + generated_point_frames[frame_index], + size=panel_size, + label="PRED POINT CLOUD", + ), + ), + axis=1, + ) + frames.append(np.concatenate((top, bottom), axis=0)) + _write_rgb_video(path, np.stack(frames), fps) + + +def export_direct_point_video( + output_dir: Path, + gt_points: np.ndarray, + generated_points: np.ndarray, + gt_rgb: np.ndarray, + generated_rgb: np.ndarray, + gt_valid: np.ndarray, + source_indices: np.ndarray, + config: dict[str, Any], + metrics: dict[str, float], + *, + fps: int, +) -> tuple[np.ndarray, np.ndarray] | None: + """Export the third head directly; never reconstruct it from generated depth.""" + + point_config = config["point_cloud"] + gt = np.asarray(gt_points, dtype=np.float32) + generated = np.asarray(generated_points, dtype=np.float32) + valid = np.asarray(gt_valid, dtype=bool) + if gt.shape != generated.shape or gt.shape[:-1] != valid.shape or gt.shape[-1] != 3: + raise ValueError("Direct point export shape mismatch") + output_dir.mkdir(parents=True, exist_ok=True) + np.save(output_dir / "gt_world_point_map_m.npy", gt) + np.save(output_dir / "generated_world_point_map_m.npy", generated) + np.save(output_dir / "gt_world_point_valid_mask.npy", valid) + + stride = int(point_config.get("point_stride", 2)) + gt_sub = gt[:, ::stride, ::stride] + generated_sub = generated[:, ::stride, ::stride] + valid_sub = valid[:, ::stride, ::stride] + gt_colors = np.asarray(gt_rgb, dtype=np.uint8)[:, ::stride, ::stride] + generated_colors = np.asarray(generated_rgb, dtype=np.uint8)[:, ::stride, ::stride] + generated_valid = np.isfinite(generated_sub).all(axis=-1) + if bool(point_config.get("export_npz", True)): + np.savez_compressed( + output_dir / "gt_world_point_cloud_video.npz", + points=gt_sub, + colors=gt_colors, + valid=valid_sub, + source_indices=source_indices, + ) + np.savez_compressed( + output_dir / "generated_world_point_cloud_video.npz", + points=generated_sub, + colors=generated_colors, + valid=generated_valid, + source_indices=source_indices, + ) + if bool(point_config.get("export_ply", True)): + for frame_index in range(len(gt_sub)): + write_binary_ply( + output_dir / "ply" / "gt" / f"frame_{frame_index:03d}.ply", + gt_sub[frame_index][valid_sub[frame_index]], + gt_colors[frame_index][valid_sub[frame_index]], + ) + write_binary_ply( + output_dir / "ply" / "generated" / f"frame_{frame_index:03d}.ply", + generated_sub[frame_index][generated_valid[frame_index]], + generated_colors[frame_index][generated_valid[frame_index]], + ) + + rendered_frames = None + if bool(point_config.get("render_video", True)): + view = str(point_config.get("render_view", "xy")) + axes_by_view = {"xy": (0, 1), "xz": (0, 2), "yz": (1, 2)} + if view not in axes_by_view: + raise ValueError(f"Unsupported point render view: {view}") + axes = axes_by_view[view] + low = np.asarray(point_config["xyz_min_m"], dtype=np.float32) + high = np.asarray(point_config["xyz_max_m"], dtype=np.float32) + bounds = (low[axes[0]], high[axes[0]], low[axes[1]], high[axes[1]]) + size = int(point_config.get("render_size", 512)) + gt_frames, generated_frames, comparisons = [], [], [] + for frame_index in range(len(gt_sub)): + gt_frame_points = gt_sub[frame_index].copy() + gt_frame_points[~valid_sub[frame_index]] = np.nan + generated_frame_points = generated_sub[frame_index].copy() + generated_frame_points[~generated_valid[frame_index]] = np.nan + gt_frame = _render_cloud( + gt_frame_points, + gt_colors[frame_index], + bounds, + axes, + size, + f"GT WORLD POINT HEAD | {view.upper()} | f={frame_index:02d}", + ) + generated_frame = _render_cloud( + generated_frame_points, + generated_colors[frame_index], + bounds, + axes, + size, + f"GEN WORLD POINT HEAD | {view.upper()} | f={frame_index:02d}", + ) + gt_frames.append(gt_frame) + generated_frames.append(generated_frame) + comparisons.append(np.concatenate((gt_frame, generated_frame), axis=1)) + _write_rgb_video(output_dir / "gt_world_point_cloud.mp4", np.stack(gt_frames), fps) + _write_rgb_video( + output_dir / "generated_world_point_cloud.mp4", + np.stack(generated_frames), + fps, + ) + _write_rgb_video( + output_dir / "gt_generated_world_point_cloud_side_by_side.mp4", + np.stack(comparisons), + fps, + ) + rendered_frames = np.stack(gt_frames), np.stack(generated_frames) + + manifest = { + "format_version": 2, + "coordinate_frame": point_config["coordinate_frame"], + "derivation": "direct_third_diffusion_head_decoded_by_shared_frozen_wan_vae", + "rgb_frame0_conditioned": True, + "depth_frame0_conditioned": True, + "point_frame0_conditioned": False, + "pose_vector_layout": point_config["pose_vector_layout"], + "pose_is": point_config["pose_is"], + "camera_to_head_transform": point_config["camera_to_head_transform"], + "xyz_min_m": point_config["xyz_min_m"], + "xyz_max_m": point_config["xyz_max_m"], + "source_indices": [int(value) for value in source_indices], + "metrics": metrics, + } + (output_dir / "geometry.json").write_text( + json.dumps(manifest, indent=2), encoding="utf-8" + ) + return rendered_frames + + +def run_test(args: argparse.Namespace) -> None: + config = validate_config(load_config(args.config)) + post_config = config.get("post_training_test", {}) + if not bool(post_config.get("enabled", False)): + raise ValueError("post_training_test.enabled must be true") + if not args.checkpoint.is_file(): + raise FileNotFoundError(args.checkpoint) + cache_dir = Path(config["paths"]["cache_dir"]) + if not (cache_dir / "manifest.json").is_file(): + raise FileNotFoundError(f"Missing triple latent cache: {cache_dir / 'manifest.json'}") + if not torch.cuda.is_available() and str(args.device).startswith("cuda"): + raise RuntimeError("CUDA is required for the real 5B post-training test") + + prepare_output_directory(args.output_dir, args.overwrite) + device = torch.device(args.device) + dtype = dtype_from_name(str(config["model"]["dtype"])) + dataset = CachedRgbDepthPointLatents(cache_dir, point_config_fingerprint(config)) + selected_indices = [ + int(index) + for index in ( + args.sample_indices + if args.sample_indices is not None + else post_config["sample_indices"] + ) + ] + if not selected_indices: + raise ValueError("At least one sample index is required") + if len(set(selected_indices)) != len(selected_indices): + raise ValueError("sample indices must be unique") + if min(selected_indices) < 0 or max(selected_indices) >= len(dataset): + raise IndexError( + f"sample indices must be in [0, {len(dataset) - 1}], " + f"got {selected_indices}" + ) + + print(f"[post-test] strict-loading {args.checkpoint}", flush=True) + model = build_joint_dit(config, args.checkpoint).to(device=device, dtype=dtype) + model.use_gradient_checkpointing = False + model.eval().requires_grad_(False) + latent_dir = args.output_dir / "latents" + latent_dir.mkdir(parents=True, exist_ok=True) + latent_results: list[dict[str, Any]] = [] + with torch.inference_mode(): + for ordinal, sample_index in enumerate(selected_indices): + sample = dataset[sample_index] + modalities = [ + sample[key].unsqueeze(0).to(device=device, dtype=dtype) + for key in ("z_rgb", "z_depth", "z_point") + ] + context = sample["context"].unsqueeze(0).to(device=device, dtype=dtype) + context_mask = sample["context_mask"].unsqueeze(0).to( + device=device, dtype=torch.bool + ) + generated = sample_joint_latents( + model, + *modalities, + context, + context_mask, + config, + seed=int(config["validation"]["fixed_noise_seed"]) + sample_index, + ) + payload = { + "sample_index": sample_index, + "sample_id": sample["sample_id"], + "prompt": sample["prompt"], + "generated_rgb": generated[0][0].cpu(), + "generated_depth": generated[1][0].cpu(), + "generated_point": generated[2][0].cpu(), + } + torch.save(payload, latent_dir / f"{ordinal:02d}_{sample['sample_id']}.pt") + latent_results.append(payload) + print( + f"[post-test sample {ordinal + 1:02d}/{len(selected_indices):02d}] " + f"{sample['sample_id']}", + flush=True, + ) + + model.to("cpu") + del model + gc.collect() + torch.cuda.empty_cache() + + print("[post-test] loading frozen Wan VAE for RGB/depth/point decode", flush=True) + vae = load_vae(Path(config["paths"]["pretrained_model_dir"]), dtype, device) + depth_config = config["data"]["depth"] + depth_min = float(depth_config["min"]) + depth_max = float(depth_config["max"]) + point_config = config["point_cloud"] + results: list[dict[str, Any]] = [] + with torch.inference_mode(): + for ordinal, latent_payload in enumerate(latent_results): + sample_index = int(latent_payload["sample_index"]) + sample_id = str(latent_payload["sample_id"]) + cache_entry = dataset.samples[sample_index] + cached = torch.load( + cache_dir / cache_entry["file"], map_location="cpu", weights_only=True + ) + decoded = [] + for key in ("generated_rgb", "generated_depth", "generated_point"): + decoded.append( + vae.decode( + latent_payload[key].unsqueeze(0).to(device=device, dtype=dtype), + device=device, + ).float() + ) + generated_rgb_uint8 = tensor_rgb_to_uint8(decoded[0]) + generated_depth_01, disagreement = decoded_depth_to_gray(decoded[1]) + generated_points = denormalize_world_point_video( + decoded[2][0].cpu().numpy(), + point_config["xyz_min_m"], + point_config["xyz_max_m"], + ) + gt_rgb_uint8 = cached["rgb_uint8"].permute(1, 2, 3, 0).numpy() + gt_depth_01 = cached["depth_gray"][0].float().numpy() + valid_mask = _valid_t_hw(cached["depth_valid_mask"].numpy()) + gt_depth_m = gt_depth_01 * (depth_max - depth_min) + depth_min + generated_depth_m = generated_depth_01 * (depth_max - depth_min) + depth_min + intrinsics = cached["point_intrinsics"].float().numpy() + camera_to_reference = cached["point_camera_to_reference"].float().numpy() + gt_points, gt_point_valid = depth_video_to_world_grid( + gt_depth_m, + intrinsics, + camera_to_reference, + stride=1, + min_depth_m=float(point_config["min_depth_m"]), + max_depth_m=float(point_config["max_depth_m"]), + valid_mask=valid_mask, + ) + depth_generated_points, depth_generated_valid = depth_video_to_world_grid( + generated_depth_m, + intrinsics, + camera_to_reference, + stride=1, + min_depth_m=float(point_config["min_depth_m"]), + max_depth_m=float(point_config["max_depth_m"]), + ) + metrics = calculate_metrics( + gt_rgb_uint8, + generated_rgb_uint8, + gt_depth_01, + generated_depth_01, + valid_mask, + depth_min, + depth_max, + disagreement, + depth_first_frame_conditioned=True, + ) + metrics.update( + calculate_world_point_cloud_metrics( + gt_points, + generated_points, + gt_point_valid, + depth_first_frame_conditioned=False, + ) + ) + metrics.update( + _point_depth_consistency( + generated_points, + depth_generated_points, + depth_generated_valid, + ) + ) + sample_dir = args.output_dir / "samples" / sample_id + sample_dir.mkdir(parents=True, exist_ok=True) + (sample_dir / "prompt.txt").write_text( + str(latent_payload["prompt"]) + "\n", encoding="utf-8" + ) + save_sample_visualization( + sample_dir, + sample_id, + sample_index, + gt_rgb_uint8, + generated_rgb_uint8, + gt_depth_01, + generated_depth_01, + valid_mask, + cached["source_indices"].numpy(), + metrics, + config, + ) + point_frames = export_direct_point_video( + sample_dir / "world_point_cloud", + gt_points, + generated_points, + gt_rgb_uint8, + generated_rgb_uint8, + gt_point_valid, + cached["source_indices"].numpy(), + config, + metrics, + fps=int(post_config["fps"]), + ) + if point_frames is None: + raise RuntimeError( + "point_cloud.render_video must be true for the 2x3 visualization" + ) + write_gt_pred_three_modal_video( + sample_dir / "gt_pred_rgb_depth_point_2x3.mp4", + gt_rgb_uint8, + generated_rgb_uint8, + gt_depth_01, + generated_depth_01, + valid_mask, + *point_frames, + fps=int(post_config["fps"]), + panel_size=int(point_config.get("render_size", 512)), + ) + result = { + "sample_index": sample_index, + "sample_id": sample_id, + "prompt": str(latent_payload["prompt"]), + **metrics, + } + results.append(result) + print( + f"[visualize {ordinal + 1:02d}/{len(latent_results):02d}] {sample_id} " + f"depth_rmse={metrics['depth_generated_rmse_m']:.4f}m " + f"point_epe={metrics['point_world_generated_euclidean_mean_m']:.4f}m", + flush=True, + ) + del decoded, cached + + vae.to("cpu") + del vae + gc.collect() + torch.cuda.empty_cache() + write_summary(args.output_dir, config, args.checkpoint, results) + with (args.output_dir / "README.md").open("a", encoding="utf-8") as handle: + handle.write( + "\n## Direct world-point output\n\n" + "`samples/*/world_point_cloud/` is decoded from the third diffusion " + "head. It is not reconstructed from generated depth. The summary also " + "reports direct point error and point-vs-depth geometric consistency.\n" + ) + + hf_enabled = bool(post_config["huggingface"].get("enabled", False)) + if hf_enabled and not args.skip_upload: + commit_url = upload_to_huggingface(args.output_dir, config, args.checkpoint) + print(f"[huggingface] uploaded: {commit_url}", flush=True) + elif hf_enabled: + print("[huggingface] upload skipped by --skip-upload", flush=True) + print(f"[post-test] complete: {args.output_dir}", flush=True) + + +if __name__ == "__main__": + run_test(parse_args()) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/precompute_robodojo_exhaustive_rgb_latents.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/precompute_robodojo_exhaustive_rgb_latents.py new file mode 100644 index 0000000000000000000000000000000000000000..aa226c253aef37bbdd9c03a95b035e12394dd09e --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/precompute_robodojo_exhaustive_rgb_latents.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""Precompute one BF16 Wan-VAE latent for every configured RoboDojo start.""" + +from __future__ import annotations + +import argparse +import copy +import json +import os +from pathlib import Path +import time +from typing import Any + +import torch + +import train_robodojo_rgb_state_joint_action_4way as fourway +import train_robodojo_rgb_state_joint_action_episode_anchor_4way as episode_anchor +import train_robodojo_rgb_state_joint_action_absolute_ee_4way as absolute_ee + + +RGB_LATENT_CONDITIONING = { + "rgb_frame0": "clean_condition", + "initial_state": "clean_action_context_token", + "exclude_rgb_frame0_from_loss": True, +} +POINT_TARGET_CONDITIONING = { + "point_condition_mode": "none", + "point_timestep_adapter_bottleneck_dim": 256, + "fix_point_first_latent": False, + "exclude_point_condition_from_loss": False, +} + + +def validate_absolute_ee_rgb_latent_config( + config: dict[str, Any], +) -> dict[str, Any]: + """Validate an absolute-EE config for RGB-only latent preprocessing. + + The RGB cache is shared by the RGB/action and RGB/depth/point/action + experiments. The legacy absolute-EE validator deliberately requires an + exact RGB-only conditioning mapping, while the point-target experiment + adds point-specific target semantics to that same mapping. Validate the + complete multimodal contract first, then project only the conditioning + view passed to the legacy RGB validator. Restore the complete mapping in + the returned config so sampling and cache fingerprints retain the source + experiment semantics. + """ + + conditioning = config.get("conditioning") + if not isinstance(conditioning, dict): + raise ValueError("Config must define a conditioning mapping") + if conditioning == RGB_LATENT_CONDITIONING: + return absolute_ee.validate_absolute_ee_with_legacy_base(config) + + expected = {**RGB_LATENT_CONDITIONING, **POINT_TARGET_CONDITIONING} + if conditioning != expected: + raise ValueError( + "Unsupported conditioning contract for RGB latent preprocessing: " + f"{conditioning} != {expected}" + ) + modalities = tuple(config.get("geometry", {}).get("modalities", [])) + if modalities != ("rgb", "depth", "point"): + raise ValueError( + "Point-target conditioning is allowed only for " + "geometry.modalities=[rgb, depth, point]" + ) + + from robodojo_native_depth_point_data import validate_native_depth_point_config + + validate_native_depth_point_config(config) + projected = copy.deepcopy(config) + projected["conditioning"] = copy.deepcopy(RGB_LATENT_CONDITIONING) + validated = absolute_ee.validate_absolute_ee_with_legacy_base(projected) + validated["conditioning"] = copy.deepcopy(conditioning) + return validated + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--max-episodes-per-rank", type=int) + parser.add_argument( + "--prefix-latent-dir", + type=Path, + help=( + "Optional complete legacy valid-window cache. Its start-aligned prefix is " + "copied after strict manifest/shard validation; only FastWAM tail windows " + "are VAE-encoded." + ), + ) + parser.add_argument( + "--prefix-source-fingerprint", + help="Required exact source fingerprint when --prefix-latent-dir is used.", + ) + parser.add_argument( + "--action-adapter", + choices=("episode_initial", "absolute_ee", "piper_joint"), + default="episode_initial", + ) + parser.add_argument("--verify-only", action="store_true") + return parser.parse_args() + + +def valid_shard(path: Path, fingerprint: str, windows: int) -> bool: + if not path.is_file(): + return False + try: + payload = torch.load(path, map_location="cpu", weights_only=True, mmap=True) + return ( + payload.get("source_fingerprint") == fingerprint + and tuple(payload["z_rgb"].shape) == (windows, 48, 9, 16, 16) + and payload["z_rgb"].dtype == torch.bfloat16 + ) + except Exception: + return False + + +def main() -> None: + args = parse_args() + if args.batch_size <= 0: + raise ValueError("batch-size must be positive") + # Use the global rank for dataset sharding. LOCAL_RANK repeats on every + # node and would make a multi-node cache build encode the same episodes on + # each host. Device placement must continue to use the node-local rank. + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + rank = int(os.environ.get("RANK", str(local_rank))) + world_size = int(os.environ.get("WORLD_SIZE", os.environ.get("LOCAL_WORLD_SIZE", "1"))) + device = torch.device(f"cuda:{local_rank}") + if args.action_adapter == "piper_joint": + import train_piper_rgb_state_joint_action_quantile_4way as piper_joint + + config = piper_joint.load_piper_config(args.config) + piper_joint.install_piper_data_adapter(config) + config = piper_joint.validate_piper_config(config) + fingerprint_function = piper_joint.piper_cache_fingerprint + elif args.action_adapter == "absolute_ee": + config = absolute_ee.load_absolute_ee_config(args.config) + config = validate_absolute_ee_rgb_latent_config(config) + fourway.base.build_egoverse_delta_targets = absolute_ee.build_absolute_ee_targets + fingerprint_function = absolute_ee.absolute_ee_cache_fingerprint + else: + config = episode_anchor.load_episode_anchor_config(args.config) + config = episode_anchor.validate_episode_anchor_with_legacy_base(config) + fourway.base.build_egoverse_delta_targets = episode_anchor.build_episode_initial_relative_targets + fingerprint_function = episode_anchor.episode_anchor_cache_fingerprint + fourway.episode_length_normalized_cache_fingerprint = fingerprint_function + cache_dir = Path(config["paths"]["cache_dir"]) + source_manifest = json.loads( + (cache_dir / "manifest.json").read_text(encoding="utf-8") + ) + fingerprint = str(source_manifest["fingerprint"]) + records = [entry for entry in source_manifest["episodes"] if entry["split"] == "train"] + prefix_manifest = None + if (args.prefix_latent_dir is None) != (args.prefix_source_fingerprint is None): + raise ValueError( + "--prefix-latent-dir and --prefix-source-fingerprint must be provided together" + ) + if args.prefix_latent_dir is not None: + prefix_manifest_path = args.prefix_latent_dir / "manifest.json" + prefix_manifest = json.loads(prefix_manifest_path.read_text(encoding="utf-8")) + expected_prefix_windows = sum( + max(0, int(record["length"]) - int(config["sampling"]["window_size"]) + 1) + for record in records + ) + expected_prefix_contract = { + "source_fingerprint": str(args.prefix_source_fingerprint), + "dtype": "bfloat16", + "latent_shape": [48, 9, 16, 16], + "episode_count": len(records), + "valid_window_count": expected_prefix_windows, + "shard_layout": "one_tensor_per_episode_indexed_by_window_start", + } + actual_prefix_contract = { + key: prefix_manifest.get(key) for key in expected_prefix_contract + } + if actual_prefix_contract != expected_prefix_contract: + raise RuntimeError( + "Legacy prefix latent manifest mismatch: " + f"actual={actual_prefix_contract} expected={expected_prefix_contract}" + ) + print( + f"[latent-cache prefix] validated manifest path={args.prefix_latent_dir} " + f"windows={expected_prefix_windows}", + flush=True, + ) + output_dir = args.output_dir + output_dir.mkdir(parents=True, exist_ok=True) + latent_manifest = { + "format_version": 1, + "source_fingerprint": fingerprint, + "dtype": "bfloat16", + "latent_shape": [48, 9, 16, 16], + "episode_count": len(records), + "valid_window_count": int(source_manifest["valid_window_count"]), + "shard_layout": "one_tensor_per_episode_indexed_by_frame_start", + "tail_padding": source_manifest["sampling"].get("tail_padding"), + } + if rank == 0: + temporary = output_dir / f"manifest.json.tmp.{os.getpid()}" + temporary.write_text(json.dumps(latent_manifest, indent=2), encoding="utf-8") + os.replace(temporary, output_dir / "manifest.json") + + if args.verify_only: + invalid = [] + for record in records: + windows = fourway.episode_sample_count(config, int(record["length"])) + path = output_dir / f"episode_{int(record['episode_index']):06d}.pt" + if not valid_shard(path, fingerprint, windows): + invalid.append(str(path)) + if invalid: + preview = "\n".join(invalid[:10]) + raise RuntimeError( + f"Latent cache incomplete/invalid: {len(invalid)}/{len(records)} shards\n{preview}" + ) + print( + f"[latent-cache verify] complete episodes={len(records)} " + f"windows={source_manifest['valid_window_count']} path={output_dir}", + flush=True, + ) + return + + dtype = fourway.base.dtype_from_name(str(config["model"]["dtype"])) + configured_vae = os.environ.get("ROBODOJO_DYNAMIC_VAE_DIR", "").strip() + model_dir = Path(configured_vae) if configured_vae else Path(config["paths"]["pretrained_model_dir"]) + vae = fourway.base.load_vae(model_dir, dtype, device) + vae.eval().requires_grad_(False) + root = Path(config["paths"]["dataset_root"]) + clip_config = fourway.base.clip_config_from_config(config) + assigned = records[rank::world_size] + if args.max_episodes_per_rank is not None: + assigned = assigned[: args.max_episodes_per_rank] + started = time.monotonic() + generated_windows = 0 + completed = 0 + for number, record in enumerate(assigned, start=1): + robodojo_episode = fourway._record_episode(record) + windows = fourway.episode_sample_count( + config, int(robodojo_episode.length) + ) + output_path = output_dir / f"episode_{int(robodojo_episode.episode_index):06d}.pt" + if valid_shard(output_path, fingerprint, windows): + print( + f"[latent-cache rank={rank}] reuse {number}/{len(assigned)} " + f"episode={robodojo_episode.episode_index} windows={windows}", + flush=True, + ) + continue + episode_latents = torch.empty( + windows, 48, 9, 16, 16, dtype=torch.bfloat16, device="cpu" + ) + first_unencoded_window = 0 + if args.prefix_latent_dir is not None: + prefix_windows = max( + 0, + int(robodojo_episode.length) + - int(config["sampling"]["window_size"]) + + 1, + ) + prefix_path = ( + args.prefix_latent_dir + / f"episode_{int(robodojo_episode.episode_index):06d}.pt" + ) + prefix_payload = torch.load( + prefix_path, map_location="cpu", weights_only=True, mmap=True + ) + if prefix_payload.get("source_fingerprint") != args.prefix_source_fingerprint: + raise RuntimeError(f"Legacy prefix shard fingerprint mismatch: {prefix_path}") + prefix_latents = prefix_payload.get("z_rgb") + expected_shape = (prefix_windows, 48, 9, 16, 16) + if not isinstance(prefix_latents, torch.Tensor) or ( + tuple(prefix_latents.shape) != expected_shape + or prefix_latents.dtype != torch.bfloat16 + ): + raise RuntimeError( + f"Legacy prefix shard mismatch: {prefix_path}; " + f"expected shape={expected_shape} dtype=torch.bfloat16" + ) + episode_latents[:prefix_windows].copy_(prefix_latents) + first_unencoded_window = prefix_windows + del prefix_latents, prefix_payload + for start in range(first_unencoded_window, windows, args.batch_size): + stop = min(start + args.batch_size, windows) + rgb_batch = [] + for window_start in range(start, stop): + indices, _, _ = fourway.window_indices_and_pad_masks( + config, + length=int(robodojo_episode.length), + start=window_start, + ) + rgb_batch.append( + fourway.base.decode_rgb_window( + fourway.base.video_path(root, robodojo_episode, clip_config.video_key), + episode_from_timestamp=robodojo_episode.video_from_timestamp, + image_indices=indices, + config=clip_config, + ) + ) + rgb = torch.stack(rgb_batch).to(device=device, dtype=dtype, non_blocking=True) + with torch.inference_mode(): + z_rgb = vae.encode(rgb, device=device) + if tuple(z_rgb.shape[1:]) != (48, 9, 16, 16): + raise RuntimeError(f"Unexpected latent shape: {tuple(z_rgb.shape)}") + episode_latents[start:stop].copy_(z_rgb.to(device="cpu", dtype=torch.bfloat16)) + temporary = output_path.with_suffix(output_path.suffix + f".tmp.{os.getpid()}") + torch.save( + { + "source_fingerprint": fingerprint, + "episode_index": int(robodojo_episode.episode_index), + "z_rgb": episode_latents, + }, + temporary, + ) + os.replace(temporary, output_path) + generated_windows += windows + completed += 1 + elapsed = time.monotonic() - started + rate = generated_windows / elapsed if elapsed > 0 else 0.0 + print( + f"[latent-cache rank={rank}] wrote {number}/{len(assigned)} " + f"episode={robodojo_episode.episode_index} windows={windows} " + f"prefix_reused={first_unencoded_window} " + f"rate={rate:.2f}_windows/s path={output_path}", + flush=True, + ) + elapsed = time.monotonic() - started + print( + f"[latent-cache rank={rank}] complete episodes={completed} " + f"windows={generated_windows} elapsed_s={elapsed:.1f}", + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/precompute_robodojo_geometry_latents.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/precompute_robodojo_geometry_latents.py new file mode 100644 index 0000000000000000000000000000000000000000..be2dd8da10903f38e2edbde3761aae305de30480 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/precompute_robodojo_geometry_latents.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""Precompute BF16 Wan-VAE depth or point latents for exhaustive windows.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import time + +import numpy as np +import torch + +import train_robodojo_rgb_state_joint_action_4way as fourway +from robodojo_geometry_data import ( + LATENT_SHAPE, + build_geometry_episode_vae_inputs, + build_geometry_vae_inputs, + depth_episode_path, + depth_source_record, + geometry_cache_fingerprint, + geometry_modalities, + load_depth_episode_bundle, +) +from train_robodojo_rgb_geometry_action_absolute_ee import resolve_geometry_config + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--modality", choices=("depth", "point"), required=True) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--max-episodes-per-rank", type=int) + parser.add_argument("--verify-only", action="store_true") + parser.add_argument("--source-smoke-test", action="store_true") + return parser.parse_args() + + +def valid_shard( + path: Path, + *, + modality: str, + source_fingerprint: str, + geometry_fingerprint: str, + depth_source: dict[str, object], + windows: int, +) -> bool: + if not path.is_file(): + return False + try: + payload = torch.load(path, map_location="cpu", weights_only=True, mmap=True) + tensor = payload[f"z_{modality}"] + return ( + payload.get("source_fingerprint") == source_fingerprint + and payload.get("geometry_fingerprint") == geometry_fingerprint + and payload.get("depth_source") == depth_source + and tuple(tensor.shape) == (windows, *LATENT_SHAPE) + and tensor.dtype == torch.bfloat16 + ) + except Exception: + return False + + +def main() -> None: + args = parse_args() + if args.batch_size <= 0: + raise ValueError("batch-size must be positive") + config = resolve_geometry_config(args.config) + if args.modality == "point" and "point" not in geometry_modalities(config): + raise ValueError("Point cache requested with a depth-only config") + cache_dir = Path(config["paths"]["cache_dir"]) + source_manifest = json.loads( + (cache_dir / "manifest.json").read_text(encoding="utf-8") + ) + source_fingerprint = str(source_manifest["fingerprint"]) + records = [entry for entry in source_manifest["episodes"] if entry["split"] == "train"] + geometry_fingerprint = geometry_cache_fingerprint( + config, source_fingerprint, args.modality + ) + default_key = f"{args.modality}_latent_cache_dir" + output_dir = args.output_dir or Path(config["paths"][default_key]) + latent_manifest = { + "format_version": 1, + "modality": args.modality, + "source_fingerprint": source_fingerprint, + "geometry_fingerprint": geometry_fingerprint, + "dtype": "bfloat16", + "latent_shape": list(LATENT_SHAPE), + "episode_count": len(records), + "valid_window_count": int(source_manifest["valid_window_count"]), + "shard_layout": "one_tensor_per_episode_indexed_by_window_start", + } + rank = int(os.environ.get("LOCAL_RANK", "0")) + world_size = int(os.environ.get("LOCAL_WORLD_SIZE", os.environ.get("WORLD_SIZE", "1"))) + + depth_root = Path(config["paths"]["depth_dataset_root"]) + if args.source_smoke_test: + record = records[0] + bundle = load_depth_episode_bundle( + depth_root, + int(record["episode_index"]), + expected_length=int(record["length"]), + ) + inputs = build_geometry_vae_inputs( + bundle, + np.arange(int(config["data"]["num_frames"]), dtype=np.int64), + config, + ) + print( + json.dumps( + { + "status": "geometry_source_smoke_ok", + "modality": args.modality, + "episode_index": int(record["episode_index"]), + "tensor_shape": list(inputs[args.modality].shape), + "tensor_dtype": str(inputs[args.modality].dtype), + "tensor_range": [ + float(inputs[args.modality].min()), + float(inputs[args.modality].max()), + ], + }, + indent=2, + ), + flush=True, + ) + return + + if args.verify_only: + manifest_path = output_dir / "manifest.json" + if not manifest_path.is_file(): + raise FileNotFoundError(manifest_path) + actual_manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if actual_manifest != latent_manifest: + raise RuntimeError( + f"Geometry cache manifest mismatch: {actual_manifest} != {latent_manifest}" + ) + invalid = [] + for record in records: + episode_index = int(record["episode_index"]) + windows = int(record["length"]) - int(config["sampling"]["window_size"]) + 1 + source = depth_episode_path(depth_root, episode_index) + path = output_dir / f"episode_{episode_index:06d}.pt" + if not valid_shard( + path, + modality=args.modality, + source_fingerprint=source_fingerprint, + geometry_fingerprint=geometry_fingerprint, + depth_source=depth_source_record(source), + windows=windows, + ): + invalid.append(str(path)) + if invalid: + raise RuntimeError( + f"Geometry cache incomplete/invalid: {len(invalid)}/{len(records)}\n" + + "\n".join(invalid[:10]) + ) + print( + f"[geometry-cache verify] modality={args.modality} " + f"episodes={len(records)} windows={source_manifest['valid_window_count']} " + f"path={output_dir}", + flush=True, + ) + return + + output_dir.mkdir(parents=True, exist_ok=True) + if rank == 0: + temporary = output_dir / f"manifest.json.tmp.{os.getpid()}" + temporary.write_text(json.dumps(latent_manifest, indent=2), encoding="utf-8") + os.replace(temporary, output_dir / "manifest.json") + + device = torch.device(f"cuda:{rank}") + dtype = fourway.base.dtype_from_name(str(config["model"]["dtype"])) + configured_vae = os.environ.get("ROBODOJO_DYNAMIC_VAE_DIR", "").strip() + model_dir = ( + Path(configured_vae) + if configured_vae + else Path(config["paths"]["pretrained_model_dir"]) + ) + vae = fourway.base.load_vae(model_dir, dtype, device) + vae.eval().requires_grad_(False) + assigned = records[rank::world_size] + if args.max_episodes_per_rank is not None: + assigned = assigned[: args.max_episodes_per_rank] + started = time.monotonic() + generated_windows = 0 + completed = 0 + for number, record in enumerate(assigned, start=1): + episode_index = int(record["episode_index"]) + windows = int(record["length"]) - int(config["sampling"]["window_size"]) + 1 + source_path = depth_episode_path(depth_root, episode_index) + source_record = depth_source_record(source_path) + output_path = output_dir / f"episode_{episode_index:06d}.pt" + if valid_shard( + output_path, + modality=args.modality, + source_fingerprint=source_fingerprint, + geometry_fingerprint=geometry_fingerprint, + depth_source=source_record, + windows=windows, + ): + print( + f"[geometry-cache rank={rank}] reuse {number}/{len(assigned)} " + f"episode={episode_index} windows={windows}", + flush=True, + ) + continue + bundle = load_depth_episode_bundle( + depth_root, episode_index, expected_length=int(record["length"]) + ) + # Windows overlap by 32/33 frames. Geometry preprocessing, especially + # dense point back-projection, must therefore happen once per source + # frame rather than once per window. Only the VAE window encoding is + # left in the inner loop. + episode_video = build_geometry_episode_vae_inputs(bundle, config)[ + args.modality + ] + episode_latents = torch.empty( + windows, *LATENT_SHAPE, dtype=torch.bfloat16, device="cpu" + ) + for start in range(0, windows, args.batch_size): + stop = min(start + args.batch_size, windows) + videos = [] + for window_start in range(start, stop): + window_stop = window_start + int(config["sampling"]["window_size"]) + videos.append(episode_video[:, window_start:window_stop]) + batch = torch.stack(videos).to( + device=device, dtype=dtype, non_blocking=True + ) + with torch.inference_mode(): + latent = vae.encode(batch, device=device) + if tuple(latent.shape[1:]) != LATENT_SHAPE: + raise RuntimeError(f"Unexpected geometry latent shape: {tuple(latent.shape)}") + episode_latents[start:stop].copy_( + latent.to(device="cpu", dtype=torch.bfloat16) + ) + temporary = output_path.with_suffix(output_path.suffix + f".tmp.{os.getpid()}") + torch.save( + { + "source_fingerprint": source_fingerprint, + "geometry_fingerprint": geometry_fingerprint, + "depth_source": source_record, + "episode_index": episode_index, + f"z_{args.modality}": episode_latents, + }, + temporary, + ) + os.replace(temporary, output_path) + completed += 1 + generated_windows += windows + elapsed = time.monotonic() - started + rate = generated_windows / elapsed if elapsed > 0 else 0.0 + print( + f"[geometry-cache rank={rank}] wrote {number}/{len(assigned)} " + f"episode={episode_index} windows={windows} rate={rate:.2f}_windows/s", + flush=True, + ) + print( + f"[geometry-cache rank={rank}] complete modality={args.modality} " + f"episodes={completed} windows={generated_windows} " + f"elapsed_s={time.monotonic() - started:.1f}", + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/precompute_robodojo_native_depth_latents_fastwam.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/precompute_robodojo_native_depth_latents_fastwam.py new file mode 100644 index 0000000000000000000000000000000000000000..e32bcd804b9d749c62b57fe06bed76abf3d9600d --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/precompute_robodojo_native_depth_latents_fastwam.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +"""Audit native RoboDojo-mini GT depth and precompute FastWAM Wan latents.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import time + +import numpy as np +import torch + +import train_robodojo_rgb_state_joint_action_4way as fourway +from robodojo_native_depth_data import ( + LATENT_SHAPE, + audit_selected_native_episodes, + load_native_depth_episode, + native_depth_cache_fingerprint, + native_depth_episode_path, + preprocess_native_depth_episode, + source_record, +) +from train_robodojo_rgb_native_depth_state_joint_action_fastwam import ( + resolve_native_depth_config, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--audit-only", action="store_true") + parser.add_argument("--verify-only", action="store_true") + parser.add_argument("--source-smoke-test", action="store_true") + parser.add_argument("--max-episodes-per-rank", type=int) + return parser.parse_args() + + +def load_source_records(config: dict) -> tuple[dict, list[dict]]: + source_manifest = json.loads( + (Path(config["paths"]["cache_dir"]) / "manifest.json").read_text( + encoding="utf-8" + ) + ) + records = [ + entry for entry in source_manifest["episodes"] if entry["split"] == "train" + ] + if ( + len(records) != 100 * len(fourway.EXPECTED_TASKS) + or int(source_manifest["valid_window_count"]) != int(config["training"]["valid_windows_per_epoch"]) + or sum(int(record["length"]) for record in records) != int(source_manifest["valid_window_count"]) + ): + raise RuntimeError("Source manifest does not match the configured RoboDojo task set") + return source_manifest, records + + +def build_audit(config: dict, source_manifest: dict, records: list[dict]) -> dict: + episodes = audit_selected_native_episodes(config, records) + fingerprint = native_depth_cache_fingerprint( + config, str(source_manifest["fingerprint"]), episodes + ) + return { + "format_version": 1, + "source": "robodojo_native_gt_hdf5", + "source_fingerprint": str(source_manifest["fingerprint"]), + "native_depth_fingerprint": fingerprint, + "episode_count": len(episodes), + "frame_count": sum(int(entry["length"]) for entry in episodes), + "task_count": len(fourway.EXPECTED_TASKS), + "episodes": episodes, + } + + +def expected_manifest(source_manifest: dict, audit: dict) -> dict: + return { + "format_version": 1, + "modality": "depth", + "source": "robodojo_native_gt_hdf5", + "source_fingerprint": str(source_manifest["fingerprint"]), + "native_depth_fingerprint": str(audit["native_depth_fingerprint"]), + "dtype": "bfloat16", + "latent_shape": list(LATENT_SHAPE), + "episode_count": int(source_manifest["episode_count"]), + "valid_window_count": int(source_manifest["valid_window_count"]), + "window_sampling": "all_episode_frames_repeat_last_tail", + } + + +def valid_shard( + path: Path, + *, + source_fingerprint: str, + native_fingerprint: str, + native_source: dict, + windows: int, +) -> bool: + if not path.is_file(): + return False + try: + payload = torch.load(path, map_location="cpu", weights_only=True, mmap=True) + tensor = payload["z_depth"] + return ( + payload.get("source_fingerprint") == source_fingerprint + and payload.get("native_depth_fingerprint") == native_fingerprint + and payload.get("native_source") == native_source + and tuple(tensor.shape) == (windows, *LATENT_SHAPE) + and tensor.dtype == torch.bfloat16 + ) + except Exception: + return False + + +def main() -> None: + args = parse_args() + if args.batch_size <= 0: + raise ValueError("batch-size must be positive") + modes = sum((args.audit_only, args.verify_only, args.source_smoke_test)) + if modes > 1: + raise ValueError("Choose at most one audit/verify/smoke mode") + config = resolve_native_depth_config(args.config) + source_manifest, records = load_source_records(config) + output_dir = args.output_dir or Path(config["paths"]["depth_latent_cache_dir"]) + audit_path = output_dir / "native_audit.json" + manifest_path = output_dir / "manifest.json" + + if args.audit_only: + output_dir.mkdir(parents=True, exist_ok=True) + audit = build_audit(config, source_manifest, records) + manifest = expected_manifest(source_manifest, audit) + audit_tmp = audit_path.with_suffix(f".json.tmp.{os.getpid()}") + manifest_tmp = manifest_path.with_suffix(f".json.tmp.{os.getpid()}") + audit_tmp.write_text(json.dumps(audit, indent=2), encoding="utf-8") + manifest_tmp.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + os.replace(audit_tmp, audit_path) + os.replace(manifest_tmp, manifest_path) + print( + f"[native-depth audit] episodes={len(records)} tasks={len(fourway.EXPECTED_TASKS)} frames={source_manifest['valid_window_count']} " + f"fingerprint={audit['native_depth_fingerprint']} path={audit_path}", + flush=True, + ) + return + + if not audit_path.is_file() or not manifest_path.is_file(): + raise FileNotFoundError("Run --audit-only before cache precompute") + audit = json.loads(audit_path.read_text(encoding="utf-8")) + actual_manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest = expected_manifest(source_manifest, audit) + if actual_manifest != manifest: + raise RuntimeError(f"Native depth cache manifest mismatch: {actual_manifest} != {manifest}") + audit_by_episode = { + int(entry["episode_index"]): entry for entry in audit["episodes"] + } + if set(audit_by_episode) != {int(record["episode_index"]) for record in records}: + raise RuntimeError("Native audit and source manifest episode sets differ") + + native_root = Path(config["paths"]["native_depth_root"]) + if args.source_smoke_test: + record = records[0] + episode_index = int(record["episode_index"]) + path = native_depth_episode_path(native_root, config, episode_index) + depth, _ = load_native_depth_episode(path, expected_length=int(record["length"])) + processed = preprocess_native_depth_episode(depth[:33], config) + print( + json.dumps( + { + "status": "native_gt_depth_source_smoke_ok", + "episode_index": episode_index, + "source": str(path), + "raw_shape": list(depth[:33].shape), + "processed_shape": list(processed.shape), + "processed_range": [float(processed.min()), float(processed.max())], + }, + indent=2, + ), + flush=True, + ) + return + + if args.verify_only: + invalid = [] + for record in records: + episode_index = int(record["episode_index"]) + source = native_depth_episode_path(native_root, config, episode_index) + shard = output_dir / f"episode_{episode_index:06d}.pt" + if not valid_shard( + shard, + source_fingerprint=str(source_manifest["fingerprint"]), + native_fingerprint=str(audit["native_depth_fingerprint"]), + native_source=source_record(source), + windows=int(record["length"]), + ): + invalid.append(str(shard)) + if invalid: + raise RuntimeError( + f"Native depth cache incomplete/invalid: {len(invalid)}/{len(records)}\n" + + "\n".join(invalid[:10]) + ) + print( + f"[native-depth verify] episodes={len(records)} windows={source_manifest['valid_window_count']} path={output_dir}", + flush=True, + ) + return + + rank = int(os.environ.get("LOCAL_RANK", "0")) + world_size = int(os.environ.get("LOCAL_WORLD_SIZE", os.environ.get("WORLD_SIZE", "1"))) + device = torch.device(f"cuda:{rank}") + dtype = fourway.base.dtype_from_name(str(config["model"]["dtype"])) + configured_vae = os.environ.get("ROBODOJO_DYNAMIC_VAE_DIR", "").strip() + model_dir = ( + Path(configured_vae) + if configured_vae + else Path(config["paths"]["pretrained_model_dir"]) + ) + vae = fourway.base.load_vae(model_dir, dtype, device) + vae.eval().requires_grad_(False) + assigned = records[rank::world_size] + if args.max_episodes_per_rank is not None: + assigned = assigned[: args.max_episodes_per_rank] + started = time.monotonic() + generated_windows = 0 + completed = 0 + window_size = int(config["sampling"]["window_size"]) + window_offsets = np.arange(window_size, dtype=np.int64) + for number, record in enumerate(assigned, start=1): + episode_index = int(record["episode_index"]) + windows = int(record["length"]) + source_path = native_depth_episode_path(native_root, config, episode_index) + native_source = source_record(source_path) + output_path = output_dir / f"episode_{episode_index:06d}.pt" + if valid_shard( + output_path, + source_fingerprint=str(source_manifest["fingerprint"]), + native_fingerprint=str(audit["native_depth_fingerprint"]), + native_source=native_source, + windows=windows, + ): + print( + f"[native-depth-cache rank={rank}] reuse {number}/{len(assigned)} " + f"episode={episode_index} windows={windows}", + flush=True, + ) + continue + depth, _ = load_native_depth_episode(source_path, expected_length=windows) + episode_video = preprocess_native_depth_episode(depth, config) + episode_latents = torch.empty( + windows, *LATENT_SHAPE, dtype=torch.bfloat16, device="cpu" + ) + for start in range(0, windows, args.batch_size): + stop = min(start + args.batch_size, windows) + videos = [] + for window_start in range(start, stop): + indices = np.minimum(window_start + window_offsets, windows - 1) + videos.append(episode_video[:, indices]) + batch = torch.stack(videos).to(device=device, dtype=dtype, non_blocking=True) + with torch.inference_mode(): + latent = vae.encode(batch, device=device) + if tuple(latent.shape[1:]) != LATENT_SHAPE: + raise RuntimeError(f"Unexpected depth latent shape {tuple(latent.shape)}") + episode_latents[start:stop].copy_( + latent.to(device="cpu", dtype=torch.bfloat16) + ) + temporary = output_path.with_suffix(output_path.suffix + f".tmp.{os.getpid()}") + torch.save( + { + "source_fingerprint": str(source_manifest["fingerprint"]), + "native_depth_fingerprint": str(audit["native_depth_fingerprint"]), + "native_source": native_source, + "episode_index": episode_index, + "z_depth": episode_latents, + }, + temporary, + ) + os.replace(temporary, output_path) + completed += 1 + generated_windows += windows + elapsed = time.monotonic() - started + rate = generated_windows / elapsed if elapsed > 0 else 0.0 + print( + f"[native-depth-cache rank={rank}] wrote {number}/{len(assigned)} " + f"episode={episode_index} windows={windows} rate={rate:.2f}_windows/s", + flush=True, + ) + print( + f"[native-depth-cache rank={rank}] complete episodes={completed} " + f"windows={generated_windows} elapsed_s={time.monotonic() - started:.1f}", + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/precompute_robodojo_native_point_latents_fastwam.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/precompute_robodojo_native_point_latents_fastwam.py new file mode 100644 index 0000000000000000000000000000000000000000..a183990422c60eaa23d6504bdd92f2219f5dac1f --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/precompute_robodojo_native_point_latents_fastwam.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +"""Audit native RoboDojo point geometry and precompute 33-frame Wan latents.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import time + +import numpy as np +import torch + +import train_robodojo_rgb_state_joint_action_4way as fourway +from robodojo_native_depth_point_data import ( + LATENT_SHAPE, + audit_selected_native_point_episodes, + expected_native_point_manifest, + load_native_point_episode, + native_depth_episode_path, + native_point_cache_fingerprint, + preprocess_native_point_episode, + source_record, +) +from train_robodojo_rgb_native_depth_point_state_joint_action_fastwam import ( + resolve_native_depth_point_config, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--audit-only", action="store_true") + parser.add_argument("--verify-only", action="store_true") + parser.add_argument("--source-smoke-test", action="store_true") + parser.add_argument("--max-episodes-per-rank", type=int) + return parser.parse_args() + + +def load_source_records(config: dict) -> tuple[dict, list[dict]]: + manifest = json.loads( + (Path(config["paths"]["cache_dir"]) / "manifest.json").read_text( + encoding="utf-8" + ) + ) + records = [entry for entry in manifest["episodes"] if entry["split"] == "train"] + if ( + len(records) != 100 * len(fourway.EXPECTED_TASKS) + or int(manifest["valid_window_count"]) != int(config["training"]["valid_windows_per_epoch"]) + or sum(int(record["length"]) for record in records) != int(manifest["valid_window_count"]) + ): + raise RuntimeError( + "Source manifest does not match the configured RoboDojo task set" + ) + return manifest, records + + +def load_verified_depth_contract(config: dict, source_manifest: dict) -> tuple[dict, dict]: + cache = Path(config["paths"]["depth_latent_cache_dir"]) + manifest = json.loads((cache / "manifest.json").read_text(encoding="utf-8")) + audit = json.loads((cache / "native_audit.json").read_text(encoding="utf-8")) + expected = { + "format_version": 1, + "modality": "depth", + "source": "robodojo_native_gt_hdf5", + "source_fingerprint": str(source_manifest["fingerprint"]), + "native_depth_fingerprint": str(audit["native_depth_fingerprint"]), + "dtype": "bfloat16", + "latent_shape": list(LATENT_SHAPE), + "episode_count": int(source_manifest["episode_count"]), + "valid_window_count": int(source_manifest["valid_window_count"]), + "window_sampling": "all_episode_frames_repeat_last_tail", + } + if manifest != expected: + raise RuntimeError(f"Depth cache contract mismatch before point cache: {manifest}") + if audit.get("source_fingerprint") != str(source_manifest["fingerprint"]): + raise RuntimeError("Depth/source fingerprint mismatch before point cache") + return manifest, audit + + +def build_audit( + config: dict, + source_manifest: dict, + depth_audit: dict, + records: list[dict], +) -> dict: + episodes = audit_selected_native_point_episodes(config, records) + fingerprint = native_point_cache_fingerprint( + config, + str(source_manifest["fingerprint"]), + str(depth_audit["native_depth_fingerprint"]), + episodes, + ) + return { + "format_version": 1, + "source": "robodojo_native_gt_hdf5_backprojection", + "source_fingerprint": str(source_manifest["fingerprint"]), + "native_depth_fingerprint": str(depth_audit["native_depth_fingerprint"]), + "native_point_fingerprint": fingerprint, + "episode_count": len(episodes), + "frame_count": sum(int(entry["length"]) for entry in episodes), + "task_count": len(fourway.EXPECTED_TASKS), + "episodes": episodes, + } + + +def expected_manifest(source_manifest: dict, depth_audit: dict, audit: dict) -> dict: + return expected_native_point_manifest( + source_fingerprint=str(source_manifest["fingerprint"]), + native_depth_fingerprint=str(depth_audit["native_depth_fingerprint"]), + native_point_fingerprint=str(audit["native_point_fingerprint"]), + episode_count=int(source_manifest["episode_count"]), + valid_window_count=int(source_manifest["valid_window_count"]), + ) + + +def valid_shard( + path: Path, + *, + source_fingerprint: str, + native_depth_fingerprint: str, + native_point_fingerprint: str, + native_source: dict, + windows: int, +) -> bool: + if not path.is_file(): + return False + try: + payload = torch.load(path, map_location="cpu", weights_only=True, mmap=True) + tensor = payload["z_point"] + return ( + payload.get("source_fingerprint") == source_fingerprint + and payload.get("native_depth_fingerprint") == native_depth_fingerprint + and payload.get("native_point_fingerprint") == native_point_fingerprint + and payload.get("native_source") == native_source + and tuple(tensor.shape) == (windows, *LATENT_SHAPE) + and tensor.dtype == torch.bfloat16 + ) + except Exception: + return False + + +def main() -> None: + args = parse_args() + if args.batch_size <= 0: + raise ValueError("batch-size must be positive") + if args.max_episodes_per_rank is not None and args.max_episodes_per_rank <= 0: + raise ValueError("max-episodes-per-rank must be positive") + modes = sum((args.audit_only, args.verify_only, args.source_smoke_test)) + if modes > 1: + raise ValueError("Choose at most one audit/verify/smoke mode") + + config = resolve_native_depth_point_config(args.config) + source_manifest, records = load_source_records(config) + _, depth_audit = load_verified_depth_contract(config, source_manifest) + output_dir = args.output_dir or Path(config["paths"]["point_latent_cache_dir"]) + audit_path = output_dir / "native_point_audit.json" + manifest_path = output_dir / "manifest.json" + + if args.audit_only: + output_dir.mkdir(parents=True, exist_ok=True) + audit = build_audit(config, source_manifest, depth_audit, records) + manifest = expected_manifest(source_manifest, depth_audit, audit) + audit_tmp = audit_path.with_suffix(f".json.tmp.{os.getpid()}") + manifest_tmp = manifest_path.with_suffix(f".json.tmp.{os.getpid()}") + audit_tmp.write_text(json.dumps(audit, indent=2), encoding="utf-8") + manifest_tmp.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + os.replace(audit_tmp, audit_path) + os.replace(manifest_tmp, manifest_path) + max_translation = max( + float(entry["native_source"]["episode_first_camera_translation_max_m"]) + for entry in audit["episodes"] + ) + print( + f"[native-point audit] episodes={len(records)} tasks={len(fourway.EXPECTED_TASKS)} frames={source_manifest['valid_window_count']} " + f"camera_translation_max_m={max_translation:.9g} " + f"fingerprint={audit['native_point_fingerprint']} path={audit_path}", + flush=True, + ) + return + + if not audit_path.is_file() or not manifest_path.is_file(): + raise FileNotFoundError("Run --audit-only before point cache precompute") + audit = json.loads(audit_path.read_text(encoding="utf-8")) + manifest = expected_manifest(source_manifest, depth_audit, audit) + actual_manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if actual_manifest != manifest: + raise RuntimeError( + f"Native point cache manifest mismatch: {actual_manifest} != {manifest}" + ) + audit_by_episode = { + int(entry["episode_index"]): entry for entry in audit["episodes"] + } + if set(audit_by_episode) != { + int(record["episode_index"]) for record in records + }: + raise RuntimeError("Native point audit/source episode sets differ") + + native_root = Path(config["paths"]["native_depth_root"]) + if args.source_smoke_test: + record = records[0] + episode_index = int(record["episode_index"]) + path = native_depth_episode_path(native_root, config, episode_index) + depth, intrinsics, extrinsics = load_native_point_episode( + path, expected_length=int(record["length"]) + ) + point, metrics = preprocess_native_point_episode( + depth[:33], intrinsics, extrinsics[:33], config + ) + print( + json.dumps( + { + "status": "native_gt_point_source_smoke_ok", + "episode_index": episode_index, + "source": str(path), + "raw_depth_shape": list(depth[:33].shape), + "processed_point_shape": list(point.shape), + "processed_point_range": [float(point.min()), float(point.max())], + **metrics, + }, + indent=2, + ), + flush=True, + ) + return + + shard_contract = { + "source_fingerprint": str(source_manifest["fingerprint"]), + "native_depth_fingerprint": str(depth_audit["native_depth_fingerprint"]), + "native_point_fingerprint": str(audit["native_point_fingerprint"]), + } + if args.verify_only: + invalid = [] + for record in records: + episode_index = int(record["episode_index"]) + source = native_depth_episode_path(native_root, config, episode_index) + shard = output_dir / f"episode_{episode_index:06d}.pt" + if not valid_shard( + shard, + **shard_contract, + native_source=source_record(source), + windows=int(record["length"]), + ): + invalid.append(str(shard)) + if invalid: + raise RuntimeError( + f"Native point cache incomplete/invalid: {len(invalid)}/{len(records)}\n" + + "\n".join(invalid[:10]) + ) + print( + f"[native-point verify] episodes={len(records)} windows={source_manifest['valid_window_count']} path={output_dir}", + flush=True, + ) + return + + rank = int(os.environ.get("LOCAL_RANK", "0")) + world_size = int( + os.environ.get("LOCAL_WORLD_SIZE", os.environ.get("WORLD_SIZE", "1")) + ) + device = torch.device(f"cuda:{rank}") + dtype = fourway.base.dtype_from_name(str(config["model"]["dtype"])) + configured_vae = os.environ.get("ROBODOJO_DYNAMIC_VAE_DIR", "").strip() + model_dir = ( + Path(configured_vae) + if configured_vae + else Path(config["paths"]["pretrained_model_dir"]) + ) + vae = fourway.base.load_vae(model_dir, dtype, device) + vae.eval().requires_grad_(False) + assigned = records[rank::world_size] + if args.max_episodes_per_rank is not None: + assigned = assigned[: args.max_episodes_per_rank] + started = time.monotonic() + generated_windows = 0 + completed = 0 + window_size = int(config["sampling"]["window_size"]) + window_offsets = np.arange(window_size, dtype=np.int64) + for number, record in enumerate(assigned, start=1): + episode_index = int(record["episode_index"]) + windows = int(record["length"]) + source_path = native_depth_episode_path(native_root, config, episode_index) + native_source = source_record(source_path) + output_path = output_dir / f"episode_{episode_index:06d}.pt" + if valid_shard( + output_path, + **shard_contract, + native_source=native_source, + windows=windows, + ): + print( + f"[native-point-cache rank={rank}] reuse {number}/{len(assigned)} " + f"episode={episode_index} windows={windows}", + flush=True, + ) + continue + depth, intrinsics, extrinsics = load_native_point_episode( + source_path, expected_length=windows + ) + episode_video, geometry_metrics = preprocess_native_point_episode( + depth, intrinsics, extrinsics, config + ) + episode_latents = torch.empty( + windows, *LATENT_SHAPE, dtype=torch.bfloat16, device="cpu" + ) + for start in range(0, windows, args.batch_size): + stop = min(start + args.batch_size, windows) + videos = [] + for window_start in range(start, stop): + indices = np.minimum(window_start + window_offsets, windows - 1) + videos.append(episode_video[:, indices]) + batch = torch.stack(videos).to( + device=device, dtype=dtype, non_blocking=True + ) + with torch.inference_mode(): + latent = vae.encode(batch, device=device) + if tuple(latent.shape[1:]) != LATENT_SHAPE: + raise RuntimeError(f"Unexpected point latent shape {tuple(latent.shape)}") + episode_latents[start:stop].copy_( + latent.to(device="cpu", dtype=torch.bfloat16) + ) + temporary = output_path.with_suffix( + output_path.suffix + f".tmp.{os.getpid()}" + ) + torch.save( + { + **shard_contract, + "native_source": native_source, + "episode_index": episode_index, + "geometry_metrics": geometry_metrics, + "z_point": episode_latents, + }, + temporary, + ) + os.replace(temporary, output_path) + completed += 1 + generated_windows += windows + elapsed = time.monotonic() - started + rate = generated_windows / elapsed if elapsed > 0 else 0.0 + print( + f"[native-point-cache rank={rank}] wrote {number}/{len(assigned)} " + f"episode={episode_index} windows={windows} " + f"valid={geometry_metrics['valid_fraction']:.6f} " + f"clipped={geometry_metrics['clipped_fraction']:.6f} " + f"rate={rate:.2f}_windows/s", + flush=True, + ) + print( + f"[native-point-cache rank={rank}] complete episodes={completed} " + f"windows={generated_windows} elapsed_s={time.monotonic() - started:.1f}", + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/preflight_robodojo_ego_first_cam.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/preflight_robodojo_ego_first_cam.py new file mode 100644 index 0000000000000000000000000000000000000000..ed837eb98a5870f4243748586d57e7804748641e --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/preflight_robodojo_ego_first_cam.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Fail-closed CPU preflight for the RoboDojo C0-frame transfer experiment.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import torch + +import train_robodojo_rgb_state_joint_action_4way as fourway +import train_robodojo_rgb_state_joint_action_absolute_ee_4way as absolute +import train_robodojo_rgb_state_joint_action_ego_first_cam_meanstd_4way as ego +from robodojo_ego_first_cam import ( + transform_pose16_base_to_c0, + transform_pose16_c0_to_base, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--require-cache", action="store_true") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + ego.install_hooks() + fourway.base.build_egoverse_delta_targets = absolute.build_absolute_ee_targets + config = fourway.validate_config(fourway.load_config(args.config)) + splits = fourway.base.resolve_splits(config) + split_counts = {name: len(values) for name, values in splits.items()} + if split_counts != {"train": 1000, "validation": 0, "test": 0}: + raise RuntimeError(f"Unexpected split counts: {split_counts}") + + extrinsic_hashes: dict[int, str] = {} + for episode in splits["train"]: + _, audit = ego.load_episode_t_b_c0(config, episode) + extrinsic_hashes[int(episode.episode_index)] = str(audit["matrix_sha256"]) + if len(extrinsic_hashes) != 1000: + raise RuntimeError("Did not audit exactly 1000 native cam_head extrinsics") + + episode = splits["train"][0] + base_state, base_action, _ = ego._original_load_episode_arrays(config, episode) + t_b_c0, _ = ego.load_episode_t_b_c0(config, episode) + for name, pose in (("state", base_state[0]), ("action", base_action[0])): + recovered = transform_pose16_c0_to_base( + transform_pose16_base_to_c0(pose, t_b_c0), t_b_c0 + ) + translation_error = max( + float(np.max(np.abs(recovered[0:3] - pose[0:3]))), + float(np.max(np.abs(recovered[8:11] - pose[8:11]))), + ) + if translation_error >= 1.0e-6: + raise RuntimeError(f"{name} round-trip error={translation_error}") + + source_path = Path(config["initialization"]["checkpoint"]) + payload = torch.load(source_path, map_location="cpu", weights_only=True, mmap=True) + if set(payload) != {"metadata", "state_dict"}: + raise RuntimeError("Source checkpoint top-level key contract changed") + source = payload["state_dict"] + block_prefix = "mot.mixtures.action.blocks." + block_indices = { + int(key[len(block_prefix) :].split(".", 1)[0]) + for key in source + if key.startswith(block_prefix) + } + if block_indices != set(range(30)): + raise RuntimeError(f"Source ActionDiT block coverage mismatch: {block_indices}") + required_prefixes = { + "action_encoder": "mot.mixtures.action.action_encoder.", + "action_head": "mot.mixtures.action.head.", + "action_state_encoder": "action_state_encoder.", + "action_text": "mot.mixtures.action.text_embedding.", + "action_time": "mot.mixtures.action.time_embedding.", + "video": "mot.mixtures.video.", + } + source_prefix_counts = { + name: sum(key.startswith(prefix) for key in source) + for name, prefix in required_prefixes.items() + } + if any(count == 0 for count in source_prefix_counts.values()): + raise RuntimeError(f"Source checkpoint module coverage failed: {source_prefix_counts}") + source_normalizer = { + key: source[key] + for key in source + if key.startswith("action_normalizer.") + } + if set(source_normalizer) != { + "action_normalizer.mean", + "action_normalizer.std", + "action_normalizer.count", + "action_normalizer.fitted", + }: + raise RuntimeError("Source action-normalizer key contract changed") + + cache_audit = None + if args.require_cache: + cache_dir = Path(config["paths"]["cache_dir"]) + manifest = json.loads((cache_dir / "manifest.json").read_text(encoding="utf-8")) + stats = torch.load( + cache_dir / "action_normalizer.pt", map_location="cpu", weights_only=True + ) + expected = { + "normalization_type": "mean_std", + "fit_split": "train", + "coordinate_frame": ego.COORDINATE_FRAME, + "state_uses_action_statistics": True, + } + actual = {key: stats.get(key) for key in expected} + if actual != expected: + raise RuntimeError(f"Target normalizer metadata mismatch: {actual}") + if stats.get("fingerprint") != manifest.get("fingerprint"): + raise RuntimeError("Target normalizer/cache fingerprint mismatch") + if torch.equal(stats["mean"], source_normalizer["action_normalizer.mean"]): + raise RuntimeError("Target mean unexpectedly equals SpaceDreamer source mean") + if torch.equal(stats["std"], source_normalizer["action_normalizer.std"]): + raise RuntimeError("Target std unexpectedly equals SpaceDreamer source std") + cache_audit = { + "cache_fingerprint": manifest["fingerprint"], + "statistics_fingerprint": stats["statistics_fingerprint"], + "count": int(stats["count"]), + "mean": stats["mean"].tolist(), + "std": stats["std"].tolist(), + "normalizer_not_inherited": True, + } + + print( + json.dumps( + { + "status": "passed", + "split_counts": split_counts, + "extrinsics_audited": len(extrinsic_hashes), + "unique_extrinsic_payloads": len(set(extrinsic_hashes.values())), + "source_checkpoint": str(source_path), + "source_sha256": config["initialization"]["source_sha256"], + "source_state_keys": len(source), + "source_module_key_counts": source_prefix_counts, + "source_action_blocks": len(block_indices), + "cache": cache_audit, + }, + indent=2, + sort_keys=True, + ), + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/prepare_point_cloud_cache.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/prepare_point_cloud_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..eebebc314202a95a9f2e28d8e2d8c2948fc593e8 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/prepare_point_cloud_cache.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Add frozen-Wan-VAE world-point latents to an existing RGB-depth cache.""" + +from __future__ import annotations + +import argparse +import gc +import json +from pathlib import Path +import shutil +import time +from typing import Any + +import numpy as np +import torch + +from train_joint_rgb_depth import ( + dtype_from_name, + load_config, + load_vae, +) +from train_joint_rgb_depth_point import point_config_fingerprint, validate_config +from world_point_cloud import ( + depth_video_to_world_grid, + load_egoverse_world_geometry, + normalize_world_point_video, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + return parser.parse_args() + + +def _valid_mask_t_hw(mask: torch.Tensor) -> np.ndarray: + valid = mask.bool().numpy() + if valid.ndim == 4 and valid.shape[1] == 1: + valid = valid[:, 0] + if valid.ndim != 3: + raise ValueError(f"Expected depth valid mask [T,H,W], got {valid.shape}") + return valid + + +def prepare_point_cache(config: dict[str, Any]) -> None: + point_config = config["point_cloud"] + source_dir = Path(config["paths"]["rgb_depth_cache_dir"]) + target_dir = Path(config["paths"]["cache_dir"]) + source_manifest_path = source_dir / "manifest.json" + target_manifest_path = target_dir / "manifest.json" + if not source_manifest_path.is_file(): + raise FileNotFoundError(source_manifest_path) + source_manifest = json.loads(source_manifest_path.read_text(encoding="utf-8")) + expected_samples = int(config["_num_samples"]) + if int(source_manifest.get("num_samples", -1)) != expected_samples: + raise RuntimeError("RGB-depth source cache sample count does not match config") + + fingerprint = point_config_fingerprint(config) + if target_manifest_path.is_file() and not bool(config["cache"]["overwrite"]): + existing = json.loads(target_manifest_path.read_text(encoding="utf-8")) + if existing.get("fingerprint") != fingerprint: + raise RuntimeError( + f"Existing point cache fingerprint mismatch: {target_manifest_path}" + ) + print(f"[point-cache] reuse {target_manifest_path}", flush=True) + return + if not torch.cuda.is_available() and str(config["cache"]["device"]).startswith("cuda"): + raise RuntimeError("Point cache preparation requires configured CUDA device") + + target_dir.mkdir(parents=True, exist_ok=True) + source_prompt = source_dir / "prompt_contexts.pt" + if not source_prompt.is_file(): + raise FileNotFoundError(source_prompt) + shutil.copy2(source_prompt, target_dir / "prompt_contexts.pt") + + device = torch.device(str(config["cache"]["device"])) + dtype = dtype_from_name(str(config["model"]["dtype"])) + vae = load_vae(Path(config["paths"]["pretrained_model_dir"]), dtype, device) + xyz_min = point_config["xyz_min_m"] + xyz_max = point_config["xyz_max_m"] + invalid_fill = point_config["invalid_fill_reference_m"] + depth_min = float(config["data"]["depth"]["min"]) + depth_max = float(config["data"]["depth"]["max"]) + resolution = int(config["data"]["resolution"]) + expected_shape = ( + 1, + int(config["model"]["point_channels"]), + (int(config["data"]["num_frames"]) - 1) // 4 + 1, + resolution // 16, + resolution // 16, + ) + entries: list[dict[str, Any]] = [] + clipped_fractions: list[float] = [] + + for index, entry in enumerate(source_manifest["samples"]): + started = time.monotonic() + source_sample = torch.load( + source_dir / entry["file"], map_location="cpu", weights_only=True + ) + source_indices = source_sample["source_indices"].numpy() + geometry = load_egoverse_world_geometry( + Path(source_sample["annotation_path"]), + source_indices, + resolution, + point_config, + ) + depth_01 = source_sample["depth_gray"][0].float().numpy() + depth_m = depth_01 * (depth_max - depth_min) + depth_min + valid = _valid_mask_t_hw(source_sample["depth_valid_mask"]) + world_points, world_valid = depth_video_to_world_grid( + depth_m, + geometry.intrinsics, + geometry.camera_to_reference, + stride=1, + min_depth_m=float(point_config["min_depth_m"]), + max_depth_m=float(point_config["max_depth_m"]), + valid_mask=valid, + ) + point_video, clipped_fraction = normalize_world_point_video( + world_points, + world_valid, + xyz_min, + xyz_max, + invalid_fill_reference=invalid_fill, + ) + with torch.inference_mode(): + z_point = vae.encode( + torch.from_numpy(point_video).unsqueeze(0).to( + device=device, dtype=dtype + ), + device=device, + ) + if tuple(z_point.shape) != expected_shape: + raise RuntimeError( + f"Unexpected point latent for {source_sample['sample_id']}: " + f"{tuple(z_point.shape)} != {expected_shape}" + ) + payload = dict(source_sample) + payload.update( + { + "z_point": z_point[0].detach().cpu(), + "point_intrinsics": torch.from_numpy(geometry.intrinsics.copy()), + "point_camera_to_reference": torch.from_numpy( + geometry.camera_to_reference.copy() + ), + "point_clipped_fraction": float(clipped_fraction), + } + ) + cache_name = str(entry["file"]) + torch.save(payload, target_dir / cache_name) + entries.append(dict(entry)) + clipped_fractions.append(float(clipped_fraction)) + print( + f"[point-cache {index + 1:03d}/{expected_samples:03d}] " + f"{source_sample['sample_id']} clipped={clipped_fraction:.6f} " + f"{time.monotonic() - started:.1f}s", + flush=True, + ) + + vae.to("cpu") + del vae + gc.collect() + if device.type == "cuda": + torch.cuda.empty_cache() + manifest = { + "format_version": 2, + "fingerprint": fingerprint, + "source_rgb_depth_cache": str(source_dir), + "source_rgb_depth_fingerprint": source_manifest.get("fingerprint"), + "num_episodes": int(config["_num_episodes"]), + "clips_per_episode": int(config["_clips_per_episode"]), + "num_samples": len(entries), + "latent_shape": list(expected_shape[1:]), + "modalities": ["rgb", "depth", "world_point"], + "point_representation": "dense_first_camera_xyz_point_map_video", + "xyz_min_m": list(xyz_min), + "xyz_max_m": list(xyz_max), + "mean_clipped_fraction": float(np.mean(clipped_fractions)), + "max_clipped_fraction": float(np.max(clipped_fractions)), + "samples": entries, + } + target_manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + print(f"[point-cache] wrote {target_manifest_path}", flush=True) + + +def main() -> None: + args = parse_args() + config = validate_config(load_config(args.config)) + if "point_cloud" not in config or not bool(config["point_cloud"].get("enabled", False)): + raise ValueError("point_cloud.enabled must be true") + prepare_point_cache(config) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/rgb_action_model_factory.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/rgb_action_model_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..12757ec555f51509936e4cae7ba0a06e31b441ae --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/rgb_action_model_factory.py @@ -0,0 +1,96 @@ +"""Factory for the isolated RGB-video + action ablation model.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +import torch + +from online_egoverse_action import OnlineActionNormalizer +from train_joint_rgb_depth import ( + WanVideoDiT, + dtype_from_name, + precompute_freqs_cis_3d, + strict_load_original_dit, + temporary_default_dtype, +) +from wan_rgb_action_mot import RgbActionMoT, build_rgb_action_mot + + +def validate_rgb_action_model_config(config: dict[str, Any]) -> None: + model = config["model"] + if model.get("ablation_architecture") != "rgb_action_mot": + raise ValueError("model.ablation_architecture must be rgb_action_mot") + dit = model.get("dit") + action_dit = model.get("action_dit") + if not isinstance(dit, dict) or not isinstance(action_dit, dict): + raise ValueError("model.dit and model.action_dit must be mappings") + if int(dit.get("in_dim", -1)) != 48 or int(dit.get("out_dim", -1)) != 48: + raise ValueError("The RGB-only Wan expert must use 48 input/output channels") + for key in ("num_layers", "num_heads", "attn_head_dim"): + if int(action_dit.get(key, -1)) != int(dit.get(key, -2)): + raise ValueError(f"model.action_dit.{key} must match model.dit.{key}") + if int(action_dit.get("action_dim", -1)) != int(model.get("action_state_dim", -2)): + raise ValueError("action_dim and action_state_dim must match") + if model.get("cross_modal_attention") not in { + "bidirectional", + "action_reads_video", + }: + raise ValueError("Unsupported model.cross_modal_attention") + if model.get("action_loss_to_video_stop_gradient") is not False: + raise ValueError( + "This sibling requires model.action_loss_to_video_stop_gradient=false" + ) + + +def rgb_action_checkpoint_metadata(model: RgbActionMoT) -> dict[str, Any]: + return model.architecture_metadata() + + +def build_fresh_rgb_action_mot(config: dict[str, Any]) -> RgbActionMoT: + """Load only the Wan2.2 RGB base and randomly initialize action modules.""" + + validate_rgb_action_model_config(config) + model_config = config["model"] + dit_config = dict(model_config["dit"]) + dtype = dtype_from_name(str(model_config["dtype"])) + load_weights_on_this_rank = int(os.environ.get("LOCAL_RANK", "0")) == 0 + with torch.device("meta"), temporary_default_dtype(dtype): + video_expert = WanVideoDiT(**dit_config) + video_expert.to_empty(device="cpu") + video_expert.freqs = precompute_freqs_cis_3d( + int(dit_config["attn_head_dim"]) + ) + if any(parameter.is_meta for parameter in video_expert.parameters()): + raise RuntimeError("WanVideoDiT still contains meta parameters") + if load_weights_on_this_rank: + strict_load_original_dit( + video_expert, Path(config["paths"]["pretrained_model_dir"]) + ) + else: + print( + "[model-load] nonzero local rank awaits distributed module-state sync", + flush=True, + ) + model = build_rgb_action_mot( + video_expert, + model_config["action_dit"], + random_seed=int(model_config["action_random_seed"]), + action_state_dim=int(model_config["action_state_dim"]), + mot_checkpoint_mixed_attn=bool( + model_config.get("mot_checkpoint_mixed_attn", True) + ), + cross_modal_attention=str(model_config["cross_modal_attention"]), + ) + normalization = config["action"]["normalization"] + model.add_module( + "action_normalizer", + OnlineActionNormalizer( + action_dim=int(model_config["action_state_dim"]), + eps=float(normalization.get("eps", 1.0e-6)), + clip=normalization.get("clip", 10.0), + ), + ) + return model diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/rgb_depth_point_action_model_factory.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/rgb_depth_point_action_model_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..a975cc4938ba51e007f2cce96d4c369e9b93f6cf --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/rgb_depth_point_action_model_factory.py @@ -0,0 +1,175 @@ +"""Production construction and checkpoint helpers for the four-output MoT.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +import torch + +from online_egoverse_action import OnlineActionNormalizer, OnlineCameraNormalizer +from train_joint_rgb_depth_point import build_joint_dit, joint_checkpoint_metadata +from wan_rgb_depth_point_action_mot import RgbDepthPointActionMoT, build_action_mot + + +def validate_action_mot_config(config: dict[str, Any]) -> None: + model = config.get("model", {}) + if model.get("joint_rgb_depth_point_action_mot") is not True: + raise ValueError("model.joint_rgb_depth_point_action_mot must be true") + if model.get("action_initialization") != "random": + raise ValueError("This sibling requires model.action_initialization=random") + action_config = model.get("action_dit") + if not isinstance(action_config, dict): + raise ValueError("model.action_dit must be a mapping") + video_config = model.get("dit", {}) + for key in ("num_layers", "num_heads", "attn_head_dim"): + if int(action_config.get(key, -1)) != int(video_config.get(key, -2)): + raise ValueError(f"model.action_dit.{key} must match model.dit.{key}") + if int(action_config.get("action_dim", 0)) <= 0: + raise ValueError("model.action_dit.action_dim must be positive") + if int(action_config.get("hidden_dim", 0)) <= 0: + raise ValueError("model.action_dit.hidden_dim must be positive") + if int(action_config.get("ffn_dim", 0)) <= 0: + raise ValueError("model.action_dit.ffn_dim must be positive") + action = config.get("action", {}) + if action.get("source") != "online_egoverse_pose_keypoints": + raise ValueError("action.source must be online_egoverse_pose_keypoints") + if action.get("representation") != "dual_arm_se3_body_log_plus_grip": + raise ValueError("Unsupported online action representation") + if action.get("persist_derived_data") is not False: + raise ValueError("Online action pipeline requires persist_derived_data=false") + if int(action.get("action_dim", -1)) != int(action_config["action_dim"]): + raise ValueError("action.action_dim must match model.action_dit.action_dim") + if int(model.get("action_state_dim", -1)) != int(action_config["action_dim"]): + raise ValueError( + "model.action_state_dim must match action_dit.action_dim for aligned slots" + ) + if model.get("action_state_conditioning") != "initial_state_context_token": + raise ValueError( + "model.action_state_conditioning must be initial_state_context_token" + ) + initial_state = action.get("initial_state", {}) + expected_initial_state = { + "role": "clean_action_context", + "representation": "dual_arm_absolute_se3_log_plus_grip", + "action_slot_aligned_dim": int(action_config["action_dim"]), + "source_frame": "first", + "add_diffusion_noise": False, + "prediction_target": False, + "loss": "none", + } + for key, expected in expected_initial_state.items(): + if initial_state.get(key) != expected: + raise ValueError( + f"action.initial_state.{key} must be {expected!r}, " + f"got {initial_state.get(key)!r}" + ) + camera = config.get("camera", {}) + if camera.get("enabled") is not True: + raise ValueError("camera.enabled must be true") + if camera.get("representation") != "first_camera_relative_translation_so3_log": + raise ValueError("Unsupported camera representation") + if int(camera.get("camera_dim", -1)) != 6: + raise ValueError("camera.camera_dim must be 6") + if int(model.get("camera_dim", -1)) != int(camera["camera_dim"]): + raise ValueError("model.camera_dim must match camera.camera_dim") + if camera.get("initial_state_conditioning") != "none": + raise ValueError("Camera must not receive a clean initial camera state") + cross_modal = model.get("cross_modal_attention", "bidirectional") + if cross_modal not in {"bidirectional", "action_reads_video"}: + raise ValueError("Unsupported model.cross_modal_attention") + if model.get("action_loss_to_video_stop_gradient") is not False: + raise ValueError( + "This sibling requires model.action_loss_to_video_stop_gradient=false" + ) + + +def mot_checkpoint_metadata(model: RgbDepthPointActionMoT) -> dict[str, Any]: + return { + "video": joint_checkpoint_metadata(model.video_expert), + "mot": model.architecture_metadata(), + } + + +def build_joint_action_mot( + config: dict[str, Any], + resume_model: Path | None = None, + *, + load_pretrained: bool = True, +) -> RgbDepthPointActionMoT: + """Load/upgrade Wan video weights, then randomly create the action expert. + + On a fresh run, only the RGB Wan checkpoint is loaded. Depth/point + projections follow the base sibling's initialization and ActionDiT is + random. ``resume_model`` is the only path allowed to overwrite action + weights, and it must be a full combined MoT checkpoint. + """ + + validate_action_mot_config(config) + # A full MoT resume is loaded only after both experts exist. The base + # builder therefore takes no resume path here. + video_expert = build_joint_dit( + config, resume_model=None, load_pretrained=load_pretrained + ) + model_config = config["model"] + model = build_action_mot( + video_expert, + model_config["action_dit"], + random_seed=int(model_config.get("action_random_seed", config["experiment"]["seed"])), + action_state_dim=int(model_config["action_state_dim"]), + camera_dim=int(model_config["camera_dim"]), + mot_checkpoint_mixed_attn=bool( + model_config.get("mot_checkpoint_mixed_attn", True) + ), + cross_modal_attention=str( + model_config.get("cross_modal_attention", "bidirectional") + ), + ) + normalization = config.get("action", {}).get("normalization", {}) + model.add_module( + "action_normalizer", + OnlineActionNormalizer( + action_dim=int(model.action_expert.action_dim), + eps=float(normalization.get("eps", 1.0e-6)), + clip=normalization.get("clip", 10.0), + ), + ) + camera_normalization = config.get("camera", {}).get("normalization", {}) + model.add_module( + "camera_normalizer", + OnlineCameraNormalizer( + camera_dim=int(model.camera_dim), + eps=float(camera_normalization.get("eps", 1.0e-6)), + clip=camera_normalization.get("clip", 10.0), + ), + ) + load_weights_on_this_rank = int(os.environ.get("LOCAL_RANK", "0")) == 0 + if resume_model is not None and load_weights_on_this_rank: + payload = torch.load( + resume_model, map_location="cpu", weights_only=True, mmap=True + ) + if not isinstance(payload, dict) or "state_dict" not in payload: + raise ValueError(f"Invalid RGB/depth/point/action MoT checkpoint: {resume_model}") + expected = mot_checkpoint_metadata(model) + if payload.get("metadata") != expected: + raise RuntimeError( + f"MoT checkpoint metadata mismatch: {payload.get('metadata')} != {expected}" + ) + incompatible = model.load_state_dict(payload["state_dict"], strict=True) + if incompatible.missing_keys or incompatible.unexpected_keys: + raise RuntimeError(f"Strict MoT checkpoint mismatch: {incompatible}") + return model + + +def configure_trainable_parameters( + model: RgbDepthPointActionMoT, + *, + train_video_expert: bool = True, +) -> None: + """Train ActionDiT always; optionally freeze the existing video expert.""" + + model.requires_grad_(True) + if not train_video_expert: + model.video_expert.requires_grad_(False) + model.action_expert.requires_grad_(True) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_ego_first_cam.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_ego_first_cam.py new file mode 100644 index 0000000000000000000000000000000000000000..4c9501190bf48f9e415212c8ee26c8a35d5f90d9 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_ego_first_cam.py @@ -0,0 +1,349 @@ +"""Shared RoboDojo episode-first-camera SE(3) action-frame bridge. + +The RoboDojo command/state frame is denoted ``B`` and the episode's first +``cam_head`` frame is ``C0``. Matrices follow the unambiguous +``T_target_source`` convention: ``T_B_C0`` maps C0 coordinates into B and +``T_C0_B = inverse(T_B_C0)`` maps B coordinates into C0. + +All geometry is evaluated in float64. Dataset/model boundary tensors are +returned as float32. Quaternions are scalar-first ``wxyz`` throughout. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + + +ACTION_DIM = 14 +POSE16_DIM = 16 +QUATERNION_LAYOUT = "wxyz" +MATRIX_CONVENTION = "T_target_source" +ACTION_DIMENSION_ORDER = ( + "left_x", + "left_y", + "left_z", + "left_so3_x", + "left_so3_y", + "left_so3_z", + "left_grip", + "right_x", + "right_y", + "right_z", + "right_so3_x", + "right_so3_y", + "right_so3_z", + "right_grip", +) + + +def _as_float64(value: np.ndarray, shape: tuple[int, ...], name: str) -> np.ndarray: + array = np.asarray(value, dtype=np.float64) + if array.shape != shape: + raise ValueError(f"{name} must have shape {shape}, got {array.shape}") + if not np.isfinite(array).all(): + raise ValueError(f"{name} contains NaN/Inf") + return array + + +def validate_rigid_transform( + transform: np.ndarray, + *, + name: str = "transform", + atol: float = 1.0e-6, +) -> np.ndarray: + """Return a float64 SE(3) matrix or fail closed.""" + + matrix = _as_float64(transform, (4, 4), name) + if not np.allclose(matrix[3], [0.0, 0.0, 0.0, 1.0], atol=atol, rtol=0.0): + raise ValueError(f"{name} has an invalid homogeneous last row") + rotation = matrix[:3, :3] + if not np.allclose(rotation.T @ rotation, np.eye(3), atol=atol, rtol=0.0): + raise ValueError(f"{name} rotation is not orthonormal") + determinant = float(np.linalg.det(rotation)) + if not np.isclose(determinant, 1.0, atol=atol, rtol=0.0): + raise ValueError(f"{name} rotation determinant is {determinant}, expected +1") + return matrix + + +def rigid_inverse(transform: np.ndarray) -> np.ndarray: + matrix = validate_rigid_transform(transform) + output = np.eye(4, dtype=np.float64) + output[:3, :3] = matrix[:3, :3].T + output[:3, 3] = -output[:3, :3] @ matrix[:3, 3] + return output + + +def quaternion_wxyz_to_rotation(quaternion: np.ndarray) -> np.ndarray: + q = _as_float64(quaternion, (4,), "quaternion_wxyz") + norm = float(np.linalg.norm(q)) + if norm < 1.0e-8: + raise ValueError("quaternion_wxyz has near-zero norm") + w, x, y, z = q / norm + return np.array( + [ + [1 - 2 * (y * y + z * z), 2 * (x * y - w * z), 2 * (x * z + w * y)], + [2 * (x * y + w * z), 1 - 2 * (x * x + z * z), 2 * (y * z - w * x)], + [2 * (x * z - w * y), 2 * (y * z + w * x), 1 - 2 * (x * x + y * y)], + ], + dtype=np.float64, + ) + + +def rotation_to_quaternion_wxyz(rotation: np.ndarray) -> np.ndarray: + matrix = _as_float64(rotation, (3, 3), "rotation") + validate_rigid_transform( + np.block([[matrix, np.zeros((3, 1))], [np.zeros((1, 3)), np.ones((1, 1))]]), + name="rotation_as_transform", + ) + trace = float(np.trace(matrix)) + if trace > 0.0: + scale = np.sqrt(trace + 1.0) * 2.0 + quaternion = np.array( + [ + 0.25 * scale, + (matrix[2, 1] - matrix[1, 2]) / scale, + (matrix[0, 2] - matrix[2, 0]) / scale, + (matrix[1, 0] - matrix[0, 1]) / scale, + ] + ) + else: + axis = int(np.argmax(np.diag(matrix))) + if axis == 0: + scale = np.sqrt(1.0 + matrix[0, 0] - matrix[1, 1] - matrix[2, 2]) * 2.0 + quaternion = np.array( + [ + (matrix[2, 1] - matrix[1, 2]) / scale, + 0.25 * scale, + (matrix[0, 1] + matrix[1, 0]) / scale, + (matrix[0, 2] + matrix[2, 0]) / scale, + ] + ) + elif axis == 1: + scale = np.sqrt(1.0 + matrix[1, 1] - matrix[0, 0] - matrix[2, 2]) * 2.0 + quaternion = np.array( + [ + (matrix[0, 2] - matrix[2, 0]) / scale, + (matrix[0, 1] + matrix[1, 0]) / scale, + 0.25 * scale, + (matrix[1, 2] + matrix[2, 1]) / scale, + ] + ) + else: + scale = np.sqrt(1.0 + matrix[2, 2] - matrix[0, 0] - matrix[1, 1]) * 2.0 + quaternion = np.array( + [ + (matrix[1, 0] - matrix[0, 1]) / scale, + (matrix[0, 2] + matrix[2, 0]) / scale, + (matrix[1, 2] + matrix[2, 1]) / scale, + 0.25 * scale, + ] + ) + quaternion /= np.linalg.norm(quaternion) + return quaternion if quaternion[0] >= 0.0 else -quaternion + + +def pose7_wxyz_to_matrix(pose: np.ndarray) -> np.ndarray: + value = _as_float64(pose, (7,), "pose_txyz_qwxyz") + output = np.eye(4, dtype=np.float64) + output[:3, :3] = quaternion_wxyz_to_rotation(value[3:7]) + output[:3, 3] = value[:3] + return output + + +def matrix_to_pose7_wxyz(transform: np.ndarray) -> np.ndarray: + matrix = validate_rigid_transform(transform) + return np.concatenate( + [matrix[:3, 3], rotation_to_quaternion_wxyz(matrix[:3, :3])] + ) + + +def _skew(vector: np.ndarray) -> np.ndarray: + x, y, z = _as_float64(vector, (3,), "rotation_vector") + return np.array([[0.0, -z, y], [z, 0.0, -x], [-y, x, 0.0]]) + + +def so3_log(rotation: np.ndarray) -> np.ndarray: + matrix = _as_float64(rotation, (3, 3), "rotation") + cosine = float(np.clip((np.trace(matrix) - 1.0) * 0.5, -1.0, 1.0)) + angle = float(np.arccos(cosine)) + vee = 0.5 * np.array( + [ + matrix[2, 1] - matrix[1, 2], + matrix[0, 2] - matrix[2, 0], + matrix[1, 0] - matrix[0, 1], + ], + dtype=np.float64, + ) + if angle < 1.0e-7: + return vee + sine = float(np.sin(angle)) + if abs(sine) > 1.0e-6: + return vee * (angle / sine) + diagonal = np.maximum((np.diag(matrix) + 1.0) * 0.5, 0.0) + axis = np.sqrt(diagonal) + largest = int(np.argmax(axis)) + if axis[largest] < 1.0e-7: + raise ValueError("Cannot recover SO(3) axis near pi") + if largest == 0: + axis[1] = np.copysign(axis[1], matrix[0, 1] + matrix[1, 0]) + axis[2] = np.copysign(axis[2], matrix[0, 2] + matrix[2, 0]) + elif largest == 1: + axis[0] = np.copysign(axis[0], matrix[0, 1] + matrix[1, 0]) + axis[2] = np.copysign(axis[2], matrix[1, 2] + matrix[2, 1]) + else: + axis[0] = np.copysign(axis[0], matrix[0, 2] + matrix[2, 0]) + axis[1] = np.copysign(axis[1], matrix[1, 2] + matrix[2, 1]) + axis /= np.linalg.norm(axis) + return axis * angle + + +def so3_exp(rotation_vector: np.ndarray) -> np.ndarray: + vector = _as_float64(rotation_vector, (3,), "rotation_vector") + angle = float(np.linalg.norm(vector)) + omega = _skew(vector) + identity = np.eye(3, dtype=np.float64) + if angle < 1.0e-7: + return identity + omega + 0.5 * (omega @ omega) + return ( + identity + + (np.sin(angle) / angle) * omega + + ((1.0 - np.cos(angle)) / (angle * angle)) * (omega @ omega) + ) + + +def absolute_xyz_so3_to_matrix(value: np.ndarray) -> np.ndarray: + vector = _as_float64(value, (6,), "absolute_xyz_so3") + output = np.eye(4, dtype=np.float64) + output[:3, :3] = so3_exp(vector[3:6]) + output[:3, 3] = vector[:3] + return output + + +def matrix_to_absolute_xyz_so3(transform: np.ndarray) -> np.ndarray: + matrix = validate_rigid_transform(transform) + return np.concatenate([matrix[:3, 3], so3_log(matrix[:3, :3])]) + + +def _split_pose16(pose16: np.ndarray) -> tuple[np.ndarray, float, np.ndarray, float]: + value = _as_float64(pose16, (POSE16_DIM,), "dual_arm_pose16") + return value[:7], float(value[7]), value[8:15], float(value[15]) + + +def transform_pose16_base_to_c0( + pose16: np.ndarray, t_b_c0: np.ndarray +) -> np.ndarray: + """Apply ``T_C0_E = inverse(T_B_C0) @ T_B_E`` to both arms.""" + + left, left_grip, right, right_grip = _split_pose16(pose16) + t_c0_b = rigid_inverse(t_b_c0) + left_c0 = t_c0_b @ pose7_wxyz_to_matrix(left) + right_c0 = t_c0_b @ pose7_wxyz_to_matrix(right) + return np.concatenate( + [ + matrix_to_pose7_wxyz(left_c0), + [left_grip], + matrix_to_pose7_wxyz(right_c0), + [right_grip], + ] + ).astype(np.float32) + + +def transform_pose16_c0_to_base( + pose16: np.ndarray, t_b_c0: np.ndarray +) -> np.ndarray: + """Apply ``T_B_E = T_B_C0 @ T_C0_E`` to both arms.""" + + left, left_grip, right, right_grip = _split_pose16(pose16) + t_b_c0 = validate_rigid_transform(t_b_c0, name="T_B_C0") + left_base = t_b_c0 @ pose7_wxyz_to_matrix(left) + right_base = t_b_c0 @ pose7_wxyz_to_matrix(right) + return np.concatenate( + [ + matrix_to_pose7_wxyz(left_base), + [left_grip], + matrix_to_pose7_wxyz(right_base), + [right_grip], + ] + ).astype(np.float32) + + +def transform_pose16_sequence_base_to_c0( + pose16: np.ndarray, t_b_c0: np.ndarray +) -> np.ndarray: + sequence = np.asarray(pose16) + if sequence.ndim != 2 or sequence.shape[1] != POSE16_DIM: + raise ValueError(f"dual_arm_pose16 sequence must be [T,16], got {sequence.shape}") + return np.stack( + [transform_pose16_base_to_c0(row, t_b_c0) for row in sequence], axis=0 + ) + + +def pose16_to_action14(pose16: np.ndarray) -> np.ndarray: + left, left_grip, right, right_grip = _split_pose16(pose16) + return np.concatenate( + [ + matrix_to_absolute_xyz_so3(pose7_wxyz_to_matrix(left)), + [left_grip], + matrix_to_absolute_xyz_so3(pose7_wxyz_to_matrix(right)), + [right_grip], + ] + ).astype(np.float32) + + +def action14_c0_to_pose16_base( + action14: np.ndarray, t_b_c0: np.ndarray +) -> np.ndarray: + """Decode one model row in C0 and deterministically return a B-frame pose16.""" + + row = _as_float64(action14, (ACTION_DIM,), "action14") + left_c0 = absolute_xyz_so3_to_matrix(row[0:6]) + right_c0 = absolute_xyz_so3_to_matrix(row[7:13]) + pose_c0 = np.concatenate( + [ + matrix_to_pose7_wxyz(left_c0), + [row[6]], + matrix_to_pose7_wxyz(right_c0), + [row[13]], + ] + ) + return transform_pose16_c0_to_base(pose_c0, t_b_c0) + + +def base_pose16_to_action14_c0( + pose16: np.ndarray, t_b_c0: np.ndarray +) -> np.ndarray: + """Training/inference shared state conversion from B-frame pose16 to C0 14D.""" + + return pose16_to_action14(transform_pose16_base_to_c0(pose16, t_b_c0)) + + +@dataclass(frozen=True) +class EpisodeFirstCameraBridge: + """Immutable episode bridge holding the fixed first camera pose.""" + + t_b_c0: np.ndarray + + def __post_init__(self) -> None: + object.__setattr__( + self, + "t_b_c0", + validate_rigid_transform(self.t_b_c0, name="T_B_C0").copy(), + ) + + @property + def t_c0_b(self) -> np.ndarray: + return rigid_inverse(self.t_b_c0) + + def state_base_to_c0(self, pose16: np.ndarray) -> np.ndarray: + return base_pose16_to_action14_c0(pose16, self.t_b_c0) + + def pose_base_to_c0(self, pose16: np.ndarray) -> np.ndarray: + return transform_pose16_base_to_c0(pose16, self.t_b_c0) + + def pose_c0_to_base(self, pose16: np.ndarray) -> np.ndarray: + return transform_pose16_c0_to_base(pose16, self.t_b_c0) + + def action_c0_to_base(self, action14: np.ndarray) -> np.ndarray: + return action14_c0_to_pose16_base(action14, self.t_b_c0) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_geometry_data.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_geometry_data.py new file mode 100644 index 0000000000000000000000000000000000000000..641f469272c18860322dab0b5d4996d9d7b9fee5 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_geometry_data.py @@ -0,0 +1,320 @@ +"""RoboDojo VDA depth and deterministic camera-frame point-map utilities. + +The geometry annotations are aligned one-to-one with the LeRobot episode and +frame indices. This module deliberately has no training-loop dependencies so +the same validation and preprocessing contract is used by cache preparation, +dataset smoke tests, and formal training. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +from depth_preprocessing import DepthPreprocessingConfig, preprocess_depth_grayscale +from world_point_cloud import depth_video_to_world_grid, normalize_world_point_video + + +DEPTH_KEY = "depth_mm" +LATENT_SHAPE = (48, 9, 16, 16) +EXPECTED_DEPTH_DATASET_ROOT = Path( + "/dataset_rc_rl/qshou/robodojo_depth_vda_cam_high_25fps_256" +) + + +def geometry_modalities(config: dict[str, Any]) -> tuple[str, ...]: + modalities = tuple(str(value) for value in config["geometry"]["modalities"]) + allowed = { + ("rgb", "depth"), + ("rgb", "depth", "point"), + } + if modalities not in allowed: + raise ValueError( + "geometry.modalities must be exactly [rgb, depth] or " + f"[rgb, depth, point], got {modalities}" + ) + return modalities + + +def validate_geometry_config(config: dict[str, Any]) -> dict[str, Any]: + modalities = geometry_modalities(config) + paths = config["paths"] + depth_root = Path(paths["depth_dataset_root"]) + if not depth_root.is_dir(): + raise FileNotFoundError(depth_root) + if depth_root.resolve() != EXPECTED_DEPTH_DATASET_ROOT.resolve(): + raise ValueError( + "Controlled RoboDojo geometry experiments must use the complete " + f"depth annotation root {EXPECTED_DEPTH_DATASET_ROOT}, got {depth_root}" + ) + stats_path = depth_root / "manifests" / "episodes.stats.json" + if not stats_path.is_file(): + raise FileNotFoundError(stats_path) + stats = json.loads(stats_path.read_text(encoding="utf-8")) + expected_stats = { + "adapter": "robodojo_lerobot_v3", + "episodes": 3500, + "eligible": 3500, + "ineligible": 0, + "source_fps": 25.0, + "output_size": 256, + "spatial_transform": "center_square_resize", + } + actual_stats = {key: stats.get(key) for key in expected_stats} + if actual_stats != expected_stats: + raise ValueError( + f"RoboDojo depth annotation audit mismatch: {actual_stats} != {expected_stats}" + ) + episode_manifest = depth_root / "manifests" / "episodes.jsonl" + if not episode_manifest.is_file(): + raise FileNotFoundError(episode_manifest) + with episode_manifest.open("rb") as handle: + manifest_records = sum(1 for line in handle if line.strip()) + if manifest_records != 3500: + raise ValueError( + f"RoboDojo depth manifest must contain 3500 episodes, got {manifest_records}" + ) + + initialization = config.get("initialization", {}) + if initialization.get("mode") != "fresh": + raise ValueError("Geometry experiments require initialization.mode=fresh") + forbidden_initialization = { + "checkpoint", + "local_checkpoint", + "flavor", + }.intersection(initialization) + if forbidden_initialization: + raise ValueError( + "Fresh geometry experiments may not define checkpoint initialization: " + f"{sorted(forbidden_initialization)}" + ) + if "checkpoint" in paths: + raise ValueError("Fresh geometry experiments may not define paths.checkpoint") + model = config["model"] + expected_model_flags = { + "action_initialization": "random", + "cross_modal_attention": "bidirectional", + "action_loss_to_video_stop_gradient": False, + "mot_checkpoint_mixed_attn": False, + } + actual_model_flags = {key: model.get(key) for key in expected_model_flags} + if actual_model_flags != expected_model_flags: + raise ValueError( + f"No-stopgrad geometry model mismatch: {actual_model_flags} " + f"!= {expected_model_flags}" + ) + + depth = config["geometry"]["depth"] + expected_depth = { + "representation": "linear_metric_grayscale", + "input_unit": "millimeter", + "min_m": 0.0, + "max_m": 10.0, + "invalid_value": 0.0, + "invalid_fill": "far", + "first_frame_condition": True, + "exclude_first_frame_from_loss": True, + } + actual_depth = {key: depth.get(key) for key in expected_depth} + if actual_depth != expected_depth: + raise ValueError(f"Depth contract mismatch: {actual_depth} != {expected_depth}") + if float(config["diffusion"].get("lambda_depth", 0.0)) != 1.0: + raise ValueError("The controlled ablation requires diffusion.lambda_depth=1.0") + if "depth_latent_cache_dir" not in paths: + raise KeyError("paths.depth_latent_cache_dir is required") + + point = config["geometry"].get("point") + if "point" in modalities: + if not isinstance(point, dict): + raise ValueError("RGB/depth/point experiment requires geometry.point") + expected_point = { + "representation": "dense_xyz_point_map_video", + "coordinate_frame": "opencv_camera_x_right_y_down_z_forward", + "derived_from": "depth_mm_and_K", + "min_depth_m": 0.05, + "max_depth_m": 10.0, + "xyz_min_m": [-8.5, -8.5, 0.0], + "xyz_max_m": [8.5, 8.5, 10.0], + "invalid_fill_m": [0.0, 0.0, 5.0], + "first_frame_condition": False, + "exclude_first_frame_from_loss": False, + "timestep_adapter_bottleneck_dim": 256, + } + actual_point = {key: point.get(key) for key in expected_point} + if actual_point != expected_point: + raise ValueError(f"Point contract mismatch: {actual_point} != {expected_point}") + if float(config["diffusion"].get("lambda_point", 0.0)) != 1.0: + raise ValueError("The controlled ablation requires diffusion.lambda_point=1.0") + if "point_latent_cache_dir" not in paths: + raise KeyError("paths.point_latent_cache_dir is required") + elif point is not None: + raise ValueError("Depth-only experiment must not define geometry.point") + return config + + +def depth_episode_path(depth_root: Path, episode_index: int) -> Path: + index = int(episode_index) + return ( + depth_root + / "episodes" + / "RoboDojo" + / f"chunk-{index // 1000:03d}" + / f"episode_{index:06d}.npz" + ) + + +def depth_source_record(path: Path) -> dict[str, Any]: + stat = path.stat() + return { + "path": str(path.resolve()), + "size": int(stat.st_size), + "mtime_ns": int(stat.st_mtime_ns), + } + + +def load_depth_episode_bundle( + depth_root: Path, + episode_index: int, + *, + expected_length: int | None = None, +) -> dict[str, Any]: + path = depth_episode_path(depth_root, episode_index) + if not path.is_file(): + raise FileNotFoundError(path) + with np.load(path, allow_pickle=False) as payload: + required = {DEPTH_KEY, "frame_index", "timestamp", "K", "metadata_json"} + missing = required - set(payload.files) + if missing: + raise KeyError(f"Depth annotation {path} is missing {sorted(missing)}") + bundle = {key: payload[key].copy() for key in required} + depth = bundle[DEPTH_KEY] + frame_index = bundle["frame_index"] + if depth.dtype != np.uint16 or depth.ndim != 3 or depth.shape[1:] != (256, 256): + raise ValueError(f"Unexpected depth tensor in {path}: {depth.shape} {depth.dtype}") + if frame_index.shape != (len(depth),) or not np.array_equal( + frame_index, np.arange(len(depth), dtype=frame_index.dtype) + ): + raise ValueError(f"Depth frame indices are not contiguous in {path}") + if expected_length is not None and len(depth) != int(expected_length): + raise ValueError( + f"Depth/RoboDojo episode length mismatch for {path}: " + f"{len(depth)} != {expected_length}" + ) + k = np.asarray(bundle["K"], dtype=np.float32) + if k.shape != (3, 3) or not np.isfinite(k).all() or min(k[0, 0], k[1, 1]) <= 0: + raise ValueError(f"Invalid camera intrinsics in {path}: {k}") + return bundle + + +def build_geometry_vae_inputs( + bundle: dict[str, Any], + indices: np.ndarray, + config: dict[str, Any], +) -> dict[str, torch.Tensor | float]: + indices = np.asarray(indices, dtype=np.int64) + if indices.ndim != 1 or len(indices) != int(config["data"]["num_frames"]): + raise ValueError("Geometry indices must contain exactly one configured window") + return _build_geometry_vae_inputs(bundle, indices, config) + + +def build_geometry_episode_vae_inputs( + bundle: dict[str, Any], config: dict[str, Any] +) -> dict[str, torch.Tensor | float]: + """Preprocess every episode frame once before slicing overlapping windows.""" + + depth_all = bundle[DEPTH_KEY] + if depth_all.ndim != 3 or len(depth_all) < int(config["data"]["num_frames"]): + raise ValueError("Geometry episode is shorter than one configured window") + indices = np.arange(len(depth_all), dtype=np.int64) + return _build_geometry_vae_inputs(bundle, indices, config) + + +def _build_geometry_vae_inputs( + bundle: dict[str, Any], + indices: np.ndarray, + config: dict[str, Any], +) -> dict[str, torch.Tensor | float]: + if np.any(np.diff(indices) != 1): + raise ValueError("RoboDojo geometry windows must use consecutive native frames") + depth_all = bundle[DEPTH_KEY] + if int(indices[0]) < 0 or int(indices[-1]) >= len(depth_all): + raise IndexError("Geometry window lies outside the annotated episode") + depth_mm = depth_all[indices] + depth_config = config["geometry"]["depth"] + processed = preprocess_depth_grayscale( + depth_mm, + config=DepthPreprocessingConfig( + depth_min=float(depth_config["min_m"]), + depth_max=float(depth_config["max_m"]), + depth_invalid_value=float(depth_config["invalid_value"]), + depth_invalid_fill=str(depth_config["invalid_fill"]), + ), + input_unit=str(depth_config["input_unit"]), + ) + result: dict[str, torch.Tensor | float] = { + "depth": torch.from_numpy(processed.vae_video).permute(1, 0, 2, 3), + "depth_valid_fraction": float(processed.valid_mask.mean()), + } + if "point" not in geometry_modalities(config): + return result + + point_config = config["geometry"]["point"] + depth_m = depth_mm.astype(np.float32) / np.float32(1000.0) + identity = np.broadcast_to( + np.eye(4, dtype=np.float32), (len(indices), 4, 4) + ).copy() + points, valid = depth_video_to_world_grid( + depth_m, + np.asarray(bundle["K"], dtype=np.float32), + identity, + stride=1, + min_depth_m=float(point_config["min_depth_m"]), + max_depth_m=float(point_config["max_depth_m"]), + valid_mask=processed.valid_mask[:, 0], + ) + point_video, clipped_fraction = normalize_world_point_video( + points, + valid, + point_config["xyz_min_m"], + point_config["xyz_max_m"], + invalid_fill_reference=point_config["invalid_fill_m"], + ) + result["point"] = torch.from_numpy(point_video) + result["point_valid_fraction"] = float(valid.mean()) + result["point_clipped_fraction"] = clipped_fraction + return result + + +def geometry_cache_fingerprint( + config: dict[str, Any], source_fingerprint: str, modality: str +) -> str: + if modality not in {"depth", "point"}: + raise ValueError(f"Unsupported geometry cache modality: {modality}") + if modality == "point" and "point" not in geometry_modalities(config): + raise ValueError("Point fingerprint requested for a depth-only config") + depth_root = Path(config["paths"]["depth_dataset_root"]) + stats_path = depth_root / "manifests" / "episodes.stats.json" + stats_stat = stats_path.stat() + payload: dict[str, Any] = { + "format_version": 1, + "source_fingerprint": str(source_fingerprint), + "modality": modality, + "depth_dataset_root": str(depth_root.resolve()), + "depth_stats": { + "size": int(stats_stat.st_size), + "mtime_ns": int(stats_stat.st_mtime_ns), + "content": json.loads(stats_path.read_text(encoding="utf-8")), + }, + "depth": config["geometry"]["depth"], + "num_frames": int(config["data"]["num_frames"]), + "resolution": int(config["data"]["resolution"]), + "fps": float(config["data"]["fps"]), + } + if modality == "point": + payload["point"] = config["geometry"]["point"] + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_lerobot_v30.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_lerobot_v30.py new file mode 100644 index 0000000000000000000000000000000000000000..0d535040f52d3748bb3ad8298f08237fc044add2 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_lerobot_v30.py @@ -0,0 +1,444 @@ +"""RoboDojo LeRobot v3.0 adapter for the isolated RGB+Action ablation. + +RoboDojo stores a 16D current EE state and a 16D next absolute EE target: +``[left xyz, left quaternion wxyz, left grip, right xyz, right quaternion +wxyz, right grip]``. This adapter converts the command to the Egoverse-style +14D target ``[left body SE(3) delta, next grip, right body SE(3) delta, next +grip]`` by evaluating ``log(inv(T_state_t) @ T_action_t)``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +import random +from typing import Any, Mapping + +import av +import cv2 +import numpy as np +import polars as pl +import torch + +from galexea_mcap import pose_to_matrix +from online_egoverse_action import ACTION_DIM, _rigid_inverse, se3_log + + +@dataclass(frozen=True) +class RoboDojoClipConfig: + num_frames: int = 33 + resolution: int = 256 + clips_per_episode: int = 2 + fps: float = 25.0 + video_key: str = "observation.images.cam_high" + + +@dataclass(frozen=True) +class RoboDojoEpisode: + episode_index: int + task: str + length: int + data_chunk_index: int + data_file_index: int + video_chunk_index: int + video_file_index: int + video_from_timestamp: float + video_to_timestamp: float + + +def read_dataset_info(root: Path) -> dict[str, Any]: + import json + + payload = json.loads((root / "meta" / "info.json").read_text(encoding="utf-8")) + if payload.get("codebase_version") != "v3.0": + raise ValueError("Expected a LeRobot v3.0 dataset") + return payload + + +def read_task_ids(path: Path) -> list[str]: + """Read a newline task allowlist while ignoring comments and blank lines.""" + + task_ids = [] + for raw_line in path.read_text(encoding="utf-8").splitlines(): + task_id = raw_line.split("#", 1)[0].strip() + if task_id: + task_ids.append(task_id) + if not task_ids: + raise ValueError(f"Task file is empty: {path}") + duplicates = sorted({task_id for task_id in task_ids if task_ids.count(task_id) > 1}) + if duplicates: + raise ValueError(f"Duplicate task ids in {path}: {duplicates}") + return task_ids + + +def read_dataset_task_prompts(root: Path) -> dict[int, str]: + """Return the LeRobot v3 task-index to natural-language prompt table.""" + + table = pl.read_parquet(root / "meta" / "tasks.parquet") + if "task_index" not in table.columns: + raise ValueError("RoboDojo tasks.parquet has no task_index column") + prompt_columns = [name for name in table.columns if name != "task_index"] + if len(prompt_columns) != 1: + raise ValueError(f"Expected one task prompt column, found {prompt_columns}") + prompt_column = prompt_columns[0] + result = { + int(row["task_index"]): str(row[prompt_column]) + for row in table.select("task_index", prompt_column).to_dicts() + } + if len(result) != table.height: + raise ValueError("RoboDojo tasks.parquet contains duplicate task indices") + return result + + +def resolve_training_task_prompts( + root: Path, + task_file: Path, + task_index_by_id: Mapping[str, int], +) -> dict[str, str]: + """Resolve symbolic task ids to prompts stored in LeRobot metadata. + + Random-layout eval ids deliberately map to their base-task demonstration + index. Requiring the YAML mapping to exactly match the task file makes a + later task-list edit fail closed instead of silently changing training data. + """ + + task_ids = read_task_ids(task_file) + configured = {str(key): int(value) for key, value in task_index_by_id.items()} + missing = sorted(set(task_ids) - set(configured)) + extra = sorted(set(configured) - set(task_ids)) + if missing or extra: + raise ValueError( + f"task_index_by_id must exactly match {task_file}; " + f"missing={missing}, extra={extra}" + ) + indices = [configured[task_id] for task_id in task_ids] + if len(set(indices)) != len(indices): + raise ValueError("Each selected task id must map to a distinct training task") + prompt_by_index = read_dataset_task_prompts(root) + unknown = sorted(set(indices) - set(prompt_by_index)) + if unknown: + raise ValueError(f"Unknown LeRobot task indices: {unknown}") + return {task_id: prompt_by_index[configured[task_id]] for task_id in task_ids} + + +def filter_episodes_by_task_prompts( + episodes: list[RoboDojoEpisode], task_prompts: Mapping[str, str] +) -> list[RoboDojoEpisode]: + selected_prompts = set(task_prompts.values()) + if len(selected_prompts) != len(task_prompts): + raise ValueError("Selected task ids resolve to duplicate prompts") + selected = [episode for episode in episodes if episode.task in selected_prompts] + found = {episode.task for episode in selected} + missing = sorted(selected_prompts - found) + if missing: + raise RuntimeError(f"Selected RoboDojo tasks have no episodes: {missing}") + return selected + + +def load_episodes(root: Path, video_key: str) -> list[RoboDojoEpisode]: + path = root / "meta" / "episodes" / "chunk-000" / "file-000.parquet" + prefix = f"videos/{video_key}" + columns = [ + "episode_index", + "tasks", + "length", + "data/chunk_index", + "data/file_index", + f"{prefix}/chunk_index", + f"{prefix}/file_index", + f"{prefix}/from_timestamp", + f"{prefix}/to_timestamp", + ] + rows = pl.read_parquet(path, columns=columns).sort("episode_index").to_dicts() + episodes = [] + for row in rows: + tasks = row["tasks"] + if not isinstance(tasks, list) or len(tasks) != 1: + raise ValueError( + f"Episode {row['episode_index']} must contain exactly one task" + ) + episodes.append( + RoboDojoEpisode( + episode_index=int(row["episode_index"]), + task=str(tasks[0]), + length=int(row["length"]), + data_chunk_index=int(row["data/chunk_index"]), + data_file_index=int(row["data/file_index"]), + video_chunk_index=int(row[f"{prefix}/chunk_index"]), + video_file_index=int(row[f"{prefix}/file_index"]), + video_from_timestamp=float(row[f"{prefix}/from_timestamp"]), + video_to_timestamp=float(row[f"{prefix}/to_timestamp"]), + ) + ) + return episodes + + +def stratified_episode_split( + episodes: list[RoboDojoEpisode], + per_task: dict[str, int], + seed: int, +) -> dict[str, list[RoboDojoEpisode]]: + expected_names = {"train", "validation", "test"} + if set(per_task) != expected_names: + raise ValueError(f"per_task split must contain {sorted(expected_names)}") + groups: dict[str, list[RoboDojoEpisode]] = {} + for episode in episodes: + groups.setdefault(episode.task, []).append(episode) + outputs = {name: [] for name in ("train", "validation", "test")} + required = sum(int(per_task[name]) for name in outputs) + rng = random.Random(int(seed)) + for task in sorted(groups): + group = sorted(groups[task], key=lambda value: value.episode_index) + if len(group) != required: + raise ValueError( + f"Task {task!r} has {len(group)} episodes; split requests {required}" + ) + rng.shuffle(group) + offset = 0 + for name in outputs: + count = int(per_task[name]) + outputs[name].extend(group[offset : offset + count]) + offset += count + for name in outputs: + outputs[name].sort(key=lambda value: value.episode_index) + return outputs + + +def select_clip_indices(length: int, config: RoboDojoClipConfig) -> list[np.ndarray]: + if config.num_frames < 2 or config.clips_per_episode <= 0: + raise ValueError("Invalid RoboDojo clip geometry") + available_start = int(length) - int(config.num_frames) + if available_start < 0: + raise RuntimeError(f"Episode length {length} is shorter than the clip") + starts = [ + (clip_index + 1) * available_start // (config.clips_per_episode + 1) + for clip_index in range(config.clips_per_episode) + ] + if any( + right - left < config.num_frames + for left, right in zip(starts, starts[1:], strict=False) + ): + # Short episodes can still contain multiple non-overlapping clips even + # when the preferred interior, evenly-spaced starts overlap. Preserve + # that layout for existing caches, but fall back to the widest possible + # placement (the first and last valid starts for two clips). + if config.clips_per_episode == 1: + starts = [available_start // 2] + else: + starts = [ + clip_index * available_start // (config.clips_per_episode - 1) + for clip_index in range(config.clips_per_episode) + ] + if any( + right - left < config.num_frames + for left, right in zip(starts, starts[1:], strict=False) + ): + raise RuntimeError("Requested RoboDojo clips overlap") + return [start + np.arange(config.num_frames, dtype=np.int64) for start in starts] + + +def data_parquet_path(root: Path, episode: RoboDojoEpisode) -> Path: + return ( + root + / "data" + / f"chunk-{episode.data_chunk_index:03d}" + / f"file-{episode.data_file_index:03d}.parquet" + ) + + +def video_path(root: Path, episode: RoboDojoEpisode, video_key: str) -> Path: + return ( + root + / "videos" + / video_key + / f"chunk-{episode.video_chunk_index:03d}" + / f"file-{episode.video_file_index:03d}.mp4" + ) + + +def load_data_shard(path: Path) -> pl.DataFrame: + return pl.read_parquet( + path, + columns=[ + "observation.state", + "action", + "timestamp", + "frame_index", + "episode_index", + "task_index", + ], + ) + + +def episode_arrays( + shard: pl.DataFrame, episode: RoboDojoEpisode +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + rows = shard.filter(pl.col("episode_index") == episode.episode_index).sort( + "frame_index" + ) + if rows.height != episode.length: + raise RuntimeError( + f"Episode {episode.episode_index} rows {rows.height} != {episode.length}" + ) + frame_index = rows["frame_index"].to_numpy() + if not np.array_equal(frame_index, np.arange(episode.length)): + raise RuntimeError("RoboDojo frame_index must be consecutive from zero") + timestamps = rows["timestamp"].to_numpy().astype(np.float64) + state = np.asarray(rows["observation.state"].to_list(), dtype=np.float64) + action = np.asarray(rows["action"].to_list(), dtype=np.float64) + if state.shape != (episode.length, 16) or action.shape != (episode.length, 16): + raise RuntimeError("RoboDojo state/action must be [T,16]") + if not np.isfinite(state).all() or not np.isfinite(action).all(): + raise RuntimeError("RoboDojo state/action contains non-finite values") + return state, action, timestamps + + +def _pose16_matrix(rows: np.ndarray, side: str) -> np.ndarray: + if side == "left": + position, quaternion = rows[:, 0:3], rows[:, 3:7] + elif side == "right": + position, quaternion = rows[:, 8:11], rows[:, 11:15] + else: + raise ValueError(f"Unsupported side {side!r}") + # Dataset order is [w, wx, wy, wz]; pose_to_matrix expects [x, y, z, w]. + quaternion_xyzw = quaternion[:, [1, 2, 3, 0]] + return np.stack( + [pose_to_matrix(p, q) for p, q in zip(position, quaternion_xyzw, strict=True)] + ) + + +def build_egoverse_delta_targets( + state: np.ndarray, + absolute_action: np.ndarray, + image_indices: np.ndarray, +) -> dict[str, torch.Tensor]: + indices = np.asarray(image_indices, dtype=np.int64) + if len(indices) < 2 or np.any(np.diff(indices) != 1): + raise ValueError("RoboDojo clips must use consecutive native frames") + current = state[indices[:-1]] + target = absolute_action[indices[:-1]] + left_current = _pose16_matrix(current, "left") + right_current = _pose16_matrix(current, "right") + left_target = _pose16_matrix(target, "left") + right_target = _pose16_matrix(target, "right") + action = np.zeros((len(indices) - 1, ACTION_DIM), dtype=np.float32) + for row in range(len(indices) - 1): + action[row, 0:6] = se3_log( + _rigid_inverse(left_current[row]) @ left_target[row] + ) + action[row, 7:13] = se3_log( + _rigid_inverse(right_current[row]) @ right_target[row] + ) + action[:, 6] = target[:, 7] + action[:, 13] = target[:, 15] + first = state[int(indices[0]) : int(indices[0]) + 1] + left_initial = _pose16_matrix(first, "left")[0] + right_initial = _pose16_matrix(first, "right")[0] + initial_state = np.concatenate( + [ + se3_log(left_initial), + [first[0, 7]], + se3_log(right_initial), + [first[0, 15]], + ] + ).astype(np.float32) + if not np.isfinite(action).all() or not np.isfinite(initial_state).all(): + raise RuntimeError("Converted RoboDojo targets contain non-finite values") + return { + "action_raw": torch.from_numpy(action), + "action_is_pad": torch.zeros(len(indices) - 1, dtype=torch.bool), + "initial_state": torch.from_numpy(initial_state), + "left_initial_pose": torch.from_numpy(left_initial.astype(np.float32)), + "right_initial_pose": torch.from_numpy(right_initial.astype(np.float32)), + } + + +def decode_rgb_window( + path: Path, + *, + episode_from_timestamp: float, + image_indices: np.ndarray, + config: RoboDojoClipConfig, +) -> torch.Tensor: + requested = np.asarray(image_indices, dtype=np.int64) + if requested.ndim != 1 or requested.size == 0 or np.any(np.diff(requested) < 0): + raise ValueError("image_indices must be a nonempty nondecreasing sequence") + target_times = episode_from_timestamp + requested.astype(np.float64) / config.fps + frames: list[np.ndarray] = [] + with av.open(str(path)) as container: + stream = container.streams.video[0] + if abs(float(stream.average_rate) - config.fps) > 1.0e-6: + raise RuntimeError( + f"Video FPS {stream.average_rate} does not match configured {config.fps}" + ) + seek_pts = int(round(float(target_times[0]) / float(stream.time_base))) + container.seek(seek_pts, stream=stream, backward=True, any_frame=False) + target_slot = 0 + tolerance = 0.25 / config.fps + for frame in container.decode(stream): + frame_time = float(frame.time) + if frame_time + tolerance < target_times[target_slot]: + continue + if abs(frame_time - target_times[target_slot]) > tolerance: + raise RuntimeError( + f"Video timestamp mismatch {frame_time} vs {target_times[target_slot]}" + ) + rgb = frame.to_ndarray(format="rgb24") + height, width = rgb.shape[:2] + crop = min(height, width) + top, left = (height - crop) // 2, (width - crop) // 2 + rgb = rgb[top : top + crop, left : left + crop] + resized = cv2.resize( + rgb, + (config.resolution, config.resolution), + interpolation=cv2.INTER_AREA, + ) + # One decoded terminal frame may satisfy multiple identical target + # timestamps introduced by repeat-last episode padding. + while ( + target_slot < len(target_times) + and abs(frame_time - target_times[target_slot]) <= tolerance + ): + frames.append(resized) + target_slot += 1 + if target_slot == len(target_times): + break + if len(frames) != len(target_times): + raise RuntimeError(f"Decoded {len(frames)}/{len(target_times)} frames") + array = np.stack(frames).astype(np.float32) / np.float32(127.5) - np.float32(1.0) + return torch.from_numpy(array).permute(3, 0, 1, 2).contiguous() + + +def extract_clip( + root: Path, + episode: RoboDojoEpisode, + shard: pl.DataFrame, + image_indices: np.ndarray, + config: RoboDojoClipConfig, +) -> dict[str, Any]: + state, absolute_action, timestamps = episode_arrays(shard, episode) + output = build_egoverse_delta_targets(state, absolute_action, image_indices) + selected_timestamps = timestamps[image_indices] + observed_next = state[image_indices[1:]] + commanded_next = absolute_action[image_indices[:-1]] + output.update( + { + "rgb": decode_rgb_window( + video_path(root, episode, config.video_key), + episode_from_timestamp=episode.video_from_timestamp, + image_indices=image_indices, + config=config, + ), + "image_indices": torch.from_numpy(image_indices.copy()), + "timestamps": torch.from_numpy(selected_timestamps.copy()), + "effective_fps": torch.tensor( + (len(selected_timestamps) - 1) + / float(selected_timestamps[-1] - selected_timestamps[0]), + dtype=torch.float64, + ), + "absolute_action_next_state_mean_abs_error": torch.tensor( + np.abs(commanded_next - observed_next).mean(), dtype=torch.float64 + ), + } + ) + return output diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_native_depth_action_model.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_native_depth_action_model.py new file mode 100644 index 0000000000000000000000000000000000000000..f4c747edcbc1d4b057e59d0a832518ec27df1e5f --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_native_depth_action_model.py @@ -0,0 +1,366 @@ +"""Wan RGB/depth video expert plus RoboDojo action expert initialization.""" + +from __future__ import annotations + +import gc +import os +from pathlib import Path +from typing import Any + +import torch + +from online_egoverse_action import OnlineActionNormalizer +from rgb_action_model_factory import ( + WanVideoDiT, + dtype_from_name, + precompute_freqs_cis_3d, + strict_load_original_dit, + temporary_default_dtype, +) +from robodojo_native_depth_data import validate_native_depth_config +from robodojo_rgb_geometry_action_model import _build_geometry_action_mot +from robodojo_v2_checkpoint_contract import verify_v2_baseframe_checkpoint_contract +from train_robodojo_rgb_state_joint_action_4way import ( + SOURCE_ACTION_NORMALIZER_TRANSFER, + TARGET_ACTION_NORMALIZER_TRANSFER, +) +from wan_rgb_depth_initialization import upgrade_wan_dit_for_rgb_depth + + +V2_RGB_DEPTH_TRANSFER_MODE = ( + "joint_checkpoint_rgb_depth_slice_full_action_baseframe_v2_contract" +) +SOURCE_ONLY_POINT_CAMERA_KEYS = { + "structured_type_embedding", + "camera_encoder.bias", + "camera_encoder.weight", + "camera_head.bias", + "camera_head.weight", + "camera_normalizer.count", + "camera_normalizer.fitted", + "camera_normalizer.mean", + "camera_normalizer.std", + "mot.mixtures.video.point_first_frame_timestep_adapter.net.1.bias", + "mot.mixtures.video.point_first_frame_timestep_adapter.net.1.weight", + "mot.mixtures.video.point_first_frame_timestep_adapter.net.3.bias", + "mot.mixtures.video.point_first_frame_timestep_adapter.net.3.weight", +} + + +def _rgb_depth_head_slice(source: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """Drop point rows while preserving position-major RGB/depth output rows.""" + + patch_volume, source_channels, target_channels = 4, 144, 96 + if source.ndim == 2: + expected = (patch_volume * source_channels, target.shape[1]) + if tuple(source.shape) != expected: + raise RuntimeError(f"Unexpected source head weight shape: {tuple(source.shape)}") + return source.view(patch_volume, source_channels, target.shape[1])[ + :, :target_channels + ].reshape_as(target) + if source.ndim == 1: + if tuple(source.shape) != (patch_volume * source_channels,): + raise RuntimeError(f"Unexpected source head bias shape: {tuple(source.shape)}") + return source.view(patch_volume, source_channels)[:, :target_channels].reshape_as(target) + raise RuntimeError(f"Unsupported output-head tensor rank: {source.ndim}") + + +def _validate_v2_source_metadata(metadata: Any, checkpoint: Path) -> None: + if not isinstance(metadata, dict): + raise ValueError(f"Checkpoint metadata must be a mapping: {checkpoint}") + video, mot = metadata.get("video"), metadata.get("mot") + if not isinstance(video, dict) or not isinstance(mot, dict): + raise ValueError(f"Checkpoint is missing video/MoT metadata: {checkpoint}") + expected_video = { + "joint_rgb_depth_point": True, + "rgb_latent_channels": 48, + "depth_latent_channels": 48, + "point_latent_channels": 48, + "joint_input_channels": 144, + "patch_embedding_in_channels": 144, + "output_head_out_features": 576, + "rgb_frame0_conditioned": True, + "depth_frame0_conditioned": True, + "point_frame0_conditioned": False, + } + actual_video = {key: video.get(key) for key in expected_video} + if actual_video != expected_video: + raise RuntimeError( + f"AgiBot V2 video contract mismatch: {actual_video} != {expected_video}" + ) + expected_mot = { + "model_class": "RgbDepthPointActionMoT", + "experts": ["rgb_depth_point_video", "action"], + "cross_modal_attention": "bidirectional", + "action_loss_to_video_gradient": "enabled_through_video_kv", + "num_layers": 30, + "num_heads": 24, + "attn_head_dim": 128, + "action_dim": 14, + "action_state_dim": 14, + } + actual_mot = {key: mot.get(key) for key in expected_mot} + if actual_mot != expected_mot: + raise RuntimeError( + f"AgiBot V2 MoT contract mismatch: {actual_mot} != {expected_mot}" + ) + + +@torch.no_grad() +def load_v2_rgb_depth_full_action_checkpoint( + model: torch.nn.Module, + checkpoint: Path, + *, + preserve_source_action_normalizer: bool = False, +) -> dict[str, Any]: + """Load RGB, depth, the unchanged DiT trunk, and the full action branch.""" + + payload = torch.load(checkpoint, map_location="cpu", weights_only=True, mmap=True) + if not isinstance(payload, dict) or set(payload) != {"metadata", "state_dict"}: + raise ValueError(f"Invalid joint checkpoint payload: {checkpoint}") + _validate_v2_source_metadata(payload["metadata"], checkpoint) + source = payload["state_dict"] + if not isinstance(source, dict): + raise TypeError("AgiBot checkpoint state_dict must be a mapping") + + # Preserve the source FP32 values. All ranks are promoted to FP32 master + # weights by the training loop before optimizer construction. + model.float() + target = model.state_dict(keep_vars=True) + source_keys, target_keys = set(source), set(target) + missing = target_keys - source_keys + source_only = source_keys - target_keys + if missing: + raise RuntimeError(f"Checkpoint missing target keys: {sorted(missing)}") + if source_only != SOURCE_ONLY_POINT_CAMERA_KEYS: + raise RuntimeError( + f"Unexpected checkpoint-only keys: {sorted(source_only)}; expected " + f"point/camera keys {sorted(SOURCE_ONLY_POINT_CAMERA_KEYS)}" + ) + + normalizer_keys = { + key for key in target_keys if key.startswith("action_normalizer.") + } + expected_normalizer_keys = { + "action_normalizer.count", + "action_normalizer.fitted", + "action_normalizer.mean", + "action_normalizer.std", + } + if normalizer_keys != expected_normalizer_keys: + raise RuntimeError(f"Action normalizer key contract changed: {normalizer_keys}") + preserved_normalizer = { + key: target[key].detach().clone() for key in sorted(normalizer_keys) + } + load_keys = target_keys if preserve_source_action_normalizer else target_keys - normalizer_keys + converted: list[str] = [] + for key in sorted(load_keys): + source_tensor, target_tensor = source[key], target[key] + if tuple(source_tensor.shape) == tuple(target_tensor.shape): + value = source_tensor + elif key == "mot.mixtures.video.patch_embedding.weight": + expected = (target_tensor.shape[0], 144, *target_tensor.shape[2:]) + if tuple(source_tensor.shape) != tuple(expected): + raise RuntimeError( + f"Unexpected joint patch weight shape: {tuple(source_tensor.shape)}" + ) + value = source_tensor[:, :96] + converted.append(key) + elif key in { + "mot.mixtures.video.head.head.weight", + "mot.mixtures.video.head.head.bias", + }: + value = _rgb_depth_head_slice(source_tensor, target_tensor) + converted.append(key) + else: + raise RuntimeError( + f"Unapproved shape mismatch for {key}: source={tuple(source_tensor.shape)} " + f"target={tuple(target_tensor.shape)}" + ) + if value.dtype != target_tensor.dtype: + raise RuntimeError( + f"Checkpoint dtype mismatch for {key}: {value.dtype} != {target_tensor.dtype}" + ) + target_tensor.copy_(value) + if not torch.equal(target_tensor, value): + raise RuntimeError(f"Post-load numerical equality failed for {key}") + + expected_converted = { + "mot.mixtures.video.patch_embedding.weight", + "mot.mixtures.video.head.head.weight", + "mot.mixtures.video.head.head.bias", + } + if set(converted) != expected_converted: + raise RuntimeError(f"RGB/depth slice conversion audit failed: {converted}") + if preserve_source_action_normalizer: + unequal_normalizer = [ + key for key in sorted(normalizer_keys) + if not torch.equal(target[key], source[key]) + ] + if unequal_normalizer: + raise RuntimeError( + f"Source action normalizer was not copied exactly: {unequal_normalizer}" + ) + else: + changed_normalizer = [ + key for key, before in preserved_normalizer.items() + if not torch.equal(target[key], before) + ] + if changed_normalizer: + raise RuntimeError(f"Source action normalizer leaked into target: {changed_normalizer}") + + audit = { + "loaded_key_count": len(load_keys), + "unchanged_dit_trunk": True, + "converted_projection_keys": sorted(converted), + "dropped_source_only_keys": sorted(source_only), + "action_normalizer_transfer": ( + SOURCE_ACTION_NORMALIZER_TRANSFER + if preserve_source_action_normalizer + else TARGET_ACTION_NORMALIZER_TRANSFER + ), + "optimizer_scheduler_step_restored": False, + } + print(f"[initialization] AgiBot V2 RGB+depth+action transfer passed: {audit}", flush=True) + del preserved_normalizer, source, target, payload + gc.collect() + return audit + + +def build_fresh_native_rgb_depth_action_mot( + config: dict[str, Any], +) -> torch.nn.Module: + """Build the RGB/depth/action model and apply the selected initialization.""" + + validate_native_depth_config(config) + model_config = config["model"] + dit_config = dict(model_config["dit"]) + if int(dit_config.get("in_dim", -1)) != 48 or int(dit_config.get("out_dim", -1)) != 48: + raise ValueError("Fresh initialization must start from the original 48-channel Wan") + dtype = dtype_from_name(str(model_config["dtype"])) + original_model_dir = Path(config["paths"]["pretrained_model_dir"]) + local_model_value = str( + config.get("initialization", {}).get("local_wan_dit_dir", "") + ).strip() + local_model_dir = Path(local_model_value) if local_model_value else None + model_dir = ( + local_model_dir + if local_model_dir is not None and local_model_dir.is_dir() + else original_model_dir + ) + load_weights_on_this_rank = int(os.environ.get("LOCAL_RANK", "0")) == 0 + initialization = config.get("initialization", {"mode": "fresh"}) + initialization_mode = str(initialization.get("mode", "fresh")) + normalizer_transfer = str( + initialization.get("action_normalizer_transfer", TARGET_ACTION_NORMALIZER_TRANSFER) + ) + preserve_source_action_normalizer = ( + normalizer_transfer == SOURCE_ACTION_NORMALIZER_TRANSFER + ) + + with torch.device("meta"), temporary_default_dtype(dtype): + video_expert = WanVideoDiT(**dit_config) + video_expert.to_empty(device="cpu") + video_expert.freqs = precompute_freqs_cis_3d(int(dit_config["attn_head_dim"])) + if load_weights_on_this_rank and initialization_mode == "fresh": + strict_load_original_dit(video_expert, model_dir) + elif load_weights_on_this_rank: + print( + "[model-load] original Wan shard load skipped; every RGB/depth/action " + "target parameter will be supplied by the verified AgiBot checkpoint", + flush=True, + ) + else: + print("[model-load] nonzero rank awaits distributed state sync", flush=True) + + video_expert = upgrade_wan_dit_for_rgb_depth(video_expert) + model = _build_geometry_action_mot( + video_expert, + model_config["action_dit"], + modalities=("rgb", "depth"), + random_seed=int(model_config["action_random_seed"]), + action_state_dim=int(model_config["action_state_dim"]), + mot_checkpoint_mixed_attn=False, + cross_modal_attention="bidirectional", + ) + normalization = config["action"]["normalization"] + model.add_module( + "action_normalizer", + OnlineActionNormalizer( + action_dim=int(model_config["action_state_dim"]), + eps=float(normalization.get("eps", 1.0e-6)), + clip=normalization.get("clip"), + ), + ) + if initialization_mode == V2_RGB_DEPTH_TRANSFER_MODE: + if load_weights_on_this_rank: + local_checkpoint = initialization.get("local_checkpoint") + checkpoint = Path( + local_checkpoint + if local_checkpoint and Path(local_checkpoint).is_file() + else initialization["checkpoint"] + ) + contract_audit = verify_v2_baseframe_checkpoint_contract( + checkpoint, + Path(initialization["contract_sidecar"]), + expected_checkpoint_sha256=str(initialization["source_sha256"]), + expected_action_contract_fingerprint=str( + initialization["source_action_contract_fingerprint"] + ), + ) + tensor_audit = load_v2_rgb_depth_full_action_checkpoint( + model, + checkpoint, + preserve_source_action_normalizer=preserve_source_action_normalizer, + ) + initialization_metadata = { + "mode": initialization_mode, + "source_checkpoint": str(initialization["checkpoint"]), + "checkpoint_contract": contract_audit, + "tensor_transfer": tensor_audit, + "input_conditions": [ + "rgb_frame0", + "depth_frame0", + "prompt", + "initial_state", + ], + "outputs": ["rgb_video", "depth_video", "action_chunk_32"], + "action_normalizer": normalizer_transfer, + } + else: + # Rank zero loads and audits the full checkpoint. FSDP's + # sync_module_states then broadcasts those exact parameters to the + # other ranks, matching the existing fresh-Wan initialization path. + initialization_metadata = { + "mode": initialization_mode, + "source_checkpoint": str(initialization["checkpoint"]), + "distributed_state": "awaiting_rank0_fsdp_sync", + "input_conditions": [ + "rgb_frame0", + "depth_frame0", + "prompt", + "initial_state", + ], + "outputs": ["rgb_video", "depth_video", "action_chunk_32"], + "action_normalizer": normalizer_transfer, + } + elif initialization_mode == "fresh": + initialization_metadata = { + "mode": "fresh", + "source_checkpoint": None, + "rgb_slice_from_joint_video": False, + "wan_dit_source": str(model_dir), + "depth_input_projection": "zero_initialized", + "depth_output_projection": "zero_initialized", + } + else: + raise ValueError( + f"Unsupported RGB/depth/action initialization mode: {initialization_mode}" + ) + model._robodojo_initialization = initialization_metadata + # The shared 4-way trainer uses this exact sentinel to decide whether the + # video expert is trainable. Keep modality detail in a separate metadata + # field; any value other than ``joint`` is treated as action-only. + model._robodojo_objective = str(config["experiment_variant"]["objective"]) + model._robodojo_joint_modalities = "rgb_depth_action" + return model diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_native_depth_data.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_native_depth_data.py new file mode 100644 index 0000000000000000000000000000000000000000..46fa5812eb20ee2267bd50f302cf9aeadfbc41ba --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_native_depth_data.py @@ -0,0 +1,375 @@ +"""Native-GT RoboDojo-mini depth contract for FastWAM training. + +Only the ten RoboDojo-mini tasks selected by the RGB/action experiment are +accepted. Depth is read directly from the simulator-exported HDF5 files; the +VDA annotation tree is deliberately neither accepted nor referenced here. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Iterable + +import cv2 +import h5py +import numpy as np +import torch + +from depth_preprocessing import DepthPreprocessingConfig, preprocess_depth_grayscale +from train_robodojo_rgb_state_joint_action_4way import EXPECTED_TASKS + + +NATIVE_DEPTH_KEY = "vision/cam_head/approximate_depths" +NATIVE_INTRINSICS_KEY = "vision/cam_head/intrinsic_matrix" +NATIVE_FREQUENCY_KEY = "additional_info/frequency" +NATIVE_INSTRUCTION_KEY = "instruction" +LATENT_SHAPE = (48, 9, 16, 16) +EXPECTED_NATIVE_ROOT = Path("/dataset_rc_rl/qshou/RoboDojo_depth_head_native") +FORBIDDEN_VDA_ROOT = Path("/dataset_rc_rl/qshou/robodojo_depth_vda_cam_high_25fps_256") + + +def _decode_scalar(value: Any) -> str: + if isinstance(value, bytes): + return value.decode("utf-8") + if isinstance(value, np.ndarray) and value.shape == (): + return _decode_scalar(value.item()) + return str(value) + + +def task_for_episode(config: dict[str, Any], episode_index: int) -> tuple[str, int]: + episode_index = int(episode_index) + task_index = episode_index // 100 + local_index = episode_index % 100 + matches = [ + task + for task, configured_index in config["robodojo"]["task_index_by_id"].items() + if int(configured_index) == task_index + ] + if len(matches) != 1: + raise ValueError( + f"Episode {episode_index} does not map uniquely to a selected RoboDojo-mini task" + ) + return str(matches[0]), local_index + + +def native_depth_episode_path( + native_root: Path, config: dict[str, Any], episode_index: int +) -> Path: + task, local_index = task_for_episode(config, episode_index) + return ( + Path(native_root) + / task + / "arx_x5" + / "data" + / f"episode_{local_index:07d}.hdf5" + ) + + +def source_record(path: Path) -> dict[str, Any]: + stat = path.stat() + return { + "path": str(path.resolve()), + "size": int(stat.st_size), + "mtime_ns": int(stat.st_mtime_ns), + } + + +def validate_native_depth_config(config: dict[str, Any]) -> dict[str, Any]: + root = Path(config["paths"]["native_depth_root"]) + if not root.is_dir(): + raise FileNotFoundError(root) + if root.resolve() != EXPECTED_NATIVE_ROOT.resolve(): + raise ValueError( + f"Native-GT experiment requires {EXPECTED_NATIVE_ROOT}, got {root}" + ) + if root.resolve() == FORBIDDEN_VDA_ROOT.resolve(): + raise ValueError("VDA depth annotations are forbidden for this experiment") + if "depth_dataset_root" in config["paths"]: + raise ValueError("paths.depth_dataset_root is ambiguous; use native_depth_root only") + if "depth_latent_cache_dir" not in config["paths"]: + raise KeyError("paths.depth_latent_cache_dir is required") + + expected_indices = {task: values[0] for task, values in EXPECTED_TASKS.items()} + actual_indices = { + str(task): int(index) + for task, index in config["robodojo"]["task_index_by_id"].items() + } + if actual_indices != expected_indices: + raise ValueError(f"RoboDojo-mini task mapping mismatch: {actual_indices}") + split = config["data"]["episode_split_per_task"] + if split != {"train": 100, "validation": 0, "test": 0}: + raise ValueError(f"Expected 100 train episodes for each of {len(EXPECTED_TASKS)} tasks, got {split}") + if float(config["data"]["fps"]) != 25.0: + raise ValueError("RoboDojo-mini native depth must remain at 25 FPS") + if int(config["data"]["num_frames"]) != 33 or int(config["data"]["resolution"]) != 256: + raise ValueError("Native depth experiment requires 33 frames at 256x256") + + objective = str(config.get("experiment_variant", {}).get("objective", "")) + if objective != "joint": + raise ValueError( + "Native RGB/depth/action training requires experiment_variant.objective=joint" + ) + if config["training"].get("train_video_expert") is not True: + raise ValueError("Native RGB/depth/action training requires train_video_expert=true") + + model_expected = { + "joint_rgb_depth": True, + "rgb_channels": 48, + "depth_channels": 48, + "share_rgb_depth_vae": True, + "freeze_vae": True, + "dtype": "bfloat16", + "action_initialization": "random", + "mot_checkpoint_mixed_attn": False, + "cross_modal_attention": "bidirectional", + "action_loss_to_video_stop_gradient": False, + } + model_actual = {key: config["model"].get(key) for key in model_expected} + if model_actual != model_expected: + raise ValueError(f"RGB/depth model contract mismatch: {model_actual} != {model_expected}") + + depth_expected = { + "representation": "linear", + "input_unit": "millimeter", + "min": 0.0, + "max": 5.0, + "invalid_value": 0.0, + "invalid_fill": "far", + "spatial_transform": "center_square_resize", + "resize_interpolation": "nearest", + "condition_mode": "first_frame_oracle", + "first_frame_timestep_mode": "wan_zero", + "fix_first_latent": True, + "exclude_condition_from_loss": True, + } + modalities = tuple(config["geometry"].get("modalities", [])) + if modalities != ("rgb", "depth"): + raise ValueError(f"geometry.modalities must be [rgb, depth], got {modalities}") + depth_actual = { + key: config["geometry"]["depth"].get(key) for key in depth_expected + } + if depth_actual != depth_expected: + raise ValueError(f"32-GPU SpaceDreamer depth preprocessing mismatch: {depth_actual}") + conditioning_expected = { + "rgb_frame0": "clean_condition", + "initial_state": "clean_action_context_token", + "exclude_rgb_frame0_from_loss": True, + } + conditioning_actual = { + key: config["conditioning"].get(key) for key in conditioning_expected + } + if conditioning_actual != conditioning_expected: + raise ValueError( + f"RGB/action base conditioning contract mismatch: {conditioning_actual}" + ) + diffusion_expected = { + "rgb_depth_noise_mode": "independent", + "lambda_rgb": 1.0, + "lambda_depth": 1.0, + "lambda_action": 1.0, + } + diffusion_actual = { + key: config["diffusion"].get(key) for key in diffusion_expected + } + if diffusion_actual != diffusion_expected: + raise ValueError(f"RGB/depth diffusion contract mismatch: {diffusion_actual}") + if config.get("performance", {}).get("rope_compute_dtype") != "float32": + raise ValueError("performance.rope_compute_dtype must be float32") + if not bool(config["training"].get("fp32_master_weights")): + raise ValueError("FP32 master weights are required") + initialization = config.get("initialization", {}) + initialization_mode = str(initialization.get("mode", "fresh")) + if initialization_mode == "fresh": + forbidden = {"checkpoint", "local_checkpoint", "flavor"}.intersection(initialization) + if forbidden: + raise ValueError( + "Fresh native-GT experiments may not define checkpoint " + f"initialization: {sorted(forbidden)}" + ) + elif initialization_mode in { + "joint_checkpoint_full", + "joint_checkpoint_rgb_depth_slice_full_action_baseframe_v2_contract", + "joint_checkpoint_rgb_depth_point_full_action_baseframe_v2_contract", + }: + checkpoint = Path(initialization["checkpoint"]) + if not checkpoint.is_file(): + raise FileNotFoundError(checkpoint) + if str(initialization.get("flavor")) != "camera6d": + raise ValueError( + "Native full-checkpoint initialization requires flavor=camera6d" + ) + if initialization_mode.endswith("baseframe_v2_contract"): + for key in ("source_sha256", "source_action_contract_fingerprint"): + value = str(initialization.get(key, "")) + if len(value) != 64: + raise ValueError(f"initialization.{key} must be a 64-character hash") + try: + int(value, 16) + except ValueError as error: + raise ValueError( + f"initialization.{key} must be hexadecimal" + ) from error + sidecar = Path(str(initialization.get("contract_sidecar", ""))) + if not sidecar.is_file(): + raise FileNotFoundError(sidecar) + else: + raise ValueError( + "Native-GT initialization.mode must be fresh, joint_checkpoint_full, " + "or a V2 RGB/depth[/point]/full-action transfer mode" + ) + if "checkpoint" in config["paths"]: + raise ValueError("Use initialization.checkpoint, not paths.checkpoint") + return config + + +def audit_native_episode( + path: Path, + *, + expected_length: int, + expected_prompt: str, +) -> dict[str, Any]: + if not path.is_file(): + raise FileNotFoundError(path) + with h5py.File(path, "r") as handle: + for key in ( + NATIVE_DEPTH_KEY, + NATIVE_INTRINSICS_KEY, + NATIVE_FREQUENCY_KEY, + NATIVE_INSTRUCTION_KEY, + ): + if key not in handle: + raise KeyError(f"{path} is missing {key}") + depth = handle[NATIVE_DEPTH_KEY] + if depth.dtype != np.uint16 or tuple(depth.shape) != ( + int(expected_length), 480, 640 + ): + raise ValueError( + f"Native depth shape/dtype mismatch in {path}: {depth.shape} {depth.dtype}" + ) + if int(handle[NATIVE_FREQUENCY_KEY][()]) != 25: + raise ValueError(f"Native depth frequency mismatch in {path}") + prompt = _decode_scalar(handle[NATIVE_INSTRUCTION_KEY][()]) + if prompt != expected_prompt: + raise ValueError(f"Native instruction mismatch in {path}: {prompt!r}") + intrinsics = np.asarray(handle[NATIVE_INTRINSICS_KEY][()], dtype=np.float64) + if intrinsics.shape != (3, 3) or not np.isfinite(intrinsics).all(): + raise ValueError(f"Invalid native intrinsics in {path}") + if min(float(intrinsics[0, 0]), float(intrinsics[1, 1])) <= 0: + raise ValueError(f"Invalid native focal length in {path}") + compression = depth.compression + return { + **source_record(path), + "length": int(expected_length), + "prompt": expected_prompt, + "frequency": 25, + "depth_shape": [int(expected_length), 480, 640], + "depth_dtype": "uint16", + "compression": compression, + "intrinsics": intrinsics.tolist(), + } + + +def audit_selected_native_episodes( + config: dict[str, Any], records: Iterable[dict[str, Any]] +) -> list[dict[str, Any]]: + root = Path(config["paths"]["native_depth_root"]) + prompts = {task: values[1] for task, values in EXPECTED_TASKS.items()} + audited = [] + seen: set[int] = set() + for record in records: + episode_index = int(record["episode_index"]) + if episode_index in seen: + raise ValueError(f"Duplicate episode index {episode_index}") + seen.add(episode_index) + task, _ = task_for_episode(config, episode_index) + if str(record["task"]) != prompts[task]: + raise ValueError( + f"LeRobot/native prompt mapping mismatch for episode {episode_index}" + ) + path = native_depth_episode_path(root, config, episode_index) + native = audit_native_episode( + path, + expected_length=int(record["length"]), + expected_prompt=prompts[task], + ) + audited.append( + { + "episode_index": episode_index, + "task_id": task, + "length": int(record["length"]), + "native_source": native, + } + ) + expected_count = 100 * len(EXPECTED_TASKS) + if len(audited) != expected_count: + raise ValueError(f"Expected {expected_count} selected native episodes, got {len(audited)}") + return audited + + +def load_native_depth_episode( + path: Path, *, expected_length: int +) -> tuple[np.ndarray, np.ndarray]: + with h5py.File(path, "r") as handle: + depth = handle[NATIVE_DEPTH_KEY][()] + intrinsics = np.asarray(handle[NATIVE_INTRINSICS_KEY][()], dtype=np.float32) + if depth.dtype != np.uint16 or tuple(depth.shape) != (expected_length, 480, 640): + raise ValueError(f"Native depth changed after audit: {path}") + return depth, intrinsics + + +def preprocess_native_depth_episode( + depth_mm: np.ndarray, config: dict[str, Any] +) -> torch.Tensor: + if depth_mm.dtype != np.uint16 or depth_mm.ndim != 3: + raise ValueError(f"Expected native uint16 [T,H,W] depth, got {depth_mm.shape}") + resolution = int(config["data"]["resolution"]) + height, width = depth_mm.shape[1:] + crop = min(height, width) + top, left = (height - crop) // 2, (width - crop) // 2 + cropped = depth_mm[:, top : top + crop, left : left + crop] + resized = np.stack( + [ + cv2.resize( + frame, + (resolution, resolution), + interpolation=cv2.INTER_NEAREST, + ) + for frame in cropped + ] + ) + depth_config = config["geometry"]["depth"] + processed = preprocess_depth_grayscale( + resized, + config=DepthPreprocessingConfig( + depth_min=float(depth_config["min"]), + depth_max=float(depth_config["max"]), + depth_invalid_value=float(depth_config["invalid_value"]), + depth_invalid_fill=str(depth_config["invalid_fill"]), + ), + input_unit=str(depth_config["input_unit"]), + ) + # Wan VAE layout is [C,T,H,W]. + return torch.from_numpy(processed.vae_video).permute(1, 0, 2, 3).contiguous() + + +def native_depth_cache_fingerprint( + config: dict[str, Any], source_fingerprint: str, audited: list[dict[str, Any]] +) -> str: + payload = { + "format_version": 1, + "source_fingerprint": str(source_fingerprint), + "native_depth_root": str(Path(config["paths"]["native_depth_root"]).resolve()), + "episodes": audited, + "depth": config["geometry"]["depth"], + "num_frames": int(config["data"]["num_frames"]), + "resolution": int(config["data"]["resolution"]), + "fps": float(config["data"]["fps"]), + "conditioning": config["conditioning"], + "rgb_depth_noise_mode": config["diffusion"]["rgb_depth_noise_mode"], + "vae": "shared_frozen_wan_vae", + "latent_dtype": "bfloat16", + "tail_padding": config["sampling"]["tail_padding"], + } + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_native_depth_point_action_model.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_native_depth_point_action_model.py new file mode 100644 index 0000000000000000000000000000000000000000..876a7871473bd176c589369c1e6e0de73745efb1 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_native_depth_point_action_model.py @@ -0,0 +1,435 @@ +"""Wan RGB/depth/point video expert plus RoboDojo action expert initialization.""" + +from __future__ import annotations + +import gc +import os +from pathlib import Path +from typing import Any + +import torch + +from online_egoverse_action import OnlineActionNormalizer +from rgb_action_model_factory import ( + WanVideoDiT, + dtype_from_name, + precompute_freqs_cis_3d, + strict_load_original_dit, + temporary_default_dtype, +) +from robodojo_native_depth_point_data import validate_native_depth_point_config +from robodojo_native_depth_action_model import _validate_v2_source_metadata +from robodojo_rgb_geometry_action_model import _build_geometry_action_mot +from robodojo_v2_checkpoint_contract import verify_v2_baseframe_checkpoint_contract +from train_robodojo_rgb_state_joint_action_4way import ( + SOURCE_ACTION_NORMALIZER_TRANSFER, + TARGET_ACTION_NORMALIZER_TRANSFER, +) +from wan_rgb_depth_point_initialization import ( + enable_point_first_frame_timestep_adapter, + upgrade_wan_dit_for_rgb_depth_point, +) + +CAMERA6D_CHECKPOINT_ONLY_KEYS = { + "structured_type_embedding", + "camera_encoder.bias", + "camera_encoder.weight", + "camera_head.bias", + "camera_head.weight", + "camera_normalizer.count", + "camera_normalizer.fitted", + "camera_normalizer.mean", + "camera_normalizer.std", +} +V2_RGB_DEPTH_POINT_TRANSFER_MODE = ( + "joint_checkpoint_rgb_depth_point_full_action_baseframe_v2_contract" +) +ACTION_NORMALIZER_KEYS = { + "action_normalizer.count", + "action_normalizer.fitted", + "action_normalizer.mean", + "action_normalizer.std", +} + + +def _validate_joint_checkpoint_metadata(metadata: Any, checkpoint: Path) -> None: + if not isinstance(metadata, dict): + raise ValueError(f"Checkpoint metadata must be a mapping: {checkpoint}") + video = metadata.get("video") + mot = metadata.get("mot") + if not isinstance(video, dict) or not isinstance(mot, dict): + raise ValueError(f"Checkpoint is missing video/MoT metadata: {checkpoint}") + expected_video = { + "joint_rgb_depth_point": True, + "rgb_latent_channels": 48, + "depth_latent_channels": 48, + "point_latent_channels": 48, + "joint_input_channels": 144, + "point_frame0_conditioned": False, + "point_timestep_adapter_bottleneck_dim": 256, + } + actual_video = {key: video.get(key) for key in expected_video} + if actual_video != expected_video: + raise ValueError( + f"Checkpoint video contract mismatch: {actual_video} != {expected_video}" + ) + expected_mot = { + "experts": ["rgb_depth_point_video", "action"], + "cross_modal_attention": "bidirectional", + "action_loss_to_video_gradient": "stopped_at_video_kv", + "num_layers": 30, + "num_heads": 24, + "attn_head_dim": 128, + "action_dim": 14, + "camera_dim": 6, + "structured_token_types": ["action", "camera"], + "action_state_dim": 14, + } + actual_mot = {key: mot.get(key) for key in expected_mot} + if actual_mot != expected_mot: + raise ValueError( + f"Checkpoint MoT contract mismatch: {actual_mot} != {expected_mot}" + ) + + +@torch.no_grad() +def load_joint_checkpoint_full( + model: torch.nn.Module, + checkpoint: Path, +) -> None: + """Strict-load all target weights, ignoring only the unused camera6D branch.""" + + payload = torch.load(checkpoint, map_location="cpu", weights_only=True, mmap=True) + if not isinstance(payload, dict) or set(payload) != {"metadata", "state_dict"}: + raise ValueError(f"Invalid joint checkpoint payload: {checkpoint}") + _validate_joint_checkpoint_metadata(payload["metadata"], checkpoint) + source = payload["state_dict"] + if not isinstance(source, dict): + raise ValueError(f"Checkpoint state_dict must be a mapping: {checkpoint}") + + # Preserve the source checkpoint's FP32 values exactly. The training loop + # promotes every rank to FP32 master weights immediately after model build. + model.float() + target = model.state_dict(keep_vars=True) + source_keys, target_keys = set(source), set(target) + if target_keys - source_keys: + raise RuntimeError( + f"Checkpoint missing target keys: {sorted(target_keys - source_keys)}" + ) + source_only = source_keys - target_keys + if source_only != CAMERA6D_CHECKPOINT_ONLY_KEYS: + raise RuntimeError( + f"Unexpected checkpoint-only keys: {sorted(source_only)}; expected only " + f"the unused camera6D branch {sorted(CAMERA6D_CHECKPOINT_ONLY_KEYS)}" + ) + + for key in sorted(target_keys): + source_tensor = source[key] + target_tensor = target[key] + if tuple(source_tensor.shape) != tuple(target_tensor.shape): + raise RuntimeError( + f"Checkpoint shape mismatch for {key}: " + f"source={tuple(source_tensor.shape)} target={tuple(target_tensor.shape)}" + ) + if source_tensor.dtype != target_tensor.dtype: + raise RuntimeError( + f"Checkpoint dtype mismatch for {key}: " + f"source={source_tensor.dtype} target={target_tensor.dtype}" + ) + target_tensor.copy_(source_tensor) + + print( + f"[initialization] strict-loaded {len(target_keys)} full " + f"RGB+depth+point+action keys from {checkpoint}; " + f"ignored_camera6d_keys={sorted(source_only)}; " + "optimizer_scheduler_step=not_restored", + flush=True, + ) + del source, target, payload + gc.collect() + + +@torch.no_grad() +def load_v2_rgb_depth_point_full_action_checkpoint( + model: torch.nn.Module, + checkpoint: Path, + *, + preserve_source_action_normalizer: bool = False, +) -> dict[str, Any]: + """Strictly transfer all RGB/depth/point/action tensors from AgiBot V2. + + The source and target architectures are identical apart from the unused + camera6D branch. The normalizer policy is explicit: legacy callers retain + the target buffers, while the pretraining-normalizer ablation copies them. + """ + + payload = torch.load(checkpoint, map_location="cpu", weights_only=True, mmap=True) + if not isinstance(payload, dict) or set(payload) != {"metadata", "state_dict"}: + raise ValueError(f"Invalid AgiBot V2 joint checkpoint payload: {checkpoint}") + _validate_v2_source_metadata(payload["metadata"], checkpoint) + source = payload["state_dict"] + if not isinstance(source, dict): + raise TypeError("AgiBot V2 checkpoint state_dict must be a mapping") + + model.float() + target = model.state_dict(keep_vars=True) + source_keys, target_keys = set(source), set(target) + missing = target_keys - source_keys + source_only = source_keys - target_keys + if missing: + raise RuntimeError(f"Checkpoint missing target keys: {sorted(missing)}") + if source_only != CAMERA6D_CHECKPOINT_ONLY_KEYS: + raise RuntimeError( + f"Unexpected checkpoint-only keys: {sorted(source_only)}; expected only " + f"the unused camera6D branch {sorted(CAMERA6D_CHECKPOINT_ONLY_KEYS)}" + ) + actual_normalizer_keys = { + key for key in target_keys if key.startswith("action_normalizer.") + } + if actual_normalizer_keys != ACTION_NORMALIZER_KEYS: + raise RuntimeError( + f"Action normalizer key contract changed: {sorted(actual_normalizer_keys)}" + ) + preserved_normalizer = { + key: target[key].detach().clone() for key in sorted(ACTION_NORMALIZER_KEYS) + } + load_keys = target_keys if preserve_source_action_normalizer else target_keys - ACTION_NORMALIZER_KEYS + for key in sorted(load_keys): + source_tensor, target_tensor = source[key], target[key] + if tuple(source_tensor.shape) != tuple(target_tensor.shape): + raise RuntimeError( + f"Checkpoint shape mismatch for {key}: " + f"source={tuple(source_tensor.shape)} target={tuple(target_tensor.shape)}" + ) + if source_tensor.dtype != target_tensor.dtype: + raise RuntimeError( + f"Checkpoint dtype mismatch for {key}: " + f"source={source_tensor.dtype} target={target_tensor.dtype}" + ) + target_tensor.copy_(source_tensor) + if not torch.equal(target_tensor, source_tensor): + raise RuntimeError(f"Post-load numerical equality failed for {key}") + if preserve_source_action_normalizer: + unequal_normalizer = [ + key for key in sorted(ACTION_NORMALIZER_KEYS) + if not torch.equal(target[key], source[key]) + ] + if unequal_normalizer: + raise RuntimeError( + f"Source action normalizer was not copied exactly: {unequal_normalizer}" + ) + else: + changed_normalizer = [ + key + for key, before in preserved_normalizer.items() + if not torch.equal(target[key], before) + ] + if changed_normalizer: + raise RuntimeError( + f"AgiBot action normalizer leaked into RoboDojo target: {changed_normalizer}" + ) + + audit = { + "loaded_key_count": len(load_keys), + "loaded_modalities": ["rgb", "depth", "point", "action"], + "projection_conversion": "none_identical_144_in_576_out_architecture", + "dropped_source_only_keys": sorted(source_only), + "action_normalizer_transfer": ( + SOURCE_ACTION_NORMALIZER_TRANSFER + if preserve_source_action_normalizer + else TARGET_ACTION_NORMALIZER_TRANSFER + ), + "optimizer_scheduler_step_restored": False, + } + print( + "[initialization] AgiBot V2 full RGB+depth+point+action transfer passed: " + f"{audit}", + flush=True, + ) + del preserved_normalizer, source, target, payload + gc.collect() + return audit + + +def build_fresh_native_rgb_depth_point_action_mot( + config: dict[str, Any], +) -> torch.nn.Module: + """Strict-load original Wan, then zero-init only depth/point additions.""" + + validate_native_depth_point_config(config) + model_config = config["model"] + dit_config = dict(model_config["dit"]) + if int(dit_config.get("in_dim", -1)) != 48 or int(dit_config.get("out_dim", -1)) != 48: + raise ValueError("Fresh initialization must start from the original 48-channel Wan") + dtype = dtype_from_name(str(model_config["dtype"])) + original_model_dir = Path(config["paths"]["pretrained_model_dir"]) + local_value = str(config.get("initialization", {}).get("local_wan_dit_dir", "")).strip() + local_dir = Path(local_value) if local_value else None + model_dir = local_dir if local_dir is not None and local_dir.is_dir() else original_model_dir + load_weights = int(os.environ.get("LOCAL_RANK", "0")) == 0 + initialization = config.get("initialization", {"mode": "fresh"}) + initialization_mode = str(initialization.get("mode", "fresh")) + normalizer_transfer = str( + initialization.get("action_normalizer_transfer", TARGET_ACTION_NORMALIZER_TRANSFER) + ) + preserve_source_action_normalizer = ( + normalizer_transfer == SOURCE_ACTION_NORMALIZER_TRANSFER + ) + + with torch.device("meta"), temporary_default_dtype(dtype): + video_expert = WanVideoDiT(**dit_config) + video_expert.to_empty(device="cpu") + video_expert.freqs = precompute_freqs_cis_3d(int(dit_config["attn_head_dim"])) + if load_weights and initialization_mode == "fresh": + strict_load_original_dit(video_expert, model_dir) + elif load_weights: + print( + "[model-load] original Wan shard load skipped; the selected joint " + "checkpoint supplies every RGB/depth/point/action target parameter", + flush=True, + ) + else: + print("[model-load] nonzero rank awaits distributed state sync", flush=True) + + video_expert = upgrade_wan_dit_for_rgb_depth_point(video_expert) + video_expert = enable_point_first_frame_timestep_adapter( + video_expert, + point_bottleneck_dim=int( + config["geometry"]["point"]["timestep_adapter_bottleneck_dim"] + ), + ) + model = _build_geometry_action_mot( + video_expert, + model_config["action_dit"], + modalities=("rgb", "depth", "point"), + random_seed=int(model_config["action_random_seed"]), + action_state_dim=int(model_config["action_state_dim"]), + mot_checkpoint_mixed_attn=False, + cross_modal_attention="bidirectional", + ) + model.condition_depth_frame0 = bool( + config.get("modality_contract", {}).get("depth_condition", True) + ) + normalization = config["action"]["normalization"] + model.add_module( + "action_normalizer", + OnlineActionNormalizer( + action_dim=int(model_config["action_state_dim"]), + eps=float(normalization.get("eps", 1.0e-6)), + clip=normalization.get("clip"), + ), + ) + if initialization_mode == "joint_checkpoint_full" and load_weights: + local_checkpoint = initialization.get("local_checkpoint") + checkpoint = Path( + local_checkpoint + if local_checkpoint and Path(local_checkpoint).is_file() + else initialization["checkpoint"] + ) + load_joint_checkpoint_full(model, checkpoint) + + if initialization_mode == V2_RGB_DEPTH_POINT_TRANSFER_MODE: + if load_weights: + local_checkpoint = initialization.get("local_checkpoint") + checkpoint = Path( + local_checkpoint + if local_checkpoint and Path(local_checkpoint).is_file() + else initialization["checkpoint"] + ) + contract_audit = verify_v2_baseframe_checkpoint_contract( + checkpoint, + Path(initialization["contract_sidecar"]), + expected_checkpoint_sha256=str(initialization["source_sha256"]), + expected_action_contract_fingerprint=str( + initialization["source_action_contract_fingerprint"] + ), + ) + tensor_audit = load_v2_rgb_depth_point_full_action_checkpoint( + model, + checkpoint, + preserve_source_action_normalizer=preserve_source_action_normalizer, + ) + initialization_metadata = { + "mode": initialization_mode, + "source_checkpoint": str(initialization["checkpoint"]), + "checkpoint_contract": contract_audit, + "tensor_transfer": tensor_audit, + "input_conditions": ( + ["rgb_frame0", "depth_frame0", "prompt", "initial_state"] + if model.condition_depth_frame0 + else ["rgb_frame0", "prompt", "initial_state"] + ), + "outputs": [ + "rgb_video", + "depth_video", + "point_video", + "absolute_action_chunk_32", + ], + "action_normalizer": normalizer_transfer, + } + else: + initialization_metadata = { + "mode": initialization_mode, + "source_checkpoint": str(initialization["checkpoint"]), + "distributed_state": "awaiting_rank0_fsdp_sync", + "input_conditions": ( + ["rgb_frame0", "depth_frame0", "prompt", "initial_state"] + if model.condition_depth_frame0 + else ["rgb_frame0", "prompt", "initial_state"] + ), + "outputs": [ + "rgb_video", + "depth_video", + "point_video", + "absolute_action_chunk_32", + ], + "action_normalizer": normalizer_transfer, + } + + if initialization_mode == "fresh": + initialization_metadata = { + "mode": "fresh", + "source_checkpoint": None, + "reference_checkpoint_usage": "configuration_only_not_loaded", + "rgb_slice_from_joint_video": False, + "wan_dit_source": str(model_dir), + "depth_input_output_projections": "zero_initialized", + "point_input_output_projections": "zero_initialized", + "point_first_frame_timestep_adapter": "zero_initialized", + "action_expert": "random_seeded", + "rgb_condition": True, + "depth_condition": True, + "point_condition": False, + "predict_rgb": True, + "predict_depth": True, + "predict_point": True, + "predict_action": True, + "depth_source": "robodojo_native_simulator_gt", + "point_source": "deterministic_unprojection_of_native_gt_depth", + } + elif initialization_mode == "joint_checkpoint_full": + initialization_metadata = { + "mode": "joint_checkpoint_full", + "source_checkpoint": str(initialization["checkpoint"]), + "loaded_target_keys": "all_rgb_depth_point_action_model_state", + "ignored_source_keys": sorted(CAMERA6D_CHECKPOINT_ONLY_KEYS), + "source_action_loss_to_video_gradient": "stopped_at_video_kv", + "target_action_loss_to_video_gradient": "full_bidirectional", + "optimizer_state_restored": False, + "scheduler_state_restored": False, + "global_step_restored": False, + "action_normalizer": ( + "replaced_from_robodojo_train_cache_before_training" + ), + } + elif initialization_mode != V2_RGB_DEPTH_POINT_TRANSFER_MODE: + raise ValueError( + "Unsupported RGB/depth/point/action initialization mode: " + f"{initialization_mode}" + ) + model._robodojo_initialization = initialization_metadata + if load_weights: + print(f"[initialization-contract] {initialization_metadata}", flush=True) + model._robodojo_objective = str(config["experiment_variant"]["objective"]) + model._robodojo_joint_modalities = "rgb_depth_point_action" + return model diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_native_depth_point_data.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_native_depth_point_data.py new file mode 100644 index 0000000000000000000000000000000000000000..44978af18a08cf4381c20afdac28097caa173e09 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_native_depth_point_data.py @@ -0,0 +1,319 @@ +"""Native-GT point-map contract layered on the verified RoboDojo depth data. + +RGB and metric depth frame zero are model conditions. Dense XYZ point maps +are derived only from the simulator-exported depth/intrinsics/extrinsics and +all 33 point frames remain diffusion targets. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +from pathlib import Path +from typing import Any, Iterable + +import cv2 +import h5py +import numpy as np +import torch + +from robodojo_native_depth_data import ( + LATENT_SHAPE, + native_depth_episode_path, + source_record, + task_for_episode, + validate_native_depth_config, +) +from train_robodojo_rgb_state_joint_action_4way import EXPECTED_TASKS +from world_point_cloud import ( + depth_video_to_world_grid, + normalize_world_point_video, + scale_intrinsics_after_center_crop, +) + + +NATIVE_DEPTH_KEY = "vision/cam_head/approximate_depths" +NATIVE_INTRINSICS_KEY = "vision/cam_head/intrinsic_matrix" +NATIVE_EXTRINSICS_KEY = "vision/cam_head/extrinsic_matrix" + + +def validate_native_depth_point_config(config: dict[str, Any]) -> dict[str, Any]: + """Prove the new experiment is the depth baseline plus one point target.""" + + depth_view = copy.deepcopy(config) + depth_view["geometry"] = { + "modalities": ["rgb", "depth"], + "depth": copy.deepcopy(config["geometry"]["depth"]), + } + validate_native_depth_config(depth_view) + + modalities = tuple(config["geometry"].get("modalities", [])) + if modalities != ("rgb", "depth", "point"): + raise ValueError( + "Native point experiment requires geometry.modalities=[rgb, depth, point]" + ) + model_expected = { + "joint_rgb_depth_point": True, + "point_channels": 48, + "share_rgb_depth_point_vae": True, + } + model_actual = {key: config["model"].get(key) for key in model_expected} + if model_actual != model_expected: + raise ValueError( + f"Native point model contract mismatch: {model_actual} != {model_expected}" + ) + point_expected = { + "representation": "dense_episode_first_camera_xyz_point_map_video", + "coordinate_frame": "episode_first_camera_frame", + "derived_from": "native_depth_intrinsics_extrinsics", + "pose_array_key": NATIVE_EXTRINSICS_KEY, + "pose_is": "camera_to_world", + "min_depth_m": 0.05, + "max_depth_m": 5.0, + "xyz_min_m": [-5.0, -5.5, 0.0], + "xyz_max_m": [5.0, 4.5, 6.0], + "invalid_fill_reference_m": [0.0, -0.5, 3.0], + "point_stride": 1, + "timestep_adapter_bottleneck_dim": 256, + } + point = config["geometry"].get("point") + if not isinstance(point, dict): + raise ValueError("Native point experiment requires geometry.point") + point_actual = {key: point.get(key) for key in point_expected} + if point_actual != point_expected: + raise ValueError( + f"32-GPU SpaceDreamer point contract mismatch: {point_actual} != {point_expected}" + ) + conditioning_expected = { + "point_condition_mode": "none", + "fix_point_first_latent": False, + "exclude_point_condition_from_loss": False, + } + conditioning_actual = { + key: config["conditioning"].get(key) for key in conditioning_expected + } + if conditioning_actual != conditioning_expected: + raise ValueError( + f"Point conditioning contract mismatch: {conditioning_actual}" + ) + diffusion_expected = { + "rgb_depth_point_noise_mode": "independent", + "lambda_point": 1.0, + } + diffusion_actual = { + key: config["diffusion"].get(key) for key in diffusion_expected + } + if diffusion_actual != diffusion_expected: + raise ValueError(f"Point diffusion contract mismatch: {diffusion_actual}") + if "point_latent_cache_dir" not in config["paths"]: + raise KeyError("paths.point_latent_cache_dir is required") + if int(config["data"]["num_frames"]) != 33: + raise ValueError("Point output must contain all 33 raw frames") + return config + + +def _validate_extrinsics(extrinsics: np.ndarray, path: Path) -> dict[str, float]: + matrices = np.asarray(extrinsics, dtype=np.float64) + if matrices.ndim != 3 or matrices.shape[1:] != (4, 4): + raise ValueError(f"Invalid native extrinsic shape in {path}: {matrices.shape}") + if not np.isfinite(matrices).all(): + raise ValueError(f"Non-finite native extrinsics in {path}") + bottom_error = float( + np.max(np.abs(matrices[:, 3] - np.array([0.0, 0.0, 0.0, 1.0]))) + ) + rotations = matrices[:, :3, :3] + identity = np.eye(3, dtype=np.float64) + orthogonal_error = float( + np.max(np.abs(np.einsum("tji,tjk->tik", rotations, rotations) - identity)) + ) + determinant_error = float(np.max(np.abs(np.linalg.det(rotations) - 1.0))) + if max(bottom_error, orthogonal_error, determinant_error) > 1.0e-4: + raise ValueError( + f"Native camera-to-world matrices are not rigid in {path}: " + f"bottom={bottom_error} orthogonal={orthogonal_error} " + f"determinant={determinant_error}" + ) + reference_from_world = np.linalg.inv(matrices[0]) + relative = np.einsum("ij,tjk->tik", reference_from_world, matrices) + translation = np.linalg.norm(relative[:, :3, 3], axis=1) + return { + "bottom_row_max_error": bottom_error, + "rotation_orthogonal_max_error": orthogonal_error, + "rotation_determinant_max_error": determinant_error, + "episode_first_camera_translation_max_m": float(translation.max()), + } + + +def audit_native_point_episode(path: Path, *, expected_length: int) -> dict[str, Any]: + with h5py.File(path, "r") as handle: + if NATIVE_EXTRINSICS_KEY not in handle: + raise KeyError(f"{path} is missing {NATIVE_EXTRINSICS_KEY}") + dataset = handle[NATIVE_EXTRINSICS_KEY] + if tuple(dataset.shape) != (int(expected_length), 4, 4): + raise ValueError( + f"Native extrinsic length mismatch in {path}: {dataset.shape}" + ) + dtype = str(dataset.dtype) + audit = _validate_extrinsics(dataset[()], path) + return { + **source_record(path), + "extrinsic_shape": [int(expected_length), 4, 4], + "extrinsic_dtype": dtype, + "pose_is": "camera_to_world", + **audit, + } + + +def audit_selected_native_point_episodes( + config: dict[str, Any], records: Iterable[dict[str, Any]] +) -> list[dict[str, Any]]: + root = Path(config["paths"]["native_depth_root"]) + prompts = {task: values[1] for task, values in EXPECTED_TASKS.items()} + audited: list[dict[str, Any]] = [] + for record in records: + episode_index = int(record["episode_index"]) + task, _ = task_for_episode(config, episode_index) + if str(record["task"]) != prompts[task]: + raise ValueError(f"Point/native prompt mismatch for episode {episode_index}") + path = native_depth_episode_path(root, config, episode_index) + audited.append( + { + "episode_index": episode_index, + "task_id": task, + "length": int(record["length"]), + "native_source": audit_native_point_episode( + path, expected_length=int(record["length"]) + ), + } + ) + expected_count = 100 * len(EXPECTED_TASKS) + if len(audited) != expected_count: + raise ValueError(f"Expected {expected_count} native point episodes, got {len(audited)}") + return audited + + +def native_point_cache_fingerprint( + config: dict[str, Any], + source_fingerprint: str, + native_depth_fingerprint: str, + audited: list[dict[str, Any]], +) -> str: + payload = { + "format_version": 1, + "source_fingerprint": str(source_fingerprint), + "native_depth_fingerprint": str(native_depth_fingerprint), + "episodes": audited, + "point": config["geometry"]["point"], + "conditioning": { + key: config["conditioning"][key] + for key in ( + "point_condition_mode", + "fix_point_first_latent", + "exclude_point_condition_from_loss", + ) + }, + "num_frames": int(config["data"]["num_frames"]), + "resolution": int(config["data"]["resolution"]), + "fps": float(config["data"]["fps"]), + "vae": "shared_frozen_wan_vae", + "latent_dtype": "bfloat16", + "tail_padding": config["sampling"]["tail_padding"], + } + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest() + + +def load_native_point_episode( + path: Path, *, expected_length: int +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + with h5py.File(path, "r") as handle: + depth = handle[NATIVE_DEPTH_KEY][()] + intrinsics = np.asarray(handle[NATIVE_INTRINSICS_KEY][()], dtype=np.float32) + extrinsics = np.asarray(handle[NATIVE_EXTRINSICS_KEY][()], dtype=np.float32) + if depth.dtype != np.uint16 or tuple(depth.shape) != (expected_length, 480, 640): + raise ValueError(f"Native depth changed after point audit: {path}") + if intrinsics.shape != (3, 3) or extrinsics.shape != (expected_length, 4, 4): + raise ValueError(f"Native calibration changed after point audit: {path}") + _validate_extrinsics(extrinsics, path) + return depth, intrinsics, extrinsics + + +def preprocess_native_point_episode( + depth_mm: np.ndarray, + intrinsics: np.ndarray, + camera_to_world: np.ndarray, + config: dict[str, Any], +) -> tuple[torch.Tensor, dict[str, float]]: + """Build normalized episode-first-camera XYZ maps in Wan-VAE layout.""" + + if depth_mm.dtype != np.uint16 or depth_mm.ndim != 3: + raise ValueError(f"Expected native uint16 [T,H,W] depth, got {depth_mm.shape}") + frames, height, width = depth_mm.shape + if camera_to_world.shape != (frames, 4, 4): + raise ValueError("Native extrinsics do not align with depth frames") + resolution = int(config["data"]["resolution"]) + crop = min(height, width) + top, left = (height - crop) // 2, (width - crop) // 2 + cropped = depth_mm[:, top : top + crop, left : left + crop] + resized = np.stack( + [ + cv2.resize(frame, (resolution, resolution), interpolation=cv2.INTER_NEAREST) + for frame in cropped + ] + ) + processed_intrinsics = scale_intrinsics_after_center_crop( + intrinsics, height, width, resolution + ) + reference_from_world = np.linalg.inv(camera_to_world[0]).astype(np.float32) + camera_to_reference = np.einsum( + "ij,tjk->tik", reference_from_world, camera_to_world + ).astype(np.float32) + camera_to_reference[0] = np.eye(4, dtype=np.float32) + point_config = config["geometry"]["point"] + points, valid = depth_video_to_world_grid( + resized.astype(np.float32) / np.float32(1000.0), + processed_intrinsics, + camera_to_reference, + stride=int(point_config["point_stride"]), + min_depth_m=float(point_config["min_depth_m"]), + max_depth_m=float(point_config["max_depth_m"]), + valid_mask=resized != 0, + ) + point_video, clipped_fraction = normalize_world_point_video( + points, + valid, + point_config["xyz_min_m"], + point_config["xyz_max_m"], + invalid_fill_reference=point_config["invalid_fill_reference_m"], + ) + translation = np.linalg.norm(camera_to_reference[:, :3, 3], axis=1) + return torch.from_numpy(point_video).contiguous(), { + "valid_fraction": float(valid.mean()), + "clipped_fraction": float(clipped_fraction), + "camera_translation_max_m": float(translation.max()), + } + + +def expected_native_point_manifest( + *, + source_fingerprint: str, + native_depth_fingerprint: str, + native_point_fingerprint: str, + episode_count: int = 1000, + valid_window_count: int = 429700, +) -> dict[str, Any]: + return { + "format_version": 1, + "modality": "point", + "source": "robodojo_native_gt_hdf5_backprojection", + "source_fingerprint": str(source_fingerprint), + "native_depth_fingerprint": str(native_depth_fingerprint), + "native_point_fingerprint": str(native_point_fingerprint), + "dtype": "bfloat16", + "latent_shape": list(LATENT_SHAPE), + "episode_count": int(episode_count), + "valid_window_count": int(valid_window_count), + "window_sampling": "all_episode_frames_repeat_last_tail", + "raw_point_frames_per_window": 33, + "point_frame0_conditioned": False, + } diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_rgb_geometry_action_model.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_rgb_geometry_action_model.py new file mode 100644 index 0000000000000000000000000000000000000000..84434b2dcb4b4ca6acd3117c6f77a71207033bcb --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_rgb_geometry_action_model.py @@ -0,0 +1,195 @@ +"""Isolated RoboDojo RGB/depth[/point] video + Action MoT factory.""" + +from __future__ import annotations + +from contextlib import nullcontext +import os +from pathlib import Path +from typing import Any, Mapping + +import torch + +from online_egoverse_action import OnlineActionNormalizer +from rgb_action_model_factory import ( + dtype_from_name, + precompute_freqs_cis_3d, + strict_load_original_dit, + temporary_default_dtype, + WanVideoDiT, +) +from robodojo_geometry_data import geometry_modalities, validate_geometry_config +from wan_rgb_action_mot import ActionDiT, RgbActionMoT, action_dit_05b_config +from wan_rgb_depth_initialization import upgrade_wan_dit_for_rgb_depth +from wan_rgb_depth_point_initialization import ( + enable_point_first_frame_timestep_adapter, + upgrade_wan_dit_for_rgb_depth_point, +) + + +class RgbGeometryActionMoT(RgbActionMoT): + """The existing two-expert MoT with an expanded dense video expert.""" + + def __init__( + self, + video_expert: torch.nn.Module, + action_expert: ActionDiT, + *, + modalities: tuple[str, ...], + action_state_dim: int, + mot_checkpoint_mixed_attn: bool, + cross_modal_attention: str, + ) -> None: + super().__init__( + video_expert, + action_expert, + action_state_dim=action_state_dim, + mot_checkpoint_mixed_attn=mot_checkpoint_mixed_attn, + cross_modal_attention=cross_modal_attention, + ) + self.geometry_modalities = tuple(modalities) + if self.geometry_modalities == ("rgb", "depth"): + if not getattr(video_expert, "joint_rgb_depth", False): + raise ValueError("Depth experiment requires an RGB/depth video expert") + elif self.geometry_modalities == ("rgb", "depth", "point"): + if not getattr(video_expert, "joint_rgb_depth_point", False): + raise ValueError("Point experiment requires an RGB/depth/point video expert") + else: + raise ValueError(f"Unsupported modalities: {self.geometry_modalities}") + + def architecture_metadata(self) -> dict[str, Any]: + metadata = super().architecture_metadata() + video = self.video_expert + outputs = [f"{name}_video" for name in self.geometry_modalities] + ["action"] + metadata.update( + { + "format_version": 2, + "model_class": type(self).__name__, + "experts": ["_".join(self.geometry_modalities) + "_video", "action"], + "outputs": outputs, + "conditioning": ( + ["rgb_frame0", "depth_frame0", "text", "initial_state"] + if bool(getattr(self, "condition_depth_frame0", True)) + else ["rgb_frame0", "text", "initial_state"] + ), + "video_modalities": list(self.geometry_modalities), + "video_latent_channels": int(video.in_dim), + "rgb_latent_channels": int(video.rgb_latent_channels), + "depth_latent_channels": int(video.depth_latent_channels), + "point_latent_channels": int( + getattr(video, "point_latent_channels", 0) + ), + "point_first_frame_timestep_adapter": bool( + hasattr(video, "point_first_frame_timestep_adapter") + ), + } + ) + return metadata + + +def _build_geometry_action_mot( + video_expert: torch.nn.Module, + action_dit_config: Mapping[str, Any], + *, + modalities: tuple[str, ...], + random_seed: int, + action_state_dim: int, + mot_checkpoint_mixed_attn: bool, + cross_modal_attention: str, +) -> RgbGeometryActionMoT: + action_config = action_dit_05b_config(**dict(action_dit_config)) + for name, actual in ( + ("num_layers", len(video_expert.blocks)), + ("num_heads", int(video_expert.num_heads)), + ("attn_head_dim", int(video_expert.attn_head_dim)), + ): + if int(action_config.get(name, -1)) != actual: + raise ValueError(f"action_dit.{name} must match video expert ({actual})") + reference = next(video_expert.parameters()) + device, dtype = reference.device, reference.dtype + devices = [device.index or 0] if device.type == "cuda" else [] + rng_context = torch.random.fork_rng(devices=devices) if devices else nullcontext() + previous_dtype = torch.get_default_dtype() + try: + torch.set_default_dtype(dtype) + with rng_context, torch.device(device): + torch.manual_seed(int(random_seed)) + action_expert = ActionDiT(**action_config) + return RgbGeometryActionMoT( + video_expert, + action_expert, + modalities=modalities, + action_state_dim=action_state_dim, + mot_checkpoint_mixed_attn=mot_checkpoint_mixed_attn, + cross_modal_attention=cross_modal_attention, + ) + finally: + torch.set_default_dtype(previous_dtype) + + +def build_fresh_rgb_geometry_action_mot( + config: dict[str, Any], +) -> RgbGeometryActionMoT: + """Load the same Wan RGB base, then add only the configured new modalities.""" + + validate_geometry_config(config) + modalities = geometry_modalities(config) + model_config = config["model"] + dit_config = dict(model_config["dit"]) + if int(dit_config.get("in_dim", -1)) != 48 or int(dit_config.get("out_dim", -1)) != 48: + raise ValueError("The common initialization must start from the 48-channel RGB Wan") + dtype = dtype_from_name(str(model_config["dtype"])) + original_model_dir = str(config["paths"]["pretrained_model_dir"]) + local_model_dir = str(config.get("initialization", {}).get("local_wan_dit_dir", "")) + model_dir = ( + Path(local_model_dir) + if local_model_dir and Path(local_model_dir).is_dir() + else Path(original_model_dir) + ) + load_weights_on_this_rank = int(os.environ.get("LOCAL_RANK", "0")) == 0 + with torch.device("meta"), temporary_default_dtype(dtype): + video_expert = WanVideoDiT(**dit_config) + video_expert.to_empty(device="cpu") + video_expert.freqs = precompute_freqs_cis_3d(int(dit_config["attn_head_dim"])) + if load_weights_on_this_rank: + strict_load_original_dit(video_expert, model_dir) + else: + print("[model-load] nonzero rank awaits distributed state sync", flush=True) + + if modalities == ("rgb", "depth"): + video_expert = upgrade_wan_dit_for_rgb_depth(video_expert) + else: + video_expert = upgrade_wan_dit_for_rgb_depth_point(video_expert) + video_expert = enable_point_first_frame_timestep_adapter( + video_expert, + point_bottleneck_dim=int( + config["geometry"]["point"]["timestep_adapter_bottleneck_dim"] + ), + ) + model = _build_geometry_action_mot( + video_expert, + model_config["action_dit"], + modalities=modalities, + random_seed=int(model_config["action_random_seed"]), + action_state_dim=int(model_config["action_state_dim"]), + mot_checkpoint_mixed_attn=bool( + model_config.get("mot_checkpoint_mixed_attn", True) + ), + cross_modal_attention=str(model_config["cross_modal_attention"]), + ) + normalization = config["action"]["normalization"] + model.add_module( + "action_normalizer", + OnlineActionNormalizer( + action_dim=int(model_config["action_state_dim"]), + eps=float(normalization.get("eps", 1.0e-6)), + clip=normalization.get("clip", 10.0), + ), + ) + model._robodojo_initialization = { + "mode": "fresh", + "source_checkpoint": None, + "rgb_slice_from_joint_video": False, + "wan_dit_source": str(model_dir), + } + model._robodojo_objective = "joint" + return model diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_v2_checkpoint_contract.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_v2_checkpoint_contract.py new file mode 100644 index 0000000000000000000000000000000000000000..2709d5d46816c5f03bbbb82e45f1bd06e8727e36 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/robodojo_v2_checkpoint_contract.py @@ -0,0 +1,165 @@ +"""Fail-closed verifier for base-frame SpaceDreamer-v2 action transfer.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + + +SCHEMA = "spacedreamer-v2-checkpoint-contract-v1" +EXPECTED_LAYOUT = [ + "left_x", + "left_y", + "left_z", + "left_so3log_x", + "left_so3log_y", + "left_so3log_z", + "left_gripper_open_fraction", + "right_x", + "right_y", + "right_z", + "right_so3log_x", + "right_so3log_y", + "right_so3log_z", + "right_gripper_open_fraction", +] +TRANSFERABLE_COMPONENTS = [ + "action_encoder", + "state_encoder", + "action_dit", + "action_head", +] + + +def sha256_file(path: Path) -> str: + value = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(8 * 1024 * 1024), b""): + value.update(block) + return value.hexdigest() + + +def default_sidecar_path(checkpoint: Path) -> Path: + return checkpoint.with_name(f"{checkpoint.name}.v2_contract.json") + + +def verify_v2_baseframe_checkpoint_contract( + checkpoint: Path, + sidecar: Path, + *, + expected_checkpoint_sha256: str, + expected_action_contract_fingerprint: str, +) -> dict[str, Any]: + """Verify checkpoint bytes and the exact source/target transfer semantics.""" + + checkpoint = checkpoint.resolve() + sidecar = sidecar.resolve() + if len(expected_checkpoint_sha256) != 64: + raise ValueError("Expected checkpoint SHA256 must contain 64 hex characters") + if len(expected_action_contract_fingerprint) != 64: + raise ValueError("Expected action-contract fingerprint must contain 64 hex characters") + try: + int(expected_checkpoint_sha256, 16) + int(expected_action_contract_fingerprint, 16) + except ValueError as error: + raise ValueError("Expected checkpoint/contract hashes must be hexadecimal") from error + if not checkpoint.is_file(): + raise FileNotFoundError(checkpoint) + if not sidecar.is_file(): + raise FileNotFoundError(sidecar) + actual_checkpoint_sha256 = sha256_file(checkpoint) + if actual_checkpoint_sha256 != expected_checkpoint_sha256: + raise RuntimeError( + "SpaceDreamer-v2 checkpoint SHA256 mismatch: " + f"{actual_checkpoint_sha256} != {expected_checkpoint_sha256}" + ) + payload = json.loads(sidecar.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or ( + payload.get("complete") is not True + or payload.get("status") != "sealed_checkpoint_contract" + or payload.get("schema") != SCHEMA + ): + raise RuntimeError(f"Invalid SpaceDreamer-v2 checkpoint sidecar: {sidecar}") + if payload.get("checkpoint", {}).get("sha256") != actual_checkpoint_sha256: + raise RuntimeError("Checkpoint sidecar does not bind the selected checkpoint bytes") + + action_contract = payload.get("action_contract") + if not isinstance(action_contract, dict): + raise RuntimeError("Checkpoint sidecar has no sealed action contract") + fingerprint = action_contract.get("fingerprint") + if fingerprint != expected_action_contract_fingerprint: + raise RuntimeError( + "SpaceDreamer-v2 action-contract fingerprint mismatch: " + f"{fingerprint} != {expected_action_contract_fingerprint}" + ) + semantics = action_contract.get("semantics") + if not isinstance(semantics, dict): + raise RuntimeError("Checkpoint sidecar has no action semantic payload") + required_semantics = { + "contract_name": "agibot_robodojo_action", + "contract_version": 2, + "action_dimension": 14, + "action_layout": EXPECTED_LAYOUT, + "action_semantics": "absolute_next_observed_ee_state_t_plus_1", + "coordinate_frame": "source_robot_base", + "endpoint_type": "dual_arm_flange", + "fps": 25.0, + "horizon": 32, + "observation_window_frames": 33, + "initial_state": "state_at_observation_t", + "integration": "none", + "rotation_encoding": "absolute_so3_log", + "translation_encoding": "absolute_xyz", + "target_alignment": "state_at_next_observation_t_plus_1_through_t_plus_32", + } + mismatches = { + key: {"expected": expected, "actual": semantics.get(key)} + for key, expected in required_semantics.items() + if semantics.get(key) != expected + } + if mismatches: + raise RuntimeError(f"SpaceDreamer-v2 source action semantics mismatch: {mismatches}") + normalization = semantics.get("normalization", {}) + if normalization != { + "clip": False, + "fit_population": "agibot_joint_training_next_observation_targets_only", + "type": "pi05_q01_q99_all_action_v1", + }: + raise RuntimeError("SpaceDreamer-v2 source normalization contract is not exact") + source_policy = semantics.get("source_command_policy", {}) + if source_policy.get("action/end") != "provenance_only_never_training_target" or ( + source_policy.get("target") + != "resampled_state/end_and_state/effector_at_t_plus_1" + ): + raise RuntimeError("SpaceDreamer-v2 action source/alignment contract is unsafe") + + transfer = payload.get("transfer_policy") + if not isinstance(transfer, dict) or ( + transfer.get("transferable_components") != TRANSFERABLE_COMPONENTS + or transfer.get("target_action_normalizer") + != "refit_target_dataset_q01_q99_do_not_reuse_source" + or transfer.get("semantic_match_required") is not True + or transfer.get("checkpoint_sha256_sidecar_required") is not True + ): + raise RuntimeError("SpaceDreamer-v2 transfer policy is unsafe") + return { + "verified": True, + "schema": SCHEMA, + "checkpoint_sha256": actual_checkpoint_sha256, + "sidecar_path": str(sidecar), + "sidecar_sha256": sha256_file(sidecar), + "action_contract_sha256": str(action_contract["sha256"]), + "action_contract_fingerprint": str(fingerprint), + "source_action_semantics": required_semantics, + "source_normalization": normalization, + "target_action_semantics": { + "representation": "dual_arm_absolute_translation_so3_log_plus_grip", + "coordinate_frame": "robodojo_robot_base", + "target": "commanded_absolute_ee_action_t_empirically_matches_state_t_plus_1", + "dimension": 14, + "horizon": 32, + }, + "target_normalizer_policy": transfer["target_action_normalizer"], + } diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_piper_pretrained018126.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_piper_pretrained018126.py new file mode 100644 index 0000000000000000000000000000000000000000..f525a740eb878154a215b631fda3546296f47eb4 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_piper_pretrained018126.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Fail-closed audit for the controlled Piper spatial-motion transfer run.""" + +from __future__ import annotations + +import argparse +import json +import sys +from copy import deepcopy +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import train_piper_rgb_state_joint_action_quantile_4way as piper # noqa: E402 + + +BASE = ROOT / "ablation_test/piper_experiment/config.yaml" +CANDIDATE = ( + ROOT + / "ablation_test/piper_pretrained018126_source_norm_gbs256_8gpu_20epoch/config.yaml" +) +BASE_INDEX = ROOT / "ablation_test/piper_experiment/piper_episode_index.json" + + +def _strip_runtime_differences(config: dict) -> dict: + value = deepcopy(config) + value["experiment"].pop("name", None) + value["experiment"].pop("output_dir", None) + for key in ( + "dataset_root", + "piper_index_file", + "cache_dir", + "rgb_latent_cache_dir", + "rgb_latent_compatible_source_fingerprints", + ): + value["paths"].pop(key, None) + value.pop("initialization", None) + value.pop("tracking", None) + value.pop("transfer_contract", None) + value["training"].pop("expected_num_processes", None) + value["training"].pop("gradient_accumulation_steps", None) + return value + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--config", type=Path, default=CANDIDATE) + args = parser.parse_args() + + base = piper.load_piper_config(BASE) + candidate_path = args.config.resolve() + candidate = piper.load_piper_config(candidate_path) + if _strip_runtime_differences(base) != _strip_runtime_differences(candidate): + raise SystemExit("Candidate changes training fields outside the controlled transfer") + + piper.validate_piper_config(candidate) + old_index = json.loads(BASE_INDEX.read_text(encoding="utf-8")) + new_index = json.loads( + Path(candidate["paths"]["piper_index_file"]).read_text(encoding="utf-8") + ) + for payload in (old_index, new_index): + payload.pop("dataset_root", None) + payload.pop("relocation_audit", None) + if old_index != new_index: + raise SystemExit("Relocated Piper index changed content records") + + initialization = candidate["initialization"] + report = { + "status": "PASS", + "comparison": "controlled_spatial_motion_pretraining_ablation", + "baseline": str(BASE), + "candidate": str(candidate_path), + "unchanged_training_contract": True, + "dataset_relocation_only": True, + "episode_count": 423, + "task_counts": { + name: int(spec["perfect"]) + for name, spec in candidate["piper"]["expected_episode_counts"].items() + }, + "epochs": int(candidate["training"]["exact_epochs"]), + "optimizer_steps": int(candidate["training"]["max_steps"]), + "global_batch_size": int(candidate["training"]["global_batch_size"]), + "num_processes": int(candidate["training"]["expected_num_processes"]), + "gradient_accumulation_steps": int( + candidate["training"]["gradient_accumulation_steps"] + ), + "save_every": int(candidate["training"]["save_every"]), + "changed_scientific_factors": { + "initialization": initialization["mode"], + "source_checkpoint": initialization["checkpoint"], + "source_action_normalizer": initialization["action_normalizer_transfer"], + }, + "normalizer_semantic_warning": ( + "source absolute xyz/so3 statistics are intentionally reused for Piper joint targets" + ), + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + print(json.dumps(report, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_piper_pretrained018126_target_norm.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_piper_pretrained018126_target_norm.py new file mode 100644 index 0000000000000000000000000000000000000000..f89cb1f36a89e418ac66ff604f01292b7a2044af --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_piper_pretrained018126_target_norm.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Fail-closed audit for the Piper-q01/q99 SpaceDreamer-v3 transfer.""" +import argparse +import json +from copy import deepcopy +from pathlib import Path +import sys +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT)) +import train_piper_rgb_state_joint_action_quantile_4way as piper + +REFERENCE=ROOT/'ablation_test/piper_pretrained018126_source_norm_gbs256_16gpu_20epoch/config.yaml' +TARGET=ROOT/'ablation_test/piper_pretrained018126_target_norm_gbs256_16gpu_20epoch/config.yaml' + +def scrub(c): + c=deepcopy(c) + for k in ('experiment','tracking','transfer_contract'): c.pop(k,None) + for k in ('piper_index_file','cache_dir','rgb_latent_cache_dir'): c['paths'].pop(k,None) + c['initialization'].pop('action_normalizer_transfer',None) + c['initialization'].pop('local_checkpoint',None) + return c + +def main(): + ap=argparse.ArgumentParser();ap.add_argument('--config',type=Path,default=TARGET);ap.add_argument('--output',type=Path,required=True);a=ap.parse_args() + old=piper.load_piper_config(REFERENCE);new=piper.load_piper_config(a.config) + piper.validate_piper_config(new) + if scrub(old)!=scrub(new):raise RuntimeError('Non-normalizer training configuration changed') + if new['initialization']['action_normalizer_transfer']!=piper.TARGET_NORMALIZER_TRANSFER:raise RuntimeError('Piper q01/q99 target refit not selected') + if new['tracking'].get('mode')!='online' or new['tracking'].get('backend')!='wandb':raise RuntimeError('Online W&B missing') + old_idx=json.loads(Path(old['paths']['piper_index_file']).read_text()) + new_idx=json.loads(Path(new['paths']['piper_index_file']).read_text()) + if old_idx!=new_idx:raise RuntimeError('Piper episode index changed') + report={'status':'PASS','source_checkpoint':new['initialization']['checkpoint'],'source_sha256':new['initialization']['source_sha256'],'normalizer':'Piper train split q01/q99','other_training_config':'identical to source-normalizer run','episodes':423,'epochs':20,'global_batch_size':256,'processes':16,'wandb':'online'} + a.output.parent.mkdir(parents=True,exist_ok=True);a.output.write_text(json.dumps(report,indent=2,sort_keys=True)+'\n');print(json.dumps(report,sort_keys=True)) +if __name__=='__main__':main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_robodojo_full35_training.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_robodojo_full35_training.py new file mode 100644 index 0000000000000000000000000000000000000000..b46cda836c04a10d1d2cc1a243cba66b093c0b0c --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_robodojo_full35_training.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Check the full 35-task RoboDojo training inventory and epoch geometry.""" + +from __future__ import annotations + +import json +import math +from pathlib import Path + +import h5py +import pyarrow.parquet as pq +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = ROOT / "ablation_test/spacedreamer_v3_robodojo_full35_rgb_depth_to_rgb_depth_point_action_gbs256_32gpu_20epoch_save5000/config.yaml" +TASKS = ROOT / "robodojo_full_35_tasks.json" +BASE_RESOLVED = Path("/workspace/shouqx@xiaopeng.com/training_runs/spacedreamer_v3_robodojo_full_rgb_depth_to_rgb_depth_point_action_srs_trail1_gbs256_16gpu_20epoch/run/resolved_config.yaml") + + +def merge(base: dict, override: dict) -> dict: + result = dict(base) + for key, value in override.items(): + if key == "task_index_by_id": + result[key] = value + elif isinstance(value, dict) and isinstance(result.get(key), dict): + result[key] = merge(result[key], value) + else: + result[key] = value + return result + + +def main() -> None: + overlay = yaml.safe_load(CONFIG.read_text()) + overlay.pop("base_config") + config = merge(yaml.safe_load(BASE_RESOLVED.read_text()), overlay) + tasks = json.loads(TASKS.read_text()) + assert len(tasks) == 35 + assert config["robodojo"]["dataset_scope"] == "full_35_tasks" + assert config["robodojo"]["task_index_by_id"] == { + name: index for name, (index, _) in tasks.items() + } + task_lines = [line.strip() for line in Path(config["robodojo"]["task_file"]).read_text().splitlines() + if line.strip() and not line.startswith("#")] + assert task_lines == sorted(tasks) + + dataset = Path(config["paths"]["dataset_root"]) + native = Path(config["paths"]["native_depth_root"]) + info = json.loads((dataset / "meta/info.json").read_text()) + assert info["total_tasks"] == 35 and info["total_episodes"] == 3500 + prompts = pq.read_table(dataset / "meta/tasks.parquet").to_pydict() + episodes = pq.read_table(dataset / "meta/episodes/chunk-000/file-000.parquet", + columns=["episode_index", "length", "tasks"]).to_pydict() + assert len(episodes["episode_index"]) == 3500 + assert sorted(episodes["episode_index"]) == list(range(3500)) + assert sum(episodes["length"]) == info["total_frames"] == 1_859_602 + for name, (index, prompt) in tasks.items(): + assert prompts["task_index"][index] == index + assert prompts["__index_level_0__"][index] == prompt + for local_index in range(100): + episode_index = 100 * index + local_index + assert episodes["tasks"][episode_index] == [prompt] + source = native / name / "arx_x5/data" / f"episode_{local_index:07d}.hdf5" + assert source.is_file() and source.stat().st_size > 0, source + with h5py.File(native / name / "arx_x5/data/episode_0000000.hdf5") as handle: + instruction = handle["instruction"][()] + if isinstance(instruction, bytes): + instruction = instruction.decode() + assert instruction == prompt + + training = config["training"] + expected_steps = 20 * math.ceil(info["total_frames"] / 256) + assert config["data"]["num_clips"] == 7000 + assert config["data"]["episode_split_per_task"] == {"train": 100, "validation": 0, "test": 0} + assert training["valid_windows_per_epoch"] == info["total_frames"] + assert training["global_batch_size"] == 256 + assert config["data"]["batch_size"] * 32 * training["gradient_accumulation_steps"] == 256 + assert training["exact_epochs"] == 20 + assert training["max_steps"] == expected_steps == 145300 + assert training["save_every"] == 5000 + assert training["checkpoint_sweep_interval"] is None + assert "resume" not in config + assert config["initialization"]["action_normalizer_transfer"] == "preserve_source_pretraining_q01_q99" + assert config["modality_contract"]["input_conditions"] == ["rgb_frame0", "depth_frame0", "prompt", "initial_state"] + assert config["modality_contract"]["predictions"] == ["rgb_frames_1_to_32", "depth_frames_1_to_32", "point_frames_1_to_32", "action_chunk_32"] + print(json.dumps({"status": "PASS", "tasks": 35, "episodes": 3500, + "windows_per_epoch": info["total_frames"], "epochs": 20, + "optimizer_steps": expected_steps, "world_size": 32, "save_every": 5000, + "fixed_step_evaluation": False}, sort_keys=True), flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_robodojo_full_normalizer_noaug_configs.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_robodojo_full_normalizer_noaug_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..407119ca1d30b6010b219f595730bac69469c133 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_robodojo_full_normalizer_noaug_configs.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Audit RoboDojo-mini transfers that retain the pretraining RoboDojo-full q01/q99.""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +from pathlib import Path +import sys +from types import SimpleNamespace +from typing import Any + +import torch + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +import train_robodojo_rgb_state_joint_action_4way as fourway +from scripts.audit_source_normalizer_noaug_configs import resolve + + +TRANSFER = "load_robodojo_full_q01_q99" +EXPECTED_SHA256 = "efe619631560fdb6b7d0b7523bb9866109e8cabc9d01a2e5eb91ffe23ba8ec09" +NAMES = { + "rgb": "rgb_action", + "depth": "rgb_depth_action", + "point": "rgb_depth_point_action", +} + + +def without_operational_changes(config: dict[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(config) + for key in ("experiment", "tracking", "modality_contract"): + result.pop(key, None) + for key in ("rgb_latent_cache_dir", "depth_latent_cache_dir", "point_latent_cache_dir"): + result.get("paths", {}).pop(key, None) + initialization = result.get("initialization", {}) + for key in ( + "action_normalizer_transfer", + "action_normalizer_path", + "action_normalizer_sha256", + "local_checkpoint", + ): + initialization.pop(key, None) + return result + + +class BufferNormalizer(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.register_buffer("count", torch.tensor(0, dtype=torch.long)) + self.register_buffer("fitted", torch.tensor(False, dtype=torch.bool)) + self.register_buffer("mean", torch.zeros(14, dtype=torch.float32)) + self.register_buffer("std", torch.ones(14, dtype=torch.float32)) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--variant", choices=tuple(NAMES), required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + stem = f"robodojo_mini_pretrained018126_{NAMES[args.variant]}" + suffix = "_norm_noaug_gbs256_8gpu_20epoch" + config_path = ROOT / "ablation_test" / f"{stem}_robodojo_full{suffix}" / "config.yaml" + target_path = ROOT / "ablation_test" / f"{stem}_target{suffix}" / "config.yaml" + config = resolve(args.variant, config_path) + target = resolve(args.variant, target_path) + if without_operational_changes(config) != without_operational_changes(target): + raise RuntimeError("A non-normalizer model/data/training setting changed") + + initialization = config["initialization"] + if initialization.get("action_normalizer_transfer") != TRANSFER: + raise RuntimeError("RoboDojo-full action normalizer transfer was not selected") + stats_path = Path(initialization["action_normalizer_path"]) + actual_sha256 = hashlib.sha256(stats_path.read_bytes()).hexdigest() + if actual_sha256 != EXPECTED_SHA256: + raise RuntimeError(f"RoboDojo-full normalizer hash mismatch: {actual_sha256}") + if initialization.get("action_normalizer_sha256") != actual_sha256: + raise RuntimeError("Config normalizer hash does not match the audited file") + + payload = torch.load(stats_path, map_location="cpu", weights_only=True) + q01 = torch.as_tensor(payload["q01"], dtype=torch.float32) + q99 = torch.as_tensor(payload["q99"], dtype=torch.float32) + mean = torch.as_tensor(payload["mean"], dtype=torch.float32) + std = torch.as_tensor(payload["std"], dtype=torch.float32) + if payload.get("normalization_type") != "pi05_q01_q99_all_action_v1": + raise RuntimeError("Unexpected RoboDojo-full normalization type") + if not bool(payload["fitted"]) or int(payload["count"]) != 57_659_104: + raise RuntimeError("Unexpected RoboDojo-full normalizer population") + if not torch.allclose(mean, (q01 + q99) * 0.5, atol=1.0e-6, rtol=0.0): + raise RuntimeError("RoboDojo-full q01/q99 center mismatch") + if not torch.allclose(std, (q99 - q01) * 0.5, atol=1.0e-6, rtol=0.0): + raise RuntimeError("RoboDojo-full q01/q99 scale mismatch") + + model = SimpleNamespace(action_normalizer=BufferNormalizer()) + previous = fourway._ACTIVE_CONFIG + fourway._ACTIVE_CONFIG = config + try: + fourway.fit_action_normalizer_cached( + model, SimpleNamespace(cache_dir=config["paths"]["cache_dir"]) + ) + finally: + fourway._ACTIVE_CONFIG = previous + if not torch.equal(model.action_normalizer.mean, mean): + raise RuntimeError("Runtime loader did not copy the RoboDojo-full center exactly") + if not torch.equal(model.action_normalizer.std, std): + raise RuntimeError("Runtime loader did not copy the RoboDojo-full scale exactly") + if int(model.action_normalizer.count) != int(payload["count"]): + raise RuntimeError("Runtime loader did not copy the RoboDojo-full count") + + training = config["training"] + expected_training = { + "exact_epochs": 20, + "max_steps": 33580, + "max_steps_cap": 33580, + "global_batch_size": 256, + "save_every": 10000, + } + if {key: training.get(key) for key in expected_training} != expected_training: + raise RuntimeError("Training geometry changed") + if config["modality_contract"].get("rgb_augmentation") != "disabled": + raise RuntimeError("RGB augmentation is enabled") + + report = { + "status": "PASS", + "variant": args.variant, + "config": str(config_path), + "target_normalizer_reference": str(target_path), + "only_action_normalizer_and_operational_paths_changed": True, + "normalizer_transfer": TRANSFER, + "normalizer_path": str(stats_path), + "normalizer_sha256": actual_sha256, + "normalizer_count": int(payload["count"]), + "training": expected_training, + "augmentation": "disabled", + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + print(json.dumps(report, sort_keys=True), flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_source_normalizer_noaug_configs.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_source_normalizer_noaug_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..fb046654578e590068c69203dd574ad044c94b01 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_source_normalizer_noaug_configs.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Audit the three no-augmentation, source-normalizer RoboDojo transfers.""" + +from __future__ import annotations + +import argparse +import copy +import json +from pathlib import Path +import sys +from typing import Any + +import torch + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +import train_robodojo_rgb_state_joint_action_4way as fourway +import train_robodojo_rgb_state_joint_action_absolute_ee_4way as absolute +import train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way as quantile +from robodojo_v2_checkpoint_contract import verify_v2_baseframe_checkpoint_contract +from train_robodojo_rgb_native_depth_state_joint_action_fastwam import ( + resolve_native_depth_config, +) +from train_robodojo_rgb_native_depth_point_state_joint_action_fastwam import ( + resolve_native_depth_point_config, +) + + +SOURCE_TRANSFER = "preserve_source_pretraining_q01_q99" +EXPERIMENTS = { + "rgb": ( + "robodojo_mini_pretrained018126_rgb_action_source_norm_noaug_gbs256_8gpu_20epoch", + "robodojo_mini_absolute_xyz_so3_q01q99_full_bidir_nostopgrad_fastwam_tailpad_wanflow_fp32master_gbs256_8gpu_20epoch", + ), + "depth": ( + "robodojo_mini_pretrained018126_rgb_depth_action_source_norm_noaug_gbs256_8gpu_20epoch", + "robodojo_mini_native_gt_rgb_depth_action_fresh_aligned_rgbaction033580cfg_fastwam_tailpad_wanflow_fp32master_gbs256_8gpu_20epoch_r1", + ), + "point": ( + "robodojo_mini_pretrained018126_rgb_depth_point_action_source_norm_noaug_gbs256_8gpu_20epoch", + "robodojo_mini_native_gt_rgb_depth_point_action_fresh_aligned_rgbaction033580cfg_fastwam_tailpad_wanflow_fp32master_gbs256_8gpu_20epoch_r1", + ), +} + + +def resolve_rgb(path: Path) -> dict[str, Any]: + config = absolute.load_absolute_ee_config(path) + previous = fourway._original_validate_config + fourway._original_validate_config = quantile.validate_quantile_absolute_ee + try: + return fourway.validate_config(config) + finally: + fourway._original_validate_config = previous + + +def resolve(variant: str, path: Path) -> dict[str, Any]: + if variant == "rgb": + return resolve_rgb(path) + if variant == "depth": + return resolve_native_depth_config(path) + return resolve_native_depth_point_config(path) + + +def without_allowed_changes(config: dict[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(config) + for key in ("experiment", "initialization", "tracking", "modality_contract"): + result.pop(key, None) + for key in ( + "rgb_latent_cache_dir", + "depth_latent_cache_dir", + "point_latent_cache_dir", + ): + result.get("paths", {}).pop(key, None) + return result + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--variant", choices=tuple(EXPERIMENTS), required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--config-only", + action="store_true", + help="Skip the large checkpoint read; cluster preflight must not use this flag.", + ) + args = parser.parse_args() + + name, reference_name = EXPERIMENTS[args.variant] + config_path = ROOT / "ablation_test" / name / "config.yaml" + reference_path = ROOT / "ablation_test" / reference_name / "config.yaml" + config = resolve(args.variant, config_path) + reference = resolve(args.variant, reference_path) + if without_allowed_changes(config) != without_allowed_changes(reference): + raise RuntimeError("Effective training/data/model config differs from fresh reference") + if "rgb_augmentation" in config: + raise RuntimeError("No-augmentation experiment unexpectedly defines rgb_augmentation") + init = config["initialization"] + if init.get("action_normalizer_transfer") != SOURCE_TRANSFER: + raise RuntimeError("Source action normalizer transfer is not explicitly selected") + training = config["training"] + required_training = { + "exact_epochs": 20, + "max_steps": 33580, + "max_steps_cap": 33580, + "global_batch_size": 256, + "save_every": 10000, + } + actual_training = {key: training.get(key) for key in required_training} + if actual_training != required_training: + raise RuntimeError(f"Training geometry mismatch: {actual_training}") + + checkpoint = Path(init["checkpoint"]) + if args.config_only: + contract = { + "checkpoint_sha256": str(init["source_sha256"]), + "target_normalizer_policy": "not_read_in_config_only_audit", + } + normalizer_report: dict[str, Any] = {"checkpoint_io": "skipped"} + else: + contract = verify_v2_baseframe_checkpoint_contract( + checkpoint, + Path(init["contract_sidecar"]), + expected_checkpoint_sha256=str(init["source_sha256"]), + expected_action_contract_fingerprint=str( + init["source_action_contract_fingerprint"] + ), + ) + payload = torch.load(checkpoint, map_location="cpu", weights_only=True, mmap=True) + source = payload["state_dict"] + normalizer = { + key.removeprefix("action_normalizer."): source[key].detach().cpu() + for key in sorted(source) + if key.startswith("action_normalizer.") + } + if set(normalizer) != {"count", "fitted", "mean", "std"}: + raise RuntimeError(f"Source normalizer keys changed: {sorted(normalizer)}") + if not bool(normalizer["fitted"].item()) or int(normalizer["count"].item()) < 2: + raise RuntimeError("Source action normalizer is not fitted") + if not torch.isfinite(normalizer["mean"]).all() or not torch.all(normalizer["std"] > 0): + raise RuntimeError("Source action normalizer statistics are invalid") + normalizer_report = { + "count": int(normalizer["count"].item()), + "fitted": bool(normalizer["fitted"].item()), + "mean": normalizer["mean"].tolist(), + "std": normalizer["std"].tolist(), + } + + report = { + "status": "PASS", + "variant": args.variant, + "experiment": name, + "fresh_reference": reference_name, + "only_allowed_config_sections_differ": True, + "rgb_augmentation": "disabled_absent_from_effective_config", + "training": required_training, + "checkpoint": str(checkpoint), + "checkpoint_sha256": contract["checkpoint_sha256"], + "source_contract_transfer_policy": contract["target_normalizer_policy"], + "explicit_experiment_normalizer_override": SOURCE_TRANSFER, + "source_normalizer": normalizer_report, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + print(json.dumps(report, sort_keys=True), flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_spacedreamer_v3_robodojo_srs_resume5000.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_spacedreamer_v3_robodojo_srs_resume5000.py new file mode 100644 index 0000000000000000000000000000000000000000..c0f885c3e86892d161e5bbe4cf39676330564890 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_spacedreamer_v3_robodojo_srs_resume5000.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Verify the 2k model-only continuation changes only save/resume bookkeeping.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from audit_spacedreamer_v3_robodojo_srs_trail1 import flatten, load + + +ROOT = Path(__file__).resolve().parents[1] +OLD = ROOT / "ablation_test/spacedreamer_v3_robodojo_full_rgb_depth_to_rgb_depth_point_action_srs_trail1_gbs256_16gpu_20epoch/config.yaml" +NEW = ROOT / "ablation_test/spacedreamer_v3_robodojo_full_rgb_depth_to_rgb_depth_point_action_srs_trail1_resume002000_save5000_16gpu_20epoch/config.yaml" +ALLOWED = { + "experiment.name", "experiment.output_dir", + "training.save_every", "training.checkpoint_sweep_interval", + "tracking.run_name", "tracking.tags", + "resume.mode", "resume.checkpoint", "resume.global_step", + "resume.expected_initialization_checkpoint", + "resume.optimizer_state", "resume.scheduler_state", + "resume.dataloader_position", +} + + +def main() -> None: + old, new = load(OLD), load(NEW) + before, after = flatten(old), flatten(new) + changed = {key for key in before | after if before.get(key) != after.get(key)} + unexpected = sorted(changed - ALLOWED) + if unexpected: + raise SystemExit(f"unexpected continuation changes: {unexpected}") + training = new["training"] + assert training["save_every"] == 5000 + assert training["checkpoint_sweep_interval"] is None + assert training["max_steps"] == 33580 and training["exact_epochs"] == 20 + assert new["resume"] == { + "mode": "model_only", + "checkpoint": str(Path(new["resume"]["checkpoint"])), + "expected_initialization_checkpoint": old["initialization"]["checkpoint"], + "global_step": 2000, + "optimizer_state": "reset", + "scheduler_state": "reconstruct_from_global_step", + "dataloader_position": "reconstruct_from_global_step", + } + checkpoint = Path(new["resume"]["checkpoint"]) + assert checkpoint.is_file() and checkpoint.stat().st_size > 1_000_000_000 + assert checkpoint.name == "robodojo_rgb_state_joint_action_002000.pt" + assert new["sampling"] == old["sampling"] + assert new["modality_contract"] == old["modality_contract"] + assert new["initialization"] == old["initialization"] + assert new["tracking"]["mode"] == "online" + assert new.get("rgb_augmentation") is None + print(json.dumps({ + "status": "PASS", "changed_fields": sorted(changed), + "resume_step": 2000, "save_every": 5000, + "fixed_step_evaluation": False, "optimizer_state": "reset", + "checkpoint_bytes": checkpoint.stat().st_size, + }, sort_keys=True), flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_spacedreamer_v3_robodojo_srs_trail1.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_spacedreamer_v3_robodojo_srs_trail1.py new file mode 100644 index 0000000000000000000000000000000000000000..8eb6c2b7a97d5befd9361784d62e1a17964ec8d5 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_spacedreamer_v3_robodojo_srs_trail1.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +REFERENCE = ROOT / "ablation_test/robodojo_mini_pretrained018126_rgb_depth_point_action_source_norm_noaug_gbs256_8gpu_20epoch/config.yaml" +TARGET = ROOT / "ablation_test/spacedreamer_v3_robodojo_full_rgb_depth_to_rgb_depth_point_action_srs_trail1_gbs256_16gpu_20epoch/config.yaml" +ALLOWED = { + "experiment.name", "experiment.output_dir", + "paths.rgb_latent_cache_dir", "paths.depth_latent_cache_dir", "paths.point_latent_cache_dir", + "data.batch_size", "training.gradient_accumulation_steps", "training.save_every", + "training.checkpoint_sweep_interval", + "tracking.group", "tracking.run_name", "tracking.tags", +} + + +def merge(left: dict[str, Any], right: dict[str, Any]) -> dict[str, Any]: + for key, value in right.items(): + if isinstance(value, dict) and isinstance(left.get(key), dict): + merge(left[key], value) + else: + left[key] = value + return left + + +def load(path: Path) -> dict[str, Any]: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + base = data.pop("base_config", None) + return merge(load(Path(base)), data) if base else data + + +def flatten(value: Any, prefix: str = "") -> dict[str, Any]: + if isinstance(value, dict): + result = {} + for key, child in value.items(): + result.update(flatten(child, f"{prefix}.{key}" if prefix else str(key))) + return result + return {prefix: value} + + +def main() -> None: + reference, target = load(REFERENCE), load(TARGET) + left, right = flatten(reference), flatten(target) + changed = {key for key in left | right if left.get(key) != right.get(key)} + unexpected = sorted(changed - ALLOWED) + if unexpected: + raise SystemExit(f"unexpected differences from aligned prior run: {unexpected}") + assert target["modality_contract"]["input_conditions"] == ["rgb_frame0", "depth_frame0", "prompt", "initial_state"] + assert target["modality_contract"]["predictions"] == ["rgb_frames_1_to_32", "depth_frames_1_to_32", "point_frames_1_to_32", "action_chunk_32"] + assert target["initialization"]["action_normalizer_transfer"] == "preserve_source_pretraining_q01_q99" + assert target.get("rgb_augmentation") is None + assert target["sampling"]["shuffle_each_epoch"] is True + assert target["training"]["exact_epochs"] == 20 and target["training"]["max_steps"] == 33580 + assert target["training"]["save_every"] == target["training"]["checkpoint_sweep_interval"] == 2000 + assert target["data"]["batch_size"] * 16 * target["training"]["gradient_accumulation_steps"] == target["training"]["global_batch_size"] == 256 + report = { + "status": "PASS", "reference": str(REFERENCE), "target": str(TARGET), + "only_allowed_fields_differ": True, "changed_fields": sorted(changed), + "input_conditions": target["modality_contract"]["input_conditions"], + "predictions": target["modality_contract"]["predictions"], + "global_batch_size": 256, "world_size": 16, "save_every": 2000, + "exact_epochs": 20, "max_steps": 33580, "augmentation": "disabled", + "action_normalizer": "preserve_source_pretraining_q01_q99", + } + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_target_normalizer_noaug_configs.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_target_normalizer_noaug_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..ea0dd0acb3abe82c68b11262e4db5a4e9918586f --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/audit_target_normalizer_noaug_configs.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Check SpaceDreamer-v3 RoboDojo target-normalizer transfer configs.""" +import argparse +import json +from pathlib import Path +import torch +from audit_source_normalizer_noaug_configs import ROOT, resolve, without_allowed_changes + +NAMES = { + "rgb": "rgb_action", + "depth": "rgb_depth_action", + "point": "rgb_depth_point_action", +} + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--variant", choices=NAMES, required=True) + p.add_argument("--output", type=Path, required=True) + a = p.parse_args() + stem = "robodojo_mini_pretrained018126_" + NAMES[a.variant] + suffix = "_norm_noaug_gbs256_8gpu_20epoch" + target = ROOT / "ablation_test" / (stem + "_target" + suffix) / "config.yaml" + source = ROOT / "ablation_test" / (stem + "_source" + suffix) / "config.yaml" + cfg = resolve(a.variant, target) + ref = resolve(a.variant, source) + if without_allowed_changes(cfg) != without_allowed_changes(ref): + raise RuntimeError("Effective model/data/training configuration differs from source-normalizer reference") + init = cfg["initialization"] + if init["action_normalizer_transfer"] != "refit_robodojo_q01_q99": + raise RuntimeError("Target q01/q99 refit was not selected") + if any(init[k] != ref["initialization"][k] for k in + ("mode", "checkpoint", "source_sha256", "source_action_contract_fingerprint")): + raise RuntimeError("Source checkpoint transfer differs") + if cfg["tracking"].get("mode") != "online" or cfg["tracking"].get("backend") != "wandb": + raise RuntimeError("Online W&B is not configured") + if cfg["modality_contract"].get("rgb_augmentation") != "disabled": + raise RuntimeError("RGB augmentation is enabled") + cache = Path(cfg["paths"]["cache_dir"]) + stats = torch.load(cache / "action_normalizer.pt", map_location="cpu", weights_only=True) + manifest = json.loads((cache / "manifest.json").read_text()) + if stats["fingerprint"] != manifest["fingerprint"] or not bool(stats["fitted"]): + raise RuntimeError("RoboDojo cached q01/q99 stats do not match training cache") + if tuple(stats["mean"].shape) != (14,) or not bool((stats["std"] > 0).all()): + raise RuntimeError("RoboDojo cached q01/q99 stats invalid") + report = {"status":"PASS", "variant":a.variant, "config":str(target), + "source_checkpoint":init["checkpoint"], "source_sha256":init["source_sha256"], + "normalizer":"refit_robodojo_q01_q99", "cache":str(cache), + "cache_fingerprint":stats["fingerprint"], "wandb":"online", "augmentation":"disabled"} + a.output.parent.mkdir(parents=True,exist_ok=True) + a.output.write_text(json.dumps(report,indent=2,sort_keys=True)+"\n") + print(json.dumps(report,sort_keys=True)) + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/build_egoverse_canonical_motion_cache.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/build_egoverse_canonical_motion_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..1ad42ffa1fc639119aec1ba6e169813bbeec001e --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/build_egoverse_canonical_motion_cache.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Build resumable Egoverse absolute action14/camera6 canonical motion caches.""" + +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +import json +import os +from pathlib import Path +import sys +import time +from typing import Any + +import numpy as np + +REPO = Path(__file__).resolve().parents[1] +if str(REPO) not in sys.path: + sys.path.insert(0, str(REPO)) + +from canonical_spacedreamer_data import VALID_ACTION, VALID_CAMERA # noqa: E402 +from online_egoverse_action import _rigid_inverse, so3_log # noqa: E402 +from spacedreamer_ego_dataset import EgoverseEpisode, load_motion_payload # noqa: E402 + + +WORKSPACE = Path("/workspace-wl/shouqx@xiaopeng.com") +DEFAULT_MANIFEST = WORKSPACE / "dataset/Egoverse_3D_30fps/manifests/episodes.jsonl" +DEFAULT_ROOT = WORKSPACE / "dataset/Egoverse_3D_30fps" +DEFAULT_OUTPUT = WORKSPACE / "dataset/SpaceDreamerEgo_Mix_Canonical_v1/cache/egoverse_motion" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) + parser.add_argument("--annotation-root", type=Path, default=DEFAULT_ROOT) + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument("--workers", type=int, default=32) + return parser.parse_args() + + +def rows(path: Path) -> list[dict[str, Any]]: + selected = [] + with path.open("r", encoding="utf-8") as handle: + for line in handle: + row = json.loads(line) + if row.get("dataset") == "Egoverse" and row.get("metric_depth_eligible") is True: + selected.append(row) + return selected + + +def episode(row: dict[str, Any], root: Path) -> EgoverseEpisode: + index, chunk = int(row["episode_index"]), int(row["chunk_index"]) + return EgoverseEpisode( + episode_id=str(row["episode_id"]), episode_index=index, chunk_index=chunk, + length=int(row["raw_frames"]), + annotation_path=root / "episodes/Egoverse" / f"chunk-{chunk:03d}" / f"episode_{index:06d}.npz", + video_path=Path(row["video_path"]), source_metadata_path=Path(row["source_metadata_path"]), + output_width=int(row["output_width"]), output_height=int(row["output_height"]), + k_output=tuple(tuple(float(v) for v in values) for values in row["K_output"]), + distortion=tuple(float(v) for v in row.get("distortion", [])), prompt="unused", + ) + + +def so3_log_batch(rotation: np.ndarray) -> np.ndarray: + """Vectorized equivalent of ``online_egoverse_action.so3_log``.""" + + rotation = np.asarray(rotation, dtype=np.float64) + cosine = np.clip( + (np.trace(rotation, axis1=-2, axis2=-1) - 1.0) * 0.5, + -1.0, + 1.0, + ) + angle = np.arccos(cosine) + vee = 0.5 * np.stack( + [ + rotation[:, 2, 1] - rotation[:, 1, 2], + rotation[:, 0, 2] - rotation[:, 2, 0], + rotation[:, 1, 0] - rotation[:, 0, 1], + ], + axis=1, + ) + result = vee.copy() + regular = (angle >= 1.0e-7) & (np.abs(np.sin(angle)) > 1.0e-6) + result[regular] *= (angle[regular] / np.sin(angle[regular]))[:, None] + # Preserve the exact scalar implementation for the rare near-pi branch. + singular = (angle >= 1.0e-7) & ~regular + for index in np.flatnonzero(singular): + result[index] = so3_log(rotation[index]) + return result + + +def convert(row: dict[str, Any], root: Path, offset: int) -> dict[str, Any]: + item = episode(row, root) + payload = load_motion_payload(item) + reference_from_world = _rigid_inverse(payload.head_matrix[0]) + left = reference_from_world[None] @ payload.left_matrix + right = reference_from_world[None] @ payload.right_matrix + head = reference_from_world[None] @ payload.head_matrix + length = item.length + action = np.zeros((length, 14), np.float32) + camera = np.zeros((length, 6), np.float32) + action_valid = payload.left_valid & payload.right_valid & payload.left_grip_valid & payload.right_grip_valid + camera_valid = payload.head_valid + action_indices = np.flatnonzero(action_valid) + action[action_indices, 0:3] = left[action_indices, :3, 3] + action[action_indices, 3:6] = so3_log_batch(left[action_indices, :3, :3]) + action[action_indices, 6] = payload.left_grip[action_indices] + action[action_indices, 7:10] = right[action_indices, :3, 3] + action[action_indices, 10:13] = so3_log_batch(right[action_indices, :3, :3]) + action[action_indices, 13] = payload.right_grip[action_indices] + camera_indices = np.flatnonzero(camera_valid) + camera[camera_indices, :3] = head[camera_indices, :3, 3] + camera[camera_indices, 3:6] = so3_log_batch(head[camera_indices, :3, :3]) + validity = action_valid.astype(np.uint8) * VALID_ACTION | camera_valid.astype(np.uint8) * VALID_CAMERA + raw_grip = np.concatenate([payload.left_grip[action_valid], payload.right_grip[action_valid]]) + return { + "episode_id": item.episode_id, "offset": offset, "length": length, + "action": action, "camera": camera, "validity": validity, + "invalid_action": int((~action_valid).sum()), "invalid_camera": int((~camera_valid).sum()), + "grip_outside_0_1": int(((raw_grip < 0) | (raw_grip > 1)).sum()), + "valid_grip_values": int(len(raw_grip)), + } + + +def size_file(path: Path, size: int, create: bool) -> None: + if create: + with path.open("wb") as handle: + handle.truncate(size) + elif path.stat().st_size != size: + raise RuntimeError(f"Existing cache has wrong size: {path}") + + +def main() -> None: + args = parse_args() + if args.workers <= 0: + raise ValueError("workers must be positive") + selected = rows(args.manifest) + offsets: dict[str, int] = {} + total = 0 + for row in selected: + offsets[str(row["episode_id"])] = total + total += int(row["raw_frames"]) + args.output_dir.mkdir(parents=True, exist_ok=True) + final_action_path = args.output_dir / "action14.f32" + action_path = args.output_dir / "action14_raw.f32" + camera_path = args.output_dir / "camera6.f32" + validity_path = args.output_dir / "validity.u8" + progress_path = args.output_dir / "progress.jsonl" + create = not action_path.exists() + size_file(action_path, total * 14 * 4, create) + size_file(camera_path, total * 6 * 4, create) + size_file(validity_path, total, create) + action_cache = np.memmap(action_path, dtype=" argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--egoverse-manifest", type=Path, default=DEFAULT_EGOVERSE_MANIFEST) + parser.add_argument("--egoverse-root", type=Path, default=DEFAULT_EGOVERSE_ROOT) + parser.add_argument("--agibot-manifest", type=Path, default=DEFAULT_AGIBOT_MANIFEST) + parser.add_argument("--output-root", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument("--max-episodes-per-source", type=int) + parser.add_argument("--selection", choices=("manifest", "shortest"), default="manifest") + parser.add_argument("--shard-target-gib", type=float, default=4.0) + parser.add_argument("--validation-fraction", type=float, default=0.02) + parser.add_argument("--split-seed", type=int, default=20260724) + parser.add_argument("--video-codec", default="mp4v") + parser.add_argument("--partition-count", type=int, default=1) + parser.add_argument("--partition-index", type=int, default=0) + return parser.parse_args() + + +def read_rows(path: Path, source: str) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + with path.open("r", encoding="utf-8") as handle: + for line in handle: + if not line.strip(): + continue + row = json.loads(line) + if source == "Egoverse": + if row.get("dataset") != "Egoverse" or row.get("metric_depth_eligible") is not True: + continue + else: + if row.get("dataset") != "AgiBotWorld" or row.get("five_input_eligible") is not True: + continue + if int(row.get("geometry_metadata_version", -1)) != 4: + raise ValueError(f"AgiBot row is not geometry v4: {row.get('episode_id')}") + if row.get("effective_image_geometry") != "rectified_pinhole": + raise ValueError(f"AgiBot row is not rectified: {row.get('episode_id')}") + if row.get("effective_image_distortion") != []: + raise ValueError(f"AgiBot effective distortion is not empty: {row.get('episode_id')}") + rows.append(row) + return rows + + +def select_rows(rows: list[dict[str, Any]], limit: int | None, selection: str) -> list[dict[str, Any]]: + if limit is None: + return rows + if limit <= 0: + raise ValueError("--max-episodes-per-source must be positive") + if selection == "shortest": + rows = sorted(rows, key=lambda row: ( + int(row.get("aligned_frames", row["raw_frames"])), str(row["episode_id"]) + )) + return rows[:limit] + + +def _sha(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _npz_bytes(**arrays: np.ndarray) -> bytes: + stream = io.BytesIO() + np.savez_compressed(stream, **arrays) + return stream.getvalue() + + +def _json_bytes(value: Any) -> bytes: + return (json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + + +def _write_rgb_mp4(frames: np.ndarray, path: Path, codec: str) -> None: + if frames.dtype != np.uint8 or frames.ndim != 4 or frames.shape[1:] != (256, 256, 3): + raise ValueError(f"Unexpected canonical RGB shape/dtype: {frames.shape} {frames.dtype}") + if len(codec) != 4: + raise ValueError("Video codec must be a fourcc") + writer = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*codec), 30.0, (256, 256)) + if not writer.isOpened(): + raise RuntimeError(f"Could not create canonical MP4: {path}") + try: + for rgb in frames: + writer.write(cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)) + finally: + writer.release() + if not path.is_file() or path.stat().st_size == 0: + raise RuntimeError(f"Empty canonical MP4: {path}") + + +def _decode_selected_video( + video_path: Path, frame_index: np.ndarray, *, output_width: int, + output_height: int, rectify_k: np.ndarray, distortion: np.ndarray, +) -> np.ndarray: + capture = cv2.VideoCapture(str(video_path)) + if not capture.isOpened(): + raise RuntimeError(f"Could not open source RGB: {video_path}") + map_x = map_y = None + size = (output_width, output_height) + if distortion.size and np.any(np.abs(distortion) > 0): + map_x, map_y = cv2.initUndistortRectifyMap( + rectify_k.astype(np.float64), distortion.astype(np.float64), None, + rectify_k.astype(np.float64), size, cv2.CV_32FC1, + ) + frames: list[np.ndarray] = [] + next_source = 0 + try: + for target_value in frame_index.astype(np.int64): + target = int(target_value) + if target < next_source: + raise ValueError("source frame_index must be strictly increasing") + while next_source <= target: + ok, bgr = capture.read() + if not ok: + raise RuntimeError(f"Could not decode frame {target} from {video_path}") + if next_source == target: + if (bgr.shape[1], bgr.shape[0]) != size: + bgr = cv2.resize(bgr, size, interpolation=cv2.INTER_AREA) + if map_x is not None and map_y is not None: + bgr = cv2.remap(bgr, map_x, map_y, interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT) + crop = min(output_height, output_width) + top, left = (output_height - crop) // 2, (output_width - crop) // 2 + bgr = bgr[top:top + crop, left:left + crop] + bgr = cv2.resize(bgr, (256, 256), interpolation=cv2.INTER_AREA) + frames.append(cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)) + next_source += 1 + finally: + capture.release() + return np.stack(frames) + + +def _decode_selected_video_ffmpeg( + video_path: Path, frame_index: np.ndarray, *, output_width: int, output_height: int, +) -> np.ndarray: + """Software-decode AV1/H26x without depending on OpenCV's codec build.""" + command = [ + "ffmpeg", "-v", "error", "-hwaccel", "none", "-i", str(video_path), + "-vf", f"scale={output_width}:{output_height}", "-pix_fmt", "bgr24", + "-f", "rawvideo", "pipe:1", + ] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert process.stdout is not None + frame_bytes = output_width * output_height * 3 + + def read_exact() -> bytes: + chunks: list[bytes] = [] + remaining = frame_bytes + while remaining: + block = process.stdout.read(remaining) + if not block: + break + chunks.append(block) + remaining -= len(block) + return b"".join(chunks) + + frames: list[np.ndarray] = [] + next_source = 0 + try: + for target_value in frame_index.astype(np.int64): + target = int(target_value) + if target < next_source: + raise ValueError("source frame_index must be strictly increasing") + while next_source <= target: + payload = read_exact() + if len(payload) != frame_bytes: + stderr = process.stderr.read().decode("utf-8", errors="replace") if process.stderr else "" + raise RuntimeError( + f"ffmpeg could not decode source frame {target} from {video_path}: {stderr[-2000:]}" + ) + if next_source == target: + bgr = np.frombuffer(payload, dtype=np.uint8).reshape(output_height, output_width, 3) + crop = min(output_height, output_width) + top, left = (output_height - crop) // 2, (output_width - crop) // 2 + bgr = bgr[top:top + crop, left:left + crop] + bgr = cv2.resize(bgr, (256, 256), interpolation=cv2.INTER_AREA) + frames.append(cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)) + next_source += 1 + finally: + process.stdout.close() + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + return np.stack(frames) + + +def _motion_from_egoverse(payload: Any) -> tuple[np.ndarray, np.ndarray, np.ndarray, dict[str, Any]]: + reference_from_world = _rigid_inverse(payload.head_matrix[0]) + left = reference_from_world[None] @ payload.left_matrix + right = reference_from_world[None] @ payload.right_matrix + camera_matrix = reference_from_world[None] @ payload.head_matrix + length = len(payload.frame_index) + action = np.zeros((length, 14), dtype=np.float32) + camera = np.zeros((length, 6), dtype=np.float32) + valid_action = ( + payload.left_valid & payload.right_valid & + payload.left_grip_valid & payload.right_grip_valid + ) + valid_camera = payload.head_valid.copy() + for index in range(length): + if valid_action[index]: + action[index, 0:3] = left[index, :3, 3] + action[index, 3:6] = so3_log(left[index, :3, :3]) + action[index, 6] = np.clip(payload.left_grip[index], 0.0, 1.0) + action[index, 7:10] = right[index, :3, 3] + action[index, 10:13] = so3_log(right[index, :3, :3]) + action[index, 13] = np.clip(payload.right_grip[index], 0.0, 1.0) + if valid_camera[index]: + camera[index, :3] = camera_matrix[index, :3, 3] + camera[index, 3:6] = so3_log(camera_matrix[index, :3, :3]) + validity = valid_action.astype(np.uint8) * VALID_ACTION | valid_camera.astype(np.uint8) * VALID_CAMERA + grip_values = np.concatenate([payload.left_grip[valid_action], payload.right_grip[valid_action]]) + audit = { + "input_grip": "pinch_over_palm", + "output_grip": "open_fraction_clipped_0_1", + "input_grip_outside_0_1_fraction": float(np.mean((grip_values < 0) | (grip_values > 1))) if len(grip_values) else 0.0, + } + return action, camera, validity, audit + + +def convert_egoverse(row: dict[str, Any], root: Path) -> dict[str, Any]: + metadata_path = Path(row["source_metadata_path"]) + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + prompt = str(metadata.get("attributes", {}).get("task_description", "")).strip() + episode = EgoverseEpisode( + episode_id=str(row["episode_id"]), episode_index=int(row["episode_index"]), + chunk_index=int(row["chunk_index"]), length=int(row["raw_frames"]), + annotation_path=root / "episodes/Egoverse" / f"chunk-{int(row['chunk_index']):03d}" / f"episode_{int(row['episode_index']):06d}.npz", + video_path=Path(row["video_path"]), source_metadata_path=metadata_path, + output_width=int(row["output_width"]), output_height=int(row["output_height"]), + k_output=tuple(tuple(float(v) for v in values) for values in row["K_output"]), + distortion=tuple(float(v) for v in row.get("distortion", [])), prompt=prompt, + ) + payload = load_episode_payload(episode, 256) + action, camera, validity, motion_audit = _motion_from_egoverse(payload) + return { + "episode_id": episode.episode_id, + "source": "Egoverse", + "recording_group_id": str(row["source_stem"]), + "prompt": prompt, + "rgb": payload.rgb_uint8, + "depth": payload.depth_mm, + "K": payload.intrinsics, + "action": action, + "camera": camera, + "validity": validity, + "source_frame_index": payload.frame_index.astype(np.int64), + "metadata": { + "source_episode_id": row["episode_id"], + "source_manifest": str(DEFAULT_EGOVERSE_MANIFEST), + "source_geometry": "distorted_pinhole", + "rectification": "offline_opencv_undistort_then_center_crop_resize", + "endpoint_type": "observed_human_effector", + "motion_audit": motion_audit, + }, + } + + +def _extract_agibot_rgb(row: dict[str, Any], destination: Path) -> None: + rgb = row["rgb"] + if rgb.get("storage") != "uncompressed_tar_byte_range": + raise ValueError(f"Unsupported AgiBot RGB storage: {rgb.get('storage')}") + archive = Path(rgb["archive_path"]) + remaining = int(rgb["size"]) + with archive.open("rb", buffering=0) as source, destination.open("wb") as target: + source.seek(int(rgb["offset_data"])) + while remaining: + block = source.read(min(16 * 1024 * 1024, remaining)) + if not block: + raise IOError(f"Short AgiBot RGB byte range: {archive}") + target.write(block) + remaining -= len(block) + + +def convert_agibot(row: dict[str, Any], temporary: Path) -> dict[str, Any]: + length = int(row["aligned_frames"]) + with np.load(row["annotation_path"], allow_pickle=False) as bundle: + depth = bundle["depth_mm"][:length] + frame_index = bundle["frame_index"][:length].astype(np.int64) + source_k = bundle["K"].astype(np.float32) + source_mp4 = temporary / f"agibot_{int(row['source_episode_id'])}.mp4" + _extract_agibot_rgb(row, source_mp4) + try: + rgb = _decode_selected_video_ffmpeg( + source_mp4, frame_index, output_width=int(row["output_width"]), + output_height=int(row["output_height"]), + ) + finally: + source_mp4.unlink(missing_ok=True) + crop = min(depth.shape[1], depth.shape[2]) + top, left = (depth.shape[1] - crop) // 2, (depth.shape[2] - crop) // 2 + depth = depth[:, top:top + crop, left:left + crop] + depth = np.stack([cv2.resize(value, (256, 256), interpolation=cv2.INTER_NEAREST) for value in depth]) + K = scale_intrinsics_after_center_crop( + source_k, int(row["output_height"]), int(row["output_width"]), 256 + ) + motion = row["motion_cache"] + offset, motion_length = int(motion["offset"]), int(motion["length"]) + if motion_length != length: + raise ValueError(f"AgiBot motion length mismatch: {row['episode_id']}") + action_rows = Path(motion["action_path"]).stat().st_size // (4 * 14) + camera_rows = Path(motion["camera_path"]).stat().st_size // (4 * 6) + action_memmap = np.memmap(motion["action_path"], dtype=" None: + length = len(value["rgb"]) + shapes = { + "rgb": value["rgb"].shape, + "depth": value["depth"].shape, + "action": value["action"].shape, + "camera": value["camera"].shape, + "validity": value["validity"].shape, + } + expected = { + "rgb": (length, 256, 256, 3), "depth": (length, 256, 256), + "action": (length, 14), "camera": (length, 6), "validity": (length,), + } + if shapes != expected: + raise ValueError(f"Canonical shape mismatch for {value['episode_id']}: {shapes}") + if length < 33 or value["rgb"].dtype != np.uint8 or value["depth"].dtype != np.uint16: + raise ValueError(f"Canonical dtype/length mismatch for {value['episode_id']}") + if not np.isfinite(value["K"]).all() or not np.isfinite(value["action"]).all() or not np.isfinite(value["camera"]).all(): + raise ValueError(f"Non-finite canonical values for {value['episode_id']}") + if not np.allclose(value["camera"][0], 0.0, atol=1e-4): + raise ValueError(f"Frame-0 camera is not identity for {value['episode_id']}: {value['camera'][0]}") + + +class ShardWriter: + def __init__(self, root: Path, target_bytes: int) -> None: + self.root = root + self.target_bytes = target_bytes + self.shard_number = -1 + self.tar: tarfile.TarFile | None = None + self.temporary: Path | None = None + self.final: Path | None = None + self.rows: list[dict[str, Any]] = [] + self._open_next() + + def _open_next(self) -> None: + self.shard_number += 1 + self.final = self.root / "shards" / f"shard-{self.shard_number:05d}.tar" + self.temporary = self.final.with_suffix(".tar.tmp") + self.tar = tarfile.open(self.temporary, "w") + + def _close_current(self) -> None: + assert self.tar is not None and self.temporary is not None and self.final is not None + self.tar.close() + os.replace(self.temporary, self.final) + self.tar = None + + def add(self, value: dict[str, Any], codec: str, temporary: Path) -> dict[str, Any]: + validate_converted(value) + rgb_path = temporary / "canonical_rgb.mp4" + _write_rgb_mp4(value["rgb"], rgb_path, codec) + payloads = { + "rgb": ("rgb.mp4", rgb_path.read_bytes()), + "depth": ("depth.npz", _npz_bytes( + depth_mm=value["depth"], K_256=value["K"].astype(np.float32), + source_frame_index=value["source_frame_index"].astype(np.int64), + )), + "motion": ("motion.npz", _npz_bytes( + action_abs=value["action"].astype(np.float32), + camera_abs=value["camera"].astype(np.float32), + validity=value["validity"].astype(np.uint8), + )), + } + metadata = { + "schema_name": SCHEMA_NAME, "schema_version": SCHEMA_VERSION, + "episode_id": value["episode_id"], "source": value["source"], + "frames": len(value["rgb"]), "fps": 30, "resolution": [256, 256], + "geometry": "rectified_pinhole", "distortion": [], + "depth_unit": "millimeter", "depth_invalid": 0, + "coordinate_frame": "episode_first_camera", + "action_layout": "left_xyz_so3log_grip_right_xyz_so3log_grip", + "camera_layout": "translation_so3log", + "prompt": value["prompt"], **value["metadata"], + } + payloads["metadata"] = ("meta.json", _json_bytes(metadata)) + estimated = sum(len(payload) + 1024 for _, payload in payloads.values()) + assert self.tar is not None + if self.tar.fileobj.tell() > 0 and self.tar.fileobj.tell() + estimated > self.target_bytes: + self._close_current() + self._open_next() + assert self.tar is not None and self.final is not None + members: dict[str, dict[str, Any]] = {} + safe_id = value["episode_id"].replace(":", "_") + for key, (suffix, payload) in payloads.items(): + member_name = f"{safe_id}/{suffix}" + info = tarfile.TarInfo(member_name) + info.size, info.mtime, info.mode = len(payload), 0, 0o444 + offset = int(self.tar.fileobj.tell()) + tarfile.BLOCKSIZE + self.tar.addfile(info, io.BytesIO(payload)) + members[key] = { + "path": str(self.final.relative_to(self.root)), "offset": offset, + "size": len(payload), "sha256": _sha(payload), "member": member_name, + } + row = { + "schema_name": SCHEMA_NAME, "schema_version": SCHEMA_VERSION, + "episode_id": value["episode_id"], "source": value["source"], + "recording_group_id": value["recording_group_id"], + "frames": len(value["rgb"]), "fps": 30, "resolution": [256, 256], + "geometry": "rectified_pinhole", "distortion": [], + "prompt": value["prompt"], "members": members, + } + self.rows.append(row) + rgb_path.unlink(missing_ok=True) + return row + + def close(self) -> None: + if self.tar is not None: + self._close_current() + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + with temporary.open("w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + os.replace(temporary, path) + + +def split_rows(rows: list[dict[str, Any]], fraction: float, seed: int) -> dict[str, list[dict[str, Any]]]: + result = {"train": [], "validation": []} + for source in sorted({row["source"] for row in rows}): + selected = [row for row in rows if row["source"] == source] + groups: dict[str, list[dict[str, Any]]] = {} + for row in selected: + groups.setdefault(row["recording_group_id"], []).append(row) + ordered = sorted(groups, key=lambda group: (stable_u64(f"{source}:{group}", seed), group)) + target = max(1, round(len(selected) * fraction)) if len(selected) > 1 else 0 + validation_groups: set[str] = set() + count = 0 + for group in ordered: + if count >= target: + break + validation_groups.add(group) + count += len(groups[group]) + for row in selected: + result["validation" if row["recording_group_id"] in validation_groups else "train"].append(row) + return result + + +def main() -> None: + args = parse_args() + if args.partition_count <= 0 or not 0 <= args.partition_index < args.partition_count: + raise ValueError("partition index must be in [0, partition count)") + if args.output_root.exists(): + raise FileExistsError(f"Refusing to overwrite canonical output: {args.output_root}") + args.output_root.mkdir(parents=True) + (args.output_root / "shards").mkdir() + (args.output_root / "manifests").mkdir() + (args.output_root / "reports").mkdir() + rows_by_source = { + "Egoverse": select_rows(read_rows(args.egoverse_manifest, "Egoverse"), args.max_episodes_per_source, args.selection), + "AgiBotWorld": select_rows(read_rows(args.agibot_manifest, "AgiBotWorld"), args.max_episodes_per_source, args.selection), + } + rows_by_source = { + source: [ + row for row in rows + if stable_u64(str(row["episode_id"]), 20260822) % args.partition_count + == args.partition_index + ] + for source, rows in rows_by_source.items() + } + writer = ShardWriter(args.output_root, int(args.shard_target_gib * 1024**3)) + started = __import__("time").time() + try: + with tempfile.TemporaryDirectory(prefix="spacedreamer_canonical_") as directory: + temporary = Path(directory) + for source in ("Egoverse", "AgiBotWorld"): + selected = rows_by_source[source] + for number, source_row in enumerate(selected, start=1): + print(f"[canonical] source={source} episode={number}/{len(selected)} id={source_row['episode_id']}", flush=True) + converted = ( + convert_egoverse(source_row, args.egoverse_root) + if source == "Egoverse" else convert_agibot(source_row, temporary) + ) + writer.add(converted, args.video_codec, temporary) + except BaseException: + if writer.tar is not None: + writer.tar.close() + raise + writer.close() + splits = split_rows(writer.rows, args.validation_fraction, args.split_seed) + write_jsonl(args.output_root / "manifests/all.jsonl", writer.rows) + write_jsonl(args.output_root / "manifests/train.jsonl", splits["train"]) + write_jsonl(args.output_root / "manifests/validation.jsonl", splits["validation"]) + source_summary = {} + for source in sorted(rows_by_source): + selected = [row for row in writer.rows if row["source"] == source] + source_summary[source] = { + "episodes": len(selected), "frames": sum(int(row["frames"]) for row in selected), + "hours": sum(int(row["frames"]) for row in selected) / 108000.0, + "train_episodes": sum(row["source"] == source for row in splits["train"]), + "validation_episodes": sum(row["source"] == source for row in splits["validation"]), + } + report = { + "complete": True, "ready": False, "schema_name": SCHEMA_NAME, + "schema_version": SCHEMA_VERSION, "sources": source_summary, + "sampling_ratio": {"Egoverse": 0.5, "AgiBotWorld": 0.5}, + "elapsed_seconds": __import__("time").time() - started, + "partition_count": args.partition_count, + "partition_index": args.partition_index, + "note": "ready remains false until independent validation and smoke tests pass", + } + write_json(args.output_root / "reports/build.json", report) + print(json.dumps(report, indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/build_spacedreamer_indexed_canonical_manifest.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/build_spacedreamer_indexed_canonical_manifest.py new file mode 100644 index 0000000000000000000000000000000000000000..4998bf2696c94dc658934f8ef916ecbcac8a8572 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/build_spacedreamer_indexed_canonical_manifest.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""Build the unified no-copy canonical manifest for Egoverse and AgiBot.""" + +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import math +import os +from pathlib import Path +import sys +from typing import Any + +import numpy as np + +REPO = Path(__file__).resolve().parents[1] +if str(REPO) not in sys.path: + sys.path.insert(0, str(REPO)) + +from canonical_spacedreamer_data import stable_u64 # noqa: E402 + + +WORKSPACE = Path("/workspace-wl/shouqx@xiaopeng.com") +EGOVERSE_ROOT = WORKSPACE / "dataset/Egoverse_3D_30fps" +EGOVERSE_MANIFEST = EGOVERSE_ROOT / "manifests/episodes.jsonl" +AGIBOT_MANIFEST = WORKSPACE / "dataset/AgiBotWorld_Beta_Egoverse_duration_trainable_diverse_v2/manifests/spacedreamer_ego_five_input_motion_qc_v4.jsonl" +EGOVERSE_VALIDATION = EGOVERSE_ROOT / "reports/validation.json" +AGIBOT_VALIDATION = WORKSPACE / "dataset/AgiBotWorld_Beta_Egoverse_duration_trainable_diverse_v2/reports/spacedreamer_ego_five_input_motion_qc_v4_validation.json" +DEFAULT_ROOT = WORKSPACE / "dataset/SpaceDreamerEgo_Mix_Canonical_v1" +SCHEMA = "spacedreamer-canonical-indexed" +VERSION = 1 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset-root", type=Path, default=DEFAULT_ROOT) + parser.add_argument("--workers", type=int, default=32) + parser.add_argument("--validation-fraction", type=float, default=0.02) + parser.add_argument("--split-seed", type=int, default=20260724) + return parser.parse_args() + + +def read_jsonl(path: Path) -> list[dict[str, Any]]: + with path.open("r", encoding="utf-8") as handle: + return [json.loads(line) for line in handle if line.strip()] + + +def atomic_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def atomic_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + with temporary.open("w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + os.replace(temporary, path) + + +def split_rows(rows: list[dict[str, Any]], fraction: float, seed: int) -> dict[str, list[dict[str, Any]]]: + result = {"train": [], "validation": []} + for source in sorted({row["source"] for row in rows}): + selected = [row for row in rows if row["source"] == source] + groups: dict[str, list[dict[str, Any]]] = {} + for row in selected: + groups.setdefault(row["recording_group_id"], []).append(row) + ordered = sorted(groups, key=lambda group: (stable_u64(f"{source}:{group}", seed), group)) + target = max(1, round(len(selected) * fraction)) + validation_groups, count = set(), 0 + for group in ordered: + if count >= target: + break + validation_groups.add(group); count += len(groups[group]) + for row in selected: + result["validation" if row["recording_group_id"] in validation_groups else "train"].append(row) + return result + + +def lpt_max(rows: list[dict[str, Any]], ranks: int = 8) -> tuple[list[int], int]: + loads = [0] * ranks + for windows in sorted((math.ceil(int(row["frames"]) / 33) for row in rows), reverse=True): + rank = min(range(ranks), key=lambda value: (loads[value], value)) + loads[rank] += windows + return loads, max(loads) + + +def old_annotation(row: dict[str, Any]) -> Path: + return EGOVERSE_ROOT / "episodes/Egoverse" / f"chunk-{int(row['chunk_index']):03d}" / f"episode_{int(row['episode_index']):06d}.npz" + + +def build_old( + item: tuple[dict[str, Any], dict[str, Any]], motion_dir: Path +) -> dict[str, Any]: + row, motion_index = item + annotation = old_annotation(row) + with np.load(annotation, allow_pickle=False) as bundle: + K = bundle["K"].astype(np.float32) + length = len(bundle["frame_index"]) + if ( + length != int(row["raw_frames"]) + or int(motion_index["length"]) != length + or int(motion_index["invalid_action"]) != 0 + or int(motion_index["invalid_camera"]) != 0 + ): + raise RuntimeError(f"Egoverse annotation/motion integrity mismatch: {row['episode_id']}") + metadata = json.loads(Path(row["source_metadata_path"]).read_text(encoding="utf-8")) + prompt = str(metadata.get("attributes", {}).get("task_description", "")).strip() + video = Path(row["video_path"]) + return { + "schema_name": SCHEMA, "schema_version": VERSION, + "episode_id": str(row["episode_id"]), "source": "Egoverse", + "recording_group_id": str(row["source_stem"]), "frames": length, + "fps": 30, "prompt": prompt, "depth_path": str(annotation), + "rgb": {"path": str(video), "offset": 0, "size": video.stat().st_size}, + "output_width": int(row["output_width"]), "output_height": int(row["output_height"]), + "K_input": row["K_output"], "distortion": row.get("distortion", []), + "K_rectified": K.tolist(), "effective_geometry": "rectified_pinhole_after_declared_transform", + "motion": { + "action_path": str(motion_dir / "action14.f32"), + "camera_path": str(motion_dir / "camera6.f32"), + "validity_path": str(motion_dir / "validity.u8"), + "offset": int(motion_index["offset"]), "length": length, + }, + "coordinate_frame": "episode_first_camera", "endpoint_type": "observed_human_effector", + "grip_encoding": "train_q01_q99_pinchoverpalm_to_open_fraction_0_1", + } + + +def build_new(row: dict[str, Any]) -> dict[str, Any]: + length = int(row["aligned_frames"]) + with np.load(row["annotation_path"], allow_pickle=False) as bundle: + K = bundle["K"].astype(np.float32) + annotation_frames = len(bundle["frame_index"]) + if annotation_frames < length or not np.allclose(K, row["K_output"], atol=1e-3): + raise RuntimeError(f"AgiBot annotation/K mismatch: {row['episode_id']}") + rgb, motion = row["rgb"], row["motion_cache"] + if ( + rgb.get("storage") != "uncompressed_tar_byte_range" + or motion["length"] != length + or motion.get("coordinate_frame") != "episode_first_head_camera" + or motion.get("action_layout") != "left_xyz_so3log_grip_right_xyz_so3log_grip" + or motion.get("camera_layout") != "translation_so3log" + or row.get("effective_image_geometry") != "rectified_pinhole" + or row.get("effective_image_distortion_model") != "none" + or row.get("effective_image_distortion") != [] + or row.get("point_projection") != "pinhole_K_without_reapplying_sensor_distortion" + or int(row.get("geometry_metadata_version", -1)) != 4 + or row.get("five_input_eligible") is not True + ): + raise RuntimeError(f"AgiBot indexed schema mismatch: {row['episode_id']}") + return { + "schema_name": SCHEMA, "schema_version": VERSION, + "episode_id": str(row["episode_id"]), "source": "AgiBotWorld", + "recording_group_id": str(row["source_episode_id"]), "frames": length, + "fps": 30, "prompt": str(row["task_name"]).strip(), + "depth_path": str(row["annotation_path"]), + "rgb": {"path": str(rgb["archive_path"]), "offset": int(rgb["offset_data"]), "size": int(rgb["size"])}, + "output_width": int(row["output_width"]), "output_height": int(row["output_height"]), + "K_input": row["K_output"], "distortion": [], "K_rectified": K.tolist(), + "effective_geometry": "rectified_pinhole_no_distortion_reapplication", + "motion": { + "action_path": str(motion["action_path"]), "camera_path": str(motion["camera_path"]), + "validity_path": str(motion["validity_path"]), "offset": int(motion["offset"]), "length": length, + }, + "coordinate_frame": "episode_first_camera", "endpoint_type": "flange", + "flange_to_tcp_offset_applied": False, "grip_encoding": motion["grip_encoding"], + } + + +def main() -> None: + args = parse_args() + motion_dir = args.dataset_root / "cache/egoverse_motion" + motion_report = motion_dir / "report.json" + motion_index_path = motion_dir / "episodes.jsonl" + motion_report_payload = ( + json.loads(motion_report.read_text()) if motion_report.is_file() else {} + ) + if ( + motion_report_payload.get("complete") is not True + or motion_report_payload.get("episodes") != 26796 + or motion_report_payload.get("frames") != 65735137 + or motion_report_payload.get("invalid_action_frames") != 0 + or motion_report_payload.get("invalid_camera_frames") != 0 + ): + raise RuntimeError(f"Egoverse canonical motion cache is incomplete: {motion_report}") + motion_index = {row["episode_id"]: row for row in read_jsonl(motion_index_path)} + old_rows = [row for row in read_jsonl(EGOVERSE_MANIFEST) if row.get("dataset") == "Egoverse" and row.get("metric_depth_eligible") is True] + new_rows = read_jsonl(AGIBOT_MANIFEST) + if len(motion_index) != len(old_rows): + raise RuntimeError("Egoverse motion index does not exactly cover eligible episodes") + with ThreadPoolExecutor(max_workers=args.workers) as executor: + old = list(executor.map(lambda row: build_old((row, motion_index[str(row["episode_id"])]), motion_dir), old_rows)) + new = list(executor.map(build_new, new_rows)) + rows = sorted([*old, *new], key=lambda row: (row["source"], row["episode_id"])) + ids = [row["episode_id"] for row in rows] + if len(ids) != len(set(ids)) or any(not row["prompt"] for row in rows): + raise RuntimeError("Indexed canonical IDs/prompts are invalid") + splits = split_rows(rows, args.validation_fraction, args.split_seed) + manifests = args.dataset_root / "manifests" + atomic_jsonl(manifests / "all.jsonl", rows) + atomic_jsonl(manifests / "train.jsonl", splits["train"]) + atomic_jsonl(manifests / "validation.jsonl", splits["validation"]) + sources, maxima = {}, [] + for source in sorted({row["source"] for row in rows}): + source_all = [row for row in rows if row["source"] == source] + source_train = [row for row in splits["train"] if row["source"] == source] + source_validation = [row for row in splits["validation"] if row["source"] == source] + loads, maximum = lpt_max(source_train); maxima.append(maximum) + sources[source] = { + "episodes": len(source_all), "frames": sum(row["frames"] for row in source_all), + "hours": sum(row["frames"] for row in source_all) / 108000.0, + "train_episodes": len(source_train), "validation_episodes": len(source_validation), + "train_coverage_windows": sum(math.ceil(row["frames"] / 33) for row in source_train), + "eight_rank_lpt_loads": loads, + } + slots = math.ceil(max(maxima) / 16) * 16 + digest = hashlib.sha256((manifests / "all.jsonl").read_bytes()).hexdigest() + atomic_json(args.dataset_root / "reports/index_build.json", { + "complete": True, "ready": False, "schema_name": SCHEMA, "schema_version": VERSION, + "episodes": len(rows), "sources": sources, "manifest_sha256": digest, + "sampling_ratio": {"Egoverse": 0.5, "AgiBotWorld": 0.5}, + "slots_per_rank_per_mixed_epoch": slots, + "optimizer_steps_per_mixed_epoch": slots // 16, + "optimizer_steps_three_epochs": slots // 16 * 3, + "source_manifest_sha256": { + "Egoverse": hashlib.sha256(EGOVERSE_MANIFEST.read_bytes()).hexdigest(), + "AgiBotWorld": hashlib.sha256(AGIBOT_MANIFEST.read_bytes()).hexdigest(), + }, + "source_validation_sha256": { + "Egoverse": hashlib.sha256(EGOVERSE_VALIDATION.read_bytes()).hexdigest(), + "AgiBotWorld": hashlib.sha256(AGIBOT_VALIDATION.read_bytes()).hexdigest(), + }, + "egoverse_motion_cache": motion_report_payload, + "note": "ready remains false until indexed canonical validation and smoke pass", + }) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/evaluate_robodojo_train_action_generation.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/evaluate_robodojo_train_action_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..0fddb85d42debb61ef77c9967300141155d7b456 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/evaluate_robodojo_train_action_generation.py @@ -0,0 +1,566 @@ +#!/usr/bin/env python3 +"""Measure open-loop joint action-generation error on RoboDojo train windows. + +This evaluation follows deployment-style joint sampling: it conditions only on +the cached real RGB frame-0 latent, the real initial absolute EE state, and the +task text. Future RGB latents and all 32 absolute EE actions start from noise. +No future GT video or action is fed to the model. +""" + +from __future__ import annotations + +import argparse +import gc +import json +import math +from pathlib import Path +import time +from typing import Any + +import numpy as np +import torch + +from joint_rgb_action_diffusion import sample_joint_rgb_and_action +from online_egoverse_action import so3_exp, so3_log +from train_joint_rgb_depth import WanContinuousFlowMatchScheduler, dtype_from_name +import train_robodojo_rgb_state_joint_action_4way as fourway +import train_robodojo_rgb_state_joint_action_absolute_ee_4way as absolute +import train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way as quantile + + +DIM_NAMES = ( + "left_x", + "left_y", + "left_z", + "left_rx", + "left_ry", + "left_rz", + "left_grip", + "right_x", + "right_y", + "right_z", + "right_rx", + "right_ry", + "right_rz", + "right_grip", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--checkpoint", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--samples-per-task", type=int, default=2) + parser.add_argument( + "--dataset-index", + type=int, + help="Evaluate exactly one fixed train-window index instead of sampling every task", + ) + parser.add_argument("--batch-size", type=int, default=2) + parser.add_argument("--selection-seed", type=int, default=20260820) + parser.add_argument("--noise-seed", type=int, default=20260820) + parser.add_argument("--inference-steps", type=int) + parser.add_argument( + "--schedule-mode", + choices=("config_shifted", "motus_linear"), + default="config_shifted", + help=( + "config_shifted uses this experiment's configured Wan shift; " + "motus_linear reproduces thu-ml/Motus f771216 joint inference's " + "shared linear t grid while preserving the same joint forward" + ), + ) + parser.add_argument("--min-free-gib", type=float, default=45.0) + parser.add_argument("--dry-run", action="store_true") + return parser.parse_args() + + +def activate_training_stack(config_path: Path) -> dict[str, Any]: + """Apply the exact entrypoint adapters without entering the train loop.""" + + absolute.validate_absolute_ee_with_legacy_base = ( + quantile.validate_quantile_absolute_ee + ) + absolute.absolute_ee_checkpoint_metadata = quantile.quantile_checkpoint_metadata + fourway._exhaustive_action_statistics = ( + quantile.exhaustive_action_quantile_affine_statistics + ) + fourway.prepare_episode_length_normalized_cache = quantile.prepare_quantile_cache + + fourway.load_config = absolute.load_absolute_ee_config + fourway._original_validate_config = absolute.validate_absolute_ee_with_legacy_base + fourway.episode_length_normalized_cache_fingerprint = ( + absolute.absolute_ee_cache_fingerprint + ) + fourway.checkpoint_metadata = absolute.absolute_ee_checkpoint_metadata + fourway.base.build_egoverse_delta_targets = absolute.build_absolute_ee_targets + + config = fourway.load_config(config_path) + return fourway.validate_config(config) + + +def load_action_statistics(config: dict[str, Any]) -> dict[str, Any]: + path = Path(config["paths"]["cache_dir"]) / "action_normalizer.pt" + payload = torch.load(path, map_location="cpu", weights_only=True) + if payload.get("normalization_type") != quantile.NORMALIZATION_TYPE: + raise RuntimeError( + f"Expected {quantile.NORMALIZATION_TYPE}, got " + f"{payload.get('normalization_type')!r}" + ) + if tuple(payload["mean"].shape) != (14,) or tuple(payload["std"].shape) != (14,): + raise RuntimeError("Action normalizer must contain 14D affine statistics") + if not bool(payload["fitted"].item()): + raise RuntimeError("Action normalizer is not fitted") + return payload + + +def build_dataset( + config: dict[str, Any], statistics: dict[str, Any] +) -> fourway.ExhaustiveWindowDataset: + cache_dir = Path(config["paths"]["cache_dir"]) + manifest = json.loads((cache_dir / "manifest.json").read_text(encoding="utf-8")) + return fourway.ExhaustiveWindowDataset( + cache_dir, + str(manifest["fingerprint"]), + "train", + action_mean=statistics["mean"].float(), + action_std=statistics["std"].float(), + action_clip=config["action"]["normalization"].get("clip"), + ) + + +def select_windows( + dataset: fourway.ExhaustiveWindowDataset, + config: dict[str, Any], + *, + samples_per_task: int, + seed: int, +) -> list[dict[str, Any]]: + if samples_per_task <= 0: + raise ValueError("samples_per_task must be positive") + rng = np.random.default_rng(seed) + selected: list[dict[str, Any]] = [] + previous = np.concatenate(([0], dataset.cumulative_starts[:-1])).astype(np.int64) + prompts = { + task_id: fourway.EXPECTED_TASKS[task_id][1] + for task_id in config["robodojo"]["task_index_by_id"] + } + for task_id, prompt in prompts.items(): + slots = [ + slot for slot, episode in enumerate(dataset.episodes) + if episode.task == prompt + ] + if not slots: + raise RuntimeError(f"No training episodes for task {task_id}") + choices: set[int] = set() + attempts = 0 + while len(choices) < samples_per_task: + attempts += 1 + if attempts > 10_000: + raise RuntimeError(f"Could not select unique windows for {task_id}") + slot = int(rng.choice(slots)) + start = int(rng.integers(0, int(dataset.valid_starts[slot]))) + choices.add(int(previous[slot]) + start) + for index in sorted(choices): + episode, start = dataset._window_at(index) + selected.append( + { + "dataset_index": index, + "task_id": task_id, + "prompt": prompt, + "episode_index": int(episode.episode_index), + "window_start": int(start), + } + ) + return selected + + +def select_fixed_window( + dataset: fourway.ExhaustiveWindowDataset, + config: dict[str, Any], + index: int, +) -> list[dict[str, Any]]: + if index < 0 or index >= len(dataset): + raise IndexError(f"dataset-index {index} is outside [0, {len(dataset)})") + episode, start = dataset._window_at(index) + prompt_to_task = { + fourway.EXPECTED_TASKS[task_id][1]: task_id + for task_id in config["robodojo"]["task_index_by_id"] + } + try: + task_id = prompt_to_task[episode.task] + except KeyError as error: + raise RuntimeError(f"Unknown task prompt for dataset-index {index}: {episode.task}") from error + return [ + { + "dataset_index": int(index), + "task_id": task_id, + "prompt": episode.task, + "episode_index": int(episode.episode_index), + "window_start": int(start), + } + ] + + +def audit_checkpoint_metadata(metadata: dict[str, Any]) -> None: + expected = { + "cross_modal_attention": "bidirectional", + "action_to_video_attention": True, + "video_to_action_attention": True, + "action_loss_to_video_gradient": "enabled_through_video_kv", + "action_dim": 14, + "action_state_dim": 14, + } + actual = {key: metadata.get(key) for key in expected} + if actual != expected: + raise RuntimeError(f"Checkpoint architecture mismatch: {actual} != {expected}") + semantics = metadata.get("action_semantics", {}) + if semantics.get("representation") != absolute.ACTION_REPRESENTATION: + raise RuntimeError(f"Checkpoint action semantics mismatch: {semantics}") + normalization = metadata.get("action_normalization", {}) + if normalization.get("type") != quantile.NORMALIZATION_TYPE: + raise RuntimeError(f"Checkpoint normalization mismatch: {normalization}") + + +def load_model( + config: dict[str, Any], + checkpoint: Path, + statistics: dict[str, Any], + device: torch.device, + dtype: torch.dtype, +) -> torch.nn.Module: + print(f"[model] build exact RGB+Action architecture from {checkpoint}", flush=True) + model = fourway.build_experiment_model(config) + payload = torch.load(checkpoint, map_location="cpu", weights_only=True, mmap=True) + if not isinstance(payload, dict) or set(payload) != {"metadata", "state_dict"}: + raise RuntimeError("Checkpoint must contain exactly metadata and state_dict") + audit_checkpoint_metadata(payload["metadata"]) + model.load_state_dict(payload["state_dict"], strict=True, assign=True) + for key in ("mean", "std", "count", "fitted"): + checkpoint_value = getattr(model.action_normalizer, key).cpu() + cache_value = statistics[key] + if not torch.equal(checkpoint_value, cache_value): + raise RuntimeError(f"Checkpoint/cache action normalizer mismatch: {key}") + model.to(device=device, dtype=dtype).eval().requires_grad_(False) + model.video_expert.use_gradient_checkpointing = False + model.action_expert.use_gradient_checkpointing = False + del payload + gc.collect() + torch.cuda.empty_cache() + free, total = torch.cuda.mem_get_info(device) + print( + f"[model] strict load PASS; gpu_free={free / 2**30:.1f} GiB " + f"gpu_total={total / 2**30:.1f} GiB", + flush=True, + ) + return model + + +def rotation_error_deg(predicted: np.ndarray, target: np.ndarray) -> np.ndarray: + flat_pred = predicted.reshape(-1, 3) + flat_target = target.reshape(-1, 3) + values = np.empty(len(flat_pred), dtype=np.float64) + for index, (pred, gt) in enumerate(zip(flat_pred, flat_target, strict=True)): + relative = so3_exp(pred).T @ so3_exp(gt) + values[index] = np.linalg.norm(so3_log(relative)) * 180.0 / np.pi + return values.reshape(predicted.shape[:-1]) + + +def trajectory_metrics( + predicted_normalized: np.ndarray, + target_normalized: np.ndarray, + predicted_raw: np.ndarray, + target_raw: np.ndarray, +) -> dict[str, Any]: + normalized_error = predicted_normalized - target_normalized + result: dict[str, Any] = { + "normalized_mse": float(np.mean(normalized_error**2)), + "normalized_rmse": float(np.sqrt(np.mean(normalized_error**2))), + "normalized_mae": float(np.mean(np.abs(normalized_error))), + "predicted_normalized_outside_q01q99_fraction": float( + np.mean(np.abs(predicted_normalized) > 1.0) + ), + "per_dimension_normalized_mae": { + name: float(np.mean(np.abs(normalized_error[..., dim]))) + for dim, name in enumerate(DIM_NAMES) + }, + } + for side, offset in (("left", 0), ("right", 7)): + position_error = np.linalg.norm( + predicted_raw[..., offset : offset + 3] + - target_raw[..., offset : offset + 3], + axis=-1, + ) + angle_error = rotation_error_deg( + predicted_raw[..., offset + 3 : offset + 6], + target_raw[..., offset + 3 : offset + 6], + ) + grip_pred = predicted_raw[..., offset + 6] + grip_target = target_raw[..., offset + 6] + result.update( + { + f"{side}_position_mean_m": float(np.mean(position_error)), + f"{side}_position_rmse_m": float(np.sqrt(np.mean(position_error**2))), + f"{side}_position_final_mean_m": float(np.mean(position_error[..., -1])), + f"{side}_rotation_mean_deg": float(np.mean(angle_error)), + f"{side}_rotation_rmse_deg": float(np.sqrt(np.mean(angle_error**2))), + f"{side}_rotation_final_mean_deg": float(np.mean(angle_error[..., -1])), + f"{side}_gripper_mae": float(np.mean(np.abs(grip_pred - grip_target))), + f"{side}_gripper_binary_accuracy": float( + np.mean((grip_pred >= 0.5) == (grip_target >= 0.5)) + ), + } + ) + result["per_timestep"] = [] + for timestep in range(predicted_raw.shape[-2]): + entry: dict[str, float | int] = { + "timestep": timestep, + "normalized_rmse": float( + np.sqrt(np.mean(normalized_error[..., timestep, :] ** 2)) + ), + } + for side, offset in (("left", 0), ("right", 7)): + position = np.linalg.norm( + predicted_raw[..., timestep, offset : offset + 3] + - target_raw[..., timestep, offset : offset + 3], + axis=-1, + ) + angle = rotation_error_deg( + predicted_raw[..., timestep, offset + 3 : offset + 6], + target_raw[..., timestep, offset + 3 : offset + 6], + ) + entry[f"{side}_position_rmse_m"] = float(np.sqrt(np.mean(position**2))) + entry[f"{side}_rotation_mean_deg"] = float(np.mean(angle)) + result["per_timestep"].append(entry) + return result + + +def write_markdown(path: Path, summary: dict[str, Any]) -> None: + metrics = summary["aggregate"] + lines = [ + "# RoboDojo training-set action generation error", + "", + f"- Checkpoint: `{summary['checkpoint']}`", + ( + f"- Samples: {summary['sample_count']} (fixed train dataset index " + f"{summary['dataset_index']})" + if summary["dataset_index"] is not None + else f"- Samples: {summary['sample_count']} ({summary['samples_per_task']} per task)" + ), + f"- Joint denoising steps: {summary['inference_steps']}", + f"- Schedule mode: {summary['schedule_mode']}", + f"- Video/action shifts: {summary['video_shift']}/{summary['action_shift']}", + f"- Noise seed: {summary['noise_seed']}", + "- Conditioning: GT RGB frame 0 + GT initial state + task text only", + "", + "| Metric | Value |", + "|---|---:|", + f"| Normalized MSE | {metrics['normalized_mse']:.6f} |", + f"| Normalized RMSE | {metrics['normalized_rmse']:.6f} |", + f"| Normalized MAE | {metrics['normalized_mae']:.6f} |", + f"| Left position RMSE | {metrics['left_position_rmse_m']:.6f} m |", + f"| Right position RMSE | {metrics['right_position_rmse_m']:.6f} m |", + f"| Left rotation mean | {metrics['left_rotation_mean_deg']:.3f} deg |", + f"| Right rotation mean | {metrics['right_rotation_mean_deg']:.3f} deg |", + f"| Left gripper MAE / accuracy | {metrics['left_gripper_mae']:.6f} / {metrics['left_gripper_binary_accuracy']:.3f} |", + f"| Right gripper MAE / accuracy | {metrics['right_gripper_mae']:.6f} / {metrics['right_gripper_binary_accuracy']:.3f} |", + "", + "## Per sample", + "", + "| Task | Sample | Norm RMSE | L pos RMSE (m) | R pos RMSE (m) | L rot (deg) | R rot (deg) |", + "|---|---|---:|---:|---:|---:|---:|", + ] + for sample in summary["samples"]: + value = sample["metrics"] + lines.append( + f"| {sample['task_id']} | {sample['sample_id']} | " + f"{value['normalized_rmse']:.5f} | " + f"{value['left_position_rmse_m']:.5f} | " + f"{value['right_position_rmse_m']:.5f} | " + f"{value['left_rotation_mean_deg']:.2f} | " + f"{value['right_rotation_mean_deg']:.2f} |" + ) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> None: + args = parse_args() + if args.batch_size <= 0: + raise ValueError("batch_size must be positive") + config = activate_training_stack(args.config) + statistics = load_action_statistics(config) + dataset = build_dataset(config, statistics) + if args.dataset_index is None: + selection = select_windows( + dataset, + config, + samples_per_task=args.samples_per_task, + seed=args.selection_seed, + ) + else: + selection = select_fixed_window(dataset, config, args.dataset_index) + print(json.dumps({"selection": selection}, indent=2), flush=True) + if args.dry_run: + return + if not args.checkpoint.is_file(): + raise FileNotFoundError(args.checkpoint) + if args.output_dir.exists() and any(args.output_dir.iterdir()): + raise RuntimeError(f"Refusing non-empty output directory: {args.output_dir}") + args.output_dir.mkdir(parents=True, exist_ok=True) + + device = torch.device(args.device) + if device.type != "cuda" or not torch.cuda.is_available(): + raise RuntimeError("This exact checkpoint evaluation requires CUDA") + torch.cuda.set_device(device) + free, total = torch.cuda.mem_get_info(device) + if free < args.min_free_gib * 2**30: + raise RuntimeError( + f"GPU free memory {free / 2**30:.1f} GiB is below " + f"required {args.min_free_gib:.1f} GiB" + ) + print( + f"[gpu] preflight free={free / 2**30:.1f} GiB total={total / 2**30:.1f} GiB", + flush=True, + ) + dtype = dtype_from_name(str(config["model"]["dtype"])) + model = load_model(config, args.checkpoint, statistics, device, dtype) + configured_video_shift = float(config["diffusion"]["infer_shift"]) + configured_action_shift = float(config["diffusion"]["action_infer_shift"]) + if args.schedule_mode == "motus_linear": + # Motus f771216 inference_step constructs one torch.linspace(1, 0) + # grid and feeds the same t/dt to both modalities. phi(u, shift=1) is + # exactly u, so the existing scheduler reproduces that loop without + # changing model code or numerical update semantics. + video_shift = 1.0 + action_shift = 1.0 + else: + video_shift = configured_video_shift + action_shift = configured_action_shift + video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=video_shift, + ) + action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["action_num_train_timesteps"]), + shift=action_shift, + ) + inference_steps = int( + args.inference_steps + if args.inference_steps is not None + else config["diffusion"]["inference_steps"] + ) + + predicted_normalized_parts: list[torch.Tensor] = [] + target_normalized_parts: list[torch.Tensor] = [] + sample_records: list[dict[str, Any]] = [] + started = time.monotonic() + with torch.inference_mode(): + for batch_start in range(0, len(selection), args.batch_size): + chosen = selection[batch_start : batch_start + args.batch_size] + samples = [dataset[item["dataset_index"]] for item in chosen] + batch = fourway.collate_episode_length_normalized(samples) + z_rgb = batch["z_rgb"].to(device=device, dtype=dtype) + target_action = batch["action"].to(device=device, dtype=dtype) + initial_state = batch["initial_state"].to(device=device, dtype=dtype) + context_mask_cpu = batch["context_mask"] + valid_columns = context_mask_cpu.any(dim=0) + context_length = int(valid_columns.nonzero(as_tuple=False)[-1, 0]) + 1 + context = batch["context"][:, :context_length].to(device=device, dtype=dtype) + context_mask = context_mask_cpu[:, :context_length].to( + device=device, dtype=torch.bool + ) + generator = torch.Generator(device=device).manual_seed( + args.noise_seed + batch_start + ) + generated = sample_joint_rgb_and_action( + model, + video_scheduler, + action_scheduler, + video_shape=tuple(z_rgb.shape), + action_shape=tuple(target_action.shape), + context=context, + context_mask=context_mask, + initial_state=initial_state, + rgb_condition_latent=z_rgb[:, :, 0:1], + num_inference_steps=inference_steps, + generator=generator, + dtype=dtype, + device=device, + ) + predicted = generated["action"].float().cpu() + target = target_action.float().cpu() + predicted_normalized_parts.append(predicted) + target_normalized_parts.append(target) + for item, sample, pred, gt in zip( + chosen, samples, predicted, target, strict=True + ): + item.update({"sample_id": sample["sample_id"]}) + pred_raw = pred * statistics["std"] + statistics["mean"] + gt_raw = gt * statistics["std"] + statistics["mean"] + item["metrics"] = trajectory_metrics( + pred.numpy(), gt.numpy(), pred_raw.numpy(), gt_raw.numpy() + ) + sample_records.append(item) + del generated, z_rgb, target_action, initial_state, context, context_mask + print( + f"[generation] {min(batch_start + len(chosen), len(selection))}/" + f"{len(selection)} elapsed={time.monotonic() - started:.1f}s", + flush=True, + ) + + predicted_normalized = torch.cat(predicted_normalized_parts).numpy() + target_normalized = torch.cat(target_normalized_parts).numpy() + mean = statistics["mean"].numpy() + std = statistics["std"].numpy() + predicted_raw = predicted_normalized * std + mean + target_raw = target_normalized * std + mean + aggregate = trajectory_metrics( + predicted_normalized, target_normalized, predicted_raw, target_raw + ) + summary = { + "checkpoint": str(args.checkpoint), + "config": str(args.config), + "sample_count": len(selection), + "samples_per_task": args.samples_per_task, + "dataset_index": args.dataset_index, + "selection_seed": args.selection_seed, + "noise_seed": args.noise_seed, + "inference_steps": inference_steps, + "schedule_mode": args.schedule_mode, + "video_shift": video_shift, + "action_shift": action_shift, + "configured_video_shift": configured_video_shift, + "configured_action_shift": configured_action_shift, + "motus_source_audit": { + "repository": "thu-ml/Motus", + "commit": "f771216802b8a1601599422f12088bee3c068c14", + "training_timesteps": "independent video/action draws", + "joint_inference_timesteps": "shared linear t grid", + "noise_initialization": "independent Gaussian video/action latents", + "update": "one joint forward then separate Euler updates", + "condition": "reset clean video frame-0 latent after every update", + }, + "elapsed_seconds": time.monotonic() - started, + "aggregate": aggregate, + "samples": sample_records, + } + np.savez_compressed( + args.output_dir / "generated_vs_target_actions.npz", + predicted_normalized=predicted_normalized, + target_normalized=target_normalized, + predicted_raw=predicted_raw, + target_raw=target_raw, + ) + (args.output_dir / "summary.json").write_text( + json.dumps(summary, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + write_markdown(args.output_dir / "summary.md", summary) + print(json.dumps(aggregate, indent=2), flush=True) + print(f"[done] output={args.output_dir}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/merge_spacedreamer_canonical_parts.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/merge_spacedreamer_canonical_parts.py new file mode 100644 index 0000000000000000000000000000000000000000..cbe7a378fbf04e82f506f18883e1ff67309ed42c --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/merge_spacedreamer_canonical_parts.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Merge completed canonical build partitions without rewriting shard payloads.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +from pathlib import Path +import sys +from typing import Any + +REPO = Path(__file__).resolve().parents[1] +if str(REPO) not in sys.path: + sys.path.insert(0, str(REPO)) + +from canonical_spacedreamer_data import stable_u64 # noqa: E402 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset-root", type=Path, required=True) + parser.add_argument("--partition-count", type=int, required=True) + parser.add_argument("--validation-fraction", type=float, default=0.02) + parser.add_argument("--split-seed", type=int, default=20260724) + return parser.parse_args() + + +def atomic_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def atomic_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + with temporary.open("w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + os.replace(temporary, path) + + +def split_rows(rows: list[dict[str, Any]], fraction: float, seed: int) -> dict[str, list[dict[str, Any]]]: + result = {"train": [], "validation": []} + for source in sorted({row["source"] for row in rows}): + selected = [row for row in rows if row["source"] == source] + groups: dict[str, list[dict[str, Any]]] = {} + for row in selected: + groups.setdefault(row["recording_group_id"], []).append(row) + ordered = sorted(groups, key=lambda group: (stable_u64(f"{source}:{group}", seed), group)) + target = max(1, round(len(selected) * fraction)) + validation_groups: set[str] = set() + count = 0 + for group in ordered: + if count >= target: + break + validation_groups.add(group) + count += len(groups[group]) + for row in selected: + result["validation" if row["recording_group_id"] in validation_groups else "train"].append(row) + return result + + +def lpt_max(rows: list[dict[str, Any]], ranks: int) -> tuple[list[int], int]: + loads = [0] * ranks + windows = sorted((math.ceil(int(row["frames"]) / 33) for row in rows), reverse=True) + for value in windows: + target = min(range(ranks), key=lambda rank: (loads[rank], rank)) + loads[target] += value + return loads, max(loads) + + +def main() -> None: + args = parse_args() + rows: list[dict[str, Any]] = [] + missing: list[int] = [] + for index in range(args.partition_count): + relative = Path("parts") / f"part-{index:04d}" + part = args.dataset_root / relative + report_path = part / "reports/build.json" + manifest_path = part / "manifests/all.jsonl" + if not report_path.is_file() or not manifest_path.is_file(): + missing.append(index) + continue + report = json.loads(report_path.read_text(encoding="utf-8")) + if report.get("complete") is not True or int(report.get("partition_index", -1)) != index: + missing.append(index) + continue + with manifest_path.open("r", encoding="utf-8") as handle: + for line in handle: + row = json.loads(line) + for member in row["members"].values(): + member["path"] = str(relative / member["path"]) + rows.append(row) + if missing: + raise RuntimeError(f"Canonical partitions are incomplete: count={len(missing)} first={missing[:20]}") + ids = [row["episode_id"] for row in rows] + if len(ids) != len(set(ids)): + raise RuntimeError("Duplicate episode IDs across canonical partitions") + rows.sort(key=lambda row: (row["source"], row["episode_id"])) + splits = split_rows(rows, args.validation_fraction, args.split_seed) + manifests = args.dataset_root / "manifests" + atomic_jsonl(manifests / "all.jsonl", rows) + atomic_jsonl(manifests / "train.jsonl", splits["train"]) + atomic_jsonl(manifests / "validation.jsonl", splits["validation"]) + sources: dict[str, Any] = {} + maxima = [] + for source in sorted({row["source"] for row in rows}): + all_source = [row for row in rows if row["source"] == source] + train_source = [row for row in splits["train"] if row["source"] == source] + validation_source = [row for row in splits["validation"] if row["source"] == source] + loads, maximum = lpt_max(train_source, 8) + maxima.append(maximum) + sources[source] = { + "episodes": len(all_source), + "frames": sum(int(row["frames"]) for row in all_source), + "hours": sum(int(row["frames"]) for row in all_source) / 108000.0, + "train_episodes": len(train_source), + "validation_episodes": len(validation_source), + "train_coverage_windows": sum(math.ceil(int(row["frames"]) / 33) for row in train_source), + "eight_rank_lpt_loads": loads, + } + slots = math.ceil(max(maxima) / 16) * 16 + manifest_digest = hashlib.sha256((manifests / "all.jsonl").read_bytes()).hexdigest() + atomic_json(args.dataset_root / "reports/merge.json", { + "complete": True, "ready": False, "partition_count": args.partition_count, + "episodes": len(rows), "sources": sources, + "source_sampling_ratio": {"Egoverse": 0.5, "AgiBotWorld": 0.5}, + "slots_per_rank_per_mixed_epoch": slots, + "optimizer_steps_per_mixed_epoch": slots // 16, + "optimizer_steps_three_epochs": slots // 16 * 3, + "manifest_sha256": manifest_digest, + "note": "ready remains false until full independent validation", + }) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/prepare_robodojo_full35_base_cache.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/prepare_robodojo_full35_base_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..1b1eff4daada141d81dadb6a9cc06c83ecbd2346 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/prepare_robodojo_full35_base_cache.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Prepare full35 metadata without constructing a training Accelerator.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import torch + +import train_robodojo_rgb_state_joint_action_4way as fourway +import train_robodojo_rgb_state_joint_action_absolute_ee_4way as absolute +from train_robodojo_rgb_native_depth_point_state_joint_action_fastwam import ( + resolve_native_depth_point_config, +) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + args = parser.parse_args() + config = resolve_native_depth_point_config(args.config) + fourway.base.build_egoverse_delta_targets = absolute.build_absolute_ee_targets + fourway.episode_length_normalized_cache_fingerprint = absolute.absolute_ee_cache_fingerprint + fourway.prepare_episode_length_normalized_cache( + config, device=torch.device("cuda:0"), overwrite=False + ) + manifest = json.loads( + (Path(config["paths"]["cache_dir"]) / "manifest.json").read_text() + ) + assert manifest["episode_count"] == 3500, manifest["episode_count"] + assert manifest["valid_window_count"] == 1_859_602, manifest["valid_window_count"] + print("[full35-cache] base metadata PASS episodes=3500 windows=1859602", flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/run_spacedreamer_canonical_full_build.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/run_spacedreamer_canonical_full_build.py new file mode 100644 index 0000000000000000000000000000000000000000..e71ccf83a01c1d808e2849fe6dddbbf88870d397 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/run_spacedreamer_canonical_full_build.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Run the canonical build in resumable parallel partitions, then merge.""" + +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time + + +REPO = Path(__file__).resolve().parents[1] +DEFAULT_ROOT = Path("/workspace-wl/shouqx@xiaopeng.com/dataset/SpaceDreamerEgo_Mix_Canonical_v1") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset-root", type=Path, default=DEFAULT_ROOT) + parser.add_argument("--partition-count", type=int, default=256) + parser.add_argument("--workers", type=int, default=32) + parser.add_argument("--shard-target-gib", type=float, default=4.0) + return parser.parse_args() + + +def complete(part: Path, index: int, count: int) -> bool: + path = part / "reports/build.json" + if not path.is_file(): + return False + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return False + return ( + value.get("complete") is True + and int(value.get("partition_index", -1)) == index + and int(value.get("partition_count", -1)) == count + ) + + +def run_partition(root: Path, index: int, count: int, shard_gib: float) -> tuple[int, float]: + part = root / "parts" / f"part-{index:04d}" + if complete(part, index, count): + return index, 0.0 + if part.exists(): + failed = root / "failed_parts" / f"part-{index:04d}-{int(time.time())}" + failed.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(part), str(failed)) + log = root / "logs" / f"part-{index:04d}.log" + log.parent.mkdir(parents=True, exist_ok=True) + command = [ + sys.executable, str(REPO / "scripts/build_spacedreamer_canonical_mix.py"), + "--output-root", str(part), "--partition-count", str(count), + "--partition-index", str(index), "--shard-target-gib", str(shard_gib), + ] + started = time.time() + with log.open("w", encoding="utf-8") as handle: + result = subprocess.run(command, cwd=REPO, stdout=handle, stderr=subprocess.STDOUT) + if result.returncode != 0 or not complete(part, index, count): + raise RuntimeError(f"Partition {index} failed; inspect {log}") + return index, time.time() - started + + +def main() -> None: + args = parse_args() + if args.partition_count <= 0 or args.workers <= 0: + raise ValueError("partition count and workers must be positive") + args.dataset_root.mkdir(parents=True, exist_ok=True) + pending = [ + index for index in range(args.partition_count) + if not complete(args.dataset_root / "parts" / f"part-{index:04d}", index, args.partition_count) + ] + progress = args.dataset_root / "reports/build_progress.jsonl" + progress.parent.mkdir(parents=True, exist_ok=True) + print(f"[orchestrator] complete={args.partition_count-len(pending)} pending={len(pending)} workers={args.workers}", flush=True) + with ThreadPoolExecutor(max_workers=min(args.workers, max(len(pending), 1))) as executor: + futures = { + executor.submit(run_partition, args.dataset_root, index, args.partition_count, args.shard_target_gib): index + for index in pending + } + for future in as_completed(futures): + index = futures[future] + try: + _, seconds = future.result() + event = {"partition": index, "status": "complete", "seconds": seconds, "time": time.time()} + except BaseException as error: + event = {"partition": index, "status": "failed", "error": repr(error), "time": time.time()} + with progress.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(event, sort_keys=True) + "\n") + print(f"[orchestrator] {event}", flush=True) + if event["status"] == "failed": + raise RuntimeError(event) + merge = [ + sys.executable, str(REPO / "scripts/merge_spacedreamer_canonical_parts.py"), + "--dataset-root", str(args.dataset_root), + "--partition-count", str(args.partition_count), + ] + subprocess.run(merge, cwd=REPO, check=True) + print(f"[orchestrator] merge complete: {args.dataset_root}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/smoke_robodojo_geometry_cached_dataset.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/smoke_robodojo_geometry_cached_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..324b546f690ea9491ba2119ab60a3b78a39fc48c --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/smoke_robodojo_geometry_cached_dataset.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Read and collate real cached RoboDojo geometry samples without training.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + +import torch + + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +import train_robodojo_rgb_state_joint_action_4way as fourway # noqa: E402 +from robodojo_geometry_data import geometry_modalities # noqa: E402 +from train_robodojo_rgb_geometry_action_absolute_ee import ( # noqa: E402 + GeometryExhaustiveWindowDataset, + collate_geometry, + resolve_geometry_config, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + config = resolve_geometry_config(args.config) + cache_dir = Path(config["paths"]["cache_dir"]) + manifest = json.loads((cache_dir / "manifest.json").read_text(encoding="utf-8")) + normalizer = torch.load( + cache_dir / "action_normalizer.pt", map_location="cpu", weights_only=True + ) + fingerprint = str(manifest["fingerprint"]) + if normalizer.get("fingerprint") != fingerprint: + raise RuntimeError("Action normalizer/cache fingerprint mismatch") + + # ExhaustiveWindowDataset intentionally takes the active resolved config + # from the four-way adapter, exactly as the training entrypoint does. + fourway._ACTIVE_CONFIG = config + dataset = GeometryExhaustiveWindowDataset( + cache_dir, + fingerprint, + "train", + action_mean=normalizer["mean"].float(), + action_std=normalizer["std"].float(), + action_clip=None, + ) + if len(dataset) != 397700: + raise RuntimeError(f"Unexpected exhaustive dataset length: {len(dataset)}") + indices = [0, len(dataset) // 2, len(dataset) - 1] + samples = [dataset[index] for index in indices] + batch = collate_geometry(samples) + modalities = geometry_modalities(config) + expected = { + "z_rgb": (len(indices), 48, 9, 16, 16), + "z_depth": (len(indices), 48, 9, 16, 16), + "action": (len(indices), 32, 14), + "action_is_pad": (len(indices), 32), + "initial_state": (len(indices), 14), + } + if "point" in modalities: + expected["z_point"] = (len(indices), 48, 9, 16, 16) + for key, shape in expected.items(): + if tuple(batch[key].shape) != shape: + raise RuntimeError(f"Cached batch {key} shape={tuple(batch[key].shape)} != {shape}") + if torch.is_floating_point(batch[key]) and not torch.isfinite(batch[key]).all(): + raise RuntimeError(f"Cached batch {key} contains non-finite values") + if batch["z_rgb"].dtype != torch.bfloat16: + raise RuntimeError("RGB cache must remain BF16") + if batch["z_depth"].dtype != torch.bfloat16: + raise RuntimeError("Depth cache must remain BF16") + if "z_point" in batch and batch["z_point"].dtype != torch.bfloat16: + raise RuntimeError("Point cache must remain BF16") + report = { + "status": "geometry_cached_dataset_smoke_ok", + "config": str(args.config.resolve()), + "modalities": list(modalities), + "dataset_windows": len(dataset), + "sample_indices": indices, + "sample_ids": batch["sample_id"], + "batch_shapes": {key: list(shape) for key, shape in expected.items()}, + "latent_dtype": "bfloat16", + "action_normalization": normalizer["normalization_type"], + "source_fingerprint": fingerprint, + } + print(json.dumps(report, indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/validate_spacedreamer_canonical_mix.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/validate_spacedreamer_canonical_mix.py new file mode 100644 index 0000000000000000000000000000000000000000..b448bb13fc56eec4b8a9b2ca6cb9665352a66b9b --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/validate_spacedreamer_canonical_mix.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Validate canonical SpaceDreamer data and publish its balanced normalizer.""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +from pathlib import Path +import sys +from typing import Any + +import cv2 +import numpy as np +import torch + +REPO = Path(__file__).resolve().parents[1] +if str(REPO) not in sys.path: + sys.path.insert(0, str(REPO)) + +from canonical_spacedreamer_data import ( # noqa: E402 + SCHEMA_NAME, + SCHEMA_VERSION, + VALID_ACTION, + VALID_CAMERA, + _read_blob, + camera6_to_matrices, + load_canonical_manifest, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset-root", type=Path, required=True) + parser.add_argument("--max-decode-episodes", type=int, help="Omit for full validation") + parser.add_argument("--skip-member-sha256", action="store_true") + parser.add_argument("--normalizer-sample-rows-per-source", type=int, default=2_000_000) + return parser.parse_args() + + +def atomic_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def manifest_sha(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while block := handle.read(16 * 1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def decode_mp4_count(payload: bytes, expected: int, temporary_dir: Path, digest: str) -> int: + path = temporary_dir / f"{digest}.mp4" + path.write_bytes(payload) + capture = cv2.VideoCapture(str(path)) + if not capture.isOpened(): + raise RuntimeError(f"Could not decode canonical MP4: {digest}") + count = 0 + try: + while True: + ok, frame = capture.read() + if not ok: + break + if frame.shape[:2] != (256, 256): + raise ValueError(f"Non-256 canonical MP4 frame: {frame.shape}") + count += 1 + finally: + capture.release() + path.unlink(missing_ok=True) + if count != expected: + raise RuntimeError(f"Canonical MP4 length mismatch: {count} != {expected}") + return count + + +def source_stats(chunks: Any, max_rows: int) -> dict[str, Any]: + action_sum = np.zeros(14, np.float64) + action_square = np.zeros(14, np.float64) + camera_sum = np.zeros(6, np.float64) + camera_square = np.zeros(6, np.float64) + action_count = camera_count = 0 + action_samples: list[np.ndarray] = [] + camera_samples: list[np.ndarray] = [] + sample_budget = max_rows + rng = np.random.default_rng(20260724) + for action, camera, validity in chunks: + valid_action = action[(validity & VALID_ACTION) != 0].astype(np.float64) + valid_camera = camera[(validity & VALID_CAMERA) != 0].astype(np.float64) + action_sum += valid_action.sum(0) + action_square += np.square(valid_action).sum(0) + camera_sum += valid_camera.sum(0) + camera_square += np.square(valid_camera).sum(0) + action_count += len(valid_action) + camera_count += len(valid_camera) + if sample_budget > 0: + take = min(sample_budget, len(valid_action), len(valid_camera)) + if take: + ai = rng.choice(len(valid_action), take, replace=False) + ci = rng.choice(len(valid_camera), take, replace=False) + action_samples.append(valid_action[ai]) + camera_samples.append(valid_camera[ci]) + sample_budget -= take + if action_count < 2 or camera_count < 2: + raise RuntimeError("Insufficient motion rows for normalizer") + action_mean = action_sum / action_count + camera_mean = camera_sum / camera_count + action_std = np.sqrt(np.maximum(action_square / action_count - action_mean**2, 1e-12)) + camera_std = np.sqrt(np.maximum(camera_square / camera_count - camera_mean**2, 1e-12)) + sampled_action = np.concatenate(action_samples) if action_samples else np.empty((0, 14)) + sampled_camera = np.concatenate(camera_samples) if camera_samples else np.empty((0, 6)) + return { + "action_sum": action_sum, "action_square": action_square, "action_count": action_count, + "camera_sum": camera_sum, "camera_square": camera_square, "camera_count": camera_count, + "action_mean": action_mean, "action_std": action_std, + "camera_mean": camera_mean, "camera_std": camera_std, + "action_quantiles": np.quantile(sampled_action, [0.01, 0.5, 0.99], axis=0) if len(sampled_action) else None, + "camera_quantiles": np.quantile(sampled_camera, [0.01, 0.5, 0.99], axis=0) if len(sampled_camera) else None, + } + + +def serializable_stats(stats: dict[str, Any]) -> dict[str, Any]: + return { + key: value.tolist() if isinstance(value, np.ndarray) else value + for key, value in stats.items() + if key not in {"action_sum", "action_square", "camera_sum", "camera_square"} + } + + +def main() -> None: + args = parse_args() + all_path = args.dataset_root / "manifests/all.jsonl" + train_path = args.dataset_root / "manifests/train.jsonl" + validation_path = args.dataset_root / "manifests/validation.jsonl" + all_episodes = load_canonical_manifest(all_path) + train_episodes = load_canonical_manifest(train_path) if train_path.stat().st_size else [] + validation_episodes = load_canonical_manifest(validation_path) if validation_path.stat().st_size else [] + all_ids = {episode.episode_id for episode in all_episodes} + train_ids = {episode.episode_id for episode in train_episodes} + validation_ids = {episode.episode_id for episode in validation_episodes} + if train_ids & validation_ids or train_ids | validation_ids != all_ids: + raise RuntimeError("Canonical train/validation partition is not exact") + train_groups = {(episode.source, episode.recording_group_id) for episode in train_episodes} + validation_groups = {(episode.source, episode.recording_group_id) for episode in validation_episodes} + if train_groups & validation_groups: + raise RuntimeError("Canonical recording-group leakage detected") + + limit = args.max_decode_episodes + selected = all_episodes if limit is None else all_episodes[:limit] + motion_by_source: dict[str, list[tuple[np.ndarray, np.ndarray, np.ndarray]]] = {} + member_hashes = not args.skip_member_sha256 + decoded_frames = depth_frames = 0 + import tempfile + with tempfile.TemporaryDirectory(prefix="validate_canonical_") as directory: + temporary = Path(directory) + for number, episode in enumerate(selected, start=1): + depth_payload = _read_blob(episode.depth, verify_sha256=member_hashes) + motion_payload = _read_blob(episode.motion, verify_sha256=member_hashes) + rgb_payload = _read_blob(episode.rgb, verify_sha256=member_hashes) + _read_blob(episode.metadata, verify_sha256=member_hashes) + with np.load(io.BytesIO(depth_payload), allow_pickle=False) as bundle: + depth = bundle["depth_mm"] + K = bundle["K_256"] + source_index = bundle["source_frame_index"] + with np.load(io.BytesIO(motion_payload), allow_pickle=False) as bundle: + action = bundle["action_abs"] + camera = bundle["camera_abs"] + validity = bundle["validity"] + if depth.shape != (episode.length, 256, 256) or depth.dtype != np.uint16: + raise ValueError(f"Depth schema mismatch: {episode.episode_id}") + if action.shape != (episode.length, 14) or camera.shape != (episode.length, 6): + raise ValueError(f"Motion schema mismatch: {episode.episode_id}") + if validity.shape != (episode.length,) or source_index.shape != (episode.length,): + raise ValueError(f"Index/validity schema mismatch: {episode.episode_id}") + if K.shape != (3, 3) or not np.isfinite(K).all(): + raise ValueError(f"K schema mismatch: {episode.episode_id}") + if not np.isfinite(action).all() or not np.isfinite(camera).all(): + raise ValueError(f"Non-finite motion: {episode.episode_id}") + if not np.allclose(camera[0], 0, atol=1e-4): + raise ValueError(f"Frame-0 camera is non-zero: {episode.episode_id}") + matrices = camera6_to_matrices(camera[: min(64, len(camera))]) + if not np.isfinite(matrices).all() or not np.allclose(matrices[0], np.eye(4), atol=1e-4): + raise ValueError(f"Camera matrix reconstruction failed: {episode.episode_id}") + decoded_frames += decode_mp4_count(rgb_payload, episode.length, temporary, episode.rgb.sha256) + depth_frames += len(depth) + motion_by_source.setdefault(episode.source, []).append((action, camera, validity)) + print(f"[validate] {number}/{len(selected)} {episode.episode_id}", flush=True) + + # Normalization must always use the complete training split, not a decode subset. + def source_motion(source: str) -> Any: + for episode in train_episodes: + if episode.source != source: + continue + with np.load(io.BytesIO(_read_blob(episode.motion, verify_sha256=member_hashes)), allow_pickle=False) as bundle: + yield bundle["action_abs"], bundle["camera_abs"], bundle["validity"] + + per_source = { + source: source_stats(source_motion(source), args.normalizer_sample_rows_per_source) + for source in sorted({episode.source for episode in train_episodes}) + } + if len(per_source) != 2: + raise RuntimeError(f"Expected two training sources, found {sorted(per_source)}") + source_values = list(per_source.values()) + action_mean = sum(value["action_mean"] for value in source_values) / 2.0 + action_second = sum( + value["action_square"] / value["action_count"] for value in source_values + ) / 2.0 + camera_mean = sum(value["camera_mean"] for value in source_values) / 2.0 + camera_second = sum( + value["camera_square"] / value["camera_count"] for value in source_values + ) / 2.0 + action_std = np.sqrt(np.maximum(action_second - action_mean**2, 1e-12)) + camera_std = np.sqrt(np.maximum(camera_second - camera_mean**2, 1e-12)) + fingerprint = hashlib.sha256( + f"{SCHEMA_NAME}:{SCHEMA_VERSION}:{manifest_sha(train_path)}:balanced-1to1".encode() + ).hexdigest() + statistics_dir = args.dataset_root / "statistics" + statistics_dir.mkdir(exist_ok=True) + torch.save({ + "fingerprint": fingerprint, "sampling": "equal_source_moments_1_to_1", + "action_mean": torch.from_numpy(action_mean.astype(np.float32)), + "action_std": torch.from_numpy(action_std.astype(np.float32)), + "action_count": torch.tensor(sum(v["action_count"] for v in source_values)), + "camera_mean": torch.from_numpy(camera_mean.astype(np.float32)), + "camera_std": torch.from_numpy(camera_std.astype(np.float32)), + "camera_count": torch.tensor(sum(v["camera_count"] for v in source_values)), + }, statistics_dir / "action_camera_normalizer.pt") + atomic_json(statistics_dir / "per_source_statistics.json", { + "fingerprint": fingerprint, + "sources": {source: serializable_stats(value) for source, value in per_source.items()}, + }) + full_validation = limit is None or limit >= len(all_episodes) + report = { + "ready": bool(full_validation), "complete_manifest": True, + "full_payload_validation": full_validation, + "schema_name": SCHEMA_NAME, "schema_version": SCHEMA_VERSION, + "episodes": len(all_episodes), "train_episodes": len(train_episodes), + "validation_episodes": len(validation_episodes), + "decoded_episodes": len(selected), "decoded_rgb_frames": decoded_frames, + "validated_depth_frames": depth_frames, + "train_validation_episode_overlap": 0, + "train_validation_recording_group_overlap": 0, + "member_sha256_verified": member_hashes, + "normalizer_fingerprint": fingerprint, + "manifest_sha256": manifest_sha(all_path), + "train_manifest_sha256": manifest_sha(train_path), + "validation_manifest_sha256": manifest_sha(validation_path), + } + atomic_json(args.dataset_root / "reports/ready.json", report) + print(json.dumps(report, indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/validate_spacedreamer_indexed_canonical.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/validate_spacedreamer_indexed_canonical.py new file mode 100644 index 0000000000000000000000000000000000000000..27d50ec03e6788160eb28db554b0826e5e5b8db6 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/validate_spacedreamer_indexed_canonical.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +"""Full metadata/motion validation and sampled decode for indexed canonical data.""" + +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +import hashlib +import json +import os +from pathlib import Path +import sys +import tempfile +from typing import Any + +import numpy as np +import torch + +REPO = Path(__file__).resolve().parents[1] +if str(REPO) not in sys.path: + sys.path.insert(0, str(REPO)) + +from canonical_spacedreamer_data import ( # noqa: E402 + VALID_ACTION, VALID_CAMERA, _ffmpeg_decode_indexed_rgb, + _materialize_indexed_rgb, load_indexed_canonical_manifest, stable_u64, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset-root", type=Path, required=True) + parser.add_argument("--decode-samples-per-source", type=int, default=128) + parser.add_argument("--decode-workers", type=int, default=8) + return parser.parse_args() + + +def atomic_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def motion_slice( + episode: Any, + cache: dict[tuple[Path, Path, Path], tuple[np.memmap, np.memmap, np.memmap]], +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + key = (episode.action_path, episode.camera_path, episode.validity_path) + if key not in cache: + action_rows = episode.action_path.stat().st_size // (14 * 4) + camera_rows = episode.camera_path.stat().st_size // (6 * 4) + cache[key] = ( + np.memmap(episode.action_path, dtype=" tuple[str, str, int]: + with np.load(episode.depth_path, allow_pickle=False) as bundle: + depth = bundle["depth_mm"][:episode.length] + frame_index = bundle["frame_index"][:episode.length].astype(np.int64) + source_k = bundle["K"].astype(np.float32) + if ( + depth.dtype != np.uint16 + or depth.shape != (episode.length, episode.output_height, episode.output_width) + or frame_index.shape != (episode.length,) + or frame_index[0] < 0 + or np.any(np.diff(frame_index) <= 0) + or not np.allclose(source_k, episode.k_rectified, atol=1e-3) + ): + raise RuntimeError(f"Sampled depth/geometry mismatch: {episode.episode_id}") + del depth + rgb_path = _materialize_indexed_rgb(episode.rgb, cache) + ranged_rgb = not ( + episode.rgb.offset == 0 + and episode.rgb.size == episode.rgb.path.stat().st_size + ) + try: + rgb = _ffmpeg_decode_indexed_rgb( + rgb_path, frame_index, output_width=episode.output_width, + output_height=episode.output_height, + k_input=np.asarray(episode.k_input, np.float32), + distortion=np.asarray(episode.distortion, np.float64), + k_rectified=np.asarray(episode.k_rectified, np.float32), + ) + finally: + if ranged_rgb: + rgb_path.unlink(missing_ok=True) + if rgb.shape != (episode.length, 256, 256, 3): + raise RuntimeError(f"Sampled RGB decode mismatch: {episode.episode_id}") + return episode.source, episode.episode_id, len(rgb) + + +def main() -> None: + args = parse_args() + if args.decode_samples_per_source <= 0 or args.decode_workers <= 0: + raise ValueError("decode sample/worker counts must be positive") + workspace = Path("/workspace-wl/shouqx@xiaopeng.com") + source_paths = { + "Egoverse": ( + workspace / "dataset/Egoverse_3D_30fps/manifests/episodes.jsonl", + workspace / "dataset/Egoverse_3D_30fps/reports/validation.json", + ), + "AgiBotWorld": ( + workspace / "dataset/AgiBotWorld_Beta_Egoverse_duration_trainable_diverse_v2/manifests/spacedreamer_ego_five_input_motion_qc_v4.jsonl", + workspace / "dataset/AgiBotWorld_Beta_Egoverse_duration_trainable_diverse_v2/reports/spacedreamer_ego_five_input_motion_qc_v4_validation.json", + ), + } + build_report_path = args.dataset_root / "reports/index_build.json" + build_report = json.loads(build_report_path.read_text(encoding="utf-8")) + if build_report.get("complete") is not True: + raise RuntimeError(f"Indexed build report is incomplete: {build_report_path}") + source_provenance: dict[str, dict[str, str]] = {} + for source, (manifest_path, validation_report_path) in source_paths.items(): + manifest_hash = hashlib.sha256(manifest_path.read_bytes()).hexdigest() + validation_hash = hashlib.sha256(validation_report_path.read_bytes()).hexdigest() + if build_report.get("source_manifest_sha256", {}).get(source) != manifest_hash: + raise RuntimeError(f"{source} source manifest changed after indexed build") + if build_report.get("source_validation_sha256", {}).get(source) != validation_hash: + raise RuntimeError(f"{source} source validation report changed after indexed build") + source_provenance[source] = { + "manifest_sha256": manifest_hash, + "validation_report_sha256": validation_hash, + } + egoverse_validation = json.loads(source_paths["Egoverse"][1].read_text(encoding="utf-8")) + agibot_validation = json.loads(source_paths["AgiBotWorld"][1].read_text(encoding="utf-8")) + if not ( + egoverse_validation.get("valid_annotations") == 26796 + and egoverse_validation.get("missing_annotations") == 0 + and egoverse_validation.get("invalid_annotations") == 0 + ): + raise RuntimeError("Egoverse source validation report is not clean") + if not ( + agibot_validation.get("complete") is True + and agibot_validation.get("effective_geometry") == "rectified_pinhole" + and agibot_validation.get("effective_distortion_model") == "none" + and agibot_validation.get("invariants", {}).get("all_effective_distortion_empty") is True + and agibot_validation.get("output_manifest_sha256") == source_provenance["AgiBotWorld"]["manifest_sha256"] + ): + raise RuntimeError("AgiBot v4 source validation report is not clean") + all_path = args.dataset_root / "manifests/all.jsonl" + train_path = args.dataset_root / "manifests/train.jsonl" + validation_path = args.dataset_root / "manifests/validation.jsonl" + episodes = load_indexed_canonical_manifest(all_path) + train = load_indexed_canonical_manifest(train_path, verify_paths=False) + validation = load_indexed_canonical_manifest(validation_path, verify_paths=False) + train_ids, validation_ids = {e.episode_id for e in train}, {e.episode_id for e in validation} + if train_ids & validation_ids or train_ids | validation_ids != {e.episode_id for e in episodes}: + raise RuntimeError("Indexed canonical train/validation partition mismatch") + train_groups = {(e.source, e.recording_group_id) for e in train} + validation_groups = {(e.source, e.recording_group_id) for e in validation} + if train_groups & validation_groups: + raise RuntimeError("Indexed canonical recording-group leakage") + + accumulators = { + source: { + "action_sum": np.zeros(14, np.float64), "action_square": np.zeros(14, np.float64), + "camera_sum": np.zeros(6, np.float64), "camera_square": np.zeros(6, np.float64), + "action_count": 0, "camera_count": 0, + "action_weight": 0.0, "camera_weight": 0.0, + } + for source in sorted({e.source for e in train}) + } + invalid_action = invalid_camera = nonfinite = bad_frame0 = bad_ftyp = bad_grip = 0 + rgb_ranges: dict[Path, list[Any]] = {} + for episode in episodes: + rgb_ranges.setdefault(episode.rgb.path, []).append(episode.rgb) + def inspect_rgb_file(item: tuple[Path, list[Any]]) -> tuple[int, int]: + rgb_path, ranges = item + local_bad = 0 + with rgb_path.open("rb", buffering=0) as handle: + for blob in ranges: + handle.seek(blob.offset) + header = handle.read(16) + if len(header) < 8 or header[4:8] != b"ftyp": + local_bad += 1 + return len(ranges), local_bad + + checked_ranges = 0 + with ThreadPoolExecutor(max_workers=min(32, len(rgb_ranges))) as executor: + futures = [executor.submit(inspect_rgb_file, item) for item in rgb_ranges.items()] + for path_number, future in enumerate(as_completed(futures), start=1): + ranges_checked, local_bad = future.result() + checked_ranges += ranges_checked + bad_ftyp += local_bad + if path_number == 1 or path_number % 2000 == 0 or path_number == len(rgb_ranges): + print( + f"[indexed-validate] rgb_ranges={checked_ranges}/{len(episodes)} " + f"files={path_number}/{len(rgb_ranges)}", + flush=True, + ) + train_set = {e.episode_id for e in train} + motion_maps: dict[ + tuple[Path, Path, Path], tuple[np.memmap, np.memmap, np.memmap] + ] = {} + for number, episode in enumerate(episodes, start=1): + action, camera, validity = motion_slice(episode, motion_maps) + if len(action) != episode.length or len(camera) != episode.length or len(validity) != episode.length: + raise RuntimeError(f"Motion range mismatch: {episode.episode_id}") + invalid_action += int(((validity & VALID_ACTION) == 0).sum()) + invalid_camera += int(((validity & VALID_CAMERA) == 0).sum()) + nonfinite += int((~np.isfinite(action)).sum() + (~np.isfinite(camera)).sum()) + bad_frame0 += int(not np.allclose(camera[0], 0, atol=1e-4)) + grips = action[:, (6, 13)] + bad_grip += int(((grips < -1e-6) | (grips > 1.0 + 1e-6)).sum()) + if episode.episode_id in train_set: + acc = accumulators[episode.source] + # Frame 0 is conditioning state/identity camera and is never a + # prediction target. Fit only the possible target rows 1..T-1. + target_validity = validity[1:] + action_valid = action[1:][(target_validity & VALID_ACTION) != 0].astype(np.float64) + camera_valid = camera[1:][(target_validity & VALID_CAMERA) != 0].astype(np.float64) + episode_weight = 1.0 / float(episode.length - 1) + acc["action_sum"] += action_valid.sum(0) * episode_weight + acc["action_square"] += np.square(action_valid).sum(0) * episode_weight + acc["camera_sum"] += camera_valid.sum(0) * episode_weight + acc["camera_square"] += np.square(camera_valid).sum(0) * episode_weight + acc["action_count"] += len(action_valid); acc["camera_count"] += len(camera_valid) + acc["action_weight"] += len(action_valid) * episode_weight + acc["camera_weight"] += len(camera_valid) * episode_weight + if number == 1 or number % 2000 == 0 or number == len(episodes): + print(f"[indexed-validate] metadata_motion={number}/{len(episodes)}", flush=True) + if invalid_action or invalid_camera or nonfinite or bad_frame0 or bad_ftyp or bad_grip: + raise RuntimeError( + f"Indexed validation failed: invalid_action={invalid_action} invalid_camera={invalid_camera} " + f"nonfinite={nonfinite} bad_frame0={bad_frame0} bad_ftyp={bad_ftyp} " + f"bad_grip={bad_grip}" + ) + if len(accumulators) != 2: + raise RuntimeError(f"Expected two sources, found {sorted(accumulators)}") + source_stats = {} + for source, acc in accumulators.items(): + action_mean = acc["action_sum"] / acc["action_weight"] + camera_mean = acc["camera_sum"] / acc["camera_weight"] + action_second = acc["action_square"] / acc["action_weight"] + camera_second = acc["camera_square"] / acc["camera_weight"] + source_stats[source] = { + **acc, "action_mean": action_mean, "camera_mean": camera_mean, + "action_second": action_second, "camera_second": camera_second, + "action_std": np.sqrt(np.maximum(action_second - action_mean**2, 1e-12)), + "camera_std": np.sqrt(np.maximum(camera_second - camera_mean**2, 1e-12)), + } + values = list(source_stats.values()) + action_mean = sum(v["action_mean"] for v in values) / 2 + camera_mean = sum(v["camera_mean"] for v in values) / 2 + action_second = sum(v["action_second"] for v in values) / 2 + camera_second = sum(v["camera_second"] for v in values) / 2 + action_std = np.sqrt(np.maximum(action_second - action_mean**2, 1e-12)) + camera_std = np.sqrt(np.maximum(camera_second - camera_mean**2, 1e-12)) + + decoded = {} + with tempfile.TemporaryDirectory(prefix="indexed_canonical_decode_") as directory: + cache = Path(directory) + selected_by_source = {} + for source in sorted({e.source for e in episodes}): + selected_by_source[source] = sorted( + (e for e in episodes if e.source == source), + key=lambda e: (stable_u64(e.episode_id, 20260822), e.episode_id), + )[:args.decode_samples_per_source] + decoded[source] = {"episodes": 0, "frames": 0} + selected = [episode for source in sorted(selected_by_source) for episode in selected_by_source[source]] + completed_by_source = {source: 0 for source in selected_by_source} + with ThreadPoolExecutor(max_workers=min(args.decode_workers, max(len(selected), 1))) as executor: + futures = [executor.submit(validate_decode, episode, cache) for episode in selected] + for future in as_completed(futures): + source, episode_id, frames = future.result() + decoded[source]["episodes"] += 1 + decoded[source]["frames"] += frames + completed_by_source[source] += 1 + number = completed_by_source[source] + total = len(selected_by_source[source]) + if number == 1 or number % 8 == 0 or number == total: + print( + f"[indexed-validate] decode source={source} {number}/{total} " + f"last={episode_id}", + flush=True, + ) + + fingerprint = hashlib.sha256( + ( + "spacedreamer-canonical-indexed:1:" + f"{hashlib.sha256(train_path.read_bytes()).hexdigest()}:" + "equal-source-equal-episode-target-frames-v2" + ).encode() + ).hexdigest() + statistics = args.dataset_root / "statistics"; statistics.mkdir(exist_ok=True) + torch.save({ + "fingerprint": fingerprint, + "sampling": "equal_source_equal_episode_target_frames_1_to_T_minus_1", + "action_mean": torch.from_numpy(action_mean.astype(np.float32)), + "action_std": torch.from_numpy(action_std.astype(np.float32)), + "action_count": torch.tensor(sum(v["action_count"] for v in values)), + "camera_mean": torch.from_numpy(camera_mean.astype(np.float32)), + "camera_std": torch.from_numpy(camera_std.astype(np.float32)), + "camera_count": torch.tensor(sum(v["camera_count"] for v in values)), + }, statistics / "action_camera_normalizer.pt") + atomic_json(statistics / "per_source_statistics.json", { + "fingerprint": fingerprint, + "sources": { + source: { + key: value.tolist() if isinstance(value, np.ndarray) else value + for key, value in stats.items() + if key not in {"action_sum", "action_square", "camera_sum", "camera_square", "action_second", "camera_second"} + } + for source, stats in source_stats.items() + }, + }) + report = { + "ready": True, "full_payload_validation": True, + "full_metadata_motion_validation": True, + "episodes": len(episodes), "train_episodes": len(train), + "validation_episodes": len(validation), "invalid_action_frames": 0, + "invalid_camera_frames": 0, "bad_rgb_ftyp": 0, "grip_outside_unit_interval": 0, + "sampled_software_decode": decoded, + "train_validation_episode_overlap": 0, + "train_validation_recording_group_overlap": 0, + "normalizer_fingerprint": fingerprint, + "manifest_sha256": hashlib.sha256(all_path.read_bytes()).hexdigest(), + "train_manifest_sha256": hashlib.sha256(train_path.read_bytes()).hexdigest(), + "validation_manifest_sha256": hashlib.sha256(validation_path.read_bytes()).hexdigest(), + "source_provenance": source_provenance, + } + atomic_json(args.dataset_root / "reports/ready.json", report) + print(json.dumps(report, indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/verify_robodojo_geometry_experiment.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/verify_robodojo_geometry_experiment.py new file mode 100644 index 0000000000000000000000000000000000000000..58996e2dcdaf837774a592f82185ef68594338e7 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/verify_robodojo_geometry_experiment.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Fail-closed config/data audit for controlled RoboDojo geometry experiments.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import sys +from typing import Any + +import torch + + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +import train_robodojo_rgb_state_joint_action_4way as fourway # noqa: E402 +from robodojo_geometry_data import geometry_modalities # noqa: E402 +from train_robodojo_rgb_geometry_action_absolute_ee import ( # noqa: E402 + resolve_geometry_config, +) + + +BASELINE = ROOT / ( + "ablation_test/" + "robodojo_mini_absolute_xyz_so3_pi05_q01q99_full_bidir_nostopgrad_" + "fresh_joint_gbs256_8gpu_20epoch/config.yaml" +) +ALLOWED_EXACT_DIFFS = { + "experiment.name", + "experiment.output_dir", + "paths.depth_dataset_root", + "paths.depth_latent_cache_dir", + "paths.point_latent_cache_dir", + "diffusion.lambda_depth", + "diffusion.lambda_point", + "tracking.group", + "tracking.run_name", + "tracking.tags", +} +ALLOWED_PREFIX_DIFFS = ("geometry.",) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--baseline-config", type=Path, default=BASELINE) + parser.add_argument("--require-empty-output", action="store_true") + return parser.parse_args() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while block := handle.read(8 << 20): + digest.update(block) + return digest.hexdigest() + + +def flatten(value: Any, prefix: str = "") -> dict[str, Any]: + if not isinstance(value, dict): + return {prefix: value} + result: dict[str, Any] = {} + for key, child in value.items(): + path = f"{prefix}.{key}" if prefix else str(key) + result.update(flatten(child, path)) + return result + + +def controlled_config_differences( + baseline: dict[str, Any], experiment: dict[str, Any] +) -> dict[str, dict[str, Any]]: + left, right = flatten(baseline), flatten(experiment) + differences: dict[str, dict[str, Any]] = {} + for key in sorted(set(left) | set(right)): + if left.get(key) != right.get(key): + differences[key] = { + "baseline": left.get(key, ""), + "experiment": right.get(key, ""), + } + unexpected = { + key: value + for key, value in differences.items() + if key not in ALLOWED_EXACT_DIFFS + and not any(key.startswith(prefix) for prefix in ALLOWED_PREFIX_DIFFS) + } + if unexpected: + raise RuntimeError( + "Geometry experiment drifted from the GBS256 baseline:\n" + + json.dumps(unexpected, indent=2, sort_keys=True) + ) + return differences + + +def load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def main() -> None: + args = parse_args() + baseline_raw = fourway.load_config(args.baseline_config) + experiment_raw = fourway.load_config(args.config) + differences = controlled_config_differences(baseline_raw, experiment_raw) + config = resolve_geometry_config(args.config) + modalities = geometry_modalities(config) + + required_training = { + "max_steps": 31080, + "max_steps_cap": 31080, + "exact_epochs": 20, + "valid_windows_per_epoch": 397700, + "global_batch_size": 256, + "gradient_accumulation_steps": 4, + "fp32_master_weights": False, + "warmup_steps": 1000, + "save_every": 10000, + "train_video_expert": True, + } + actual_training = { + key: config["training"].get(key) for key in required_training + } + if actual_training != required_training: + raise RuntimeError( + f"Training contract mismatch: {actual_training} != {required_training}" + ) + expected_data = { + "num_frames": 33, + "resolution": 256, + "fps": 25.0, + "batch_size": 8, + "num_workers": 0, + } + actual_data = {key: config["data"].get(key) for key in expected_data} + if actual_data != expected_data: + raise RuntimeError(f"Data contract mismatch: {actual_data} != {expected_data}") + if config["action"]["normalization"].get("type") != ( + "pi05_q01_q99_all_action_v1" + ): + raise RuntimeError("The geometry experiment lost pi0.5 q01/q99 normalization") + + cache_dir = Path(config["paths"]["cache_dir"]) + action_manifest_path = cache_dir / "manifest.json" + normalizer_path = cache_dir / "action_normalizer.pt" + if not action_manifest_path.is_file() or not normalizer_path.is_file(): + raise FileNotFoundError(f"Absolute-EE cache is incomplete: {cache_dir}") + action_manifest = load_json(action_manifest_path) + if ( + int(action_manifest.get("valid_window_count", -1)) != 397700 + or len(action_manifest.get("episodes", [])) != 1000 + ): + raise RuntimeError("Absolute-EE cache does not contain the controlled split") + normalizer = torch.load(normalizer_path, map_location="cpu", weights_only=True) + if ( + normalizer.get("fingerprint") != action_manifest.get("fingerprint") + or normalizer.get("normalization_type") + != "pi05_q01_q99_all_action_v1" + or int(normalizer["count"].item()) != 397700 * 32 + or not bool(normalizer["fitted"].item()) + ): + raise RuntimeError("Absolute-EE q01/q99 normalizer is incompatible") + + rgb_cache = Path(config["paths"]["rgb_latent_cache_dir"]) + rgb_manifest_path = rgb_cache / "manifest.json" + if not rgb_manifest_path.is_file(): + raise FileNotFoundError(rgb_manifest_path) + rgb_manifest = load_json(rgb_manifest_path) + compatible_rgb = set(config["paths"]["rgb_latent_compatible_source_fingerprints"]) + if ( + rgb_manifest.get("source_fingerprint") not in compatible_rgb + or int(rgb_manifest.get("episode_count", -1)) != 1000 + or int(rgb_manifest.get("valid_window_count", -1)) != 397700 + or rgb_manifest.get("latent_shape") != [48, 9, 16, 16] + or rgb_manifest.get("dtype") != "bfloat16" + ): + raise RuntimeError("RGB latent cache is incompatible with the experiment") + + output_dir = Path(config["experiment"]["output_dir"]) + if args.require_empty_output and output_dir.exists() and any(output_dir.iterdir()): + raise RuntimeError(f"Formal output directory is not empty: {output_dir}") + + report = { + "complete": True, + "config": str(args.config.resolve()), + "config_sha256": sha256_file(args.config), + "baseline_config": str(args.baseline_config.resolve()), + "baseline_config_sha256": sha256_file(args.baseline_config), + "modalities": list(modalities), + "controlled_config_differences": differences, + "episodes": 1000, + "valid_windows": 397700, + "frames": 33, + "resolution": 256, + "fps": 25.0, + "global_batch_size": 256, + "optimizer_steps": 31080, + "exact_epochs": 20, + "fresh_joint_checkpoint_loaded": False, + "wan_base_initialization": config["paths"]["pretrained_model_dir"], + "action_initialization": "random", + "action_normalization": "pi05_q01_q99_all_action_v1", + "cross_modal_attention": "bidirectional", + "action_loss_to_video_stop_gradient": False, + "action_cache_fingerprint": action_manifest["fingerprint"], + "rgb_cache_fingerprint": rgb_manifest["source_fingerprint"], + "output_dir": str(output_dir), + } + print(json.dumps(report, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/visualize_depth_preprocessing.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/visualize_depth_preprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..a468e1d918b0d6260933c962cd57dcb8bbc36aee --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/scripts/visualize_depth_preprocessing.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Create aligned RGB, color-depth, and grayscale-depth GIF triptychs.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image, ImageDraw + +from depth_preprocessing import DepthPreprocessingConfig, preprocess_depth_grayscale + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--annotations", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--samples", type=int, default=20) + parser.add_argument("--frames-per-sample", type=int, default=24) + parser.add_argument("--panel-width", type=int, default=320) + parser.add_argument("--panel-height", type=int, default=180) + parser.add_argument("--depth-min", type=float, default=0.0) + parser.add_argument("--depth-max", type=float, default=5.0) + return parser.parse_args() + + +def selected_positions(frame_count: int, count: int) -> np.ndarray: + if frame_count < 1 or count < 1: + raise ValueError("frame_count and count must be positive") + return np.unique(np.linspace(0, frame_count - 1, min(count, frame_count), dtype=np.int64)) + + +def read_selected_rgb( + video_path: Path, + source_indices: np.ndarray, + output_size: tuple[int, int], + metadata: dict, + rectified_K: np.ndarray, +) -> list[np.ndarray]: + capture = cv2.VideoCapture(str(video_path)) + if not capture.isOpened(): + raise RuntimeError(f"Could not open RGB video: {video_path}") + + episode = metadata["episode"] + distortion = np.asarray(episode.get("distortion") or [], dtype=np.float64) + map_x = map_y = None + if metadata.get("image_geometry") == "rectified_pinhole" and np.any(np.abs(distortion) > 0): + map_x, map_y = cv2.initUndistortRectifyMap( + np.asarray(episode["K_output"], dtype=np.float64), + distortion, + None, + np.asarray(rectified_K, dtype=np.float64), + output_size, + cv2.CV_32FC1, + ) + + frames: list[np.ndarray] = [] + try: + for frame_index in source_indices: + capture.set(cv2.CAP_PROP_POS_FRAMES, int(frame_index)) + ok, bgr = capture.read() + if not ok: + raise RuntimeError(f"Could not decode source frame {frame_index} from {video_path}") + if (bgr.shape[1], bgr.shape[0]) != output_size: + bgr = cv2.resize(bgr, output_size, interpolation=cv2.INTER_AREA) + if map_x is not None and map_y is not None: + bgr = cv2.remap( + bgr, + map_x, + map_y, + interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + ) + frames.append(cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)) + finally: + capture.release() + return frames + + +def render_triptych( + rgb: np.ndarray, + grayscale: np.ndarray, + *, + panel_size: tuple[int, int], + episode_name: str, + source_frame: int, +) -> Image.Image: + gray_u8 = np.round(grayscale * 255.0).astype(np.uint8) + color_bgr = cv2.applyColorMap(255 - gray_u8, cv2.COLORMAP_TURBO) + color_rgb = cv2.cvtColor(color_bgr, cv2.COLOR_BGR2RGB) + gray_rgb = np.repeat(gray_u8[..., None], 3, axis=2) + + panels = [] + for array in (rgb, color_rgb, gray_rgb): + panels.append(Image.fromarray(array).resize(panel_size, Image.Resampling.BILINEAR)) + + title_height = 30 + canvas = Image.new("RGB", (panel_size[0] * 3, panel_size[1] + title_height), "white") + labels = ("RGB", "Depth color (0-5 m)", "Depth grayscale (Wan input)") + draw = ImageDraw.Draw(canvas) + for index, (panel, label) in enumerate(zip(panels, labels, strict=True)): + x = index * panel_size[0] + canvas.paste(panel, (x, title_height)) + draw.text((x + 8, 8), label, fill="black") + if index: + draw.line((x, 0, x, canvas.height), fill=(220, 220, 220), width=2) + footer = f"{episode_name} | source frame {source_frame}" + draw.text((canvas.width - 8, 8), footer, fill=(70, 70, 70), anchor="ra") + return canvas + + +def visualize_one( + annotation_path: Path, + output_path: Path, + *, + frames_per_sample: int, + panel_size: tuple[int, int], + config: DepthPreprocessingConfig, +) -> int: + with np.load(annotation_path, allow_pickle=False) as bundle: + depth_mm = bundle["depth_mm"] + frame_index = bundle["frame_index"] + rectified_K = bundle["K"] + metadata = json.loads(bundle["metadata_json"].item()) + + positions = selected_positions(len(depth_mm), frames_per_sample) + depth_selected = depth_mm[positions] + source_indices = frame_index[positions] + processed = preprocess_depth_grayscale( + depth_selected, + config=config, + input_unit="millimeter", + ) + + output_size = (int(depth_mm.shape[2]), int(depth_mm.shape[1])) + rgb_frames = read_selected_rgb( + Path(metadata["episode"]["video_path"]), + source_indices, + output_size, + metadata, + rectified_K, + ) + frames = [ + render_triptych( + rgb, + processed.grayscale[index, 0], + panel_size=panel_size, + episode_name=annotation_path.stem, + source_frame=int(source_indices[index]), + ) + for index, rgb in enumerate(rgb_frames) + ] + output_path.parent.mkdir(parents=True, exist_ok=True) + duration_ms = round(1000.0 / min(float(metadata.get("target_fps", 10.0)), 8.0)) + frames[0].save( + output_path, + save_all=True, + append_images=frames[1:], + duration=duration_ms, + loop=0, + optimize=False, + ) + return len(frames) + + +def main() -> None: + args = parse_args() + annotation_paths = sorted(args.annotations.glob("*.npz"))[: args.samples] + if len(annotation_paths) != args.samples: + raise RuntimeError(f"Expected {args.samples} NPZ files, found {len(annotation_paths)}") + config = DepthPreprocessingConfig( + depth_min=args.depth_min, + depth_max=args.depth_max, + depth_invalid_value=0.0, + depth_invalid_fill="far", + ) + for index, annotation_path in enumerate(annotation_paths, start=1): + output_path = args.output_dir / f"{annotation_path.stem}_rgb_depth_gray.gif" + frame_count = visualize_one( + annotation_path, + output_path, + frames_per_sample=args.frames_per_sample, + panel_size=(args.panel_width, args.panel_height), + config=config, + ) + print(f"[{index:02d}/{len(annotation_paths):02d}] {output_path} ({frame_count} frames)", flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/spacedreamer_ego_dataset.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/spacedreamer_ego_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..dbd9a7da9d13d89e088566f49d3a6c43ca5f3c65 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/spacedreamer_ego_dataset.py @@ -0,0 +1,960 @@ +"""Egoverse-30fps data adapter for the isolated SpaceDreamer-ego run. + +The model architecture is intentionally not defined here. This module only +implements episode splitting, coverage-window scheduling, aligned RGB/depth/ +point targets, and absolute bimanual EE/camera targets in each episode's +frame-0 camera coordinate system. +""" + +from __future__ import annotations + +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass +import hashlib +import json +import math +from pathlib import Path +from typing import Any, Iterator + +import cv2 +import numpy as np +import torch +from torch.utils.data import Dataset + +from depth_preprocessing import DepthPreprocessingConfig, preprocess_depth_grayscale +from online_egoverse_action import ( + ACTION_DIM, + CAMERA_DIM, + _rigid_inverse, + _safe_pose_matrices, + so3_log, +) +from world_point_cloud import ( + _read_zarr_v3_sharded_numeric_rows, + depth_video_to_world_grid, + normalize_world_point_video, + scale_intrinsics_after_center_crop, +) + + +WINDOW_SIZE = 33 +ACTION_HORIZON = WINDOW_SIZE - 1 + + +def configure_spacedreamer_worker( + _worker_id: int, + *, + opencv_threads: int, + torch_threads: int, +) -> None: + """Bound per-worker CPU pools to avoid 8-rank node oversubscription.""" + + cv2.setNumThreads(int(opencv_threads)) + torch.set_num_threads(int(torch_threads)) + + +@dataclass(frozen=True) +class EgoverseEpisode: + episode_id: str + episode_index: int + chunk_index: int + length: int + annotation_path: Path + video_path: Path + source_metadata_path: Path + output_width: int + output_height: int + k_output: tuple[tuple[float, ...], ...] + distortion: tuple[float, ...] + prompt: str + + +@dataclass(frozen=True) +class WindowRecord: + episode_slot: int + start: int + sample_weight: float + is_padding: bool = False + + +@dataclass +class MotionPayload: + frame_index: np.ndarray + head_matrix: np.ndarray + head_valid: np.ndarray + left_matrix: np.ndarray + left_valid: np.ndarray + right_matrix: np.ndarray + right_valid: np.ndarray + left_grip: np.ndarray + left_grip_valid: np.ndarray + right_grip: np.ndarray + right_grip_valid: np.ndarray + + +@dataclass +class EpisodePayload(MotionPayload): + rgb_uint8: np.ndarray + depth_mm: np.ndarray + intrinsics: np.ndarray + + +def _stable_u64(text: str, seed: int) -> int: + digest = hashlib.sha256(f"{seed}:{text}".encode("utf-8")).digest() + return int.from_bytes(digest[:8], byteorder="big", signed=False) + + +def load_eligible_episodes( + manifest_path: Path, + annotation_root: Path, + *, + metadata_workers: int = 32, +) -> list[EgoverseEpisode]: + """Load eligible rows with ordered parallel metadata/path validation.""" + + if metadata_workers <= 0: + raise ValueError("metadata_workers must be positive") + selected: list[tuple[int, dict[str, Any]]] = [] + with manifest_path.open("r", encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + row = json.loads(line) + if row.get("dataset") == "Egoverse" and row.get("metric_depth_eligible") is True: + selected.append((line_number, row)) + if not selected: + raise RuntimeError(f"No eligible Egoverse episodes in {manifest_path}") + + def build(item: tuple[int, dict[str, Any]]) -> EgoverseEpisode: + line_number, row = item + length = int(row["raw_frames"]) + if length < WINDOW_SIZE: + raise ValueError( + f"Eligible episode {row['episode_id']} has only {length} frames" + ) + source_metadata_path = Path(row["source_metadata_path"]) + source_metadata = json.loads(source_metadata_path.read_text(encoding="utf-8")) + prompt = str(source_metadata.get("attributes", {}).get("task_description", "")).strip() + if not prompt: + raise ValueError( + f"Missing task_description at manifest line {line_number}: {source_metadata_path}" + ) + episode_index = int(row["episode_index"]) + chunk_index = int(row["chunk_index"]) + annotation_path = ( + annotation_root / "episodes" / "Egoverse" / f"chunk-{chunk_index:03d}" + / f"episode_{episode_index:06d}.npz" + ) + video_path = Path(row["video_path"]) + if not annotation_path.is_file(): + raise FileNotFoundError(annotation_path) + if not video_path.is_file(): + raise FileNotFoundError(video_path) + return EgoverseEpisode( + episode_id=str(row["episode_id"]), episode_index=episode_index, + chunk_index=chunk_index, length=length, annotation_path=annotation_path, + video_path=video_path, source_metadata_path=source_metadata_path, + output_width=int(row["output_width"]), output_height=int(row["output_height"]), + k_output=tuple(tuple(float(value) for value in values) for values in row["K_output"]), + distortion=tuple(float(value) for value in row.get("distortion", [])), + prompt=prompt, + ) + + episodes: list[EgoverseEpisode] = [] + with ThreadPoolExecutor(max_workers=min(metadata_workers, len(selected))) as executor: + for number, episode in enumerate(executor.map(build, selected), start=1): + episodes.append(episode) + if number == 1 or number % 2000 == 0 or number == len(selected): + print(f"[manifest] validated={number}/{len(selected)}", flush=True) + ids = [episode.episode_id for episode in episodes] + if len(ids) != len(set(ids)): + raise RuntimeError("Eligible manifest contains duplicate episode IDs") + return episodes + + +def split_episodes( + episodes: list[EgoverseEpisode], + *, + validation_fraction: float, + seed: int, +) -> dict[str, list[EgoverseEpisode]]: + """Make an exact deterministic episode-level train/validation split.""" + + if not 0.0 < validation_fraction < 0.5: + raise ValueError("validation_fraction must be in (0, 0.5)") + ordered = sorted( + episodes, + key=lambda episode: (_stable_u64(episode.episode_id, seed), episode.episode_id), + ) + validation_count = max(1, int(round(len(ordered) * validation_fraction))) + validation = ordered[:validation_count] + train = ordered[validation_count:] + if not train: + raise RuntimeError("Episode split produced an empty training set") + train_ids = {episode.episode_id for episode in train} + validation_ids = {episode.episode_id for episode in validation} + if train_ids & validation_ids: + raise RuntimeError("Train/validation episode leakage detected") + return {"train": train, "validation": validation} + + +def coverage_starts(length: int, *, seed: int) -> list[int]: + """Return randomized 33-frame windows whose union covers every frame.""" + + if length < WINDOW_SIZE: + raise ValueError(f"Episode length {length} is shorter than {WINDOW_SIZE}") + if length == WINDOW_SIZE: + return [0] + count = int(math.ceil(length / WINDOW_SIZE)) + gap_count = count - 1 + distance = length - WINDOW_SIZE + # Start with one frame per gap, then distribute the remaining distance while + # respecting the maximum gap of WINDOW_SIZE. A deterministic shuffle makes + # the overlap pattern vary by epoch without losing endpoint coverage. + gaps = np.ones(gap_count, dtype=np.int64) + remaining = distance - gap_count + rng = np.random.default_rng(seed) + order = rng.permutation(gap_count) + while remaining > 0: + progressed = False + for slot in order: + capacity = WINDOW_SIZE - int(gaps[slot]) + if capacity <= 0: + continue + amount = min(capacity, remaining) + if amount > 1: + amount = int(rng.integers(1, amount + 1)) + gaps[slot] += amount + remaining -= amount + progressed = True + if remaining == 0: + break + if not progressed: + raise RuntimeError("Could not construct bounded coverage gaps") + order = rng.permutation(gap_count) + starts = [0, *np.cumsum(gaps).astype(np.int64).tolist()] + if starts[-1] != length - WINDOW_SIZE: + raise RuntimeError("Coverage schedule does not end at the final valid start") + if any(right - left > WINDOW_SIZE for left, right in zip(starts, starts[1:])): + raise RuntimeError("Coverage schedule contains an uncovered temporal gap") + covered = np.zeros(length, dtype=np.bool_) + for start in starts: + covered[start : start + WINDOW_SIZE] = True + if not bool(covered.all()): + raise RuntimeError("Coverage schedule failed to use every episode frame") + return starts + + +def fixed_validation_start(length: int, episode_id: str, seed: int) -> int: + valid_starts = length - WINDOW_SIZE + 1 + return _stable_u64(episode_id, seed) % valid_starts + + +def build_rank_schedules( + episodes: list[EgoverseEpisode], + *, + world_size: int, + epoch: int, + seed: int, + pad_to_multiple: int = 1, +) -> list[list[WindowRecord]]: + """Assign whole episode groups to ranks and pad ranks to equal lengths.""" + + if world_size <= 0: + raise ValueError("world_size must be positive") + if pad_to_multiple <= 0: + raise ValueError("pad_to_multiple must be positive") + episode_windows: list[tuple[int, list[int]]] = [] + for slot, episode in enumerate(episodes): + starts = coverage_starts( + episode.length, + seed=_stable_u64(episode.episode_id, seed + epoch), + ) + episode_windows.append((slot, starts)) + + # Longest-processing-time bin packing balances synchronous ranks while + # retaining every episode on exactly one rank (important for NPZ/RGB cache). + tie_rng = np.random.default_rng(seed + epoch) + tie_break = tie_rng.random(len(episode_windows)) + order = sorted( + range(len(episode_windows)), + key=lambda index: (-len(episode_windows[index][1]), tie_break[index]), + ) + bins: list[list[tuple[int, list[int]]]] = [[] for _ in range(world_size)] + loads = [0] * world_size + for index in order: + target = min(range(world_size), key=lambda rank: (loads[rank], rank)) + bins[target].append(episode_windows[index]) + loads[target] += len(episode_windows[index][1]) + + schedules: list[list[WindowRecord]] = [] + all_weights: list[float] = [] + for rank, groups in enumerate(bins): + rank_rng = np.random.default_rng(seed + epoch * 1_000_003 + rank) + rank_rng.shuffle(groups) + schedule: list[WindowRecord] = [] + for episode_slot, starts in groups: + episode = episodes[episode_slot] + starts = list(starts) + rank_rng.shuffle(starts) + weight = 1.0 / float(episode.length) + all_weights.extend([weight] * len(starts)) + schedule.extend( + WindowRecord(episode_slot, int(start), weight) for start in starts + ) + schedules.append(schedule) + + if not all_weights: + raise RuntimeError("Coverage schedule is empty") + maximum = max(len(schedule) for schedule in schedules) + maximum = int(math.ceil(maximum / pad_to_multiple) * pad_to_multiple) + for schedule in schedules: + if not schedule: + raise RuntimeError("A distributed rank received no episodes") + while len(schedule) < maximum: + source = schedule[len(schedule) % len(schedule)] + schedule.append( + WindowRecord( + source.episode_slot, + source.start, + 0.0, + is_padding=True, + ) + ) + # Include synchronization padding in the denominator. This preserves unit + # mean weight over all rank/step slots while padding retains zero gradient. + mean_weight = float( + np.sum(np.asarray(all_weights, dtype=np.float64)) + / float(world_size * maximum) + ) + schedules = [ + [ + WindowRecord( + record.episode_slot, + record.start, + record.sample_weight / mean_weight, + record.is_padding, + ) + for record in schedule + ] + for schedule in schedules + ] + return schedules + + +def build_validation_schedules( + episodes: list[EgoverseEpisode], + *, + world_size: int, + seed: int, +) -> list[list[WindowRecord]]: + schedules: list[list[WindowRecord]] = [[] for _ in range(world_size)] + for index, episode in enumerate(episodes): + schedules[index % world_size].append( + WindowRecord( + episode_slot=index, + start=fixed_validation_start(episode.length, episode.episode_id, seed), + sample_weight=1.0, + ) + ) + maximum = max(len(schedule) for schedule in schedules) + for schedule in schedules: + if not schedule: + raise RuntimeError("Validation split is smaller than world_size") + while len(schedule) < maximum: + source = schedule[len(schedule) % len(schedule)] + schedule.append( + WindowRecord(source.episode_slot, source.start, 0.0, True) + ) + return schedules + + +def _grip_from_keypoints(keypoints: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + points = np.asarray(keypoints, dtype=np.float64).reshape(-1, 21, 3) + required = points[:, [4, 8, 5, 17]] + valid = np.isfinite(required).all(axis=(1, 2)) + pinch = np.linalg.norm(points[:, 4] - points[:, 8], axis=1) + palm = np.linalg.norm(points[:, 5] - points[:, 17], axis=1) + valid &= np.isfinite(pinch) & np.isfinite(palm) & (palm >= 1.0e-4) + grip = pinch / np.maximum(palm, 1.0e-4) + grip[~valid] = 0.0 + return grip, valid + + +def absolute_targets_in_episode_first_camera( + payload: MotionPayload, + start: int, +) -> dict[str, torch.Tensor]: + stop = start + WINDOW_SIZE + if not 0 <= start <= len(payload.frame_index) - WINDOW_SIZE: + raise IndexError(f"Invalid window start {start}") + indices = np.arange(start, stop, dtype=np.int64) + if not payload.head_valid[0] or not payload.head_valid[indices].all(): + raise ValueError("Episode frame 0 or a window frame has an invalid head pose") + initial_valid = bool( + payload.left_valid[start] + and payload.right_valid[start] + and payload.left_grip_valid[start] + and payload.right_grip_valid[start] + ) + if not initial_valid: + raise ValueError("A window has an invalid clean initial EE state") + + reference_from_world = _rigid_inverse(payload.head_matrix[0]) + left_reference = reference_from_world[None] @ payload.left_matrix[indices] + right_reference = reference_from_world[None] @ payload.right_matrix[indices] + camera_reference = reference_from_world[None] @ payload.head_matrix[indices] + action = np.zeros((ACTION_HORIZON, ACTION_DIM), dtype=np.float32) + camera = np.zeros((ACTION_HORIZON, CAMERA_DIM), dtype=np.float32) + action_valid = ( + payload.left_valid[indices[1:]] + & payload.right_valid[indices[1:]] + & payload.left_grip_valid[indices[1:]] + & payload.right_grip_valid[indices[1:]] + ) + for row in range(ACTION_HORIZON): + target = row + 1 + if action_valid[row]: + action[row, 0:3] = left_reference[target, :3, 3] + action[row, 3:6] = so3_log(left_reference[target, :3, :3]) + action[row, 6] = payload.left_grip[indices[target]] + action[row, 7:10] = right_reference[target, :3, 3] + action[row, 10:13] = so3_log(right_reference[target, :3, :3]) + action[row, 13] = payload.right_grip[indices[target]] + camera[row, 0:3] = camera_reference[target, :3, 3] + camera[row, 3:6] = so3_log(camera_reference[target, :3, :3]) + + initial_state = np.concatenate( + [ + left_reference[0, :3, 3], + so3_log(left_reference[0, :3, :3]), + [payload.left_grip[start]], + right_reference[0, :3, 3], + so3_log(right_reference[0, :3, :3]), + [payload.right_grip[start]], + ] + ).astype(np.float32) + if not np.isfinite(action[action_valid]).all() or not np.isfinite(camera).all(): + raise ValueError("Absolute action/camera targets contain non-finite values") + if not np.isfinite(initial_state).all(): + raise ValueError("Initial EE state contains non-finite values") + return { + "action_raw": torch.from_numpy(action), + "action_is_pad": torch.from_numpy(~action_valid), + "camera_raw": torch.from_numpy(camera), + "camera_is_pad": torch.zeros(ACTION_HORIZON, dtype=torch.bool), + "initial_state": torch.from_numpy(initial_state), + "camera_to_reference": camera_reference.astype(np.float32), + } + + +def _decode_episode_rgb( + episode: EgoverseEpisode, + frame_index: np.ndarray, + rectified_k: np.ndarray, + resolution: int, +) -> np.ndarray: + capture = cv2.VideoCapture(str(episode.video_path)) + if not capture.isOpened(): + raise RuntimeError(f"Could not open RGB video: {episode.video_path}") + distortion = np.asarray(episode.distortion, dtype=np.float64) + output_size = (episode.output_width, episode.output_height) + map_x = map_y = None + if distortion.size and np.any(np.abs(distortion) > 0): + map_x, map_y = cv2.initUndistortRectifyMap( + np.asarray(episode.k_output, dtype=np.float64), + distortion, + None, + np.asarray(rectified_k, dtype=np.float64), + output_size, + cv2.CV_32FC1, + ) + frames: list[np.ndarray] = [] + next_source = 0 + try: + for target in frame_index.astype(np.int64): + if int(target) < next_source: + raise ValueError("frame_index must be strictly increasing") + while next_source <= int(target): + ok, bgr = capture.read() + if not ok: + raise RuntimeError( + f"Could not decode source frame {target} from {episode.video_path}" + ) + if next_source == int(target): + if (bgr.shape[1], bgr.shape[0]) != output_size: + bgr = cv2.resize(bgr, output_size, interpolation=cv2.INTER_AREA) + if map_x is not None and map_y is not None: + bgr = cv2.remap( + bgr, + map_x, + map_y, + interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + ) + crop = min(bgr.shape[0], bgr.shape[1]) + top = (bgr.shape[0] - crop) // 2 + left = (bgr.shape[1] - crop) // 2 + bgr = bgr[top : top + crop, left : left + crop] + bgr = cv2.resize( + bgr, (resolution, resolution), interpolation=cv2.INTER_AREA + ) + frames.append(cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)) + next_source += 1 + finally: + capture.release() + return np.stack(frames) + + +def _resize_episode_depth(depth_mm: np.ndarray, resolution: int) -> np.ndarray: + height, width = depth_mm.shape[1:] + crop = min(height, width) + top = (height - crop) // 2 + left = (width - crop) // 2 + cropped = depth_mm[:, top : top + crop, left : left + crop] + return np.stack( + [ + cv2.resize(frame, (resolution, resolution), interpolation=cv2.INTER_NEAREST) + for frame in cropped + ] + ) + + +def load_motion_payload( + episode: EgoverseEpisode, + frame_index: np.ndarray | None = None, +) -> MotionPayload: + """Read EE/keypoint/head metadata without decoding RGB or depth.""" + + if frame_index is None: + with np.load(episode.annotation_path, allow_pickle=False) as bundle: + frame_index = bundle["frame_index"].astype(np.int64) + if len(frame_index) != episode.length: + raise RuntimeError( + f"Motion length mismatch for {episode.episode_id}: " + f"manifest={episode.length}, annotation={len(frame_index)}" + ) + root = episode.source_metadata_path.parent + arrays = { + key: _read_zarr_v3_sharded_numeric_rows(root / key, frame_index) + for key in ( + "left.obs_ee_pose", + "right.obs_ee_pose", + "left.obs_keypoints", + "right.obs_keypoints", + "obs_head_pose", + ) + } + return _motion_payload_from_arrays(frame_index, arrays) + + +def _motion_payload_from_arrays( + frame_index: np.ndarray, + arrays: dict[str, np.ndarray], +) -> MotionPayload: + left_matrix, left_valid = _safe_pose_matrices(arrays["left.obs_ee_pose"], 1.0e-8) + right_matrix, right_valid = _safe_pose_matrices(arrays["right.obs_ee_pose"], 1.0e-8) + head_matrix, head_valid = _safe_pose_matrices(arrays["obs_head_pose"], 1.0e-8) + left_grip, left_grip_valid = _grip_from_keypoints(arrays["left.obs_keypoints"]) + right_grip, right_grip_valid = _grip_from_keypoints(arrays["right.obs_keypoints"]) + return MotionPayload( + frame_index=frame_index, + head_matrix=head_matrix, head_valid=head_valid, + left_matrix=left_matrix, left_valid=left_valid, + right_matrix=right_matrix, right_valid=right_valid, + left_grip=left_grip, left_grip_valid=left_grip_valid, + right_grip=right_grip, right_grip_valid=right_grip_valid, + ) + + +def load_motion_window_payload( + episode: EgoverseEpisode, + start: int, +) -> MotionPayload: + """Read only episode-frame-0 plus one 33-frame statistics window.""" + + if not 0 <= start <= episode.length - WINDOW_SIZE: + raise IndexError(f"Invalid window start {start} for {episode.episode_id}") + with np.load(episode.annotation_path, allow_pickle=False) as bundle: + full_frame_index = bundle["frame_index"].astype(np.int64) + if len(full_frame_index) != episode.length: + raise RuntimeError( + f"Motion length mismatch for {episode.episode_id}: " + f"manifest={episode.length}, annotation={len(full_frame_index)}" + ) + # Keep a separate reference slot even when start == 0. The common target + # function then sees episode frame 0 at payload index 0 and the exact + # 33-frame window at payload indices 1..33. + selected_frame_index = np.concatenate( + [full_frame_index[0:1], full_frame_index[start : start + WINDOW_SIZE]] + ) + root = episode.source_metadata_path.parent + arrays = { + key: _read_zarr_v3_sharded_numeric_rows(root / key, selected_frame_index) + for key in ( + "left.obs_ee_pose", + "right.obs_ee_pose", + "left.obs_keypoints", + "right.obs_keypoints", + "obs_head_pose", + ) + } + return _motion_payload_from_arrays(selected_frame_index, arrays) + + +def load_episode_payload(episode: EgoverseEpisode, resolution: int) -> EpisodePayload: + """Decompress and decode one complete episode once for grouped windows.""" + + with np.load(episode.annotation_path, allow_pickle=False) as bundle: + depth_mm = _resize_episode_depth(bundle["depth_mm"], resolution) + frame_index = bundle["frame_index"].astype(np.int64) + rectified_k = bundle["K"].astype(np.float32) + if len(frame_index) != episode.length or len(depth_mm) != episode.length: + raise RuntimeError( + f"Episode length mismatch for {episode.episode_id}: " + f"manifest={episode.length}, annotation={len(frame_index)}" + ) + intrinsics = scale_intrinsics_after_center_crop( + rectified_k, episode.output_height, episode.output_width, resolution + ) + rgb_uint8 = _decode_episode_rgb( + episode, frame_index, rectified_k, resolution + ) + motion = load_motion_payload(episode, frame_index) + return EpisodePayload( + rgb_uint8=rgb_uint8, + depth_mm=depth_mm, + intrinsics=intrinsics, + **motion.__dict__, + ) + + +class SpaceDreamerEgoDataset(Dataset): + def __init__( + self, + episodes: list[EgoverseEpisode], + schedule: list[WindowRecord], + *, + resolution: int, + prompt_cache_dir: Path, + action_mean: torch.Tensor, + action_std: torch.Tensor, + camera_mean: torch.Tensor, + camera_std: torch.Tensor, + normalization_clip: float | None, + depth_config: dict[str, Any], + point_config: dict[str, Any], + cache_derived_modalities: bool = False, + prefetch_next_episode: bool = False, + ) -> None: + self.episodes = episodes + self.schedule = schedule + self.resolution = int(resolution) + self.prompt_cache_dir = prompt_cache_dir + self.action_mean = action_mean.float().cpu() + self.action_std = action_std.float().cpu() + self.camera_mean = camera_mean.float().cpu() + self.camera_std = camera_std.float().cpu() + self.normalization_clip = normalization_clip + self.depth_config = depth_config + self.point_config = point_config + self.cache_derived_modalities = bool(cache_derived_modalities) + self.prefetch_next_episode = bool(prefetch_next_episode) + self._cached_slot: int | None = None + self._cached_payload: EpisodePayload | None = None + self._cached_prompt: dict[str, Any] | None = None + self._cached_depth: np.ndarray | None = None + self._cached_point: np.ndarray | None = None + self._cached_point_outside: np.ndarray | None = None + self._cached_point_valid: np.ndarray | None = None + self._prefetch_executor: ThreadPoolExecutor | None = None + self._prefetch_slot: int | None = None + self._prefetch_future: Future[tuple[Any, ...]] | None = None + + def _shutdown_prefetch(self, *, wait: bool) -> None: + if self._prefetch_future is not None: + self._prefetch_future.cancel() + if self._prefetch_executor is not None: + self._prefetch_executor.shutdown(wait=wait, cancel_futures=True) + self._prefetch_executor = None + self._prefetch_slot = None + self._prefetch_future = None + + def close(self) -> None: + """Release the optional episode-prefetch worker explicitly.""" + + self._shutdown_prefetch(wait=True) + + def __del__(self) -> None: + # Dataset destruction can happen during interpreter shutdown, where + # referenced modules may already be partially torn down. + try: + self._shutdown_prefetch(wait=False) + except Exception: + pass + + def set_schedule(self, schedule: list[WindowRecord]) -> None: + # This is normally called on the parent-process dataset, while loading + # and prefetching happen in the short-lived DataLoader worker. Still, + # close any local executor so num_workers=0 remains race-free. + self._shutdown_prefetch(wait=True) + self.schedule = schedule + self._cached_slot = None + self._cached_payload = None + self._cached_prompt = None + self._cached_depth = None + self._cached_point = None + self._cached_point_outside = None + self._cached_point_valid = None + + def _load_slot_payload(self, slot: int) -> tuple[Any, ...]: + episode = self.episodes[slot] + payload = load_episode_payload(episode, self.resolution) + prompt_key = hashlib.sha256(episode.prompt.encode("utf-8")).hexdigest() + prompt_path = self.prompt_cache_dir / f"{prompt_key}.pt" + if not prompt_path.is_file(): + raise FileNotFoundError(prompt_path) + prompt = torch.load(prompt_path, map_location="cpu", weights_only=True) + if not self.cache_derived_modalities: + return payload, prompt, None, None, None, None + + length = len(payload.frame_index) + depth_video = np.empty( + (3, length, self.resolution, self.resolution), dtype=np.float32 + ) + point_video = np.empty( + (3, length, self.resolution, self.resolution), dtype=np.float32 + ) + outside_counts = np.zeros(length, dtype=np.int64) + valid_counts = np.zeros(length, dtype=np.int64) + reference_from_world = _rigid_inverse(payload.head_matrix[0]) + camera_to_reference = ( + reference_from_world[None] @ payload.head_matrix + ).astype(np.float32) + low = np.asarray(self.point_config["xyz_min_m"], dtype=np.float32) + high = np.asarray(self.point_config["xyz_max_m"], dtype=np.float32) + chunk_frames = 64 + depth_preprocessing = DepthPreprocessingConfig( + depth_min=float(self.depth_config["min"]), + depth_max=float(self.depth_config["max"]), + depth_invalid_value=float(self.depth_config["invalid_value"]), + depth_invalid_fill=str(self.depth_config["invalid_fill"]), + ) + for begin in range(0, length, chunk_frames): + end = min(begin + chunk_frames, length) + processed = preprocess_depth_grayscale( + payload.depth_mm[begin:end], + config=depth_preprocessing, + input_unit=str(self.depth_config["input_unit"]), + ) + depth_video[:, begin:end] = processed.vae_video.transpose(1, 0, 2, 3) + depth_m = ( + payload.depth_mm[begin:end].astype(np.float32) / np.float32(1000.0) + ) + world_points, world_valid = depth_video_to_world_grid( + depth_m, + payload.intrinsics, + camera_to_reference[begin:end], + stride=1, + min_depth_m=float(self.point_config["min_depth_m"]), + max_depth_m=float(self.point_config["max_depth_m"]), + valid_mask=processed.valid_mask[:, 0], + ) + normalized, _ = normalize_world_point_video( + world_points, + world_valid, + low, + high, + invalid_fill_reference=self.point_config["invalid_fill_reference_m"], + ) + point_video[:, begin:end] = normalized + finite_valid = world_valid & np.isfinite(world_points).all(axis=-1) + outside = finite_valid & ( + ((world_points < low) | (world_points > high)).any(axis=-1) + ) + outside_counts[begin:end] = outside.sum(axis=(1, 2), dtype=np.int64) + valid_counts[begin:end] = finite_valid.sum(axis=(1, 2), dtype=np.int64) + return ( + payload, + prompt, + depth_video, + point_video, + outside_counts, + valid_counts, + ) + + def _queue_prefetch(self, index: int) -> None: + if not self.prefetch_next_episode or index >= len(self.schedule): + return + # Schedules keep an episode's windows contiguous. Find the next + # different episode immediately after loading the current one, so its + # full decode/geometry work overlaps all remaining current windows. + slot = self.schedule[index].episode_slot + while slot == self._cached_slot: + index += 1 + if index >= len(self.schedule): + return + slot = self.schedule[index].episode_slot + if slot == self._prefetch_slot: + return + if self._prefetch_executor is None: + self._prefetch_executor = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="egoverse-episode-prefetch" + ) + if self._prefetch_future is not None and not self._prefetch_future.done(): + return + self._prefetch_slot = slot + self._prefetch_future = self._prefetch_executor.submit( + self._load_slot_payload, slot + ) + + def __len__(self) -> int: + return len(self.schedule) + + def _load_slot(self, slot: int) -> tuple[EpisodePayload, dict[str, Any]]: + if self._cached_slot != slot: + if self._prefetch_slot == slot and self._prefetch_future is not None: + loaded = self._prefetch_future.result() + else: + loaded = self._load_slot_payload(slot) + ( + self._cached_payload, + self._cached_prompt, + self._cached_depth, + self._cached_point, + self._cached_point_outside, + self._cached_point_valid, + ) = loaded + self._cached_slot = slot + self._prefetch_slot = None + self._prefetch_future = None + assert self._cached_payload is not None and self._cached_prompt is not None + return self._cached_payload, self._cached_prompt + + def __getitem__(self, index: int) -> dict[str, Any]: + record = self.schedule[index] + episode = self.episodes[record.episode_slot] + payload, prompt = self._load_slot(record.episode_slot) + self._queue_prefetch(index + 1) + start = record.start + stop = start + WINDOW_SIZE + targets = absolute_targets_in_episode_first_camera(payload, start) + + rgb = torch.from_numpy( + payload.rgb_uint8[start:stop].astype(np.float32) / np.float32(127.5) + - np.float32(1.0) + ).permute(3, 0, 1, 2) + if self._cached_depth is not None and self._cached_point is not None: + depth = torch.from_numpy(self._cached_depth[:, start:stop]) + point = self._cached_point[:, start:stop] + assert self._cached_point_outside is not None + assert self._cached_point_valid is not None + clipped_fraction = float( + self._cached_point_outside[start:stop].sum(dtype=np.int64) + / max(self._cached_point_valid[start:stop].sum(dtype=np.int64), 1) + ) + else: + processed_depth = preprocess_depth_grayscale( + payload.depth_mm[start:stop], + config=DepthPreprocessingConfig( + depth_min=float(self.depth_config["min"]), + depth_max=float(self.depth_config["max"]), + depth_invalid_value=float(self.depth_config["invalid_value"]), + depth_invalid_fill=str(self.depth_config["invalid_fill"]), + ), + input_unit=str(self.depth_config["input_unit"]), + ) + depth = torch.from_numpy(processed_depth.vae_video).permute(1, 0, 2, 3) + depth_m = payload.depth_mm[start:stop].astype(np.float32) / np.float32(1000.0) + world_points, world_valid = depth_video_to_world_grid( + depth_m, + payload.intrinsics, + targets["camera_to_reference"], + stride=1, + min_depth_m=float(self.point_config["min_depth_m"]), + max_depth_m=float(self.point_config["max_depth_m"]), + valid_mask=processed_depth.valid_mask[:, 0], + ) + point, clipped_fraction = normalize_world_point_video( + world_points, + world_valid, + self.point_config["xyz_min_m"], + self.point_config["xyz_max_m"], + invalid_fill_reference=self.point_config["invalid_fill_reference_m"], + ) + + action = (targets["action_raw"] - self.action_mean) / self.action_std + camera = (targets["camera_raw"] - self.camera_mean) / self.camera_std + if self.normalization_clip is not None: + clip = float(self.normalization_clip) + action = action.clamp(-clip, clip) + camera = camera.clamp(-clip, clip) + action = action.masked_fill(targets["action_is_pad"].unsqueeze(-1), 0) + camera = camera.masked_fill(targets["camera_is_pad"].unsqueeze(-1), 0) + return { + "sample_id": f"{episode.episode_id}_start_{start:06d}", + "episode_id": episode.episode_id, + "window_start": torch.tensor(start, dtype=torch.long), + "rgb": rgb, + "depth": depth, + "point": torch.from_numpy(point), + "action": action, + "camera": camera, + "initial_state": targets["initial_state"], + "action_is_pad": targets["action_is_pad"], + "camera_is_pad": targets["camera_is_pad"], + "sample_weight": torch.tensor(record.sample_weight, dtype=torch.float32), + "is_padding": torch.tensor(record.is_padding, dtype=torch.bool), + "point_clipped_fraction": torch.tensor(clipped_fraction, dtype=torch.float32), + "context": prompt["context"], + "context_mask": prompt["context_mask"].to(dtype=torch.bool), + "prompt": episode.prompt, + } + + +def collate_spacedreamer(samples: list[dict[str, Any]]) -> dict[str, Any]: + if not samples: + raise ValueError("Cannot collate an empty batch") + tensor_keys = ( + "rgb", + "depth", + "point", + "action", + "camera", + "initial_state", + "action_is_pad", + "camera_is_pad", + "sample_weight", + "is_padding", + "point_clipped_fraction", + "window_start", + ) + maximum_context = max(int(sample["context"].shape[0]) for sample in samples) + contexts = [] + context_masks = [] + for sample in samples: + context = sample["context"] + mask = sample["context_mask"] + padding = maximum_context - int(context.shape[0]) + if padding: + context = torch.nn.functional.pad(context, (0, 0, 0, padding)) + mask = torch.nn.functional.pad(mask, (0, padding), value=False) + contexts.append(context) + context_masks.append(mask) + return { + **{key: torch.stack([sample[key] for sample in samples]) for key in tensor_keys}, + "context": torch.stack(contexts), + "context_mask": torch.stack(context_masks), + "sample_id": [sample["sample_id"] for sample in samples], + "episode_id": [sample["episode_id"] for sample in samples], + "prompt": [sample["prompt"] for sample in samples], + } + + +def iter_center_window_targets( + episodes: list[EgoverseEpisode], +) -> Iterator[dict[str, torch.Tensor]]: + """Stream one representative absolute-target window per training episode.""" + + for episode in episodes: + payload = load_motion_payload(episode) + start = (episode.length - WINDOW_SIZE) // 2 + yield absolute_targets_in_episode_first_camera(payload, start) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_galexea_rgb_state_joint_action_ablation.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_galexea_rgb_state_joint_action_ablation.py new file mode 100644 index 0000000000000000000000000000000000000000..f6cf417081e98d943bfaa49acdc943de73c473e8 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_galexea_rgb_state_joint_action_ablation.py @@ -0,0 +1,722 @@ +#!/usr/bin/env python3 +"""Isolated real-robot ablation: RGB frame 0 + state -> RGB video + action. + +The experiment loads the Wan2.2 RGB base weights only. It never loads the +existing RGB/depth/point/action/camera joint checkpoint, and depth, point-map, +and camera tokens do not exist in its model, cache, loss, or checkpoint. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +from accelerate import Accelerator +from accelerate.utils import DistributedType, set_seed +import torch +from torch.utils.data import DataLoader, Dataset +import yaml + +from galexea_mcap import ( + GalexeaClipConfig, + GalexeaTopics, + discover_episodes, + extract_clip, + scan_episode, + select_clip_image_indices, +) +from joint_rgb_action_diffusion import joint_rgb_action_flow_matching_loss +from rgb_action_model_factory import ( + build_fresh_rgb_action_mot, + rgb_action_checkpoint_metadata, + validate_rgb_action_model_config, +) +from train_joint_rgb_depth import ( + WanContinuousFlowMatchScheduler, + append_jsonl, + build_lr_scheduler, + dtype_from_name, + load_text_encoder_and_tokenizer, + load_vae, +) +from train_joint_rgb_depth_point import FixedSeedPermutationSampler +from train_joint_rgb_depth_point_action import enable_fp32_master_weights + + +SPLITS = ("train", "validation", "test") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--max-steps", type=int) + parser.add_argument("--prepare-cache-only", action="store_true") + parser.add_argument("--dataset-smoke-test", action="store_true") + parser.add_argument("--overwrite-cache", action="store_true") + return parser.parse_args() + + +def load_config(path: Path) -> dict[str, Any]: + config = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(config, dict): + raise ValueError("Config must be a YAML mapping") + return config + + +def topics_from_config(config: dict[str, Any]) -> GalexeaTopics: + values = config["galexea"].get("topics", {}) + defaults = GalexeaTopics() + return GalexeaTopics( + **{ + name: values.get(name, getattr(defaults, name)) + for name in defaults.__dataclass_fields__ + } + ) + + +def clip_config_from_config(config: dict[str, Any]) -> GalexeaClipConfig: + data, galexea = config["data"], config["galexea"] + gripper = galexea["gripper"] + configured_fps = data.get("target_fps", "native") + return GalexeaClipConfig( + num_frames=int(data["num_frames"]), + target_fps=None if configured_fps == "native" else float(configured_fps), + resolution=int(data["resolution"]), + clips_per_episode=int(data["clips_per_episode"]), + max_alignment_error_s=float(galexea["max_alignment_error_s"]), + gripper_min=float(gripper["min"]), + gripper_max=float(gripper["max"]), + gripper_invert=bool(gripper["invert"]), + ) + + +def validate_config(config: dict[str, Any]) -> dict[str, Any]: + required = { + "experiment", + "paths", + "model", + "data", + "conditioning", + "diffusion", + "training", + "action", + "galexea", + "cache", + } + missing = required - set(config) + if missing: + raise KeyError(f"Missing config sections: {sorted(missing)}") + forbidden = {"depth", "point_cloud", "camera"}.intersection(config) + if forbidden: + raise ValueError( + f"RGB+state ablation must not define unrelated modalities: {sorted(forbidden)}" + ) + if "checkpoint" in config["paths"]: + raise ValueError("This fresh ablation must not define paths.checkpoint") + validate_rgb_action_model_config(config) + data = config["data"] + if int(data["num_frames"]) != 33 or int(data["resolution"]) != 256: + raise ValueError("This ablation requires 33 frames at 256x256") + if data.get("target_fps") != "native": + raise ValueError("This ablation preserves native RGB timestamps") + split_counts = data.get("episode_splits", {}) + if set(split_counts) != set(SPLITS): + raise ValueError(f"data.episode_splits must contain exactly {SPLITS}") + if any(int(split_counts[name]) < 0 for name in SPLITS): + raise ValueError("Episode split counts must be non-negative") + total_episodes = sum(int(split_counts[name]) for name in SPLITS) + expected_clips = total_episodes * int(data["clips_per_episode"]) + if int(data["num_clips"]) != expected_clips: + raise ValueError( + f"data.num_clips must be {expected_clips}, got {data['num_clips']}" + ) + conditioning = config["conditioning"] + if conditioning != { + "rgb_frame0": "clean_condition", + "initial_state": "clean_action_context_token", + "exclude_rgb_frame0_from_loss": True, + }: + raise ValueError("conditioning must contain only clean RGB frame0 and state") + normalization = config["action"]["normalization"] + if normalization.get("fit") != "galexea_train_split_cache": + raise ValueError("Action normalization must be fitted on the train split") + for key in ("lambda_rgb", "lambda_action"): + if float(config["diffusion"].get(key, 0)) <= 0: + raise ValueError(f"diffusion.{key} must be positive") + if int(config["training"]["max_steps"]) <= 0: + raise ValueError("training.max_steps must be positive") + root = Path(config["paths"]["galexea_root"]).resolve() + model_dir = Path(config["paths"]["pretrained_model_dir"]).resolve() + cache_dir = Path(config["paths"]["cache_dir"]).resolve() + output_dir = Path(config["experiment"]["output_dir"]).resolve() + if not root.is_dir() or not model_dir.is_dir(): + raise FileNotFoundError("Galexea root or Wan2.2 base directory is missing") + for writable in (cache_dir, output_dir): + if writable == root or writable.is_relative_to(root): + raise ValueError("Ablation outputs must not be inside the source dataset") + if writable == model_dir or writable.is_relative_to(model_dir): + raise ValueError("Ablation outputs must not be inside the Wan base directory") + if ( + cache_dir == output_dir + or cache_dir.is_relative_to(output_dir) + or output_dir.is_relative_to(cache_dir) + ): + raise ValueError("cache_dir and output_dir must be separate trees") + return config + + +def resolve_split_episodes(config: dict[str, Any]) -> dict[str, list[Path]]: + episodes = discover_episodes(Path(config["paths"]["galexea_root"])) + counts = config["data"]["episode_splits"] + required = sum(int(counts[name]) for name in SPLITS) + if len(episodes) < required: + raise RuntimeError(f"Need {required} episodes, found {len(episodes)}") + result: dict[str, list[Path]] = {} + offset = 0 + for name in SPLITS: + count = int(counts[name]) + result[name] = episodes[offset : offset + count] + offset += count + return result + + +def cache_fingerprint( + config: dict[str, Any], split_episodes: dict[str, list[Path]] +) -> str: + episode_records = [] + for split in SPLITS: + for path in split_episodes[split]: + stat = path.stat() + episode_records.append( + { + "split": split, + "path": str(path), + "size": stat.st_size, + "mtime_ns": stat.st_mtime_ns, + } + ) + selected = { + "format_version": 2, + "episodes": episode_records, + "data": config["data"], + "galexea": config["galexea"], + "prompt": config["data"]["prompt"], + "pretrained_model_dir": config["paths"]["pretrained_model_dir"], + "modalities": ["rgb", "action", "initial_state"], + } + return hashlib.sha256(json.dumps(selected, sort_keys=True).encode()).hexdigest() + + +def dataset_smoke_test(config: dict[str, Any]) -> None: + split_episodes = resolve_split_episodes(config) + mcap = split_episodes["train"][0] + topics, clip_config = topics_from_config(config), clip_config_from_config(config) + index = scan_episode(mcap, topics) + indices = select_clip_image_indices(index.image_time_ns, clip_config)[0] + sample = extract_clip(index, topics, clip_config, indices) + print( + json.dumps( + { + "status": "rgb_state_joint_action_data_ok", + "episode": index.episode_id, + "rgb_shape": list(sample["rgb"].shape), + "action_shape": list(sample["action_raw"].shape), + "initial_state_shape": list(sample["initial_state"].shape), + "effective_fps": float(sample["effective_fps"]), + "model_inputs": ["noisy_rgb_with_clean_frame0", "initial_state"], + "model_outputs": ["rgb_video", "action"], + "excluded_modalities": ["depth", "point", "camera"], + }, + indent=2, + ), + flush=True, + ) + + +def _cache_is_complete(cache_dir: Path, manifest: dict[str, Any]) -> bool: + if not (cache_dir / "prompt_contexts.pt").is_file(): + return False + return all((cache_dir / entry["file"]).is_file() for entry in manifest["samples"]) + + +@torch.no_grad() +def prepare_cache( + config: dict[str, Any], *, device: torch.device, overwrite: bool +) -> None: + cache_dir = Path(config["paths"]["cache_dir"]) + split_episodes = resolve_split_episodes(config) + fingerprint = cache_fingerprint(config, split_episodes) + manifest_path = cache_dir / "manifest.json" + if manifest_path.is_file() and not overwrite: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("fingerprint") != fingerprint: + raise RuntimeError("Cache fingerprint mismatch; use a new cache directory") + if not _cache_is_complete(cache_dir, manifest): + raise RuntimeError("Cache manifest exists but sample files are incomplete") + print(f"[cache] reuse {manifest_path}", flush=True) + return + cache_dir.mkdir(parents=True, exist_ok=True) + dtype = dtype_from_name(str(config["model"]["dtype"])) + model_dir = Path(config["paths"]["pretrained_model_dir"]) + text_encoder, tokenizer = load_text_encoder_and_tokenizer( + model_dir, + dtype, + device, + int(config["cache"].get("tokenizer_max_length", 512)), + ) + prompt = str(config["data"]["prompt"]) + ids, mask = tokenizer([prompt], return_mask=True, add_special_tokens=True) + context = text_encoder(ids.to(device), mask.to(device=device, dtype=torch.bool))[0] + prompt_path = cache_dir / "prompt_contexts.pt" + prompt_tmp = cache_dir / "prompt_contexts.pt.tmp" + torch.save( + { + "entries": [ + { + "prompt": prompt, + "context": context.cpu(), + "context_mask": mask[0].to(dtype=torch.bool).cpu(), + } + ] + }, + prompt_tmp, + ) + prompt_tmp.replace(prompt_path) + text_encoder.to("cpu") + del text_encoder, tokenizer, ids, mask, context + if device.type == "cuda": + torch.cuda.empty_cache() + vae = load_vae(model_dir, dtype, device) + topics, clip_config = topics_from_config(config), clip_config_from_config(config) + expected_shape = ( + 1, + 48, + (int(config["data"]["num_frames"]) - 1) // 4 + 1, + int(config["data"]["resolution"]) // 16, + int(config["data"]["resolution"]) // 16, + ) + entries: list[dict[str, Any]] = [] + sample_number = 0 + for split in SPLITS: + for episode_number, mcap in enumerate(split_episodes[split]): + print(f"[cache] {split} scan {mcap.parent.name}", flush=True) + index = scan_episode(mcap, topics) + windows = select_clip_image_indices(index.image_time_ns, clip_config) + for clip_number, image_indices in enumerate(windows): + sample = extract_clip(index, topics, clip_config, image_indices) + rgb = sample["rgb"].unsqueeze(0).to(device=device, dtype=dtype) + z_rgb = vae.encode(rgb, device=device) + if tuple(z_rgb.shape) != expected_shape: + raise RuntimeError( + f"Unexpected RGB latent {tuple(z_rgb.shape)} != {expected_shape}" + ) + sample_id = f"{index.episode_id}_clip_{clip_number:02d}" + filename = f"{sample_number:04d}_{sample_id}.pt" + target = cache_dir / filename + temporary = cache_dir / f"{filename}.tmp" + torch.save( + { + "sample_id": sample_id, + "z_rgb": z_rgb[0].cpu(), + "action_raw": sample["action_raw"], + "action_is_pad": sample["action_is_pad"], + "initial_state": sample["initial_state"], + "image_time_ns": sample["image_time_ns"], + "effective_fps": sample["effective_fps"], + }, + temporary, + ) + temporary.replace(target) + entries.append( + { + "sample_id": sample_id, + "file": filename, + "split": split, + "episode_index": episode_number, + "clip_index": clip_number, + "prompt_index": 0, + } + ) + sample_number += 1 + manifest = { + "format_version": 2, + "fingerprint": fingerprint, + "source": "galexea_ros2_mcap_0728", + "modalities": ["rgb", "action", "initial_state"], + "explicitly_excluded_modalities": ["depth", "point", "camera"], + "samples": entries, + } + manifest_tmp = cache_dir / "manifest.json.tmp" + manifest_tmp.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + manifest_tmp.replace(manifest_path) + vae.to("cpu") + del vae + if device.type == "cuda": + torch.cuda.empty_cache() + + +class RgbActionCacheDataset(Dataset): + def __init__( + self, + cache_dir: Path, + expected_fingerprint: str, + split: str, + *, + action_mean: torch.Tensor | None = None, + action_std: torch.Tensor | None = None, + action_clip: float | None = None, + ) -> None: + manifest = json.loads((cache_dir / "manifest.json").read_text(encoding="utf-8")) + if manifest.get("fingerprint") != expected_fingerprint: + raise RuntimeError("RGB-action cache does not match config") + if manifest.get("modalities") != ["rgb", "action", "initial_state"]: + raise RuntimeError("Cache contains unexpected modalities") + self.cache_dir = cache_dir + self.entries = [entry for entry in manifest["samples"] if entry["split"] == split] + if not self.entries: + raise RuntimeError(f"Cache split {split!r} is empty") + prompt = torch.load( + cache_dir / "prompt_contexts.pt", map_location="cpu", weights_only=True + )["entries"][0] + self.context = prompt["context"] + self.context_mask = prompt["context_mask"].to(dtype=torch.bool) + self.prompt = prompt["prompt"] + self.action_mean = action_mean + self.action_std = action_std + self.action_clip = action_clip + + def __len__(self) -> int: + return len(self.entries) + + def raw_payload(self, index: int) -> dict[str, Any]: + return torch.load( + self.cache_dir / self.entries[index]["file"], + map_location="cpu", + weights_only=True, + ) + + def __getitem__(self, index: int) -> dict[str, Any]: + payload = self.raw_payload(index) + action = payload["action_raw"] + if self.action_mean is not None: + action = (action - self.action_mean) / self.action_std + if self.action_clip is not None: + action = action.clamp(-self.action_clip, self.action_clip) + action = action.masked_fill(payload["action_is_pad"].unsqueeze(-1), 0) + return { + "sample_id": payload["sample_id"], + "z_rgb": payload["z_rgb"], + "action": action, + "action_is_pad": payload["action_is_pad"], + "initial_state": payload["initial_state"], + "context": self.context, + "context_mask": self.context_mask, + "prompt": self.prompt, + } + + +@torch.no_grad() +def fit_action_normalizer(model: torch.nn.Module, dataset: RgbActionCacheDataset) -> None: + action_dim = int(model.action_normalizer.action_dim) + total = torch.zeros(action_dim, dtype=torch.float64) + squared = torch.zeros(action_dim, dtype=torch.float64) + count = 0 + for index in range(len(dataset)): + payload = dataset.raw_payload(index) + valid = ~payload["action_is_pad"].to(dtype=torch.bool) + values = payload["action_raw"][valid].to(dtype=torch.float64) + total += values.sum(dim=0) + squared += values.square().sum(dim=0) + count += int(values.shape[0]) + if count < 2: + raise RuntimeError("At least two valid train actions are required") + mean = total / count + variance = (squared / count - mean.square()).clamp( + min=float(model.action_normalizer.eps) ** 2 + ) + model.action_normalizer.mean.copy_(mean.float()) + model.action_normalizer.std.copy_(variance.sqrt().float()) + model.action_normalizer.count.fill_(count) + model.action_normalizer.fitted.fill_(True) + + +def collate_rgb_action(samples: list[dict[str, Any]]) -> dict[str, Any]: + tensor_keys = ( + "z_rgb", + "action", + "action_is_pad", + "initial_state", + "context", + "context_mask", + ) + return { + **{key: torch.stack([sample[key] for sample in samples]) for key in tensor_keys}, + "sample_id": [sample["sample_id"] for sample in samples], + "prompt": [sample["prompt"] for sample in samples], + } + + +def configure_trainable(model: torch.nn.Module) -> None: + model.requires_grad_(True) + model.video_expert.patch_embedding.bias.requires_grad_(False) + model.video_expert.head.modulation.requires_grad_(False) + model.action_normalizer.requires_grad_(False) + + +def build_optimizer(model: torch.nn.Module, config: dict[str, Any]) -> torch.optim.Optimizer: + video_parameters = [p for p in model.video_expert.parameters() if p.requires_grad] + action_parameters = [ + p + for module in (model.action_expert, model.action_state_encoder) + for p in module.parameters() + if p.requires_grad + ] + return torch.optim.AdamW( + [ + { + "name": "rgb_video_expert", + "params": video_parameters, + "lr": float(config["training"]["video_lr"]), + }, + { + "name": "action_expert_and_state_encoder", + "params": action_parameters, + "lr": float(config["training"]["action_lr"]), + }, + ], + betas=(0.9, 0.95), + eps=1.0e-8, + weight_decay=float(config["training"].get("weight_decay", 0.0)), + ) + + +def compute_loss( + model: torch.nn.Module, + batch: dict[str, Any], + video_scheduler: Any, + action_scheduler: Any, + config: dict[str, Any], + device: torch.device, + dtype: torch.dtype, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + z_rgb = batch["z_rgb"].to(device=device, dtype=dtype, non_blocking=True) + action = batch["action"].to(device=device, dtype=dtype, non_blocking=True) + initial_state = batch["initial_state"].to( + device=device, dtype=dtype, non_blocking=True + ) + context_mask_cpu = batch["context_mask"] + valid_columns = context_mask_cpu.any(dim=0) + context_length = int(valid_columns.nonzero(as_tuple=False)[-1, 0]) + 1 + context = batch["context"][:, :context_length].to( + device=device, dtype=dtype, non_blocking=True + ) + context_mask = context_mask_cpu[:, :context_length].to( + device=device, dtype=torch.bool, non_blocking=True + ) + return joint_rgb_action_flow_matching_loss( + model, + video_scheduler, + action_scheduler, + z_rgb=z_rgb, + action=action, + initial_state=initial_state, + context=context, + context_mask=context_mask, + action_is_pad=batch["action_is_pad"].to( + device=device, dtype=torch.bool, non_blocking=True + ), + image_is_pad=( + batch["image_is_pad"].to( + device=device, dtype=torch.bool, non_blocking=True + ) + if "image_is_pad" in batch + else None + ), + fix_rgb_first_latent=True, + exclude_rgb_condition_from_loss=True, + lambda_rgb=float(config["diffusion"]["lambda_rgb"]), + lambda_action=float(config["diffusion"]["lambda_action"]), + ) + + +def export_model(model: torch.nn.Module, accelerator: Accelerator, path: Path) -> None: + fsdp_plugin = None + previous_state_dict_type = None + if accelerator.distributed_type == DistributedType.FSDP: + fsdp_plugin = accelerator.state.fsdp_plugin + previous_state_dict_type = fsdp_plugin.state_dict_type + fsdp_plugin.set_state_dict_type("FULL_STATE_DICT") + state_dict = accelerator.get_state_dict(model) + if accelerator.is_main_process: + unwrapped = accelerator.unwrap_model(model) + payload = { + "metadata": rgb_action_checkpoint_metadata(unwrapped), + "state_dict": {key: value.cpu() for key, value in state_dict.items()}, + } + path.parent.mkdir(parents=True, exist_ok=True) + accelerator.save(payload, path) + print(f"[checkpoint] exported {path}", flush=True) + accelerator.wait_for_everyone() + if fsdp_plugin is not None: + fsdp_plugin.set_state_dict_type(previous_state_dict_type) + + +def train(config: dict[str, Any], output_dir: Path) -> None: + accelerator = Accelerator( + gradient_accumulation_steps=int(config["training"]["gradient_accumulation_steps"]), + mixed_precision="bf16" if config["model"]["dtype"] == "bfloat16" else "no", + ) + set_seed(int(config["experiment"]["seed"]), device_specific=True) + if accelerator.is_main_process: + if output_dir.exists() and any(output_dir.iterdir()): + raise RuntimeError(f"Refusing to reuse non-empty output directory: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "resolved_config.yaml").write_text( + yaml.safe_dump(config, sort_keys=False), encoding="utf-8" + ) + accelerator.wait_for_everyone() + model = build_fresh_rgb_action_mot(config) + if bool(config["training"].get("fp32_master_weights", True)): + enable_fp32_master_weights(model) + split_episodes = resolve_split_episodes(config) + fingerprint = cache_fingerprint(config, split_episodes) + raw_train = RgbActionCacheDataset( + Path(config["paths"]["cache_dir"]), fingerprint, "train" + ) + fit_action_normalizer(model, raw_train) + train_dataset = RgbActionCacheDataset( + Path(config["paths"]["cache_dir"]), + fingerprint, + "train", + action_mean=model.action_normalizer.mean.detach().cpu(), + action_std=model.action_normalizer.std.detach().cpu(), + action_clip=model.action_normalizer.clip, + ) + dataloader = DataLoader( + train_dataset, + batch_size=int(config["data"]["batch_size"]), + sampler=FixedSeedPermutationSampler( + train_dataset, int(config["data"]["sample_order_seed"]) + ), + num_workers=int(config["data"].get("num_workers", 0)), + pin_memory=True, + drop_last=False, + collate_fn=collate_rgb_action, + ) + configure_trainable(model) + optimizer = build_optimizer(model, config) + max_steps = int(config["training"]["max_steps"]) + lr_scheduler = build_lr_scheduler( + optimizer, + max_steps, + int(config["training"]["warmup_steps"]), + schedule_type=str(config["training"]["lr_schedule"]), + ) + model, optimizer, dataloader, lr_scheduler = accelerator.prepare( + model, optimizer, dataloader, lr_scheduler + ) + video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=float(config["diffusion"]["train_shift"]), + ) + action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["action_num_train_timesteps"]), + shift=float(config["diffusion"]["action_train_shift"]), + ) + dtype = dtype_from_name(str(config["model"]["dtype"])) + global_step = 0 + last_exported_step: int | None = None + optimizer.zero_grad(set_to_none=True) + model.train() + while global_step < max_steps: + for batch in dataloader: + if global_step >= max_steps: + break + with accelerator.accumulate(model): + loss, metrics = compute_loss( + model, + batch, + video_scheduler, + action_scheduler, + config, + accelerator.device, + dtype, + ) + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_( + [p for p in model.parameters() if p.requires_grad], + float(config["training"]["max_grad_norm"]), + ) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + if not accelerator.sync_gradients: + continue + global_step += 1 + if global_step == 1 or global_step % int(config["training"]["log_every"]) == 0: + reduced = { + key: accelerator.reduce(value.float(), reduction="mean") + for key, value in metrics.items() + if key.startswith("loss_") + } + record = { + "global_step": global_step, + "loss_total": float( + accelerator.reduce(loss.detach().float(), reduction="mean").item() + ), + **{key: float(value.item()) for key, value in reduced.items()}, + "learning_rates": [float(group["lr"]) for group in optimizer.param_groups], + } + if accelerator.is_main_process: + append_jsonl(output_dir / "metrics.jsonl", record) + print(json.dumps(record, sort_keys=True), flush=True) + if global_step % int(config["training"]["save_every"]) == 0: + export_model( + model, + accelerator, + output_dir + / "checkpoints" + / f"rgb_state_joint_action_{global_step:06d}.pt", + ) + last_exported_step = global_step + if last_exported_step != global_step: + export_model( + model, + accelerator, + output_dir / "checkpoints" / f"rgb_state_joint_action_{global_step:06d}.pt", + ) + accelerator.end_training() + + +def main() -> None: + args = parse_args() + config = load_config(args.config) + if args.output_dir is not None: + config["experiment"]["output_dir"] = str(args.output_dir) + if args.max_steps is not None: + config["training"]["max_steps"] = args.max_steps + config = validate_config(config) + if args.dataset_smoke_test: + dataset_smoke_test(config) + return + accelerator = Accelerator() + if accelerator.is_main_process: + prepare_cache( + config, + device=torch.device(str(config["cache"].get("device", accelerator.device))), + overwrite=args.overwrite_cache, + ) + accelerator.wait_for_everyone() + accelerator.end_training() + if args.prepare_cache_only: + return + train(config, Path(config["experiment"]["output_dir"])) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_joint_rgb_depth.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_joint_rgb_depth.py new file mode 100644 index 0000000000000000000000000000000000000000..43a967bda1b950598989bd9b600c5d1ddfb0d204 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_joint_rgb_depth.py @@ -0,0 +1,1752 @@ +#!/usr/bin/env python3 +"""Train one shared Wan2.2 DiT to jointly denoise RGB and depth latents. + +The script has three explicit modes: + +1. ``--validate-only`` validates paths/configuration without loading weights. +2. ``--prepare-cache-only`` creates aligned RGB/depth Wan-VAE latents and a + frozen text embedding cache. +3. The default mode trains either ``projection_only`` or ``full_dit``. + +No model or dataset downloads are performed. All pretrained components are +loaded from ``paths.pretrained_model_dir``. +""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from functools import lru_cache +import gc +import hashlib +import importlib.util +import json +import math +from pathlib import Path +import sys +import time +from typing import Any, Iterator + +import cv2 +import numpy as np +from safetensors import safe_open +import torch +import torch.nn.functional as F +from torch.utils.data import DataLoader, Dataset, Sampler +import yaml + + +PROJECT_ROOT = Path(__file__).resolve().parent +FASTWAM_SRC = PROJECT_ROOT.parent / "model_old" / "src" +if str(FASTWAM_SRC) not in sys.path: + sys.path.insert(0, str(FASTWAM_SRC)) + +from accelerate import Accelerator +from accelerate.utils import DistributedType, set_seed +from fastwam.models.wan22.schedulers.scheduler_continuous import ( + WanContinuousFlowMatchScheduler, +) +from fastwam.models.wan22.wan_video_dit import ( + WanVideoDiT, + precompute_freqs_cis_3d, +) +from fastwam.models.wan22.wan_video_text_encoder import ( + HuggingfaceTokenizer, + WanTextEncoder, +) +from fastwam.models.wan22.wan_video_vae import WanVideoVAE38 + +from depth_preprocessing import DepthPreprocessingConfig, preprocess_depth_grayscale +from scripts.visualize_depth_preprocessing import read_selected_rgb +from wan_rgb_depth_initialization import ( + enable_depth_first_frame_timestep_adapter, + joint_rgb_depth_sampling_step, + split_joint_rgb_depth_prediction, + upgrade_wan_dit_for_rgb_depth, +) + + +PHASES = {"projection_only", "full_dit"} +PROMPT_SOURCES = {"fixed", "egoverse_task_description"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--phase", choices=sorted(PHASES)) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--resume-model", type=Path) + parser.add_argument("--resume-state", type=Path) + parser.add_argument("--validate-only", action="store_true") + parser.add_argument("--prepare-cache-only", action="store_true") + return parser.parse_args() + + +def load_config(path: Path) -> dict[str, Any]: + with path.open("r", encoding="utf-8") as handle: + config = yaml.safe_load(handle) + if not isinstance(config, dict): + raise ValueError(f"Config must contain a YAML mapping: {path}") + return config + + +def dtype_from_name(name: str) -> torch.dtype: + mapping = { + "float32": torch.float32, + "float16": torch.float16, + "bfloat16": torch.bfloat16, + } + if name not in mapping: + raise ValueError(f"Unsupported model.dtype={name!r}; expected {sorted(mapping)}") + return mapping[name] + + +def validate_config(config: dict[str, Any]) -> dict[str, Any]: + required_sections = { + "experiment", + "paths", + "model", + "data", + "conditioning", + "diffusion", + "training", + "validation", + "cache", + } + missing = sorted(required_sections - set(config)) + if missing: + raise ValueError(f"Missing config sections: {missing}") + + model = config["model"] + data = config["data"] + conditioning = config["conditioning"] + diffusion = config["diffusion"] + training = config["training"] + paths = config["paths"] + tracking = config.get("tracking", {"enabled": False}) + if bool(tracking.get("enabled", False)): + if tracking.get("backend") != "wandb": + raise ValueError("Only tracking.backend=wandb is implemented") + if not str(tracking.get("project", "")).strip(): + raise ValueError("tracking.project must be non-empty") + if tracking.get("mode") not in {"online", "offline", "disabled"}: + raise ValueError("tracking.mode must be online, offline, or disabled") + tags = tracking.get("tags", []) + if not isinstance(tags, list) or not all(isinstance(tag, str) for tag in tags): + raise ValueError("tracking.tags must be a list of strings") + for optional_key in ("entity", "group", "run_id"): + value = tracking.get(optional_key) + if value is not None and not isinstance(value, str): + raise ValueError(f"tracking.{optional_key} must be a string or null") + if bool(tracking.get("log_model_artifact", False)): + raise ValueError( + "Model artifact upload is intentionally disabled for the 5B checkpoint" + ) + + if model.get("joint_rgb_depth") is not True: + raise ValueError("This entrypoint requires model.joint_rgb_depth=true") + if model.get("share_rgb_depth_vae") is not True: + raise ValueError("This experiment requires one shared RGB/depth VAE") + if model.get("freeze_vae") is not True: + raise ValueError("The shared Wan VAE must remain frozen") + if model.get("freeze_text_encoder") is not True: + raise ValueError("The text encoder must remain frozen") + if int(model["rgb_channels"]) != 48 or int(model["depth_channels"]) != 48: + raise ValueError("Wan2.2-TI2V-5B requires 48 RGB and 48 depth channels") + if int(model["dit"]["in_dim"]) != 48 or int(model["dit"]["out_dim"]) != 48: + raise ValueError("The pretrained RGB-only DiT config must remain 48/48") + if list(model["dit"]["patch_size"]) != [1, 2, 2]: + raise ValueError("Expected Wan2.2-TI2V-5B patch_size [1,2,2]") + dtype_from_name(str(model["dtype"])) + + num_frames = int(data["num_frames"]) + if num_frames % 4 != 1: + raise ValueError(f"data.num_frames must satisfy T % 4 == 1, got {num_frames}") + resolution = int(data["resolution"]) + if resolution <= 0 or resolution % 16: + raise ValueError("data.resolution must be a positive multiple of 16") + num_clips = int(data["num_clips"]) + num_episodes = int(data.get("num_episodes", num_clips)) + clips_per_episode = int(data.get("clips_per_episode", 1)) + if num_clips <= 0: + raise ValueError("data.num_clips must be a positive integer") + if num_episodes <= 0 or clips_per_episode <= 0: + raise ValueError("data.num_episodes and clips_per_episode must be positive") + if num_clips != num_episodes * clips_per_episode: + raise ValueError( + "data.num_clips must equal num_episodes * clips_per_episode" + ) + if clips_per_episode > 1 and data["window_selection"] != "uniform_nonoverlap": + raise ValueError( + "Multiple clips per episode require window_selection=uniform_nonoverlap" + ) + if int(config["validation"]["final_num_samples"]) > num_clips: + raise ValueError("validation.final_num_samples cannot exceed data.num_clips") + config["_num_episodes"] = num_episodes + config["_clips_per_episode"] = clips_per_episode + config["_num_samples"] = num_clips + if int(data["batch_size"]) <= 0: + raise ValueError("data.batch_size must be a positive integer") + if data.get("shuffle") is not False: + raise ValueError( + "Use data.sample_order=fixed_seed_permutation with data.shuffle=false" + ) + if data.get("sample_order") != "fixed_seed_permutation": + raise ValueError( + "The schedule ablation requires data.sample_order=" + "fixed_seed_permutation" + ) + if not isinstance(data.get("sample_order_seed"), int): + raise ValueError("data.sample_order_seed must be an integer") + if data.get("random_crop") is not False: + raise ValueError("The implemented deterministic pipeline requires random_crop=false") + if data.get("horizontal_flip") is not False: + raise ValueError( + "The implemented deterministic pipeline requires horizontal_flip=false" + ) + if float(data["target_fps"]) <= 0: + raise ValueError("data.target_fps must be positive") + depth_config = data["depth"] + if depth_config["representation"] != "linear": + raise ValueError("Only linear depth representation is implemented") + if depth_config["resize_interpolation"] != "nearest": + raise ValueError("Depth resize_interpolation must be nearest") + + post_test = config.get("post_training_test", {"enabled": False}) + if bool(post_test.get("enabled", False)): + sample_indices = post_test.get("sample_indices") + if not isinstance(sample_indices, list) or len(sample_indices) != 20: + raise ValueError("post_training_test.sample_indices must contain 20 indices") + if len(set(sample_indices)) != len(sample_indices): + raise ValueError("post-training sample indices must be unique") + if any( + not isinstance(index, int) or not 0 <= index < num_clips + for index in sample_indices + ): + raise ValueError("post-training sample index outside data.num_clips") + if int(post_test["fps"]) != int(round(float(data["target_fps"]))): + raise ValueError("post-training FPS must match data.target_fps") + hf_config = post_test.get("huggingface", {}) + if bool(hf_config.get("enabled", False)): + if hf_config.get("repo_type") != "dataset": + raise ValueError("Hugging Face upload repo_type must be dataset") + if not hf_config.get("repo_id") and not hf_config.get("repo_id_env"): + raise ValueError("Hugging Face upload requires repo_id or repo_id_env") + if not isinstance(hf_config.get("private"), bool): + raise ValueError("Hugging Face private must be a boolean") + + timestep_mode = str( + conditioning.get("depth_first_frame_timestep_mode", "wan_zero") + ) + if timestep_mode not in {"wan_zero", "depth_adapter"}: + raise ValueError( + "conditioning.depth_first_frame_timestep_mode must be " + "wan_zero or depth_adapter" + ) + adapter_bottleneck = int( + conditioning.get("depth_timestep_adapter_bottleneck_dim", 256) + ) + if timestep_mode == "depth_adapter" and adapter_bottleneck <= 0: + raise ValueError("depth timestep adapter bottleneck must be positive") + + mode = conditioning["depth_condition_mode"] + if mode not in {"none", "first_frame_oracle"}: + raise ValueError( + "Implemented depth_condition_mode values are none and " + f"first_frame_oracle, got {mode!r}" + ) + if mode == "first_frame_oracle": + for key in ( + "fix_rgb_first_latent", + "fix_depth_first_latent", + "exclude_rgb_condition_from_loss", + "exclude_depth_condition_from_loss", + ): + if conditioning.get(key) is not True: + raise ValueError(f"Oracle anchor requires conditioning.{key}=true") + else: + if conditioning.get("fix_rgb_first_latent") is not True: + raise ValueError("RGB-only conditioning requires fix_rgb_first_latent=true") + if conditioning.get("exclude_rgb_condition_from_loss") is not True: + raise ValueError( + "RGB-only conditioning requires exclude_rgb_condition_from_loss=true" + ) + if conditioning.get("fix_depth_first_latent") is not False: + raise ValueError( + "depth_condition_mode=none requires fix_depth_first_latent=false" + ) + if conditioning.get("exclude_depth_condition_from_loss") is not False: + raise ValueError( + "depth_condition_mode=none requires " + "exclude_depth_condition_from_loss=false" + ) + + if diffusion["rgb_depth_noise_mode"] != "independent": + raise ValueError("The first ablation implements independent RGB/depth noise") + if float(training["weight_decay"]) != 0.0: + raise ValueError( + "weight_decay must be 0 because gradient masks protect original " + "slices but decoupled weight decay would still modify them" + ) + for phase in PHASES: + if phase not in training: + raise ValueError(f"Missing training.{phase}") + phase_config = training[phase] + lr_schedule = str(phase_config.get("lr_schedule", "cosine")) + if lr_schedule not in {"cosine", "constant"}: + raise ValueError( + f"training.{phase}.lr_schedule must be cosine or constant, " + f"got {lr_schedule!r}" + ) + max_steps = int(phase_config["max_steps"]) + warmup_steps = int(phase_config["warmup_steps"]) + if max_steps <= 0: + raise ValueError(f"training.{phase}.max_steps must be positive") + if not 0 <= warmup_steps < max_steps: + raise ValueError( + f"training.{phase}.warmup_steps must be in [0, max_steps)" + ) + if timestep_mode == "depth_adapter" and float( + phase_config.get("depth_adapter_lr", 0.0) + ) <= 0: + raise ValueError( + f"training.{phase}.depth_adapter_lr must be positive" + ) + + model_dir = Path(paths["pretrained_model_dir"]) + required_model_files = [ + model_dir / "config.json", + model_dir / "diffusion_pytorch_model.safetensors.index.json", + model_dir / "Wan2.2_VAE.pth", + model_dir / "models_t5_umt5-xxl-enc-bf16.pth", + model_dir / "google" / "umt5-xxl" / "tokenizer.json", + ] + missing_files = [str(path) for path in required_model_files if not path.is_file()] + if missing_files: + raise FileNotFoundError(f"Missing local pretrained files: {missing_files}") + + annotation_dir = Path(paths["annotation_dir"]) + annotations = sorted(annotation_dir.glob(str(paths["annotation_glob"]))) + expected_episodes = config["_num_episodes"] + if len(annotations) < expected_episodes: + raise FileNotFoundError( + f"Expected at least {expected_episodes} annotations in {annotation_dir}, " + f"found {len(annotations)}" + ) + config["_resolved_annotations"] = [ + str(path) for path in annotations[:expected_episodes] + ] + prompt_source = str(data.get("prompt_source", "fixed")) + if prompt_source not in PROMPT_SOURCES: + raise ValueError( + f"Unsupported data.prompt_source={prompt_source!r}; " + f"expected one of {sorted(PROMPT_SOURCES)}" + ) + if prompt_source == "fixed": + prompt = str(data.get("prompt", "")).strip() + if not prompt: + raise ValueError("data.prompt must be non-empty for prompt_source=fixed") + config["_resolved_prompts"] = [prompt] * expected_episodes + else: + config["_resolved_prompts"] = [ + read_egoverse_task_description(Path(path)) + for path in config["_resolved_annotations"] + ] + return config + + +@lru_cache(maxsize=None) +def read_egoverse_task_description(annotation_path: Path) -> str: + """Resolve the source EgoVerse task description stored by an episode.""" + + with np.load(annotation_path, allow_pickle=False) as payload: + if "metadata_json" not in payload: + raise KeyError(f"Missing metadata_json in {annotation_path}") + metadata = json.loads(payload["metadata_json"].item()) + source_metadata = metadata.get("episode", {}).get("source_metadata_path") + if not source_metadata: + raise KeyError( + f"Missing episode.source_metadata_path in {annotation_path}" + ) + source_metadata_path = Path(source_metadata) + if not source_metadata_path.is_file(): + raise FileNotFoundError( + f"Missing EgoVerse source metadata for {annotation_path}: " + f"{source_metadata_path}" + ) + with source_metadata_path.open("r", encoding="utf-8") as handle: + source = json.load(handle) + prompt = str(source.get("attributes", {}).get("task_description", "")).strip() + if not prompt: + raise ValueError( + f"Missing attributes.task_description in {source_metadata_path}" + ) + return prompt + + +@contextmanager +def temporary_default_dtype(dtype: torch.dtype) -> Iterator[None]: + previous = torch.get_default_dtype() + torch.set_default_dtype(dtype) + try: + yield + finally: + torch.set_default_dtype(previous) + + +def unwrap_checkpoint_mapping(payload: Any) -> dict[str, torch.Tensor]: + if not isinstance(payload, dict): + raise TypeError(f"Expected checkpoint dict, got {type(payload).__name__}") + while len(payload) == 1: + key = next(iter(payload)) + if key not in {"state_dict", "model_state", "module"}: + break + payload = payload[key] + if not isinstance(payload, dict): + raise TypeError(f"Checkpoint field {key!r} must contain a dict") + return payload + + +def strict_load_original_dit( + model: WanVideoDiT, + model_dir: Path, +) -> None: + """Load sharded local safetensors with exact key and shape validation.""" + + index_path = model_dir / "diffusion_pytorch_model.safetensors.index.json" + with index_path.open("r", encoding="utf-8") as handle: + index = json.load(handle) + weight_map = index.get("weight_map") + if not isinstance(weight_map, dict): + raise ValueError(f"Invalid safetensors index: {index_path}") + + target_state = model.state_dict(keep_vars=True) + expected_keys = set(target_state) + checkpoint_keys = set(weight_map) + if checkpoint_keys != expected_keys: + missing = sorted(expected_keys - checkpoint_keys) + unexpected = sorted(checkpoint_keys - expected_keys) + raise RuntimeError( + "Original DiT checkpoint key mismatch: " + f"missing={missing}, unexpected={unexpected}" + ) + + loaded: set[str] = set() + shard_names = sorted(set(weight_map.values())) + load_started = time.monotonic() + for shard_index, shard_name in enumerate(shard_names, start=1): + shard_path = model_dir / shard_name + shard_started = time.monotonic() + shard_gib = shard_path.stat().st_size / (1024 ** 3) + print( + f"[model-load] shard {shard_index}/{len(shard_names)} " + f"start: {shard_name} ({shard_gib:.2f} GiB)", + flush=True, + ) + if not shard_path.is_file(): + raise FileNotFoundError(shard_path) + with safe_open(shard_path, framework="pt", device="cpu") as shard: + for key in shard.keys(): + if weight_map.get(key) != shard_name: + raise RuntimeError(f"Index/shard disagreement for {key}") + source = shard.get_tensor(key) + target = target_state[key] + if tuple(source.shape) != tuple(target.shape): + raise RuntimeError( + f"Shape mismatch for {key}: checkpoint={tuple(source.shape)}, " + f"model={tuple(target.shape)}" + ) + with torch.no_grad(): + target.copy_(source.to(dtype=target.dtype)) + loaded.add(key) + print( + f"[model-load] shard {shard_index}/{len(shard_names)} " + f"complete in {time.monotonic() - shard_started:.1f}s", + flush=True, + ) + if loaded != expected_keys: + raise RuntimeError(f"Did not load all DiT keys: {sorted(expected_keys - loaded)}") + print( + f"[model-load] all DiT shards strict-loaded in " + f"{time.monotonic() - load_started:.1f}s", + flush=True, + ) + + +def build_joint_dit(config: dict[str, Any], resume_model: Path | None) -> WanVideoDiT: + model_config = config["model"] + dtype = dtype_from_name(str(model_config["dtype"])) + dit_config = dict(model_config["dit"]) + allocation_started = time.monotonic() + with torch.device("meta"), temporary_default_dtype(dtype): + dit = WanVideoDiT(**dit_config) + dit.to_empty(device="cpu") + dit.freqs = precompute_freqs_cis_3d(int(dit_config["attn_head_dim"])) + if any(parameter.is_meta for parameter in dit.parameters()): + raise RuntimeError("WanVideoDiT still contains meta parameters after to_empty") + print( + f"[model-load] allocated uninitialized WanVideoDiT on CPU in " + f"{time.monotonic() - allocation_started:.1f}s; skipped random initialization", + flush=True, + ) + + if resume_model is None: + strict_load_original_dit( + dit, + Path(config["paths"]["pretrained_model_dir"]), + ) + dit = upgrade_wan_dit_for_rgb_depth( + dit, + rgb_channels=int(model_config["rgb_channels"]), + depth_channels=int(model_config["depth_channels"]), + ) + print("[model-load] RGB-depth projections expanded", flush=True) + if config["conditioning"].get( + "depth_first_frame_timestep_mode", "wan_zero" + ) == "depth_adapter": + dit = enable_depth_first_frame_timestep_adapter( + dit, + bottleneck_dim=int( + config["conditioning"]["depth_timestep_adapter_bottleneck_dim"] + ), + ) + print( + "[model-load] depth first-frame timestep adapter enabled", + flush=True, + ) + + if resume_model is not None: + payload = torch.load( + resume_model, + map_location="cpu", + weights_only=True, + mmap=True, + ) + if not isinstance(payload, dict) or "state_dict" not in payload: + raise ValueError(f"Invalid joint model checkpoint: {resume_model}") + metadata = payload.get("metadata", {}) + expected_metadata = joint_checkpoint_metadata(dit) + if metadata != expected_metadata: + raise RuntimeError( + f"Joint checkpoint metadata mismatch: {metadata} != {expected_metadata}" + ) + incompatible = dit.load_state_dict(payload["state_dict"], strict=True) + if incompatible.missing_keys or incompatible.unexpected_keys: + raise RuntimeError( + f"Strict joint load mismatch: {incompatible}" + ) + return dit + + +def load_text_encoder_and_tokenizer( + model_dir: Path, + dtype: torch.dtype, + device: torch.device, + tokenizer_max_length: int, +) -> tuple[WanTextEncoder, HuggingfaceTokenizer]: + with temporary_default_dtype(dtype): + text_encoder = WanTextEncoder() + payload = torch.load( + model_dir / "models_t5_umt5-xxl-enc-bf16.pth", + map_location="cpu", + weights_only=True, + mmap=True, + ) + state = unwrap_checkpoint_mapping(payload) + incompatible = text_encoder.load_state_dict(state, strict=True) + if incompatible.missing_keys or incompatible.unexpected_keys: + raise RuntimeError(f"Strict text encoder load mismatch: {incompatible}") + del payload, state + text_encoder = text_encoder.to(device=device, dtype=dtype).eval().requires_grad_(False) + tokenizer = HuggingfaceTokenizer( + name=model_dir / "google" / "umt5-xxl", + seq_len=tokenizer_max_length, + clean="whitespace", + local_files_only=True, + ) + return text_encoder, tokenizer + + +def load_vae( + model_dir: Path, + dtype: torch.dtype, + device: torch.device, +) -> WanVideoVAE38: + with temporary_default_dtype(dtype): + vae = WanVideoVAE38() + payload = torch.load( + model_dir / "Wan2.2_VAE.pth", + map_location="cpu", + weights_only=True, + mmap=True, + ) + raw_state = unwrap_checkpoint_mapping(payload) + state = {f"model.{key}": value for key, value in raw_state.items()} + incompatible = vae.load_state_dict(state, strict=True) + if incompatible.missing_keys or incompatible.unexpected_keys: + raise RuntimeError(f"Strict VAE load mismatch: {incompatible}") + del payload, raw_state, state + return vae.to(device=device, dtype=dtype).eval().requires_grad_(False) + + +def config_fingerprint(config: dict[str, Any]) -> str: + selected = { + "annotations": config["_resolved_annotations"], + "prompts": config["_resolved_prompts"], + "data": config["data"], + "depth_condition_mode": config["conditioning"]["depth_condition_mode"], + "model_dir": config["paths"]["pretrained_model_dir"], + } + encoded = json.dumps(selected, sort_keys=True).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def choose_positions( + frame_count: int, + num_frames: int, + stride: int, + selection: str, + *, + clip_index: int = 0, + clips_per_episode: int = 1, +) -> np.ndarray: + span = (num_frames - 1) * stride + 1 + if span > frame_count: + raise ValueError( + f"Requested {num_frames} frames at stride {stride} from {frame_count}" + ) + if selection == "center": + start = (frame_count - span) // 2 + elif selection == "start": + start = 0 + elif selection == "uniform_nonoverlap": + if not 0 <= clip_index < clips_per_episode: + raise ValueError( + f"clip_index={clip_index} outside clips_per_episode={clips_per_episode}" + ) + available_start = frame_count - span + starts = [ + ((index + 1) * available_start) // (clips_per_episode + 1) + for index in range(clips_per_episode) + ] + if any( + right - left < span + for left, right in zip(starts, starts[1:], strict=False) + ): + raise ValueError( + f"Cannot place {clips_per_episode} non-overlapping windows of span " + f"{span} in {frame_count} frames" + ) + start = starts[clip_index] + else: + raise ValueError(f"Unsupported window_selection={selection!r}") + return start + np.arange(num_frames, dtype=np.int64) * stride + + +def center_crop_and_resize( + rgb_frames: list[np.ndarray], + depth_frames: np.ndarray, + resolution: int, +) -> tuple[np.ndarray, np.ndarray]: + rgb = np.stack(rgb_frames) + if rgb.ndim != 4 or depth_frames.ndim != 3: + raise ValueError(f"Unexpected RGB/depth shapes: {rgb.shape}, {depth_frames.shape}") + if rgb.shape[0] != depth_frames.shape[0] or rgb.shape[1:3] != depth_frames.shape[1:]: + raise ValueError(f"Unaligned RGB/depth shapes: {rgb.shape}, {depth_frames.shape}") + + height, width = depth_frames.shape[1:] + crop_size = min(height, width) + top = (height - crop_size) // 2 + left = (width - crop_size) // 2 + rgb = rgb[:, top : top + crop_size, left : left + crop_size] + depth = depth_frames[:, top : top + crop_size, left : left + crop_size] + + rgb_resized = np.stack( + [ + cv2.resize(frame, (resolution, resolution), interpolation=cv2.INTER_AREA) + for frame in rgb + ] + ) + depth_resized = np.stack( + [ + cv2.resize(frame, (resolution, resolution), interpolation=cv2.INTER_NEAREST) + for frame in depth + ] + ) + return rgb_resized, depth_resized + + +def read_aligned_clip( + annotation_path: Path, + config: dict[str, Any], + *, + clip_index: int = 0, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, np.ndarray]: + data_config = config["data"] + with np.load(annotation_path, allow_pickle=False) as bundle: + frame_count = int(bundle["frame_index"].shape[0]) + positions = choose_positions( + frame_count, + int(data_config["num_frames"]), + int(data_config["depth_frame_stride"]), + str(data_config["window_selection"]), + clip_index=clip_index, + clips_per_episode=int(config["_clips_per_episode"]), + ) + depth_mm = bundle["depth_mm"][positions] + source_indices = bundle["frame_index"][positions] + rectified_k = bundle["K"] + metadata = json.loads(bundle["metadata_json"].item()) + + metadata_target_fps = float(metadata["target_fps"]) + configured_target_fps = float(data_config["target_fps"]) + if not math.isclose(metadata_target_fps, configured_target_fps): + raise ValueError( + f"Target FPS mismatch for {annotation_path}: metadata=" + f"{metadata_target_fps}, config={configured_target_fps}" + ) + output_size = (int(depth_mm.shape[2]), int(depth_mm.shape[1])) + rgb_frames = read_selected_rgb( + Path(metadata["episode"]["video_path"]), + source_indices, + output_size, + metadata, + rectified_k, + ) + rgb_uint8, depth_mm = center_crop_and_resize( + rgb_frames, + depth_mm, + int(data_config["resolution"]), + ) + depth_config = data_config["depth"] + processed = preprocess_depth_grayscale( + depth_mm, + config=DepthPreprocessingConfig( + depth_min=float(depth_config["min"]), + depth_max=float(depth_config["max"]), + depth_invalid_value=float(depth_config["invalid_value"]), + depth_invalid_fill=str(depth_config["invalid_fill"]), + ), + input_unit=str(depth_config["input_unit"]), + ) + + rgb = torch.from_numpy( + rgb_uint8.astype(np.float32) / np.float32(127.5) - np.float32(1.0) + ).permute(3, 0, 1, 2) + depth = torch.from_numpy(processed.vae_video).permute(1, 0, 2, 3) + valid_mask = torch.from_numpy(processed.valid_mask) + return rgb, depth, valid_mask, source_indices + + +def prepare_cache(config: dict[str, Any]) -> None: + cache_dir = Path(config["paths"]["cache_dir"]) + manifest_path = cache_dir / "manifest.json" + fingerprint = config_fingerprint(config) + if manifest_path.is_file() and not bool(config["cache"]["overwrite"]): + with manifest_path.open("r", encoding="utf-8") as handle: + existing = json.load(handle) + if existing.get("fingerprint") != fingerprint: + raise RuntimeError( + f"Existing cache fingerprint mismatch: {manifest_path}. " + "Use a new cache_dir or set cache.overwrite=true." + ) + print(f"[cache] reuse {manifest_path}", flush=True) + return + + if not torch.cuda.is_available() and str(config["cache"]["device"]).startswith("cuda"): + raise RuntimeError("Cache preparation requested CUDA but CUDA is unavailable") + device = torch.device(str(config["cache"]["device"])) + dtype = dtype_from_name(str(config["model"]["dtype"])) + model_dir = Path(config["paths"]["pretrained_model_dir"]) + cache_dir.mkdir(parents=True, exist_ok=True) + + print("[cache] encoding per-episode prompts", flush=True) + text_encoder, tokenizer = load_text_encoder_and_tokenizer( + model_dir, + dtype, + device, + int(config["cache"]["tokenizer_max_length"]), + ) + prompts = list(config["_resolved_prompts"]) + prompt_batch_size = int(config["cache"].get("prompt_batch_size", 8)) + if prompt_batch_size <= 0: + raise ValueError("cache.prompt_batch_size must be positive") + prompt_cache: list[dict[str, Any]] = [] + for start in range(0, len(prompts), prompt_batch_size): + batch_prompts = prompts[start : start + prompt_batch_size] + ids, mask = tokenizer( + batch_prompts, + return_mask=True, + add_special_tokens=True, + ) + with torch.inference_mode(): + context = text_encoder( + ids.to(device), + mask.to(device=device, dtype=torch.bool), + ) + for local_index, prompt in enumerate(batch_prompts): + prompt_cache.append( + { + "context": context[local_index].detach().cpu(), + "context_mask": mask[local_index].to(dtype=torch.bool).cpu(), + "prompt": prompt, + } + ) + del ids, mask, context + torch.save( + {"entries": prompt_cache}, + cache_dir / "prompt_contexts.pt", + ) + text_encoder.to("cpu") + del text_encoder, tokenizer + gc.collect() + if device.type == "cuda": + torch.cuda.empty_cache() + + print("[cache] loading frozen Wan VAE", flush=True) + vae = load_vae(model_dir, dtype, device) + sample_entries: list[dict[str, Any]] = [] + expected_latent_t = (int(config["data"]["num_frames"]) - 1) // 4 + 1 + expected_latent_hw = int(config["data"]["resolution"]) // 16 + + clips_per_episode = int(config["_clips_per_episode"]) + sample_number = 0 + for episode_index, annotation_string in enumerate( + config["_resolved_annotations"] + ): + annotation_path = Path(annotation_string) + for clip_index in range(clips_per_episode): + start_time = time.time() + sample_id = ( + annotation_path.stem + if clips_per_episode == 1 + else f"{annotation_path.stem}_clip_{clip_index:02d}" + ) + rgb, depth, valid_mask, source_indices = read_aligned_clip( + annotation_path, + config, + clip_index=clip_index, + ) + with torch.inference_mode(): + z_rgb = vae.encode( + rgb.unsqueeze(0).to(device=device, dtype=dtype), + device=device, + ) + z_depth = vae.encode( + depth.unsqueeze(0).to(device=device, dtype=dtype), + device=device, + ) + expected_shape = ( + 1, 48, expected_latent_t, expected_latent_hw, expected_latent_hw + ) + if ( + tuple(z_rgb.shape) != expected_shape + or tuple(z_depth.shape) != expected_shape + ): + raise RuntimeError( + f"Unexpected latent shapes for {sample_id}: " + f"{tuple(z_rgb.shape)}, {tuple(z_depth.shape)}, " + f"expected {expected_shape}" + ) + + cache_name = f"{sample_number:03d}_{sample_id}.pt" + torch.save( + { + "sample_id": sample_id, + "annotation_path": str(annotation_path), + "clip_index": clip_index, + "source_indices": torch.from_numpy(source_indices.copy()), + "z_rgb": z_rgb[0].detach().cpu(), + "z_depth": z_depth[0].detach().cpu(), + "rgb_uint8": ((rgb + 1.0) * 127.5) + .round() + .clamp(0, 255) + .to(torch.uint8), + "depth_gray": ((depth[:1] + 1.0) * 0.5).to(torch.float16), + "depth_valid_mask": valid_mask, + }, + cache_dir / cache_name, + ) + sample_entries.append( + {"sample_id": sample_id, "file": cache_name, "prompt_index": episode_index} + ) + sample_number += 1 + print( + f"[cache {sample_number:03d}/{config['_num_samples']:03d}] " + f"{sample_id} {time.time() - start_time:.1f}s", + flush=True, + ) + + vae.to("cpu") + del vae + gc.collect() + if device.type == "cuda": + torch.cuda.empty_cache() + + manifest = { + "format_version": 1, + "fingerprint": fingerprint, + "num_episodes": int(config["_num_episodes"]), + "clips_per_episode": clips_per_episode, + "num_samples": len(sample_entries), + "latent_shape": [48, expected_latent_t, expected_latent_hw, expected_latent_hw], + "samples": sample_entries, + } + with manifest_path.open("w", encoding="utf-8") as handle: + json.dump(manifest, handle, indent=2) + print(f"[cache] wrote {manifest_path}", flush=True) + + +class CachedRgbDepthLatents(Dataset): + def __init__(self, cache_dir: Path, expected_fingerprint: str): + self.cache_dir = cache_dir + with (cache_dir / "manifest.json").open("r", encoding="utf-8") as handle: + self.manifest = json.load(handle) + if self.manifest.get("fingerprint") != expected_fingerprint: + raise RuntimeError("Latent cache does not match current config") + self.samples = list(self.manifest["samples"]) + prompt_payload = torch.load( + cache_dir / "prompt_contexts.pt", + map_location="cpu", + weights_only=True, + ) + self.prompt_cache = list(prompt_payload["entries"]) + + def __len__(self) -> int: + return len(self.samples) + + def __getitem__(self, index: int) -> dict[str, Any]: + entry = self.samples[index] + prompt = self.prompt_cache[int(entry["prompt_index"])] + sample = torch.load( + self.cache_dir / entry["file"], + map_location="cpu", + weights_only=True, + ) + return { + "sample_id": sample["sample_id"], + "z_rgb": sample["z_rgb"], + "z_depth": sample["z_depth"], + "context": prompt["context"], + "context_mask": prompt["context_mask"], + "prompt": prompt["prompt"], + } + + +class FixedSeedPermutationSampler(Sampler[int]): + """Replay one fixed random permutation on every epoch and process.""" + + def __init__(self, data_source: Dataset, seed: int): + generator = torch.Generator() + generator.manual_seed(int(seed)) + self.order = torch.randperm(len(data_source), generator=generator).tolist() + + def __iter__(self) -> Iterator[int]: + return iter(self.order) + + def __len__(self) -> int: + return len(self.order) + + +class ProjectionGradientMasks: + """Protect pretrained projection slices while training appended depth slices.""" + + def __init__( + self, + model: WanVideoDiT, + *, + phase: str, + head_warmup_steps: int, + preserve_original_slices: bool, + ): + self.phase = phase + self.phase_step = 0 + self.head_warmup_steps = int(head_warmup_steps) + self.handles: list[Any] = [] + def adapter_hook(gradient: torch.Tensor) -> torch.Tensor: + if ( + self.phase == "projection_only" + and self.phase_step < self.head_warmup_steps + ): + return torch.zeros_like(gradient) + return gradient + + adapter = getattr(model, "depth_first_frame_timestep_adapter", None) + if adapter is not None: + for parameter in adapter.parameters(): + self.handles.append(parameter.register_hook(adapter_hook)) + if not preserve_original_slices: + return + + rgb_channels = int(model.rgb_latent_channels) + joint_channels = rgb_channels + int(model.depth_latent_channels) + patch_mask = torch.zeros_like(model.patch_embedding.weight) + patch_mask[:, rgb_channels:].fill_(1) + + patch_volume = math.prod(model.patch_size) + head_weight_mask = torch.zeros_like(model.head.head.weight) + head_weight_mask.view( + patch_volume, + joint_channels, + model.head.head.in_features, + )[:, rgb_channels:].fill_(1) + head_bias_mask = torch.zeros_like(model.head.head.bias) + head_bias_mask.view(patch_volume, joint_channels)[:, rgb_channels:].fill_(1) + + def patch_hook(gradient: torch.Tensor) -> torch.Tensor: + if self.phase == "projection_only" and self.phase_step < self.head_warmup_steps: + return torch.zeros_like(gradient) + return gradient * patch_mask.to( + device=gradient.device, dtype=gradient.dtype + ) + + self.handles.append(model.patch_embedding.weight.register_hook(patch_hook)) + self.handles.append( + model.head.head.weight.register_hook( + lambda gradient: gradient + * head_weight_mask.to( + device=gradient.device, dtype=gradient.dtype + ) + ) + ) + self.handles.append( + model.head.head.bias.register_hook( + lambda gradient: gradient + * head_bias_mask.to( + device=gradient.device, dtype=gradient.dtype + ) + ) + ) + + def set_step(self, phase_step: int) -> None: + self.phase_step = int(phase_step) + + def close(self) -> None: + for handle in self.handles: + handle.remove() + self.handles.clear() + + +def configure_trainable_parameters( + model: WanVideoDiT, + phase: str, +) -> None: + if phase == "projection_only": + model.requires_grad_(False) + model.patch_embedding.weight.requires_grad_(True) + model.head.head.weight.requires_grad_(True) + model.head.head.bias.requires_grad_(True) + adapter = getattr(model, "depth_first_frame_timestep_adapter", None) + if adapter is not None: + adapter.requires_grad_(True) + elif phase == "full_dit": + model.requires_grad_(True) + # These tensors have no modality-specific slice and remain pretrained. + model.patch_embedding.bias.requires_grad_(False) + model.head.modulation.requires_grad_(False) + else: + raise ValueError(f"Unsupported phase: {phase}") + + +def build_optimizer( + model: WanVideoDiT, + config: dict[str, Any], + phase: str, +) -> torch.optim.Optimizer: + phase_config = config["training"][phase] + projection_parameters = { + id(model.patch_embedding.weight), + id(model.head.head.weight), + id(model.head.head.bias), + } + adapter_module = getattr( + model, + "depth_first_frame_timestep_adapter", + None, + ) + adapter = list(adapter_module.parameters()) if adapter_module is not None else [] + projection_parameters.update(id(parameter) for parameter in adapter) + patch = [model.patch_embedding.weight] + head = [model.head.head.weight, model.head.head.bias] + groups: list[dict[str, Any]] = [ + {"params": patch, "lr": float(phase_config["depth_patch_lr"])}, + {"params": head, "lr": float(phase_config["depth_head_lr"])}, + ] + if adapter: + groups.append( + {"params": adapter, "lr": float(phase_config["depth_adapter_lr"])} + ) + if phase == "full_dit": + dit_parameters = [ + parameter + for parameter in model.parameters() + if parameter.requires_grad and id(parameter) not in projection_parameters + ] + groups.append({"params": dit_parameters, "lr": float(phase_config["dit_lr"])}) + return torch.optim.AdamW( + groups, + betas=(0.9, 0.95), + eps=1.0e-8, + weight_decay=float(config["training"]["weight_decay"]), + ) + + +def build_lr_scheduler( + optimizer: torch.optim.Optimizer, + max_steps: int, + warmup_steps: int, + schedule_type: str = "cosine", +) -> torch.optim.lr_scheduler.LambdaLR: + if schedule_type not in {"cosine", "constant"}: + raise ValueError(f"Unsupported LR schedule: {schedule_type!r}") + + def scale(step: int) -> float: + if warmup_steps > 0 and step < warmup_steps: + return max(float(step + 1) / float(warmup_steps), 1.0e-8) + if schedule_type == "constant": + return 1.0 + progress = (step - warmup_steps) / max(max_steps - warmup_steps, 1) + progress = min(max(progress, 0.0), 1.0) + return 0.5 * (1.0 + math.cos(math.pi * progress)) + + return torch.optim.lr_scheduler.LambdaLR(optimizer, scale) + + +def joint_flow_matching_loss( + model: torch.nn.Module, + scheduler: WanContinuousFlowMatchScheduler, + z_rgb: torch.Tensor, + z_depth: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + config: dict[str, Any], +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + batch_size = z_rgb.shape[0] + timestep = scheduler.sample_training_t( + batch_size, + device=z_rgb.device, + dtype=z_rgb.dtype, + ) + noise_rgb = torch.randn_like(z_rgb) + noise_depth = torch.randn_like(z_depth) + noisy_rgb = scheduler.add_noise(z_rgb, noise_rgb, timestep) + noisy_depth = scheduler.add_noise(z_depth, noise_depth, timestep) + target_rgb = scheduler.training_target(z_rgb, noise_rgb, timestep) + target_depth = scheduler.training_target(z_depth, noise_depth, timestep) + + conditioning = config["conditioning"] + if conditioning["fix_rgb_first_latent"]: + noisy_rgb[:, :, 0:1] = z_rgb[:, :, 0:1] + if conditioning["fix_depth_first_latent"]: + noisy_depth[:, :, 0:1] = z_depth[:, :, 0:1] + + joint_input = torch.cat((noisy_rgb, noisy_depth), dim=1) + joint_prediction = model( + joint_input, + timestep, + context, + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + pred_rgb, pred_depth = split_joint_rgb_depth_prediction(joint_prediction) + + if conditioning["exclude_rgb_condition_from_loss"]: + pred_rgb_loss = pred_rgb[:, :, 1:] + target_rgb_loss = target_rgb[:, :, 1:] + else: + pred_rgb_loss, target_rgb_loss = pred_rgb, target_rgb + if conditioning["exclude_depth_condition_from_loss"]: + pred_depth_loss = pred_depth[:, :, 1:] + target_depth_loss = target_depth[:, :, 1:] + else: + pred_depth_loss, target_depth_loss = pred_depth, target_depth + + loss_rgb_per_sample = F.mse_loss( + pred_rgb_loss.float(), + target_rgb_loss.float(), + reduction="none", + ).mean(dim=(1, 2, 3, 4)) + loss_depth_per_sample = F.mse_loss( + pred_depth_loss.float(), + target_depth_loss.float(), + reduction="none", + ).mean(dim=(1, 2, 3, 4)) + loss_depth_frame0_per_sample = F.mse_loss( + pred_depth[:, :, 0:1].float(), + target_depth[:, :, 0:1].float(), + reduction="none", + ).mean(dim=(1, 2, 3, 4)) + if pred_depth.shape[2] > 1: + loss_depth_future_per_sample = F.mse_loss( + pred_depth[:, :, 1:].float(), + target_depth[:, :, 1:].float(), + reduction="none", + ).mean(dim=(1, 2, 3, 4)) + else: + loss_depth_future_per_sample = torch.zeros_like( + loss_depth_frame0_per_sample + ) + sample_weight = scheduler.training_weight(timestep).to( + device=z_rgb.device, + dtype=torch.float32, + ) + loss_rgb = (loss_rgb_per_sample * sample_weight).mean() + loss_depth = (loss_depth_per_sample * sample_weight).mean() + loss_depth_frame0 = ( + loss_depth_frame0_per_sample * sample_weight + ).mean() + loss_depth_future = ( + loss_depth_future_per_sample * sample_weight + ).mean() + total = ( + float(config["diffusion"]["lambda_rgb"]) * loss_rgb + + float(config["diffusion"]["lambda_depth"]) * loss_depth + ) + return total, { + "loss_rgb": loss_rgb.detach(), + "loss_depth": loss_depth.detach(), + "loss_depth_unweighted": loss_depth_per_sample.mean().detach(), + "loss_depth_frame0": loss_depth_frame0.detach(), + "loss_depth_future": loss_depth_future.detach(), + "timestep_mean": timestep.detach().float().mean(), + "sample_weight_mean": sample_weight.detach().mean(), + "prediction_rgb_norm": pred_rgb.detach().float().norm(), + "prediction_depth_norm": pred_depth.detach().float().norm(), + } + + +def sample_joint_latents( + model: torch.nn.Module, + z_rgb: torch.Tensor, + z_depth: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + config: dict[str, Any], + seed: int, +) -> tuple[torch.Tensor, torch.Tensor]: + infer_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=float(config["diffusion"]["infer_shift"]), + ) + generator = torch.Generator(device=z_rgb.device).manual_seed(seed) + noisy_rgb = torch.randn( + z_rgb.shape, + generator=generator, + device=z_rgb.device, + dtype=z_rgb.dtype, + ) + noisy_depth = torch.randn( + z_depth.shape, + generator=generator, + device=z_depth.device, + dtype=z_depth.dtype, + ) + rgb_condition = z_rgb[:, :, 0:1] + depth_condition = ( + z_depth[:, :, 0:1] + if config["conditioning"]["depth_condition_mode"] == "first_frame_oracle" + else None + ) + noisy_rgb[:, :, 0:1] = rgb_condition + if depth_condition is not None: + noisy_depth[:, :, 0:1] = depth_condition + joint_latents = torch.cat((noisy_rgb, noisy_depth), dim=1) + + timesteps, deltas = infer_scheduler.build_inference_schedule( + int(config["diffusion"]["inference_steps"]), + device=z_rgb.device, + dtype=z_rgb.dtype, + ) + for step_t, step_delta in zip(timesteps, deltas, strict=True): + timestep = step_t.expand(z_rgb.shape[0]) + joint_latents, _, _ = joint_rgb_depth_sampling_step( + model, + infer_scheduler, + joint_latents, + timestep, + step_delta, + context, + context_mask=context_mask, + rgb_condition_latent=rgb_condition, + depth_condition_latent=depth_condition, + ) + return split_joint_rgb_depth_prediction(joint_latents) + + +@torch.no_grad() +def run_sampling_validation( + model: torch.nn.Module, + dataset: CachedRgbDepthLatents, + accelerator: Accelerator, + config: dict[str, Any], + output_dir: Path, + step: int, + num_samples: int, +) -> dict[str, float]: + model.eval() + world_size = accelerator.num_processes + rank = accelerator.process_index + num_samples = min(int(num_samples), len(dataset)) + calls_per_rank = math.ceil(num_samples / world_size) + local_rgb_sum = torch.zeros((), device=accelerator.device) + local_depth_sum = torch.zeros((), device=accelerator.device) + local_count = torch.zeros((), device=accelerator.device) + validation_dir = output_dir / "validation" / f"step_{step:06d}" + validation_dir.mkdir(parents=True, exist_ok=True) + + model_dtype = dtype_from_name(str(config["model"]["dtype"])) + + for call_index in range(calls_per_rank): + sample_index = rank + call_index * world_size + is_valid = sample_index < num_samples + actual_index = sample_index if is_valid else 0 + sample = dataset[actual_index] + z_rgb = sample["z_rgb"].unsqueeze(0).to( + device=accelerator.device, + dtype=model_dtype, + ) + z_depth = sample["z_depth"].unsqueeze(0).to( + device=accelerator.device, + dtype=model_dtype, + ) + context = sample["context"].unsqueeze(0).to( + device=accelerator.device, + dtype=model_dtype, + ) + context_mask = sample["context_mask"].unsqueeze(0).to( + device=accelerator.device, + dtype=torch.bool, + ) + generated_rgb, generated_depth = sample_joint_latents( + model, + z_rgb, + z_depth, + context, + context_mask, + config, + seed=int(config["validation"]["fixed_noise_seed"]) + actual_index, + ) + rgb_mse = F.mse_loss(generated_rgb[:, :, 1:].float(), z_rgb[:, :, 1:].float()) + depth_start = 1 if config["conditioning"]["exclude_depth_condition_from_loss"] else 0 + depth_mse = F.mse_loss( + generated_depth[:, :, depth_start:].float(), + z_depth[:, :, depth_start:].float(), + ) + if is_valid: + local_rgb_sum += rgb_mse + local_depth_sum += depth_mse + local_count += 1 + if config["validation"]["save_latents"]: + torch.save( + { + "sample_id": sample["sample_id"], + "generated_rgb": generated_rgb[0].detach().cpu(), + "generated_depth": generated_depth[0].detach().cpu(), + "target_rgb": z_rgb[0].detach().cpu(), + "target_depth": z_depth[0].detach().cpu(), + }, + validation_dir / f"{actual_index:02d}_{sample['sample_id']}.pt", + ) + + totals = torch.stack((local_rgb_sum, local_depth_sum, local_count)) + totals = accelerator.reduce(totals, reduction="sum") + model.train() + count = max(float(totals[2].item()), 1.0) + return { + "validation_rgb_latent_mse": float(totals[0].item()) / count, + "validation_depth_latent_mse": float(totals[1].item()) / count, + } + + +def joint_checkpoint_metadata(model: WanVideoDiT) -> dict[str, Any]: + metadata = { + "format_version": 1, + "model_class": type(model).__name__, + "joint_rgb_depth": True, + "rgb_latent_channels": int(model.rgb_latent_channels), + "depth_latent_channels": int(model.depth_latent_channels), + "joint_input_channels": int(model.in_dim), + "patch_size": list(model.patch_size), + "patch_embedding_in_channels": int(model.patch_embedding.in_channels), + "output_head_out_features": int(model.head.head.out_features), + } + if hasattr(model, "depth_first_frame_timestep_adapter"): + metadata["depth_first_frame_timestep_mode"] = "adapter" + metadata["depth_timestep_adapter_bottleneck_dim"] = int( + model.depth_first_frame_timestep_adapter_bottleneck_dim + ) + return metadata + + +def export_joint_model( + model: torch.nn.Module, + accelerator: Accelerator, + path: Path, +) -> None: + fsdp_plugin = None + previous_state_dict_type = None + if accelerator.distributed_type == DistributedType.FSDP: + fsdp_plugin = accelerator.state.fsdp_plugin + previous_state_dict_type = fsdp_plugin.state_dict_type + fsdp_plugin.set_state_dict_type("FULL_STATE_DICT") + state_dict = accelerator.get_state_dict(model) + if accelerator.is_main_process: + unwrapped = accelerator.unwrap_model(model) + payload = { + "metadata": joint_checkpoint_metadata(unwrapped), + "state_dict": {key: value.cpu() for key, value in state_dict.items()}, + } + path.parent.mkdir(parents=True, exist_ok=True) + accelerator.save(payload, path) + print(f"[checkpoint] exported {path}", flush=True) + accelerator.wait_for_everyone() + if fsdp_plugin is not None: + fsdp_plugin.set_state_dict_type(previous_state_dict_type) + + +def append_jsonl(path: Path, record: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + + +def initialize_experiment_trackers( + accelerator: Accelerator, + config: dict[str, Any], + phase: str, +) -> None: + tracking = config.get("tracking", {"enabled": False}) + if not bool(tracking.get("enabled", False)): + return + wandb_kwargs: dict[str, Any] = { + "name": f"{config['experiment']['name']}-{phase}", + "job_type": phase, + "mode": str(tracking["mode"]), + "tags": list(tracking.get("tags", [])), + } + for key in ("entity", "group"): + value = tracking.get(key) + if value: + wandb_kwargs[key] = value + run_id = tracking.get("run_id") + if run_id: + wandb_kwargs["id"] = str(run_id) + wandb_kwargs["resume"] = "allow" + public_config = { + key: value for key, value in config.items() if not key.startswith("_") + } + public_config["runtime_phase"] = phase + accelerator.init_trackers( + project_name=str(tracking["project"]), + config=public_config, + init_kwargs={"wandb": wandb_kwargs}, + ) + + +def train( + config: dict[str, Any], + phase: str, + output_dir: Path, + resume_model: Path | None, + resume_state: Path | None, +) -> None: + tracking_enabled = bool(config.get("tracking", {}).get("enabled", False)) + if tracking_enabled and importlib.util.find_spec("wandb") is None: + raise RuntimeError( + "tracking.enabled=true but the wandb package is not installed. " + "Activate the training environment containing wandb; this script " + "does not install dependencies automatically." + ) + accelerator = Accelerator( + gradient_accumulation_steps=int( + config["training"]["gradient_accumulation_steps"] + ), + mixed_precision="bf16" if config["model"]["dtype"] == "bfloat16" else "no", + log_with="wandb" if tracking_enabled else None, + project_dir=str(output_dir), + ) + set_seed(int(config["experiment"]["seed"]), device_specific=True) + if accelerator.is_main_process: + output_dir.mkdir(parents=True, exist_ok=True) + with (output_dir / "resolved_config.yaml").open("w", encoding="utf-8") as handle: + yaml.safe_dump( + {key: value for key, value in config.items() if not key.startswith("_")}, + handle, + sort_keys=False, + ) + accelerator.wait_for_everyone() + initialize_experiment_trackers(accelerator, config, phase) + + dataset = CachedRgbDepthLatents( + Path(config["paths"]["cache_dir"]), + config_fingerprint(config), + ) + sampler = FixedSeedPermutationSampler( + dataset, + seed=int(config["data"]["sample_order_seed"]), + ) + dataloader = DataLoader( + dataset, + batch_size=int(config["data"]["batch_size"]), + shuffle=False, + sampler=sampler, + num_workers=int(config["data"]["num_workers"]), + pin_memory=True, + drop_last=False, + ) + + if accelerator.is_main_process: + print("[model] strict-loading original Wan2.2 5B and expanding RGB-depth", flush=True) + model = build_joint_dit(config, resume_model) + configure_trainable_parameters(model, phase) + phase_config = config["training"][phase] + mask_controller = ProjectionGradientMasks( + model, + phase=phase, + head_warmup_steps=int(phase_config.get("head_warmup_steps", 0)), + preserve_original_slices=bool( + config["training"]["preserve_original_projection_slices"] + ), + ) + optimizer = build_optimizer(model, config, phase) + max_steps = int(phase_config["max_steps"]) + lr_scheduler = build_lr_scheduler( + optimizer, + max_steps, + int(phase_config["warmup_steps"]), + schedule_type=str(phase_config.get("lr_schedule", "cosine")), + ) + if accelerator.is_main_process: + print("[model] moving model and optimizer through accelerator.prepare", flush=True) + model, optimizer, dataloader, lr_scheduler = accelerator.prepare( + model, + optimizer, + dataloader, + lr_scheduler, + ) + + if accelerator.is_main_process: + print("[model] accelerator.prepare complete; entering training loop", flush=True) + start_step = 0 + if resume_state is not None: + accelerator.load_state(str(resume_state)) + state_path = resume_state / "trainer_state.json" + with state_path.open("r", encoding="utf-8") as handle: + start_step = int(json.load(handle)["global_step"]) + mask_controller.set_step(start_step) + + train_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=float(config["diffusion"]["train_shift"]), + ) + model_dtype = dtype_from_name(str(config["model"]["dtype"])) + global_step = start_step + epoch = 0 + optimizer.zero_grad(set_to_none=True) + log_path = output_dir / "metrics.jsonl" + model.train() + + while global_step < max_steps: + epoch += 1 + for batch in dataloader: + if global_step >= max_steps: + break + with accelerator.accumulate(model): + z_rgb = batch["z_rgb"].to( + device=accelerator.device, + dtype=model_dtype, + non_blocking=True, + ) + z_depth = batch["z_depth"].to( + device=accelerator.device, + dtype=model_dtype, + non_blocking=True, + ) + context = batch["context"].to( + device=accelerator.device, + dtype=model_dtype, + non_blocking=True, + ) + context_mask = batch["context_mask"].to( + device=accelerator.device, + dtype=torch.bool, + non_blocking=True, + ) + + total_loss, metrics = joint_flow_matching_loss( + model, + train_scheduler, + z_rgb, + z_depth, + context, + context_mask, + config, + ) + accelerator.backward(total_loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_( + [parameter for parameter in model.parameters() if parameter.requires_grad], + float(config["training"]["max_grad_norm"]), + ) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + + if not accelerator.sync_gradients: + continue + global_step += 1 + mask_controller.set_step(global_step) + reduced = { + key: accelerator.reduce(value.float(), reduction="mean") + for key, value in metrics.items() + } + reduced["loss_total"] = accelerator.reduce( + total_loss.detach().float(), reduction="mean" + ) + should_log = ( + global_step == 1 + or global_step % int(config["training"]["log_every"]) == 0 + ) + if should_log: + learning_rates = [ + float(group["lr"]) for group in optimizer.param_groups + ] + record = { + "phase": phase, + "epoch": epoch, + "global_step": global_step, + **{key: float(value.item()) for key, value in reduced.items()}, + "learning_rates": learning_rates, + } + if tracking_enabled: + wandb_metrics = { + f"train/{key}": float(value.item()) + for key, value in reduced.items() + } + wandb_metrics["train/epoch"] = epoch + wandb_metrics.update( + { + f"train/lr_group_{index}": learning_rate + for index, learning_rate in enumerate(learning_rates) + } + ) + accelerator.log(wandb_metrics, step=global_step) + if accelerator.is_main_process: + append_jsonl(log_path, record) + print(json.dumps(record, sort_keys=True), flush=True) + + if ( + global_step < max_steps + and global_step % int(config["training"]["validate_every"]) == 0 + ): + validation_metrics = run_sampling_validation( + model, + dataset, + accelerator, + config, + output_dir, + global_step, + int(config["validation"]["periodic_num_samples"]), + ) + if tracking_enabled: + accelerator.log( + { + f"validation/{key}": value + for key, value in validation_metrics.items() + }, + step=global_step, + ) + if accelerator.is_main_process: + append_jsonl( + log_path, + {"phase": phase, "global_step": global_step, **validation_metrics}, + ) + print(json.dumps(validation_metrics, sort_keys=True), flush=True) + + if global_step % int(config["training"]["save_every"]) == 0: + state_dir = output_dir / "checkpoints" / f"state_{global_step:06d}" + accelerator.save_state(str(state_dir)) + if accelerator.is_main_process: + with (state_dir / "trainer_state.json").open( + "w", encoding="utf-8" + ) as handle: + json.dump( + {"phase": phase, "global_step": global_step}, + handle, + indent=2, + ) + + final_metrics = run_sampling_validation( + model, + dataset, + accelerator, + config, + output_dir, + global_step, + int(config["validation"]["final_num_samples"]), + ) + if tracking_enabled: + accelerator.log( + {f"final/{key}": value for key, value in final_metrics.items()}, + step=global_step, + ) + if accelerator.is_main_process: + append_jsonl( + log_path, + {"phase": phase, "global_step": global_step, **final_metrics}, + ) + export_joint_model( + model, + accelerator, + output_dir / "checkpoints" / f"joint_model_{phase}_{global_step:06d}.pt", + ) + mask_controller.close() + accelerator.end_training() + + +def main() -> None: + args = parse_args() + config = validate_config(load_config(args.config)) + if args.validate_only: + print( + json.dumps( + { + "status": "valid", + "episodes": len(config["_resolved_annotations"]), + "clips_per_episode": config["_clips_per_episode"], + "samples": config["_num_samples"], + "cache_dir": config["paths"]["cache_dir"], + "model_dir": config["paths"]["pretrained_model_dir"], + "tracking_enabled": bool(config.get("tracking", {}).get("enabled", False)), + "wandb_installed": importlib.util.find_spec("wandb") is not None, + }, + indent=2, + ) + ) + return + if args.prepare_cache_only: + prepare_cache(config) + return + if args.phase is None: + raise ValueError("--phase is required for training") + cache_manifest = Path(config["paths"]["cache_dir"]) / "manifest.json" + if not cache_manifest.is_file(): + raise FileNotFoundError( + f"Missing latent cache {cache_manifest}. Run --prepare-cache-only first." + ) + output_dir = ( + args.output_dir + if args.output_dir is not None + else Path(config["experiment"]["output_dir"]) / args.phase + ) + train( + config, + args.phase, + output_dir, + args.resume_model, + args.resume_state, + ) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_joint_rgb_depth_point.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_joint_rgb_depth_point.py new file mode 100644 index 0000000000000000000000000000000000000000..35b098758bc80a21fe06f5d21c9c37ebd33e9d49 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_joint_rgb_depth_point.py @@ -0,0 +1,998 @@ +#!/usr/bin/env python3 +"""Train one Wan DiT to jointly diffuse RGB, depth, and world-XYZ point videos.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +import os +from pathlib import Path +from typing import Any, Iterator + +import torch +import torch.nn.functional as F +from torch.utils.data import DataLoader, Dataset, Sampler, default_collate +import yaml +from accelerate import Accelerator +from accelerate.utils import DistributedType, set_seed + +from train_joint_rgb_depth import ( + WanContinuousFlowMatchScheduler, + WanVideoDiT, + append_jsonl, + build_lr_scheduler, + config_fingerprint, + dtype_from_name, + initialize_experiment_trackers, + load_config, + precompute_freqs_cis_3d, + strict_load_original_dit, + temporary_default_dtype, + validate_config as validate_rgb_depth_config, +) +from wan_rgb_depth_point_initialization import ( + enable_point_first_frame_timestep_adapter, + joint_rgb_depth_point_sampling_step, + split_joint_rgb_depth_point_prediction, + upgrade_wan_dit_for_rgb_depth_point, +) + +PHASES = {"projection_only", "full_dit"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--phase", choices=sorted(PHASES)) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--resume-model", type=Path) + parser.add_argument("--resume-state", type=Path) + parser.add_argument("--validate-only", action="store_true") + parser.add_argument("--prepare-cache-only", action="store_true") + return parser.parse_args() + + +def point_config_fingerprint(config: dict[str, Any]) -> str: + selected = { + "rgb_depth_fingerprint": config_fingerprint(config), + "point_cloud": config["point_cloud"], + "point_channels": int(config["model"]["point_channels"]), + } + return hashlib.sha256( + json.dumps(selected, sort_keys=True).encode("utf-8") + ).hexdigest() + + +def validate_config(config: dict[str, Any]) -> dict[str, Any]: + config = validate_rgb_depth_config(config) + model = config["model"] + conditioning = config["conditioning"] + diffusion = config["diffusion"] + point = config.get("point_cloud") + if model.get("joint_rgb_depth_point") is not True: + raise ValueError("This entrypoint requires model.joint_rgb_depth_point=true") + if int(model.get("point_channels", 0)) != 48: + raise ValueError("Wan point-map VAE latent must contain 48 channels") + if point is None or point.get("enabled") is not True: + raise ValueError("point_cloud.enabled must be true") + if point.get("representation") != "dense_first_camera_xyz_point_map_video": + raise ValueError("Only dense_first_camera_xyz_point_map_video is implemented") + if point.get("vae") != "shared_frozen_wan_vae": + raise ValueError("The first point experiment requires the frozen shared Wan VAE") + for key in ("xyz_min_m", "xyz_max_m", "invalid_fill_reference_m"): + value = point.get(key) + if not isinstance(value, list) or len(value) != 3: + raise ValueError(f"point_cloud.{key} must contain 3 values") + low = torch.tensor(point["xyz_min_m"], dtype=torch.float32) + high = torch.tensor(point["xyz_max_m"], dtype=torch.float32) + fill = torch.tensor(point["invalid_fill_reference_m"], dtype=torch.float32) + if not torch.all(high > low) or not torch.all((fill >= low) & (fill <= high)): + raise ValueError("Invalid point-cloud XYZ bounds/fill") + if point.get("coordinate_frame") != "first_camera_frame": + raise ValueError("Point maps must be aligned to the first camera frame") + if point.get("pose_vector_layout") != "txyz_qwxyz": + raise ValueError("Egoverse point experiment requires txyz_qwxyz pose layout") + if point.get("pose_is") != "camera_to_world": + raise ValueError("Egoverse point experiment requires camera_to_world pose") + extrinsic = torch.tensor(point.get("camera_to_head_transform"), dtype=torch.float32) + if tuple(extrinsic.shape) != (4, 4) or not torch.isfinite(extrinsic).all(): + raise ValueError("camera_to_head_transform must be finite [4,4]") + if conditioning.get("depth_condition_mode") != "first_frame_oracle": + raise ValueError( + "This sibling requires RGB and depth clean frame-zero conditions" + ) + if conditioning.get("depth_first_frame_timestep_mode") != "wan_zero": + raise ValueError( + "Clean depth frame zero uses Wan timestep zero and no depth adapter" + ) + if conditioning.get("point_condition_mode") != "none": + raise ValueError("First experiment uses point_condition_mode=none") + if conditioning.get("fix_point_first_latent") is not False: + raise ValueError("Unconditioned point frame zero must not be fixed") + if conditioning.get("exclude_point_condition_from_loss") is not False: + raise ValueError("Unconditioned point frame zero must remain in point loss") + if int(conditioning.get("point_timestep_adapter_bottleneck_dim", 0)) <= 0: + raise ValueError("point timestep adapter bottleneck must be positive") + if diffusion.get("rgb_depth_point_noise_mode") != "independent": + raise ValueError("First triple experiment requires independent modality noise") + if float(diffusion.get("lambda_point", 0.0)) <= 0: + raise ValueError("diffusion.lambda_point must be positive") + for phase in PHASES: + if float(config["training"][phase].get("point_adapter_lr", 0.0)) <= 0: + raise ValueError(f"training.{phase}.point_adapter_lr must be positive") + export_model_every = int( + config["training"].get( + "export_model_every", config["training"]["save_every"] + ) + ) + if export_model_every <= 0: + raise ValueError("training.export_model_every must be positive") + config["training"]["export_model_every"] = export_model_every + source_cache = Path(config["paths"].get("rgb_depth_cache_dir", "")) + if not (source_cache / "manifest.json").is_file(): + raise FileNotFoundError( + f"Missing reusable RGB-depth cache manifest: {source_cache / 'manifest.json'}" + ) + return config + + +def joint_checkpoint_metadata(model: WanVideoDiT) -> dict[str, Any]: + return { + "format_version": 3, + "model_class": type(model).__name__, + "joint_rgb_depth_point": True, + "rgb_latent_channels": int(model.rgb_latent_channels), + "depth_latent_channels": int(model.depth_latent_channels), + "point_latent_channels": int(model.point_latent_channels), + "joint_input_channels": int(model.in_dim), + "patch_size": list(model.patch_size), + "patch_embedding_in_channels": int(model.patch_embedding.in_channels), + "output_head_out_features": int(model.head.head.out_features), + "rgb_frame0_conditioned": True, + "depth_frame0_conditioned": True, + "point_frame0_conditioned": False, + "point_timestep_adapter_bottleneck_dim": int( + model.point_first_frame_timestep_adapter_bottleneck_dim + ), + } + + +def build_joint_dit( + config: dict[str, Any], + resume_model: Path | None, + *, + load_pretrained: bool = True, +) -> WanVideoDiT: + model_config = config["model"] + dtype = dtype_from_name(str(model_config["dtype"])) + load_weights_on_this_rank = int(os.environ.get("LOCAL_RANK", "0")) == 0 + dit_config = dict(model_config["dit"]) + with torch.device("meta"), temporary_default_dtype(dtype): + dit = WanVideoDiT(**dit_config) + dit.to_empty(device="cpu") + dit.freqs = precompute_freqs_cis_3d(int(dit_config["attn_head_dim"])) + if any(parameter.is_meta for parameter in dit.parameters()): + raise RuntimeError("WanVideoDiT still contains meta parameters") + if resume_model is None and load_pretrained and load_weights_on_this_rank: + dit_model_dir = Path( + os.environ.get( + "WAN_DIT_MODEL_DIR", + config["paths"]["pretrained_model_dir"], + ) + ) + print(f"[model-load] DiT checkpoint directory: {dit_model_dir}", flush=True) + strict_load_original_dit(dit, dit_model_dir) + dit = upgrade_wan_dit_for_rgb_depth_point( + dit, + rgb_channels=int(model_config["rgb_channels"]), + depth_channels=int(model_config["depth_channels"]), + point_channels=int(model_config["point_channels"]), + ) + dit = enable_point_first_frame_timestep_adapter( + dit, + point_bottleneck_dim=int( + config["conditioning"]["point_timestep_adapter_bottleneck_dim"] + ), + ) + print( + "[model-load] RGB-depth-point projections and point-frame0 adapter enabled", + flush=True, + ) + if resume_model is not None and load_weights_on_this_rank: + payload = torch.load( + resume_model, map_location="cpu", weights_only=True, mmap=True + ) + if not isinstance(payload, dict) or "state_dict" not in payload: + raise ValueError(f"Invalid triple checkpoint: {resume_model}") + expected = joint_checkpoint_metadata(dit) + if payload.get("metadata") != expected: + raise RuntimeError( + f"Triple checkpoint metadata mismatch: {payload.get('metadata')} != {expected}" + ) + incompatible = dit.load_state_dict(payload["state_dict"], strict=True) + if incompatible.missing_keys or incompatible.unexpected_keys: + raise RuntimeError(f"Strict triple checkpoint mismatch: {incompatible}") + if not load_weights_on_this_rank: + print( + "[model-load] nonzero local rank awaits FSDP module-state sync", + flush=True, + ) + return dit + + +class CachedRgbDepthPointLatents(Dataset): + """RAM-resident training latents with valid-prefix text contexts.""" + + def __init__(self, cache_dir: Path, expected_fingerprint: str): + self.cache_dir = cache_dir + self.manifest = json.loads( + (cache_dir / "manifest.json").read_text(encoding="utf-8") + ) + if self.manifest.get("fingerprint") != expected_fingerprint: + raise RuntimeError("RGB-depth-point latent cache does not match config") + if self.manifest.get("modalities") != ["rgb", "depth", "world_point"]: + raise RuntimeError("Cache does not declare all three modalities") + self.samples = list(self.manifest["samples"]) + prompt_payload = torch.load( + cache_dir / "prompt_contexts.pt", map_location="cpu", weights_only=True + ) + self.prompt_cache: list[dict[str, Any]] = [] + for prompt in prompt_payload["entries"]: + context_mask = prompt["context_mask"].to(dtype=torch.bool) + valid_positions = context_mask.nonzero(as_tuple=False) + if valid_positions.numel() == 0: + raise ValueError("Cached prompt contains no valid text tokens") + context_length = int(valid_positions[-1, 0]) + 1 + self.prompt_cache.append( + { + "context": prompt["context"][:context_length].clone(), + "context_mask": context_mask[:context_length].clone(), + "prompt": prompt["prompt"], + } + ) + del prompt_payload + + self.latent_cache: list[dict[str, Any]] = [] + for entry in self.samples: + sample = torch.load( + self.cache_dir / entry["file"], + map_location="cpu", + weights_only=True, + ) + cached = { + "sample_id": sample["sample_id"], + "z_rgb": sample["z_rgb"], + "z_depth": sample["z_depth"], + "z_point": sample["z_point"], + } + for key in ("annotation_path", "source_indices"): + if key in sample: + cached[key] = sample[key] + self.latent_cache.append(cached) + + def __len__(self) -> int: + return len(self.samples) + + def __getitem__(self, index: int) -> dict[str, Any]: + entry = self.samples[index] + prompt = self.prompt_cache[int(entry["prompt_index"])] + sample = self.latent_cache[index] + output = { + "sample_id": sample["sample_id"], + "z_rgb": sample["z_rgb"], + "z_depth": sample["z_depth"], + "z_point": sample["z_point"], + "context": prompt["context"], + "context_mask": prompt["context_mask"], + "prompt": prompt["prompt"], + } + for key in ("annotation_path", "source_indices"): + if key in sample: + output[key] = sample[key] + return output + + +def collate_rgb_depth_point_batch( + samples: list[dict[str, Any]], +) -> dict[str, Any]: + """Pad text only to the longest valid prompt in the current batch.""" + + if not samples: + raise ValueError("Cannot collate an empty RGB-depth-point batch") + context_length = 0 + for sample in samples: + valid_positions = sample["context_mask"].nonzero(as_tuple=False) + if valid_positions.numel() == 0: + raise ValueError("Sample contains no valid text tokens") + context_length = max(context_length, int(valid_positions[-1, 0]) + 1) + trimmed_samples = [] + for sample in samples: + trimmed = dict(sample) + context = sample["context"] + context_mask = sample["context_mask"] + padding = context_length - int(context.shape[0]) + if padding > 0: + context = F.pad(context, (0, 0, 0, padding)) + context_mask = F.pad(context_mask, (0, padding), value=False) + trimmed["context"] = context + trimmed["context_mask"] = context_mask + trimmed_samples.append(trimmed) + return default_collate(trimmed_samples) + + +class FixedSeedPermutationSampler(Sampler[int]): + def __init__(self, data_source: Dataset, seed: int): + generator = torch.Generator().manual_seed(int(seed)) + self.order = torch.randperm(len(data_source), generator=generator).tolist() + + def __iter__(self) -> Iterator[int]: + return iter(self.order) + + def __len__(self) -> int: + return len(self.order) + + +class ProjectionGradientMasks: + """Preserve original RGB/TI2V projection slices; train depth+point slices.""" + + def __init__( + self, + model: WanVideoDiT, + *, + phase: str, + head_warmup_steps: int, + preserve_original_slices: bool, + ) -> None: + self.phase = phase + self.phase_step = 0 + self.head_warmup_steps = int(head_warmup_steps) + self.handles: list[Any] = [] + + def warmup_hook(gradient: torch.Tensor) -> torch.Tensor: + if phase == "projection_only" and self.phase_step < self.head_warmup_steps: + return torch.zeros_like(gradient) + return gradient + + for parameter in model.point_first_frame_timestep_adapter.parameters(): + self.handles.append(parameter.register_hook(warmup_hook)) + if not preserve_original_slices: + return + rgb = int(model.rgb_latent_channels) + joint = rgb + int(model.depth_latent_channels) + int(model.point_latent_channels) + model.register_buffer( + "_rgb_depth_point_projection_patch_mask", + torch.zeros_like(model.patch_embedding.weight), + persistent=False, + ) + model._rgb_depth_point_projection_patch_mask[:, rgb:].fill_(1) + patch_volume = math.prod(model.patch_size) + model.register_buffer( + "_rgb_depth_point_projection_head_weight_mask", + torch.zeros_like(model.head.head.weight), + persistent=False, + ) + model._rgb_depth_point_projection_head_weight_mask.view( + patch_volume, joint, model.head.head.in_features + )[:, rgb:].fill_(1) + model.register_buffer( + "_rgb_depth_point_projection_head_bias_mask", + torch.zeros_like(model.head.head.bias), + persistent=False, + ) + model._rgb_depth_point_projection_head_bias_mask.view( + patch_volume, joint + )[:, rgb:].fill_(1) + + def patch_hook(gradient: torch.Tensor) -> torch.Tensor: + if phase == "projection_only" and self.phase_step < self.head_warmup_steps: + return torch.zeros_like(gradient) + return gradient * model._rgb_depth_point_projection_patch_mask + + self.handles.append(model.patch_embedding.weight.register_hook(patch_hook)) + self.handles.append( + model.head.head.weight.register_hook( + lambda gradient: gradient + * model._rgb_depth_point_projection_head_weight_mask + ) + ) + self.handles.append( + model.head.head.bias.register_hook( + lambda gradient: gradient + * model._rgb_depth_point_projection_head_bias_mask + ) + ) + + def set_step(self, phase_step: int) -> None: + self.phase_step = int(phase_step) + + def close(self) -> None: + for handle in self.handles: + handle.remove() + self.handles.clear() + + +def configure_trainable_parameters(model: WanVideoDiT, phase: str) -> None: + if phase == "projection_only": + model.requires_grad_(False) + model.patch_embedding.weight.requires_grad_(True) + model.head.head.weight.requires_grad_(True) + model.head.head.bias.requires_grad_(True) + model.point_first_frame_timestep_adapter.requires_grad_(True) + elif phase == "full_dit": + model.requires_grad_(True) + model.patch_embedding.bias.requires_grad_(False) + model.head.modulation.requires_grad_(False) + else: + raise ValueError(f"Unsupported phase: {phase}") + + +def build_optimizer( + model: WanVideoDiT, config: dict[str, Any], phase: str +) -> torch.optim.Optimizer: + phase_config = config["training"][phase] + point_adapter = list(model.point_first_frame_timestep_adapter.parameters()) + explicit = { + id(model.patch_embedding.weight), + id(model.head.head.weight), + id(model.head.head.bias), + *(id(parameter) for parameter in point_adapter), + } + groups: list[dict[str, Any]] = [ + { + "params": [model.patch_embedding.weight], + "lr": float(phase_config["depth_patch_lr"]), + }, + { + "params": [model.head.head.weight, model.head.head.bias], + "lr": float(phase_config["depth_head_lr"]), + }, + { + "params": point_adapter, + "lr": float(phase_config["point_adapter_lr"]), + }, + ] + if phase == "full_dit": + groups.append( + { + "params": [ + parameter + for parameter in model.parameters() + if parameter.requires_grad and id(parameter) not in explicit + ], + "lr": float(phase_config["dit_lr"]), + } + ) + return torch.optim.AdamW( + groups, + betas=(0.9, 0.95), + eps=1.0e-8, + weight_decay=float(config["training"]["weight_decay"]), + ) + + +def _mse_per_sample(prediction: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + return F.mse_loss( + prediction.float(), target.float(), reduction="none" + ).mean(dim=(1, 2, 3, 4)) + + +def joint_flow_matching_loss( + model: torch.nn.Module, + scheduler: WanContinuousFlowMatchScheduler, + z_rgb: torch.Tensor, + z_depth: torch.Tensor, + z_point: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + config: dict[str, Any], + compute_diagnostics: bool = True, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + batch = z_rgb.shape[0] + timestep = scheduler.sample_training_t(batch, z_rgb.device, z_rgb.dtype) + clean = (z_rgb, z_depth, z_point) + noise = tuple(torch.randn_like(value) for value in clean) + noisy = [ + scheduler.add_noise(value, modality_noise, timestep) + for value, modality_noise in zip(clean, noise, strict=True) + ] + targets = tuple( + scheduler.training_target(value, modality_noise, timestep) + for value, modality_noise in zip(clean, noise, strict=True) + ) + conditioning = config["conditioning"] + if conditioning["fix_rgb_first_latent"]: + noisy[0][:, :, 0:1] = z_rgb[:, :, 0:1] + if conditioning["fix_depth_first_latent"]: + noisy[1][:, :, 0:1] = z_depth[:, :, 0:1] + + prediction = model( + torch.cat(noisy, dim=1), + timestep, + context, + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + pred_rgb, pred_depth, pred_point = split_joint_rgb_depth_point_prediction( + prediction + ) + target_rgb, target_depth, target_point = targets + + rgb_start = 1 if conditioning["exclude_rgb_condition_from_loss"] else 0 + depth_start = 1 if conditioning["exclude_depth_condition_from_loss"] else 0 + point_start = 1 if conditioning["exclude_point_condition_from_loss"] else 0 + rgb_per_sample = _mse_per_sample( + pred_rgb[:, :, rgb_start:], target_rgb[:, :, rgb_start:] + ) + depth_per_sample = _mse_per_sample( + pred_depth[:, :, depth_start:], target_depth[:, :, depth_start:] + ) + point_per_sample = _mse_per_sample( + pred_point[:, :, point_start:], target_point[:, :, point_start:] + ) + weight = scheduler.training_weight(timestep).to(z_rgb.device, torch.float32) + loss_rgb = (rgb_per_sample * weight).mean() + loss_depth = (depth_per_sample * weight).mean() + loss_point = (point_per_sample * weight).mean() + total = ( + float(config["diffusion"]["lambda_rgb"]) * loss_rgb + + float(config["diffusion"]["lambda_depth"]) * loss_depth + + float(config["diffusion"]["lambda_point"]) * loss_point + ) + if not compute_diagnostics: + return total, {} + + depth_frame0 = _mse_per_sample( + pred_depth[:, :, 0:1], target_depth[:, :, 0:1] + ) + point_frame0 = _mse_per_sample( + pred_point[:, :, 0:1], target_point[:, :, 0:1] + ) + depth_future = _mse_per_sample( + pred_depth[:, :, 1:], target_depth[:, :, 1:] + ) + point_future = _mse_per_sample( + pred_point[:, :, 1:], target_point[:, :, 1:] + ) + return total, { + "loss_rgb": loss_rgb.detach(), + "loss_depth": loss_depth.detach(), + "loss_point": loss_point.detach(), + "loss_depth_unweighted": depth_per_sample.mean().detach(), + "loss_point_unweighted": point_per_sample.mean().detach(), + "loss_depth_frame0": (depth_frame0 * weight).mean().detach(), + "loss_depth_future": (depth_future * weight).mean().detach(), + "loss_point_frame0": (point_frame0 * weight).mean().detach(), + "loss_point_future": (point_future * weight).mean().detach(), + "timestep_mean": timestep.float().mean().detach(), + "sample_weight_mean": weight.mean().detach(), + "prediction_rgb_norm": pred_rgb.float().norm().detach(), + "prediction_depth_norm": pred_depth.float().norm().detach(), + "prediction_point_norm": pred_point.float().norm().detach(), + } + + +def sample_joint_latents( + model: torch.nn.Module, + z_rgb: torch.Tensor, + z_depth: torch.Tensor, + z_point: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + config: dict[str, Any], + seed: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=float(config["diffusion"]["infer_shift"]), + ) + generator = torch.Generator(device=z_rgb.device).manual_seed(seed) + noisy = [ + torch.randn( + value.shape, + generator=generator, + device=value.device, + dtype=value.dtype, + ) + for value in (z_rgb, z_depth, z_point) + ] + rgb_condition = z_rgb[:, :, 0:1] + depth_condition = z_depth[:, :, 0:1] + noisy[0][:, :, 0:1] = rgb_condition + noisy[1][:, :, 0:1] = depth_condition + joint = torch.cat(noisy, dim=1) + timesteps, deltas = scheduler.build_inference_schedule( + int(config["diffusion"]["inference_steps"]), z_rgb.device, z_rgb.dtype + ) + for step_t, delta in zip(timesteps, deltas, strict=True): + joint, _, _, _ = joint_rgb_depth_point_sampling_step( + model, + scheduler, + joint, + step_t.expand(z_rgb.shape[0]), + delta, + context, + context_mask=context_mask, + rgb_condition_latent=rgb_condition, + depth_condition_latent=depth_condition, + ) + return split_joint_rgb_depth_point_prediction(joint) + + +@torch.no_grad() +def run_sampling_validation( + model: torch.nn.Module, + dataset: CachedRgbDepthPointLatents, + accelerator: Accelerator, + config: dict[str, Any], + output_dir: Path, + step: int, + num_samples: int, +) -> dict[str, float]: + model.eval() + world_size, rank = accelerator.num_processes, accelerator.process_index + num_samples = min(int(num_samples), len(dataset)) + calls = math.ceil(num_samples / world_size) + totals = torch.zeros(4, device=accelerator.device) + validation_dir = output_dir / "validation" / f"step_{step:06d}" + validation_dir.mkdir(parents=True, exist_ok=True) + dtype = dtype_from_name(str(config["model"]["dtype"])) + for call in range(calls): + index = rank + call * world_size + valid = index < num_samples + actual = index if valid else 0 + sample = dataset[actual] + z_rgb, z_depth, z_point = [ + sample[key].unsqueeze(0).to(accelerator.device, dtype=dtype) + for key in ("z_rgb", "z_depth", "z_point") + ] + context = sample["context"].unsqueeze(0).to(accelerator.device, dtype=dtype) + mask = sample["context_mask"].unsqueeze(0).to(accelerator.device, dtype=torch.bool) + generated = sample_joint_latents( + model, z_rgb, z_depth, z_point, context, mask, config, + int(config["validation"]["fixed_noise_seed"]) + actual, + ) + mse = ( + F.mse_loss(generated[0][:, :, 1:].float(), z_rgb[:, :, 1:].float()), + F.mse_loss(generated[1][:, :, 1:].float(), z_depth[:, :, 1:].float()), + F.mse_loss(generated[2].float(), z_point.float()), + ) + if valid: + totals[:3] += torch.stack(mse) + totals[3] += 1 + if config["validation"]["save_latents"]: + torch.save( + { + "sample_id": sample["sample_id"], + "generated_rgb": generated[0][0].cpu(), + "generated_depth": generated[1][0].cpu(), + "generated_point": generated[2][0].cpu(), + "target_rgb": z_rgb[0].cpu(), + "target_depth": z_depth[0].cpu(), + "target_point": z_point[0].cpu(), + }, + validation_dir / f"{actual:03d}_{sample['sample_id']}.pt", + ) + totals = accelerator.reduce(totals, reduction="sum") + model.train() + count = max(float(totals[3]), 1.0) + return { + "validation_rgb_latent_mse": float(totals[0]) / count, + "validation_depth_latent_mse": float(totals[1]) / count, + "validation_point_latent_mse": float(totals[2]) / count, + } + + +def export_joint_model( + model: torch.nn.Module, accelerator: Accelerator, path: Path +) -> None: + fsdp_plugin = None + previous_state_dict_type = None + if accelerator.distributed_type == DistributedType.FSDP: + fsdp_plugin = accelerator.state.fsdp_plugin + previous_state_dict_type = fsdp_plugin.state_dict_type + fsdp_plugin.set_state_dict_type("FULL_STATE_DICT") + state_dict = accelerator.get_state_dict(model) + if accelerator.is_main_process: + unwrapped = accelerator.unwrap_model(model) + payload = { + "metadata": joint_checkpoint_metadata(unwrapped), + "state_dict": {key: value.cpu() for key, value in state_dict.items()}, + } + path.parent.mkdir(parents=True, exist_ok=True) + accelerator.save(payload, path) + print(f"[checkpoint] exported {path}", flush=True) + accelerator.wait_for_everyone() + if fsdp_plugin is not None: + fsdp_plugin.set_state_dict_type(previous_state_dict_type) + + +def inference_checkpoint_path( + output_dir: Path, phase: str, global_step: int +) -> Path: + """Return the stable, phase-qualified path for an inference checkpoint.""" + return ( + output_dir + / "checkpoints" + / f"joint_model_{phase}_{global_step:06d}.pt" + ) + + +def should_export_inference_checkpoint( + global_step: int, export_model_every: int +) -> bool: + if export_model_every <= 0: + raise ValueError("training.export_model_every must be positive") + return global_step > 0 and global_step % export_model_every == 0 + + +def train( + config: dict[str, Any], + phase: str, + output_dir: Path, + resume_model: Path | None, + resume_state: Path | None, +) -> None: + tracking = bool(config.get("tracking", {}).get("enabled", False)) + tracking_step_offset = int(os.environ.get("WANDB_STEP_OFFSET", "0")) + if tracking_step_offset < 0: + raise ValueError("WANDB_STEP_OFFSET must be non-negative") + if tracking and importlib.util.find_spec("wandb") is None: + raise RuntimeError("tracking.enabled=true but wandb is unavailable") + accelerator = Accelerator( + gradient_accumulation_steps=int(config["training"]["gradient_accumulation_steps"]), + mixed_precision="bf16" if config["model"]["dtype"] == "bfloat16" else "no", + log_with="wandb" if tracking else None, + project_dir=str(output_dir), + ) + set_seed(int(config["experiment"]["seed"]), device_specific=True) + if accelerator.is_main_process: + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "resolved_config.yaml").write_text( + yaml.safe_dump( + {key: value for key, value in config.items() if not key.startswith("_")}, + sort_keys=False, + ), + encoding="utf-8", + ) + accelerator.wait_for_everyone() + initialize_experiment_trackers(accelerator, config, phase) + dataset = CachedRgbDepthPointLatents( + Path(config["paths"]["cache_dir"]), point_config_fingerprint(config) + ) + dataloader = DataLoader( + dataset, + batch_size=int(config["data"]["batch_size"]), + shuffle=False, + sampler=FixedSeedPermutationSampler(dataset, int(config["data"]["sample_order_seed"])), + num_workers=int(config["data"]["num_workers"]), + pin_memory=True, + drop_last=False, + collate_fn=collate_rgb_depth_point_batch, + ) + if accelerator.is_main_process: + print("[model] strict-loading Wan2.2 and expanding RGB-depth-point", flush=True) + model = build_joint_dit(config, resume_model) + configure_trainable_parameters(model, phase) + phase_config = config["training"][phase] + masks = ProjectionGradientMasks( + model, + phase=phase, + head_warmup_steps=int(phase_config.get("head_warmup_steps", 0)), + preserve_original_slices=bool(config["training"]["preserve_original_projection_slices"]), + ) + optimizer = build_optimizer(model, config, phase) + max_steps = int(phase_config["max_steps"]) + lr_scheduler = build_lr_scheduler( + optimizer, + max_steps, + int(phase_config["warmup_steps"]), + schedule_type=str(phase_config.get("lr_schedule", "cosine")), + ) + model, optimizer, dataloader, lr_scheduler = accelerator.prepare( + model, optimizer, dataloader, lr_scheduler + ) + start_step = 0 + if resume_state is not None: + accelerator.load_state(str(resume_state)) + start_step = int( + json.loads((resume_state / "trainer_state.json").read_text())["global_step"] + ) + masks.set_step(start_step) + scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=float(config["diffusion"]["train_shift"]), + ) + dtype = dtype_from_name(str(config["model"]["dtype"])) + global_step, epoch = start_step, 0 + optimizer.zero_grad(set_to_none=True) + log_path = output_dir / "metrics.jsonl" + export_model_every = int( + config["training"].get( + "export_model_every", config["training"]["save_every"] + ) + ) + if export_model_every <= 0: + raise ValueError("training.export_model_every must be positive") + last_exported_step: int | None = None + model.train() + while global_step < max_steps: + epoch += 1 + for batch in dataloader: + if global_step >= max_steps: + break + with accelerator.accumulate(model): + next_global_step = global_step + 1 + compute_diagnostics = accelerator.sync_gradients and ( + next_global_step == 1 + or next_global_step % int(config["training"]["log_every"]) == 0 + ) + z_rgb, z_depth, z_point = [ + batch[key].to(accelerator.device, dtype=dtype, non_blocking=True) + for key in ("z_rgb", "z_depth", "z_point") + ] + context_mask_cpu = batch["context_mask"] + valid_context_columns = context_mask_cpu.any(dim=0) + if not bool(valid_context_columns.any()): + raise ValueError("Batch contains no valid text tokens") + context_length = int( + valid_context_columns.nonzero(as_tuple=False)[-1, 0] + ) + 1 + context = batch["context"][:, :context_length].to( + accelerator.device, dtype=dtype, non_blocking=True + ) + context_mask = context_mask_cpu[:, :context_length].to( + accelerator.device, dtype=torch.bool, non_blocking=True + ) + total_loss, metrics = joint_flow_matching_loss( + model, + scheduler, + z_rgb, + z_depth, + z_point, + context, + context_mask, + config, + compute_diagnostics=compute_diagnostics, + ) + accelerator.backward(total_loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_( + [p for p in model.parameters() if p.requires_grad], + float(config["training"]["max_grad_norm"]), + ) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + if not accelerator.sync_gradients: + continue + global_step += 1 + masks.set_step(global_step) + should_log = ( + global_step == 1 + or global_step % int(config["training"]["log_every"]) == 0 + ) + if should_log: + reduced = { + key: accelerator.reduce(value.float(), reduction="mean") + for key, value in metrics.items() + } + reduced["loss_total"] = accelerator.reduce( + total_loss.detach().float(), reduction="mean" + ) + lrs = [float(group["lr"]) for group in optimizer.param_groups] + record = { + "phase": phase, + "epoch": epoch, + "global_step": global_step, + **{key: float(value.item()) for key, value in reduced.items()}, + "learning_rates": lrs, + } + if tracking: + wandb_metrics = { + f"train/{key}": float(value.item()) + for key, value in reduced.items() + } + wandb_metrics.update( + {f"train/lr_group_{i}": lr for i, lr in enumerate(lrs)} + ) + wandb_metrics["train/epoch"] = epoch + wandb_metrics["train/phase_step"] = global_step + accelerator.log( + wandb_metrics, + step=tracking_step_offset + global_step, + ) + if accelerator.is_main_process: + append_jsonl(log_path, record) + print(json.dumps(record, sort_keys=True), flush=True) + if global_step < max_steps and global_step % int(config["training"]["validate_every"]) == 0: + validation = run_sampling_validation( + model, + dataset, + accelerator, + config, + output_dir, + global_step, + int(config["validation"]["periodic_num_samples"]), + ) + if tracking: + accelerator.log( + {f"validation/{key}": value for key, value in validation.items()}, + step=tracking_step_offset + global_step, + ) + if accelerator.is_main_process: + append_jsonl(log_path, {"phase": phase, "global_step": global_step, **validation}) + if global_step % int(config["training"]["save_every"]) == 0: + state_dir = output_dir / "checkpoints" / f"state_{global_step:06d}" + accelerator.save_state(str(state_dir)) + if accelerator.is_main_process: + (state_dir / "trainer_state.json").write_text( + json.dumps({"phase": phase, "global_step": global_step}, indent=2), + encoding="utf-8", + ) + if should_export_inference_checkpoint(global_step, export_model_every): + export_joint_model( + model, + accelerator, + inference_checkpoint_path(output_dir, phase, global_step), + ) + last_exported_step = global_step + final = run_sampling_validation( + model, + dataset, + accelerator, + config, + output_dir, + global_step, + int(config["validation"]["final_num_samples"]), + ) + if tracking: + accelerator.log( + {f"final/{key}": value for key, value in final.items()}, + step=tracking_step_offset + global_step, + ) + if accelerator.is_main_process: + append_jsonl(log_path, {"phase": phase, "global_step": global_step, **final}) + if last_exported_step != global_step: + export_joint_model( + model, + accelerator, + inference_checkpoint_path(output_dir, phase, global_step), + ) + masks.close() + accelerator.end_training() + + +def main() -> None: + args = parse_args() + config = validate_config(load_config(args.config)) + if args.validate_only: + print( + json.dumps( + { + "status": "valid", + "episodes": config["_num_episodes"], + "samples": config["_num_samples"], + "modalities": ["rgb", "depth", "world_point"], + "cache_dir": config["paths"]["cache_dir"], + "rgb_depth_cache_dir": config["paths"]["rgb_depth_cache_dir"], + "wandb_installed": importlib.util.find_spec("wandb") is not None, + }, + indent=2, + ) + ) + return + if args.prepare_cache_only: + from prepare_point_cloud_cache import prepare_point_cache + + prepare_point_cache(config) + return + if args.phase is None or args.output_dir is None: + raise ValueError("Training requires --phase and --output-dir") + train(config, args.phase, args.output_dir, args.resume_model, args.resume_state) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_joint_rgb_depth_point_action.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_joint_rgb_depth_point_action.py new file mode 100644 index 0000000000000000000000000000000000000000..1a9e17daddb1389b6eeacf69a575375c1d4df544 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_joint_rgb_depth_point_action.py @@ -0,0 +1,929 @@ +#!/usr/bin/env python3 +"""Train RGB/depth/point video and 14D bimanual actions with one Action MoT.""" + +from __future__ import annotations + +from datetime import timedelta + +import argparse +import importlib.util +import json +import math +import os +from pathlib import Path +from typing import Any + +import torch +import torch.nn.functional as F +from torch.utils.data import DataLoader +import yaml +from accelerate import Accelerator +from accelerate.utils import DistributedType, InitProcessGroupKwargs, set_seed + +from joint_rgb_depth_point_action_diffusion import ( + joint_flow_matching_loss, + sample_joint_latents_and_actions, +) +from online_egoverse_action import ( + OnlineEgoverseActionDataset, + collate_online_action_batch, + prepare_online_training_dataset, +) +from rgb_depth_point_action_model_factory import ( + build_joint_action_mot, + mot_checkpoint_metadata, + validate_action_mot_config, +) +from train_joint_rgb_depth import ( + WanContinuousFlowMatchScheduler, + append_jsonl, + build_lr_scheduler, + dtype_from_name, + initialize_experiment_trackers, + load_config, +) +from train_joint_rgb_depth_point import ( + PHASES, + CachedRgbDepthPointLatents, + FixedSeedPermutationSampler, + ProjectionGradientMasks, + configure_trainable_parameters as configure_video_trainable_parameters, + point_config_fingerprint, + should_export_inference_checkpoint, + validate_config as validate_point_config, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--phase", choices=sorted(PHASES)) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--resume-model", type=Path) + parser.add_argument("--resume-state", type=Path) + parser.add_argument("--validate-only", action="store_true") + parser.add_argument("--dataset-smoke-test", action="store_true") + return parser.parse_args() + + +def validate_config(config: dict[str, Any]) -> dict[str, Any]: + config = validate_point_config(config) + validate_action_mot_config(config) + data = config["data"] + action = config["action"] + diffusion = config["diffusion"] + training = config["training"] + if action.get("enabled") is not True: + raise ValueError("action.enabled must be true") + if action.get("horizon") != "data_num_frames_minus_one": + raise ValueError("action.horizon must be data_num_frames_minus_one") + if int(data["num_frames"]) - 1 <= 0: + raise ValueError("At least two frames are required for action deltas") + if action.get("coordinate_frame") != "first_camera_frame": + raise ValueError("Action state and deltas must use first_camera_frame") + normalization = action.get("normalization", {}) + camera = config["camera"] + camera_normalization = camera.get("normalization", {}) + if camera.get("horizon") != "data_num_frames_minus_one": + raise ValueError("camera.horizon must be data_num_frames_minus_one") + if camera.get("coordinate_frame") != "first_camera_frame": + raise ValueError("camera must be relative to first_camera_frame") + if camera_normalization.get("fit") != "online_training_dataset_startup": + raise ValueError("Camera normalization must be fitted online at startup") + if camera_normalization.get("checkpoint_buffers") is not True: + raise ValueError("Camera normalization statistics must be checkpoint buffers") + if camera_normalization.get("write_statistics_file") is not False: + raise ValueError("Camera training must not write a statistics sidecar") + if normalization.get("fit") != "online_training_dataset_startup": + raise ValueError("Action normalization must be fitted online at startup") + if normalization.get("checkpoint_buffers") is not True: + raise ValueError("Action normalization statistics must be checkpoint buffers") + if normalization.get("write_statistics_file") is not False: + raise ValueError("Online action training must not write a statistics sidecar") + for key in ( + "action_num_train_timesteps", + "action_train_shift", + "action_infer_shift", + "lambda_action", + "lambda_camera_translation", + "lambda_camera_rotation", + ): + if float(diffusion.get(key, 0.0)) <= 0: + raise ValueError(f"diffusion.{key} must be positive") + if float(training.get("action_dit_lr", 0.0)) <= 0: + raise ValueError("training.action_dit_lr must be positive") + if not isinstance(training.get("fp32_master_weights", False), bool): + raise ValueError("training.fp32_master_weights must be a boolean") + if int(config["model"]["action_state_dim"]) != int(action["action_dim"]): + raise ValueError("Initial state and action must use the same 14D slot layout") + return config + + +def configure_trainable_parameters( + model: torch.nn.Module, + config: dict[str, Any], + phase: str, +) -> None: + train_video = bool(config["training"].get("train_video_expert", True)) + if train_video: + configure_video_trainable_parameters(model.video_expert, phase) + else: + model.video_expert.requires_grad_(False) + model.action_expert.requires_grad_(True) + model.camera_encoder.requires_grad_(True) + model.camera_head.requires_grad_(True) + model.structured_type_embedding.requires_grad_(True) + if model.action_state_encoder is None: + raise RuntimeError("Action MoT must have an initial-state encoder") + model.action_state_encoder.requires_grad_(True) + + +def enable_fp32_master_weights(model: torch.nn.Module) -> None: + """Keep parameters/optimizer state in FP32 while autocast computes in BF16.""" + + model.float() + non_fp32_parameters = [ + name + for name, parameter in model.named_parameters() + if parameter.is_floating_point() and parameter.dtype != torch.float32 + ] + if non_fp32_parameters: + raise RuntimeError( + "FP32 master-weight promotion failed for parameters: " + f"{non_fp32_parameters[:8]}" + ) + + +def build_optimizer( + model: torch.nn.Module, + config: dict[str, Any], + phase: str, +) -> torch.optim.Optimizer: + phase_config = config["training"][phase] + groups: list[dict[str, Any]] = [] + assigned: set[int] = set() + + def add_group(name: str, parameters: list[torch.nn.Parameter], lr: float) -> None: + selected = [ + parameter + for parameter in parameters + if parameter.requires_grad and id(parameter) not in assigned + ] + if not selected: + return + assigned.update(id(parameter) for parameter in selected) + groups.append({"name": name, "params": selected, "lr": float(lr)}) + + video = model.video_expert + if bool(config["training"].get("train_video_expert", True)): + add_group( + "video_patch_projection", + [video.patch_embedding.weight], + float(phase_config["depth_patch_lr"]), + ) + add_group( + "video_output_projection", + [video.head.head.weight, video.head.head.bias], + float(phase_config["depth_head_lr"]), + ) + add_group( + "video_point_timestep_adapter", + list(video.point_first_frame_timestep_adapter.parameters()), + float(phase_config["point_adapter_lr"]), + ) + if phase == "full_dit": + add_group( + "video_dit", + list(video.parameters()), + float(phase_config["dit_lr"]), + ) + + action_parameters = ( + list(model.action_expert.parameters()) + + list(model.action_state_encoder.parameters()) + + list(model.camera_encoder.parameters()) + + list(model.camera_head.parameters()) + + [model.structured_type_embedding] + ) + add_group( + "action_dit_and_initial_state_encoder", + action_parameters, + float(config["training"]["action_dit_lr"]), + ) + missing = [ + name + for name, parameter in model.named_parameters() + if parameter.requires_grad and id(parameter) not in assigned + ] + if missing: + raise RuntimeError(f"Trainable parameters missing optimizer group: {missing[:8]}") + if not groups: + raise RuntimeError("No trainable optimizer parameters") + optimizer_kwargs: dict[str, Any] = { + "betas": (0.9, 0.95), + "eps": 1.0e-8, + "weight_decay": float(config["training"]["weight_decay"]), + } + # Preserve AdamW's historical/default dispatch exactly for every existing + # config. Only an explicit setting opts into (or disables) the fused path. + if "fused_optimizer" in config["training"]: + optimizer_kwargs["fused"] = bool(config["training"]["fused_optimizer"]) + return torch.optim.AdamW(groups, **optimizer_kwargs) + + +def compute_training_loss( + model: torch.nn.Module, + video_scheduler: WanContinuousFlowMatchScheduler, + action_scheduler: WanContinuousFlowMatchScheduler, + batch: dict[str, Any], + *, + device: torch.device, + dtype: torch.dtype, + config: dict[str, Any], +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + z_rgb, z_depth, z_point = [ + batch[key].to(device, dtype=dtype, non_blocking=True) + for key in ("z_rgb", "z_depth", "z_point") + ] + action = batch["action"].to(device, dtype=dtype, non_blocking=True) + camera = batch["camera"].to(device, dtype=dtype, non_blocking=True) + initial_state = batch["initial_state"].to( + device, dtype=dtype, non_blocking=True + ) + action_is_pad = batch["action_is_pad"].to( + device, dtype=torch.bool, non_blocking=True + ) + camera_is_pad = batch["camera_is_pad"].to( + device, dtype=torch.bool, non_blocking=True + ) + expected_horizon = int(config["data"]["num_frames"]) - 1 + expected_action_dim = int(config["action"]["action_dim"]) + if tuple(action.shape[1:]) != (expected_horizon, expected_action_dim): + raise ValueError( + f"Expected action [B,{expected_horizon},{expected_action_dim}], " + f"got {tuple(action.shape)}" + ) + expected_camera_dim = int(config["camera"]["camera_dim"]) + if tuple(camera.shape[1:]) != (expected_horizon, expected_camera_dim): + raise ValueError( + f"Expected camera [B,{expected_horizon},{expected_camera_dim}], " + f"got {tuple(camera.shape)}" + ) + if tuple(initial_state.shape[1:]) != (expected_action_dim,): + raise ValueError( + f"Expected initial_state [B,{expected_action_dim}], " + f"got {tuple(initial_state.shape)}" + ) + + context_mask_cpu = batch["context_mask"] + valid_context_columns = context_mask_cpu.any(dim=0) + if not bool(valid_context_columns.any()): + raise ValueError("Batch contains no valid text tokens") + context_length = int( + valid_context_columns.nonzero(as_tuple=False)[-1, 0] + ) + 1 + context = batch["context"][:, :context_length].to( + device, dtype=dtype, non_blocking=True + ) + context_mask = context_mask_cpu[:, :context_length].to( + device, dtype=torch.bool, non_blocking=True + ) + conditioning = config["conditioning"] + diffusion = config["diffusion"] + return joint_flow_matching_loss( + model, + video_scheduler, + action_scheduler, + z_rgb=z_rgb, + z_depth=z_depth, + z_point=z_point, + action=action, + camera=camera, + initial_state=initial_state, + context=context, + context_mask=context_mask, + action_is_pad=action_is_pad, + camera_is_pad=camera_is_pad, + fix_rgb_first_latent=bool(conditioning["fix_rgb_first_latent"]), + fix_depth_first_latent=bool(conditioning["fix_depth_first_latent"]), + exclude_rgb_condition_from_loss=bool( + conditioning["exclude_rgb_condition_from_loss"] + ), + exclude_depth_condition_from_loss=bool( + conditioning["exclude_depth_condition_from_loss"] + ), + lambda_rgb=float(diffusion["lambda_rgb"]), + lambda_depth=float(diffusion["lambda_depth"]), + lambda_point=float(diffusion["lambda_point"]), + lambda_action=float(diffusion["lambda_action"]), + lambda_camera_translation=float( + diffusion["lambda_camera_translation"] + ), + lambda_camera_rotation=float(diffusion["lambda_camera_rotation"]), + ) + + +@torch.no_grad() +def run_sampling_validation( + model: torch.nn.Module, + dataset: OnlineEgoverseActionDataset, + accelerator: Accelerator, + config: dict[str, Any], + output_dir: Path, + step: int, + num_samples: int, +) -> dict[str, float]: + model.eval() + world_size, rank = accelerator.num_processes, accelerator.process_index + num_samples = min(int(num_samples), len(dataset)) + calls = math.ceil(num_samples / world_size) + totals = torch.zeros(7, device=accelerator.device) + validation_dir = output_dir / "validation" / f"step_{step:06d}" + if accelerator.is_main_process and bool(config["validation"]["save_latents"]): + validation_dir.mkdir(parents=True, exist_ok=True) + dtype = dtype_from_name(str(config["model"]["dtype"])) + video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=float(config["diffusion"]["infer_shift"]), + ) + action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["action_num_train_timesteps"]), + shift=float(config["diffusion"]["action_infer_shift"]), + ) + for call in range(calls): + index = rank + call * world_size + valid = index < num_samples + actual = index if valid else 0 + sample = dataset[actual] + z_rgb, z_depth, z_point = [ + sample[key].unsqueeze(0).to(accelerator.device, dtype=dtype) + for key in ("z_rgb", "z_depth", "z_point") + ] + target_action = sample["action"].unsqueeze(0).to( + accelerator.device, dtype=dtype + ) + action_is_pad = sample["action_is_pad"].unsqueeze(0).to( + accelerator.device, dtype=torch.bool + ) + target_camera = sample["camera"].unsqueeze(0).to( + accelerator.device, dtype=dtype + ) + camera_is_pad = sample["camera_is_pad"].unsqueeze(0).to( + accelerator.device, dtype=torch.bool + ) + initial_state = sample["initial_state"].unsqueeze(0).to( + accelerator.device, dtype=dtype + ) + context = sample["context"].unsqueeze(0).to( + accelerator.device, dtype=dtype + ) + context_mask = sample["context_mask"].unsqueeze(0).to( + accelerator.device, dtype=torch.bool + ) + generator = torch.Generator(device=accelerator.device).manual_seed( + int(config["validation"]["fixed_noise_seed"]) + actual + ) + generated = sample_joint_latents_and_actions( + model, + video_scheduler, + action_scheduler, + video_shape=(1, 144, *z_rgb.shape[2:]), + action_shape=tuple(target_action.shape), + camera_shape=tuple(target_camera.shape), + context=context, + context_mask=context_mask, + initial_state=initial_state, + rgb_condition_latent=z_rgb[:, :, 0:1], + depth_condition_latent=z_depth[:, :, 0:1], + num_inference_steps=int(config["diffusion"]["inference_steps"]), + generator=generator, + dtype=dtype, + device=accelerator.device, + ) + valid_action = (~action_is_pad).unsqueeze(-1).expand_as(target_action) + action_mse = F.mse_loss( + generated["action"][valid_action].float(), + target_action[valid_action].float(), + ) + valid_camera = (~camera_is_pad).unsqueeze(-1) + camera_translation_mse = F.mse_loss( + generated["camera"][..., :3][valid_camera.expand_as(target_camera[..., :3])].float(), + target_camera[..., :3][valid_camera.expand_as(target_camera[..., :3])].float(), + ) + camera_rotation_mse = F.mse_loss( + generated["camera"][..., 3:][valid_camera.expand_as(target_camera[..., 3:])].float(), + target_camera[..., 3:][valid_camera.expand_as(target_camera[..., 3:])].float(), + ) + metrics = torch.stack( + [ + F.mse_loss( + generated["rgb"][:, :, 1:].float(), + z_rgb[:, :, 1:].float(), + ), + F.mse_loss( + generated["depth"][:, :, 1:].float(), + z_depth[:, :, 1:].float(), + ), + F.mse_loss(generated["point"].float(), z_point.float()), + action_mse, + camera_translation_mse, + camera_rotation_mse, + ] + ) + if valid: + totals[:6] += metrics + totals[6] += 1 + if bool(config["validation"]["save_latents"]): + torch.save( + { + "sample_id": sample["sample_id"], + "generated_rgb": generated["rgb"][0].cpu(), + "generated_depth": generated["depth"][0].cpu(), + "generated_point": generated["point"][0].cpu(), + "generated_action_normalized": generated["action"][0].cpu(), + "target_action_normalized": target_action[0].cpu(), + "generated_camera_normalized": generated["camera"][0].cpu(), + "target_camera_normalized": target_camera[0].cpu(), + "initial_state": initial_state[0].cpu(), + "action_is_pad": action_is_pad[0].cpu(), + "camera_is_pad": camera_is_pad[0].cpu(), + }, + validation_dir / f"{actual:03d}_{sample['sample_id']}.pt", + ) + totals = accelerator.reduce(totals, reduction="sum") + model.train() + count = max(float(totals[6]), 1.0) + return { + "validation_rgb_latent_mse": float(totals[0]) / count, + "validation_depth_latent_mse": float(totals[1]) / count, + "validation_point_latent_mse": float(totals[2]) / count, + "validation_action_normalized_mse": float(totals[3]) / count, + "validation_camera_translation_normalized_mse": float(totals[4]) / count, + "validation_camera_rotation_normalized_mse": float(totals[5]) / count, + } + + +def export_action_mot( + model: torch.nn.Module, + accelerator: Accelerator, + path: Path, +) -> None: + fsdp_plugin = None + previous_state_dict_type = None + if accelerator.distributed_type == DistributedType.FSDP: + fsdp_plugin = accelerator.state.fsdp_plugin + previous_state_dict_type = fsdp_plugin.state_dict_type + fsdp_plugin.set_state_dict_type("FULL_STATE_DICT") + state_dict = accelerator.get_state_dict(model) + if accelerator.is_main_process: + unwrapped = accelerator.unwrap_model(model) + payload = { + "metadata": mot_checkpoint_metadata(unwrapped), + "state_dict": {key: value.cpu() for key, value in state_dict.items()}, + } + path.parent.mkdir(parents=True, exist_ok=True) + accelerator.save(payload, path) + print(f"[checkpoint] exported {path}", flush=True) + accelerator.wait_for_everyone() + if fsdp_plugin is not None: + fsdp_plugin.set_state_dict_type(previous_state_dict_type) + + +def inference_checkpoint_path( + output_dir: Path, + phase: str, + global_step: int, +) -> Path: + return ( + output_dir + / "checkpoints" + / f"joint_action_mot_{phase}_{global_step:06d}.pt" + ) + + +def dataset_smoke_test(config: dict[str, Any]) -> None: + base_dataset = CachedRgbDepthPointLatents( + Path(config["paths"]["cache_dir"]), point_config_fingerprint(config) + ) + if len(base_dataset) != int(config["data"]["num_clips"]): + raise RuntimeError( + f"Cache has {len(base_dataset)} clips, expected {config['data']['num_clips']}" + ) + from online_egoverse_action import ( + OnlineActionNormalizer, + OnlineCameraNormalizer, + OnlineEgoverseActionDataset, + fit_online_camera_normalizer, + fit_online_normalizer, + processor_from_config, + ) + + processor = processor_from_config(config) + normalizer = fit_online_normalizer( + base_dataset, + processor, + OnlineActionNormalizer( + action_dim=int(config["action"]["action_dim"]), + eps=float(config["action"]["normalization"]["eps"]), + clip=config["action"]["normalization"].get("clip"), + ), + ) + camera_normalizer = fit_online_camera_normalizer( + base_dataset, + processor, + OnlineCameraNormalizer( + camera_dim=int(config["camera"]["camera_dim"]), + eps=float(config["camera"]["normalization"]["eps"]), + clip=config["camera"]["normalization"].get("clip"), + ), + ) + dataset = OnlineEgoverseActionDataset( + base_dataset, processor, normalizer, camera_normalizer + ) + sample = dataset[0] + batch = collate_online_action_batch([sample, dataset[1]]) + print( + json.dumps( + { + "status": "dataset_smoke_ok", + "samples": len(dataset), + "action_shape": list(sample["action"].shape), + "camera_shape": list(sample["camera"].shape), + "initial_state_shape": list(sample["initial_state"].shape), + "batch_action_shape": list(batch["action"].shape), + "batch_camera_shape": list(batch["camera"].shape), + "valid_action_transitions": int(normalizer.count), + "valid_camera_poses": int(camera_normalizer.count), + "prompt": sample["prompt"], + }, + indent=2, + ), + flush=True, + ) + + +def train( + config: dict[str, Any], + phase: str, + output_dir: Path, + resume_model: Path | None, + resume_state: Path | None, +) -> None: + tracking = bool(config.get("tracking", {}).get("enabled", False)) + tracking_step_offset = int(os.environ.get("WANDB_STEP_OFFSET", "0")) + if tracking_step_offset < 0: + raise ValueError("WANDB_STEP_OFFSET must be non-negative") + if tracking and importlib.util.find_spec("wandb") is None: + raise RuntimeError("tracking.enabled=true but wandb is unavailable") + distributed_timeout_seconds = int( + os.environ.get("WAN_DISTRIBUTED_TIMEOUT_SECONDS", "600") + ) + if distributed_timeout_seconds <= 0: + raise ValueError("WAN_DISTRIBUTED_TIMEOUT_SECONDS must be positive") + accelerator = Accelerator( + gradient_accumulation_steps=int( + config["training"]["gradient_accumulation_steps"] + ), + mixed_precision=( + "bf16" if config["model"]["dtype"] == "bfloat16" else "no" + ), + log_with="wandb" if tracking else None, + project_dir=str(output_dir), + kwargs_handlers=[ + InitProcessGroupKwargs( + timeout=timedelta(seconds=distributed_timeout_seconds) + ) + ], + ) + set_seed(int(config["experiment"]["seed"]), device_specific=True) + if accelerator.is_main_process: + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "resolved_config.yaml").write_text( + yaml.safe_dump( + { + key: value + for key, value in config.items() + if not key.startswith("_") + }, + sort_keys=False, + ), + encoding="utf-8", + ) + accelerator.wait_for_everyone() + initialize_experiment_trackers(accelerator, config, phase) + + base_dataset = CachedRgbDepthPointLatents( + Path(config["paths"]["cache_dir"]), point_config_fingerprint(config) + ) + if len(base_dataset) != int(config["data"]["num_clips"]): + raise RuntimeError( + f"Cache has {len(base_dataset)} clips, expected {config['data']['num_clips']}" + ) + if accelerator.is_main_process: + print( + "[model] loading RGB/depth/point Wan and random ActionDiT", + flush=True, + ) + model = build_joint_action_mot( + config, + resume_model, + load_pretrained=resume_state is None, + ) + if bool(config["training"].get("fp32_master_weights", False)): + enable_fp32_master_weights(model) + if accelerator.is_main_process: + print( + "[precision] FP32 model/AdamW master weights enabled; " + "BF16 autocast remains enabled", + flush=True, + ) + dataset, _ = prepare_online_training_dataset(base_dataset, model, config) + if accelerator.is_main_process: + print( + "[action] online dataset ready: " + f"samples={len(dataset)} valid_transitions={int(model.action_normalizer.count)} " + f"action_shape=({int(config['data']['num_frames']) - 1}, " + f"{int(config['action']['action_dim'])}) camera_shape=(" + f"{int(config['data']['num_frames']) - 1}, " + f"{int(config['camera']['camera_dim'])}) initial_state_dim=" + f"{int(config['model']['action_state_dim'])} " + f"valid_camera_poses={int(model.camera_normalizer.count)}", + flush=True, + ) + dataloader = DataLoader( + dataset, + batch_size=int(config["data"]["batch_size"]), + shuffle=False, + sampler=FixedSeedPermutationSampler( + dataset, int(config["data"]["sample_order_seed"]) + ), + num_workers=int(config["data"]["num_workers"]), + pin_memory=True, + drop_last=False, + collate_fn=collate_online_action_batch, + ) + + configure_trainable_parameters(model, config, phase) + phase_config = config["training"][phase] + masks = None + if bool(config["training"].get("train_video_expert", True)): + masks = ProjectionGradientMasks( + model.video_expert, + phase=phase, + head_warmup_steps=int(phase_config.get("head_warmup_steps", 0)), + preserve_original_slices=bool( + config["training"]["preserve_original_projection_slices"] + ), + ) + optimizer = build_optimizer(model, config, phase) + max_steps = int(phase_config["max_steps"]) + lr_scheduler = build_lr_scheduler( + optimizer, + max_steps, + int(phase_config["warmup_steps"]), + schedule_type=str(phase_config.get("lr_schedule", "cosine")), + ) + model, optimizer, dataloader, lr_scheduler = accelerator.prepare( + model, optimizer, dataloader, lr_scheduler + ) + start_step = 0 + if resume_state is not None: + accelerator.load_state(str(resume_state)) + start_step = int( + json.loads( + (resume_state / "trainer_state.json").read_text() + )["global_step"] + ) + if masks is not None: + masks.set_step(start_step) + + video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=float(config["diffusion"]["train_shift"]), + ) + action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int( + config["diffusion"]["action_num_train_timesteps"] + ), + shift=float(config["diffusion"]["action_train_shift"]), + ) + dtype = dtype_from_name(str(config["model"]["dtype"])) + global_step, epoch = start_step, 0 + optimizer.zero_grad(set_to_none=True) + log_path = output_dir / "metrics.jsonl" + export_model_every = int(config["training"]["export_model_every"]) + last_exported_step: int | None = None + model.train() + while global_step < max_steps: + epoch += 1 + for batch in dataloader: + if global_step >= max_steps: + break + with accelerator.accumulate(model): + total_loss, metrics = compute_training_loss( + model, + video_scheduler, + action_scheduler, + batch, + device=accelerator.device, + dtype=dtype, + config=config, + ) + accelerator.backward(total_loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_( + [ + parameter + for parameter in model.parameters() + if parameter.requires_grad + ], + float(config["training"]["max_grad_norm"]), + ) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + if not accelerator.sync_gradients: + continue + global_step += 1 + if masks is not None: + masks.set_step(global_step) + should_log = global_step == 1 or global_step % int( + config["training"]["log_every"] + ) == 0 + if should_log: + reduced = { + key: accelerator.reduce(value.float(), reduction="mean") + for key, value in metrics.items() + } + reduced["loss_total"] = accelerator.reduce( + total_loss.detach().float(), reduction="mean" + ) + lrs = [float(group["lr"]) for group in optimizer.param_groups] + record = { + "phase": phase, + "epoch": epoch, + "global_step": global_step, + **{ + key: float(value.item()) + for key, value in reduced.items() + }, + "learning_rates": lrs, + } + if tracking: + wandb_metrics = { + f"train/{key}": float(value.item()) + for key, value in reduced.items() + } + wandb_metrics.update( + { + f"train/lr_group_{index}": lr + for index, lr in enumerate(lrs) + } + ) + wandb_metrics["train/epoch"] = epoch + wandb_metrics["train/phase_step"] = global_step + accelerator.log( + wandb_metrics, + step=tracking_step_offset + global_step, + ) + if accelerator.is_main_process: + append_jsonl(log_path, record) + print(json.dumps(record, sort_keys=True), flush=True) + + if ( + global_step < max_steps + and global_step % int(config["training"]["validate_every"]) == 0 + ): + validation = run_sampling_validation( + model, + dataset, + accelerator, + config, + output_dir, + global_step, + int(config["validation"]["periodic_num_samples"]), + ) + if tracking: + accelerator.log( + { + f"validation/{key}": value + for key, value in validation.items() + }, + step=tracking_step_offset + global_step, + ) + if accelerator.is_main_process: + append_jsonl( + log_path, + { + "phase": phase, + "global_step": global_step, + **validation, + }, + ) + if ( + bool(config["training"].get("save_training_state", True)) + and global_step % int(config["training"]["save_every"]) == 0 + ): + state_dir = ( + output_dir + / "checkpoints" + / f"state_{global_step:06d}" + ) + accelerator.save_state(str(state_dir)) + if accelerator.is_main_process: + (state_dir / "trainer_state.json").write_text( + json.dumps( + {"phase": phase, "global_step": global_step}, + indent=2, + ), + encoding="utf-8", + ) + if should_export_inference_checkpoint( + global_step, export_model_every + ): + export_action_mot( + model, + accelerator, + inference_checkpoint_path(output_dir, phase, global_step), + ) + last_exported_step = global_step + + final = run_sampling_validation( + model, + dataset, + accelerator, + config, + output_dir, + global_step, + int(config["validation"]["final_num_samples"]), + ) + if tracking: + accelerator.log( + {f"final/{key}": value for key, value in final.items()}, + step=tracking_step_offset + global_step, + ) + if accelerator.is_main_process: + append_jsonl( + log_path, + {"phase": phase, "global_step": global_step, **final}, + ) + if last_exported_step != global_step: + export_action_mot( + model, + accelerator, + inference_checkpoint_path(output_dir, phase, global_step), + ) + if masks is not None: + masks.close() + accelerator.end_training() + + +def main() -> None: + args = parse_args() + config = validate_config(load_config(args.config)) + if args.validate_only: + print( + json.dumps( + { + "status": "valid", + "episodes": config["_num_episodes"], + "samples": config["_num_samples"], + "modalities": ["rgb", "depth", "world_point", "action", "camera"], + "action_shape": [ + int(config["data"]["num_frames"]) - 1, + int(config["action"]["action_dim"]), + ], + "camera_shape": [ + int(config["data"]["num_frames"]) - 1, + int(config["camera"]["camera_dim"]), + ], + "initial_state_dim": int( + config["model"]["action_state_dim"] + ), + "prompt_source": config["data"]["prompt_source"], + "cache_dir": config["paths"]["cache_dir"], + "action_persisted": False, + "wandb_installed": ( + importlib.util.find_spec("wandb") is not None + ), + }, + indent=2, + ) + ) + return + if args.dataset_smoke_test: + dataset_smoke_test(config) + return + if args.phase is None or args.output_dir is None: + raise ValueError("Training requires --phase and --output-dir") + train( + config, + args.phase, + args.output_dir, + args.resume_model, + args.resume_state, + ) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_piper_rgb_state_joint_action_quantile_4way.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_piper_rgb_state_joint_action_quantile_4way.py new file mode 100644 index 0000000000000000000000000000000000000000..a7f09237637fa91a605dd540aa86316a0da5b5d1 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_piper_rgb_state_joint_action_quantile_4way.py @@ -0,0 +1,627 @@ +#!/usr/bin/env python3 +"""Train FastWAM on native 14D Piper joint commands with q01/q99 scaling.""" + +from __future__ import annotations + +import json +import math +import os +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +import piper_hdf5 +import train_robodojo_rgb_state_joint_action_4way as fourway + + +NORMALIZATION_TYPE = "piper_joint_action_q01_q99_v1" +ACTION_REPRESENTATION = "dual_arm_joint_position_plus_gripper" +ACTION_SOURCE = "native_hdf5_action_t" +PRETRAINED_INITIALIZATION_MODE = fourway.V2_BASEFRAME_FULL_ACTION_INITIALIZATION_MODE +SOURCE_NORMALIZER_TRANSFER = fourway.SOURCE_ACTION_NORMALIZER_TRANSFER +TARGET_NORMALIZER_TRANSFER = fourway.TARGET_ACTION_NORMALIZER_TRANSFER +ACTION_DIMS = tuple(range(14)) +LOW_QUANTILE = 0.01 +HIGH_QUANTILE = 0.99 +QUANTILE_EPSILON = 1.0e-6 + +_ORIGINAL_LOAD_CONFIG = fourway.load_config +_ORIGINAL_PREPARE_CACHE = fourway.prepare_episode_length_normalized_cache +_ORIGINAL_CHECKPOINT_METADATA = fourway.checkpoint_metadata + + +def load_piper_config(path: Path) -> dict[str, Any]: + return _ORIGINAL_LOAD_CONFIG(path) + + +def install_piper_data_adapter(config: dict[str, Any] | None = None) -> None: + """Install the native-HDF5 hooks used by the inherited training loop.""" + + base = fourway.base + base.RoboDojoEpisode = piper_hdf5.PiperEpisode + base.selected_task_prompts = piper_hdf5.selected_task_prompts + base.dataset_smoke_test = piper_hdf5.dataset_smoke_test + base.clip_config_from_config = piper_hdf5.clip_config_from_config + base.resolve_splits = piper_hdf5.resolve_splits + base.data_parquet_path = piper_hdf5.data_hdf5_path + base.load_data_shard = piper_hdf5.load_data_shard + base.episode_arrays = piper_hdf5.episode_arrays + base.video_path = lambda root, episode, video_key: piper_hdf5.data_hdf5_path( + root, episode + ) + base.decode_rgb_window = piper_hdf5.decode_rgb_window + base.build_egoverse_delta_targets = piper_hdf5.build_joint_command_targets + if config is not None: + piper_hdf5.configure_source(config) + + +def _normalization_contract(config: dict[str, Any]) -> dict[str, Any]: + expected = { + "type": NORMALIZATION_TYPE, + "fit": "piper_train_split_cache", + "quantiles": [LOW_QUANTILE, HIGH_QUANTILE], + "epsilon": QUANTILE_EPSILON, + "normalized_range": [-1.0, 1.0], + "normalized_dims": list(ACTION_DIMS), + "gripper_dims": [6, 13], + "gripper_semantics": "continuous_joint_position_command", + "clip": None, + "checkpoint_buffers": True, + } + normalization = config["action"]["normalization"] + actual = {key: normalization.get(key) for key in expected} + if actual != expected: + raise ValueError(f"Piper q01/q99 normalization mismatch: {actual} != {expected}") + return expected + + +def _validate_source_contract(config: dict[str, Any]) -> None: + source = config["piper"] + expected = { + "format": "aloha_hdf5", + "image_key": "observations/images/cam_high", + "state_key": "observations/qpos", + "action_key": "action", + "image_color_order": "rgb", + "action_alignment": "action_t_approximately_observation_qpos_t_plus_1", + } + actual = {key: source.get(key) for key in expected} + if actual != expected: + raise ValueError(f"Piper source contract mismatch: {actual} != {expected}") + qualities = [str(value) for value in source.get("qualities", [])] + if qualities != ["perfect"]: + raise ValueError("Only Piper perfect trajectories may enter this training run") + quality_to_split = source.get("quality_to_split") + if quality_to_split != {"perfect": "train"}: + raise ValueError("Piper perfect trajectories must be explicitly assigned to train") + excluded = sorted(str(value) for value in source.get("exclude_files", [])) + expected_excluded = ["assemble_battery_long/perfect/episode_93.hdf5"] + if excluded != expected_excluded: + raise ValueError( + "The known-corrupt Piper episode must be the sole explicit exclusion: " + f"{excluded} != {expected_excluded}" + ) + + +def validate_piper_config(config: dict[str, Any]) -> dict[str, Any]: + """Fail closed on data, shuffle, action, precision, and epoch geometry.""" + + required = { + "experiment", + "paths", + "model", + "data", + "conditioning", + "diffusion", + "training", + "action", + "piper", + "cache", + "sampling", + "initialization", + "experiment_variant", + } + missing = required - set(config) + if missing: + raise KeyError(f"Missing Piper config sections: {sorted(missing)}") + forbidden = {"robodojo", "depth", "point_cloud", "camera"}.intersection(config) + if forbidden: + raise ValueError(f"Unrelated source/modalities are forbidden: {sorted(forbidden)}") + if config["experiment_variant"] != {"objective": "joint"}: + raise ValueError("The Piper sibling supports the joint RGB+action objective only") + + for key in ("pretrained_model_dir", "dataset_root"): + path = Path(config["paths"][key]) + if not path.is_dir(): + raise FileNotFoundError(path) + index_path = Path(config["paths"]["piper_index_file"]) + if not index_path.is_file(): + raise FileNotFoundError( + f"Piper index missing: {index_path}; run build_piper_hdf5_index.py first" + ) + install_piper_data_adapter(config) + index = piper_hdf5.load_episode_index(config, verify_source_stats=True) + _validate_source_contract(config) + + data = config["data"] + expected_data = { + "num_frames": 33, + "resolution": 256, + "fps": 25.0, + "batch_size": 8, + "num_workers": 0, + } + actual_data = { + "num_frames": int(data["num_frames"]), + "resolution": int(data["resolution"]), + "fps": float(data["fps"]), + "batch_size": int(data["batch_size"]), + "num_workers": int(data.get("num_workers", 0)), + } + if actual_data != expected_data: + raise ValueError(f"Piper data geometry mismatch: {actual_data} != {expected_data}") + if config["conditioning"] != { + "rgb_frame0": "clean_condition", + "initial_state": "clean_action_context_token", + "exclude_rgb_frame0_from_loss": True, + }: + raise ValueError("Piper conditioning must use clean RGB frame0 and qpos context") + + action = config["action"] + expected_action = { + "enabled": True, + "source": ACTION_SOURCE, + "source_layout": "left_joint6_gripper_right_joint6_gripper", + "representation": ACTION_REPRESENTATION, + "target_formula": "recorded_piper_joint_command_t", + "integration": "none", + "action_dim": 14, + "horizon": "data_num_frames_minus_one", + } + actual_action = {key: action.get(key) for key in expected_action} + if actual_action != expected_action: + raise ValueError(f"Piper action contract mismatch: {actual_action} != {expected_action}") + expected_initial = { + "representation": ACTION_REPRESENTATION, + "source": "observations/qpos_at_sample_first_frame", + "add_diffusion_noise": False, + } + actual_initial = {key: action["initial_state"].get(key) for key in expected_initial} + if actual_initial != expected_initial: + raise ValueError(f"Piper initial-state contract mismatch: {actual_initial} != {expected_initial}") + _normalization_contract(config) + + sampling = config["sampling"] + expected_sampling = { + "mode": fourway.FASTWAM_TAIL_PAD_SAMPLING_MODE, + "window_size": 33, + "replacement": False, + "shuffle_each_epoch": True, + "epoch_seed": int(data["sample_order_seed"]), + "samples_per_epoch": "all_episode_frames", + "tail_padding": { + "image": "repeat_last", + "action": "repeat_last", + "loss": "mask_padded", + }, + } + if sampling != expected_sampling: + raise ValueError( + "Piper sampling must exhaustively reshuffle every epoch without replacement: " + f"{sampling} != {expected_sampling}" + ) + + train_episodes = int(index["split_episode_counts"]["train"]) + train_windows = int(index["split_frame_counts"]["train"]) + if train_episodes != 423 or train_windows != 545_356: + raise RuntimeError( + f"Validated Piper index geometry changed: episodes={train_episodes}, " + f"windows={train_windows}" + ) + if int(index["split_episode_counts"]["validation"]) != 0 or int( + index["split_episode_counts"]["test"] + ) != 0: + raise ValueError("This Piper sibling assigns all valid episodes to train") + + training = config["training"] + expected_global_batch = ( + int(data["batch_size"]) + * int(training["expected_num_processes"]) + * int(training["gradient_accumulation_steps"]) + ) + if expected_global_batch != 256 or int(training["global_batch_size"]) != 256: + raise ValueError(f"Piper global batch must be 256, got {expected_global_batch}") + if int(training["valid_windows_per_epoch"]) != train_windows: + raise ValueError("training.valid_windows_per_epoch does not match the Piper index") + epochs = int(training["exact_epochs"]) + required_steps = epochs * math.ceil(train_windows / expected_global_batch) + smoke_mode = os.environ.get("ROBODOJO_4WAY_SMOKE", "0") == "1" + if smoke_mode and 1 <= int(training["max_steps"]) <= 50: + config.setdefault("tracking", {})["enabled"] = False + elif epochs != 20 or int(training["max_steps"]) != required_steps or required_steps != 42_620: + raise ValueError( + f"Twenty Piper epochs require 42,620 optimizer steps, got {training['max_steps']}" + ) + if int(training["max_steps_cap"]) != required_steps: + raise ValueError("training.max_steps_cap must equal the exact 20-epoch step count") + if int(training["save_every"]) != 10_000: + raise ValueError("Piper checkpoint interval must remain 10,000 steps") + if not bool(training.get("fp32_master_weights", False)): + raise ValueError("Piper experiment requires FP32 master weights") + if not bool(training.get("train_video_expert", False)): + raise ValueError("Joint Piper training must train the video expert") + + model = config["model"] + model_contract = { + "ablation_architecture": "rgb_action_mot", + "dtype": "bfloat16", + "action_initialization": "random", + "action_state_dim": 14, + "cross_modal_attention": "bidirectional", + "action_loss_to_video_stop_gradient": False, + } + actual_model = {key: model.get(key) for key in model_contract} + if actual_model != model_contract: + raise ValueError(f"Piper model contract mismatch: {actual_model} != {model_contract}") + checkpoint_flags = ( + bool(model.get("mot_checkpoint_mixed_attn", True)), + bool(model["dit"].get("use_gradient_checkpointing", True)), + bool(model["action_dit"].get("use_gradient_checkpointing", True)), + ) + if checkpoint_flags != (False, False, False): + raise ValueError(f"All model checkpointing flags must be disabled: {checkpoint_flags}") + if model["dit"].get("video_attention_mask_mode") != "bidirectional": + raise ValueError("Piper video attention must be bidirectional") + if model["dit"].get("seperated_timestep") is not True: + raise ValueError("Piper video/action diffusion timesteps must be independent") + + diffusion = config["diffusion"] + if float(diffusion["lambda_rgb"]) != 1.0 or float(diffusion["lambda_action"]) != 1.0: + raise ValueError("Piper RGB and joint-action loss weights must both be 1.0") + if int(diffusion["num_train_timesteps"]) != 1000 or int( + diffusion["action_num_train_timesteps"] + ) != 1000: + raise ValueError("Piper RGB/action flow schedulers must use 1000 train timesteps") + if float(diffusion["train_shift"]) != 5.0 or float(diffusion["action_train_shift"]) != 5.0: + raise ValueError("Piper RGB/action flow shifts must both be 5.0") + initialization = config["initialization"] + if initialization == { + "mode": "fresh", + "local_wan_dit_dir": "/tmp/robodojo_wam_gc_model", + }: + pass + elif str(initialization.get("mode")) == PRETRAINED_INITIALIZATION_MODE: + required = { + "checkpoint", + "contract_sidecar", + "source_sha256", + "source_action_contract_fingerprint", + "flavor", + "action_normalizer_transfer", + "local_wan_dit_dir", + } + missing_initialization = required - set(initialization) + if missing_initialization: + raise ValueError( + "Piper pretrained initialization is incomplete: " + f"{sorted(missing_initialization)}" + ) + if str(initialization["flavor"]) != "camera6d": + raise ValueError("Piper pretrained transfer requires flavor=camera6d") + if str(initialization["action_normalizer_transfer"]) not in { + SOURCE_NORMALIZER_TRANSFER, + TARGET_NORMALIZER_TRANSFER, + }: + raise ValueError( + "Piper transfer must explicitly preserve the source normalizer or refit Piper q01/q99" + ) + if str(initialization["local_wan_dit_dir"]) != "/tmp/robodojo_wam_gc_model": + raise ValueError("Piper local Wan staging path changed") + checkpoint = Path(str(initialization["checkpoint"])) + sidecar = Path(str(initialization["contract_sidecar"])) + if not checkpoint.is_file() or not sidecar.is_file(): + raise FileNotFoundError(checkpoint if not checkpoint.is_file() else sidecar) + for key in ("source_sha256", "source_action_contract_fingerprint"): + value = str(initialization[key]) + if len(value) != 64: + raise ValueError(f"initialization.{key} is not a SHA256 value") + try: + int(value, 16) + except ValueError as error: + raise ValueError(f"initialization.{key} is not hexadecimal") from error + sidecar_payload = json.loads(sidecar.read_text(encoding="utf-8")) + if sidecar_payload.get("checkpoint", {}).get("sha256") != str( + initialization["source_sha256"] + ): + raise ValueError("Piper initialization sidecar checkpoint SHA256 mismatch") + if sidecar_payload.get("action_contract", {}).get("fingerprint") != str( + initialization["source_action_contract_fingerprint"] + ): + raise ValueError("Piper initialization sidecar action fingerprint mismatch") + else: + raise ValueError( + "Piper initialization must be either the historical fresh baseline or the " + "sealed one-epoch SpaceDreamer full-model transfer" + ) + + dataset_root = Path(config["paths"]["dataset_root"]).resolve() + cache_dir = Path(config["paths"]["cache_dir"]).resolve() + output_dir = Path(config["experiment"]["output_dir"]).resolve() + for candidate, name in ((cache_dir, "cache"), (output_dir, "output")): + if candidate == dataset_root or candidate.is_relative_to(dataset_root): + raise ValueError(f"Piper {name} directory must not be inside the source dataset") + if cache_dir == output_dir or cache_dir.is_relative_to(output_dir) or output_dir.is_relative_to(cache_dir): + raise ValueError("Piper cache and run output directories must be separate") + + fourway._ACTIVE_CONFIG = config + return config + + +def weighted_linear_quantile( + values: np.ndarray, integer_weights: np.ndarray, quantile: float +) -> float: + """Match ``np.quantile(np.repeat(values, weights), q)`` without expansion.""" + + values = np.asarray(values, dtype=np.float64) + weights = np.asarray(integer_weights, dtype=np.int64) + if values.ndim != 1 or weights.ndim != 1 or values.shape != weights.shape: + raise ValueError("Weighted quantile expects matching one-dimensional arrays") + if not 0.0 <= float(quantile) <= 1.0: + raise ValueError("Quantile must be in [0, 1]") + if not np.isfinite(values).all() or np.any(weights < 0): + raise ValueError("Weighted quantile inputs must be finite with nonnegative weights") + positive = weights > 0 + if not np.any(positive): + raise ValueError("Weighted quantile requires positive total weight") + values = values[positive] + weights = weights[positive] + order = np.argsort(values, kind="stable") + sorted_values = values[order] + cumulative = np.cumsum(weights[order], dtype=np.int64) + total = int(cumulative[-1]) + position = (total - 1) * float(quantile) + lower_rank = int(np.floor(position)) + upper_rank = int(np.ceil(position)) + + def value_at(rank: int) -> float: + index = int(np.searchsorted(cumulative, rank, side="right")) + return float(sorted_values[index]) + + lower = value_at(lower_rank) + upper = value_at(upper_rank) + return lower + (position - lower_rank) * (upper - lower) + + +def exhaustive_piper_action_quantile_statistics( + config: dict[str, Any], episodes: list[Any] +) -> tuple[torch.Tensor, torch.Tensor, int]: + """Fit exact q01/q99 statistics to all valid tokens of all frame-start windows.""" + + _normalization_contract(config) + horizon = int(config["sampling"]["window_size"]) - 1 + value_parts: list[np.ndarray] = [] + weight_parts: list[np.ndarray] = [] + for number, episode in enumerate(episodes, start=1): + state, command, _ = fourway._load_episode_arrays(config, episode) + length = int(episode.length) + indices = np.arange(length, dtype=np.int64) + indices = np.concatenate([indices, indices[-1:]]) + action = piper_hdf5.build_joint_command_targets( + state, command, indices + )["action_raw"].numpy() + if action.shape != (length, 14): + raise RuntimeError("Tail-padded Piper action-statistics length mismatch") + counts = fourway.valid_action_occurrence_counts(length, horizon) + value_parts.append(np.asarray(action, dtype=np.float32)) + weight_parts.append(counts) + if number == 1 or number % 50 == 0 or number == len(episodes): + print( + f"[piper-quantile] {number}/{len(episodes)} " + f"episode={episode.episode_index} frames={length}", + flush=True, + ) + values = np.concatenate(value_parts, axis=0) + weights = np.concatenate(weight_parts, axis=0) + weighted_count = int(weights.sum()) + center = np.empty(14, dtype=np.float32) + scale = np.empty(14, dtype=np.float32) + for dimension in ACTION_DIMS: + q01 = weighted_linear_quantile(values[:, dimension], weights, LOW_QUANTILE) + q99 = weighted_linear_quantile(values[:, dimension], weights, HIGH_QUANTILE) + width = q99 - q01 + if not np.isfinite(width) or width <= 1.0e-8: + raise RuntimeError(f"Degenerate Piper q01/q99 interval for dimension {dimension}") + scale[dimension] = np.float32(0.5 * (width + QUANTILE_EPSILON)) + center[dimension] = np.float32(q01 + scale[dimension]) + print( + f"[piper-quantile] dim={dimension:02d} q01={q01:.9f} q99={q99:.9f}", + flush=True, + ) + return torch.from_numpy(center), torch.from_numpy(scale), weighted_count + + +@torch.no_grad() +def prepare_piper_cache( + config: dict[str, Any], *, device: torch.device, overwrite: bool +) -> None: + preserve_source = ( + str(config.get("initialization", {}).get("action_normalizer_transfer", "")) + == SOURCE_NORMALIZER_TRANSFER + ) + original_statistics = fourway._exhaustive_action_statistics + target_statistics_called = False + if preserve_source: + def source_statistics( + active_config: dict[str, Any], episodes: list[Any] + ) -> tuple[torch.Tensor, torch.Tensor, int]: + del episodes + initialization = active_config["initialization"] + local_checkpoint = initialization.get("local_checkpoint") + checkpoint = Path( + local_checkpoint + if local_checkpoint and Path(local_checkpoint).is_file() + else initialization["checkpoint"] + ) + payload = torch.load( + checkpoint, map_location="cpu", weights_only=True, mmap=True + ) + state = payload.get("state_dict") if isinstance(payload, dict) else None + if not isinstance(state, dict): + raise RuntimeError(f"Invalid source checkpoint payload: {checkpoint}") + required = { + name: state.get(f"action_normalizer.{name}") + for name in ("mean", "std", "count", "fitted") + } + if any(value is None for value in required.values()): + raise RuntimeError("Source checkpoint is missing action-normalizer buffers") + mean = required["mean"].float().clone() + std = required["std"].float().clone() + count = int(required["count"].item()) + if tuple(mean.shape) != (14,) or tuple(std.shape) != (14,): + raise RuntimeError("Source action normalizer must have 14 dimensions") + if not bool(required["fitted"].item()) or count < 2: + raise RuntimeError("Source action normalizer is not fitted") + if not torch.isfinite(mean).all() or not torch.isfinite(std).all() or not torch.all(std > 0): + raise RuntimeError("Source action normalizer contains invalid values") + print( + "[piper-cache] importing source pretraining q01/q99 statistics; " + f"count={count} target_piper_fit_skipped=true", + flush=True, + ) + return mean, std, count + + fourway._exhaustive_action_statistics = source_statistics + else: + # This function is also called by the standalone cluster metadata + # preflight, before main() installs the Piper training hooks. + def target_statistics( + active_config: dict[str, Any], episodes: list[Any] + ) -> tuple[torch.Tensor, torch.Tensor, int]: + nonlocal target_statistics_called + target_statistics_called = True + return exhaustive_piper_action_quantile_statistics(active_config, episodes) + + fourway._exhaustive_action_statistics = target_statistics + try: + _ORIGINAL_PREPARE_CACHE(config, device=device, overwrite=overwrite) + finally: + fourway._exhaustive_action_statistics = original_statistics + cache_dir = Path(config["paths"]["cache_dir"]) + stats_path = cache_dir / "action_normalizer.pt" + payload = torch.load(stats_path, map_location="cpu", weights_only=True) + if not preserve_source and not target_statistics_called and payload.get( + "fit_proof" + ) != "piper_train_split_weighted_q01_q99_exact_v1": + raise RuntimeError("Piper q01/q99 statistics were not computed or sealed") + center = payload["mean"].float() + scale = payload["std"].float() + if preserve_source: + payload.update( + { + "normalization_type": SOURCE_NORMALIZER_TRANSFER, + "source_action_semantics": "dual_arm_absolute_xyz_so3_gripper", + "target_action_semantics": ACTION_REPRESENTATION, + "semantic_equivalence_claimed": False, + } + ) + else: + payload.update( + { + "fit_proof": "piper_train_split_weighted_q01_q99_exact_v1", + "normalization_type": NORMALIZATION_TYPE, + "quantiles": torch.tensor([LOW_QUANTILE, HIGH_QUANTILE]), + "epsilon": torch.tensor(QUANTILE_EPSILON), + "normalized_range": torch.tensor([-1.0, 1.0]), + "normalized_dims": torch.tensor(ACTION_DIMS, dtype=torch.long), + "gripper_dims": torch.tensor([6, 13], dtype=torch.long), + "gripper_semantics": "continuous_joint_position_command", + "q01": center - scale, + "q99": center + scale - QUANTILE_EPSILON, + } + ) + temporary = stats_path.with_name(f"{stats_path.name}.piper.{os.getpid()}.tmp") + torch.save(payload, temporary) + os.replace(temporary, stats_path) + + manifest_path = cache_dir / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["source"] = "piper_native_aloha_hdf5" + manifest["modalities"] = ["rgb_online", "joint_action", "joint_initial_state"] + manifest["excluded_source_files"] = list(config["piper"]["exclude_files"]) + manifest["action_normalizer_usage"] = ( + "source_pretraining_checkpoint" + if preserve_source + else "piper_train_split_q01_q99" + ) + manifest_tmp = manifest_path.with_name(f"{manifest_path.name}.piper.{os.getpid()}.tmp") + manifest_tmp.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + os.replace(manifest_tmp, manifest_path) + print( + f"[piper-cache] q01/q99 metadata ready count={int(payload['count'])} " + f"path={manifest_path}", + flush=True, + ) + + +def piper_checkpoint_metadata(model: torch.nn.Module) -> dict[str, Any]: + metadata = _ORIGINAL_CHECKPOINT_METADATA(model) + metadata["data_source"] = { + "format": "piper_native_aloha_hdf5", + "image": "observations/images/cam_high", + "initial_state": "observations/qpos[sample_first_frame]", + "excluded_source_files": [ + "assemble_battery_long/perfect/episode_93.hdf5" + ], + } + metadata["action_semantics"] = { + "representation": ACTION_REPRESENTATION, + "source": ACTION_SOURCE, + "target_formula": "recorded_piper_joint_command_t", + "alignment": "action_t_approximately_observation_qpos_t_plus_1", + "integration": "none", + "gripper": "continuous_recorded_joint_command", + "dimensions": list(ACTION_DIMS), + } + metadata["action_normalization"] = { + "type": NORMALIZATION_TYPE, + "quantiles": [LOW_QUANTILE, HIGH_QUANTILE], + "epsilon": QUANTILE_EPSILON, + "normalized_range": [-1.0, 1.0], + "implicit_clip": False, + "statistics_storage": { + "center": "action_normalizer.mean", + "half_range": "action_normalizer.std", + }, + } + initialization = getattr(model, "_robodojo_initialization", {}) + if initialization.get("action_normalizer") == SOURCE_NORMALIZER_TRANSFER: + metadata["cross_embodiment_transfer"] = { + "source_action_semantics": "dual_arm_absolute_xyz_so3_gripper", + "target_action_semantics": ACTION_REPRESENTATION, + "source_normalizer_reused_by_explicit_experiment_request": True, + "semantic_equivalence_claimed": False, + } + return metadata + + +def piper_cache_fingerprint( + config: dict[str, Any], splits: dict[str, list[Any]] +) -> str: + return piper_hdf5.cache_fingerprint(config, splits) + + +def main() -> None: + install_piper_data_adapter() + fourway.load_config = load_piper_config + fourway.validate_config = validate_piper_config + fourway.episode_length_normalized_cache_fingerprint = piper_cache_fingerprint + fourway._exhaustive_action_statistics = exhaustive_piper_action_quantile_statistics + fourway.prepare_episode_length_normalized_cache = prepare_piper_cache + fourway.checkpoint_metadata = piper_checkpoint_metadata + fourway.main() + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_geometry_action_absolute_ee.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_geometry_action_absolute_ee.py new file mode 100644 index 0000000000000000000000000000000000000000..aa5d95bc9ff2ee9a1885ac86a9cb114c54d432eb --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_geometry_action_absolute_ee.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""Controlled RoboDojo RGB/depth[/point] + absolute-EE action experiments. + +This adapter intentionally installs its changes only after the existing +absolute-EE/exhaustive-window adapters have installed theirs. The original +RGB+action entrypoint and policy-server sources remain untouched. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +import train_robodojo_rgb_state_joint_action_4way as fourway +import train_robodojo_rgb_state_joint_action_absolute_ee_4way as absolute +import train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way as quantile +from joint_rgb_geometry_action_diffusion import joint_geometry_action_flow_matching_loss +from robodojo_geometry_data import ( + LATENT_SHAPE, + build_geometry_vae_inputs, + depth_episode_path, + depth_source_record, + geometry_cache_fingerprint, + geometry_modalities, + load_depth_episode_bundle, + validate_geometry_config, +) +from robodojo_rgb_geometry_action_model import build_fresh_rgb_geometry_action_mot + + +base = fourway.base +_ORIGINAL_BASE_MAIN = base.main +_GEOMETRY_SHARDS: dict[str, torch.Tensor] = {} + + +def resolve_geometry_config(path: Path) -> dict[str, Any]: + """Resolve and fully validate a geometry config outside the patched CLI.""" + + config = absolute.load_absolute_ee_config(path) + geometry = config.pop("geometry", None) + if not isinstance(geometry, dict): + raise ValueError("Config must define geometry") + previous = fourway._original_validate_config + # The controlled GBS256 baseline uses the pi0.5 q01/q99 action adapter on + # top of absolute EE targets. Config-only consumers (cache builders and + # verifiers) must run the same validator even though quantile.main() has + # not installed its runtime hooks yet. + fourway._original_validate_config = quantile.validate_quantile_absolute_ee + try: + validated = fourway.validate_config(config) + finally: + fourway._original_validate_config = previous + config["geometry"] = geometry + return validate_geometry_config(validated) + + +def _load_geometry_shard( + dataset: "GeometryExhaustiveWindowDataset", + modality: str, + episode_index: int, +) -> torch.Tensor: + cache_dir = dataset.geometry_cache_dirs[modality] + path = cache_dir / f"episode_{episode_index:06d}.pt" + key = f"{modality}:{path}" + tensor = _GEOMETRY_SHARDS.get(key) + if tensor is not None: + return tensor + payload = torch.load(path, map_location="cpu", weights_only=True, mmap=True) + expected_fingerprint = dataset.geometry_fingerprints[modality] + if payload.get("source_fingerprint") != dataset.expected_fingerprint: + raise RuntimeError(f"{modality} shard source fingerprint mismatch: {path}") + if payload.get("geometry_fingerprint") != expected_fingerprint: + raise RuntimeError(f"{modality} shard geometry fingerprint mismatch: {path}") + source_path = depth_episode_path(dataset.depth_root, episode_index) + if payload.get("depth_source") != depth_source_record(source_path): + raise RuntimeError(f"Depth source changed after {modality} cache creation: {source_path}") + slot = dataset.episode_slot_by_id[episode_index] + expected = (int(dataset.valid_starts[slot]), *LATENT_SHAPE) + tensor = payload[f"z_{modality}"] + if tuple(tensor.shape) != expected or tensor.dtype != torch.bfloat16: + raise RuntimeError( + f"{modality} shard mismatch: {path} shape={tuple(tensor.shape)} " + f"dtype={tensor.dtype} expected={expected}" + ) + _GEOMETRY_SHARDS[key] = tensor + return tensor + + +class GeometryExhaustiveWindowDataset(fourway.ExhaustiveWindowDataset): + """Add aligned precomputed geometry latents to the unchanged window index.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.modalities = geometry_modalities(self.config) + self.depth_root = Path(self.config["paths"]["depth_dataset_root"]) + self.geometry_cache_dirs = { + "depth": Path(self.config["paths"]["depth_latent_cache_dir"]), + } + if "point" in self.modalities: + self.geometry_cache_dirs["point"] = Path( + self.config["paths"]["point_latent_cache_dir"] + ) + self.geometry_fingerprints = { + name: geometry_cache_fingerprint( + self.config, self.expected_fingerprint, name + ) + for name in self.geometry_cache_dirs + } + for name, cache_dir in self.geometry_cache_dirs.items(): + manifest_path = cache_dir / "manifest.json" + if not manifest_path.is_file(): + raise FileNotFoundError(manifest_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + expected_manifest = { + "format_version": 1, + "modality": name, + "source_fingerprint": self.expected_fingerprint, + "geometry_fingerprint": self.geometry_fingerprints[name], + "dtype": "bfloat16", + "latent_shape": list(LATENT_SHAPE), + } + actual = {key: manifest.get(key) for key in expected_manifest} + if actual != expected_manifest: + raise RuntimeError( + f"Geometry latent manifest mismatch: {actual} != {expected_manifest}" + ) + print( + f"[geometry] modalities={list(self.modalities)} " + f"depth_root={self.depth_root}", + flush=True, + ) + + def __getitem__(self, index: int) -> dict[str, Any]: + sample = super().__getitem__(index) + episode_index = int(sample["episode_index"].item()) + window_start = int(sample["window_start"].item()) + sample["z_depth"] = _load_geometry_shard( + self, "depth", episode_index + )[window_start] + if "point" in self.modalities: + sample["z_point"] = _load_geometry_shard( + self, "point", episode_index + )[window_start] + return sample + + +def collate_geometry(samples: list[dict[str, Any]]) -> dict[str, Any]: + modalities = ["z_rgb", "z_depth"] + if "z_point" in samples[0]: + modalities.append("z_point") + required = modalities + [ + "action", + "action_is_pad", + "initial_state", + "context", + "context_mask", + "episode_index", + "window_start", + "sample_probability", + ] + for key in required: + if not all(key in sample for sample in samples): + raise RuntimeError(f"Geometry batch is missing {key}") + return { + **{key: torch.stack([sample[key] for sample in samples]) for key in required}, + "sample_id": [sample["sample_id"] for sample in samples], + "prompt": [sample["prompt"] for sample in samples], + } + + +def compute_geometry_loss( + model: torch.nn.Module, + batch: dict[str, Any], + video_scheduler: Any, + action_scheduler: Any, + config: dict[str, Any], + device: torch.device, + dtype: torch.dtype, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + z_rgb = batch["z_rgb"].to(device=device, dtype=dtype, non_blocking=True) + z_depth = batch["z_depth"].to(device=device, dtype=dtype, non_blocking=True) + z_point = ( + batch["z_point"].to(device=device, dtype=dtype, non_blocking=True) + if "z_point" in batch + else None + ) + action = batch["action"].to(device=device, dtype=dtype, non_blocking=True) + initial_state = batch["initial_state"].to( + device=device, dtype=dtype, non_blocking=True + ) + mask_cpu = batch["context_mask"] + valid_columns = mask_cpu.any(dim=0) + context_length = int(valid_columns.nonzero(as_tuple=False)[-1, 0]) + 1 + context = batch["context"][:, :context_length].to( + device=device, dtype=dtype, non_blocking=True + ) + context_mask = mask_cpu[:, :context_length].to( + device=device, dtype=torch.bool, non_blocking=True + ) + return joint_geometry_action_flow_matching_loss( + model, + video_scheduler, + action_scheduler, + z_rgb=z_rgb, + z_depth=z_depth, + z_point=z_point, + action=action, + initial_state=initial_state, + context=context, + context_mask=context_mask, + action_is_pad=batch["action_is_pad"].to( + device=device, dtype=torch.bool, non_blocking=True + ), + lambda_rgb=float(config["diffusion"]["lambda_rgb"]), + lambda_depth=float(config["diffusion"]["lambda_depth"]), + lambda_point=( + float(config["diffusion"]["lambda_point"]) + if z_point is not None + else None + ), + lambda_action=float(config["diffusion"]["lambda_action"]), + ) + + +def geometry_dataset_smoke_test(config: dict[str, Any]) -> None: + splits = base.resolve_splits(config) + episode = splits["train"][0] + indices = np.arange(int(config["data"]["num_frames"]), dtype=np.int64) + bundle = load_depth_episode_bundle( + Path(config["paths"]["depth_dataset_root"]), + int(episode.episode_index), + expected_length=int(episode.length), + ) + geometry = build_geometry_vae_inputs(bundle, indices, config) + root = Path(config["paths"]["dataset_root"]) + shard = base.load_data_shard(base.data_parquet_path(root, episode)) + state, action, timestamps = base.episode_arrays(shard, episode) + targets = base.build_egoverse_delta_targets(state, action, indices) + report: dict[str, Any] = { + "status": "robodojo_geometry_absolute_ee_data_ok", + "modalities": list(geometry_modalities(config)), + "depth_dataset_root": str(config["paths"]["depth_dataset_root"]), + "depth_episode_count": 3500, + "selected_task_count": len(base.selected_task_prompts(config)), + "split_episode_counts": {name: len(value) for name, value in splits.items()}, + "episode_index": int(episode.episode_index), + "episode_length": int(episode.length), + "depth_shape": list(geometry["depth"].shape), + "depth_valid_fraction": float(geometry["depth_valid_fraction"]), + "action_shape": list(targets["action_raw"].shape), + "initial_state_shape": list(targets["initial_state"].shape), + "timestamp_delta_range_s": [ + float(np.diff(timestamps[indices]).min()), + float(np.diff(timestamps[indices]).max()), + ], + } + if "point" in geometry: + report.update( + { + "point_shape": list(geometry["point"].shape), + "point_valid_fraction": float(geometry["point_valid_fraction"]), + "point_clipped_fraction": float(geometry["point_clipped_fraction"]), + "point_normalized_range": [ + float(geometry["point"].min()), + float(geometry["point"].max()), + ], + } + ) + print(json.dumps(report, indent=2), flush=True) + + +def _patched_base_main() -> None: + inherited_validate = base.validate_config + + def validate_runtime(config: dict[str, Any]) -> dict[str, Any]: + geometry = config.pop("geometry", None) + if not isinstance(geometry, dict): + raise ValueError("Config must define geometry") + try: + validated = inherited_validate(config) + finally: + config["geometry"] = geometry + return validate_geometry_config(validated) + + base.validate_config = validate_runtime + base.RoboDojoCachedDataset = GeometryExhaustiveWindowDataset + base.collate_rgb_action = collate_geometry + base.compute_loss = compute_geometry_loss + base.build_fresh_rgb_action_mot = build_fresh_rgb_geometry_action_mot + base.dataset_smoke_test = geometry_dataset_smoke_test + _ORIGINAL_BASE_MAIN() + + +def main() -> None: + base.main = _patched_base_main + # Adapter order is part of the experiment contract: + # absolute EE targets -> pi0.5 q01/q99 normalization -> geometry data/model. + # quantile.main() installs the first two and eventually reaches the patched + # base.main above, where the geometry hooks are installed last. + quantile.main() + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_native_depth_point_state_joint_action_fastwam.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_native_depth_point_state_joint_action_fastwam.py new file mode 100644 index 0000000000000000000000000000000000000000..ee48368b7fafa8c72e4566ac0aa2d9c92f5f8ec4 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_native_depth_point_state_joint_action_fastwam.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +"""RoboDojo-mini native-GT RGB/depth/point/action FastWAM adapter.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +import train_robodojo_rgb_state_joint_action_4way as fourway +import train_robodojo_rgb_state_joint_action_absolute_ee_4way as absolute +import train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way as quantile +from joint_rgb_depth_point_action_fastwam_diffusion import ( + joint_rgb_depth_point_action_fastwam_loss, +) +from robodojo_native_depth_point_action_model import ( + build_fresh_native_rgb_depth_point_action_mot, +) +from robodojo_native_depth_point_data import ( + LATENT_SHAPE, + expected_native_point_manifest, + load_native_point_episode, + native_depth_episode_path, + preprocess_native_point_episode, + source_record, + task_for_episode, + validate_native_depth_point_config, +) +from train_robodojo_rgb_native_depth_state_joint_action_fastwam import ( + NativeDepthFastWAMDataset, + _ORIGINAL_BASE_MAIN, + build_optimizer_with_fp32_audit, + collate_native_depth, + configure_native_joint_trainable, +) + + +base = fourway.base +_POINT_LATENT_CACHE: dict[str, torch.Tensor] = {} +_POINT_CONDITIONING_KEYS = ( + "point_condition_mode", + "point_timestep_adapter_bottleneck_dim", + "fix_point_first_latent", + "exclude_point_condition_from_loss", +) + + +def _remove_point_conditioning(config: dict[str, Any]) -> dict[str, Any]: + conditioning = config["conditioning"] + removed = { + key: conditioning.pop(key) + for key in _POINT_CONDITIONING_KEYS + if key in conditioning + } + if set(removed) != set(_POINT_CONDITIONING_KEYS): + raise ValueError("Config must define the complete point-conditioning contract") + return removed + + +def resolve_native_depth_point_config(path: Path) -> dict[str, Any]: + config = absolute.load_absolute_ee_config(path) + geometry = config.pop("geometry", None) + if not isinstance(geometry, dict): + raise ValueError("Config must define geometry") + point_conditioning = _remove_point_conditioning(config) + previous = fourway._original_validate_config + fourway._original_validate_config = quantile.validate_quantile_absolute_ee + try: + validated = fourway.validate_config(config) + finally: + fourway._original_validate_config = previous + config["geometry"] = geometry + config["conditioning"].update(point_conditioning) + return validate_native_depth_point_config(validated) + + +class NativeDepthPointFastWAMDataset(NativeDepthFastWAMDataset): + """Attach aligned native-GT depth and point latents to each base window.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + active = fourway._ACTIVE_CONFIG + if active is None: + raise RuntimeError("Native point dataset has no active config") + runtime = active.setdefault("runtime", {}) + benchmark_ids = runtime.pop("benchmark_episode_ids", None) + try: + # Validate all shared cache manifests against their complete + # 1000-episode contract before applying a smoke-only subset. + super().__init__(*args, **kwargs) + finally: + if benchmark_ids is not None: + runtime["benchmark_episode_ids"] = benchmark_ids + if benchmark_ids is not None: + allowed = {int(value) for value in benchmark_ids} + self.episodes = [ + episode + for episode in self.episodes + if int(episode.episode_index) in allowed + ] + if not self.episodes: + raise RuntimeError("Point smoke subset selected no episodes") + self.valid_starts = np.asarray( + [ + fourway.episode_sample_count(self.config, int(episode.length)) + for episode in self.episodes + ], + dtype=np.int64, + ) + self.cumulative_starts = np.cumsum(self.valid_starts, dtype=np.int64) + self.total_windows = int(self.cumulative_starts[-1]) + self.episode_slot_by_id = { + int(episode.episode_index): slot + for slot, episode in enumerate(self.episodes) + } + print( + f"[native-point smoke-subset] episodes={len(self.episodes)} " + f"windows={self.total_windows}", + flush=True, + ) + self.point_cache_dir = Path(self.config["paths"]["point_latent_cache_dir"]) + manifest_path = self.point_cache_dir / "manifest.json" + audit_path = self.point_cache_dir / "native_point_audit.json" + if not manifest_path.is_file() or not audit_path.is_file(): + raise FileNotFoundError( + f"Native point cache requires {manifest_path} and {audit_path}" + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + audit = json.loads(audit_path.read_text(encoding="utf-8")) + self.native_point_fingerprint = str( + audit.get("native_point_fingerprint", "") + ) + expected = expected_native_point_manifest( + source_fingerprint=self.expected_fingerprint, + native_depth_fingerprint=self.native_fingerprint, + native_point_fingerprint=self.native_point_fingerprint, + episode_count=100 * len(fourway.EXPECTED_TASKS), + valid_window_count=int(self.config["training"]["valid_windows_per_epoch"]), + ) + if manifest != expected: + raise RuntimeError( + f"Native point cache manifest mismatch: {manifest} != {expected}" + ) + if int(audit.get("episode_count", -1)) != 100 * len(fourway.EXPECTED_TASKS): + raise RuntimeError("Native point audit episode count mismatch") + if audit.get("source_fingerprint") != self.expected_fingerprint: + raise RuntimeError("Native point/source fingerprint mismatch") + if audit.get("native_depth_fingerprint") != self.native_fingerprint: + raise RuntimeError("Native point/depth fingerprint mismatch") + print( + f"[native-point] source=GT_HDF5_BACKPROJECTION " + f"episodes={len(self.episodes)} windows_per_epoch={self.total_windows} " + f"cache={self.point_cache_dir}", + flush=True, + ) + + def __getitem__(self, index: int) -> dict[str, Any]: + sample = super().__getitem__(index) + episode_index = int(sample["episode_index"].item()) + window_start = int(sample["window_start"].item()) + path = self.point_cache_dir / f"episode_{episode_index:06d}.pt" + key = str(path) + tensor = _POINT_LATENT_CACHE.get(key) + if tensor is None: + payload = torch.load(path, map_location="cpu", weights_only=True, mmap=True) + if payload.get("source_fingerprint") != self.expected_fingerprint: + raise RuntimeError(f"Point/RGB source fingerprint mismatch: {path}") + if payload.get("native_depth_fingerprint") != self.native_fingerprint: + raise RuntimeError(f"Point/depth fingerprint mismatch: {path}") + if payload.get("native_point_fingerprint") != self.native_point_fingerprint: + raise RuntimeError(f"Native point fingerprint mismatch: {path}") + source_path = native_depth_episode_path( + self.native_root, self.config, episode_index + ) + if payload.get("native_source") != source_record(source_path): + raise RuntimeError( + f"Native HDF5 changed after point cache creation: {source_path}" + ) + slot = self.episode_slot_by_id[episode_index] + expected_shape = (int(self.valid_starts[slot]), *LATENT_SHAPE) + tensor = payload["z_point"] + if tuple(tensor.shape) != expected_shape or tensor.dtype != torch.bfloat16: + raise RuntimeError( + f"Point shard mismatch {path}: {tuple(tensor.shape)} {tensor.dtype}" + ) + _POINT_LATENT_CACHE[key] = tensor + sample["z_point"] = tensor[window_start] + return sample + + +def collate_native_depth_point(samples: list[dict[str, Any]]) -> dict[str, Any]: + if not all("z_point" in sample for sample in samples): + raise RuntimeError("Native-point batch is missing z_point") + batch = collate_native_depth(samples) + batch["z_point"] = torch.stack([sample["z_point"] for sample in samples]) + return batch + + +def compute_native_depth_point_loss( + model: torch.nn.Module, + batch: dict[str, Any], + video_scheduler: Any, + action_scheduler: Any, + config: dict[str, Any], + device: torch.device, + dtype: torch.dtype, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + values = { + key: batch[key].to(device=device, dtype=dtype, non_blocking=True) + for key in ("z_rgb", "z_depth", "z_point", "action", "initial_state") + } + context_mask_cpu = batch["context_mask"] + valid_columns = context_mask_cpu.any(dim=0) + context_length = int(valid_columns.nonzero(as_tuple=False)[-1, 0]) + 1 + context = batch["context"][:, :context_length].to( + device=device, dtype=dtype, non_blocking=True + ) + context_mask = context_mask_cpu[:, :context_length].to( + device=device, dtype=torch.bool, non_blocking=True + ) + return joint_rgb_depth_point_action_fastwam_loss( + model, + video_scheduler, + action_scheduler, + z_rgb=values["z_rgb"], + z_depth=values["z_depth"], + z_point=values["z_point"], + action=values["action"], + initial_state=values["initial_state"], + context=context, + context_mask=context_mask, + image_is_pad=batch["image_is_pad"].to( + device=device, dtype=torch.bool, non_blocking=True + ), + action_is_pad=batch["action_is_pad"].to( + device=device, dtype=torch.bool, non_blocking=True + ), + lambda_rgb=float(config["diffusion"]["lambda_rgb"]), + lambda_depth=float(config["diffusion"]["lambda_depth"]), + lambda_point=float(config["diffusion"]["lambda_point"]), + lambda_action=float(config["diffusion"]["lambda_action"]), + condition_depth_frame0=bool( + config.get("modality_contract", {}).get("depth_condition", True) + ), + ) + + +def native_depth_point_dataset_smoke_test(config: dict[str, Any]) -> None: + episode = base.resolve_splits(config)["train"][0] + episode_index = int(episode.episode_index) + task_id, _ = task_for_episode(config, episode_index) + path = native_depth_episode_path( + Path(config["paths"]["native_depth_root"]), config, episode_index + ) + depth, intrinsics, extrinsics = load_native_point_episode( + path, expected_length=int(episode.length) + ) + point, metrics = preprocess_native_point_episode( + depth[:33], intrinsics, extrinsics[:33], config + ) + print( + json.dumps( + { + "status": "robodojo_native_gt_depth_point_ok", + "selected_task_count": len(fourway.EXPECTED_TASKS), + "selected_episode_count": 100 * len(fourway.EXPECTED_TASKS), + "windows_per_epoch": int(config["training"]["valid_windows_per_epoch"]), + "episode_index": episode_index, + "task_id": task_id, + "prompt": str(episode.task), + "native_source": source_record(path), + "point_preprocessed_shape": list(point.shape), + "point_preprocessed_range": [float(point.min()), float(point.max())], + "point_metrics": metrics, + "inputs": ( + ["rgb_frame0", "depth_frame0", "prompt", "initial_state"] + if bool(config.get("modality_contract", {}).get("depth_condition", True)) + else ["rgb_frame0", "prompt", "initial_state"] + ), + "outputs": [ + "rgb_future_frames_1_to_32", + "depth_future_frames_1_to_32", + "point_frames_0_to_32", + "action_chunk_32", + ], + }, + indent=2, + ), + flush=True, + ) + + +def _patched_base_main() -> None: + inherited_validate = base.validate_config + inherited_configure_trainable = base.configure_trainable + + def validate_runtime(config: dict[str, Any]) -> dict[str, Any]: + geometry = config.pop("geometry", None) + if not isinstance(geometry, dict): + raise ValueError("Config must define geometry") + point_conditioning = _remove_point_conditioning(config) + try: + validated = inherited_validate(config) + finally: + config["geometry"] = geometry + config["conditioning"].update(point_conditioning) + return validate_native_depth_point_config(validated) + + base.validate_config = validate_runtime + base.RoboDojoCachedDataset = NativeDepthPointFastWAMDataset + base.collate_rgb_action = collate_native_depth_point + base.compute_loss = compute_native_depth_point_loss + base.build_fresh_rgb_action_mot = build_fresh_native_rgb_depth_point_action_mot + base.configure_trainable = lambda model: configure_native_joint_trainable( + model, + inherited_configure_trainable, + expected_modalities="rgb_depth_point_action", + ) + base.dataset_smoke_test = native_depth_point_dataset_smoke_test + base.build_optimizer = build_optimizer_with_fp32_audit + _ORIGINAL_BASE_MAIN() + + +def main() -> None: + base.main = _patched_base_main + quantile.main() + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_native_depth_state_joint_action_fastwam.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_native_depth_state_joint_action_fastwam.py new file mode 100644 index 0000000000000000000000000000000000000000..f953d99ab34dc7b32edd41e8d2f362b27984a993 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_native_depth_state_joint_action_fastwam.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +"""RoboDojo-mini native-GT RGB/depth/action FastWAM training adapter.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +from joint_rgb_depth_point_action_fastwam_diffusion import output_gradient_evidence + +import train_robodojo_rgb_state_joint_action_4way as fourway +import train_robodojo_rgb_state_joint_action_absolute_ee_4way as absolute +import train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way as quantile +from joint_rgb_depth_action_fastwam_diffusion import ( + joint_rgb_depth_action_fastwam_loss, +) +from robodojo_native_depth_action_model import ( + build_fresh_native_rgb_depth_action_mot, +) +from robodojo_native_depth_data import ( + LATENT_SHAPE, + audit_native_episode, + native_depth_cache_fingerprint, + native_depth_episode_path, + preprocess_native_depth_episode, + source_record, + task_for_episode, + validate_native_depth_config, +) + + +base = fourway.base +_ORIGINAL_BASE_MAIN = base.main +_DEPTH_LATENT_CACHE: dict[str, torch.Tensor] = {} + + +def resolve_native_depth_config(path: Path) -> dict[str, Any]: + config = absolute.load_absolute_ee_config(path) + geometry = config.pop("geometry", None) + if not isinstance(geometry, dict): + raise ValueError("Config must define geometry") + previous = fourway._original_validate_config + fourway._original_validate_config = quantile.validate_quantile_absolute_ee + try: + validated = fourway.validate_config(config) + finally: + fourway._original_validate_config = previous + config["geometry"] = geometry + return validate_native_depth_config(validated) + + +class NativeDepthFastWAMDataset(fourway.ExhaustiveWindowDataset): + """Attach one aligned native-GT depth latent to every RGB/action window.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.native_root = Path(self.config["paths"]["native_depth_root"]) + self.depth_cache_dir = Path(self.config["paths"]["depth_latent_cache_dir"]) + manifest_path = self.depth_cache_dir / "manifest.json" + if not manifest_path.is_file(): + raise FileNotFoundError(manifest_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + expected = { + "format_version": 1, + "modality": "depth", + "source": "robodojo_native_gt_hdf5", + "source_fingerprint": self.expected_fingerprint, + "dtype": "bfloat16", + "latent_shape": list(LATENT_SHAPE), + "episode_count": 100 * len(fourway.EXPECTED_TASKS), + "valid_window_count": int(self.config["training"]["valid_windows_per_epoch"]), + "window_sampling": "all_episode_frames_repeat_last_tail", + } + actual = {key: manifest.get(key) for key in expected} + if actual != expected: + raise RuntimeError(f"Native depth cache manifest mismatch: {actual} != {expected}") + self.native_fingerprint = str(manifest["native_depth_fingerprint"]) + audit_path = self.depth_cache_dir / "native_audit.json" + audit = json.loads(audit_path.read_text(encoding="utf-8")) + if audit.get("native_depth_fingerprint") != self.native_fingerprint: + raise RuntimeError("Native depth audit/cache fingerprint mismatch") + if int(audit.get("episode_count", -1)) != 100 * len(fourway.EXPECTED_TASKS): + raise RuntimeError("Native depth audit episode count mismatch") + print( + f"[native-depth] source=GT_HDF5 episodes={len(self.episodes)} " + f"windows_per_epoch={self.total_windows} cache={self.depth_cache_dir}", + flush=True, + ) + + def __getitem__(self, index: int) -> dict[str, Any]: + sample = super().__getitem__(index) + episode_index = int(sample["episode_index"].item()) + window_start = int(sample["window_start"].item()) + path = self.depth_cache_dir / f"episode_{episode_index:06d}.pt" + key = str(path) + tensor = _DEPTH_LATENT_CACHE.get(key) + if tensor is None: + payload = torch.load(path, map_location="cpu", weights_only=True, mmap=True) + if payload.get("source_fingerprint") != self.expected_fingerprint: + raise RuntimeError(f"Depth/RGB source fingerprint mismatch: {path}") + if payload.get("native_depth_fingerprint") != self.native_fingerprint: + raise RuntimeError(f"Native depth fingerprint mismatch: {path}") + source_path = native_depth_episode_path( + self.native_root, self.config, episode_index + ) + if payload.get("native_source") != source_record(source_path): + raise RuntimeError(f"Native HDF5 changed after cache creation: {source_path}") + slot = self.episode_slot_by_id[episode_index] + expected_shape = (int(self.valid_starts[slot]), *LATENT_SHAPE) + tensor = payload["z_depth"] + if tuple(tensor.shape) != expected_shape or tensor.dtype != torch.bfloat16: + raise RuntimeError( + f"Depth shard mismatch {path}: {tuple(tensor.shape)} {tensor.dtype}" + ) + _DEPTH_LATENT_CACHE[key] = tensor + sample["z_depth"] = tensor[window_start] + return sample + + +def collate_native_depth(samples: list[dict[str, Any]]) -> dict[str, Any]: + if not all("z_depth" in sample for sample in samples): + raise RuntimeError("Native-depth batch is missing z_depth") + batch = fourway.collate_episode_length_normalized(samples) + batch["z_depth"] = torch.stack([sample["z_depth"] for sample in samples]) + return batch + + +def compute_native_depth_loss( + model: torch.nn.Module, + batch: dict[str, Any], + video_scheduler: Any, + action_scheduler: Any, + config: dict[str, Any], + device: torch.device, + dtype: torch.dtype, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + z_rgb = batch["z_rgb"].to(device=device, dtype=dtype, non_blocking=True) + z_depth = batch["z_depth"].to(device=device, dtype=dtype, non_blocking=True) + action = batch["action"].to(device=device, dtype=dtype, non_blocking=True) + initial_state = batch["initial_state"].to( + device=device, dtype=dtype, non_blocking=True + ) + context_mask_cpu = batch["context_mask"] + valid_columns = context_mask_cpu.any(dim=0) + context_length = int(valid_columns.nonzero(as_tuple=False)[-1, 0]) + 1 + context = batch["context"][:, :context_length].to( + device=device, dtype=dtype, non_blocking=True + ) + context_mask = context_mask_cpu[:, :context_length].to( + device=device, dtype=torch.bool, non_blocking=True + ) + return joint_rgb_depth_action_fastwam_loss( + model, + video_scheduler, + action_scheduler, + z_rgb=z_rgb, + z_depth=z_depth, + action=action, + initial_state=initial_state, + context=context, + context_mask=context_mask, + image_is_pad=batch["image_is_pad"].to( + device=device, dtype=torch.bool, non_blocking=True + ), + action_is_pad=batch["action_is_pad"].to( + device=device, dtype=torch.bool, non_blocking=True + ), + lambda_rgb=float(config["diffusion"]["lambda_rgb"]), + lambda_depth=float(config["diffusion"]["lambda_depth"]), + lambda_action=float(config["diffusion"]["lambda_action"]), + ) + + +def native_depth_dataset_smoke_test(config: dict[str, Any]) -> None: + splits = base.resolve_splits(config) + episode = splits["train"][0] + task_id, _ = task_for_episode(config, int(episode.episode_index)) + path = native_depth_episode_path( + Path(config["paths"]["native_depth_root"]), config, int(episode.episode_index) + ) + native = audit_native_episode( + path, + expected_length=int(episode.length), + expected_prompt=str(episode.task), + ) + import h5py + + with h5py.File(path, "r") as handle: + depth = handle["vision/cam_head/approximate_depths"][:33] + preprocessed = preprocess_native_depth_episode(depth, config) + print( + json.dumps( + { + "status": "robodojo_native_gt_depth_ok", + "selected_task_count": len(fourway.EXPECTED_TASKS), + "selected_episode_count": 100 * len(fourway.EXPECTED_TASKS), + "windows_per_epoch": int(config["training"]["valid_windows_per_epoch"]), + "episode_index": int(episode.episode_index), + "task_id": task_id, + "prompt": str(episode.task), + "native_source": native, + "preprocessed_shape": list(preprocessed.shape), + "preprocessed_range": [ + float(preprocessed.min()), + float(preprocessed.max()), + ], + "inputs": ["rgb_frame0", "depth_frame0", "prompt", "initial_state"], + "outputs": ["rgb_video", "depth_video", "action_chunk_32"], + }, + indent=2, + ), + flush=True, + ) + + +def build_optimizer_with_fp32_audit( + model: torch.nn.Module, config: dict[str, Any] +) -> torch.optim.Optimizer: + optimizer = fourway.build_optimizer(model, config) + trainable = {id(parameter): name for name, parameter in model.named_parameters() if parameter.requires_grad} + grouped: list[int] = [ + id(parameter) + for group in optimizer.param_groups + for parameter in group["params"] + ] + duplicate_ids = sorted({value for value in grouped if grouped.count(value) > 1}) + grouped_ids = set(grouped) + if duplicate_ids or grouped_ids != set(trainable): + missing = sorted(trainable[value] for value in set(trainable) - grouped_ids) + unexpected = sorted(value for value in grouped_ids - set(trainable)) + raise RuntimeError( + "Optimizer parameter partition failed: " + f"missing={missing} duplicate_count={len(duplicate_ids)} " + f"unexpected_ids={unexpected}" + ) + expected_groups = { + "rgb_video_expert": float(config["training"]["video_lr"]), + "action_expert_and_state_encoder": float(config["training"]["action_lr"]), + } + actual_groups = {str(group["name"]): float(group["lr"]) for group in optimizer.param_groups} + if actual_groups != expected_groups: + raise RuntimeError( + f"Optimizer group/LR contract failed: {actual_groups} != {expected_groups}" + ) + print( + f"[optimizer-group-audit rank={os.environ.get('LOCAL_RANK', '0')}] " + f"trainable_parameters={len(trainable)} grouped_parameters={len(grouped)} " + "missing=0 duplicates=0 video_lr=" + f"{expected_groups['rgb_video_expert']:.9g} action_lr=" + f"{expected_groups['action_expert_and_state_encoder']:.9g}", + flush=True, + ) + branch_gradient_evidence: dict[str, float] = {} + video = getattr(model, "video_expert", None) + requires_point_gradient_audit = bool( + getattr(video, "joint_rgb_depth_point", False) + ) + audited = False + + def audit_after_step( + current: torch.optim.Optimizer, args: tuple[Any, ...], kwargs: dict[str, Any] + ) -> None: + del args, kwargs + nonlocal audited + if audited: + return + parameter_dtypes = { + parameter.dtype + for group in current.param_groups + for parameter in group["params"] + if parameter.is_floating_point() + } + gradient_dtypes = { + parameter.grad.dtype + for group in current.param_groups + for parameter in group["params"] + if parameter.grad is not None and parameter.grad.is_floating_point() + } + state_dtypes = { + value.dtype + for state in current.state.values() + for key, value in state.items() + if key in {"exp_avg", "exp_avg_sq"} and torch.is_tensor(value) + } + expected = {torch.float32} + if parameter_dtypes != expected or gradient_dtypes != expected or state_dtypes != expected: + raise RuntimeError( + "FP32 master/optimizer audit failed: " + f"params={parameter_dtypes} grads={gradient_dtypes} states={state_dtypes}" + ) + group_gradient_squares = {} + for group in current.param_groups: + squared = sum( + float(parameter.grad.detach().float().square().sum().item()) + for parameter in group["params"] + if parameter.grad is not None + ) + group_gradient_squares[str(group["name"])] = squared + if requires_point_gradient_audit: + branch_gradient_evidence.update(output_gradient_evidence()) + point_local = branch_gradient_evidence.get("point_prediction", 0.0) + evidence_device = next( + parameter.device + for group in current.param_groups + for parameter in group["params"] + ) + distributed_evidence = torch.tensor( + [ + group_gradient_squares["rgb_video_expert"], + group_gradient_squares["action_expert_and_state_encoder"], + point_local**2, + ], + device=evidence_device, + dtype=torch.float64, + ) + if torch.distributed.is_available() and torch.distributed.is_initialized(): + torch.distributed.all_reduce(distributed_evidence, op=torch.distributed.ReduceOp.SUM) + video_norm, action_norm, point_norm = ( + float(value.sqrt().item()) for value in distributed_evidence + ) + group_gradient_norms = { + "rgb_video_expert": video_norm, + "action_expert_and_state_encoder": action_norm, + } + if any(not np.isfinite(value) or value <= 0.0 for value in group_gradient_norms.values()): + raise RuntimeError(f"Optimizer group gradient audit failed: {group_gradient_norms}") + if requires_point_gradient_audit: + if not np.isfinite(point_norm) or point_norm <= 0.0: + raise RuntimeError( + f"Point branch gradient audit failed: {branch_gradient_evidence}" + ) + branch_gradient_evidence["point_prediction_global"] = point_norm + print( + f"[precision-audit rank={os.environ.get('LOCAL_RANK', '0')}] " + "master_params=float32 gradients=float32 adam_exp_avg=float32 " + "adam_exp_avg_sq=float32 autocast_compute=bfloat16 " + f"group_gradient_norms={group_gradient_norms} " + f"branch_gradient_evidence={branch_gradient_evidence}", + flush=True, + ) + audited = True + + optimizer.register_step_post_hook(audit_after_step) + return optimizer + + +def configure_native_joint_trainable( + model: torch.nn.Module, + inherited_configure: Any, + *, + expected_modalities: str = "rgb_depth_action", +) -> None: + """Fail closed unless both experts are enabled for the joint objective.""" + + objective = getattr(model, "_robodojo_objective", None) + modalities = getattr(model, "_robodojo_joint_modalities", None) + if objective != "joint" or modalities != expected_modalities: + raise RuntimeError( + "Native training contract mismatch before parameter selection: " + f"objective={objective!r} modalities={modalities!r}" + ) + inherited_configure(model) + video_trainable = sum( + parameter.numel() + for parameter in model.video_expert.parameters() + if parameter.requires_grad + ) + action_trainable = sum( + parameter.numel() + for module in (model.action_expert, model.action_state_encoder) + for parameter in module.parameters() + if parameter.requires_grad + ) + if video_trainable <= 0 or action_trainable <= 0: + raise RuntimeError( + "Joint RGB/depth/action training requires trainable video and action experts: " + f"video_trainable={video_trainable} action_trainable={action_trainable}" + ) + print( + "[training-contract] objective=joint " + f"modalities={expected_modalities.replace('_', '+')} " + f"video_trainable={video_trainable} action_trainable={action_trainable}", + flush=True, + ) + + +def _patched_base_main() -> None: + inherited_validate = base.validate_config + inherited_configure_trainable = base.configure_trainable + + def validate_runtime(config: dict[str, Any]) -> dict[str, Any]: + geometry = config.pop("geometry", None) + if not isinstance(geometry, dict): + raise ValueError("Config must define geometry") + try: + validated = inherited_validate(config) + finally: + config["geometry"] = geometry + return validate_native_depth_config(validated) + + base.validate_config = validate_runtime + base.RoboDojoCachedDataset = NativeDepthFastWAMDataset + base.collate_rgb_action = collate_native_depth + base.compute_loss = compute_native_depth_loss + base.build_fresh_rgb_action_mot = build_fresh_native_rgb_depth_action_mot + base.configure_trainable = lambda model: configure_native_joint_trainable( + model, inherited_configure_trainable + ) + base.dataset_smoke_test = native_depth_dataset_smoke_test + base.build_optimizer = build_optimizer_with_fp32_audit + _ORIGINAL_BASE_MAIN() + + +def main() -> None: + base.main = _patched_base_main + quantile.main() + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_4way.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_4way.py new file mode 100644 index 0000000000000000000000000000000000000000..c89fb389da38ed918e8489357fe4d6a99195ca32 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_4way.py @@ -0,0 +1,1854 @@ +#!/usr/bin/env python3 +"""Isolated four-way RoboDojo RGB+Action experiment entrypoint. + +This file intentionally leaves the original training entrypoint untouched. It +adds: strict task/prompt auditing, action-only supervision, and an audited RGB +slice import from an RGB+depth+point+action checkpoint. +""" + +from __future__ import annotations + +import gc +import hashlib +import json +import math +import os +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +import train_galexea_rgb_state_joint_action_ablation as shared +import train_robodojo_rgb_state_joint_action_ablation as base +from robodojo_v2_checkpoint_contract import ( + verify_v2_baseframe_checkpoint_contract, +) + + +EXPECTED_TASKS = { + "stack_bowls": (30, "Stack the three bowls together."), + "sweep_blocks": ( + 34, + "Pick up the broom, hand it over to the right hand, then use the dustpan to sweep the blocks.", + ), + "press_by_number": ( + 25, + "Press the two red buttons the required number of times according to the number cards, then press the blue button to confirm.", + ), + "match_and_pick_from_conveyor": ( + 16, + "Remember the first object on the conveyor, then pick the matching object when it appears again.", + ), + "plug_in_charger": (22, "Plug the charger into the power strip."), + "deposit_coin": ( + 4, + "Pick up the coin from the holder and insert it precisely into the coin bank.", + ), + "fill_pen_holder": ( + 8, + "Hold the pen holder with one hand, place all pens into it with the other hand, then put it back down.", + ), + "classify_objects": (2, "Sort the objects by category into the three baskets."), + "stack_blocks": (29, "Stack the three blocks with different textures."), + "push_T": ( + 26, + "Push the T-shaped block to align it precisely with the gray T-shaped pad.", + ), +} +if os.environ.get("ROBODOJO_FULL_35_TASKS") == "1": + full_tasks = json.loads((Path(__file__).with_name("robodojo_full_35_tasks.json")).read_text()) + if len(full_tasks) != 35 or sorted(int(entry[0]) for entry in full_tasks.values()) != list(range(35)): + raise RuntimeError("Invalid full RoboDojo task-index contract") + EXPECTED_TASKS.clear() + EXPECTED_TASKS.update({name: (int(entry[0]), str(entry[1])) for name, entry in full_tasks.items()}) + +POINT_EXTRAS = { + "mot.mixtures.video.point_first_frame_timestep_adapter.net.1.bias", + "mot.mixtures.video.point_first_frame_timestep_adapter.net.1.weight", + "mot.mixtures.video.point_first_frame_timestep_adapter.net.3.bias", + "mot.mixtures.video.point_first_frame_timestep_adapter.net.3.weight", +} +CAMERA_EXTRAS = { + "structured_type_embedding", + "camera_encoder.bias", + "camera_encoder.weight", + "camera_head.bias", + "camera_head.weight", + "camera_normalizer.count", + "camera_normalizer.fitted", + "camera_normalizer.mean", + "camera_normalizer.std", +} + +# SpaceDreamer and RoboDojo share the ActionDiT backbone, but their action +# token/state interfaces, output semantics, and normalization statistics are +# task-specific. The selective initialization mode below therefore imports +# every ActionDiT backbone parameter while preserving these four RoboDojo +# modules exactly as they were initialized locally. +ACTION_REINITIALIZED_PREFIXES = ( + "mot.mixtures.action.action_encoder.", + "action_state_encoder.", + "mot.mixtures.action.head.", + "action_normalizer.", +) +ACTION_BACKBONE_PREFIX = "mot.mixtures.action." +VIDEO_EXPERT_PREFIX = "mot.mixtures.video." +EGO_FULL_ACTION_INITIALIZATION_MODE = ( + "joint_checkpoint_rgb_slice_full_action_ego_contract" +) +V2_BASEFRAME_FULL_ACTION_INITIALIZATION_MODE = ( + "joint_checkpoint_rgb_slice_full_action_baseframe_v2_contract" +) +V2_BASEFRAME_RGB_DEPTH_FULL_ACTION_INITIALIZATION_MODE = ( + "joint_checkpoint_rgb_depth_slice_full_action_baseframe_v2_contract" +) +V2_BASEFRAME_RGB_DEPTH_POINT_FULL_ACTION_INITIALIZATION_MODE = ( + "joint_checkpoint_rgb_depth_point_full_action_baseframe_v2_contract" +) +SOURCE_ACTION_NORMALIZER_TRANSFER = "preserve_source_pretraining_q01_q99" +TARGET_ACTION_NORMALIZER_TRANSFER = "refit_robodojo_q01_q99" +ROBODOJO_FULL_ACTION_NORMALIZER_TRANSFER = "load_robodojo_full_q01_q99" + +_original_validate_config = base.validate_config +_original_load_config = base.load_config +_original_build_model = base.build_fresh_rgb_action_mot +_original_configure_trainable = base.configure_trainable +_original_build_optimizer = base.build_optimizer +_original_fit_action_normalizer = base.fit_action_normalizer +_original_checkpoint_metadata = shared.rgb_action_checkpoint_metadata +_original_cache_fingerprint = base.cache_fingerprint +_original_compute_loss = base.compute_loss +_original_accelerator = base.Accelerator + +_ACTIVE_CONFIG: dict[str, Any] | None = None +_EPISODE_ARRAY_CACHE: dict[tuple[str, int], tuple[np.ndarray, np.ndarray, np.ndarray]] = {} +_SHARD_CACHE: dict[str, Any] = {} +_RGB_LATENT_CACHE: dict[str, torch.Tensor] = {} +_DYNAMIC_VAE: Any = None + +LEGACY_SAMPLING_MODE = "exhaustive_shuffled_window" +FASTWAM_TAIL_PAD_SAMPLING_MODE = ( + "exhaustive_shuffled_all_frame_starts_fastwam_tail_pad" +) + + +def _uses_fastwam_tail_padding(config: dict[str, Any]) -> bool: + return str(config.get("sampling", {}).get("mode", "")) == ( + FASTWAM_TAIL_PAD_SAMPLING_MODE + ) + + +def episode_sample_count(config: dict[str, Any], length: int) -> int: + """Return the number of uniformly sampled starts for one episode.""" + + length = int(length) + window = int(config["sampling"]["window_size"]) + if length <= 0: + raise ValueError("Episode length must be positive") + if _uses_fastwam_tail_padding(config): + return length + count = length - window + 1 + if count <= 0: + raise RuntimeError(f"Episode is shorter than the configured {window}-frame window") + return count + + +def window_indices_and_pad_masks( + config: dict[str, Any], *, length: int, start: int +) -> tuple[np.ndarray, torch.Tensor, torch.Tensor]: + """Build fixed-shape indices plus FastWAM-compatible tail masks. + + Repeated terminal values preserve tensor shapes. The masks retain the + original LeRobot semantics: the action at the current terminal frame is + valid, while requested timestamps strictly beyond the episode are padding. + """ + + length = int(length) + start = int(start) + window = int(config["sampling"]["window_size"]) + if not 0 <= start < episode_sample_count(config, length): + raise IndexError(f"Window start {start} is outside episode length {length}") + requested = start + np.arange(window, dtype=np.int64) + if _uses_fastwam_tail_padding(config): + image_is_pad_np = requested >= length + indices = np.minimum(requested, length - 1) + else: + if int(requested[-1]) >= length: + raise RuntimeError("Legacy valid-window sampling cannot cross the episode tail") + image_is_pad_np = np.zeros(window, dtype=bool) + indices = requested + image_is_pad = torch.from_numpy(image_is_pad_np.copy()) + action_is_pad = image_is_pad[:-1].clone() + return indices, image_is_pad, action_is_pad + + +def valid_action_occurrence_counts(length: int, horizon: int) -> np.ndarray: + """Count non-padding occurrences of each raw action over all frame starts.""" + + length = int(length) + horizon = int(horizon) + if length <= 0 or horizon <= 0: + raise ValueError("Length and horizon must be positive") + action_index = np.arange(length, dtype=np.int64) + return np.minimum(action_index + 1, horizon).astype(np.int64) + + +def build_precision_aware_accelerator(*args: Any, **kwargs: Any) -> Any: + """Keep trainable parameters/Adam states in BF16 when explicitly requested.""" + + # The training loop defines max_steps/warmup_steps in optimizer-step units. + # Accelerate's default wrapper advances a prepared scheduler once per rank + # when batches are sharded, which changes the LR curve with world size. + # The loop already calls lr_scheduler.step() after every optimizer step, so + # keep that call one-to-one for 2-GPU and 8-GPU runs alike. + kwargs["step_scheduler_with_optimizer"] = False + if _ACTIVE_CONFIG is not None and not bool( + _ACTIVE_CONFIG["training"].get("fp32_master_weights", True) + ): + kwargs["mixed_precision"] = "no" + print( + "[precision] native BF16 parameters/gradients/optimizer states; " + "FSDP AMP upcast disabled", + flush=True, + ) + accelerator = _original_accelerator(*args, **kwargs) + if _ACTIVE_CONFIG is not None and _ACTIVE_CONFIG["training"].get( + "global_batch_size" + ) is not None: + actual_global_batch = ( + int(_ACTIVE_CONFIG["data"]["batch_size"]) + * int(accelerator.num_processes) + * int(_ACTIVE_CONFIG["training"]["gradient_accumulation_steps"]) + ) + expected_global_batch = int( + _ACTIVE_CONFIG["training"]["global_batch_size"] + ) + if actual_global_batch != expected_global_batch: + raise RuntimeError( + f"Global batch mismatch: configured={expected_global_batch}, " + f"actual={actual_global_batch} " + f"(batch={_ACTIVE_CONFIG['data']['batch_size']} x " + f"world={accelerator.num_processes} x " + f"accum={_ACTIVE_CONFIG['training']['gradient_accumulation_steps']})" + ) + if accelerator.is_main_process: + print( + f"[training] global_batch={actual_global_batch} " + f"world_size={accelerator.num_processes}", + flush=True, + ) + return accelerator + + +def _deep_merge(base_value: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + merged = dict(base_value) + for key, value in override.items(): + if key == "task_index_by_id": + merged[key] = value + elif isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = _deep_merge(merged[key], value) + else: + merged[key] = value + return merged + + +def episode_length_normalized_cache_fingerprint( + config: dict[str, Any], splits: dict[str, list[Any]] +) -> str: + # Runtime loader knobs do not change cached episode/window contents. Keep + # the cache shareable across equivalent micro-batch benchmarks and runs. + cache_config = _deep_merge(config, {"data": {"batch_size": 1, "num_workers": 0}}) + source = _original_cache_fingerprint(cache_config, splits) + sampling = config.get("sampling", {}) + payload = { + "source_fingerprint": source, + "sampling": sampling, + "format": "exhaustive_shuffled_window_v2_fastwam_tail_mask", + } + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() + + +def _episode_record(episode: Any, split: str) -> dict[str, Any]: + return { + "split": split, + "episode_index": int(episode.episode_index), + "task": str(episode.task), + "length": int(episode.length), + "data_chunk_index": int(episode.data_chunk_index), + "data_file_index": int(episode.data_file_index), + "video_chunk_index": int(episode.video_chunk_index), + "video_file_index": int(episode.video_file_index), + "video_from_timestamp": float(episode.video_from_timestamp), + "video_to_timestamp": float(episode.video_to_timestamp), + } + + +def _record_episode(record: dict[str, Any]) -> Any: + return base.RoboDojoEpisode( + episode_index=int(record["episode_index"]), + task=str(record["task"]), + length=int(record["length"]), + data_chunk_index=int(record["data_chunk_index"]), + data_file_index=int(record["data_file_index"]), + video_chunk_index=int(record["video_chunk_index"]), + video_file_index=int(record["video_file_index"]), + video_from_timestamp=float(record["video_from_timestamp"]), + video_to_timestamp=float(record["video_to_timestamp"]), + ) + + +def _load_episode_arrays(config: dict[str, Any], episode: Any) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + root = Path(config["paths"]["dataset_root"]) + key = (str(root.resolve()), int(episode.episode_index)) + cached = _EPISODE_ARRAY_CACHE.get(key) + if cached is not None: + return cached + shard_path = base.data_parquet_path(root, episode) + shard_key = str(shard_path.resolve()) + shard = _SHARD_CACHE.get(shard_key) + if shard is None: + shard = base.load_data_shard(shard_path) + _SHARD_CACHE[shard_key] = shard + arrays = base.episode_arrays(shard, episode) + _EPISODE_ARRAY_CACHE[key] = arrays + return arrays + + +def _exhaustive_action_statistics( + config: dict[str, Any], episodes: list[Any] +) -> tuple[torch.Tensor, torch.Tensor, int]: + """Exact stats for the configured exhaustive sampler's valid action tokens.""" + + total = torch.zeros(14, dtype=torch.float64) + squared = torch.zeros(14, dtype=torch.float64) + total_mass = 0.0 + window = int(config["sampling"]["window_size"]) + action_horizon = window - 1 + for number, episode in enumerate(episodes, start=1): + state, absolute_action, _ = _load_episode_arrays(config, episode) + length = int(episode.length) + sample_count = episode_sample_count(config, length) + full_indices = np.arange(length, dtype=np.int64) + if _uses_fastwam_tail_padding(config): + # The final duplicate lets the absolute-action adapter expose the + # real action stored at the terminal frame. It is not padding; + # only requests strictly past the terminal frame are masked. + full_indices = np.concatenate([full_indices, full_indices[-1:]]) + action = base.build_egoverse_delta_targets( + state, absolute_action, full_indices + )["action_raw"].to(dtype=torch.float64) + if _uses_fastwam_tail_padding(config): + counts = torch.from_numpy( + valid_action_occurrence_counts(length, action_horizon) + ).to(dtype=torch.float64) + if int(action.shape[0]) != length: + raise RuntimeError("Tail-padded action-statistics length mismatch") + weights = counts + else: + transition = torch.arange(action.shape[0], dtype=torch.long) + low = torch.clamp(transition - (action_horizon - 1), min=0) + high = torch.minimum( + transition, + torch.tensor(sample_count - 1, dtype=torch.long), + ) + counts = (high - low + 1).clamp(min=0).to(dtype=torch.float64) + # Preserve the legacy cache exactly when the legacy sampler is used. + weights = counts / float(action_horizon) + total += (action * weights[:, None]).sum(dim=0) + squared += (action.square() * weights[:, None]).sum(dim=0) + total_mass += float(weights.sum().item()) + if number == 1 or number % 100 == 0 or number == len(episodes): + print( + f"[sampling-stats] {number}/{len(episodes)} " + f"episode={episode.episode_index} starts={sample_count}", + flush=True, + ) + if _uses_fastwam_tail_padding(config): + expected_mass = float( + sum( + valid_action_occurrence_counts( + int(episode.length), action_horizon + ).sum() + for episode in episodes + ) + ) + else: + expected_mass = float( + sum(episode_sample_count(config, int(episode.length)) for episode in episodes) + ) + if abs(total_mass - expected_mass) > 1.0e-6 * expected_mass: + raise RuntimeError( + f"Exhaustive-window action mass mismatch: " + f"{total_mass} != {expected_mass}" + ) + mean = total / total_mass + variance = (squared / total_mass - mean.square()).clamp(min=1.0e-12) + effective_count = max( + 2, + int(round(expected_mass)) + if _uses_fastwam_tail_padding(config) + else int(round(expected_mass * action_horizon)), + ) + return mean.float(), variance.sqrt().float(), effective_count + + +@torch.no_grad() +def prepare_episode_length_normalized_cache( + config: dict[str, Any], *, device: torch.device, overwrite: bool +) -> None: + """Cache only metadata/text/stats; RGB windows are sampled online.""" + + cache_dir = Path(config["paths"]["cache_dir"]) + manifest_path = cache_dir / "manifest.json" + splits = base.resolve_splits(config) + fingerprint = episode_length_normalized_cache_fingerprint(config, splits) + required = (manifest_path, cache_dir / "prompt_contexts.pt", cache_dir / "action_normalizer.pt") + if all(path.is_file() for path in required) and not overwrite: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("fingerprint") != fingerprint: + raise RuntimeError("Episode-normalized cache fingerprint mismatch") + print(f"[cache] reuse episode-length-normalized metadata {manifest_path}", flush=True) + return + if cache_dir.exists() and not overwrite: + existing = {entry.name for entry in cache_dir.iterdir()} + unexpected = existing - {"prompt_contexts.pt"} + if unexpected: + raise RuntimeError( + f"Refusing to overwrite non-empty cache {cache_dir}; " + f"unexpected entries={sorted(unexpected)}" + ) + cache_dir.mkdir(parents=True, exist_ok=True) + + prompts = sorted({episode.task for values in splits.values() for episode in values}) + prompt_path = cache_dir / "prompt_contexts.pt" + if prompt_path.is_file(): + prompt_payload = torch.load( + prompt_path, map_location="cpu", weights_only=True, mmap=True + ) + cached_prompts = sorted(str(entry["prompt"]) for entry in prompt_payload["entries"]) + if cached_prompts != prompts: + raise RuntimeError( + f"Existing prompt cache mismatch: {cached_prompts} != {prompts}" + ) + print(f"[cache] reuse validated prompt contexts {prompt_path}", flush=True) + else: + dtype = base.dtype_from_name(str(config["model"]["dtype"])) + model_dir = Path(config["paths"]["pretrained_model_dir"]) + text_encoder, tokenizer = base.load_text_encoder_and_tokenizer( + model_dir, + dtype, + device, + int(config["cache"].get("tokenizer_max_length", 512)), + ) + prompt_entries = [] + prompt_batch_size = int(config["cache"].get("prompt_batch_size", 8)) + for start in range(0, len(prompts), prompt_batch_size): + batch_prompts = prompts[start : start + prompt_batch_size] + ids, mask = tokenizer(batch_prompts, return_mask=True, add_special_tokens=True) + contexts = text_encoder(ids.to(device), mask.to(device=device, dtype=torch.bool)) + for index, prompt in enumerate(batch_prompts): + prompt_entries.append( + { + "prompt": prompt, + "context": contexts[index].cpu(), + "context_mask": mask[index].to(dtype=torch.bool).cpu(), + } + ) + prompt_tmp = cache_dir / f"prompt_contexts.pt.tmp.{os.getpid()}" + torch.save({"entries": prompt_entries}, prompt_tmp) + os.replace(prompt_tmp, prompt_path) + text_encoder.to("cpu") + del text_encoder, tokenizer + if device.type == "cuda": + torch.cuda.empty_cache() + + train_episodes = list(splits["train"]) + mean, std, count = _exhaustive_action_statistics(config, train_episodes) + stats_tmp = cache_dir / f"action_normalizer.pt.tmp.{os.getpid()}" + torch.save( + { + "fingerprint": fingerprint, + "mean": mean, + "std": std, + "count": torch.tensor(count, dtype=torch.long), + "fitted": torch.tensor(True, dtype=torch.bool), + "sampling": config["sampling"], + }, + stats_tmp, + ) + os.replace(stats_tmp, cache_dir / "action_normalizer.pt") + + episodes = [ + _episode_record(episode, split) + for split in base.SPLITS + for episode in splits[split] + ] + valid_starts = [ + episode_sample_count(config, int(entry["length"])) for entry in episodes + ] + manifest = { + "format_version": 6, + "fingerprint": fingerprint, + "source": "robodojo_ee_lerobot_v3.0", + "modalities": ["rgb_online", "action", "initial_state"], + "sampling": config["sampling"], + "episodes": episodes, + "episode_count": len(episodes), + "valid_window_count": int(sum(valid_starts)), + "valid_starts_min": int(min(valid_starts)), + "valid_starts_max": int(max(valid_starts)), + } + manifest_tmp = cache_dir / f"manifest.json.tmp.{os.getpid()}" + manifest_tmp.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + os.replace(manifest_tmp, manifest_path) + print( + f"[cache] exhaustive-window metadata ready: episodes={len(episodes)} " + f"valid_windows={sum(valid_starts)} path={manifest_path}", + flush=True, + ) + + +def cache_is_complete_fast(cache_dir: Path, manifest: dict[str, Any]) -> bool: + """Check an atomic cache manifest with one directory scan on remote storage.""" + + if manifest.get("format_version") == 6: + required = {"manifest.json", "prompt_contexts.pt", "action_normalizer.pt"} + present = {entry.name for entry in os.scandir(cache_dir)} + return required.issubset(present) + expected = {"prompt_contexts.pt"} + expected.update(str(entry["file"]) for entry in manifest["samples"]) + present = {entry.name for entry in os.scandir(cache_dir)} + return expected.issubset(present) + + +class ExhaustiveWindowDataset(torch.utils.data.Dataset): + """Index every configured frame start exactly once per dataset epoch.""" + + def __init__( + self, + cache_dir: Path, + expected_fingerprint: str, + split: str, + *, + action_mean: torch.Tensor | None = None, + action_std: torch.Tensor | None = None, + action_clip: float | None = None, + ) -> None: + if _ACTIVE_CONFIG is None: + raise RuntimeError("Exhaustive-window dataset has no active config") + self.config = _ACTIVE_CONFIG + self.expected_fingerprint = str(expected_fingerprint) + self.cache_dir = Path(cache_dir) + manifest = json.loads((self.cache_dir / "manifest.json").read_text(encoding="utf-8")) + if manifest.get("format_version") != 6: + raise RuntimeError("Expected exhaustive-window cache format_version=6") + if manifest.get("fingerprint") != expected_fingerprint: + raise RuntimeError("Exhaustive-window cache does not match config") + self.episodes = [ + _record_episode(entry) + for entry in manifest["episodes"] + if entry["split"] == split + ] + benchmark_episode_ids = self.config.get("runtime", {}).get( + "benchmark_episode_ids" + ) + if benchmark_episode_ids is not None: + allowed = {int(value) for value in benchmark_episode_ids} + self.episodes = [ + episode + for episode in self.episodes + if int(episode.episode_index) in allowed + ] + if not self.episodes: + raise RuntimeError(f"No episodes for split {split!r}") + self.window_size = int(self.config["sampling"]["window_size"]) + self.valid_starts = np.asarray( + [ + episode_sample_count(self.config, int(episode.length)) + for episode in self.episodes + ], + dtype=np.int64, + ) + if np.any(self.valid_starts <= 0): + raise RuntimeError("An episode is shorter than the configured window") + self.cumulative_starts = np.cumsum(self.valid_starts, dtype=np.int64) + self.total_windows = int(self.cumulative_starts[-1]) + self.episode_slot_by_id = { + int(episode.episode_index): slot + for slot, episode in enumerate(self.episodes) + } + prompt_entries = torch.load( + self.cache_dir / "prompt_contexts.pt", map_location="cpu", weights_only=True + )["entries"] + self.prompt_by_text = {str(entry["prompt"]): entry for entry in prompt_entries} + missing = sorted({episode.task for episode in self.episodes} - set(self.prompt_by_text)) + if missing: + raise RuntimeError(f"Missing cached prompt contexts: {missing}") + self.action_mean = action_mean + self.action_std = action_std + self.action_clip = action_clip + self.rgb_latent_source_fingerprint = expected_fingerprint + configured_latents = self.config["paths"].get("rgb_latent_cache_dir") + self.rgb_latent_cache_dir = ( + Path(configured_latents) if configured_latents else None + ) + if self.rgb_latent_cache_dir is not None: + latent_manifest_path = self.rgb_latent_cache_dir / "manifest.json" + if not latent_manifest_path.is_file(): + raise FileNotFoundError(latent_manifest_path) + latent_manifest = json.loads( + latent_manifest_path.read_text(encoding="utf-8") + ) + latent_source_fingerprint = str( + latent_manifest.get("source_fingerprint", "") + ) + if latent_source_fingerprint != expected_fingerprint: + compatible = { + str(value) + for value in self.config["paths"].get( + "rgb_latent_compatible_source_fingerprints", [] + ) + } + if latent_source_fingerprint not in compatible: + raise RuntimeError("RGB latent cache fingerprint mismatch") + self.rgb_latent_source_fingerprint = latent_source_fingerprint + print( + "[latent-cache] accepting explicitly compatible RGB-only " + f"source fingerprint {latent_source_fingerprint}", + flush=True, + ) + if latent_manifest.get("latent_shape") != [48, 9, 16, 16]: + raise RuntimeError("RGB latent cache shape mismatch") + if int(latent_manifest.get("episode_count", -1)) != len(self.episodes): + raise RuntimeError("RGB latent cache episode-count mismatch") + if int(latent_manifest.get("valid_window_count", -1)) != self.total_windows: + raise RuntimeError("RGB latent cache window-count mismatch") + print( + f"[sampling] split={split} episodes={len(self.episodes)} " + f"windows_per_epoch={self.total_windows} replacement=false " + f"shuffle_each_epoch=true " + f"rgb_source={'precomputed_latent' if self.rgb_latent_cache_dir else 'video'}", + flush=True, + ) + + def __len__(self) -> int: + return self.total_windows + + def _window_at(self, index: int) -> tuple[Any, int]: + if not 0 <= int(index) < self.total_windows: + raise IndexError(index) + episode_slot = int(np.searchsorted(self.cumulative_starts, index, side="right")) + previous = 0 if episode_slot == 0 else int(self.cumulative_starts[episode_slot - 1]) + return self.episodes[episode_slot], int(index) - previous + + def raw_payload(self, index: int) -> dict[str, Any]: + episode, start = self._window_at(int(index)) + state, absolute_action, timestamps = _load_episode_arrays(self.config, episode) + indices, image_is_pad, action_is_pad = window_indices_and_pad_masks( + self.config, length=int(episode.length), start=start + ) + targets = base.build_egoverse_delta_targets(state, absolute_action, indices) + if tuple(targets["action_raw"].shape[:1]) != (self.window_size - 1,): + raise RuntimeError("Action target horizon does not match the sampling window") + return { + "sample_id": f"episode_{episode.episode_index:06d}_start_{start:06d}", + "episode_index": int(episode.episode_index), + "window_start": start, + "sample_probability": 1.0 / float(self.total_windows), + "action_raw": targets["action_raw"], + "action_is_pad": action_is_pad, + "image_is_pad": image_is_pad, + "initial_state": targets["initial_state"], + "indices": indices, + "timestamps": timestamps[indices], + "episode": episode, + } + + def __getitem__(self, index: int) -> dict[str, Any]: + payload = self.raw_payload(index) + episode = payload.pop("episode") + visual: dict[str, torch.Tensor] + if self.rgb_latent_cache_dir is None: + root = Path(self.config["paths"]["dataset_root"]) + clip_config = base.clip_config_from_config(self.config) + visual = { + "rgb": base.decode_rgb_window( + base.video_path(root, episode, clip_config.video_key), + episode_from_timestamp=episode.video_from_timestamp, + image_indices=payload["indices"], + config=clip_config, + ) + } + else: + latent_path = self.rgb_latent_cache_dir / ( + f"episode_{int(episode.episode_index):06d}.pt" + ) + latent_key = str(latent_path) + latents = _RGB_LATENT_CACHE.get(latent_key) + if latents is None: + shard = torch.load( + latent_path, map_location="cpu", weights_only=True, mmap=True + ) + if shard.get("source_fingerprint") != self.rgb_latent_source_fingerprint: + raise RuntimeError(f"RGB latent shard fingerprint mismatch: {latent_path}") + latents = shard["z_rgb"] + episode_slot = self.episode_slot_by_id[int(episode.episode_index)] + expected = (int(self.valid_starts[episode_slot]), 48, 9, 16, 16) + if tuple(latents.shape) != expected or latents.dtype != torch.bfloat16: + raise RuntimeError( + f"RGB latent shard mismatch: {latent_path} " + f"shape={tuple(latents.shape)} dtype={latents.dtype} expected={expected}" + ) + _RGB_LATENT_CACHE[latent_key] = latents + visual = {"z_rgb": latents[int(payload["window_start"])]} + action = payload["action_raw"] + if self.action_mean is not None: + action = (action - self.action_mean) / self.action_std + if self.action_clip is not None: + action = action.clamp(-self.action_clip, self.action_clip) + action = action.masked_fill(payload["action_is_pad"].unsqueeze(-1), 0) + prompt = self.prompt_by_text[episode.task] + return { + **visual, + "sample_id": payload["sample_id"], + "episode_index": torch.tensor(payload["episode_index"], dtype=torch.long), + "window_start": torch.tensor(payload["window_start"], dtype=torch.long), + "sample_probability": torch.tensor(payload["sample_probability"], dtype=torch.float64), + "action": action, + "action_is_pad": payload["action_is_pad"], + "image_is_pad": payload["image_is_pad"], + "initial_state": payload["initial_state"], + "context": prompt["context"], + "context_mask": prompt["context_mask"].to(dtype=torch.bool), + "prompt": prompt["prompt"], + } + + +class EpochSeedPermutationSampler(torch.utils.data.Sampler[int]): + """Shuffle all windows without replacement using a new seed each epoch.""" + + def __init__(self, data_source: torch.utils.data.Dataset, seed: int) -> None: + self.data_source = data_source + self.seed = int(seed) + self.epoch = 0 + + def __iter__(self): + epoch = self.epoch + self.epoch += 1 + generator = torch.Generator() + generator.manual_seed(self.seed + epoch) + print( + f"[sampling] shuffle epoch={epoch} seed={self.seed + epoch} " + f"windows={len(self.data_source)}", + flush=True, + ) + return iter(torch.randperm(len(self.data_source), generator=generator).tolist()) + + def __len__(self) -> int: + return len(self.data_source) + + +def collate_episode_length_normalized(samples: list[dict[str, Any]]) -> dict[str, Any]: + visual_key = "z_rgb" if "z_rgb" in samples[0] else "rgb" + if not all(visual_key in sample for sample in samples): + raise RuntimeError("Mixed RGB and latent samples in one batch") + tensor_keys = ( + visual_key, + "action", + "action_is_pad", + "image_is_pad", + "initial_state", + "context", + "context_mask", + "episode_index", + "window_start", + "sample_probability", + ) + return { + **{key: torch.stack([sample[key] for sample in samples]) for key in tensor_keys}, + "sample_id": [sample["sample_id"] for sample in samples], + "prompt": [sample["prompt"] for sample in samples], + } + + +def compute_loss_with_online_vae( + model: torch.nn.Module, + batch: dict[str, Any], + video_scheduler: Any, + action_scheduler: Any, + config: dict[str, Any], + device: torch.device, + dtype: torch.dtype, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + global _DYNAMIC_VAE + if "z_rgb" in batch: + z_rgb = batch["z_rgb"].to(device=device, dtype=dtype, non_blocking=True) + else: + if "rgb" not in batch: + raise RuntimeError("Exhaustive-window batch is missing RGB or z_rgb") + if _DYNAMIC_VAE is None: + configured = os.environ.get("ROBODOJO_DYNAMIC_VAE_DIR", "").strip() + model_dir = Path(configured) if configured else Path(config["paths"]["pretrained_model_dir"]) + print(f"[sampling] loading frozen online VAE from {model_dir}", flush=True) + _DYNAMIC_VAE = base.load_vae(model_dir, dtype, device) + _DYNAMIC_VAE.eval().requires_grad_(False) + rgb = batch["rgb"].to(device=device, dtype=dtype, non_blocking=True) + with torch.inference_mode(): + z_rgb = _DYNAMIC_VAE.encode(rgb, device=device) + if tuple(z_rgb.shape[1:]) != (48, 9, 16, 16): + raise RuntimeError(f"Unexpected online VAE latent shape: {tuple(z_rgb.shape)}") + encoded = dict(batch) + encoded["z_rgb"] = z_rgb + encoded.pop("rgb", None) + return _original_compute_loss( + model, + encoded, + video_scheduler, + action_scheduler, + config, + device, + dtype, + ) + + +@torch.no_grad() +def fit_action_normalizer_cached( + model: torch.nn.Module, dataset: Any +) -> None: + if _ACTIVE_CONFIG is None: + raise RuntimeError("Active config is unavailable for action normalizer setup") + init_config = _ACTIVE_CONFIG["initialization"] + normalizer_transfer = str( + init_config.get( + "action_normalizer_transfer", TARGET_ACTION_NORMALIZER_TRANSFER + ) + ) + if normalizer_transfer == ROBODOJO_FULL_ACTION_NORMALIZER_TRANSFER: + stats_path = Path(str(init_config["action_normalizer_path"])) + expected_sha256 = str(init_config["action_normalizer_sha256"]) + actual_sha256 = hashlib.sha256(stats_path.read_bytes()).hexdigest() + if actual_sha256 != expected_sha256: + raise RuntimeError( + "RoboDojo-full action normalizer hash mismatch: " + f"{actual_sha256} != {expected_sha256}" + ) + payload = torch.load(stats_path, map_location="cpu", weights_only=True) + required = { + "normalization_type", + "q01", + "q99", + "mean", + "std", + "count", + "fitted", + } + if not isinstance(payload, dict) or not required.issubset(payload): + raise RuntimeError( + f"Invalid RoboDojo-full action normalizer payload: {stats_path}" + ) + if payload["normalization_type"] != "pi05_q01_q99_all_action_v1": + raise RuntimeError("Unexpected RoboDojo-full normalization type") + q01 = torch.as_tensor(payload["q01"], dtype=torch.float32) + q99 = torch.as_tensor(payload["q99"], dtype=torch.float32) + mean = torch.as_tensor(payload["mean"], dtype=torch.float32) + std = torch.as_tensor(payload["std"], dtype=torch.float32) + if any(tuple(value.shape) != (14,) for value in (q01, q99, mean, std)): + raise RuntimeError("RoboDojo-full action normalizer must contain 14-D stats") + if not all(torch.isfinite(value).all() for value in (q01, q99, mean, std)): + raise RuntimeError("RoboDojo-full action normalizer contains non-finite values") + if not torch.all(q99 > q01) or not torch.all(std > 0): + raise RuntimeError("RoboDojo-full action normalizer has invalid ranges") + if not torch.allclose(mean, (q01 + q99) * 0.5, atol=1.0e-6, rtol=0.0): + raise RuntimeError("RoboDojo-full q01/q99 center does not match mean buffer") + if not torch.allclose(std, (q99 - q01) * 0.5, atol=1.0e-6, rtol=0.0): + raise RuntimeError("RoboDojo-full q01/q99 scale does not match std buffer") + if not bool(payload["fitted"]) or int(payload["count"]) < 2: + raise RuntimeError("RoboDojo-full action normalizer is not fitted") + normalizer = model.action_normalizer + target = normalizer.state_dict(keep_vars=True) + target["mean"].copy_(mean.to(target["mean"])) + target["std"].copy_(std.to(target["std"])) + target["count"].fill_(int(payload["count"])) + target["fitted"].fill_(True) + print( + "[normalizer] loaded pretraining RoboDojo-full q01/q99; " + f"path={stats_path} sha256={actual_sha256} " + f"count={int(payload['count'])} target_cache_ignored={dataset.cache_dir}", + flush=True, + ) + return + initialization = getattr(model, "_robodojo_initialization", {}) + if initialization.get("action_normalizer") == SOURCE_ACTION_NORMALIZER_TRANSFER: + normalizer = model.action_normalizer + # Joint checkpoints are loaded only on rank zero before FSDP wrapping. + # The dataset consumes these buffers before that broadcast, so every + # rank must import the four small buffers here as well. + if not bool(normalizer.fitted.item()): + local_checkpoint = init_config.get("local_checkpoint") + checkpoint = Path( + local_checkpoint + if local_checkpoint and Path(local_checkpoint).is_file() + else init_config["checkpoint"] + ) + payload = torch.load(checkpoint, map_location="cpu", weights_only=True, mmap=True) + source = payload.get("state_dict") if isinstance(payload, dict) else None + if not isinstance(source, dict): + raise RuntimeError(f"Invalid source checkpoint payload: {checkpoint}") + target = normalizer.state_dict(keep_vars=True) + for short_key in ("count", "fitted", "mean", "std"): + source_key = f"action_normalizer.{short_key}" + if source_key not in source or short_key not in target: + raise RuntimeError(f"Missing action normalizer buffer {source_key}") + value = source[source_key] + if tuple(value.shape) != tuple(target[short_key].shape): + raise RuntimeError(f"Action normalizer shape mismatch for {source_key}") + target[short_key].copy_(value.to(dtype=target[short_key].dtype)) + del source, payload + if not bool(normalizer.fitted.item()) or int(normalizer.count.item()) < 2: + raise RuntimeError( + "Requested source action-normalizer preservation, but the imported " + "checkpoint normalizer is not fitted" + ) + if not torch.isfinite(normalizer.mean).all() or not torch.isfinite(normalizer.std).all(): + raise RuntimeError("Imported source action normalizer contains non-finite values") + if not torch.all(normalizer.std > 0): + raise RuntimeError("Imported source action normalizer has non-positive scales") + print( + "[normalizer] preserving source pretraining q01/q99 buffers; " + f"count={int(normalizer.count.item())} target_cache_ignored={dataset.cache_dir}", + flush=True, + ) + return + cache_dir = Path(dataset.cache_dir) + manifest = json.loads((cache_dir / "manifest.json").read_text(encoding="utf-8")) + stats_path = cache_dir / "action_normalizer.pt" + if not stats_path.is_file(): + _original_fit_action_normalizer(model, dataset) + if int(os.environ.get("LOCAL_RANK", "0")) == 0: + temporary = cache_dir / f"action_normalizer.pt.rank0.{os.getpid()}.tmp" + torch.save( + { + "fingerprint": manifest["fingerprint"], + "mean": model.action_normalizer.mean.detach().cpu(), + "std": model.action_normalizer.std.detach().cpu(), + "count": model.action_normalizer.count.detach().cpu(), + "fitted": model.action_normalizer.fitted.detach().cpu(), + }, + temporary, + ) + os.replace(temporary, stats_path) + if torch.distributed.is_available() and torch.distributed.is_initialized(): + torch.distributed.barrier() + if int(os.environ.get("LOCAL_RANK", "0")) == 0: + print(f"[normalizer] fitted and cached {stats_path}", flush=True) + return + + payload = torch.load(stats_path, map_location="cpu", weights_only=True) + if payload.get("fingerprint") != manifest.get("fingerprint"): + raise RuntimeError("Cached action normalizer fingerprint mismatch") + normalizer = model.action_normalizer + if tuple(payload["mean"].shape) != tuple(normalizer.mean.shape): + raise RuntimeError("Cached action normalizer mean shape mismatch") + if tuple(payload["std"].shape) != tuple(normalizer.std.shape): + raise RuntimeError("Cached action normalizer std shape mismatch") + if int(payload["count"].item()) < 2 or not bool(payload["fitted"].item()): + raise RuntimeError("Cached action normalizer is not fitted") + normalizer.mean.copy_(payload["mean"].to(normalizer.mean)) + normalizer.std.copy_(payload["std"].to(normalizer.std)) + normalizer.count.copy_(payload["count"].to(normalizer.count)) + normalizer.fitted.copy_(payload["fitted"].to(normalizer.fitted)) + print(f"[normalizer] reuse {stats_path}", flush=True) + + +def _load_config_recursive(path: Path) -> dict[str, Any]: + overlay = _original_load_config(path) + base_path = overlay.pop("base_config", None) + if base_path is None: + return overlay + base_path = Path(base_path) + if not base_path.is_absolute(): + base_path = path.parent / base_path + return _deep_merge(_load_config_recursive(base_path), overlay) + + +def load_config(path: Path) -> dict[str, Any]: + return _load_config_recursive(path) + + +def validate_config(config: dict[str, Any]) -> dict[str, Any]: + global _ACTIVE_CONFIG + objective = str(config.get("experiment_variant", {}).get("objective", "joint")) + if objective not in {"joint", "action_only"}: + raise ValueError("experiment_variant.objective must be joint or action_only") + + lambda_rgb = float(config["diffusion"]["lambda_rgb"]) + if objective == "action_only": + if lambda_rgb != 0.0: + raise ValueError("action_only requires diffusion.lambda_rgb=0") + config["diffusion"]["lambda_rgb"] = 1.0 + try: + validated = _original_validate_config(config) + finally: + config["diffusion"]["lambda_rgb"] = lambda_rgb + + expected_indices = {name: value[0] for name, value in EXPECTED_TASKS.items()} + configured_indices = { + str(name): int(index) + for name, index in config["robodojo"]["task_index_by_id"].items() + } + if configured_indices != expected_indices: + raise ValueError( + f"task_index_by_id mismatch: {configured_indices} != {expected_indices}" + ) + resolved = base.selected_task_prompts(config) + expected_prompts = {name: value[1] for name, value in EXPECTED_TASKS.items()} + if resolved != expected_prompts: + raise ValueError(f"Resolved task prompts mismatch: {resolved} != {expected_prompts}") + + train_video = bool(config["training"].get("train_video_expert", True)) + if train_video != (objective == "joint"): + raise ValueError("train_video_expert must be true only for joint objective") + model = config["model"] + if model.get("action_loss_to_video_stop_gradient") is not False: + raise ValueError( + "This sibling requires action loss gradients to reach video K/V" + ) + checkpoint_flags = ( + bool(model.get("mot_checkpoint_mixed_attn", True)), + bool(model["dit"].get("use_gradient_checkpointing", True)), + bool(model["action_dit"].get("use_gradient_checkpointing", True)), + ) + if checkpoint_flags != (False, False, False): + raise ValueError(f"All gradient checkpointing must be disabled: {checkpoint_flags}") + max_steps = int(config["training"]["max_steps"]) + smoke_mode = os.environ.get("ROBODOJO_4WAY_SMOKE", "0") == "1" + exact_epochs = config["training"].get("exact_epochs") + if smoke_mode and 1 <= max_steps <= 50: + config.setdefault("tracking", {})["enabled"] = False + elif exact_epochs is not None: + exact_epochs = int(exact_epochs) + valid_windows = int(config["training"].get("valid_windows_per_epoch", -1)) + global_batch = int(config["training"].get("global_batch_size", -1)) + expected_steps = exact_epochs * math.ceil(valid_windows / global_batch) + if exact_epochs <= 0 or valid_windows <= 0 or global_batch <= 0: + raise ValueError("Exact-epoch training requires positive epoch/window/batch values") + if max_steps != expected_steps: + raise ValueError( + f"Exact {exact_epochs}-epoch run requires max_steps={expected_steps}, " + f"got {max_steps}" + ) + elif max_steps != 65_000: + raise ValueError("These four exhaustive experiments require exactly 65000 steps") + save_every = int(config["training"]["save_every"]) + sweep_interval = config["training"].get("checkpoint_sweep_interval") + if sweep_interval is None: + allowed_intervals = {10_000} if exact_epochs is None else {5_000, 10_000} + if save_every not in allowed_intervals: + raise ValueError( + "Exhaustive experiments without checkpoint sweeps require " + f"save_every in {sorted(allowed_intervals)}" + ) + elif save_every != int(sweep_interval) or save_every != 2_000: + raise ValueError( + "Success-rate checkpoint sweeps require save_every=" + "checkpoint_sweep_interval=2000" + ) + scheduler_step_unit = config["training"].get("scheduler_step_unit") + if scheduler_step_unit is not None and scheduler_step_unit != "optimizer_step": + raise ValueError( + "training.scheduler_step_unit must be optimizer_step when explicitly set" + ) + + sampling = config.get("sampling") + legacy_sampling = { + "mode": "exhaustive_shuffled_window", + "window_size": 33, + "replacement": False, + "shuffle_each_epoch": True, + "epoch_seed": int(config["data"]["sample_order_seed"]), + "samples_per_epoch": "all_valid_windows", + } + fastwam_tail_sampling = { + "mode": FASTWAM_TAIL_PAD_SAMPLING_MODE, + "window_size": 33, + "replacement": False, + "shuffle_each_epoch": True, + "epoch_seed": int(config["data"]["sample_order_seed"]), + "samples_per_epoch": "all_episode_frames", + "tail_padding": { + "image": "repeat_last", + "action": "repeat_last", + "loss": "mask_padded", + }, + } + if sampling not in (legacy_sampling, fastwam_tail_sampling): + raise ValueError( + "sampling must use either the legacy valid-window contract or the " + "FastWAM all-frame tail-mask contract: " + f"actual={sampling}" + ) + if int(config["data"].get("num_workers", 0)) != 0: + raise ValueError("Online exhaustive-window sampling currently requires data.num_workers=0") + + initialization = config.get("initialization", {}) + mode = str(initialization.get("mode", "fresh")) + if mode not in { + "fresh", + "joint_checkpoint_rgb_slice", + "joint_checkpoint_rgb_slice_action_backbone", + EGO_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_RGB_DEPTH_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_RGB_DEPTH_POINT_FULL_ACTION_INITIALIZATION_MODE, + "joint_checkpoint_full", + }: + raise ValueError("initialization.mode is invalid") + if mode in { + "joint_checkpoint_rgb_slice", + "joint_checkpoint_rgb_slice_action_backbone", + EGO_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_RGB_DEPTH_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_RGB_DEPTH_POINT_FULL_ACTION_INITIALIZATION_MODE, + }: + checkpoint = Path(initialization["checkpoint"]) + if not checkpoint.is_file(): + raise FileNotFoundError(checkpoint) + flavor = str(initialization.get("flavor")) + if flavor not in {"camera6d", "no_camera"}: + raise ValueError("checkpoint initialization flavor must be camera6d or no_camera") + if mode in { + V2_BASEFRAME_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_RGB_DEPTH_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_RGB_DEPTH_POINT_FULL_ACTION_INITIALIZATION_MODE, + }: + for key in ("source_sha256", "source_action_contract_fingerprint"): + value = str(initialization.get(key, "")) + if len(value) != 64: + raise ValueError( + f"initialization.{key} must contain 64 hexadecimal characters" + ) + try: + int(value, 16) + except ValueError as error: + raise ValueError( + f"initialization.{key} must contain 64 hexadecimal characters" + ) from error + sidecar = Path(str(initialization.get("contract_sidecar", ""))) + if not sidecar.is_file(): + raise FileNotFoundError(sidecar) + normalizer_transfer = str( + initialization.get("action_normalizer_transfer", TARGET_ACTION_NORMALIZER_TRANSFER) + ) + if normalizer_transfer not in { + TARGET_ACTION_NORMALIZER_TRANSFER, + SOURCE_ACTION_NORMALIZER_TRANSFER, + ROBODOJO_FULL_ACTION_NORMALIZER_TRANSFER, + }: + raise ValueError( + "initialization.action_normalizer_transfer must be " + f"{TARGET_ACTION_NORMALIZER_TRANSFER!r}, " + f"{SOURCE_ACTION_NORMALIZER_TRANSFER!r}, or " + f"{ROBODOJO_FULL_ACTION_NORMALIZER_TRANSFER!r}" + ) + if normalizer_transfer == ROBODOJO_FULL_ACTION_NORMALIZER_TRANSFER: + normalizer_path = Path(str(initialization.get("action_normalizer_path", ""))) + if not normalizer_path.is_file(): + raise FileNotFoundError(normalizer_path) + normalizer_sha256 = str(initialization.get("action_normalizer_sha256", "")) + if len(normalizer_sha256) != 64: + raise ValueError( + "initialization.action_normalizer_sha256 must contain 64 hexadecimal characters" + ) + try: + int(normalizer_sha256, 16) + except ValueError as error: + raise ValueError( + "initialization.action_normalizer_sha256 must contain 64 hexadecimal characters" + ) from error + if ( + normalizer_transfer == SOURCE_ACTION_NORMALIZER_TRANSFER + and mode not in { + V2_BASEFRAME_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_RGB_DEPTH_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_RGB_DEPTH_POINT_FULL_ACTION_INITIALIZATION_MODE, + } + ): + raise ValueError( + "Source action-normalizer preservation is allowed only for a sealed " + "V2 base-frame full-action checkpoint transfer" + ) + elif mode == "joint_checkpoint_full": + if objective != "joint": + raise ValueError("Full checkpoint initialization requires the joint objective") + checkpoint = Path(initialization["checkpoint"]) + if not checkpoint.is_file(): + raise FileNotFoundError(checkpoint) + if str(initialization.get("flavor")) != "camera6d": + raise ValueError("Full checkpoint initialization requires flavor=camera6d") + _ACTIVE_CONFIG = validated + return validated + + +def _rgb_head_slice(source: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + patch_volume = 4 + source_channels = 144 + target_channels = 48 + if source.ndim == 2: + expected = (patch_volume * source_channels, target.shape[1]) + if tuple(source.shape) != expected: + raise RuntimeError(f"Unexpected source head weight shape: {tuple(source.shape)}") + return source.view(patch_volume, source_channels, target.shape[1])[ + :, :target_channels + ].reshape_as(target) + if source.ndim == 1: + if tuple(source.shape) != (patch_volume * source_channels,): + raise RuntimeError(f"Unexpected source head bias shape: {tuple(source.shape)}") + return source.view(patch_volume, source_channels)[:, :target_channels].reshape_as(target) + raise RuntimeError(f"Unsupported output-head tensor rank: {source.ndim}") + + +@torch.no_grad() +def _load_joint_checkpoint_rgb_slice( + model: torch.nn.Module, checkpoint: Path, flavor: str +) -> None: + payload = torch.load(checkpoint, map_location="cpu", weights_only=True, mmap=True) + if not isinstance(payload, dict) or set(payload) != {"metadata", "state_dict"}: + raise ValueError(f"Invalid joint checkpoint payload: {checkpoint}") + source = payload["state_dict"] + target = model.state_dict(keep_vars=True) + source_keys, target_keys = set(source), set(target) + expected_extras = POINT_EXTRAS | (CAMERA_EXTRAS if flavor == "camera6d" else set()) + if target_keys - source_keys: + raise RuntimeError(f"Checkpoint missing target keys: {sorted(target_keys - source_keys)}") + if source_keys - target_keys != expected_extras: + raise RuntimeError( + f"Unexpected checkpoint-only keys: {sorted(source_keys - target_keys)}; " + f"expected {sorted(expected_extras)}" + ) + + converted = [] + for key in sorted(target_keys): + source_tensor = source[key] + target_tensor = target[key] + if tuple(source_tensor.shape) == tuple(target_tensor.shape): + value = source_tensor + elif key == "mot.mixtures.video.patch_embedding.weight": + expected_source = ( + target_tensor.shape[0], + 144, + *target_tensor.shape[2:], + ) + if tuple(source_tensor.shape) != tuple(expected_source): + raise RuntimeError( + f"Unexpected joint patch weight shape: {tuple(source_tensor.shape)}" + ) + value = source_tensor[:, :48] + converted.append(key) + elif key in { + "mot.mixtures.video.head.head.weight", + "mot.mixtures.video.head.head.bias", + }: + value = _rgb_head_slice(source_tensor, target_tensor) + converted.append(key) + else: + raise RuntimeError( + f"Unapproved shape mismatch for {key}: " + f"source={tuple(source_tensor.shape)} target={tuple(target_tensor.shape)}" + ) + target_tensor.copy_(value.to(dtype=target_tensor.dtype)) + + expected_converted = { + "mot.mixtures.video.patch_embedding.weight", + "mot.mixtures.video.head.head.weight", + "mot.mixtures.video.head.head.bias", + } + if set(converted) != expected_converted: + raise RuntimeError(f"RGB slice conversion audit failed: {converted}") + print( + f"[initialization] strict-loaded {len(target_keys)} RGB+Action keys from {checkpoint}; " + f"converted={sorted(converted)}; ignored={sorted(expected_extras)}", + flush=True, + ) + del source, payload + gc.collect() + + +def _is_action_backbone_key(key: str) -> bool: + return key.startswith(ACTION_BACKBONE_PREFIX) and not any( + key.startswith(prefix) for prefix in ACTION_REINITIALIZED_PREFIXES + ) + + +def _audit_action_backbone_keys( + model: torch.nn.Module, action_keys: set[str] +) -> dict[str, Any]: + """Prove that the selected keys cover the complete 30-layer ActionDiT.""" + + block_prefix = f"{ACTION_BACKBONE_PREFIX}blocks." + block_indices: set[int] = set() + for key in action_keys: + if key.startswith(block_prefix): + suffix = key[len(block_prefix):] + try: + block_indices.add(int(suffix.split(".", 1)[0])) + except (ValueError, IndexError) as exc: + raise RuntimeError(f"Malformed ActionDiT block key: {key}") from exc + + expected_layers = len(model.action_expert.blocks) + if expected_layers != 30 or block_indices != set(range(expected_layers)): + raise RuntimeError( + "ActionDiT block coverage mismatch: " + f"model_layers={expected_layers} checkpoint_layers={sorted(block_indices)}" + ) + + required_global_prefixes = { + "text_embedding": f"{ACTION_BACKBONE_PREFIX}text_embedding.", + "time_embedding": f"{ACTION_BACKBONE_PREFIX}time_embedding.", + "time_projection": f"{ACTION_BACKBONE_PREFIX}time_projection.", + } + missing_global = [ + name + for name, prefix in required_global_prefixes.items() + if not any(key.startswith(prefix) for key in action_keys) + ] + if missing_global: + raise RuntimeError( + f"ActionDiT backbone is missing global modules: {missing_global}" + ) + + required_block_modules = ( + "self_attn.", + "cross_attn.", + "ffn.", + "norm3.", + "modulation", + ) + for layer in range(expected_layers): + prefix = f"{block_prefix}{layer}." + layer_keys = {key[len(prefix):] for key in action_keys if key.startswith(prefix)} + missing = [ + module + for module in required_block_modules + if not any(key.startswith(module) for key in layer_keys) + ] + if missing: + raise RuntimeError( + f"ActionDiT block {layer} is missing modules: {missing}" + ) + + return { + "layers": expected_layers, + "keys": len(action_keys), + "modules": [ + "self/mixed_attention", + "text_cross_attention", + "ffn", + "norm", + "modulation", + "text_embedding", + "time_embedding", + "time_projection", + ], + } + + +@torch.no_grad() +def _load_joint_checkpoint_rgb_slice_action_backbone( + model: torch.nn.Module, checkpoint: Path, flavor: str +) -> None: + """Load SpaceDreamer RGB plus ActionDiT backbone, preserving RoboDojo I/O.""" + + payload = torch.load(checkpoint, map_location="cpu", weights_only=True, mmap=True) + if not isinstance(payload, dict) or set(payload) != {"metadata", "state_dict"}: + raise ValueError(f"Invalid joint checkpoint payload: {checkpoint}") + source = payload["state_dict"] + target = model.state_dict(keep_vars=True) + source_keys, target_keys = set(source), set(target) + expected_extras = POINT_EXTRAS | (CAMERA_EXTRAS if flavor == "camera6d" else set()) + if target_keys - source_keys: + raise RuntimeError(f"Checkpoint missing target keys: {sorted(target_keys - source_keys)}") + if source_keys - target_keys != expected_extras: + raise RuntimeError( + f"Unexpected checkpoint-only keys: {sorted(source_keys - target_keys)}; " + f"expected {sorted(expected_extras)}" + ) + + video_keys = {key for key in target_keys if key.startswith(VIDEO_EXPERT_PREFIX)} + action_backbone_keys = {key for key in target_keys if _is_action_backbone_key(key)} + reinitialized_keys = { + key + for key in target_keys + if any(key.startswith(prefix) for prefix in ACTION_REINITIALIZED_PREFIXES) + } + uncategorized = target_keys - video_keys - action_backbone_keys - reinitialized_keys + if uncategorized: + raise RuntimeError( + f"Selective initialization has uncategorized target keys: {sorted(uncategorized)}" + ) + for prefix in ACTION_REINITIALIZED_PREFIXES: + if not any(key.startswith(prefix) for key in reinitialized_keys): + raise RuntimeError(f"RoboDojo reinitialization prefix matched no keys: {prefix}") + backbone_audit = _audit_action_backbone_keys(model, action_backbone_keys) + + # These clones are deliberately retained until after all copies so an + # accidental future broadening of the transfer policy fails immediately. + preserved = { + key: target[key].detach().clone() for key in sorted(reinitialized_keys) + } + converted: list[str] = [] + load_keys = video_keys | action_backbone_keys + for key in sorted(load_keys): + source_tensor = source[key] + target_tensor = target[key] + if tuple(source_tensor.shape) == tuple(target_tensor.shape): + value = source_tensor + elif key == "mot.mixtures.video.patch_embedding.weight": + expected_source = ( + target_tensor.shape[0], + 144, + *target_tensor.shape[2:], + ) + if tuple(source_tensor.shape) != tuple(expected_source): + raise RuntimeError( + f"Unexpected joint patch weight shape: {tuple(source_tensor.shape)}" + ) + value = source_tensor[:, :48] + converted.append(key) + elif key in { + "mot.mixtures.video.head.head.weight", + "mot.mixtures.video.head.head.bias", + }: + value = _rgb_head_slice(source_tensor, target_tensor) + converted.append(key) + else: + raise RuntimeError( + f"Unapproved shape mismatch for {key}: " + f"source={tuple(source_tensor.shape)} target={tuple(target_tensor.shape)}" + ) + target_tensor.copy_(value.to(dtype=target_tensor.dtype)) + + expected_converted = { + "mot.mixtures.video.patch_embedding.weight", + "mot.mixtures.video.head.head.weight", + "mot.mixtures.video.head.head.bias", + } + if set(converted) != expected_converted: + raise RuntimeError(f"RGB slice conversion audit failed: {converted}") + changed_preserved = [ + key for key, before in preserved.items() if not torch.equal(target[key], before) + ] + if changed_preserved: + raise RuntimeError( + f"RoboDojo-specific modules were overwritten: {changed_preserved}" + ) + + reinitialized_counts = { + prefix.rstrip("."): sum(key.startswith(prefix) for key in reinitialized_keys) + for prefix in ACTION_REINITIALIZED_PREFIXES + } + print( + "[initialization] selective SpaceDreamer transfer passed: " + f"video_keys={len(video_keys)} action_backbone={backbone_audit} " + f"reinitialized={reinitialized_counts} converted={sorted(converted)} " + f"ignored_checkpoint_extras={sorted(expected_extras)} source={checkpoint}", + flush=True, + ) + del preserved, source, payload + gc.collect() + + +@torch.no_grad() +def _load_joint_checkpoint_rgb_slice_full_action_ego_contract( + model: torch.nn.Module, + checkpoint: Path, + flavor: str, + *, + preserve_source_action_normalizer: bool = False, +) -> dict[str, Any]: + """Fail-closed SpaceDreamer RGB slice plus complete action-branch import. + + The source action-normalizer buffers are required to exist but are never + copied. RoboDojo C0-frame mean/std statistics are fitted after this load. + Every trainable target tensor is copied and then compared numerically. + """ + + payload = torch.load(checkpoint, map_location="cpu", weights_only=True, mmap=True) + if not isinstance(payload, dict) or set(payload) != {"metadata", "state_dict"}: + raise ValueError(f"Invalid joint checkpoint payload: {checkpoint}") + metadata = payload["metadata"] + if not isinstance(metadata, dict): + raise TypeError("SpaceDreamer checkpoint metadata must be a mapping") + source_mot = metadata.get("mot") + required_source_mot = { + "model_class": "RgbDepthPointActionMoT", + "num_layers": 30, + "num_heads": 24, + "attn_head_dim": 128, + "action_dim": 14, + "action_state_dim": 14, + } + actual_source_mot = { + key: source_mot.get(key) if isinstance(source_mot, dict) else None + for key in required_source_mot + } + if actual_source_mot != required_source_mot: + raise RuntimeError( + "SpaceDreamer source action architecture mismatch: " + f"{actual_source_mot} != {required_source_mot}" + ) + + source = payload["state_dict"] + if not isinstance(source, dict): + raise TypeError("SpaceDreamer state_dict must be a mapping") + target = model.state_dict(keep_vars=True) + source_keys, target_keys = set(source), set(target) + expected_extras = POINT_EXTRAS | (CAMERA_EXTRAS if flavor == "camera6d" else set()) + missing = target_keys - source_keys + unexpected = source_keys - target_keys + if missing: + raise RuntimeError(f"Checkpoint missing target keys: {sorted(missing)}") + if unexpected != expected_extras: + raise RuntimeError( + f"Unexpected checkpoint-only keys: {sorted(unexpected)}; " + f"expected {sorted(expected_extras)}" + ) + + normalizer_keys = { + key for key in target_keys if key.startswith("action_normalizer.") + } + expected_normalizer_keys = { + "action_normalizer.count", + "action_normalizer.fitted", + "action_normalizer.mean", + "action_normalizer.std", + } + if normalizer_keys != expected_normalizer_keys: + raise RuntimeError( + f"Action normalizer key contract changed: {sorted(normalizer_keys)}" + ) + preserved_normalizer = { + key: target[key].detach().clone() for key in sorted(normalizer_keys) + } + load_keys = target_keys if preserve_source_action_normalizer else target_keys - normalizer_keys + trainable_keys = { + name for name, parameter in model.named_parameters() if parameter.requires_grad + } + if trainable_keys - load_keys: + raise RuntimeError( + "Trainable target parameters omitted from SpaceDreamer import: " + f"{sorted(trainable_keys - load_keys)}" + ) + + action_prefixes = { + "action_encoder": "mot.mixtures.action.action_encoder.", + "action_backbone": "mot.mixtures.action.blocks.", + "action_text_embedding": "mot.mixtures.action.text_embedding.", + "action_time_embedding": "mot.mixtures.action.time_embedding.", + "action_time_projection": "mot.mixtures.action.time_projection.", + "action_head": "mot.mixtures.action.head.", + "action_state_encoder": "action_state_encoder.", + } + module_key_counts = { + name: sum(key.startswith(prefix) for key in load_keys) + for name, prefix in action_prefixes.items() + } + if any(count == 0 for count in module_key_counts.values()): + raise RuntimeError( + f"Full action import has empty required modules: {module_key_counts}" + ) + backbone_keys = { + key + for key in load_keys + if _is_action_backbone_key(key) + or key.startswith("mot.mixtures.action.action_encoder.") + or key.startswith("mot.mixtures.action.head.") + } + _audit_action_backbone_keys( + model, + {key for key in backbone_keys if _is_action_backbone_key(key)}, + ) + + converted: list[str] = [] + expected_values: dict[str, torch.Tensor] = {} + for key in sorted(load_keys): + source_tensor = source[key] + target_tensor = target[key] + if tuple(source_tensor.shape) == tuple(target_tensor.shape): + value = source_tensor + elif key == "mot.mixtures.video.patch_embedding.weight": + expected_source = ( + target_tensor.shape[0], + 144, + *target_tensor.shape[2:], + ) + if tuple(source_tensor.shape) != tuple(expected_source): + raise RuntimeError( + f"Unexpected joint patch weight shape: {tuple(source_tensor.shape)}" + ) + value = source_tensor[:, :48] + converted.append(key) + elif key in { + "mot.mixtures.video.head.head.weight", + "mot.mixtures.video.head.head.bias", + }: + value = _rgb_head_slice(source_tensor, target_tensor) + converted.append(key) + else: + raise RuntimeError( + f"Unapproved shape mismatch for {key}: " + f"source={tuple(source_tensor.shape)} target={tuple(target_tensor.shape)}" + ) + expected = value.to(dtype=target_tensor.dtype) + target_tensor.copy_(expected) + expected_values[key] = expected + + expected_converted = { + "mot.mixtures.video.patch_embedding.weight", + "mot.mixtures.video.head.head.weight", + "mot.mixtures.video.head.head.bias", + } + if set(converted) != expected_converted: + raise RuntimeError(f"RGB slice conversion audit failed: {converted}") + unequal = [ + key for key in sorted(load_keys) if not torch.equal(target[key], expected_values[key]) + ] + if unequal: + raise RuntimeError(f"Post-load numerical equality failed: {unequal[:16]}") + if preserve_source_action_normalizer: + unequal_normalizer = [ + key + for key in sorted(normalizer_keys) + if not torch.equal(target[key], source[key].to(dtype=target[key].dtype)) + ] + if unequal_normalizer: + raise RuntimeError( + f"Source action normalizer was not copied exactly: {unequal_normalizer}" + ) + else: + changed_normalizer = [ + key + for key, before in preserved_normalizer.items() + if not torch.equal(target[key], before) + ] + if changed_normalizer: + raise RuntimeError( + f"Source action normalizer leaked into target: {changed_normalizer}" + ) + + audit = { + "loaded_key_count": len(load_keys), + "source_key_count": len(source_keys), + "target_key_count": len(target_keys), + "converted_rgb_slice_keys": sorted(converted), + "ignored_source_only_keys": sorted(expected_extras), + "action_normalizer_transfer": ( + SOURCE_ACTION_NORMALIZER_TRANSFER + if preserve_source_action_normalizer + else TARGET_ACTION_NORMALIZER_TRANSFER + ), + "target_refit_normalizer_keys": ( + [] if preserve_source_action_normalizer else sorted(normalizer_keys) + ), + "action_module_key_counts": module_key_counts, + "numerical_equality": "torch.equal_after_target_dtype_cast", + } + print( + "[initialization] fail-closed full SpaceDreamer action transfer passed: " + + json.dumps(audit, sort_keys=True), + flush=True, + ) + del expected_values, preserved_normalizer, source, payload + gc.collect() + return audit + + +def build_experiment_model(config: dict[str, Any]) -> torch.nn.Module: + initialization = config.get("initialization", {"mode": "fresh"}) + mode = str(initialization.get("mode", "fresh")) + normalizer_transfer = str( + initialization.get("action_normalizer_transfer", TARGET_ACTION_NORMALIZER_TRANSFER) + ) + preserve_source_action_normalizer = ( + normalizer_transfer == SOURCE_ACTION_NORMALIZER_TRANSFER + ) + if mode == "joint_checkpoint_full": + raise ValueError( + "joint_checkpoint_full is only supported by the native " + "RGB+depth+point+action model builder" + ) + original_model_dir = config["paths"]["pretrained_model_dir"] + local_model_dir = initialization.get("local_wan_dit_dir") + if local_model_dir and Path(local_model_dir).is_dir(): + config["paths"]["pretrained_model_dir"] = str(local_model_dir) + try: + model = _original_build_model(config) + finally: + config["paths"]["pretrained_model_dir"] = original_model_dir + + rgb_slice_modes = { + "joint_checkpoint_rgb_slice", + "joint_checkpoint_rgb_slice_action_backbone", + EGO_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_FULL_ACTION_INITIALIZATION_MODE, + } + load_audit = None + if mode in rgb_slice_modes and int(os.environ.get("LOCAL_RANK", "0")) == 0: + checkpoint = Path( + initialization.get("local_checkpoint") + if initialization.get("local_checkpoint") + and Path(initialization["local_checkpoint"]).is_file() + else initialization["checkpoint"] + ) + if mode == V2_BASEFRAME_FULL_ACTION_INITIALIZATION_MODE: + contract_audit = verify_v2_baseframe_checkpoint_contract( + checkpoint, + Path(initialization["contract_sidecar"]), + expected_checkpoint_sha256=str(initialization["source_sha256"]), + expected_action_contract_fingerprint=str( + initialization["source_action_contract_fingerprint"] + ), + ) + tensor_audit = _load_joint_checkpoint_rgb_slice_full_action_ego_contract( + model, + checkpoint, + str(initialization["flavor"]), + preserve_source_action_normalizer=preserve_source_action_normalizer, + ) + load_audit = { + "checkpoint_contract": contract_audit, + "tensor_transfer": tensor_audit, + } + elif mode == EGO_FULL_ACTION_INITIALIZATION_MODE: + load_audit = _load_joint_checkpoint_rgb_slice_full_action_ego_contract( + model, checkpoint, str(initialization["flavor"]) + ) + elif mode == "joint_checkpoint_rgb_slice_action_backbone": + _load_joint_checkpoint_rgb_slice_action_backbone( + model, checkpoint, str(initialization["flavor"]) + ) + else: + _load_joint_checkpoint_rgb_slice( + model, checkpoint, str(initialization["flavor"]) + ) + model._robodojo_initialization = { + "mode": mode, + "source_checkpoint": initialization.get("checkpoint"), + "rgb_slice_from_joint_video": mode in rgb_slice_modes, + "action_backbone_from_joint_checkpoint": ( + mode in { + "joint_checkpoint_rgb_slice_action_backbone", + EGO_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_FULL_ACTION_INITIALIZATION_MODE, + } + ), + "full_action_trainable_branch_from_joint_checkpoint": ( + mode in { + EGO_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_FULL_ACTION_INITIALIZATION_MODE, + } + ), + "action_normalizer": ( + ( + SOURCE_ACTION_NORMALIZER_TRANSFER + if preserve_source_action_normalizer + else ( + ROBODOJO_FULL_ACTION_NORMALIZER_TRANSFER + if normalizer_transfer + == ROBODOJO_FULL_ACTION_NORMALIZER_TRANSFER + else "target_refit" + ) + ) + if mode in { + EGO_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_FULL_ACTION_INITIALIZATION_MODE, + } + else "source_or_fresh_by_mode" + ), + "load_audit": ( + load_audit + if mode in { + EGO_FULL_ACTION_INITIALIZATION_MODE, + V2_BASEFRAME_FULL_ACTION_INITIALIZATION_MODE, + } + else None + ), + "robodojo_reinitialized_prefixes": ( + list(ACTION_REINITIALIZED_PREFIXES) + if mode == "joint_checkpoint_rgb_slice_action_backbone" + else [] + ), + } + model._robodojo_objective = str(config["experiment_variant"]["objective"]) + return model + + +def configure_trainable(model: torch.nn.Module) -> None: + if getattr(model, "_robodojo_objective") == "joint": + _original_configure_trainable(model) + return + model.requires_grad_(False) + model.action_expert.requires_grad_(True) + model.action_state_encoder.requires_grad_(True) + model.action_normalizer.requires_grad_(False) + print("[training] action-only objective: video expert frozen, lambda_rgb=0", flush=True) + + +def build_optimizer(model: torch.nn.Module, config: dict[str, Any]) -> torch.optim.Optimizer: + if str(config["experiment_variant"]["objective"]) == "joint": + return _original_build_optimizer(model, config) + parameters = [ + parameter + for module in (model.action_expert, model.action_state_encoder) + for parameter in module.parameters() + if parameter.requires_grad + ] + if not parameters: + raise RuntimeError("Action-only experiment has no trainable parameters") + return torch.optim.AdamW( + [{"name": "action_expert_and_state_encoder", "params": parameters, + "lr": float(config["training"]["action_lr"])}], + betas=(0.9, 0.95), + eps=1.0e-8, + weight_decay=float(config["training"].get("weight_decay", 0.0)), + ) + + +def checkpoint_metadata(model: torch.nn.Module) -> dict[str, Any]: + metadata = _original_checkpoint_metadata(model) + metadata["initialization"] = getattr(model, "_robodojo_initialization", {}) + metadata["training_objective"] = getattr(model, "_robodojo_objective", "unknown") + return metadata + + +def main() -> None: + base.load_config = load_config + base.validate_config = validate_config + base.cache_fingerprint = episode_length_normalized_cache_fingerprint + base.prepare_cache = prepare_episode_length_normalized_cache + base.RoboDojoCachedDataset = ExhaustiveWindowDataset + base.FixedSeedPermutationSampler = EpochSeedPermutationSampler + base.collate_rgb_action = collate_episode_length_normalized + base.compute_loss = compute_loss_with_online_vae + base.build_fresh_rgb_action_mot = build_experiment_model + base.configure_trainable = configure_trainable + base.build_optimizer = build_optimizer + base._cache_is_complete = cache_is_complete_fast + base.fit_action_normalizer = fit_action_normalizer_cached + base.Accelerator = build_precision_aware_accelerator + shared.rgb_action_checkpoint_metadata = checkpoint_metadata + if os.environ.get("ROBODOJO_4WAY_SMOKE", "0") == "1": + def skip_smoke_export( + model: torch.nn.Module, accelerator: Any, path: Path + ) -> None: + if accelerator.is_main_process: + print(f"[smoke] checkpoint export skipped: {path}", flush=True) + accelerator.wait_for_everyone() + + base.export_model = skip_smoke_export + base.main() + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_ablation.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_ablation.py new file mode 100644 index 0000000000000000000000000000000000000000..8a42e0ca268a75c8c4ebac6678b4ca580652dd45 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_ablation.py @@ -0,0 +1,996 @@ +#!/usr/bin/env python3 +"""Train RGB frame0 + EE state -> joint RGB video + EE delta action on RoboDojo.""" + +from __future__ import annotations + +import argparse +from datetime import timedelta +import hashlib +import json +import math +from pathlib import Path +import subprocess +import time +from typing import Any + +from accelerate import Accelerator, skip_first_batches +from accelerate.utils import InitProcessGroupKwargs, set_seed +import numpy as np +import torch +from torch.utils.data import DataLoader, Dataset +import yaml + +from optimizer_step_scheduler import ( + assert_lr_scheduler_optimizer_step_index, + lr_scheduler_audit_record, + position_lr_scheduler_at_optimizer_step, + step_lr_scheduler_on_optimizer_update, +) +from rgb_action_model_factory import build_fresh_rgb_action_mot +from robodojo_lerobot_v30 import ( + RoboDojoClipConfig, + RoboDojoEpisode, + build_egoverse_delta_targets, + data_parquet_path, + decode_rgb_window, + episode_arrays, + filter_episodes_by_task_prompts, + load_data_shard, + load_episodes, + read_dataset_info, + read_dataset_task_prompts, + resolve_training_task_prompts, + select_clip_indices, + stratified_episode_split, + video_path, +) +from train_galexea_rgb_state_joint_action_ablation import ( + FixedSeedPermutationSampler, + build_optimizer, + collate_rgb_action, + compute_loss, + configure_trainable, + enable_fp32_master_weights, + export_model, + fit_action_normalizer, +) +from train_joint_rgb_depth import ( + WanContinuousFlowMatchScheduler, + append_jsonl, + build_lr_scheduler, + dtype_from_name, + load_text_encoder_and_tokenizer, + load_vae, +) + + +SPLITS = ("train", "validation", "test") + + +def query_nvidia_smi_metrics() -> dict[str, float]: + """Collect per-GPU runtime telemetry for persisted/W&B milestone records.""" + + output = subprocess.check_output( + [ + "nvidia-smi", + "--query-gpu=index,memory.used,memory.total,utilization.gpu,temperature.gpu", + "--format=csv,noheader,nounits", + ], + text=True, + timeout=10, + ) + result: dict[str, float] = {} + for line in output.splitlines(): + fields = [field.strip() for field in line.split(",")] + if len(fields) != 5: + raise RuntimeError(f"Unexpected nvidia-smi telemetry row: {line!r}") + index, used, total, utilization, temperature = fields + prefix = f"gpu/{int(index)}" + result[f"{prefix}/memory_used_mib"] = float(used) + result[f"{prefix}/memory_total_mib"] = float(total) + result[f"{prefix}/utilization_percent"] = float(utilization) + result[f"{prefix}/temperature_c"] = float(temperature) + if not result: + raise RuntimeError("nvidia-smi returned no GPU telemetry") + return result + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--max-steps", type=int) + parser.add_argument("--prepare-cache-only", action="store_true") + parser.add_argument("--dataset-smoke-test", action="store_true") + parser.add_argument("--overwrite-cache", action="store_true") + return parser.parse_args() + + +def load_config(path: Path) -> dict[str, Any]: + config = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(config, dict): + raise ValueError("Config must be a YAML mapping") + return config + + +def clip_config_from_config(config: dict[str, Any]) -> RoboDojoClipConfig: + data, source = config["data"], config["robodojo"] + return RoboDojoClipConfig( + num_frames=int(data["num_frames"]), + resolution=int(data["resolution"]), + clips_per_episode=int(data["clips_per_episode"]), + fps=float(data["fps"]), + video_key=str(source["video_key"]), + ) + + +def selected_task_prompts(config: dict[str, Any]) -> dict[str, str]: + source = config["robodojo"] + root = Path(config["paths"]["dataset_root"]) + task_file = source.get("task_file") + if task_file is None: + return { + f"task_index_{index}": prompt + for index, prompt in sorted(read_dataset_task_prompts(root).items()) + } + mapping = source.get("task_index_by_id") + if not isinstance(mapping, dict): + raise ValueError("robodojo.task_index_by_id must be a mapping") + return resolve_training_task_prompts(root, Path(task_file), mapping) + + +def validate_config(config: dict[str, Any]) -> dict[str, Any]: + required = { + "experiment", + "paths", + "model", + "data", + "conditioning", + "diffusion", + "training", + "action", + "robodojo", + "cache", + } + missing = required - set(config) + if missing: + raise KeyError(f"Missing config sections: {sorted(missing)}") + forbidden = {"depth", "point_cloud", "camera"}.intersection(config) + if forbidden: + raise ValueError(f"Unrelated modalities are forbidden: {sorted(forbidden)}") + if "checkpoint" in config["paths"]: + raise ValueError("Fresh RoboDojo ablation must not define paths.checkpoint") + root = Path(config["paths"]["dataset_root"]) + info = read_dataset_info(root) + video_key = str(config["robodojo"]["video_key"]) + features = info["features"] + if video_key not in features or features[video_key].get("dtype") != "video": + raise ValueError(f"Dataset does not contain video feature {video_key!r}") + source_fps = float(info["fps"]) + video_fps = float(features[video_key]["info"]["video.fps"]) + configured_fps = float(config["data"]["fps"]) + if configured_fps != source_fps or configured_fps != video_fps: + raise ValueError( + f"Configured FPS {configured_fps} must equal dataset/video FPS {source_fps}/{video_fps}" + ) + if int(config["data"]["resolution"]) != 256: + raise ValueError("RoboDojo RGB must be resized to 256x256") + if int(config["data"]["num_frames"]) != 33: + raise ValueError("RoboDojo ablation uses 33 consecutive frames") + if config["conditioning"] != { + "rgb_frame0": "clean_condition", + "initial_state": "clean_action_context_token", + "exclude_rgb_frame0_from_loss": True, + }: + raise ValueError("Only clean RGB frame0 and EE state may condition the model") + action = config["action"] + if action.get("representation") != "dual_arm_se3_body_log_plus_grip": + raise ValueError("RoboDojo action must use Egoverse-style body SE(3) deltas") + if action.get("source") != "state_t_to_absolute_ee_action_t": + raise ValueError("Action source must be current EE state to commanded EE target") + if int(action.get("action_dim", -1)) != 14: + raise ValueError("Converted action must be 14D") + if action["normalization"].get("fit") != "robodojo_train_split_cache": + raise ValueError("Action normalizer must be fitted only on the train split") + per_task = config["data"]["episode_split_per_task"] + if set(per_task) != set(SPLITS): + raise ValueError(f"episode_split_per_task must contain {SPLITS}") + episodes_per_task = sum(int(per_task[name]) for name in SPLITS) + task_prompts = selected_task_prompts(config) + task_count = len(task_prompts) + expected_episodes = episodes_per_task * task_count + expected_clips = expected_episodes * int(config["data"]["clips_per_episode"]) + if int(config["data"]["num_clips"]) != expected_clips: + raise ValueError(f"data.num_clips must be {expected_clips}") + for key in ("lambda_rgb", "lambda_action"): + if float(config["diffusion"].get(key, 0)) <= 0: + raise ValueError(f"diffusion.{key} must be positive") + max_steps = int(config["training"]["max_steps"]) + max_steps_cap = int(config["training"].get("max_steps_cap", 500_000)) + if max_steps <= 0 or max_steps > max_steps_cap: + raise ValueError( + f"training.max_steps must be in [1, {max_steps_cap}], got {max_steps}" + ) + for directory in ( + Path(config["paths"]["pretrained_model_dir"]), + root, + ): + if not directory.is_dir(): + raise FileNotFoundError(directory) + task_file = config["robodojo"].get("task_file") + if task_file is not None and not Path(task_file).is_file(): + raise FileNotFoundError(task_file) + cache_dir = Path(config["paths"]["cache_dir"]).resolve() + output_dir = Path(config["experiment"]["output_dir"]).resolve() + dataset_root = root.resolve() + if cache_dir == output_dir or cache_dir.is_relative_to(output_dir) or output_dir.is_relative_to(cache_dir): + raise ValueError("cache_dir and output_dir must be separate") + if cache_dir.is_relative_to(dataset_root) or output_dir.is_relative_to(dataset_root): + raise ValueError("Cache/run outputs must not be written into the dataset") + return config + + +def resolve_splits(config: dict[str, Any]) -> dict[str, list[RoboDojoEpisode]]: + episodes = load_episodes( + Path(config["paths"]["dataset_root"]), str(config["robodojo"]["video_key"]) + ) + episodes = filter_episodes_by_task_prompts( + episodes, selected_task_prompts(config) + ) + return stratified_episode_split( + episodes, + {key: int(value) for key, value in config["data"]["episode_split_per_task"].items()}, + int(config["data"]["episode_split_seed"]), + ) + + +def cache_fingerprint( + config: dict[str, Any], splits: dict[str, list[RoboDojoEpisode]] +) -> str: + root = Path(config["paths"]["dataset_root"]) + sources: set[Path] = { + root / "meta" / "info.json", + root / "meta" / "episodes" / "chunk-000" / "file-000.parquet", + root / "meta" / "tasks.parquet", + } + task_file = config["robodojo"].get("task_file") + if task_file is not None: + sources.add(Path(task_file)) + records = [] + for split in SPLITS: + for episode in splits[split]: + sources.add(data_parquet_path(root, episode)) + sources.add(video_path(root, episode, str(config["robodojo"]["video_key"]))) + records.append( + { + "split": split, + "episode_index": episode.episode_index, + "task": episode.task, + "length": episode.length, + } + ) + source_stats = [] + for path in sorted(sources): + stat = path.stat() + source_stats.append( + {"path": str(path), "size": stat.st_size, "mtime_ns": stat.st_mtime_ns} + ) + payload = { + "format_version": 1, + "episodes": records, + "sources": source_stats, + "data": config["data"], + "robodojo": config["robodojo"], + "pretrained_model_dir": config["paths"]["pretrained_model_dir"], + "modalities": ["rgb", "action", "initial_state"], + } + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() + + +def dataset_smoke_test(config: dict[str, Any]) -> None: + splits = resolve_splits(config) + task_prompts = selected_task_prompts(config) + episode = splits["train"][0] + root = Path(config["paths"]["dataset_root"]) + clip_config = clip_config_from_config(config) + shard = load_data_shard(data_parquet_path(root, episode)) + state, absolute_action, timestamps = episode_arrays(shard, episode) + indices = select_clip_indices(episode.length, clip_config)[0] + targets = build_egoverse_delta_targets(state, absolute_action, indices) + rgb = decode_rgb_window( + video_path(root, episode, clip_config.video_key), + episode_from_timestamp=episode.video_from_timestamp, + image_indices=indices, + config=clip_config, + ) + observed_next = state[indices[1:]] + commanded_next = absolute_action[indices[:-1]] + print( + json.dumps( + { + "status": "robodojo_rgb_state_joint_action_data_ok", + "episode_index": episode.episode_index, + "task": episode.task, + "selected_task_ids": list(task_prompts), + "selected_task_count": len(task_prompts), + "split_episode_counts": { + name: len(values) for name, values in splits.items() + }, + "source_fps": clip_config.fps, + "rgb_shape": list(rgb.shape), + "action_shape": list(targets["action_raw"].shape), + "initial_state_shape": list(targets["initial_state"].shape), + "timestamp_delta_range_s": [ + float(np.diff(timestamps[indices]).min()), + float(np.diff(timestamps[indices]).max()), + ], + "absolute_action_next_state_mean_abs_error": float( + np.abs(commanded_next - observed_next).mean() + ), + "action_translation_abs_max_m": float( + targets["action_raw"][:, [0, 1, 2, 7, 8, 9]].abs().max() + ), + "action_rotation_abs_max_rad": float( + targets["action_raw"][:, [3, 4, 5, 10, 11, 12]].abs().max() + ), + "model_inputs": ["noisy_rgb_with_clean_frame0", "initial_ee_state"], + "model_outputs": ["rgb_video", "ee_delta_action"], + "excluded_modalities": ["depth", "point", "camera"], + }, + indent=2, + ), + flush=True, + ) + + +def _cache_is_complete(cache_dir: Path, manifest: dict[str, Any]) -> bool: + if not (cache_dir / "prompt_contexts.pt").is_file(): + return False + return all((cache_dir / entry["file"]).is_file() for entry in manifest["samples"]) + + +@torch.no_grad() +def prepare_cache( + config: dict[str, Any], *, device: torch.device, overwrite: bool +) -> None: + root = Path(config["paths"]["dataset_root"]) + cache_dir = Path(config["paths"]["cache_dir"]) + splits = resolve_splits(config) + task_prompts = selected_task_prompts(config) + prompt_to_task_id = { + prompt: task_id for task_id, prompt in task_prompts.items() + } + fingerprint = cache_fingerprint(config, splits) + manifest_path = cache_dir / "manifest.json" + if manifest_path.is_file() and not overwrite: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("fingerprint") != fingerprint: + raise RuntimeError("RoboDojo cache fingerprint mismatch") + if not _cache_is_complete(cache_dir, manifest): + raise RuntimeError("RoboDojo cache is incomplete") + print(f"[cache] reuse {manifest_path}", flush=True) + return + cache_dir.mkdir(parents=True, exist_ok=True) + dtype = dtype_from_name(str(config["model"]["dtype"])) + model_dir = Path(config["paths"]["pretrained_model_dir"]) + text_encoder, tokenizer = load_text_encoder_and_tokenizer( + model_dir, + dtype, + device, + int(config["cache"].get("tokenizer_max_length", 512)), + ) + prompts = sorted({episode.task for values in splits.values() for episode in values}) + prompt_entries = [] + prompt_batch_size = int(config["cache"].get("prompt_batch_size", 8)) + for start in range(0, len(prompts), prompt_batch_size): + batch_prompts = prompts[start : start + prompt_batch_size] + ids, mask = tokenizer(batch_prompts, return_mask=True, add_special_tokens=True) + contexts = text_encoder(ids.to(device), mask.to(device=device, dtype=torch.bool)) + for index, prompt in enumerate(batch_prompts): + prompt_entries.append( + { + "prompt": prompt, + "context": contexts[index].cpu(), + "context_mask": mask[index].to(dtype=torch.bool).cpu(), + } + ) + prompt_to_index = {entry["prompt"]: i for i, entry in enumerate(prompt_entries)} + prompt_tmp = cache_dir / "prompt_contexts.pt.tmp" + torch.save({"entries": prompt_entries}, prompt_tmp) + prompt_tmp.replace(cache_dir / "prompt_contexts.pt") + text_encoder.to("cpu") + del text_encoder, tokenizer + if device.type == "cuda": + torch.cuda.empty_cache() + vae = load_vae(model_dir, dtype, device) + clip_config = clip_config_from_config(config) + expected_shape = (1, 48, 9, 16, 16) + assigned = { + episode.episode_index: split + for split in SPLITS + for episode in splits[split] + } + all_episodes = sorted( + [episode for values in splits.values() for episode in values], + key=lambda episode: (episode.data_chunk_index, episode.data_file_index, episode.episode_index), + ) + entries: list[dict[str, Any]] = [] + sample_number = 0 + current_shard_key = None + shard = None + for episode_number, episode in enumerate(all_episodes): + shard_key = (episode.data_chunk_index, episode.data_file_index) + if shard_key != current_shard_key: + shard = load_data_shard(data_parquet_path(root, episode)) + current_shard_key = shard_key + assert shard is not None + state, absolute_action, timestamps = episode_arrays(shard, episode) + windows = select_clip_indices(episode.length, clip_config) + split = assigned[episode.episode_index] + print( + f"[cache] {episode_number + 1}/{len(all_episodes)} {split} " + f"episode={episode.episode_index} task={episode.task}", + flush=True, + ) + for clip_number, indices in enumerate(windows): + targets = build_egoverse_delta_targets(state, absolute_action, indices) + rgb = decode_rgb_window( + video_path(root, episode, clip_config.video_key), + episode_from_timestamp=episode.video_from_timestamp, + image_indices=indices, + config=clip_config, + ).unsqueeze(0).to(device=device, dtype=dtype) + z_rgb = vae.encode(rgb, device=device) + if tuple(z_rgb.shape) != expected_shape: + raise RuntimeError(f"Unexpected RGB latent shape {tuple(z_rgb.shape)}") + sample_id = f"episode_{episode.episode_index:06d}_clip_{clip_number:02d}" + filename = f"{sample_number:06d}_{sample_id}.pt" + temporary = cache_dir / f"{filename}.tmp" + torch.save( + { + "sample_id": sample_id, + "z_rgb": z_rgb[0].cpu(), + "action_raw": targets["action_raw"], + "action_is_pad": targets["action_is_pad"], + "initial_state": targets["initial_state"], + "image_indices": torch.from_numpy(indices.copy()), + "timestamps": torch.from_numpy(timestamps[indices].copy()), + "effective_fps": torch.tensor(clip_config.fps, dtype=torch.float64), + }, + temporary, + ) + temporary.replace(cache_dir / filename) + entries.append( + { + "sample_id": sample_id, + "file": filename, + "split": split, + "episode_index": episode.episode_index, + "clip_index": clip_number, + "prompt_index": prompt_to_index[episode.task], + "task_id": prompt_to_task_id[episode.task], + } + ) + sample_number += 1 + manifest = { + "format_version": 2, + "fingerprint": fingerprint, + "source": "robodojo_ee_lerobot_v3.0", + "fps": clip_config.fps, + "modalities": ["rgb", "action", "initial_state"], + "explicitly_excluded_modalities": ["depth", "point", "camera"], + "action_representation": "dual_arm_se3_body_log_plus_grip", + "selected_tasks": task_prompts, + "samples": entries, + } + manifest_tmp = cache_dir / "manifest.json.tmp" + manifest_tmp.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + manifest_tmp.replace(manifest_path) + vae.to("cpu") + del vae + + +class RoboDojoCachedDataset(Dataset): + def __init__( + self, + cache_dir: Path, + expected_fingerprint: str, + split: str, + *, + action_mean: torch.Tensor | None = None, + action_std: torch.Tensor | None = None, + action_clip: float | None = None, + ) -> None: + manifest = json.loads((cache_dir / "manifest.json").read_text(encoding="utf-8")) + if manifest.get("fingerprint") != expected_fingerprint: + raise RuntimeError("RoboDojo cache does not match config") + if manifest.get("modalities") != ["rgb", "action", "initial_state"]: + raise RuntimeError("RoboDojo cache contains unexpected modalities") + self.cache_dir = cache_dir + self.entries = [entry for entry in manifest["samples"] if entry["split"] == split] + if not self.entries: + raise RuntimeError(f"Cache split {split!r} is empty") + self.prompts = torch.load( + cache_dir / "prompt_contexts.pt", map_location="cpu", weights_only=True + )["entries"] + self.action_mean = action_mean + self.action_std = action_std + self.action_clip = action_clip + + def __len__(self) -> int: + return len(self.entries) + + def raw_payload(self, index: int) -> dict[str, Any]: + return torch.load( + self.cache_dir / self.entries[index]["file"], + map_location="cpu", + weights_only=True, + ) + + def __getitem__(self, index: int) -> dict[str, Any]: + payload = self.raw_payload(index) + prompt = self.prompts[int(self.entries[index]["prompt_index"])] + action = payload["action_raw"] + if self.action_mean is not None: + action = (action - self.action_mean) / self.action_std + if self.action_clip is not None: + action = action.clamp(-self.action_clip, self.action_clip) + action = action.masked_fill(payload["action_is_pad"].unsqueeze(-1), 0) + return { + "sample_id": payload["sample_id"], + "z_rgb": payload["z_rgb"], + "action": action, + "action_is_pad": payload["action_is_pad"], + "initial_state": payload["initial_state"], + "context": prompt["context"], + "context_mask": prompt["context_mask"].to(dtype=torch.bool), + "prompt": prompt["prompt"], + } + + +def _validate_model_only_resume(config: dict[str, Any]) -> dict[str, Any] | None: + resume = config.get("resume") + if resume is None: + return None + if not isinstance(resume, dict): + raise TypeError("resume must be a mapping") + expected = { + "mode": "model_only", + "optimizer_state": "reset", + "scheduler_state": "reconstruct_from_global_step", + "dataloader_position": "reconstruct_from_global_step", + } + actual = {key: resume.get(key) for key in expected} + if actual != expected: + raise ValueError(f"Model-only resume contract mismatch: {actual} != {expected}") + checkpoint = Path(str(resume.get("checkpoint", ""))) + if not checkpoint.is_file(): + raise FileNotFoundError(checkpoint) + global_step = int(resume.get("global_step", -1)) + if not 0 < global_step < int(config["training"]["max_steps"]): + raise ValueError( + "resume.global_step must be between zero and training.max_steps, " + f"got {global_step}" + ) + expected_name = f"robodojo_rgb_state_joint_action_{global_step:06d}.pt" + if checkpoint.name != expected_name: + raise ValueError( + f"Resume checkpoint name {checkpoint.name!r} does not encode step {global_step}" + ) + return resume + + +@torch.no_grad() +def _load_model_only_resume( + model: torch.nn.Module, + resume: dict[str, Any], +) -> dict[str, Any]: + checkpoint = Path(str(resume["checkpoint"])) + payload = torch.load(checkpoint, map_location="cpu", weights_only=True, mmap=True) + if not isinstance(payload, dict) or set(payload) != {"metadata", "state_dict"}: + raise ValueError(f"Invalid model-only resume payload: {checkpoint}") + metadata = payload["metadata"] + source = payload["state_dict"] + target = model.state_dict(keep_vars=True) + source_keys, target_keys = set(source), set(target) + if source_keys != target_keys: + raise RuntimeError( + "Model-only resume key mismatch: " + f"missing={sorted(target_keys - source_keys)} " + f"unexpected={sorted(source_keys - target_keys)}" + ) + expected_source = resume.get("expected_initialization_checkpoint") + initialization = metadata.get("initialization", {}) if isinstance(metadata, dict) else {} + actual_source = initialization.get("source_checkpoint") + if expected_source is not None and str(actual_source) != str(expected_source): + raise RuntimeError( + "Resume checkpoint initialization lineage mismatch: " + f"actual={actual_source!r} expected={expected_source!r}" + ) + for key in sorted(target_keys): + source_tensor, target_tensor = source[key], target[key] + if tuple(source_tensor.shape) != tuple(target_tensor.shape): + raise RuntimeError( + f"Model-only resume shape mismatch for {key}: " + f"source={tuple(source_tensor.shape)} target={tuple(target_tensor.shape)}" + ) + target_tensor.copy_(source_tensor.to(dtype=target_tensor.dtype)) + audit = { + "mode": "model_only", + "checkpoint": str(checkpoint), + "global_step": int(resume["global_step"]), + "loaded_key_count": len(target_keys), + "optimizer_state": "reset", + "scheduler_state": "reconstruct_from_global_step", + "initialization_source_checkpoint": actual_source, + } + print(json.dumps({"resume_model_load": audit}, sort_keys=True), flush=True) + del source, payload + return audit + + +def train( + config: dict[str, Any], output_dir: Path, accelerator: Accelerator | None = None +) -> None: + resume = _validate_model_only_resume(config) + if accelerator is None: + accelerator = Accelerator( + gradient_accumulation_steps=int(config["training"]["gradient_accumulation_steps"]), + mixed_precision="bf16" if config["model"]["dtype"] == "bfloat16" else "no", + kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(hours=2))], + ) + set_seed(int(config["experiment"]["seed"]), device_specific=True) + if accelerator.is_main_process: + if output_dir.exists() and any(output_dir.iterdir()): + raise RuntimeError(f"Refusing to reuse non-empty output directory: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "resolved_config.yaml").write_text( + yaml.safe_dump(config, sort_keys=False), encoding="utf-8" + ) + accelerator.wait_for_everyone() + wandb_run = None + tracking = config.get("tracking", {}) + if accelerator.is_main_process and bool(tracking.get("enabled", False)): + if tracking.get("backend") != "wandb": + raise ValueError("Only the wandb tracking backend is supported") + import wandb + + wandb_run = wandb.init( + project=str(tracking["project"]), + entity=tracking.get("entity") or None, + id=tracking.get("run_id") or None, + name=str(tracking.get("run_name", config["experiment"]["name"])), + group=tracking.get("group") or None, + tags=list(tracking.get("tags", [])), + mode=str(tracking.get("mode", "online")), + dir=str(output_dir), + config=config, + resume=tracking.get("resume", False), + ) + tracking_audit = { + "backend": "wandb", + "mode": str(tracking.get("mode", "online")), + "run_id": str(wandb_run.id), + "run_url": str(wandb_run.url), + "run_name": str(wandb_run.name), + } + (output_dir / "wandb_run.json").write_text( + json.dumps(tracking_audit, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(json.dumps({"wandb_run": tracking_audit}, sort_keys=True), flush=True) + model = build_fresh_rgb_action_mot(config) + if bool(config["training"].get("fp32_master_weights", True)): + enable_fp32_master_weights(model) + splits = resolve_splits(config) + fingerprint = cache_fingerprint(config, splits) + raw_train = RoboDojoCachedDataset( + Path(config["paths"]["cache_dir"]), fingerprint, "train" + ) + fit_action_normalizer(model, raw_train) + resume_audit = None + if resume is not None and accelerator.is_main_process: + resume_audit = _load_model_only_resume(model, resume) + (output_dir / "resume_model_only_audit.json").write_text( + json.dumps(resume_audit, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + accelerator.wait_for_everyone() + train_dataset = RoboDojoCachedDataset( + Path(config["paths"]["cache_dir"]), + fingerprint, + "train", + action_mean=model.action_normalizer.mean.detach().cpu(), + action_std=model.action_normalizer.std.detach().cpu(), + action_clip=model.action_normalizer.clip, + ) + sampler = FixedSeedPermutationSampler( + train_dataset, int(config["data"]["sample_order_seed"]) + ) + dataloader = DataLoader( + train_dataset, + batch_size=int(config["data"]["batch_size"]), + sampler=sampler, + num_workers=int(config["data"].get("num_workers", 0)), + pin_memory=True, + drop_last=False, + collate_fn=collate_rgb_action, + ) + configure_trainable(model) + optimizer = build_optimizer(model, config) + max_steps = int(config["training"]["max_steps"]) + lr_scheduler = build_lr_scheduler( + optimizer, + max_steps, + int(config["training"]["warmup_steps"]), + schedule_type=str(config["training"]["lr_schedule"]), + ) + resume_global_step = 0 if resume is None else int(resume["global_step"]) + position_lr_scheduler_at_optimizer_step( + lr_scheduler, global_step=resume_global_step + ) + optimizer_updates_per_epoch = math.ceil( + int(config["training"]["valid_windows_per_epoch"]) + / int(config["training"]["global_batch_size"]) + ) + resume_epoch, resume_updates_in_epoch = divmod( + resume_global_step, optimizer_updates_per_epoch + ) + if hasattr(sampler, "epoch"): + sampler.epoch = resume_epoch + elif resume is not None: + raise RuntimeError("Model-only resume requires an epoch-addressable sampler") + model, optimizer, dataloader, lr_scheduler = accelerator.prepare( + model, optimizer, dataloader, lr_scheduler + ) + accumulation_steps = int(config["training"]["gradient_accumulation_steps"]) + micro_steps_per_epoch = len(dataloader) + if math.ceil(micro_steps_per_epoch / accumulation_steps) != optimizer_updates_per_epoch: + raise RuntimeError( + "Resume epoch geometry mismatch: " + f"micro_steps_per_epoch={micro_steps_per_epoch} " + f"accumulation_steps={accumulation_steps} " + f"optimizer_updates_per_epoch={optimizer_updates_per_epoch}" + ) + resume_skip_micro_steps = resume_updates_in_epoch * accumulation_steps + if resume_skip_micro_steps > micro_steps_per_epoch: + raise RuntimeError( + "Resume micro-step offset is outside the reconstructed epoch: " + f"skip={resume_skip_micro_steps} epoch_length={micro_steps_per_epoch}" + ) + video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=float(config["diffusion"]["train_shift"]), + ) + action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["action_num_train_timesteps"]), + shift=float(config["diffusion"]["action_train_shift"]), + ) + dtype = dtype_from_name(str(config["model"]["dtype"])) + global_step = resume_global_step + micro_step = resume_epoch * micro_steps_per_epoch + resume_skip_micro_steps + last_exported_step = None + optimizer.zero_grad(set_to_none=True) + model.train() + assert_lr_scheduler_optimizer_step_index( + lr_scheduler, global_step=global_step, config=config + ) + if config["training"].get("scheduler_step_unit") == "optimizer_step": + initial_audit = lr_scheduler_audit_record( + lr_scheduler, + optimizer, + micro_step=micro_step, + global_step=global_step, + ) + if accelerator.is_main_process: + append_jsonl(output_dir / "scheduler_audit.jsonl", initial_audit) + print( + json.dumps({"scheduler_audit": initial_audit}, sort_keys=True), + flush=True, + ) + if resume is not None and accelerator.is_main_process: + resume_position = { + "global_step": global_step, + "micro_step": micro_step, + "epoch": resume_epoch, + "optimizer_updates_in_epoch": resume_updates_in_epoch, + "micro_steps_to_skip_in_epoch": resume_skip_micro_steps, + "micro_steps_per_epoch": micro_steps_per_epoch, + "optimizer_updates_per_epoch": optimizer_updates_per_epoch, + } + (output_dir / "resume_position_audit.json").write_text( + json.dumps(resume_position, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(json.dumps({"resume_position": resume_position}, sort_keys=True), flush=True) + pending_skip_micro_steps = resume_skip_micro_steps + training_started_at = time.monotonic() + last_gradient_norm = float("nan") + while global_step < max_steps: + epoch_dataloader = dataloader + if pending_skip_micro_steps: + epoch_dataloader = skip_first_batches( + dataloader, num_batches=pending_skip_micro_steps + ) + pending_skip_micro_steps = 0 + for batch in epoch_dataloader: + if global_step >= max_steps: + break + micro_step += 1 + with accelerator.accumulate(model): + loss, metrics = compute_loss( + model, + batch, + video_scheduler, + action_scheduler, + config, + accelerator.device, + dtype, + ) + if not bool(torch.isfinite(loss.detach()).all().item()): + raise FloatingPointError( + f"Non-finite loss at micro_step={micro_step} " + f"global_step={global_step}" + ) + accelerator.backward(loss) + if accelerator.sync_gradients: + gradient_norm = accelerator.clip_grad_norm_( + [parameter for parameter in model.parameters() if parameter.requires_grad], + float(config["training"]["max_grad_norm"]), + ) + last_gradient_norm = float( + gradient_norm.detach().float().item() + if torch.is_tensor(gradient_norm) + else gradient_norm + ) + if not math.isfinite(last_gradient_norm): + raise FloatingPointError( + f"Non-finite gradient norm at micro_step={micro_step}" + ) + learning_rates_used_for_update = [ + float(group["lr"]) for group in optimizer.param_groups + ] + else: + learning_rates_used_for_update = None + optimizer.step() + optimizer_update_completed = step_lr_scheduler_on_optimizer_update( + lr_scheduler, + accelerator, + optimizer=optimizer, + expected_global_step=( + global_step + if config["training"].get("scheduler_step_unit") + == "optimizer_step" + else None + ), + ) + optimizer.zero_grad(set_to_none=True) + if not optimizer_update_completed: + continue + if learning_rates_used_for_update is None: + raise RuntimeError( + "Optimizer update completed without a captured pre-update LR" + ) + global_step += 1 + assert_lr_scheduler_optimizer_step_index( + lr_scheduler, global_step=global_step, config=config + ) + if config["training"].get("scheduler_step_unit") == "optimizer_step": + scheduler_audit = lr_scheduler_audit_record( + lr_scheduler, + optimizer, + micro_step=micro_step, + global_step=global_step, + learning_rates_used_for_update=learning_rates_used_for_update, + ) + if accelerator.is_main_process: + append_jsonl(output_dir / "scheduler_audit.jsonl", scheduler_audit) + print( + json.dumps({"scheduler_audit": scheduler_audit}, sort_keys=True), + flush=True, + ) + audit_milestones = {1, 10, 100, 250, 500, 999, 1000, 1001} + if ( + global_step in audit_milestones + or global_step % int(config["training"]["log_every"]) == 0 + ): + reduced = { + key: accelerator.reduce(value.float(), reduction="mean") + for key, value in metrics.items() + if key.startswith("loss_") + } + elapsed_seconds = max(time.monotonic() - training_started_at, 1.0e-9) + memory_local = torch.tensor( + [ + torch.cuda.memory_allocated(accelerator.device), + torch.cuda.memory_reserved(accelerator.device), + torch.cuda.max_memory_allocated(accelerator.device), + ], + device=accelerator.device, + dtype=torch.float64, + ) + memory_max = accelerator.gather(memory_local).reshape(-1, 3).max(dim=0).values + record = { + "global_step": global_step, + "optimizer_global_step": global_step, + "micro_step": micro_step, + "scheduler_update_index": int( + getattr(lr_scheduler, "scheduler", lr_scheduler).last_epoch + ), + "loss_total": float( + accelerator.reduce(loss.detach().float(), reduction="mean").item() + ), + **{key: float(value.item()) for key, value in reduced.items()}, + "grad_norm": last_gradient_norm, + "optimizer_steps_per_second": float( + (global_step - resume_global_step) / elapsed_seconds + ), + "samples_per_second": float( + (global_step - resume_global_step) + * int(config["training"]["global_batch_size"]) + / elapsed_seconds + ), + "gpu_memory_allocated_gib": float(memory_max[0].item() / 2**30), + "gpu_memory_reserved_gib": float(memory_max[1].item() / 2**30), + "gpu_max_memory_allocated_gib": float(memory_max[2].item() / 2**30), + "learning_rates_used_for_update": learning_rates_used_for_update, + "learning_rates_after_scheduler_step": [ + float(group["lr"]) for group in optimizer.param_groups + ], + } + if accelerator.is_main_process: + record.update(query_nvidia_smi_metrics()) + append_jsonl(output_dir / "metrics.jsonl", record) + print(json.dumps(record, sort_keys=True), flush=True) + wandb_log_after_step = int(tracking.get("log_after_step", -1)) + if wandb_run is not None and global_step > wandb_log_after_step: + wandb_run.log(record, step=global_step) + if global_step % int(config["training"]["save_every"]) == 0: + export_model( + model, + accelerator, + output_dir / "checkpoints" / f"robodojo_rgb_state_joint_action_{global_step:06d}.pt", + ) + last_exported_step = global_step + if last_exported_step != global_step: + export_model( + model, + accelerator, + output_dir / "checkpoints" / f"robodojo_rgb_state_joint_action_{global_step:06d}.pt", + ) + if wandb_run is not None: + wandb_run.finish() + accelerator.end_training() + + +def main() -> None: + args = parse_args() + config = load_config(args.config) + if args.output_dir is not None: + config["experiment"]["output_dir"] = str(args.output_dir) + if args.max_steps is not None: + config["training"]["max_steps"] = args.max_steps + config = validate_config(config) + if args.dataset_smoke_test: + dataset_smoke_test(config) + return + accelerator = Accelerator( + gradient_accumulation_steps=int(config["training"]["gradient_accumulation_steps"]), + mixed_precision="bf16" if config["model"]["dtype"] == "bfloat16" else "no", + kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(hours=2))], + ) + if accelerator.is_main_process: + prepare_cache( + config, + device=torch.device(str(config["cache"].get("device", accelerator.device))), + overwrite=args.overwrite_cache, + ) + accelerator.wait_for_everyone() + if args.prepare_cache_only: + accelerator.end_training() + return + train(config, Path(config["experiment"]["output_dir"]), accelerator) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_4way.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_4way.py new file mode 100644 index 0000000000000000000000000000000000000000..fc78320d4e191717cc31be4837917c487a00ca2c --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_4way.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Eight/four-way RoboDojo training with absolute base-frame EE targets. + +This isolated adapter keeps the exhaustive-window and model implementation in +``train_robodojo_rgb_state_joint_action_4way`` unchanged. Each action is the +commanded absolute end-effector pose in the robot base frame represented as +literal xyz (3), SO(3) log rotation (3), and gripper (1), independently for the +two arms (14 total). No SE(3) translation twist is used. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +import train_robodojo_rgb_state_joint_action_4way as fourway +from online_egoverse_action import ACTION_DIM, so3_log +from robodojo_lerobot_v30 import _pose16_matrix + + +ACTION_REPRESENTATION = ( + "dual_arm_absolute_translation_so3_log_plus_grip" +) +ACTION_SOURCE = "absolute_ee_action_t" + +_fourway_load_config = fourway.load_config +_fourway_validate_base_config = fourway._original_validate_config +_fourway_cache_fingerprint = fourway.episode_length_normalized_cache_fingerprint +_fourway_checkpoint_metadata = fourway.checkpoint_metadata + + +def absolute_xyz_so3_vector(transform: np.ndarray) -> np.ndarray: + """Encode an absolute pose as literal xyz followed by SO(3) log rotation.""" + + transform = np.asarray(transform, dtype=np.float64) + if transform.shape != (4, 4): + raise ValueError(f"Expected 4x4 transform, got {transform.shape}") + return np.concatenate( + [transform[:3, 3], so3_log(transform[:3, :3])] + ) + + +def build_absolute_ee_targets( + state: np.ndarray, + absolute_action: np.ndarray, + image_indices: np.ndarray, +) -> dict[str, torch.Tensor]: + """Return 32 absolute base-frame EE commands and the current absolute EE state.""" + + indices = np.asarray(image_indices, dtype=np.int64) + steps = np.diff(indices) + if len(indices) < 2 or np.any((steps != 0) & (steps != 1)): + raise ValueError( + "RoboDojo clips must use consecutive frames with optional terminal repeats" + ) + if int(indices[0]) < 0 or int(indices[-1]) >= len(state): + raise IndexError("RoboDojo clip indices are outside the episode") + if state.shape != absolute_action.shape or state.ndim != 2 or state.shape[1] != 16: + raise ValueError("RoboDojo state/action must have matching [T,16] shapes") + repeated = steps == 0 + if np.any(repeated) and np.any(indices[:-1][repeated] != len(state) - 1): + raise ValueError("Only the terminal RoboDojo frame may be repeated") + + commanded = absolute_action[indices[:-1]] + left_commanded = _pose16_matrix(commanded, "left") + right_commanded = _pose16_matrix(commanded, "right") + action = np.zeros((len(indices) - 1, ACTION_DIM), dtype=np.float32) + for row in range(len(indices) - 1): + action[row, 0:6] = absolute_xyz_so3_vector(left_commanded[row]) + action[row, 7:13] = absolute_xyz_so3_vector(right_commanded[row]) + action[:, 6] = commanded[:, 7] + action[:, 13] = commanded[:, 15] + + current = state[int(indices[0]) : int(indices[0]) + 1] + left_current = _pose16_matrix(current, "left")[0] + right_current = _pose16_matrix(current, "right")[0] + initial_state = np.concatenate( + [ + absolute_xyz_so3_vector(left_current), + [current[0, 7]], + absolute_xyz_so3_vector(right_current), + [current[0, 15]], + ] + ).astype(np.float32) + + if not np.isfinite(action).all() or not np.isfinite(initial_state).all(): + raise RuntimeError("Absolute-EE RoboDojo targets contain non-finite values") + return { + "action_raw": torch.from_numpy(action), + "action_is_pad": torch.zeros(len(indices) - 1, dtype=torch.bool), + "initial_state": torch.from_numpy(initial_state), + "left_initial_pose": torch.from_numpy(left_current.astype(np.float32)), + "right_initial_pose": torch.from_numpy(right_current.astype(np.float32)), + } + + +def load_absolute_ee_config(path: Path) -> dict[str, Any]: + config = _fourway_load_config(path) + action = config.pop("absolute_ee_action", None) + if not isinstance(action, dict): + raise ValueError("Config must define a complete absolute_ee_action mapping") + config["action"] = action + return config + + +def validate_absolute_ee_with_legacy_base(config: dict[str, Any]) -> dict[str, Any]: + """Reuse legacy structural checks while enforcing absolute-target semantics.""" + + action = config["action"] + required = { + "representation": ACTION_REPRESENTATION, + "source": ACTION_SOURCE, + "target_formula": "base_frame_commanded_target_ee", + "integration": "none", + "coordinate_frame": "robodojo_robot_base", + "translation_encoding": "absolute_xyz", + "rotation_encoding": "absolute_so3_log", + } + actual = {key: action.get(key) for key in required} + if actual != required: + raise ValueError(f"Absolute EE action config mismatch: {actual} != {required}") + if int(action.get("action_dim", -1)) != ACTION_DIM: + raise ValueError(f"Absolute EE action_dim must be {ACTION_DIM}") + initial_state = action.get("initial_state", {}) + if initial_state.get("representation") != ACTION_REPRESENTATION: + raise ValueError("Current state must use absolute base-frame EE coordinates") + if initial_state.get("source") != "sample_first_state": + raise ValueError("State conditioning must use the current sample-first state") + if initial_state.get("translation_encoding") != "absolute_xyz": + raise ValueError("Current-state translation must use literal absolute xyz") + if initial_state.get("rotation_encoding") != "absolute_so3_log": + raise ValueError("Current-state rotation must use absolute SO(3) log") + if initial_state.get("formula") != "base_frame_current_ee": + raise ValueError("State conditioning formula is inconsistent with absolute actions") + + absolute_action = copy.deepcopy(action) + legacy_action = copy.deepcopy(action) + legacy_action["representation"] = "dual_arm_se3_body_log_plus_grip" + legacy_action["source"] = "state_t_to_absolute_ee_action_t" + config["action"] = legacy_action + try: + validated = _fourway_validate_base_config(config) + finally: + config["action"] = absolute_action + validated["action"] = absolute_action + return validated + + +def absolute_ee_cache_fingerprint( + config: dict[str, Any], splits: dict[str, list[Any]] +) -> str: + source = _fourway_cache_fingerprint(config, splits) + payload = { + "source_fingerprint": source, + "action": config["action"], + "format": "absolute_base_frame_xyz_so3_action_v2", + } + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() + + +def absolute_ee_checkpoint_metadata(model: torch.nn.Module) -> dict[str, Any]: + metadata = _fourway_checkpoint_metadata(model) + metadata["action_semantics"] = { + "representation": ACTION_REPRESENTATION, + "source": ACTION_SOURCE, + "target_formula": "base_frame_commanded_target_ee", + "state_formula": "base_frame_current_ee", + "coordinate_frame": "robodojo_robot_base", + "integration": "none", + "gripper": "absolute_command", + "translation_encoding": "absolute_xyz", + "rotation_encoding": "absolute_so3_log", + } + return metadata + + +def main() -> None: + fourway.load_config = load_absolute_ee_config + fourway._original_validate_config = validate_absolute_ee_with_legacy_base + fourway.episode_length_normalized_cache_fingerprint = absolute_ee_cache_fingerprint + fourway.checkpoint_metadata = absolute_ee_checkpoint_metadata + fourway.base.build_egoverse_delta_targets = build_absolute_ee_targets + fourway.main() + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way.py new file mode 100644 index 0000000000000000000000000000000000000000..0d724a67754382210b31001f90c0332ccdd8589d --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Absolute-EE RoboDojo training with pi0.5-style q01/q99 scaling. + +All fourteen action dimensions, including the two absolute binary grippers, +use ``(x-q01)/(q99-q01+1e-6)*2-1`` without clipping, matching openpi's pi0.5 +``Normalize(use_quantiles=True)`` transform. Binary grippers therefore map +from 0/1 to approximately -1/+1 rather than using data-dependent mean/std. +The affine center and half-range are stored in the existing checkpoint- +compatible action-normalizer buffers, so training and inference use one source +of truth without changing the model architecture. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +import train_robodojo_rgb_state_joint_action_4way as fourway +import train_robodojo_rgb_state_joint_action_absolute_ee_4way as absolute + + +NORMALIZATION_TYPE = "pi05_q01_q99_all_action_v1" +ACTION_DIMS = tuple(range(14)) +LOW_QUANTILE = 0.01 +HIGH_QUANTILE = 0.99 +QUANTILE_EPSILON = 1.0e-6 + +_original_absolute_validation = absolute.validate_absolute_ee_with_legacy_base +_original_prepare_cache = fourway.prepare_episode_length_normalized_cache +_original_checkpoint_metadata = absolute.absolute_ee_checkpoint_metadata + + +def _normalization_contract(config: dict[str, Any]) -> dict[str, Any]: + normalization = config["action"]["normalization"] + expected = { + "type": NORMALIZATION_TYPE, + "fit": "robodojo_train_split_cache", + "quantiles": [LOW_QUANTILE, HIGH_QUANTILE], + "epsilon": QUANTILE_EPSILON, + "normalized_range": [-1.0, 1.0], + "normalized_dims": list(ACTION_DIMS), + "gripper_dims": [6, 13], + "gripper_semantics": "absolute_binary_0_1_normalized_to_minus1_plus1", + "clip": None, + "checkpoint_buffers": True, + } + actual = {key: normalization.get(key) for key in expected} + if actual != expected: + raise ValueError( + f"Quantile action-normalization mismatch: {actual} != {expected}" + ) + if ACTION_DIMS != tuple(range(14)): + raise RuntimeError("Pi0.5 quantile normalization must cover all 14 dimensions") + return expected + + +def validate_quantile_absolute_ee(config: dict[str, Any]) -> dict[str, Any]: + _normalization_contract(config) + return _original_absolute_validation(config) + + +def weighted_linear_quantile( + values: np.ndarray, integer_weights: np.ndarray, quantile: float +) -> float: + """Match ``np.quantile(np.repeat(values, weights), q)`` without expansion.""" + + values = np.asarray(values, dtype=np.float64) + weights = np.asarray(integer_weights, dtype=np.int64) + if values.ndim != 1 or weights.ndim != 1 or values.shape != weights.shape: + raise ValueError("Weighted quantile expects matching one-dimensional arrays") + if not 0.0 <= float(quantile) <= 1.0: + raise ValueError("Quantile must be in [0, 1]") + if not np.isfinite(values).all() or np.any(weights < 0): + raise ValueError("Weighted quantile inputs must be finite with nonnegative weights") + positive = weights > 0 + if not np.any(positive): + raise ValueError("Weighted quantile requires positive total weight") + values = values[positive] + weights = weights[positive] + order = np.argsort(values, kind="stable") + sorted_values = values[order] + cumulative = np.cumsum(weights[order], dtype=np.int64) + total = int(cumulative[-1]) + position = (total - 1) * float(quantile) + lower_rank = int(np.floor(position)) + upper_rank = int(np.ceil(position)) + + def value_at(rank: int) -> float: + index = int(np.searchsorted(cumulative, rank, side="right")) + return float(sorted_values[index]) + + lower = value_at(lower_rank) + upper = value_at(upper_rank) + return lower + (position - lower_rank) * (upper - lower) + + +def exhaustive_action_quantile_affine_statistics( + config: dict[str, Any], episodes: list[Any] +) -> tuple[torch.Tensor, torch.Tensor, int]: + """Fit exact q01/q99 affine statistics to the exhaustive window sampler.""" + + _normalization_contract(config) + window = int(config["sampling"]["window_size"]) + horizon = window - 1 + value_parts: list[np.ndarray] = [] + weight_parts: list[np.ndarray] = [] + expected_count = 0 + for number, episode in enumerate(episodes, start=1): + state, absolute_action, _ = fourway._load_episode_arrays(config, episode) + length = int(episode.length) + starts = fourway.episode_sample_count(config, length) + indices = np.arange(length, dtype=np.int64) + if fourway._uses_fastwam_tail_padding(config): + indices = np.concatenate([indices, indices[-1:]]) + action = fourway.base.build_egoverse_delta_targets( + state, absolute_action, indices + )["action_raw"].numpy() + if fourway._uses_fastwam_tail_padding(config): + counts = fourway.valid_action_occurrence_counts(length, horizon) + if action.shape[0] != length: + raise RuntimeError("Tail-padded quantile action length mismatch") + else: + transition = np.arange(action.shape[0], dtype=np.int64) + low = np.maximum(transition - (horizon - 1), 0) + high = np.minimum(transition, starts - 1) + counts = np.maximum(high - low + 1, 0).astype(np.int64) + value_parts.append(np.asarray(action, dtype=np.float32)) + weight_parts.append(counts) + expected_count += int(counts.sum()) + if number == 1 or number % 100 == 0 or number == len(episodes): + print( + f"[quantile-stats] {number}/{len(episodes)} " + f"episode={episode.episode_index} starts={starts} " + f"valid_action_tokens={int(counts.sum())}", + flush=True, + ) + + values = np.concatenate(value_parts, axis=0) + weights = np.concatenate(weight_parts, axis=0) + weighted_count = int(weights.sum()) + if weighted_count != expected_count: + raise RuntimeError( + f"Exhaustive-window weighted count {weighted_count} != {expected_count}" + ) + + center = np.empty(14, dtype=np.float32) + scale = np.empty(14, dtype=np.float32) + q01 = np.empty(14, dtype=np.float64) + q99 = np.empty(14, dtype=np.float64) + for dimension in ACTION_DIMS: + q01[dimension] = weighted_linear_quantile( + values[:, dimension], weights, LOW_QUANTILE + ) + q99[dimension] = weighted_linear_quantile( + values[:, dimension], weights, HIGH_QUANTILE + ) + width = q99[dimension] - q01[dimension] + if not np.isfinite(width) or width <= 1.0e-8: + raise RuntimeError(f"Degenerate q01/q99 interval for dimension {dimension}") + # Exact affine form of openpi's quantile transform: + # (x - q01) / (q99 - q01 + 1e-6) * 2 - 1. + scale[dimension] = np.float32( + 0.5 * (width + QUANTILE_EPSILON) + ) + center[dimension] = np.float32(q01[dimension] + scale[dimension]) + print( + f"[quantile-stats] dim={dimension:02d} " + f"q01={q01[dimension]:.9f} q99={q99[dimension]:.9f}", + flush=True, + ) + + return torch.from_numpy(center), torch.from_numpy(scale), weighted_count + + +@torch.no_grad() +def prepare_quantile_cache( + config: dict[str, Any], *, device: torch.device, overwrite: bool +) -> None: + _normalization_contract(config) + _original_prepare_cache(config, device=device, overwrite=overwrite) + cache_dir = Path(config["paths"]["cache_dir"]) + stats_path = cache_dir / "action_normalizer.pt" + manifest = json.loads( + (cache_dir / "manifest.json").read_text(encoding="utf-8") + ) + payload = torch.load(stats_path, map_location="cpu", weights_only=True) + if payload.get("fingerprint") != manifest.get("fingerprint"): + raise RuntimeError("Quantile normalizer fingerprint mismatch") + center = payload["mean"].float() + scale = payload["std"].float() + if tuple(center.shape) != (14,) or tuple(scale.shape) != (14,): + raise RuntimeError("Quantile affine buffers must be 14D") + q01 = center - scale + q99 = center + scale - QUANTILE_EPSILON + contract = _normalization_contract(config) + expected_metadata = { + "normalization_type": NORMALIZATION_TYPE, + "quantiles": torch.tensor([LOW_QUANTILE, HIGH_QUANTILE]), + "epsilon": torch.tensor(QUANTILE_EPSILON), + "normalized_range": torch.tensor([-1.0, 1.0]), + "normalized_dims": torch.tensor(ACTION_DIMS, dtype=torch.long), + "gripper_dims": torch.tensor([6, 13], dtype=torch.long), + "gripper_semantics": contract["gripper_semantics"], + "q01": q01, + "q99": q99, + } + needs_write = any( + key not in payload + or ( + torch.is_tensor(value) + and not torch.equal(torch.as_tensor(payload[key]), value) + ) + or (not torch.is_tensor(value) and payload[key] != value) + for key, value in expected_metadata.items() + ) + if needs_write: + payload.update(expected_metadata) + temporary = stats_path.with_name( + f"{stats_path.name}.quantile.{os.getpid()}.tmp" + ) + torch.save(payload, temporary) + os.replace(temporary, stats_path) + print( + f"[quantile-normalizer] validated {stats_path} " + f"weighted_count={int(payload['count'])}", + flush=True, + ) + + +def quantile_checkpoint_metadata(model: torch.nn.Module) -> dict[str, Any]: + metadata = _original_checkpoint_metadata(model) + metadata["action_normalization"] = { + "type": NORMALIZATION_TYPE, + "quantiles": [LOW_QUANTILE, HIGH_QUANTILE], + "epsilon": QUANTILE_EPSILON, + "normalized_range": [-1.0, 1.0], + "normalized_dims": list(ACTION_DIMS), + "gripper_dims": [6, 13], + "gripper_semantics": "absolute_binary_0_1_normalized_to_minus1_plus1", + "implicit_clip": False, + "statistics_storage": { + "center": "action_normalizer.mean", + "half_range": "action_normalizer.std", + }, + } + return metadata + + +def main() -> None: + absolute.validate_absolute_ee_with_legacy_base = validate_quantile_absolute_ee + absolute.absolute_ee_checkpoint_metadata = quantile_checkpoint_metadata + fourway._exhaustive_action_statistics = ( + exhaustive_action_quantile_affine_statistics + ) + fourway.prepare_episode_length_normalized_cache = prepare_quantile_cache + absolute.main() + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_quantile_shared_timestep_4way.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_quantile_shared_timestep_4way.py new file mode 100644 index 0000000000000000000000000000000000000000..1ba55200d92ad209a65b69291ee758c429f8166a --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_quantile_shared_timestep_4way.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Isolated RoboDojo absolute-EE q01/q99 joint training with one timestep. + +The existing quantile absolute-EE stack is reused at runtime. This entrypoint +only replaces its training loss with the shared-video/action-timestep sibling +and adds strict configuration/checkpoint audit metadata. Model architecture, +bidirectional attention, video/action outputs, stop-gradient, cache generation, +normalization, and inference code are unchanged. +""" + +from __future__ import annotations + +import math +from typing import Any + +import torch + +import joint_rgb_action_diffusion_shared_timestep as shared_timestep +import train_galexea_rgb_state_joint_action_ablation as galexea +import train_robodojo_rgb_state_joint_action_4way as fourway +import train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way as quantile + + +TIMESTEP_COUPLING = "shared_video_action" + +_original_quantile_validation = quantile.validate_quantile_absolute_ee +_original_quantile_checkpoint_metadata = quantile.quantile_checkpoint_metadata + + +def _validate_shared_timestep_contract(config: dict[str, Any]) -> None: + diffusion = config["diffusion"] + if diffusion.get("timestep_coupling") != TIMESTEP_COUPLING: + raise ValueError( + "Shared-timestep sibling requires " + f"diffusion.timestep_coupling={TIMESTEP_COUPLING!r}" + ) + matching_pairs = ( + ("num_train_timesteps", "action_num_train_timesteps"), + ("train_shift", "action_train_shift"), + ("infer_shift", "action_infer_shift"), + ) + for video_key, action_key in matching_pairs: + video_value = float(diffusion[video_key]) + action_value = float(diffusion[action_key]) + if not math.isclose(video_value, action_value, rel_tol=0.0, abs_tol=0.0): + raise ValueError( + "Shared-timestep sibling requires matching schedules: " + f"diffusion.{video_key}={video_value} != " + f"diffusion.{action_key}={action_value}" + ) + if config["model"].get("cross_modal_attention") != "bidirectional": + raise ValueError("Shared-timestep sibling keeps bidirectional attention") + if config.get("experiment_variant", {}).get("objective") != "joint": + raise ValueError("Shared-timestep sibling requires joint video/action loss") + if not bool(config["training"].get("train_video_expert")): + raise ValueError("Shared-timestep sibling must train the video expert") + + +def validate_shared_timestep_quantile_absolute_ee( + config: dict[str, Any] +) -> dict[str, Any]: + validated = _original_quantile_validation(config) + _validate_shared_timestep_contract(validated) + return validated + + +def shared_timestep_checkpoint_metadata(model: torch.nn.Module) -> dict[str, Any]: + metadata = _original_quantile_checkpoint_metadata(model) + metadata["diffusion_timestep_coupling"] = { + "training": TIMESTEP_COUPLING, + "video_action_timestep_identical_per_sample": True, + "video_action_noise": "independent_gaussian", + "inference": "synchronized_joint_schedule", + } + return metadata + + +def main() -> None: + if fourway._original_compute_loss is not galexea.compute_loss: + raise RuntimeError("Unexpected compute-loss wiring; refusing silent fallback") + galexea.joint_rgb_action_flow_matching_loss = ( + shared_timestep.joint_rgb_action_shared_timestep_flow_matching_loss + ) + quantile.validate_quantile_absolute_ee = ( + validate_shared_timestep_quantile_absolute_ee + ) + quantile.quantile_checkpoint_metadata = shared_timestep_checkpoint_metadata + print( + "[shared-timestep] enabled: one sampled timestep per video/action pair; " + "noise remains independent; architecture and inference unchanged", + flush=True, + ) + quantile.main() + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_ego_first_cam_meanstd_4way.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_ego_first_cam_meanstd_4way.py new file mode 100644 index 0000000000000000000000000000000000000000..702f384fccda8c33a479faa3cd9023dc2c5824af --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_ego_first_cam_meanstd_4way.py @@ -0,0 +1,638 @@ +#!/usr/bin/env python3 +"""RoboDojo RGB/action training in the episode-first ``cam_head`` frame. + +This adapter keeps the established exhaustive FastWAM/Wan-flow training loop, +but replaces its pose source at one shared boundary: + + T_C0_E(t) = inverse(T_B_C0) @ T_B_E(t) + +``T_B_C0`` is read per episode from the native RoboDojo ``cam_head`` HDF5 +extrinsic. State and action use the same target-refitted mean/std transform. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +import h5py +import numpy as np +import torch + +import train_robodojo_rgb_state_joint_action_4way as fourway +import train_robodojo_rgb_state_joint_action_absolute_ee_4way as absolute +from robodojo_ego_first_cam import ( + ACTION_DIMENSION_ORDER, + MATRIX_CONVENTION, + QUATERNION_LAYOUT, + transform_pose16_sequence_base_to_c0, + validate_rigid_transform, +) + + +ACTION_REPRESENTATION = "dual_arm_absolute_translation_so3_log_plus_grip" +COORDINATE_FRAME = "episode_first_cam_head" +INVERSE_DEPLOYMENT_FRAME = "robodojo_robot_base" +NORMALIZATION_TYPE = "mean_std" +NORMALIZATION_EPS = 1.0e-6 +NORMALIZATION_CLIP = 10.0 + +_original_load_config = fourway.load_config +_original_absolute_validation = absolute.validate_absolute_ee_with_legacy_base +_original_cache_fingerprint = absolute.absolute_ee_cache_fingerprint +_original_load_episode_arrays = fourway._load_episode_arrays +_original_prepare_cache = fourway.prepare_episode_length_normalized_cache +_original_fit_normalizer = fourway.fit_action_normalizer_cached +_original_build_model = fourway.build_experiment_model +_original_checkpoint_metadata = fourway.checkpoint_metadata + +_TRANSFORMED_EPISODE_CACHE: dict[ + tuple[str, str, int], tuple[np.ndarray, np.ndarray, np.ndarray] +] = {} +_EXTRINSIC_CACHE: dict[tuple[str, int], tuple[np.ndarray, dict[str, Any]]] = {} + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(16 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _task_identity(episode: Any) -> tuple[str, int, int]: + prompt_to_task = { + prompt: (task_id, int(index)) + for task_id, (index, prompt) in fourway.EXPECTED_TASKS.items() + } + identity = prompt_to_task.get(str(episode.task)) + if identity is None: + raise RuntimeError(f"No audited native-data mapping for task: {episode.task!r}") + task_id, task_index = identity + episode_index = int(episode.episode_index) + native_index = episode_index - task_index * 100 + if not 0 <= native_index < 100 or episode_index // 100 != task_index: + raise RuntimeError( + f"LeRobot/native episode mapping is not the audited 100-per-task layout: " + f"episode={episode_index} task={task_id} task_index={task_index}" + ) + return task_id, task_index, native_index + + +def native_extrinsic_path(config: dict[str, Any], episode: Any) -> Path: + task_id, _, native_index = _task_identity(episode) + return ( + Path(config["paths"]["robodojo_cam_head_extrinsics_root"]) + / task_id + / "arx_x5" + / "data" + / f"episode_{native_index:07d}.hdf5" + ) + + +def load_episode_t_b_c0( + config: dict[str, Any], episode: Any +) -> tuple[np.ndarray, dict[str, Any]]: + """Load the episode's exact first camera-to-command-frame transform.""" + + path = native_extrinsic_path(config, episode) + key = (str(path), int(episode.length)) + cached = _EXTRINSIC_CACHE.get(key) + if cached is not None: + return cached + if not path.is_file(): + raise FileNotFoundError(path) + expected_source = f"data/RoboDojo_depth/{path.parts[-4]}/arx_x5/data/{path.name}" + with h5py.File(path, "r") as handle: + if handle.attrs.get("format") != "robodojo_depth_only_v1": + raise RuntimeError(f"Unexpected native extrinsic container format: {path}") + cameras = json.loads(str(handle.attrs.get("cameras", "[]"))) + if cameras != ["cam_head"]: + raise RuntimeError(f"Native extrinsic camera contract mismatch: {cameras}") + if str(handle.attrs.get("source_file", "")) != path.name: + raise RuntimeError(f"Native extrinsic source_file mismatch: {path}") + if str(handle.attrs.get("source_repo_path", "")) != expected_source: + raise RuntimeError( + f"Native extrinsic source_repo_path mismatch for {path}: " + f"{handle.attrs.get('source_repo_path')!r} != {expected_source!r}" + ) + dataset_key = "vision/cam_head/extrinsic_matrix" + if dataset_key not in handle: + raise KeyError(f"Missing {dataset_key} in {path}") + matrices = np.asarray(handle[dataset_key], dtype=np.float64) + expected_shape = (int(episode.length), 4, 4) + if matrices.shape != expected_shape: + raise RuntimeError( + f"Native extrinsic length mismatch: {path} {matrices.shape} != {expected_shape}" + ) + if not np.isfinite(matrices).all(): + raise RuntimeError(f"Native extrinsics contain NaN/Inf: {path}") + for index, matrix in enumerate(matrices): + validate_rigid_transform(matrix, name=f"{path}:T_B_C[{index}]") + t_b_c0 = matrices[0].copy() + matrix_sha256 = hashlib.sha256(matrices.tobytes(order="C")).hexdigest() + audit = { + "path": str(path), + "matrix_sha256": matrix_sha256, + "frames": int(len(matrices)), + "matrix_semantics": "camera_to_robodojo_env_robot_command_frame", + "selected_frame": 0, + } + _EXTRINSIC_CACHE[key] = (t_b_c0, audit) + return t_b_c0, audit + + +def load_transformed_episode_arrays( + config: dict[str, Any], episode: Any +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + root = str(Path(config["paths"]["dataset_root"]).resolve()) + extrinsic_root = str( + Path(config["paths"]["robodojo_cam_head_extrinsics_root"]).resolve() + ) + key = (root, extrinsic_root, int(episode.episode_index)) + cached = _TRANSFORMED_EPISODE_CACHE.get(key) + if cached is not None: + return cached + state_base, action_base, timestamps = _original_load_episode_arrays(config, episode) + t_b_c0, _ = load_episode_t_b_c0(config, episode) + state_c0 = transform_pose16_sequence_base_to_c0(state_base, t_b_c0) + action_c0 = transform_pose16_sequence_base_to_c0(action_base, t_b_c0) + if not np.array_equal(state_c0[:, [7, 15]], state_base[:, [7, 15]]): + raise RuntimeError("State grippers changed during base-to-C0 conversion") + if not np.array_equal(action_c0[:, [7, 15]], action_base[:, [7, 15]]): + raise RuntimeError("Action grippers changed during base-to-C0 conversion") + transformed = (state_c0, action_c0, timestamps) + _TRANSFORMED_EPISODE_CACHE[key] = transformed + return transformed + + +def load_ego_config(path: Path) -> dict[str, Any]: + config = _original_load_config(path) + action = config.pop("ego_first_cam_action", None) + if not isinstance(action, dict): + raise ValueError("Config must define a complete ego_first_cam_action mapping") + config["action"] = action + return config + + +def _normalization_contract(config: dict[str, Any]) -> dict[str, Any]: + normalization = config["action"]["normalization"] + expected = { + "type": NORMALIZATION_TYPE, + "fit": "transformed_robodojo_train_split_only", + "eps": NORMALIZATION_EPS, + "clip": NORMALIZATION_CLIP, + "checkpoint_buffers": True, + "state_uses_action_statistics": True, + } + actual = {key: normalization.get(key) for key in expected} + if actual != expected: + raise ValueError(f"C0 mean/std normalization mismatch: {actual} != {expected}") + return expected + + +def validate_ego_config(config: dict[str, Any]) -> dict[str, Any]: + action = config["action"] + required = { + "representation": ACTION_REPRESENTATION, + "source": "absolute_ee_action_t", + "target_formula": "T_C0_B_matmul_base_frame_commanded_target_ee", + "integration": "none", + "coordinate_frame": COORDINATE_FRAME, + "translation_encoding": "absolute_xyz", + "rotation_encoding": "absolute_so3_log", + } + actual = {key: action.get(key) for key in required} + if actual != required: + raise ValueError(f"C0 action config mismatch: {actual} != {required}") + if int(action.get("action_dim", -1)) != 14: + raise ValueError("C0 action_dim must be 14") + initial = action.get("initial_state", {}) + initial_expected = { + "representation": ACTION_REPRESENTATION, + "source": "sample_first_state", + "formula": "T_C0_B_matmul_base_frame_current_ee", + "coordinate_frame": COORDINATE_FRAME, + "translation_encoding": "absolute_xyz", + "rotation_encoding": "absolute_so3_log", + "normalization": "shared_action_mean_std", + } + initial_actual = {key: initial.get(key) for key in initial_expected} + if initial_actual != initial_expected: + raise ValueError( + f"C0 initial-state config mismatch: {initial_actual} != {initial_expected}" + ) + _normalization_contract(config) + if config["robodojo"].get("cam_head_extrinsic_key") != ( + "vision/cam_head/extrinsic_matrix" + ): + raise ValueError("Training must use native cam_head extrinsic_matrix") + if config["robodojo"].get("cam_head_extrinsic_semantics") != "T_B_C_camera_to_base": + raise ValueError("cam_head extrinsic matrix semantics must be explicit") + if config["robodojo"].get("quaternion_layout") != QUATERNION_LAYOUT: + raise ValueError("RoboDojo quaternion layout must be wxyz") + if config["robodojo"].get("matrix_convention") != MATRIX_CONVENTION: + raise ValueError("SE(3) matrix convention must be T_target_source") + + training = config["training"] + smoke_mode = os.environ.get("ROBODOJO_4WAY_SMOKE", "0") == "1" + expected_training = { + "video_lr": 5.0e-6, + "action_backbone_lr": 1.0e-5, + "action_io_lr": 5.0e-5, + "warmup_steps": 1000, + "lr_schedule": "constant", + "scheduler_step_unit": "optimizer_step", + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "global_batch_size": 256, + "exact_epochs": 20, + } + training_actual = {key: training.get(key) for key in expected_training} + if training_actual != expected_training: + raise ValueError( + f"Pretrained param-group training contract mismatch: " + f"{training_actual} != {expected_training}" + ) + if smoke_mode: + if not 1 <= int(training.get("max_steps", 0)) <= 50: + raise ValueError("C0 smoke max_steps must be within [1,50]") + elif int(training.get("max_steps", -1)) != 33580: + raise ValueError("Formal C0 training requires exactly 33580 steps") + if not bool(training.get("fp32_master_weights")): + raise ValueError("This experiment requires FP32 master weights/Adam states") + if int(config["experiment"]["seed"]) != 20260724: + raise ValueError("Experiment seed must be 20260724") + if int(config["data"]["sample_order_seed"]) != 20260724: + raise ValueError("Data seed must be 20260724") + + initialization = config["initialization"] + if initialization.get("mode") != fourway.EGO_FULL_ACTION_INITIALIZATION_MODE: + raise ValueError("The fail-closed full-action initialization mode is required") + expected_sha = str(initialization.get("source_sha256", "")) + if len(expected_sha) != 64: + raise ValueError("initialization.source_sha256 must be a full SHA256") + + # Reuse the mature absolute-target/base training validation after replacing + # only its coordinate-frame-specific declarations. + compatible = copy.deepcopy(action) + compatible.update( + { + "target_formula": "base_frame_commanded_target_ee", + "coordinate_frame": "robodojo_robot_base", + } + ) + compatible["normalization"] = copy.deepcopy(action["normalization"]) + compatible["normalization"]["fit"] = "robodojo_train_split_cache" + compatible["initial_state"].update( + { + "formula": "base_frame_current_ee", + "coordinate_frame": "robodojo_robot_base", + } + ) + original_action = config["action"] + config["action"] = compatible + try: + validated = _original_absolute_validation(config) + finally: + config["action"] = original_action + validated["action"] = original_action + if smoke_mode: + validated["training"]["warmup_steps"] = min( + 10, int(validated["training"]["max_steps"]) + ) + validated["training"]["log_every"] = 1 + # The smoke is a preflight gate, not a second tracked experiment. + # Keep the user's W&B project clean: only the formal 20-epoch run is + # created online after this gate passes. + validated["tracking"]["enabled"] = False + return validated + + +def ego_cache_fingerprint( + config: dict[str, Any], splits: dict[str, list[Any]] +) -> str: + source = _original_cache_fingerprint(config, splits) + extrinsics: list[dict[str, Any]] = [] + for split in fourway.base.SPLITS: + for episode in splits[split]: + path = native_extrinsic_path(config, episode) + t_b_c0, audit = load_episode_t_b_c0(config, episode) + stat = path.stat() + extrinsics.append( + { + "split": split, + "episode_index": int(episode.episode_index), + "path": str(path), + "size": int(stat.st_size), + "mtime_ns": int(stat.st_mtime_ns), + "matrix_sha256": audit["matrix_sha256"], + "T_B_C0": t_b_c0.tolist(), + } + ) + payload = { + "source_fingerprint": source, + "coordinate_frame": COORDINATE_FRAME, + "matrix_convention": MATRIX_CONVENTION, + "quaternion_layout": QUATERNION_LAYOUT, + "dimension_order": ACTION_DIMENSION_ORDER, + "extrinsics": extrinsics, + "format": "robodojo_episode_first_cam_head_absolute_xyz_so3_v1", + } + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() + + +def _statistics_fingerprint(payload: dict[str, Any], cache_fingerprint: str) -> str: + digest = hashlib.sha256() + digest.update(cache_fingerprint.encode()) + for key in ("mean", "std", "count", "fitted"): + tensor = torch.as_tensor(payload[key]).contiguous().cpu() + digest.update(str(tensor.dtype).encode()) + digest.update(str(tuple(tensor.shape)).encode()) + digest.update(tensor.numpy().tobytes()) + digest.update(json.dumps(ACTION_DIMENSION_ORDER).encode()) + digest.update(COORDINATE_FRAME.encode()) + return digest.hexdigest() + + +@torch.no_grad() +def prepare_ego_cache( + config: dict[str, Any], *, device: torch.device, overwrite: bool +) -> None: + _original_prepare_cache(config, device=device, overwrite=overwrite) + cache_dir = Path(config["paths"]["cache_dir"]) + manifest_path = cache_dir / "manifest.json" + stats_path = cache_dir / "action_normalizer.pt" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + stats = torch.load(stats_path, map_location="cpu", weights_only=True) + if stats.get("fingerprint") != manifest.get("fingerprint"): + raise RuntimeError("C0 normalizer/cache fingerprint mismatch") + statistics_fingerprint = _statistics_fingerprint( + stats, str(manifest["fingerprint"]) + ) + expected_stats_metadata = { + "normalization_type": NORMALIZATION_TYPE, + "eps": NORMALIZATION_EPS, + "clip": NORMALIZATION_CLIP, + "fit_split": "train", + "coordinate_frame": COORDINATE_FRAME, + "dimension_order": list(ACTION_DIMENSION_ORDER), + "state_uses_action_statistics": True, + "statistics_fingerprint": statistics_fingerprint, + } + stats.update(expected_stats_metadata) + temporary = stats_path.with_name(f"{stats_path.name}.ego.{os.getpid()}.tmp") + torch.save(stats, temporary) + os.replace(temporary, stats_path) + manifest["coordinate_contract"] = { + "coordinate_frame": COORDINATE_FRAME, + "source_frame": INVERSE_DEPLOYMENT_FRAME, + "formula": "T_C0_E=inv(T_B_C0)@T_B_E", + "matrix_convention": MATRIX_CONVENTION, + "quaternion_layout": QUATERNION_LAYOUT, + "extrinsic_source": "native_hdf5:vision/cam_head/extrinsic_matrix[0]", + } + manifest["action_normalization"] = expected_stats_metadata + temporary_manifest = manifest_path.with_name( + f"{manifest_path.name}.ego.{os.getpid()}.tmp" + ) + temporary_manifest.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + os.replace(temporary_manifest, manifest_path) + print( + f"[ego-normalizer] train-only mean/std ready count={int(stats['count'])} " + f"fingerprint={statistics_fingerprint} path={stats_path}", + flush=True, + ) + + +class EgoFirstCameraDataset(fourway.ExhaustiveWindowDataset): + """Normalize clean C0 state with the exact same buffers as C0 actions.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + smoke_limit = int(os.environ.get("ROBODOJO_EGO_SMOKE_SAMPLES", "0")) + if smoke_limit: + if not 16 <= smoke_limit <= 32: + raise ValueError("ROBODOJO_EGO_SMOKE_SAMPLES must be within [16,32]") + self.total_windows = min(self.total_windows, smoke_limit) + print( + f"[ego-smoke] fixed overfit subset samples={self.total_windows}", + flush=True, + ) + + def __getitem__(self, index: int) -> dict[str, Any]: + sample = super().__getitem__(index) + if self.action_mean is not None: + state = (sample["initial_state"] - self.action_mean) / self.action_std + if self.action_clip is not None: + state = state.clamp(-float(self.action_clip), float(self.action_clip)) + sample["initial_state"] = state + return sample + + +@torch.no_grad() +def fit_ego_normalizer(model: torch.nn.Module, dataset: Any) -> None: + _original_fit_normalizer(model, dataset) + stats_path = Path(dataset.cache_dir) / "action_normalizer.pt" + stats = torch.load(stats_path, map_location="cpu", weights_only=True) + required = { + "normalization_type": NORMALIZATION_TYPE, + "fit_split": "train", + "coordinate_frame": COORDINATE_FRAME, + "dimension_order": list(ACTION_DIMENSION_ORDER), + "state_uses_action_statistics": True, + } + actual = {key: stats.get(key) for key in required} + if actual != required: + raise RuntimeError(f"C0 normalizer metadata mismatch: {actual} != {required}") + model._robodojo_action_normalization = { + **required, + "eps": NORMALIZATION_EPS, + "clip": NORMALIZATION_CLIP, + "checkpoint_buffers": True, + "statistics_fingerprint": str(stats["statistics_fingerprint"]), + "count": int(stats["count"]), + "buffers": { + "mean": "action_normalizer.mean", + "std": "action_normalizer.std", + "count": "action_normalizer.count", + "fitted": "action_normalizer.fitted", + }, + } + + +def build_ego_model(config: dict[str, Any]) -> torch.nn.Module: + initialization = config["initialization"] + expected_sha = str(initialization["source_sha256"]) + selected = Path( + initialization.get("local_checkpoint") + if initialization.get("local_checkpoint") + and Path(initialization["local_checkpoint"]).is_file() + else initialization["checkpoint"] + ) + actual_sha = expected_sha + if int(os.environ.get("LOCAL_RANK", "0")) == 0: + actual_sha = _sha256_file(selected) + if actual_sha != expected_sha: + raise RuntimeError( + f"SpaceDreamer checkpoint SHA256 mismatch: {actual_sha} != {expected_sha}" + ) + print( + f"[initialization] source_sha256_verified={actual_sha} selected={selected}", + flush=True, + ) + model = _original_build_model(config) + model._robodojo_ego_contract = { + "source_checkpoint_path": str(Path(initialization["checkpoint"]).resolve()), + "source_checkpoint_sha256": expected_sha, + "video_slice_source": "SpaceDreamer RGB channels [0:48] from RGB+depth+point expert", + "action_encoder": "source_checkpoint", + "action_backbone_30_layers": "source_checkpoint", + "action_head": "source_checkpoint", + "action_state_encoder": "source_checkpoint", + "action_normalizer": "target_refit", + "coordinate_frame": COORDINATE_FRAME, + "action_representation": "absolute_xyz_so3_log_plus_grip", + "inverse_deployment_frame": INVERSE_DEPLOYMENT_FRAME, + "matrix_convention": MATRIX_CONVENTION, + "quaternion_layout": QUATERNION_LAYOUT, + "dimension_order": list(ACTION_DIMENSION_ORDER), + "state_normalization": "shared_action_mean_std", + } + return model + + +def build_param_group_optimizer( + model: torch.nn.Module, config: dict[str, Any] +) -> torch.optim.Optimizer: + categories: dict[str, list[tuple[str, torch.nn.Parameter]]] = { + "pretrained_rgb_video_expert": [], + "pretrained_action_backbone": [], + "pretrained_action_io": [], + } + for name, parameter in model.named_parameters(): + if not parameter.requires_grad: + continue + if name.startswith("mot.mixtures.video."): + group = "pretrained_rgb_video_expert" + elif name.startswith("mot.mixtures.action.action_encoder.") or name.startswith( + "mot.mixtures.action.head." + ) or name.startswith("action_state_encoder."): + group = "pretrained_action_io" + elif name.startswith("mot.mixtures.action."): + group = "pretrained_action_backbone" + else: + raise RuntimeError(f"Trainable parameter has no optimizer group: {name}") + categories[group].append((name, parameter)) + + lrs = { + "pretrained_rgb_video_expert": float(config["training"]["video_lr"]), + "pretrained_action_backbone": float( + config["training"]["action_backbone_lr"] + ), + "pretrained_action_io": float(config["training"]["action_io_lr"]), + } + all_trainable = { + id(parameter) + for parameter in model.parameters() + if parameter.requires_grad + } + assigned = [ + id(parameter) + for values in categories.values() + for _, parameter in values + ] + if len(assigned) != len(set(assigned)): + raise RuntimeError("A trainable parameter was assigned to multiple optimizer groups") + if set(assigned) != all_trainable: + raise RuntimeError("Optimizer group coverage does not equal trainable parameters") + audit = { + name: { + "parameter_count": len(values), + "numel": sum(parameter.numel() for _, parameter in values), + "lr": lrs[name], + } + for name, values in categories.items() + } + if any(value["parameter_count"] == 0 for value in audit.values()): + raise RuntimeError(f"Optimizer has an empty required group: {audit}") + if any( + parameter.dtype != torch.float32 + for values in categories.values() + for _, parameter in values + if parameter.is_floating_point() + ): + raise RuntimeError("FP32 master parameter audit failed before AdamW creation") + model._robodojo_optimizer_audit = audit + print(f"[optimizer-audit] {json.dumps(audit, sort_keys=True)}", flush=True) + return torch.optim.AdamW( + [ + { + "name": name, + "params": [parameter for _, parameter in values], + "lr": lrs[name], + } + for name, values in categories.items() + ], + betas=(0.9, 0.95), + eps=1.0e-8, + weight_decay=float(config["training"]["weight_decay"]), + ) + + +def ego_checkpoint_metadata(model: torch.nn.Module) -> dict[str, Any]: + metadata = _original_checkpoint_metadata(model) + ego_contract = getattr(model, "_robodojo_ego_contract", None) + normalization = getattr(model, "_robodojo_action_normalization", None) + optimizer_audit = getattr(model, "_robodojo_optimizer_audit", None) + if not isinstance(ego_contract, dict): + raise RuntimeError("Checkpoint export is missing the ego coordinate contract") + if not isinstance(normalization, dict): + raise RuntimeError("Checkpoint export is missing target-refit normalization metadata") + if not isinstance(optimizer_audit, dict): + raise RuntimeError("Checkpoint export is missing optimizer param-group audit") + metadata["ego_action_contract"] = ego_contract + metadata["action_semantics"] = { + "representation": ACTION_REPRESENTATION, + "source": "absolute_ee_action_t", + "target_formula": "T_C0_B_matmul_base_frame_commanded_target_ee", + "state_formula": "T_C0_B_matmul_base_frame_current_ee", + "coordinate_frame": COORDINATE_FRAME, + "integration": "none", + "gripper": "absolute_command", + "translation_encoding": "absolute_xyz", + "rotation_encoding": "absolute_so3_log", + "inverse_deployment_frame": INVERSE_DEPLOYMENT_FRAME, + "matrix_convention": MATRIX_CONVENTION, + "quaternion_layout": QUATERNION_LAYOUT, + } + metadata["action_normalization"] = normalization + metadata["optimizer_param_groups"] = optimizer_audit + return metadata + + +def install_hooks() -> None: + fourway.load_config = load_ego_config + fourway._original_validate_config = validate_ego_config + fourway.episode_length_normalized_cache_fingerprint = ego_cache_fingerprint + fourway._load_episode_arrays = load_transformed_episode_arrays + fourway.prepare_episode_length_normalized_cache = prepare_ego_cache + fourway.ExhaustiveWindowDataset = EgoFirstCameraDataset + fourway.fit_action_normalizer_cached = fit_ego_normalizer + fourway.build_experiment_model = build_ego_model + fourway.build_optimizer = build_param_group_optimizer + fourway.checkpoint_metadata = ego_checkpoint_metadata + + +def main() -> None: + install_hooks() + fourway.base.build_egoverse_delta_targets = absolute.build_absolute_ee_targets + fourway.main() + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_episode_anchor_4way.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_episode_anchor_4way.py new file mode 100644 index 0000000000000000000000000000000000000000..e72cc4be1c89eefefbb313df21831ceb0910c828 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_robodojo_rgb_state_joint_action_episode_anchor_4way.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Four-way RoboDojo training with episode-initial-frame EE targets. + +This is an isolated adapter around ``train_robodojo_rgb_state_joint_action_4way``. +The original RoboDojo and four-way training implementations remain untouched. + +For a sample beginning at frame ``s`` in an episode, the clean state context is +``log(inv(T_episode_0) @ T_state_s)`` and every future command is independently +represented as ``log(inv(T_episode_0) @ T_action_t)``. Consequently no action +target depends on a preceding predicted action. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +import train_robodojo_rgb_state_joint_action_4way as fourway +from online_egoverse_action import ACTION_DIM, _rigid_inverse, se3_log +from robodojo_lerobot_v30 import _pose16_matrix + + +ACTION_REPRESENTATION = "dual_arm_episode_initial_relative_se3_log_plus_grip" +ACTION_SOURCE = "episode_initial_state_to_absolute_ee_action_t" + +_fourway_load_config = fourway.load_config +_fourway_validate_base_config = fourway._original_validate_config +_fourway_cache_fingerprint = fourway.episode_length_normalized_cache_fingerprint +_fourway_checkpoint_metadata = fourway.checkpoint_metadata + + +def build_episode_initial_relative_targets( + state: np.ndarray, + absolute_action: np.ndarray, + image_indices: np.ndarray, +) -> dict[str, torch.Tensor]: + """Build 32 independent EE targets in the episode-initial EE frames.""" + + indices = np.asarray(image_indices, dtype=np.int64) + if len(indices) < 2 or np.any(np.diff(indices) != 1): + raise ValueError("RoboDojo clips must use consecutive native frames") + if int(indices[0]) < 0 or int(indices[-1]) >= len(state): + raise IndexError("RoboDojo clip indices are outside the episode") + if state.shape != absolute_action.shape or state.ndim != 2 or state.shape[1] != 16: + raise ValueError("RoboDojo state/action must have matching [T,16] shapes") + + left_anchor = _pose16_matrix(state[0:1], "left")[0] + right_anchor = _pose16_matrix(state[0:1], "right")[0] + left_anchor_inverse = _rigid_inverse(left_anchor) + right_anchor_inverse = _rigid_inverse(right_anchor) + + commanded = absolute_action[indices[:-1]] + left_commanded = _pose16_matrix(commanded, "left") + right_commanded = _pose16_matrix(commanded, "right") + action = np.zeros((len(indices) - 1, ACTION_DIM), dtype=np.float32) + for row in range(len(indices) - 1): + action[row, 0:6] = se3_log(left_anchor_inverse @ left_commanded[row]) + action[row, 7:13] = se3_log(right_anchor_inverse @ right_commanded[row]) + action[:, 6] = commanded[:, 7] + action[:, 13] = commanded[:, 15] + + current = state[int(indices[0]) : int(indices[0]) + 1] + left_current = _pose16_matrix(current, "left")[0] + right_current = _pose16_matrix(current, "right")[0] + initial_state = np.concatenate( + [ + se3_log(left_anchor_inverse @ left_current), + [current[0, 7]], + se3_log(right_anchor_inverse @ right_current), + [current[0, 15]], + ] + ).astype(np.float32) + + if not np.isfinite(action).all() or not np.isfinite(initial_state).all(): + raise RuntimeError("Episode-initial RoboDojo targets contain non-finite values") + return { + "action_raw": torch.from_numpy(action), + "action_is_pad": torch.zeros(len(indices) - 1, dtype=torch.bool), + "initial_state": torch.from_numpy(initial_state), + "left_initial_pose": torch.from_numpy(left_anchor.astype(np.float32)), + "right_initial_pose": torch.from_numpy(right_anchor.astype(np.float32)), + } + + +def load_episode_anchor_config(path: Path) -> dict[str, Any]: + config = _fourway_load_config(path) + action = config.pop("episode_initial_action", None) + if not isinstance(action, dict): + raise ValueError("Config must define a complete episode_initial_action mapping") + config["action"] = action + return config + + +def validate_episode_anchor_with_legacy_base(config: dict[str, Any]) -> dict[str, Any]: + """Reuse legacy structural checks without accepting legacy action semantics.""" + + action = config["action"] + required = { + "representation": ACTION_REPRESENTATION, + "source": ACTION_SOURCE, + "anchor": "episode_first_state", + "target_formula": "inverse_episode_initial_ee_matmul_commanded_target_ee", + "integration": "none", + "coordinate_frame": "episode_initial_ee_frame", + } + actual = {key: action.get(key) for key in required} + if actual != required: + raise ValueError(f"Episode-initial action config mismatch: {actual} != {required}") + if int(action.get("action_dim", -1)) != ACTION_DIM: + raise ValueError(f"Episode-initial action_dim must be {ACTION_DIM}") + initial_state = action.get("initial_state", {}) + if initial_state.get("representation") != ACTION_REPRESENTATION: + raise ValueError("Current state must use the episode-initial action frame") + if initial_state.get("source") != "sample_first_state": + raise ValueError("State conditioning must use the current sample-first state") + if initial_state.get("formula") != "inverse_episode_initial_ee_matmul_current_ee": + raise ValueError("State conditioning formula is inconsistent with the action frame") + + episode_action = copy.deepcopy(action) + legacy_action = copy.deepcopy(action) + legacy_action["representation"] = "dual_arm_se3_body_log_plus_grip" + legacy_action["source"] = "state_t_to_absolute_ee_action_t" + config["action"] = legacy_action + try: + validated = _fourway_validate_base_config(config) + finally: + config["action"] = episode_action + validated["action"] = episode_action + return validated + + +def episode_anchor_cache_fingerprint( + config: dict[str, Any], splits: dict[str, list[Any]] +) -> str: + source = _fourway_cache_fingerprint(config, splits) + payload = { + "source_fingerprint": source, + "action": config["action"], + "format": "episode_initial_relative_ee_action_v1", + } + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() + + +def episode_anchor_checkpoint_metadata(model: torch.nn.Module) -> dict[str, Any]: + metadata = _fourway_checkpoint_metadata(model) + metadata["action_semantics"] = { + "representation": ACTION_REPRESENTATION, + "source": ACTION_SOURCE, + "anchor": "episode_first_state", + "target_formula": "inverse_episode_initial_ee_matmul_commanded_target_ee", + "state_formula": "inverse_episode_initial_ee_matmul_current_ee", + "integration": "none", + "gripper": "absolute_command", + } + return metadata + + +def main() -> None: + fourway.load_config = load_episode_anchor_config + fourway._original_validate_config = validate_episode_anchor_with_legacy_base + fourway.episode_length_normalized_cache_fingerprint = episode_anchor_cache_fingerprint + fourway.checkpoint_metadata = episode_anchor_checkpoint_metadata + # Both online samples and the exact weighted normalizer statistics call this + # module attribute, so a single replacement keeps them definition-identical. + fourway.base.build_egoverse_delta_targets = build_episode_initial_relative_targets + fourway.main() + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_spacedreamer_ego.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_spacedreamer_ego.py new file mode 100644 index 0000000000000000000000000000000000000000..ff4f92b4032ccc94ba4987a8cf3b5b874bea0795 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/train_spacedreamer_ego.py @@ -0,0 +1,1555 @@ +#!/usr/bin/env python3 +"""Isolated full-Egoverse SpaceDreamer RGB/depth/point/action/camera trainer.""" + +from __future__ import annotations + +import argparse +import copy +from datetime import timedelta +from functools import partial +import gc +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import time +from typing import Any + +import torch +from torch.utils.data import DataLoader +import yaml +from accelerate import Accelerator +from accelerate.utils import ( + DistributedDataParallelKwargs, + InitProcessGroupKwargs, + broadcast_object_list, + set_seed, +) + +from controlled_directional_mot import ForwardOnBackwardOnTwoCallMoT +from rgb_depth_point_action_model_factory import ( + build_joint_action_mot, + validate_action_mot_config, +) +from spacedreamer_ego_dataset import ( + SpaceDreamerEgoDataset, + absolute_targets_in_episode_first_camera, + build_rank_schedules, + build_validation_schedules, + collate_spacedreamer, + configure_spacedreamer_worker, + fixed_validation_start, + load_eligible_episodes, + load_motion_window_payload, + split_episodes, +) +from canonical_spacedreamer_data import ( + CanonicalSpaceDreamerDataset, + IndexedCanonicalSpaceDreamerDataset, + build_balanced_rank_schedules, + canonical_collate, + load_canonical_manifest, + load_indexed_canonical_manifest, +) +from train_joint_rgb_depth import ( + WanContinuousFlowMatchScheduler, + append_jsonl, + build_lr_scheduler, + dtype_from_name, + initialize_experiment_trackers, + load_config, + load_text_encoder_and_tokenizer, + load_vae, +) +from train_joint_rgb_depth_point import ProjectionGradientMasks +from train_joint_rgb_depth_point_action import ( + build_optimizer, + compute_training_loss, + configure_trainable_parameters, + enable_fp32_master_weights, + export_action_mot, + inference_checkpoint_path, +) + + +_DYNAMIC_VAE: Any = None +_VAE_BATCHED_MODALITIES_OK: bool | None = None +_VALIDATION_LATENT_CACHE: list[dict[str, Any]] | None = None + + +def distributed_max_scalar( + accelerator: Accelerator, value: float +) -> torch.Tensor: + """Return a scalar max without relying on unsupported Accelerate reductions.""" + + gathered = accelerator.gather( + torch.tensor([value], device=accelerator.device) + ) + return gathered.max() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--resume-model", type=Path) + parser.add_argument("--resume-global-step", type=int, default=0) + parser.add_argument( + "--wandb-log-after-step", + type=int, + default=0, + help=( + "When resuming an existing W&B run whose online history is ahead of " + "the durable model checkpoint, suppress only W&B writes through this " + "step. Local resume metrics are still recorded." + ), + ) + parser.add_argument("--validate-only", action="store_true") + parser.add_argument("--prepare-metadata-only", action="store_true") + parser.add_argument( + "--smoke-max-steps", + type=int, + help="Run an isolated short benchmark; never use its checkpoint as a formal run.", + ) + parser.add_argument( + "--smoke-expected-num-processes", + type=int, + help=( + "Override the formal distributed world size only for --smoke-max-steps; " + "this is used for a local throughput smoke test." + ), + ) + parser.add_argument( + "--smoke-prompt-cache-dir", + type=Path, + help=( + "Override the shared formal prompt cache only for --smoke-max-steps; " + "useful when a complete node-local cache already exists." + ), + ) + return parser.parse_args() + + +def _require(config: dict[str, Any], dotted: str, expected: Any) -> None: + value: Any = config + for key in dotted.split("."): + if not isinstance(value, dict) or key not in value: + raise ValueError(f"Missing required config field: {dotted}") + value = value[key] + if value != expected: + raise ValueError(f"{dotted} must be {expected!r}, got {value!r}") + + +def _deep_update(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(base) + for key, value in override.items(): + if isinstance(value, dict) and isinstance(result.get(key), dict): + result[key] = _deep_update(result[key], value) + else: + result[key] = copy.deepcopy(value) + return result + + +def load_spacedreamer_config(path: Path) -> dict[str, Any]: + """Load a normal config or a small ``_base_`` override config.""" + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError(f"Configuration must be a mapping: {path}") + base_value = raw.pop("_base_", None) + if base_value is None: + return raw + base_path = Path(base_value) + if not base_path.is_absolute(): + base_path = path.parent / base_path + return _deep_update(load_spacedreamer_config(base_path.resolve()), raw) + + +def _factory_compatible_config(config: dict[str, Any]) -> dict[str, Any]: + """Adapt only legacy validator labels; architecture and numeric fields stay exact.""" + + compatible = copy.deepcopy(config) + compatible["action"]["representation"] = "dual_arm_se3_body_log_plus_grip" + # The factory validates tensor semantics, not the offline storage backend. + compatible["action"]["source"] = "online_egoverse_pose_keypoints" + compatible["action"]["initial_state"] = { + "role": "clean_action_context", + "representation": "dual_arm_absolute_se3_log_plus_grip", + "action_slot_aligned_dim": 14, + "source_frame": "first", + "add_diffusion_noise": False, + "prediction_target": False, + "loss": "none", + } + compatible["camera"]["representation"] = "first_camera_relative_translation_so3_log" + compatible["camera"]["source"] = "online_egoverse_obs_head_pose" + return compatible + + +def validate_config(config: dict[str, Any]) -> dict[str, Any]: + """Validate data semantics and exact architecture switches before allocation.""" + + validate_action_mot_config(_factory_compatible_config(config)) + exact = { + "model.dtype": "bfloat16", + "model.dit.hidden_dim": 3072, + "model.dit.ffn_dim": 14336, + "model.dit.num_heads": 24, + "model.dit.attn_head_dim": 128, + "model.dit.num_layers": 30, + "model.dit.video_attention_mask_mode": "bidirectional", + "model.dit.use_gradient_checkpointing": False, + "model.action_dit.hidden_dim": 576, + "model.action_dit.ffn_dim": 2304, + "model.action_dit.num_heads": 24, + "model.action_dit.attn_head_dim": 128, + "model.action_dit.num_layers": 30, + "model.action_dit.use_gradient_checkpointing": False, + "model.mot_checkpoint_mixed_attn": False, + "model.cross_modal_attention": "bidirectional", + "model.action_loss_to_video_stop_gradient": False, + "conditioning.fix_rgb_first_latent": True, + "conditioning.fix_depth_first_latent": True, + "conditioning.exclude_rgb_condition_from_loss": True, + "conditioning.exclude_depth_condition_from_loss": True, + "conditioning.point_condition_mode": "none", + "conditioning.fix_point_first_latent": False, + "conditioning.exclude_point_condition_from_loss": False, + "conditioning.point_timestep_adapter_bottleneck_dim": 256, + "data.num_frames": 33, + "data.resolution": 256, + "data.target_fps": 30.0, + "data.batch_size": 1, + "data.num_workers": 1, + "data.metadata_workers": 32, + "data.frame_stride": 1, + "action.representation": "dual_arm_absolute_translation_so3_log_plus_grip", + "action.target_type": "absolute_ee_pose_in_episode_first_camera", + "action.coordinate_frame": "episode_first_camera_frame", + "action.action_dim": 14, + "camera.representation": "episode_first_camera_relative_translation_so3_log", + "camera.coordinate_frame": "episode_first_camera_frame", + "camera.camera_dim": 6, + "camera.token_order": "frames_1_to_32", + "point_cloud.representation": "dense_episode_first_camera_xyz_point_map_video", + "point_cloud.coordinate_frame": "episode_first_camera_frame", + "point_cloud.point_stride": 1, + "diffusion.rgb_depth_point_noise_mode": "independent", + "diffusion.train_shift": 5.0, + "diffusion.action_train_shift": 5.0, + "training.train_video_expert": True, + "training.fp32_master_weights": True, + "training.save_training_state": False, + "training.full_dit.lr_schedule": "constant", + } + for dotted, expected in exact.items(): + _require(config, dotted, expected) + if not 0.0 < float(config["data"]["validation_fraction"]) < 0.5: + raise ValueError("data.validation_fraction must be in (0, 0.5)") + expected_processes = int(config["training"]["expected_num_processes"]) + if expected_processes not in (8, 16, 32): + raise ValueError("training.expected_num_processes must be 8, 16, or 32") + max_epochs = config["training"].get("max_coverage_epochs") + smoke_test = bool(config["training"].get("smoke_test", False)) + if max_epochs is None: + if ( + not smoke_test + and int(config["training"]["full_dit"]["max_steps"]) < 50_000_000 + ): + raise ValueError("full_dit.max_steps must retain the 50M long-tail ceiling") + elif int(max_epochs) <= 0: + raise ValueError("training.max_coverage_epochs must be positive") + for dotted in ( + "paths.pretrained_model_dir", + "paths.prompt_pretrained_model_dir", + "paths.prompt_cache_dir", + "paths.dataset_root", + "paths.episode_manifest", + "paths.metadata_cache_dir", + ): + value: Any = config + for key in dotted.split("."): + value = value[key] + if not str(value): + raise ValueError(f"{dotted} cannot be empty") + for path_key in ( + "pretrained_model_dir", + "prompt_pretrained_model_dir", + "prompt_cache_dir", + "dataset_root", + "episode_manifest", + ): + if not Path(config["paths"][path_key]).exists(): + raise FileNotFoundError(config["paths"][path_key]) + if config["data"].get("format") in {"spacedreamer_canonical_v1", "spacedreamer_canonical_indexed_v1"}: + for path_key in ("validation_manifest", "canonical_normalizer"): + if path_key not in config["paths"]: + raise ValueError(f"Missing canonical path: paths.{path_key}") + if not Path(config["paths"][path_key]).is_file(): + raise FileNotFoundError(config["paths"][path_key]) + if config["data"].get("source_sampling_ratio") != { + "Egoverse": 0.5, "AgiBotWorld": 0.5 + }: + raise ValueError("Canonical mixing requires exact Egoverse:AgiBotWorld=1:1") + if int(config["training"]["validate_every"]) <= 0: + raise ValueError("training.validate_every must be positive") + if int(config["training"]["export_model_every"]) <= 0: + raise ValueError("training.export_model_every must be positive") + if int(config["data"]["prefetch_factor"]) <= 0: + raise ValueError("data.prefetch_factor must be positive") + for key in ("opencv_worker_threads", "torch_worker_threads"): + if key in config["data"] and int(config["data"][key]) <= 0: + raise ValueError(f"data.{key} must be positive") + performance = config.get("performance", {}) + if bool(performance.get("omit_all_true_attention_mask", False)): + _require(config, "model.cross_modal_attention", "bidirectional") + _require(config, "model.dit.video_attention_mask_mode", "bidirectional") + if str(performance.get("rope_compute_dtype", "float64")) not in { + "float32", "float64" + }: + raise ValueError("performance.rope_compute_dtype must be float32 or float64") + for key in ( + "schedule_pad_to_gradient_accumulation", + "optimized_ddp", + "ddp_static_graph", + "scheduler_one_step_per_optimizer", + "cache_derived_modalities", + "prefetch_next_episode", + "omit_all_true_attention_mask", + "vae_batch_modalities_verified", + "profile_timing", + "aggregate_accumulation_metrics", + ): + if key in performance and not isinstance(performance[key], bool): + raise ValueError(f"performance.{key} must be a boolean") + if bool(performance.get("ddp_static_graph", False)) and int( + config["training"]["gradient_accumulation_steps"] + ) > 1: + raise ValueError( + "performance.ddp_static_graph is incompatible with DDP no_sync " + "gradient accumulation in the installed PyTorch 2.5.1 runtime" + ) + if config["training"].get("max_coverage_epochs") is not None and not bool( + performance.get("schedule_pad_to_gradient_accumulation", False) + ): + raise ValueError( + "Finite coverage-epoch training requires " + "performance.schedule_pad_to_gradient_accumulation=true" + ) + _require(config, "cache.prompt_padding_mode", "full_length_512") + return config + + +def _cache_fingerprint(config: dict[str, Any], episodes: list[Any]) -> str: + payload = { + "format": "spacedreamer_ego_episode_first_absolute_v2", + "episodes": [(episode.episode_id, episode.length) for episode in episodes], + "split_seed": int(config["data"]["split_seed"]), + "validation_fraction": float(config["data"]["validation_fraction"]), + "normalizer_windows_per_episode": 1, + "target": config["action"]["target_type"], + "camera": config["camera"]["representation"], + "point": config["point_cloud"]["representation"], + } + return hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + +def _atomic_torch_save(payload: Any, path: Path) -> None: + temporary = path.with_name(f"{path.name}.tmp.{os.getpid()}") + torch.save(payload, temporary) + os.replace(temporary, path) + + +def _prepare_prompt_cache( + config: dict[str, Any], + episodes: list[Any], + device: torch.device, + *, + process_index: int, + num_processes: int, +) -> None: + """Encode unchanged global prompt batches across ranks into one cache.""" + + if not 0 <= process_index < num_processes: + raise ValueError("Invalid distributed prompt-cache rank") + cache_dir = Path(config["paths"]["prompt_cache_dir"]) + cache_dir.mkdir(parents=True, exist_ok=True) + prompts = sorted({episode.prompt for episode in episodes}) + missing_box: list[Any] = [None] + if process_index == 0: + cached_keys = {path.stem for path in cache_dir.glob("*.pt")} + missing_box[0] = [ + prompt + for prompt in prompts + if hashlib.sha256(prompt.encode("utf-8")).hexdigest() not in cached_keys + ] + broadcast_object_list(missing_box, from_process=0) + missing = missing_box[0] + if not isinstance(missing, list): + raise RuntimeError("Prompt-cache worklist broadcast failed") + batch_size = int(config["cache"]["prompt_batch_size"]) + global_batches = [ + missing[start : start + batch_size] + for start in range(0, len(missing), batch_size) + ] + local_batches = global_batches[process_index::num_processes] + print( + f"[prompt-cache rank={process_index}] total={len(prompts)} " + f"cached={len(prompts) - len(missing)} missing={len(missing)} " + f"local_batches={len(local_batches)}", + flush=True, + ) + if not local_batches: + return + dtype = dtype_from_name(str(config["model"]["dtype"])) + encoder, tokenizer = load_text_encoder_and_tokenizer( + Path(config["paths"]["prompt_pretrained_model_dir"]), + dtype, + device, + int(config["cache"]["tokenizer_max_length"]), + ) + for batch_number, texts in enumerate(local_batches, start=1): + ids, masks = tokenizer(texts, return_mask=True, add_special_tokens=True) + with torch.inference_mode(): + contexts = encoder( + ids.to(device), masks.to(device=device, dtype=torch.bool) + ) + for row, prompt in enumerate(texts): + mask = masks[row].to(dtype=torch.bool) + positions = mask.nonzero(as_tuple=False) + if positions.numel() == 0: + raise ValueError(f"Prompt produced no valid tokens: {prompt!r}") + length = int(positions[-1, 0]) + 1 + key = hashlib.sha256(prompt.encode("utf-8")).hexdigest() + _atomic_torch_save( + { + "prompt": prompt, + "context": contexts[row, :length].detach().cpu(), + "context_mask": mask[:length].cpu(), + }, + cache_dir / f"{key}.pt", + ) + if batch_number == 1 or batch_number % 25 == 0 or batch_number == len(local_batches): + print( + f"[prompt-cache rank={process_index}] " + f"batches={batch_number}/{len(local_batches)}", + flush=True, + ) + encoder.to("cpu") + del encoder, tokenizer + gc.collect() + if device.type == "cuda": + torch.cuda.empty_cache() + + +def _normalizer_statistics( + accelerator: Accelerator, + config: dict[str, Any], + train_episodes: list[Any], + fingerprint: str, +) -> dict[str, torch.Tensor]: + cache_dir = Path(config["paths"]["metadata_cache_dir"]) + stats_path = cache_dir / "episode_first_action_camera_normalizers.pt" + if stats_path.is_file(): + payload = torch.load(stats_path, map_location="cpu", weights_only=True) + if payload.get("fingerprint") != fingerprint: + raise RuntimeError(f"Normalizer fingerprint mismatch: {stats_path}") + return payload + + action_sum = torch.zeros(14, dtype=torch.float64) + action_square = torch.zeros(14, dtype=torch.float64) + camera_sum = torch.zeros(6, dtype=torch.float64) + camera_square = torch.zeros(6, dtype=torch.float64) + action_count = 0 + camera_count = 0 + seed = int(config["data"]["sample_order_seed"]) + 91_117 + local = train_episodes[accelerator.process_index :: accelerator.num_processes] + for number, episode in enumerate(local, start=1): + start = fixed_validation_start(episode.length, episode.episode_id, seed) + payload = load_motion_window_payload(episode, start) + targets = absolute_targets_in_episode_first_camera(payload, 1) + valid_action = ~targets["action_is_pad"] + action = targets["action_raw"][valid_action].double() + camera = targets["camera_raw"].double() + action_sum += action.sum(dim=0) + action_square += action.square().sum(dim=0) + camera_sum += camera.sum(dim=0) + camera_square += camera.square().sum(dim=0) + action_count += int(action.shape[0]) + camera_count += int(camera.shape[0]) + if number == 1 or number % 250 == 0 or number == len(local): + print( + f"[normalizer rank={accelerator.process_index}] " + f"{number}/{len(local)} episode={episode.episode_id}", + flush=True, + ) + packed = torch.cat( + [ + action_sum, + action_square, + camera_sum, + camera_square, + torch.tensor([action_count, camera_count], dtype=torch.float64), + ] + ).to(accelerator.device) + packed = accelerator.reduce(packed, reduction="sum").cpu() + if accelerator.is_main_process: + action_sum = packed[0:14] + action_square = packed[14:28] + camera_sum = packed[28:34] + camera_square = packed[34:40] + action_count = int(packed[40].item()) + camera_count = int(packed[41].item()) + if action_count < 2 or camera_count < 2: + raise RuntimeError("Insufficient valid action/camera rows for normalization") + eps = float(config["action"]["normalization"]["eps"]) + action_mean = action_sum / action_count + camera_mean = camera_sum / camera_count + action_std = ( + action_square / action_count - action_mean.square() + ).clamp(min=eps**2).sqrt() + camera_std = ( + camera_square / camera_count - camera_mean.square() + ).clamp(min=eps**2).sqrt() + _atomic_torch_save( + { + "fingerprint": fingerprint, + "sampling": "one deterministic uniformly-hashed window per train episode", + "action_mean": action_mean.float(), + "action_std": action_std.float(), + "action_count": torch.tensor(action_count, dtype=torch.long), + "camera_mean": camera_mean.float(), + "camera_std": camera_std.float(), + "camera_count": torch.tensor(camera_count, dtype=torch.long), + }, + stats_path, + ) + accelerator.wait_for_everyone() + return torch.load(stats_path, map_location="cpu", weights_only=True) + + +def prepare_metadata( + accelerator: Accelerator, config: dict[str, Any] +) -> tuple[list[Any], list[Any], dict[str, torch.Tensor]]: + if config["data"].get("format") in {"spacedreamer_canonical_v1", "spacedreamer_canonical_indexed_v1"}: + return _prepare_canonical_metadata(accelerator, config) + episode_box: list[Any] = [None] + if accelerator.is_main_process: + episode_box[0] = load_eligible_episodes( + Path(config["paths"]["episode_manifest"]), + Path(config["paths"]["dataset_root"]), + metadata_workers=int(config["data"]["metadata_workers"]), + ) + broadcast_object_list(episode_box, from_process=0) + episodes = episode_box[0] + if not isinstance(episodes, list) or not episodes: + raise RuntimeError("Episode metadata broadcast failed") + splits = split_episodes( + episodes, + validation_fraction=float(config["data"]["validation_fraction"]), + seed=int(config["data"]["split_seed"]), + ) + fingerprint = _cache_fingerprint(config, episodes) + cache_dir = Path(config["paths"]["metadata_cache_dir"]) + if accelerator.is_main_process: + cache_dir.mkdir(parents=True, exist_ok=True) + split_path = cache_dir / "episode_split.json" + record = { + "fingerprint": fingerprint, + "train_count": len(splits["train"]), + "validation_count": len(splits["validation"]), + "train_episode_ids": [episode.episode_id for episode in splits["train"]], + "validation_episode_ids": [episode.episode_id for episode in splits["validation"]], + } + if split_path.is_file(): + existing = json.loads(split_path.read_text(encoding="utf-8")) + if existing != record: + raise RuntimeError(f"Existing split does not match config: {split_path}") + else: + temporary = split_path.with_name(f"{split_path.name}.tmp.{os.getpid()}") + temporary.write_text(json.dumps(record, indent=2), encoding="utf-8") + os.replace(temporary, split_path) + accelerator.wait_for_everyone() + _prepare_prompt_cache( + config, + episodes, + accelerator.device, + process_index=accelerator.process_index, + num_processes=accelerator.num_processes, + ) + accelerator.wait_for_everyone() + if accelerator.is_main_process: + expected_prompt_keys = { + hashlib.sha256(episode.prompt.encode("utf-8")).hexdigest() + for episode in episodes + } + prompt_dir = Path(config["paths"]["prompt_cache_dir"]) + cached_prompt_keys = {path.stem for path in prompt_dir.glob("*.pt")} + missing_prompt_keys = expected_prompt_keys - cached_prompt_keys + if missing_prompt_keys: + raise RuntimeError( + f"Prompt cache is incomplete: missing={len(missing_prompt_keys)}" + ) + print( + f"[prompt-cache] complete embeddings={len(expected_prompt_keys)}", + flush=True, + ) + accelerator.wait_for_everyone() + stats = _normalizer_statistics( + accelerator, config, splits["train"], fingerprint + ) + return splits["train"], splits["validation"], stats + + +def _prepare_canonical_metadata( + accelerator: Accelerator, config: dict[str, Any] +) -> tuple[list[Any], list[Any], dict[str, torch.Tensor]]: + """Load an independently validated canonical dataset without source adapters.""" + payload_box: list[Any] = [None] + if accelerator.is_main_process: + root = Path(config["paths"]["dataset_root"]) + ready_path = root / "reports/ready.json" + if not ready_path.is_file(): + raise FileNotFoundError(f"Missing canonical ready gate: {ready_path}") + ready = json.loads(ready_path.read_text(encoding="utf-8")) + if ready.get("ready") is not True or ready.get("full_payload_validation") is not True: + raise RuntimeError(f"Canonical dataset has not passed full validation: {ready_path}") + all_manifest = root / "manifests/all.jsonl" + if not all_manifest.is_file(): + raise FileNotFoundError(all_manifest) + actual_manifest_sha256 = hashlib.sha256(all_manifest.read_bytes()).hexdigest() + if ready.get("manifest_sha256") != actual_manifest_sha256: + raise RuntimeError( + "Canonical ready gate does not match manifests/all.jsonl: " + f"ready={ready.get('manifest_sha256')} actual={actual_manifest_sha256}" + ) + for key, ready_key in ( + ("episode_manifest", "train_manifest_sha256"), + ("validation_manifest", "validation_manifest_sha256"), + ): + configured_manifest = Path(config["paths"][key]) + actual = hashlib.sha256(configured_manifest.read_bytes()).hexdigest() + if ready.get(ready_key) != actual: + raise RuntimeError( + f"Canonical ready gate does not match paths.{key}: " + f"ready={ready.get(ready_key)} actual={actual}" + ) + manifest_loader = ( + load_indexed_canonical_manifest + if config["data"].get("format") == "spacedreamer_canonical_indexed_v1" + else load_canonical_manifest + ) + train_episodes = manifest_loader(Path(config["paths"]["episode_manifest"])) + validation_episodes = manifest_loader(Path(config["paths"]["validation_manifest"])) + payload_box[0] = ( + train_episodes, validation_episodes, + str(ready.get("normalizer_fingerprint", "")), + ) + broadcast_object_list(payload_box, from_process=0) + train_episodes, validation_episodes, ready_normalizer_fingerprint = payload_box[0] + episodes = [*train_episodes, *validation_episodes] + _prepare_prompt_cache( + config, episodes, accelerator.device, + process_index=accelerator.process_index, + num_processes=accelerator.num_processes, + ) + accelerator.wait_for_everyone() + stats_path = Path(config["paths"]["canonical_normalizer"]) + if not stats_path.is_file(): + raise FileNotFoundError(stats_path) + stats = torch.load(stats_path, map_location="cpu", weights_only=True) + required = {"action_mean", "action_std", "action_count", "camera_mean", "camera_std", "camera_count", "fingerprint"} + missing = required - set(stats) + if missing: + raise RuntimeError(f"Canonical normalizer is incomplete: missing={sorted(missing)}") + if stats["fingerprint"] != ready_normalizer_fingerprint: + raise RuntimeError("Canonical normalizer fingerprint does not match ready gate") + return train_episodes, validation_episodes, stats + + +def _restore_normalizers(model: torch.nn.Module, stats: dict[str, torch.Tensor]) -> None: + pairs = ( + (model.action_normalizer, "action"), + (model.camera_normalizer, "camera"), + ) + for normalizer, prefix in pairs: + normalizer.mean.copy_(stats[f"{prefix}_mean"]) + normalizer.std.copy_(stats[f"{prefix}_std"]) + normalizer.count.copy_(stats[f"{prefix}_count"]) + normalizer.fitted.fill_(True) + + +def _configure_rope_cache( + model: torch.nn.Module, compute_dtype: torch.dtype, device: torch.device +) -> None: + if compute_dtype == torch.float64: + return + if compute_dtype != torch.float32: + raise ValueError(f"Unsupported RoPE compute dtype: {compute_dtype}") + model.video_expert.freqs = tuple( + value.to(device=device, dtype=torch.complex64) + for value in model.video_expert.freqs + ) + model.action_expert.freqs = model.action_expert.freqs.to( + device=device, dtype=torch.complex64 + ) + + +def _assert_fp32_optimizer_state( + model: torch.nn.Module, optimizer: torch.optim.Optimizer +) -> None: + non_fp32_parameters = [ + name + for name, parameter in model.named_parameters() + if parameter.requires_grad + and parameter.is_floating_point() + and parameter.dtype != torch.float32 + ] + if non_fp32_parameters: + raise RuntimeError( + f"Trainable parameters are not FP32: {non_fp32_parameters[:8]}" + ) + non_fp32_states: list[str] = [] + for state in optimizer.state.values(): + for name in ("exp_avg", "exp_avg_sq"): + value = state.get(name) + if torch.is_tensor(value) and value.dtype != torch.float32: + non_fp32_states.append(f"{name}:{value.dtype}") + if non_fp32_states: + raise RuntimeError(f"AdamW moment states are not FP32: {non_fp32_states[:8]}") + + +def assert_runtime_architecture(model: torch.nn.Module) -> None: + if not isinstance(model.mot, ForwardOnBackwardOnTwoCallMoT): + raise RuntimeError( + f"Expected ForwardOnBackwardOnTwoCallMoT, got {type(model.mot)}" + ) + if model.cross_modal_attention != "bidirectional": + raise RuntimeError("Cross-modal attention is not bidirectional") + if not model.mot.video_reads_action or model.mot.detach_video_for_action: + raise RuntimeError("Expected fully bidirectional MoT without stop-gradient") + if model.video_expert.video_attention_mask_mode != "bidirectional": + raise RuntimeError("Video temporal attention is not bidirectional") + if bool(model.video_expert.use_gradient_checkpointing): + raise RuntimeError("Video gradient checkpointing must be disabled") + if bool(model.action_expert.use_gradient_checkpointing): + raise RuntimeError("Action gradient checkpointing must be disabled") + if bool(model.mot.mot_checkpoint_mixed_attn): + raise RuntimeError("MoT mixed-attention checkpointing must be disabled") + if int(model.video_expert.point_first_frame_timestep_adapter_bottleneck_dim) != 256: + raise RuntimeError("Point adapter bottleneck must be 256") + mask = model.build_shared_attention_mask( + video_seq_len=4, + action_seq_len=4, + video_tokens_per_frame=2, + device=torch.device("cpu"), + ) + if mask.shape != (8, 8) or not bool(mask.all()): + raise RuntimeError("Runtime attention mask is not fully bidirectional") + metadata = model.architecture_metadata() + if metadata["action_loss_to_video_gradient"] != "enabled_through_video_kv": + raise RuntimeError("Runtime metadata does not confirm full gradient coupling") + + +def _load_dynamic_vae(config: dict[str, Any], device: torch.device, dtype: torch.dtype) -> Any: + global _DYNAMIC_VAE + if _DYNAMIC_VAE is None: + vae_model_dir = Path( + os.environ.get( + "WAN_VAE_MODEL_DIR", + config["paths"]["pretrained_model_dir"], + ) + ) + _DYNAMIC_VAE = load_vae( + vae_model_dir, dtype, device + ) + _DYNAMIC_VAE.eval().requires_grad_(False) + return _DYNAMIC_VAE + + +def _encode_video_modalities( + batch: dict[str, Any], config: dict[str, Any], device: torch.device, dtype: torch.dtype +) -> dict[str, Any]: + global _VAE_BATCHED_MODALITIES_OK + videos = [batch[key].to(device=device, dtype=dtype, non_blocking=True) for key in ("rgb", "depth", "point")] + batch_size = int(videos[0].shape[0]) + expected_video = (3, 33, 256, 256) + if any(tuple(video.shape[1:]) != expected_video for video in videos): + raise RuntimeError(f"Unexpected VAE video shapes: {[tuple(v.shape) for v in videos]}") + vae = _load_dynamic_vae(config, device, dtype) + with torch.inference_mode(): + if _VAE_BATCHED_MODALITIES_OK is False: + chunks = [vae.encode(video, device=device) for video in videos] + else: + batched = vae.encode(torch.cat(videos, dim=0), device=device) + chunks = list(batched.split(batch_size, dim=0)) + if _VAE_BATCHED_MODALITIES_OK is None: + sequential = [vae.encode(video, device=device) for video in videos] + _VAE_BATCHED_MODALITIES_OK = all( + torch.allclose(a.float(), b.float(), rtol=3.0e-4, atol=3.0e-4) + for a, b in zip(chunks, sequential, strict=True) + ) + if not _VAE_BATCHED_MODALITIES_OK: + chunks = sequential + print( + f"[vae] batched_modalities_verified={_VAE_BATCHED_MODALITIES_OK}", + flush=True, + ) + expected_latent = (48, 9, 16, 16) + if any(tuple(latent.shape[1:]) != expected_latent for latent in chunks): + raise RuntimeError(f"Unexpected VAE latent shapes: {[tuple(z.shape) for z in chunks]}") + encoded = dict(batch) + for key, latent in zip(("z_rgb", "z_depth", "z_point"), chunks, strict=True): + encoded[key] = latent + for key in ("rgb", "depth", "point"): + del encoded[key] + return encoded + + +def _compute_loss( + model: torch.nn.Module, + video_scheduler: Any, + action_scheduler: Any, + batch: dict[str, Any], + accelerator: Accelerator, + config: dict[str, Any], + dtype: torch.dtype, +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + if int(batch["sample_weight"].numel()) != 1: + raise RuntimeError("Exact 1/L weighting currently requires batch_size=1 per GPU") + encoded = _encode_video_modalities(batch, config, accelerator.device, dtype) + loss, metrics = compute_training_loss( + model, + video_scheduler, + action_scheduler, + encoded, + device=accelerator.device, + dtype=dtype, + config=config, + ) + weight = batch["sample_weight"].to(accelerator.device, dtype=torch.float32).reshape(()) + weighted = loss * weight + output = { + key: (value * weight if key.startswith("loss_") else value) + for key, value in metrics.items() + } + output["sample_weight"] = weight.detach() + output["point_clipped_fraction"] = ( + batch["point_clipped_fraction"].to(accelerator.device).float().mean() + ) + output["action_valid_fraction"] = ( + (~batch["action_is_pad"]).to(accelerator.device).float().mean() + ) + return weighted, output + + +@torch.no_grad() +def run_validation_loss( + model: torch.nn.Module, + dataloader: DataLoader, + accelerator: Accelerator, + config: dict[str, Any], + dtype: torch.dtype, + step: int, +) -> dict[str, float]: + global _VALIDATION_LATENT_CACHE + model.eval() + video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=float(config["diffusion"]["train_shift"]), + ) + action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["action_num_train_timesteps"]), + shift=float(config["diffusion"]["action_train_shift"]), + ) + keys = ( + "loss_rgb", "loss_depth", "loss_point", "loss_action", + "loss_camera_translation", "loss_camera_rotation", + ) + totals = torch.zeros(len(keys) + 2, device=accelerator.device, dtype=torch.float64) + if _VALIDATION_LATENT_CACHE is None: + cached_batches: list[dict[str, Any]] = [] + live_batches: list[dict[str, Any]] = [] + for batch in dataloader: + encoded = _encode_video_modalities(batch, config, accelerator.device, dtype) + live_batches.append(encoded) + cached_batches.append({ + key: (value.detach().cpu() if torch.is_tensor(value) else value) + for key, value in encoded.items() + }) + _VALIDATION_LATENT_CACHE = cached_batches + validation_batches = live_batches + print(f"[validation-cache] rank={accelerator.process_index} batches={len(cached_batches)}", flush=True) + else: + validation_batches = _VALIDATION_LATENT_CACHE + for index, encoded in enumerate(validation_batches): + valid = float((~encoded["is_padding"]).item()) + devices = [accelerator.device.index] if accelerator.device.type == "cuda" else [] + with torch.random.fork_rng(devices=devices): + seed = int(config["validation"]["fixed_noise_seed"]) + accelerator.process_index * 1_000_003 + index + torch.manual_seed(seed) + if accelerator.device.type == "cuda": + torch.cuda.manual_seed(seed) + loss, metrics = compute_training_loss( + model, video_scheduler, action_scheduler, encoded, + device=accelerator.device, dtype=dtype, config=config, + ) + totals[: len(keys)] += torch.stack([metrics[key].double() for key in keys]) * valid + totals[len(keys)] += loss.double() * valid + totals[len(keys) + 1] += valid + totals = accelerator.reduce(totals, reduction="sum") + count = max(float(totals[-1].item()), 1.0) + result = {f"validation_{key}": float(totals[i].item()) / count for i, key in enumerate(keys)} + result["validation_loss_total"] = float(totals[-2].item()) / count + result["validation_step"] = float(step) + model.train() + return result + + +def _ordered_data_loader( + dataset: Any, + config: dict[str, Any], +) -> DataLoader: + """Overlap one ordered CPU producer with GPU work without changing samples.""" + + num_workers = int(config["data"]["num_workers"]) + kwargs: dict[str, Any] = { + "dataset": dataset, + "batch_size": int(config["data"]["batch_size"]), + "shuffle": False, + "num_workers": num_workers, + "pin_memory": True, + "collate_fn": ( + canonical_collate + if isinstance(dataset, CanonicalSpaceDreamerDataset) + else collate_spacedreamer + ), + } + if num_workers > 0: + kwargs["prefetch_factor"] = int(config["data"]["prefetch_factor"]) + if "opencv_worker_threads" in config["data"]: + kwargs["worker_init_fn"] = partial( + configure_spacedreamer_worker, + opencv_threads=int(config["data"]["opencv_worker_threads"]), + torch_threads=int(config["data"].get("torch_worker_threads", 1)), + ) + # The loader is rebuilt at every coverage-epoch boundary. A + # non-persistent worker therefore always receives the new schedule. + kwargs["persistent_workers"] = False + return DataLoader(**kwargs) + + +def train( + config: dict[str, Any], + output_dir: Path, + prepare_only: bool, + resume_model: Path | None = None, + resume_global_step: int = 0, + wandb_log_after_step: int = 0, +) -> None: + tracking = bool(config.get("tracking", {}).get("enabled", False)) + if tracking and importlib.util.find_spec("wandb") is None: + raise RuntimeError("tracking.enabled=true but wandb is unavailable") + performance = config.get("performance", {}) + schedule_pad_multiple = ( + int(config["training"]["gradient_accumulation_steps"]) + if bool(performance.get("schedule_pad_to_gradient_accumulation", False)) + else 1 + ) + kwargs_handlers: list[Any] = [ + InitProcessGroupKwargs(timeout=timedelta(seconds=3600)) + ] + if bool(performance.get("optimized_ddp", False)): + kwargs_handlers.append( + DistributedDataParallelKwargs( + broadcast_buffers=False, + gradient_as_bucket_view=True, + static_graph=bool(performance.get("ddp_static_graph", False)), + bucket_cap_mb=int(performance.get("ddp_bucket_cap_mb", 128)), + find_unused_parameters=False, + ) + ) + accelerator = Accelerator( + gradient_accumulation_steps=int(config["training"]["gradient_accumulation_steps"]), + mixed_precision="bf16", + log_with="wandb" if tracking else None, + project_dir=str(output_dir), + step_scheduler_with_optimizer=not bool( + performance.get("scheduler_one_step_per_optimizer", False) + ), + kwargs_handlers=kwargs_handlers, + ) + expected_processes = int(config["training"]["expected_num_processes"]) + if accelerator.num_processes != expected_processes: + raise RuntimeError( + f"Expected {expected_processes} processes, got {accelerator.num_processes}" + ) + set_seed(int(config["experiment"]["seed"]), device_specific=True) + if accelerator.is_main_process: + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "resolved_config.yaml").write_text( + yaml.safe_dump(config, sort_keys=False), encoding="utf-8" + ) + accelerator.wait_for_everyone() + train_episodes, validation_episodes, stats = prepare_metadata(accelerator, config) + if prepare_only: + if accelerator.is_main_process: + print("[metadata] preparation complete", flush=True) + return + + model = build_joint_action_mot( + _factory_compatible_config(config), + resume_model=resume_model, + load_pretrained=resume_model is None, + ) + _restore_normalizers(model, stats) + assert_runtime_architecture(model) + model.mot.omit_all_true_attention_mask = bool( + performance.get("omit_all_true_attention_mask", False) + ) + model.mot.rope_compute_dtype = { + "float32": torch.float32, + "float64": torch.float64, + }[str(performance.get("rope_compute_dtype", "float64"))] + _configure_rope_cache(model, model.mot.rope_compute_dtype, accelerator.device) + global _VAE_BATCHED_MODALITIES_OK + if bool(performance.get("vae_batch_modalities_verified", False)): + _VAE_BATCHED_MODALITIES_OK = True + if accelerator.is_main_process: + print("[vae] batched_modalities pre-verified by configuration", flush=True) + enable_fp32_master_weights(model) + configure_trainable_parameters(model, config, "full_dit") + masks = ProjectionGradientMasks( + model.video_expert, + phase="full_dit", + head_warmup_steps=int(config["training"]["full_dit"].get("head_warmup_steps", 0)), + preserve_original_slices=bool(config["training"]["preserve_original_projection_slices"]), + ) + optimizer = build_optimizer(model, config, "full_dit") + optimizer_groups = {str(group.get("name")): group for group in optimizer.param_groups} + if "video_point_timestep_adapter" not in optimizer_groups: + raise RuntimeError("Point adapter is missing from optimizer parameter groups") + adapter_ids = { + id(parameter) + for parameter in model.video_expert.point_first_frame_timestep_adapter.parameters() + if parameter.requires_grad + } + optimized_adapter_ids = { + id(parameter) + for parameter in optimizer_groups["video_point_timestep_adapter"]["params"] + } + if not adapter_ids or adapter_ids != optimized_adapter_ids: + raise RuntimeError("Point adapter optimizer coverage is incomplete or duplicated") + max_steps = int(config["training"]["full_dit"]["max_steps"]) + lr_scheduler = build_lr_scheduler( + optimizer, + max_steps, + int(config["training"]["full_dit"]["warmup_steps"]), + schedule_type=str(config["training"]["full_dit"]["lr_schedule"]), + ) + if resume_global_step: + # The durable SpaceDreamer export does not contain optimizer/scheduler + # state. Re-create the exact LR position explicitly instead of silently + # repeating the 500-step warmup after an epoch-boundary resume. + resumed_lrs = [ + float(base_lr) * float(lr_lambda(resume_global_step)) + for base_lr, lr_lambda in zip( + lr_scheduler.base_lrs, lr_scheduler.lr_lambdas, strict=True + ) + ] + lr_scheduler.last_epoch = resume_global_step + lr_scheduler._step_count = resume_global_step + 1 + lr_scheduler._last_lr = resumed_lrs + for group, learning_rate in zip( + optimizer.param_groups, resumed_lrs, strict=True + ): + group["lr"] = learning_rate + model, optimizer, lr_scheduler = accelerator.prepare(model, optimizer, lr_scheduler) + masks.set_step(resume_global_step) + initialize_experiment_trackers(accelerator, config, "full_dit") + + clip = config["action"]["normalization"].get("clip") + dataset_kwargs = { + "resolution": int(config["data"]["resolution"]), + "prompt_cache_dir": Path(config["paths"]["prompt_cache_dir"]), + "action_mean": stats["action_mean"], "action_std": stats["action_std"], + "camera_mean": stats["camera_mean"], "camera_std": stats["camera_std"], + "normalization_clip": clip, + "depth_config": config["data"]["depth"], + "point_config": config["point_cloud"], + "cache_derived_modalities": bool( + performance.get("cache_derived_modalities", False) + ), + "prefetch_next_episode": bool( + performance.get("prefetch_next_episode", False) + ), + } + world_size, rank = accelerator.num_processes, accelerator.process_index + canonical_data = config["data"].get("format") in { + "spacedreamer_canonical_v1", "spacedreamer_canonical_indexed_v1" + } + schedule_builder = build_balanced_rank_schedules if canonical_data else build_rank_schedules + dataset_class = ( + IndexedCanonicalSpaceDreamerDataset + if config["data"].get("format") == "spacedreamer_canonical_indexed_v1" + else CanonicalSpaceDreamerDataset if canonical_data else SpaceDreamerEgoDataset + ) + if canonical_data: + dataset_kwargs["rgb_cache_dir"] = ( + Path(config["paths"]["rgb_cache_dir"]) + if config["paths"].get("rgb_cache_dir") else None + ) + train_schedules = schedule_builder( + train_episodes, world_size=world_size, epoch=0, + seed=int(config["data"]["sample_order_seed"]), + pad_to_multiple=schedule_pad_multiple, + ) + train_schedule = train_schedules[rank] + schedule_slots_per_rank = len(train_schedule) + train_dataset = dataset_class( + train_episodes, train_schedule, **dataset_kwargs + ) + # Every process deterministically constructs the same balanced assignment, + # but a rank only consumes its own list. Do not retain the other 31 lists. + del train_schedules, train_schedule + validation_limit = min(int(config["validation"]["periodic_num_episodes"]), len(validation_episodes)) + if canonical_data: + by_source = { + source: [episode for episode in validation_episodes if episode.source == source] + for source in sorted({episode.source for episode in validation_episodes}) + } + validation_subset = [] + for offset in range(max(map(len, by_source.values()))): + for source in by_source: + if offset < len(by_source[source]): + validation_subset.append(by_source[source][offset]) + if len(validation_subset) == validation_limit: + break + if len(validation_subset) == validation_limit: + break + else: + validation_subset = validation_episodes[:validation_limit] + validation_schedules = build_validation_schedules( + validation_subset, world_size=world_size, + seed=int(config["validation"]["fixed_window_seed"]), + ) + validation_dataset = dataset_class( + validation_subset, + validation_schedules[rank], + **{ + **dataset_kwargs, + # Validation uses only one fixed window per episode. Building and + # retaining every frame's derived point/depth tensors would add + # work without any reuse. + "cache_derived_modalities": False, + "prefetch_next_episode": False, + }, + ) + del validation_schedules + validation_loader = _ordered_data_loader(validation_dataset, config) + + video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["num_train_timesteps"]), + shift=float(config["diffusion"]["train_shift"]), + ) + action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=int(config["diffusion"]["action_num_train_timesteps"]), + shift=float(config["diffusion"]["action_train_shift"]), + ) + dtype = dtype_from_name(str(config["model"]["dtype"])) + global_step = resume_global_step + initial_global_step = resume_global_step + coverage_epoch = 0 + max_coverage_epochs = config["training"].get("max_coverage_epochs") + if max_coverage_epochs is not None: + max_coverage_epochs = int(max_coverage_epochs) + steps_per_epoch = schedule_slots_per_rank // int( + config["training"]["gradient_accumulation_steps"] + ) + planned_steps = steps_per_epoch * max_coverage_epochs + if max_steps != planned_steps: + raise RuntimeError( + f"Configured max_steps={max_steps} but the exact coverage plan " + f"requires {planned_steps} ({steps_per_epoch} x {max_coverage_epochs})" + ) + if accelerator.is_main_process: + print( + f"[coverage-plan] epochs={max_coverage_epochs} " + f"schedule_slots_per_rank={schedule_slots_per_rank} " + f"optimizer_steps_per_epoch={steps_per_epoch} " + f"planned_optimizer_steps={planned_steps}", + flush=True, + ) + if resume_global_step: + if resume_global_step % steps_per_epoch: + raise ValueError( + "SpaceDreamer resume is only supported at an exact coverage " + f"epoch boundary: step={resume_global_step}, " + f"steps_per_epoch={steps_per_epoch}" + ) + coverage_epoch = resume_global_step // steps_per_epoch + if coverage_epoch >= max_coverage_epochs: + raise ValueError( + f"Resume step {resume_global_step} is already at or beyond " + f"the configured {max_coverage_epochs} coverage epochs" + ) + if accelerator.is_main_process: + print( + f"[resume] model={resume_model} global_step={global_step} " + f"next_coverage_epoch={coverage_epoch}", + flush=True, + ) + elif resume_global_step: + raise ValueError("Resume requires finite coverage-epoch training") + last_exported: int | None = resume_global_step or None + log_path = output_dir / ( + f"metrics_resume_{resume_global_step:06d}.jsonl" + if resume_global_step + else "metrics.jsonl" + ) + optimizer.zero_grad(set_to_none=True) + model.train() + performance_interval_start = time.perf_counter() + performance_interval_steps = 0 + data_wait_seconds = 0.0 + previous_iteration_end = performance_interval_start + aggregate_accumulation_metrics = bool( + performance.get("aggregate_accumulation_metrics", False) + ) + accumulated_metrics: dict[str, torch.Tensor] = {} + accumulated_total_loss: torch.Tensor | None = None + accumulation_microsteps = 0 + accumulated_source_counts = {"Egoverse": 0, "AgiBotWorld": 0} + while global_step < max_steps and ( + max_coverage_epochs is None or coverage_epoch < max_coverage_epochs + ): + if coverage_epoch > 0: + schedules = schedule_builder( + train_episodes, world_size=world_size, epoch=coverage_epoch, + seed=int(config["data"]["sample_order_seed"]), + pad_to_multiple=schedule_pad_multiple, + ) + train_dataset.set_schedule(schedules[rank]) + del schedules + train_loader = _ordered_data_loader(train_dataset, config) + coverage_epoch += 1 + for batch in train_loader: + data_wait_seconds += time.perf_counter() - previous_iteration_end + gradient_norm: torch.Tensor | None = None + if canonical_data: + batch_sources = list(batch.get("source", [])) + unexpected = set(batch_sources) - set(accumulated_source_counts) + if unexpected: + raise RuntimeError(f"Unexpected canonical sources: {sorted(unexpected)}") + for source in batch_sources: + accumulated_source_counts[source] += 1 + with accelerator.accumulate(model): + total_loss, metrics = _compute_loss( + model, video_scheduler, action_scheduler, batch, + accelerator, config, dtype, + ) + if aggregate_accumulation_metrics: + for key, value in metrics.items(): + detached = value.detach().float() + accumulated_metrics[key] = ( + accumulated_metrics.get(key, torch.zeros_like(detached)) + + detached + ) + detached_loss = total_loss.detach().float() + accumulated_total_loss = ( + detached_loss + if accumulated_total_loss is None + else accumulated_total_loss + detached_loss + ) + accumulation_microsteps += 1 + accelerator.backward(total_loss) + if accelerator.sync_gradients: + gradient_norm = accelerator.clip_grad_norm_( + [p for p in model.parameters() if p.requires_grad], + float(config["training"]["max_grad_norm"]), + ) + optimizer.step() + if ( + not bool(performance.get("scheduler_one_step_per_optimizer", False)) + or accelerator.sync_gradients + ): + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + if not accelerator.sync_gradients: + previous_iteration_end = time.perf_counter() + continue + if aggregate_accumulation_metrics: + expected_microsteps = int( + config["training"]["gradient_accumulation_steps"] + ) + if accumulation_microsteps != expected_microsteps: + raise RuntimeError( + "An optimizer step did not contain the configured number " + f"of micro-batches: expected={expected_microsteps} " + f"actual={accumulation_microsteps}" + ) + metrics_for_log = { + key: value / accumulation_microsteps + for key, value in accumulated_metrics.items() + } + if accumulated_total_loss is None: + raise RuntimeError("Missing accumulated loss at optimizer step") + total_loss_for_log = accumulated_total_loss / accumulation_microsteps + if gradient_norm is None: + raise RuntimeError("Missing gradient norm at optimizer step") + metrics_for_log["gradient_norm_before_clip"] = ( + gradient_norm.detach().float() + ) + accumulated_metrics = {} + accumulated_total_loss = None + accumulation_microsteps = 0 + else: + metrics_for_log = metrics + total_loss_for_log = total_loss.detach().float() + if canonical_data: + local_counts = torch.tensor( + [accumulated_source_counts["Egoverse"], accumulated_source_counts["AgiBotWorld"]], + dtype=torch.float32, device=accelerator.device, + ) + global_counts = accelerator.reduce(local_counts, reduction="sum") + expected_per_source = ( + int(config["training"]["gradient_accumulation_steps"]) + * world_size * int(config["data"]["batch_size"]) // 2 + ) + if tuple(int(value) for value in global_counts.tolist()) != ( + expected_per_source, expected_per_source + ): + raise RuntimeError( + "Canonical optimizer batch is not exact 1:1: " + f"Egoverse={int(global_counts[0])} " + f"AgiBotWorld={int(global_counts[1])} expected={expected_per_source}" + ) + metrics_for_log["source_fraction_egoverse"] = global_counts[0] / global_counts.sum() + metrics_for_log["source_fraction_agibot"] = global_counts[1] / global_counts.sum() + accumulated_source_counts = {"Egoverse": 0, "AgiBotWorld": 0} + global_step += 1 + performance_interval_steps += 1 + if global_step == initial_global_step + 1: + _assert_fp32_optimizer_state(model, optimizer) + masks.set_step(global_step) + should_log = global_step == 1 or global_step % int(config["training"]["log_every"]) == 0 + if should_log: + profile_timing = bool(performance.get("profile_timing", False)) + if profile_timing and accelerator.device.type == "cuda": + torch.cuda.synchronize(accelerator.device) + interval_seconds = time.perf_counter() - performance_interval_start + reduced = { + key: accelerator.reduce(value.float(), reduction="mean") + for key, value in metrics_for_log.items() + } + reduced["loss_total"] = accelerator.reduce( + total_loss_for_log.float(), reduction="mean" + ) + if profile_timing: + # Accelerate 0.15 only implements sum/mean/none. Passing + # "max" silently performs a sum, which under-reports + # throughput by exactly world_size. Gather explicitly and + # take the slowest rank instead. + interval_tensor = distributed_max_scalar( + accelerator, interval_seconds + ) + data_wait_tensor = accelerator.reduce( + torch.tensor(data_wait_seconds, device=accelerator.device), + reduction="mean", + ) + reduced["performance_interval_seconds"] = interval_tensor + reduced["performance_data_wait_seconds"] = data_wait_tensor + reduced["performance_windows_per_second"] = torch.tensor( + performance_interval_steps + * int(config["data"]["batch_size"]) + * world_size + * int(config["training"]["gradient_accumulation_steps"]) + / max(float(interval_tensor.item()), 1.0e-9), + device=accelerator.device, + ) + record = { + "phase": "full_dit", "coverage_epoch": coverage_epoch, + "global_step": global_step, + **{key: float(value.item()) for key, value in reduced.items()}, + "learning_rates": [float(group["lr"]) for group in optimizer.param_groups], + } + if tracking and global_step > wandb_log_after_step: + accelerator.log({f"train/{key}": value for key, value in record.items() if isinstance(value, (int, float))}, step=global_step) + if accelerator.is_main_process: + append_jsonl(log_path, record) + print(json.dumps(record, sort_keys=True), flush=True) + performance_interval_start = time.perf_counter() + performance_interval_steps = 0 + data_wait_seconds = 0.0 + + validate_now = global_step % int(config["training"]["validate_every"]) == 0 + if validate_now or global_step == max_steps: + validation = run_validation_loss( + model, validation_loader, accelerator, config, dtype, global_step + ) + if tracking and global_step > wandb_log_after_step: + accelerator.log({f"validation/{key}": value for key, value in validation.items()}, step=global_step) + if accelerator.is_main_process: + append_jsonl(log_path, validation) + print(json.dumps(validation, sort_keys=True), flush=True) + if bool(performance.get("profile_timing", False)): + # Exclude validation from the next training-throughput + # interval. + performance_interval_start = time.perf_counter() + performance_interval_steps = 0 + data_wait_seconds = 0.0 + periodic_export = global_step % int(config["training"]["export_model_every"]) == 0 + if periodic_export or global_step == max_steps: + export_action_mot( + model, accelerator, + inference_checkpoint_path(output_dir, "full_dit", global_step), + ) + last_exported = global_step + if global_step >= max_steps: + break + previous_iteration_end = time.perf_counter() + + epoch_boundary = max_coverage_epochs is not None + if epoch_boundary and global_step < max_steps: + validation = run_validation_loss( + model, validation_loader, accelerator, config, dtype, global_step + ) + if tracking and global_step > wandb_log_after_step: + accelerator.log( + {f"validation/{key}": value for key, value in validation.items()}, + step=global_step, + ) + if accelerator.is_main_process: + append_jsonl(log_path, validation) + print(json.dumps(validation, sort_keys=True), flush=True) + if last_exported != global_step: + export_action_mot( + model, accelerator, + inference_checkpoint_path(output_dir, "full_dit", global_step), + ) + last_exported = global_step + performance_interval_start = time.perf_counter() + performance_interval_steps = 0 + data_wait_seconds = 0.0 + previous_iteration_end = performance_interval_start + if last_exported != global_step: + export_action_mot( + model, accelerator, + inference_checkpoint_path(output_dir, "full_dit", global_step), + ) + masks.close() + accelerator.end_training() + + +def main() -> None: + args = parse_args() + if args.resume_global_step < 0: + raise ValueError("--resume-global-step must be non-negative") + if (args.resume_model is None) != (args.resume_global_step == 0): + raise ValueError( + "--resume-model and a positive --resume-global-step must be provided together" + ) + if args.wandb_log_after_step < 0: + raise ValueError("--wandb-log-after-step must be non-negative") + if args.wandb_log_after_step and args.resume_model is None: + raise ValueError("--wandb-log-after-step is only valid with --resume-model") + if args.wandb_log_after_step and args.wandb_log_after_step < args.resume_global_step: + raise ValueError( + "--wandb-log-after-step cannot precede --resume-global-step" + ) + if args.resume_model is not None: + args.resume_model = args.resume_model.resolve() + if not args.resume_model.is_file(): + raise FileNotFoundError(args.resume_model) + config = load_spacedreamer_config(args.config) + if ( + ( + args.smoke_expected_num_processes is not None + or args.smoke_prompt_cache_dir is not None + ) + and args.smoke_max_steps is None + ): + raise ValueError( + "smoke overrides are only valid with --smoke-max-steps" + ) + if args.smoke_max_steps is not None: + if args.smoke_max_steps <= 0: + raise ValueError("--smoke-max-steps must be positive") + if ( + args.smoke_expected_num_processes is not None + and args.smoke_expected_num_processes <= 0 + ): + raise ValueError("--smoke-expected-num-processes must be positive") + config = copy.deepcopy(config) + config["training"].pop("max_coverage_epochs", None) + config["training"]["smoke_test"] = True + if args.smoke_expected_num_processes is not None: + config["training"]["expected_num_processes"] = int( + args.smoke_expected_num_processes + ) + if args.smoke_prompt_cache_dir is not None: + config["paths"]["prompt_cache_dir"] = str( + args.smoke_prompt_cache_dir.resolve() + ) + config["training"]["full_dit"]["max_steps"] = int(args.smoke_max_steps) + config["training"]["full_dit"]["warmup_steps"] = min( + 20, int(args.smoke_max_steps) + ) + config["training"]["validate_every"] = int(args.smoke_max_steps) + config["training"]["export_model_every"] = int(args.smoke_max_steps) + config["experiment"]["name"] = ( + f"{config['experiment']['name']}-smoke{args.smoke_max_steps}" + ) + config = validate_config(config) + output_dir = args.output_dir or Path(config["experiment"]["output_dir"]) + if args.validate_only: + if config["data"].get("format") in {"spacedreamer_canonical_v1", "spacedreamer_canonical_indexed_v1"}: + manifest_loader = ( + load_indexed_canonical_manifest + if config["data"].get("format") == "spacedreamer_canonical_indexed_v1" + else load_canonical_manifest + ) + splits = { + "train": manifest_loader(Path(config["paths"]["episode_manifest"])), + "validation": manifest_loader(Path(config["paths"]["validation_manifest"])), + } + episodes = [*splits["train"], *splits["validation"]] + else: + episodes = load_eligible_episodes( + Path(config["paths"]["episode_manifest"]), + Path(config["paths"]["dataset_root"]), + metadata_workers=int(config["data"]["metadata_workers"]), + ) + splits = split_episodes( + episodes, + validation_fraction=float(config["data"]["validation_fraction"]), + seed=int(config["data"]["split_seed"]), + ) + print(json.dumps({ + "status": "valid", "output_dir": str(output_dir), + "frames": 33, "resolution": 256, "fps": 30, + "action_target": config["action"]["target_type"], + "attention": "full_bidirectional_forward_and_gradient", + "gradient_checkpointing": False, + "max_steps": int(config["training"]["full_dit"]["max_steps"]), + "eligible_episodes": len(episodes), + "train_episodes": len(splits["train"]), + "validation_episodes": len(splits["validation"]), + "valid_window_starts": sum(episode.length - 32 for episode in episodes), + "effective_global_batch": int(config["data"]["batch_size"]) * int(config["training"]["expected_num_processes"]) * int(config["training"]["gradient_accumulation_steps"]), + }, indent=2)) + return + train( + config, + output_dir, + args.prepare_metadata_only, + resume_model=args.resume_model, + resume_global_step=args.resume_global_step, + wandb_log_after_step=args.wandb_log_after_step, + ) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/wan_rgb_action_mot.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/wan_rgb_action_mot.py new file mode 100644 index 0000000000000000000000000000000000000000..375c308fc8ddfbd32839f51ea5f2c03d15015679 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/wan_rgb_action_mot.py @@ -0,0 +1,277 @@ +"""Isolated two-expert MoT for RGB video and bimanual action denoising. + +This sibling intentionally has no depth, point-map, or camera-token path. It +reuses the original Wan RGB expert and a randomly initialized ActionDiT while +leaving the production five-output Action-MoT implementation untouched. +""" + +from __future__ import annotations + +from contextlib import nullcontext +from copy import deepcopy +from pathlib import Path +import sys +from typing import Any, Mapping + +import torch +from torch import nn + + +PROJECT_ROOT = Path(__file__).resolve().parent +FASTWAM_SRC = PROJECT_ROOT.parent / "model_old" / "src" +if str(FASTWAM_SRC) not in sys.path: + sys.path.insert(0, str(FASTWAM_SRC)) + +from fastwam.models.wan22.action_dit import ActionDiT +from controlled_directional_mot import ForwardOnBackwardOnTwoCallMoT as MoT + + +ACTION_DIT_05B_CONFIG: dict[str, Any] = { + "action_dim": 14, + "hidden_dim": 576, + "ffn_dim": 2304, + "num_heads": 24, + "attn_head_dim": 128, + "num_layers": 30, + "text_dim": 4096, + "freq_dim": 256, + "eps": 1.0e-6, + "use_gradient_checkpointing": True, +} + + +def action_dit_05b_config(**overrides: Any) -> dict[str, Any]: + config = deepcopy(ACTION_DIT_05B_CONFIG) + config.update(overrides) + return config + + +def count_parameters(module: nn.Module) -> int: + return sum(parameter.numel() for parameter in module.parameters()) + + +class RgbActionMoT(nn.Module): + """Wan RGB expert plus one ActionDiT with layer-wise mixed attention.""" + + def __init__( + self, + video_expert: nn.Module, + action_expert: ActionDiT, + *, + action_state_dim: int = 14, + mot_checkpoint_mixed_attn: bool = True, + cross_modal_attention: str = "bidirectional", + ) -> None: + super().__init__() + if cross_modal_attention not in {"bidirectional", "action_reads_video"}: + raise ValueError( + "cross_modal_attention must be 'bidirectional' or 'action_reads_video'" + ) + if len(video_expert.blocks) != len(action_expert.blocks): + raise ValueError("Video and ActionDiT layer counts must match") + if int(video_expert.num_heads) != int(action_expert.num_heads): + raise ValueError("Video and ActionDiT head counts must match") + if int(video_expert.attn_head_dim) != int(action_expert.attn_head_dim): + raise ValueError("Video and ActionDiT head dimensions must match") + self.mot = MoT( + mixtures={"video": video_expert, "action": action_expert}, + mot_checkpoint_mixed_attn=mot_checkpoint_mixed_attn, + ) + self.cross_modal_attention = cross_modal_attention + self.action_state_dim = int(action_state_dim) + if self.action_state_dim <= 0: + raise ValueError("action_state_dim must be positive") + self.action_state_encoder = nn.Sequential( + nn.Linear(self.action_state_dim, action_expert.hidden_dim), + nn.SiLU(), + nn.Linear(action_expert.hidden_dim, action_expert.hidden_dim), + ) + + @property + def video_expert(self) -> nn.Module: + return self.mot.mixtures["video"] + + @property + def action_expert(self) -> ActionDiT: + return self.mot.mixtures["action"] + + @property + def action_parameter_count(self) -> int: + return count_parameters(self.action_expert) + + def architecture_metadata(self) -> dict[str, Any]: + video, action = self.video_expert, self.action_expert + return { + "format_version": 1, + "model_class": type(self).__name__, + "experts": ["rgb_video", "action"], + "outputs": ["rgb_video", "action"], + "conditioning": ["rgb_frame0", "text", "initial_state"], + "cross_modal_attention": self.cross_modal_attention, + "action_to_video_attention": bool(self.mot.video_reads_action), + "video_to_action_attention": True, + "action_loss_to_video_gradient": ( + "stopped_at_video_kv" + if self.mot.detach_video_for_action + else "enabled_through_video_kv" + ), + "num_layers": len(video.blocks), + "num_heads": int(video.num_heads), + "attn_head_dim": int(video.attn_head_dim), + "video_hidden_dim": int(video.hidden_dim), + "action_hidden_dim": int(action.hidden_dim), + "action_ffn_dim": int(action.ffn_dim), + "action_dim": int(action.action_dim), + "action_state_dim": self.action_state_dim, + "action_parameter_count": self.action_parameter_count, + "action_initialization": "random", + "rgb_latent_channels": int(video.in_dim), + } + + @torch.no_grad() + def build_shared_attention_mask( + self, + *, + video_seq_len: int, + action_seq_len: int, + video_tokens_per_frame: int, + device: torch.device, + ) -> torch.Tensor: + if video_seq_len <= 0 or action_seq_len <= 0: + raise ValueError("Video and action sequences must be non-empty") + total = video_seq_len + action_seq_len + mask = torch.zeros((total, total), dtype=torch.bool, device=device) + mask[:video_seq_len, :video_seq_len] = ( + self.video_expert.build_video_to_video_mask( + video_seq_len=video_seq_len, + video_tokens_per_frame=video_tokens_per_frame, + device=device, + ) + ) + mask[video_seq_len:, video_seq_len:] = True + mask[video_seq_len:, :video_seq_len] = True + if self.cross_modal_attention == "bidirectional": + mask[:video_seq_len, video_seq_len:] = True + return mask + + def forward( + self, + x: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + *, + action_tokens: torch.Tensor, + initial_state: torch.Tensor, + action_timestep: torch.Tensor | None = None, + context_mask: torch.Tensor | None = None, + fuse_vae_embedding_in_latents: bool = True, + ) -> dict[str, torch.Tensor]: + if action_timestep is None: + action_timestep = timestep + expected_state = (int(action_tokens.shape[0]), self.action_state_dim) + if tuple(initial_state.shape) != expected_state: + raise ValueError( + f"initial_state must have shape {expected_state}, got {tuple(initial_state.shape)}" + ) + video_pre = self.video_expert.pre_dit( + x=x, + timestep=timestep, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + action_pre = self.action_expert.pre_dit( + action_tokens=action_tokens, + timestep=action_timestep, + context=context, + context_mask=context_mask, + ) + state_context = self.action_state_encoder( + initial_state.to(device=action_tokens.device, dtype=action_tokens.dtype) + ).unsqueeze(1) + action_pre["context"] = torch.cat( + [action_pre["context"], state_context], dim=1 + ) + state_mask = torch.ones( + (*action_pre["context_mask"].shape[:-1], 1), + device=action_pre["context_mask"].device, + dtype=torch.bool, + ) + action_pre["context_mask"] = torch.cat( + [action_pre["context_mask"], state_mask], dim=-1 + ) + shared_mask = self.build_shared_attention_mask( + video_seq_len=int(video_pre["tokens"].shape[1]), + action_seq_len=int(action_pre["tokens"].shape[1]), + video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]), + device=video_pre["tokens"].device, + ) + tokens = self.mot( + embeds_all={"video": video_pre["tokens"], "action": action_pre["tokens"]}, + attention_mask=shared_mask, + freqs_all={"video": video_pre["freqs"], "action": action_pre["freqs"]}, + context_all={ + "video": { + "context": video_pre["context"], + "mask": video_pre["context_mask"], + }, + "action": { + "context": action_pre["context"], + "mask": action_pre["context_mask"], + }, + }, + t_mod_all={"video": video_pre["t_mod"], "action": action_pre["t_mod"]}, + ) + return { + "video": self.video_expert.post_dit(tokens["video"], video_pre), + "action": self.action_expert.post_dit(tokens["action"], action_pre), + } + + +def build_rgb_action_mot( + video_expert: nn.Module, + action_dit_config: Mapping[str, Any] | None = None, + *, + random_seed: int | None = None, + action_state_dim: int = 14, + mot_checkpoint_mixed_attn: bool = True, + cross_modal_attention: str = "bidirectional", +) -> RgbActionMoT: + config = ( + action_dit_05b_config() + if action_dit_config is None + else dict(action_dit_config) + ) + for name, actual in ( + ("num_layers", len(video_expert.blocks)), + ("num_heads", int(video_expert.num_heads)), + ("attn_head_dim", int(video_expert.attn_head_dim)), + ): + if int(config.get(name, -1)) != actual: + raise ValueError(f"action_dit.{name} must match video expert ({actual})") + reference = next(video_expert.parameters()) + device, dtype = reference.device, reference.dtype + devices = [device.index or 0] if device.type == "cuda" else [] + rng_context = ( + torch.random.fork_rng(devices=devices) + if random_seed is not None + else nullcontext() + ) + previous_dtype = torch.get_default_dtype() + try: + torch.set_default_dtype(dtype) + with rng_context, torch.device(device): + if random_seed is not None: + torch.manual_seed(int(random_seed)) + action_expert = ActionDiT(**config) + model = RgbActionMoT( + video_expert, + action_expert, + action_state_dim=action_state_dim, + mot_checkpoint_mixed_attn=mot_checkpoint_mixed_attn, + cross_modal_attention=cross_modal_attention, + ) + finally: + torch.set_default_dtype(previous_dtype) + return model diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/wan_rgb_depth_initialization.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/wan_rgb_depth_initialization.py new file mode 100644 index 0000000000000000000000000000000000000000..1dbb519e269825a7dc9799e58ce5bf7c4445a62c --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/wan_rgb_depth_initialization.py @@ -0,0 +1,692 @@ +"""Explicit weight migration helpers for the joint RGB-depth Wan model.""" + +from __future__ import annotations + +import math +from types import MethodType +from typing import Any + +import torch +import torch.nn.functional as F +from torch import nn + +def _sinusoidal_embedding_1d(dim: int, position: torch.Tensor) -> torch.Tensor: + sinusoid = torch.outer( + position.to(torch.float64), + torch.pow( + 10000, + -torch.arange( + dim // 2, + dtype=torch.float64, + device=position.device, + ).div(dim // 2), + ), + ) + embedding = torch.cat((torch.cos(sinusoid), torch.sin(sinusoid)), dim=1) + return embedding.to(position.dtype) + + +def _rope_apply_fp32( + x: torch.Tensor, + freqs: torch.Tensor, + num_heads: int, +) -> torch.Tensor: + """Apply RoPE in complex64 instead of the original complex128 path.""" + + input_dtype = x.dtype + batch, sequence, hidden = x.shape + if hidden % num_heads: + raise ValueError(f"Hidden size {hidden} is not divisible by {num_heads} heads") + head_dim = hidden // num_heads + if head_dim % 2: + raise ValueError(f"RoPE head dimension must be even, got {head_dim}") + x = x.reshape(batch, sequence, num_heads, head_dim) + x_complex = torch.view_as_complex( + x.float().reshape(batch, sequence, num_heads, head_dim // 2, 2) + ) + rotated = x_complex * freqs.to(device=x.device, dtype=torch.complex64) + return torch.view_as_real(rotated).flatten(2).to(input_dtype) + + +def _self_attention_with_fp32_rope( + self: nn.Module, + x: torch.Tensor, + freqs: torch.Tensor, + self_attn_mask: torch.Tensor | None = None, +) -> torch.Tensor: + q = _rope_apply_fp32(self.norm_q(self.q(x)), freqs, self.num_heads) + k = _rope_apply_fp32(self.norm_k(self.k(x)), freqs, self.num_heads) + v = self.v(x) + q = q.unflatten(-1, (self.num_heads, -1)).transpose(1, 2) + k = k.unflatten(-1, (self.num_heads, -1)).transpose(1, 2) + v = v.unflatten(-1, (self.num_heads, -1)).transpose(1, 2) + attended = F.scaled_dot_product_attention( + q, + k, + v, + attn_mask=self_attn_mask, + ) + attended = attended.transpose(1, 2).flatten(2) + return self.o(attended) + + +def _pre_dit_with_deduplicated_timesteps( + self: nn.Module, + x: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor | None = None, + action: torch.Tensor | None = None, + fuse_vae_embedding_in_latents: bool = False, + control_camera_latents_input: torch.Tensor | None = None, +) -> dict[str, Any]: + """Compute the two distinct timestep embeddings once per sample.""" + + if ( + self.action_conditioned + or action is not None + or not self.seperated_timestep + or not fuse_vae_embedding_in_latents + ): + return self._joint_rgb_depth_original_pre_dit( + x=x, + timestep=timestep, + context=context, + context_mask=context_mask, + action=action, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + control_camera_latents_input=control_camera_latents_input, + ) + + x, timestep, context_mask = self._validate_forward_inputs( + x=x, + timestep=timestep, + context=context, + context_mask=context_mask, + action=action, + ) + batch_size, _, latent_frames, latent_height, latent_width = x.shape + patch_h = int(self.patch_size[1]) + patch_w = int(self.patch_size[2]) + if latent_height % patch_h or latent_width % patch_w: + raise ValueError( + "Latent spatial shape must be divisible by DiT patch size, " + f"got HxW=({latent_height}, {latent_width}), " + f"patch=({patch_h}, {patch_w})" + ) + tokens_per_frame = (latent_height // patch_h) * (latent_width // patch_w) + video_sequence_length = latent_frames * tokens_per_frame + + distinct_timesteps = torch.stack( + (torch.zeros_like(timestep), timestep), + dim=1, + ) + distinct_t = self.time_embedding( + _sinusoidal_embedding_1d( + self.freq_dim, + distinct_timesteps.reshape(-1), + ) + ).reshape(batch_size, 2, self.hidden_dim) + distinct_t_mod = self.time_projection(distinct_t).unflatten( + 2, + (6, self.hidden_dim), + ) + future_tokens = video_sequence_length - tokens_per_frame + t = torch.cat( + ( + distinct_t[:, 0:1].expand(-1, tokens_per_frame, -1), + distinct_t[:, 1:2].expand(-1, future_tokens, -1), + ), + dim=1, + ) + t_mod = torch.cat( + ( + distinct_t_mod[:, 0:1].expand(-1, tokens_per_frame, -1, -1), + distinct_t_mod[:, 1:2].expand(-1, future_tokens, -1, -1), + ), + dim=1, + ) + + x = self.patchify( + x, + control_camera_latents_input=control_camera_latents_input, + ) + patch_frames, patch_height, patch_width = x.shape[2:] + context = self.text_embedding(context) + context_mask = context_mask.unsqueeze(1).expand( + -1, + patch_frames * patch_height * patch_width, + -1, + ) + x_tokens = x.flatten(2).transpose(1, 2).contiguous() + freqs = torch.cat( + [ + self.freqs[0][:patch_frames] + .view(patch_frames, 1, 1, -1) + .expand(patch_frames, patch_height, patch_width, -1), + self.freqs[1][:patch_height] + .view(1, patch_height, 1, -1) + .expand(patch_frames, patch_height, patch_width, -1), + self.freqs[2][:patch_width] + .view(1, 1, patch_width, -1) + .expand(patch_frames, patch_height, patch_width, -1), + ], + dim=-1, + ).reshape(patch_frames * patch_height * patch_width, 1, -1) + freqs = freqs.to(device=x_tokens.device, dtype=torch.complex64) + return { + "tokens": x_tokens, + "freqs": freqs, + "t": t, + "t_mod": t_mod, + "context": context, + "context_mask": context_mask, + "current_t_embedding": distinct_t[:, 1], + "meta": { + "grid_size": (patch_frames, patch_height, patch_width), + "tokens_per_frame": tokens_per_frame, + "batch_size": batch_size, + }, + } + + +def enable_joint_rgb_depth_runtime_optimizations(dit: nn.Module) -> nn.Module: + """Install local timestep-deduplication and fp32-RoPE fast paths.""" + + if hasattr(dit, "_joint_rgb_depth_original_pre_dit"): + return dit + dit._joint_rgb_depth_original_pre_dit = dit.pre_dit + dit.pre_dit = MethodType(_pre_dit_with_deduplicated_timesteps, dit) + for block in dit.blocks: + block.self_attn.forward = MethodType( + _self_attention_with_fp32_rope, + block.self_attn, + ) + dit.joint_rgb_depth_runtime_optimizations = True + return dit + + +class DepthFirstFrameTimestepAdapter(nn.Module): + """Timestep-conditioned FiLM scale for the first depth patch contribution.""" + + def __init__( + self, + hidden_dim: int, + bottleneck_dim: int, + *, + device: torch.device, + dtype: torch.dtype, + ) -> None: + super().__init__() + if bottleneck_dim <= 0: + raise ValueError( + f"bottleneck_dim must be positive, got {bottleneck_dim}" + ) + self.net = nn.Sequential( + nn.SiLU(), + nn.Linear( + hidden_dim, + bottleneck_dim, + device=device, + dtype=dtype, + ), + nn.SiLU(), + nn.Linear( + bottleneck_dim, + hidden_dim, + device=device, + dtype=dtype, + ), + ) + nn.init.zeros_(self.net[-1].weight) + nn.init.zeros_(self.net[-1].bias) + + def forward(self, timestep_embedding: torch.Tensor) -> torch.Tensor: + return self.net(timestep_embedding) + + +def _pre_dit_with_depth_first_frame_timestep_adapter( + self: nn.Module, + x: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor | None = None, + action: torch.Tensor | None = None, + fuse_vae_embedding_in_latents: bool = False, + control_camera_latents_input: torch.Tensor | None = None, +) -> dict[str, Any]: + """Preserve RGB t=0 while adding actual-t conditioning to depth frame zero.""" + + pre_state = self._depth_first_frame_original_pre_dit( + x=x, + timestep=timestep, + context=context, + context_mask=context_mask, + action=action, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + control_camera_latents_input=control_camera_latents_input, + ) + if not fuse_vae_embedding_in_latents: + raise ValueError( + "Depth first-frame timestep adapter requires " + "fuse_vae_embedding_in_latents=True" + ) + if x.ndim != 5 or x.shape[1] != self.in_dim: + raise ValueError( + f"Expected joint latent [B,{self.in_dim},T,H,W], got {tuple(x.shape)}" + ) + + batch_size = int(pre_state["meta"]["batch_size"]) + if x.shape[0] == 1 and batch_size > 1: + x = x.expand(batch_size, -1, -1, -1, -1) + if timestep.shape[0] == 1 and batch_size > 1: + timestep = timestep.expand(batch_size) + if timestep.shape[0] != batch_size: + raise ValueError( + f"Expected timestep batch {batch_size}, got {timestep.shape[0]}" + ) + + rgb_channels = int(self.rgb_latent_channels) + depth_channels = int(self.depth_latent_channels) + if rgb_channels + depth_channels != self.in_dim: + raise ValueError( + "RGB/depth channel metadata does not match joint DiT in_dim" + ) + temporal_patch = ( + int(self.patch_embedding.kernel_size[0]), + int(self.patch_embedding.stride[0]), + int(self.patch_embedding.padding[0]), + ) + if temporal_patch != (1, 1, 0): + raise ValueError( + "Depth first-frame adapter requires temporal " + f"kernel/stride/padding=(1,1,0), got {temporal_patch}" + ) + depth_weight = self.patch_embedding.weight[:, rgb_channels:] + depth_patches = F.conv3d( + x[:, rgb_channels:, 0:1], + depth_weight, + bias=None, + stride=self.patch_embedding.stride, + padding=self.patch_embedding.padding, + dilation=self.patch_embedding.dilation, + groups=self.patch_embedding.groups, + ) + if depth_patches.shape[2] != 1: + raise ValueError( + "Depth first-frame convolution must return exactly one temporal patch" + ) + + actual_t_embedding = self.time_embedding( + _sinusoidal_embedding_1d(self.freq_dim, timestep) + ) + depth_scale = self.depth_first_frame_timestep_adapter(actual_t_embedding) + depth_first_delta = depth_patches[:, :, 0] * depth_scale[:, :, None, None] + depth_first_delta = depth_first_delta.permute(0, 2, 3, 1).reshape( + batch_size, + -1, + self.hidden_dim, + ) + tokens_per_frame = int(pre_state["meta"]["tokens_per_frame"]) + if depth_first_delta.shape[1] != tokens_per_frame: + raise ValueError( + "Depth adapter token count mismatch: " + f"{depth_first_delta.shape[1]} != {tokens_per_frame}" + ) + tokens = pre_state["tokens"].clone() + tokens[:, :tokens_per_frame] = ( + tokens[:, :tokens_per_frame] + depth_first_delta + ) + pre_state["tokens"] = tokens + return pre_state + + +def enable_depth_first_frame_timestep_adapter( + dit: nn.Module, + *, + bottleneck_dim: int = 256, +) -> nn.Module: + """Attach a zero-init modality-specific timestep adapter to one Wan DiT.""" + + if not getattr(dit, "joint_rgb_depth", False): + raise ValueError("Upgrade Wan DiT for RGB-depth before enabling adapter") + if hasattr(dit, "depth_first_frame_timestep_adapter"): + raise ValueError("Depth first-frame timestep adapter is already enabled") + adapter = DepthFirstFrameTimestepAdapter( + hidden_dim=int(dit.hidden_dim), + bottleneck_dim=int(bottleneck_dim), + device=dit.patch_embedding.weight.device, + dtype=dit.patch_embedding.weight.dtype, + ) + dit.add_module("depth_first_frame_timestep_adapter", adapter) + dit.depth_first_frame_timestep_mode = "adapter" + dit.depth_first_frame_timestep_adapter_bottleneck_dim = int(bottleneck_dim) + dit._depth_first_frame_original_pre_dit = dit.pre_dit + dit.pre_dit = MethodType( + _pre_dit_with_depth_first_frame_timestep_adapter, + dit, + ) + return dit + + +def expand_wan_patch_embedding_for_rgb_depth( + patch_embedding: nn.Conv3d, + *, + rgb_channels: int = 48, + depth_channels: int = 48, +) -> nn.Conv3d: + """Append zero-initialized depth input channels to Wan's patch Conv3d. + + The resulting channel layout is strictly ``[original RGB inputs | depth]``. + Original weights and bias are copied bit-for-bit; the new depth slice is zero. + """ + + if not isinstance(patch_embedding, nn.Conv3d): + raise TypeError( + f"Expected torch.nn.Conv3d, got {type(patch_embedding).__name__}" + ) + if patch_embedding.in_channels != rgb_channels: + raise ValueError( + f"Expected {rgb_channels} original Wan input channels, " + f"got {patch_embedding.in_channels}" + ) + if depth_channels <= 0: + raise ValueError(f"depth_channels must be positive, got {depth_channels}") + if patch_embedding.groups != 1: + raise ValueError( + f"Grouped patch embedding is unsupported, got groups={patch_embedding.groups}" + ) + + expanded = nn.Conv3d( + in_channels=rgb_channels + depth_channels, + out_channels=patch_embedding.out_channels, + kernel_size=patch_embedding.kernel_size, + stride=patch_embedding.stride, + padding=patch_embedding.padding, + dilation=patch_embedding.dilation, + groups=1, + bias=patch_embedding.bias is not None, + padding_mode=patch_embedding.padding_mode, + device=patch_embedding.weight.device, + dtype=patch_embedding.weight.dtype, + ) + expanded.train(patch_embedding.training) + with torch.no_grad(): + expanded.weight.zero_() + expanded.weight[:, :rgb_channels].copy_(patch_embedding.weight) + if patch_embedding.bias is not None: + expanded.bias.copy_(patch_embedding.bias) + + expanded.weight.requires_grad_(patch_embedding.weight.requires_grad) + if expanded.bias is not None: + expanded.bias.requires_grad_(patch_embedding.bias.requires_grad) + return expanded + +def expand_wan_output_head_for_rgb_depth( + output_head: nn.Linear, + *, + patch_size: tuple[int, int, int] = (1, 2, 2), + rgb_channels: int = 48, + depth_channels: int = 48, +) -> nn.Linear: + """Expand Wan output rows using position-major channel mapping.""" + + if not isinstance(output_head, nn.Linear): + raise TypeError(f"Expected torch.nn.Linear, got {type(output_head).__name__}") + if len(patch_size) != 3 or any(size <= 0 for size in patch_size): + raise ValueError( + f"patch_size must contain three positive integers, got {patch_size}" + ) + if rgb_channels <= 0 or depth_channels <= 0: + raise ValueError("rgb_channels and depth_channels must be positive") + + patch_volume = math.prod(patch_size) + expected_old_rows = patch_volume * rgb_channels + if output_head.out_features != expected_old_rows: + raise ValueError( + f"Expected {expected_old_rows} output rows for patch_size={patch_size} " + f"and rgb_channels={rgb_channels}, got {output_head.out_features}" + ) + + joint_channels = rgb_channels + depth_channels + expanded = nn.Linear( + output_head.in_features, + patch_volume * joint_channels, + bias=output_head.bias is not None, + device=output_head.weight.device, + dtype=output_head.weight.dtype, + ) + expanded.train(output_head.training) + with torch.no_grad(): + expanded.weight.zero_() + if expanded.bias is not None: + expanded.bias.zero_() + for patch_position in range(patch_volume): + old_start = patch_position * rgb_channels + old_stop = old_start + rgb_channels + new_start = patch_position * joint_channels + new_stop = new_start + rgb_channels + expanded.weight[new_start:new_stop].copy_( + output_head.weight[old_start:old_stop] + ) + if output_head.bias is not None: + expanded.bias[new_start:new_stop].copy_( + output_head.bias[old_start:old_stop] + ) + + expanded.weight.requires_grad_(output_head.weight.requires_grad) + if expanded.bias is not None: + expanded.bias.requires_grad_(output_head.bias.requires_grad) + return expanded + +def upgrade_wan_dit_for_rgb_depth( + dit: nn.Module, + *, + rgb_channels: int = 48, + depth_channels: int = 48, + optimize_runtime: bool = True, +) -> nn.Module: + """Replace a Wan DiT input projection and output linear for RGB-depth.""" + + if not hasattr(dit, "patch_embedding") or not hasattr(dit, "head"): + raise TypeError("Expected a Wan DiT with patch_embedding and head") + if not hasattr(dit.head, "head") or not hasattr(dit, "patch_size"): + raise TypeError("Expected a Wan DiT head.head and patch_size") + dit.patch_embedding = expand_wan_patch_embedding_for_rgb_depth( + dit.patch_embedding, + rgb_channels=rgb_channels, + depth_channels=depth_channels, + ) + dit.head.head = expand_wan_output_head_for_rgb_depth( + dit.head.head, + patch_size=tuple(dit.patch_size), + rgb_channels=rgb_channels, + depth_channels=depth_channels, + ) + dit.in_dim = rgb_channels + depth_channels + dit.rgb_latent_channels = rgb_channels + dit.depth_latent_channels = depth_channels + dit.joint_rgb_depth = True + if optimize_runtime: + dit = enable_joint_rgb_depth_runtime_optimizations(dit) + return dit + + +def split_joint_rgb_depth_prediction( + prediction: torch.Tensor, + *, + rgb_channels: int = 48, + depth_channels: int = 48, +) -> tuple[torch.Tensor, torch.Tensor]: + """Validate and split a joint [B,C,T,H,W] prediction by channel.""" + + if prediction.ndim != 5: + raise ValueError( + f"Expected joint prediction [B,C,T,H,W], got {tuple(prediction.shape)}" + ) + expected_channels = rgb_channels + depth_channels + if prediction.shape[1] != expected_channels: + raise ValueError( + f"Expected {expected_channels} joint channels, got {prediction.shape[1]}" + ) + return ( + prediction[:, :rgb_channels], + prediction[:, rgb_channels:expected_channels], + ) + +def joint_rgb_depth_sampling_step( + dit: nn.Module, + scheduler, + joint_latents: torch.Tensor, + timestep: torch.Tensor, + step_delta: torch.Tensor, + context: torch.Tensor, + *, + context_mask: torch.Tensor | None = None, + rgb_condition_latent: torch.Tensor | None = None, + depth_condition_latent: torch.Tensor | None = None, + rgb_channels: int = 48, + depth_channels: int = 48, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Run one shared-DiT forward and one joint flow-matching update. + + Channel layout is ``[original Wan RGB/TI2V channels | depth channels]``. + Supplied first-frame conditions are fixed at both the DiT input and output. + """ + + if joint_latents.ndim != 5: + raise ValueError( + "Expected joint_latents [B,C,T,H,W], " + f"got {tuple(joint_latents.shape)}" + ) + expected_channels = rgb_channels + depth_channels + if joint_latents.shape[1] != expected_channels: + raise ValueError( + f"Expected {expected_channels} joint latent channels, " + f"got {joint_latents.shape[1]}" + ) + + model_input = joint_latents + if rgb_condition_latent is not None or depth_condition_latent is not None: + model_input = joint_latents.clone() + if rgb_condition_latent is not None: + expected_rgb_shape = ( + joint_latents.shape[0], rgb_channels, 1, + joint_latents.shape[3], joint_latents.shape[4], + ) + if tuple(rgb_condition_latent.shape) != expected_rgb_shape: + raise ValueError( + "Expected rgb_condition_latent shape " + f"{expected_rgb_shape}, got {tuple(rgb_condition_latent.shape)}" + ) + model_input[:, :rgb_channels, 0:1].copy_( + rgb_condition_latent.to(device=model_input.device, dtype=model_input.dtype) + ) + if depth_condition_latent is not None: + expected_depth_shape = ( + joint_latents.shape[0], depth_channels, 1, + joint_latents.shape[3], joint_latents.shape[4], + ) + if tuple(depth_condition_latent.shape) != expected_depth_shape: + raise ValueError( + "Expected depth_condition_latent shape " + f"{expected_depth_shape}, got {tuple(depth_condition_latent.shape)}" + ) + model_input[:, rgb_channels:expected_channels, 0:1].copy_( + depth_condition_latent.to(device=model_input.device, dtype=model_input.dtype) + ) + + joint_prediction = dit( + model_input, + timestep, + context, + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + pred_rgb, pred_depth = split_joint_rgb_depth_prediction( + joint_prediction, + rgb_channels=rgb_channels, + depth_channels=depth_channels, + ) + if joint_prediction.shape != model_input.shape: + raise ValueError( + "Joint prediction and latent shapes must match, " + f"got {tuple(joint_prediction.shape)} and {tuple(model_input.shape)}" + ) + + next_joint_latents = scheduler.step(joint_prediction, step_delta, model_input) + if not isinstance(next_joint_latents, torch.Tensor): + raise TypeError( + "Expected scheduler.step to return torch.Tensor, " + f"got {type(next_joint_latents).__name__}" + ) + if next_joint_latents.shape != model_input.shape: + raise ValueError( + "Scheduler output and latent shapes must match, " + f"got {tuple(next_joint_latents.shape)} and {tuple(model_input.shape)}" + ) + + if rgb_condition_latent is not None or depth_condition_latent is not None: + next_joint_latents = next_joint_latents.clone() + if rgb_condition_latent is not None: + next_joint_latents[:, :rgb_channels, 0:1].copy_( + rgb_condition_latent.to( + device=next_joint_latents.device, dtype=next_joint_latents.dtype + ) + ) + if depth_condition_latent is not None: + next_joint_latents[:, rgb_channels:expected_channels, 0:1].copy_( + depth_condition_latent.to( + device=next_joint_latents.device, dtype=next_joint_latents.dtype + ) + ) + return next_joint_latents, pred_rgb, pred_depth + + +def configure_wan_dit_for_rgb_depth( + dit: nn.Module, + *, + joint_rgb_depth: bool, + rgb_channels: int = 48, + depth_channels: int = 48, +) -> nn.Module: + """Apply the joint architecture only when explicitly enabled. + + The disabled branch validates the original Wan 48-channel input/output + contract and returns the exact same module without replacing projections. + """ + + if not isinstance(joint_rgb_depth, bool): + raise TypeError( + "joint_rgb_depth must be bool, " + f"got {type(joint_rgb_depth).__name__}" + ) + if joint_rgb_depth: + return upgrade_wan_dit_for_rgb_depth( + dit, + rgb_channels=rgb_channels, + depth_channels=depth_channels, + ) + + if not hasattr(dit, "patch_embedding") or not hasattr(dit, "head"): + raise TypeError("Expected a Wan DiT with patch_embedding and head") + if not hasattr(dit.head, "head") or not hasattr(dit, "patch_size"): + raise TypeError("Expected a Wan DiT head.head and patch_size") + if dit.patch_embedding.in_channels != rgb_channels: + raise ValueError( + f"Expected original patch input channels {rgb_channels}, " + f"got {dit.patch_embedding.in_channels}" + ) + expected_output_rows = math.prod(tuple(dit.patch_size)) * rgb_channels + if dit.head.head.out_features != expected_output_rows: + raise ValueError( + f"Expected original output head rows {expected_output_rows}, " + f"got {dit.head.head.out_features}" + ) + if getattr(dit, "in_dim", rgb_channels) != rgb_channels: + raise ValueError( + f"Expected original in_dim {rgb_channels}, got {dit.in_dim}" + ) + + dit.joint_rgb_depth = False + return dit diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/wan_rgb_depth_point_action_mot.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/wan_rgb_depth_point_action_mot.py new file mode 100644 index 0000000000000000000000000000000000000000..985eb09d9905edf5f9366142dfb28b8954515370 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/wan_rgb_depth_point_action_mot.py @@ -0,0 +1,428 @@ +"""MoT unification of the RGB/depth/point-cloud Wan DiT and an ActionDiT. + +The three dense video modalities are already represented by one expanded Wan +expert. This module adds a separate, randomly initialized action expert. At +every transformer layer the two experts project their own Q/K/V tensors into +the same ``num_heads * attn_head_dim`` space, concatenate them, and execute one +shared self-attention operation. Norms, modulation, cross-attention, FFNs and +output heads remain expert-specific, following FastWAM's MoT design. +""" + +from __future__ import annotations + +from contextlib import nullcontext +from copy import deepcopy +from pathlib import Path +import sys +from typing import Any, Mapping + +import torch +from torch import nn + + +PROJECT_ROOT = Path(__file__).resolve().parent +FASTWAM_SRC = PROJECT_ROOT.parent / "model_old" / "src" +if str(FASTWAM_SRC) not in sys.path: + sys.path.insert(0, str(FASTWAM_SRC)) + +from fastwam.models.wan22.action_dit import ActionDiT +from controlled_directional_mot import ForwardOnBackwardOnTwoCallMoT as MoT + + +ACTION_DIT_05B_CONFIG: dict[str, Any] = { + "action_dim": 14, + "hidden_dim": 576, + "ffn_dim": 2304, + "num_heads": 24, + "attn_head_dim": 128, + "num_layers": 30, + "text_dim": 4096, + "freq_dim": 256, + "eps": 1.0e-6, + "use_gradient_checkpointing": True, +} + + +def action_dit_05b_config(**overrides: Any) -> dict[str, Any]: + """Return a fresh 0.51B ActionDiT config, optionally overriding fields.""" + + config = deepcopy(ACTION_DIT_05B_CONFIG) + config.update(overrides) + return config + + +def count_parameters(module: nn.Module) -> int: + return sum(parameter.numel() for parameter in module.parameters()) + + +class RgbDepthPointActionMoT(nn.Module): + """One video expert plus one action expert with layer-wise shared attention. + + ``video_expert`` is the existing RGB/depth/point-cloud DiT. It continues to + own the joint 144-channel patch embedding and output head. ``action_expert`` + is deliberately independent and randomly initialized by :func:`build_action_mot`. + """ + + def __init__( + self, + video_expert: nn.Module, + action_expert: ActionDiT, + *, + action_state_dim: int | None = None, + camera_dim: int = 6, + mot_checkpoint_mixed_attn: bool = True, + cross_modal_attention: str = "bidirectional", + ) -> None: + super().__init__() + if not getattr(video_expert, "joint_rgb_depth_point", False): + raise ValueError( + "video_expert must first be upgraded for joint RGB/depth/point denoising" + ) + if cross_modal_attention not in {"bidirectional", "action_reads_video"}: + raise ValueError( + "cross_modal_attention must be 'bidirectional' or 'action_reads_video'" + ) + if len(video_expert.blocks) != len(action_expert.blocks): + raise ValueError( + "ActionDiT layer count must equal the RGB/depth/point DiT layer count: " + f"{len(action_expert.blocks)} != {len(video_expert.blocks)}" + ) + if int(video_expert.num_heads) != int(action_expert.num_heads): + raise ValueError("Experts must use the same number of attention heads") + if int(video_expert.attn_head_dim) != int(action_expert.attn_head_dim): + raise ValueError("Experts must use the same attention head dimension") + + # Keep a single registration path. Accessors below expose both experts + # without duplicating their keys in state_dict/checkpoint payloads. + self.mot = MoT( + mixtures={"video": video_expert, "action": action_expert}, + mot_checkpoint_mixed_attn=mot_checkpoint_mixed_attn, + ) + self.cross_modal_attention = cross_modal_attention + self.camera_dim = int(camera_dim) + if self.camera_dim <= 0: + raise ValueError("camera_dim must be positive") + self.camera_encoder = nn.Linear(self.camera_dim, action_expert.hidden_dim) + self.camera_head = nn.Linear(action_expert.hidden_dim, self.camera_dim) + self.structured_type_embedding = nn.Parameter( + torch.randn(2, 1, action_expert.hidden_dim) + / action_expert.hidden_dim**0.5 + ) + self.action_state_dim = ( + None if action_state_dim is None else int(action_state_dim) + ) + if self.action_state_dim is not None: + if self.action_state_dim <= 0: + raise ValueError("action_state_dim must be positive") + self.action_state_encoder = nn.Sequential( + nn.Linear(self.action_state_dim, action_expert.hidden_dim), + nn.SiLU(), + nn.Linear(action_expert.hidden_dim, action_expert.hidden_dim), + ) + else: + self.action_state_encoder = None + + @property + def video_expert(self) -> nn.Module: + return self.mot.mixtures["video"] + + @property + def action_expert(self) -> ActionDiT: + return self.mot.mixtures["action"] + + @property + def num_layers(self) -> int: + return self.mot.num_layers + + @property + def action_parameter_count(self) -> int: + return count_parameters(self.action_expert) + + def architecture_metadata(self) -> dict[str, Any]: + video = self.video_expert + action = self.action_expert + return { + "format_version": 2, + "model_class": type(self).__name__, + "experts": ["rgb_depth_point_video", "action"], + "shared_self_attention_per_layer": True, + "cross_modal_attention": self.cross_modal_attention, + "action_to_video_attention": bool( + self.mot.video_reads_action + ), + "video_to_action_attention": True, + "action_loss_to_video_gradient": ( + "stopped_at_video_kv" + if self.mot.detach_video_for_action + else "enabled_through_video_kv" + ), + "attention_calls_per_layer": 2, + "num_layers": self.num_layers, + "num_heads": int(video.num_heads), + "attn_head_dim": int(video.attn_head_dim), + "video_hidden_dim": int(video.hidden_dim), + "action_hidden_dim": int(action.hidden_dim), + "action_ffn_dim": int(action.ffn_dim), + "action_dim": int(action.action_dim), + "camera_dim": self.camera_dim, + "structured_token_types": ["action", "camera"], + "camera_conditioning": "none", + "action_state_dim": self.action_state_dim, + "action_state_conditioning": ( + "cross_attention_token" + if self.action_state_encoder is not None + else "none" + ), + "action_parameter_count": self.action_parameter_count, + "action_initialization": "random", + "rgb_latent_channels": int(video.rgb_latent_channels), + "depth_latent_channels": int(video.depth_latent_channels), + "point_latent_channels": int(video.point_latent_channels), + } + + @torch.no_grad() + def build_shared_attention_mask( + self, + *, + video_seq_len: int, + action_seq_len: int, + video_tokens_per_frame: int, + device: torch.device, + ) -> torch.Tensor: + """Build the mask for one attention call over ``[video, action]`` tokens.""" + + if video_seq_len <= 0 or action_seq_len <= 0: + raise ValueError("Both video and action token sequences must be non-empty") + total = video_seq_len + action_seq_len + mask = torch.zeros((total, total), dtype=torch.bool, device=device) + mask[:video_seq_len, :video_seq_len] = self.video_expert.build_video_to_video_mask( + video_seq_len=video_seq_len, + video_tokens_per_frame=video_tokens_per_frame, + device=device, + ) + mask[video_seq_len:, video_seq_len:] = True + # Action always reads the full three-modality video stream. + mask[video_seq_len:, :video_seq_len] = True + if self.cross_modal_attention == "bidirectional": + mask[:video_seq_len, video_seq_len:] = True + return mask + + def forward( + self, + x: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + *, + action_tokens: torch.Tensor, + camera_tokens: torch.Tensor, + initial_state: torch.Tensor | None = None, + action_timestep: torch.Tensor | None = None, + context_mask: torch.Tensor | None = None, + fuse_vae_embedding_in_latents: bool = True, + ) -> dict[str, torch.Tensor]: + """Jointly denoise three video modalities and action tokens. + + Args: + x: Concatenated RGB/depth/point latents ``[B, 144, T, H, W]``. + timestep: Video diffusion timestep ``[B]``. + context: Text context ``[B, L, text_dim]``. + action_tokens: Noisy actions ``[B, T, action_dim]``. + camera_tokens: Noisy first-camera-relative poses ``[B, T, 6]``. + initial_state: Clean bimanual state ``[B, action_state_dim]``. + action_timestep: Independent action timestep; defaults to ``timestep``. + + Returns: + ``{"video": ..., "action": ...}``, where ``video`` preserves the + concatenated RGB/depth/point layout and ``action`` preserves the + input action token layout. + """ + + if action_timestep is None: + action_timestep = timestep + expected_camera = ( + int(action_tokens.shape[0]), + int(action_tokens.shape[1]), + self.camera_dim, + ) + if tuple(camera_tokens.shape) != expected_camera: + raise ValueError( + f"camera_tokens must have shape {expected_camera}, " + f"got {tuple(camera_tokens.shape)}" + ) + video_pre = self.video_expert.pre_dit( + x=x, + timestep=timestep, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + action_pre = self.action_expert.pre_dit( + action_tokens=action_tokens, + timestep=action_timestep, + context=context, + context_mask=context_mask, + ) + if self.action_state_encoder is not None: + if initial_state is None: + raise ValueError("initial_state is required by this Action MoT") + expected = (int(action_tokens.shape[0]), int(self.action_state_dim)) + if tuple(initial_state.shape) != expected: + raise ValueError( + f"initial_state must have shape {expected}, " + f"got {tuple(initial_state.shape)}" + ) + state_context = self.action_state_encoder( + initial_state.to( + device=action_tokens.device, + dtype=action_tokens.dtype, + ) + ).unsqueeze(1) + action_pre["context"] = torch.cat( + [action_pre["context"], state_context], dim=1 + ) + state_mask = torch.ones( + (*action_pre["context_mask"].shape[:-1], 1), + device=action_pre["context_mask"].device, + dtype=torch.bool, + ) + action_pre["context_mask"] = torch.cat( + [action_pre["context_mask"], state_mask], dim=-1 + ) + elif initial_state is not None: + raise ValueError( + "initial_state was provided but state conditioning is disabled" + ) + + # Action and camera are two token types processed by the same Action + # Expert. They share temporal RoPE positions and diffusion timestep. + action_seq_len = int(action_pre["tokens"].shape[1]) + action_freqs = action_pre["freqs"] + camera_hidden = self.camera_encoder(camera_tokens) + action_pre["tokens"] = torch.cat( + [ + action_pre["tokens"] + self.structured_type_embedding[0], + camera_hidden + self.structured_type_embedding[1], + ], + dim=1, + ) + action_pre["freqs"] = torch.cat([action_freqs, action_freqs], dim=0) + action_pre["context_mask"] = torch.cat( + [action_pre["context_mask"], action_pre["context_mask"]], dim=1 + ) + + shared_mask = None + if not bool(getattr(self.mot, "omit_all_true_attention_mask", False)): + shared_mask = self.build_shared_attention_mask( + video_seq_len=int(video_pre["tokens"].shape[1]), + action_seq_len=int(action_pre["tokens"].shape[1]), + video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]), + device=video_pre["tokens"].device, + ) + tokens = self.mot( + embeds_all={ + "video": video_pre["tokens"], + "action": action_pre["tokens"], + }, + attention_mask=shared_mask, + freqs_all={ + "video": video_pre["freqs"], + "action": action_pre["freqs"], + }, + context_all={ + "video": { + "context": video_pre["context"], + "mask": video_pre["context_mask"], + }, + "action": { + "context": action_pre["context"], + "mask": action_pre["context_mask"], + }, + }, + t_mod_all={ + "video": video_pre["t_mod"], + "action": action_pre["t_mod"], + }, + ) + structured = tokens["action"] + action_hidden = structured[:, :action_seq_len] + camera_hidden = structured[:, action_seq_len:] + return { + "video": self.video_expert.post_dit(tokens["video"], video_pre), + "action": self.action_expert.post_dit(action_hidden, action_pre), + "camera": self.camera_head(camera_hidden), + } + + +def build_action_mot( + video_expert: nn.Module, + action_dit_config: Mapping[str, Any] | None = None, + *, + random_seed: int | None = None, + device: torch.device | str | None = None, + dtype: torch.dtype | None = None, + action_state_dim: int | None = None, + camera_dim: int = 6, + mot_checkpoint_mixed_attn: bool = True, + cross_modal_attention: str = "bidirectional", +) -> RgbDepthPointActionMoT: + """Attach a *randomly initialized* ActionDiT to a prepared video expert. + + This builder intentionally has no pretrained-action argument. A combined + MoT checkpoint can be loaded afterwards with ``load_state_dict`` when + resuming, but first construction always initializes the action expert from + the configured RNG seed. + """ + + config = action_dit_05b_config() if action_dit_config is None else dict(action_dit_config) + if int(config.get("num_layers", -1)) != len(video_expert.blocks): + raise ValueError( + "action_dit.num_layers must match the existing video DiT: " + f"{config.get('num_layers')} != {len(video_expert.blocks)}" + ) + if int(config.get("num_heads", -1)) != int(video_expert.num_heads): + raise ValueError("action_dit.num_heads must match the existing video DiT") + if int(config.get("attn_head_dim", -1)) != int(video_expert.attn_head_dim): + raise ValueError("action_dit.attn_head_dim must match the existing video DiT") + + reference_parameter = next(video_expert.parameters()) + if device is None: + device = reference_parameter.device + if dtype is None: + dtype = reference_parameter.dtype + devices = [] + if device is not None and torch.device(device).type == "cuda": + devices = [torch.device(device).index or 0] + rng_context = ( + torch.random.fork_rng(devices=devices) + if random_seed is not None + else nullcontext() + ) + previous_dtype = torch.get_default_dtype() + try: + torch.set_default_dtype(dtype) + with rng_context, torch.device(device): + if random_seed is not None: + torch.manual_seed(int(random_seed)) + # Construct directly in the video's dtype/device. This avoids a + # transient fp32 copy for the 0.51B production action expert. + action_expert = ActionDiT(**config) + model = RgbDepthPointActionMoT( + video_expert=video_expert, + action_expert=action_expert, + action_state_dim=action_state_dim, + camera_dim=camera_dim, + mot_checkpoint_mixed_attn=mot_checkpoint_mixed_attn, + cross_modal_attention=cross_modal_attention, + ) + finally: + torch.set_default_dtype(previous_dtype) + + # The default production config is 0.5107B; keep the accepted range broad + # enough for a different action_dim while rejecting accidental tiny models. + if action_dit_config is None and not 450_000_000 <= model.action_parameter_count <= 600_000_000: + raise RuntimeError( + f"Default ActionDiT must be approximately 0.5B parameters, got " + f"{model.action_parameter_count:,}" + ) + return model diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/wan_rgb_depth_point_initialization.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/wan_rgb_depth_point_initialization.py new file mode 100644 index 0000000000000000000000000000000000000000..d3030df597e52341f0743fae05189257f8f4fee6 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/wan_rgb_depth_point_initialization.py @@ -0,0 +1,365 @@ +"""Explicit RGB-depth-point Wan DiT expansion and joint sampling helpers.""" + +from __future__ import annotations + +import math +from types import MethodType +from typing import Any + +import torch +import torch.nn.functional as F +from torch import nn + +from wan_rgb_depth_initialization import ( + DepthFirstFrameTimestepAdapter, + _sinusoidal_embedding_1d, + upgrade_wan_dit_for_rgb_depth, +) + + +class PointFirstFrameTimestepAdapter(DepthFirstFrameTimestepAdapter): + """Zero-initialized timestep correction for noisy point-map frame zero.""" + + +def _append_point_patch_channels( + patch_embedding: nn.Conv3d, + *, + rgb_channels: int, + depth_channels: int, + point_channels: int, +) -> nn.Conv3d: + old_channels = rgb_channels + depth_channels + if not isinstance(patch_embedding, nn.Conv3d): + raise TypeError("Wan patch embedding must be Conv3d") + if patch_embedding.in_channels != old_channels or patch_embedding.groups != 1: + raise ValueError( + f"Expected ungrouped RGB-depth patch input {old_channels}, " + f"got {patch_embedding.in_channels}" + ) + expanded = nn.Conv3d( + old_channels + point_channels, + patch_embedding.out_channels, + kernel_size=patch_embedding.kernel_size, + stride=patch_embedding.stride, + padding=patch_embedding.padding, + dilation=patch_embedding.dilation, + groups=1, + bias=patch_embedding.bias is not None, + padding_mode=patch_embedding.padding_mode, + device=patch_embedding.weight.device, + dtype=patch_embedding.weight.dtype, + ) + expanded.train(patch_embedding.training) + with torch.no_grad(): + expanded.weight.zero_() + expanded.weight[:, :old_channels].copy_(patch_embedding.weight) + if patch_embedding.bias is not None: + expanded.bias.copy_(patch_embedding.bias) + expanded.weight.requires_grad_(patch_embedding.weight.requires_grad) + if expanded.bias is not None: + expanded.bias.requires_grad_(patch_embedding.bias.requires_grad) + return expanded + + +def _append_point_output_rows( + output_head: nn.Linear, + *, + patch_size: tuple[int, int, int], + rgb_channels: int, + depth_channels: int, + point_channels: int, +) -> nn.Linear: + if not isinstance(output_head, nn.Linear): + raise TypeError("Wan output head must be Linear") + patch_volume = math.prod(patch_size) + old_channels = rgb_channels + depth_channels + joint_channels = old_channels + point_channels + if output_head.out_features != patch_volume * old_channels: + raise ValueError( + f"Expected RGB-depth output rows {patch_volume * old_channels}, " + f"got {output_head.out_features}" + ) + expanded = nn.Linear( + output_head.in_features, + patch_volume * joint_channels, + bias=output_head.bias is not None, + device=output_head.weight.device, + dtype=output_head.weight.dtype, + ) + expanded.train(output_head.training) + with torch.no_grad(): + expanded.weight.zero_() + if expanded.bias is not None: + expanded.bias.zero_() + for patch_position in range(patch_volume): + old_start = patch_position * old_channels + new_start = patch_position * joint_channels + expanded.weight[new_start : new_start + old_channels].copy_( + output_head.weight[old_start : old_start + old_channels] + ) + if output_head.bias is not None: + expanded.bias[new_start : new_start + old_channels].copy_( + output_head.bias[old_start : old_start + old_channels] + ) + expanded.weight.requires_grad_(output_head.weight.requires_grad) + if expanded.bias is not None: + expanded.bias.requires_grad_(output_head.bias.requires_grad) + return expanded + + +def upgrade_wan_dit_for_rgb_depth_point( + dit: nn.Module, + *, + rgb_channels: int = 48, + depth_channels: int = 48, + point_channels: int = 48, + optimize_runtime: bool = True, +) -> nn.Module: + """Expand original Wan in two explicit steps while preserving old slices.""" + + if min(rgb_channels, depth_channels, point_channels) <= 0: + raise ValueError("RGB, depth, and point latent channels must be positive") + dit = upgrade_wan_dit_for_rgb_depth( + dit, + rgb_channels=rgb_channels, + depth_channels=depth_channels, + optimize_runtime=optimize_runtime, + ) + dit.patch_embedding = _append_point_patch_channels( + dit.patch_embedding, + rgb_channels=rgb_channels, + depth_channels=depth_channels, + point_channels=point_channels, + ) + dit.head.head = _append_point_output_rows( + dit.head.head, + patch_size=tuple(dit.patch_size), + rgb_channels=rgb_channels, + depth_channels=depth_channels, + point_channels=point_channels, + ) + dit.in_dim = rgb_channels + depth_channels + point_channels + dit.point_latent_channels = point_channels + dit.joint_rgb_depth_point = True + return dit + + +def _first_frame_patch_contribution( + dit: nn.Module, + x: torch.Tensor, + channel_start: int, + channel_stop: int, +) -> torch.Tensor: + temporal_patch = ( + int(dit.patch_embedding.kernel_size[0]), + int(dit.patch_embedding.stride[0]), + int(dit.patch_embedding.padding[0]), + ) + if temporal_patch != (1, 1, 0): + raise ValueError( + "Modality timestep adapters require temporal " + f"kernel/stride/padding=(1,1,0), got {temporal_patch}" + ) + contribution = F.conv3d( + x[:, channel_start:channel_stop, 0:1], + dit.patch_embedding.weight[:, channel_start:channel_stop], + bias=None, + stride=dit.patch_embedding.stride, + padding=dit.patch_embedding.padding, + dilation=dit.patch_embedding.dilation, + groups=dit.patch_embedding.groups, + ) + if contribution.shape[2] != 1: + raise ValueError("First-frame projection must return one temporal patch") + return contribution[:, :, 0] + + +def _pre_dit_with_point_first_frame_timestep_adapter( + self: nn.Module, + x: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor | None = None, + action: torch.Tensor | None = None, + fuse_vae_embedding_in_latents: bool = False, + control_camera_latents_input: torch.Tensor | None = None, +) -> dict[str, Any]: + """Correct only the noisy point-map contribution in frame-zero tokens.""" + pre_state = self._rgb_depth_point_original_pre_dit( + x=x, + timestep=timestep, + context=context, + context_mask=context_mask, + action=action, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + control_camera_latents_input=control_camera_latents_input, + ) + if not fuse_vae_embedding_in_latents: + raise ValueError("Point timestep adapter requires fused VAE latents") + if x.ndim != 5 or x.shape[1] != self.in_dim: + raise ValueError(f"Expected [B,{self.in_dim},T,H,W], got {tuple(x.shape)}") + batch_size = int(pre_state["meta"]["batch_size"]) + if x.shape[0] == 1 and batch_size > 1: + x = x.expand(batch_size, -1, -1, -1, -1) + if timestep.shape[0] == 1 and batch_size > 1: + timestep = timestep.expand(batch_size) + if timestep.shape[0] != batch_size: + raise ValueError("Timestep batch does not match token batch") + + rgb = int(self.rgb_latent_channels) + depth = int(self.depth_latent_channels) + point = int(self.point_latent_channels) + if rgb + depth + point != self.in_dim: + raise ValueError("Triple-modality channel metadata does not match in_dim") + actual_t = pre_state.get("current_t_embedding") + if actual_t is None: + actual_t = self.time_embedding( + _sinusoidal_embedding_1d(self.freq_dim, timestep) + ) + point_patch = _first_frame_patch_contribution( + self, x, rgb + depth, rgb + depth + point + ) + point_scale = self.point_first_frame_timestep_adapter(actual_t) + delta = point_patch * point_scale[:, :, None, None] + delta = delta.permute(0, 2, 3, 1).reshape(batch_size, -1, self.hidden_dim) + tokens_per_frame = int(pre_state["meta"]["tokens_per_frame"]) + if delta.shape[1] != tokens_per_frame: + raise ValueError("Point timestep adapter token count mismatch") + tokens = pre_state["tokens"].clone() + tokens[:, :tokens_per_frame] += delta + pre_state["tokens"] = tokens + return pre_state + + +def enable_point_first_frame_timestep_adapter( + dit: nn.Module, + *, + point_bottleneck_dim: int = 256, +) -> nn.Module: + """Attach only the point-frame-zero adapter; RGB/depth frame zero stay clean.""" + if not getattr(dit, "joint_rgb_depth_point", False): + raise ValueError("Upgrade Wan DiT to RGB-depth-point before the adapter") + if hasattr(dit, "point_first_frame_timestep_adapter"): + raise ValueError("Point first-frame timestep adapter already exists") + if hasattr(dit, "depth_first_frame_timestep_adapter"): + raise ValueError("This sibling must not contain a depth timestep adapter") + kwargs = { + "hidden_dim": int(dit.hidden_dim), + "device": dit.patch_embedding.weight.device, + "dtype": dit.patch_embedding.weight.dtype, + } + dit.add_module( + "point_first_frame_timestep_adapter", + PointFirstFrameTimestepAdapter( + bottleneck_dim=int(point_bottleneck_dim), **kwargs + ), + ) + dit.point_first_frame_timestep_adapter_bottleneck_dim = int( + point_bottleneck_dim + ) + dit._rgb_depth_point_original_pre_dit = dit.pre_dit + dit.pre_dit = MethodType( + _pre_dit_with_point_first_frame_timestep_adapter, dit + ) + return dit + + +def split_joint_rgb_depth_point_prediction( + prediction: torch.Tensor, + *, + rgb_channels: int = 48, + depth_channels: int = 48, + point_channels: int = 48, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if prediction.ndim != 5: + raise ValueError(f"Expected prediction [B,C,T,H,W], got {prediction.shape}") + expected = rgb_channels + depth_channels + point_channels + if prediction.shape[1] != expected: + raise ValueError(f"Expected {expected} prediction channels, got {prediction.shape[1]}") + depth_stop = rgb_channels + depth_channels + return ( + prediction[:, :rgb_channels], + prediction[:, rgb_channels:depth_stop], + prediction[:, depth_stop:expected], + ) + + +def joint_rgb_depth_point_sampling_step( + dit: nn.Module, + scheduler: Any, + joint_latents: torch.Tensor, + timestep: torch.Tensor, + step_delta: torch.Tensor, + context: torch.Tensor, + *, + context_mask: torch.Tensor | None = None, + rgb_condition_latent: torch.Tensor | None = None, + depth_condition_latent: torch.Tensor | None = None, + rgb_channels: int = 48, + depth_channels: int = 48, + point_channels: int = 48, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + expected = rgb_channels + depth_channels + point_channels + if joint_latents.ndim != 5 or joint_latents.shape[1] != expected: + raise ValueError( + f"Expected joint latents [B,{expected},T,H,W], got {joint_latents.shape}" + ) + + expected_spatial = ( + joint_latents.shape[0], + 1, + joint_latents.shape[3], + joint_latents.shape[4], + ) + for name, condition, channels in ( + ("RGB", rgb_condition_latent, rgb_channels), + ("depth", depth_condition_latent, depth_channels), + ): + if condition is not None: + expected_condition = (expected_spatial[0], channels, *expected_spatial[1:]) + if tuple(condition.shape) != expected_condition: + raise ValueError( + f"Expected {name} condition {expected_condition}, got {condition.shape}" + ) + + model_input = joint_latents + if rgb_condition_latent is not None or depth_condition_latent is not None: + model_input = joint_latents.clone() + if rgb_condition_latent is not None: + model_input[:, :rgb_channels, 0:1].copy_(rgb_condition_latent.to(model_input)) + if depth_condition_latent is not None: + depth_stop = rgb_channels + depth_channels + model_input[:, rgb_channels:depth_stop, 0:1].copy_( + depth_condition_latent.to(model_input) + ) + + prediction = dit( + model_input, + timestep, + context, + context_mask=context_mask, + fuse_vae_embedding_in_latents=True, + ) + if prediction.shape != model_input.shape: + raise ValueError("Triple prediction and joint latent shapes must match") + pred_rgb, pred_depth, pred_point = split_joint_rgb_depth_point_prediction( + prediction, + rgb_channels=rgb_channels, + depth_channels=depth_channels, + point_channels=point_channels, + ) + next_latents = scheduler.step(prediction, step_delta, model_input) + if not isinstance(next_latents, torch.Tensor) or next_latents.shape != model_input.shape: + raise ValueError("Scheduler must return a tensor matching triple joint latents") + if rgb_condition_latent is not None or depth_condition_latent is not None: + next_latents = next_latents.clone() + if rgb_condition_latent is not None: + next_latents[:, :rgb_channels, 0:1].copy_( + rgb_condition_latent.to(next_latents) + ) + if depth_condition_latent is not None: + depth_stop = rgb_channels + depth_channels + next_latents[:, rgb_channels:depth_stop, 0:1].copy_( + depth_condition_latent.to(next_latents) + ) + return next_latents, pred_rgb, pred_depth, pred_point diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/world_point_cloud.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/world_point_cloud.py new file mode 100644 index 0000000000000000000000000000000000000000..9c2046a9b6c4c43b57b9b96e7af21d90128c257a --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/multimodal/world_point_cloud.py @@ -0,0 +1,595 @@ +"""World-frame point-map construction, normalization, metrics, and export. + +The RGB-depth-point branch uses these utilities both to build the learned third +diffusion target and to evaluate its direct decoded output. Egoverse +``obs_head_pose`` is interpreted explicitly as +``[tx, ty, tz, qw, qx, qy, qz]`` camera-to-world pose; an optional fixed +camera-to-head transform can be supplied when a calibrated extrinsic exists. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +import math +from pathlib import Path +from typing import Any + +import cv2 +import numpy as np + +try: + import zstandard as zstd +except ImportError as error: # pragma: no cover - checked by Fuyao preflight + raise ImportError( + "Egoverse Zarr pose loading requires the zstandard package" + ) from error + + +_UINT64_MISSING = np.iinfo(np.uint64).max + + +@dataclass(frozen=True) +class WorldGeometry: + intrinsics: np.ndarray + camera_to_reference: np.ndarray + source_indices: np.ndarray + source_metadata_path: Path + pose_assumption: str + + +def _read_zarr_v3_sharded_numeric_rows( + array_dir: Path, + row_indices: np.ndarray, +) -> np.ndarray: + """Read selected rows from the numeric Zarr-v3 sharded arrays used by Egoverse.""" + + metadata_path = array_dir / "zarr.json" + if not metadata_path.is_file(): + raise FileNotFoundError(metadata_path) + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + shape = tuple(int(value) for value in metadata["shape"]) + if len(shape) != 2: + raise ValueError(f"Expected a rank-2 pose array, got shape={shape}") + codecs = metadata.get("codecs", []) + if len(codecs) != 1 or codecs[0].get("name") != "sharding_indexed": + raise ValueError("Only Egoverse sharding_indexed numeric Zarr arrays are supported") + shard_config = codecs[0]["configuration"] + if shard_config.get("index_location") != "end": + raise ValueError("Expected Zarr shard index_location=end") + inner_codecs = shard_config.get("codecs", []) + if [codec.get("name") for codec in inner_codecs] != ["bytes", "zstd"]: + raise ValueError(f"Unexpected numeric inner codecs: {inner_codecs}") + endian = inner_codecs[0].get("configuration", {}).get("endian") + if endian != "little": + raise ValueError(f"Only little-endian numeric arrays are supported, got {endian}") + + outer_shape = tuple( + int(value) + for value in metadata["chunk_grid"]["configuration"]["chunk_shape"] + ) + inner_shape = tuple(int(value) for value in shard_config["chunk_shape"]) + if len(outer_shape) != 2 or len(inner_shape) != 2: + raise ValueError("Expected rank-2 outer and inner chunk shapes") + inner_grid = tuple( + int(math.ceil(outer / inner)) + for outer, inner in zip(outer_shape, inner_shape, strict=True) + ) + index_slots = math.prod(inner_grid) + index_size = index_slots * 16 + 4 # uint64 offset/length pairs + CRC32C + + requested = np.asarray(row_indices, dtype=np.int64).reshape(-1) + if np.any(requested < 0) or np.any(requested >= shape[0]): + raise IndexError(f"Pose row outside [0,{shape[0]}): {requested.tolist()}") + + dtype = np.dtype(metadata["data_type"]).newbyteorder("<") + decompressor = zstd.ZstdDecompressor() + shard_cache: dict[tuple[int, int], tuple[bytes, np.ndarray]] = {} + chunk_cache: dict[tuple[int, int, int, int], np.ndarray] = {} + rows: list[np.ndarray] = [] + + for row in requested: + outer_row = int(row) // outer_shape[0] + row_inside_outer = int(row) % outer_shape[0] + inner_row = row_inside_outer // inner_shape[0] + row_inside_inner = row_inside_outer % inner_shape[0] + outer_col = 0 + inner_col = 0 + shard_key = (outer_row, outer_col) + if shard_key not in shard_cache: + shard_path = array_dir / "c" / str(outer_row) / str(outer_col) + raw = shard_path.read_bytes() + if len(raw) <= index_size: + raise ValueError(f"Invalid Zarr shard: {shard_path}") + index = np.frombuffer(raw[-index_size:-4], dtype="= chunk.shape[0]: + raise ValueError(f"Pose row {int(row)} is absent from its final chunk") + rows.append(chunk[row_inside_inner].copy()) + return np.stack(rows) + + +def quaternion_wxyz_to_rotation(quaternion: np.ndarray) -> np.ndarray: + q = np.asarray(quaternion, dtype=np.float64) + if q.shape[-1] != 4: + raise ValueError(f"Expected [...,4] quaternion, got {q.shape}") + norm = np.linalg.norm(q, axis=-1, keepdims=True) + if np.any(~np.isfinite(norm)) or np.any(norm < 1.0e-8): + raise ValueError("Invalid zero/non-finite quaternion") + q = q / norm + w, x, y, z = np.moveaxis(q, -1, 0) + rotation = np.empty(q.shape[:-1] + (3, 3), dtype=np.float64) + rotation[..., 0, 0] = 1 - 2 * (y * y + z * z) + rotation[..., 0, 1] = 2 * (x * y - z * w) + rotation[..., 0, 2] = 2 * (x * z + y * w) + rotation[..., 1, 0] = 2 * (x * y + z * w) + rotation[..., 1, 1] = 1 - 2 * (x * x + z * z) + rotation[..., 1, 2] = 2 * (y * z - x * w) + rotation[..., 2, 0] = 2 * (x * z - y * w) + rotation[..., 2, 1] = 2 * (y * z + x * w) + rotation[..., 2, 2] = 1 - 2 * (x * x + y * y) + return rotation.astype(np.float32) + + +def pose_txyz_qwxyz_to_matrix(pose: np.ndarray) -> np.ndarray: + pose = np.asarray(pose, dtype=np.float64) + if pose.ndim != 2 or pose.shape[1] != 7: + raise ValueError(f"Expected [T,7] txyz+qwxyz pose, got {pose.shape}") + if not np.all(np.isfinite(pose)): + raise ValueError("Camera pose contains NaN or Inf") + matrix = np.broadcast_to(np.eye(4, dtype=np.float32), (len(pose), 4, 4)).copy() + matrix[:, :3, :3] = quaternion_wxyz_to_rotation(pose[:, 3:7]) + matrix[:, :3, 3] = pose[:, :3].astype(np.float32) + return matrix + + +def scale_intrinsics_after_center_crop( + intrinsics: np.ndarray, + source_height: int, + source_width: int, + target_resolution: int, +) -> np.ndarray: + if min(source_height, source_width, target_resolution) <= 0: + raise ValueError("Image dimensions must be positive") + k = np.asarray(intrinsics, dtype=np.float64).copy() + if k.shape != (3, 3): + raise ValueError(f"Expected K [3,3], got {k.shape}") + crop_size = min(source_height, source_width) + top = (source_height - crop_size) // 2 + left = (source_width - crop_size) // 2 + scale = float(target_resolution) / float(crop_size) + k[0, 0] *= scale + k[1, 1] *= scale + k[0, 2] = (k[0, 2] - left) * scale + k[1, 2] = (k[1, 2] - top) * scale + return k.astype(np.float32) + + +def load_egoverse_world_geometry( + annotation_path: Path, + source_indices: np.ndarray, + target_resolution: int, + config: dict[str, Any], +) -> WorldGeometry: + """Load processed intrinsics and camera-to-world matrices for one clip.""" + + with np.load(annotation_path, allow_pickle=False) as bundle: + k = bundle["K"].astype(np.float32) + metadata = json.loads(bundle["metadata_json"].item()) + episode = metadata["episode"] + source_height = int(episode["output_height"]) + source_width = int(episode["output_width"]) + processed_k = scale_intrinsics_after_center_crop( + k, source_height, source_width, target_resolution + ) + source_metadata_path = Path(episode["source_metadata_path"]) + pose_key = str(config.get("pose_array_key", "obs_head_pose")) + poses = _read_zarr_v3_sharded_numeric_rows( + source_metadata_path.parent / pose_key, + np.asarray(source_indices, dtype=np.int64), + ) + layout = str(config.get("pose_vector_layout", "txyz_qwxyz")) + if layout != "txyz_qwxyz": + raise ValueError(f"Only pose_vector_layout=txyz_qwxyz is implemented, got {layout}") + camera_to_world = pose_txyz_qwxyz_to_matrix(poses) + pose_is = str(config.get("pose_is", "camera_to_world")) + if pose_is == "world_to_camera": + camera_to_world = np.linalg.inv(camera_to_world).astype(np.float32) + elif pose_is != "camera_to_world": + raise ValueError(f"pose_is must be camera_to_world or world_to_camera, got {pose_is}") + + camera_to_head = np.asarray( + config.get("camera_to_head_transform", np.eye(4).tolist()), + dtype=np.float32, + ) + if camera_to_head.shape != (4, 4) or not np.all(np.isfinite(camera_to_head)): + raise ValueError("camera_to_head_transform must be a finite [4,4] matrix") + camera_to_world = np.einsum( + "tij,jk->tik", camera_to_world, camera_to_head + ).astype(np.float32) + coordinate_frame = str(config.get("coordinate_frame", "first_camera_frame")) + if coordinate_frame == "first_camera_frame": + reference_from_world = np.linalg.inv(camera_to_world[0]).astype(np.float32) + camera_to_reference = np.einsum( + "ij,tjk->tik", reference_from_world, camera_to_world + ).astype(np.float32) + camera_to_reference[0] = np.eye(4, dtype=np.float32) + elif coordinate_frame == "egoverse_world": + camera_to_reference = camera_to_world + else: + raise ValueError( + "coordinate_frame must be first_camera_frame or egoverse_world, " + f"got {coordinate_frame!r}" + ) + assumption = ( + f"{pose_key} interpreted as {layout} {pose_is}; " + "T_world_camera=T_world_head@T_head_camera; " + f"output_coordinate_frame={coordinate_frame}" + ) + return WorldGeometry( + intrinsics=processed_k, + camera_to_reference=camera_to_reference, + source_indices=np.asarray(source_indices, dtype=np.int64), + source_metadata_path=source_metadata_path, + pose_assumption=assumption, + ) + + +def depth_video_to_world_grid( + depth_m: np.ndarray, + intrinsics: np.ndarray, + camera_to_reference: np.ndarray, + *, + stride: int, + min_depth_m: float, + max_depth_m: float, + valid_mask: np.ndarray | None = None, +) -> tuple[np.ndarray, np.ndarray]: + depth = np.asarray(depth_m, dtype=np.float32) + if depth.ndim != 3: + raise ValueError(f"Expected depth [T,H,W], got {depth.shape}") + if stride <= 0: + raise ValueError("point stride must be positive") + if not 0 <= min_depth_m < max_depth_m: + raise ValueError("Expected 0 <= min_depth_m < max_depth_m") + k = np.asarray(intrinsics, dtype=np.float32) + c2r = np.asarray(camera_to_reference, dtype=np.float32) + if k.shape != (3, 3) or c2r.shape != (depth.shape[0], 4, 4): + raise ValueError( + f"Geometry shape mismatch: depth={depth.shape}, K={k.shape}, c2r={c2r.shape}" + ) + + z = depth[:, ::stride, ::stride] + v = np.arange(0, depth.shape[1], stride, dtype=np.float32) + u = np.arange(0, depth.shape[2], stride, dtype=np.float32) + grid_u, grid_v = np.meshgrid(u, v, indexing="xy") + x = (grid_u[None] - k[0, 2]) * z / k[0, 0] + y = (grid_v[None] - k[1, 2]) * z / k[1, 1] + camera_points = np.stack((x, y, z), axis=-1) + world_points = np.einsum( + "tij,thwj->thwi", c2r[:, :3, :3], camera_points + ) + c2r[:, None, None, :3, 3] + valid = np.isfinite(z) & (z >= min_depth_m) & (z <= max_depth_m) + if valid_mask is not None: + supplied = np.asarray(valid_mask, dtype=bool) + if supplied.shape != depth.shape: + raise ValueError(f"valid mask {supplied.shape} != depth {depth.shape}") + valid &= supplied[:, ::stride, ::stride] + world_points = world_points.astype(np.float32) + world_points[~valid] = np.nan + return world_points, valid + + +def calculate_world_point_cloud_metrics( + gt_points: np.ndarray, + generated_points: np.ndarray, + correspondence_valid: np.ndarray, + *, + depth_first_frame_conditioned: bool, +) -> dict[str, float]: + gt = np.asarray(gt_points, dtype=np.float32) + generated = np.asarray(generated_points, dtype=np.float32) + valid = np.asarray(correspondence_valid, dtype=bool) + if gt.shape != generated.shape or gt.shape[:-1] != valid.shape or gt.shape[-1] != 3: + raise ValueError("Point cloud metric shapes do not match") + valid &= np.isfinite(gt).all(axis=-1) & np.isfinite(generated).all(axis=-1) + if not np.any(valid): + raise ValueError("No valid point correspondences") + start = 1 if depth_first_frame_conditioned else 0 + generated_valid = valid[start:] + if not np.any(generated_valid): + raise ValueError("No valid generated-frame point correspondences") + error = generated - gt + distance = np.linalg.norm(error, axis=-1) + component_square = np.square(error) + return { + "point_world_l1_m": float(np.mean(np.abs(error[valid]))), + "point_world_rmse_m": float(np.sqrt(np.mean(component_square[valid]))), + "point_world_euclidean_mean_m": float(np.mean(distance[valid])), + "point_world_generated_rmse_m": float( + np.sqrt(np.mean(component_square[start:][generated_valid])) + ), + "point_world_generated_euclidean_mean_m": float( + np.mean(distance[start:][generated_valid]) + ), + "point_world_frame0_euclidean_mean_m": float(np.mean(distance[0][valid[0]])), + "point_world_valid_fraction": float(np.mean(valid)), + } + + +def write_binary_ply(path: Path, points: np.ndarray, colors_rgb: np.ndarray) -> None: + xyz = np.asarray(points, dtype=np.float32).reshape(-1, 3) + rgb = np.asarray(colors_rgb, dtype=np.uint8).reshape(-1, 3) + if len(xyz) != len(rgb): + raise ValueError("PLY points and colors have different lengths") + finite = np.isfinite(xyz).all(axis=1) + xyz, rgb = xyz[finite], rgb[finite] + vertices = np.empty( + len(xyz), + dtype=[("x", " tuple[float, float, float, float]: + arrays = [cloud.reshape(-1, 3) for cloud in clouds] + finite = np.concatenate([array[np.isfinite(array).all(axis=1)] for array in arrays]) + if len(finite) == 0: + raise ValueError("Cannot render an empty point cloud") + xy = finite[:, axes] + low = np.percentile(xy, 1.0, axis=0) + high = np.percentile(xy, 99.0, axis=0) + span = np.maximum(high - low, 0.1) + center = (low + high) / 2 + span *= 1.1 + return ( + float(center[0] - span[0] / 2), + float(center[0] + span[0] / 2), + float(center[1] - span[1] / 2), + float(center[1] + span[1] / 2), + ) + + +def _render_cloud( + points: np.ndarray, + colors: np.ndarray, + bounds: tuple[float, float, float, float], + axes: tuple[int, int], + size: int, + label: str, +) -> np.ndarray: + xyz = points.reshape(-1, 3) + rgb = colors.reshape(-1, 3) + finite = np.isfinite(xyz).all(axis=1) + xyz, rgb = xyz[finite], rgb[finite] + canvas = np.zeros((size, size, 3), dtype=np.uint8) + if len(xyz): + x0, x1, y0, y1 = bounds + px = np.rint((xyz[:, axes[0]] - x0) / max(x1 - x0, 1e-8) * (size - 1)) + py = np.rint((1.0 - (xyz[:, axes[1]] - y0) / max(y1 - y0, 1e-8)) * (size - 1)) + inside = (px >= 0) & (px < size) & (py >= 0) & (py < size) + canvas[py[inside].astype(np.int64), px[inside].astype(np.int64)] = rgb[inside] + cv2.rectangle(canvas, (0, 0), (size - 1, 28), (20, 20, 20), -1) + cv2.putText(canvas, label, (6, 19), cv2.FONT_HERSHEY_SIMPLEX, 0.48, + (255, 255, 255), 1, cv2.LINE_AA) + return canvas + + +def _write_rgb_video(path: Path, frames: np.ndarray, fps: int) -> None: + frames = np.asarray(frames, dtype=np.uint8) + height, width = frames.shape[1:3] + path.parent.mkdir(parents=True, exist_ok=True) + writer = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*"mp4v"), float(fps), + (width, height)) + if not writer.isOpened(): + raise RuntimeError(f"Failed to open point-cloud video writer: {path}") + try: + for frame in frames: + writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)) + finally: + writer.release() + if not path.is_file() or path.stat().st_size == 0: + raise RuntimeError(f"Empty point-cloud video: {path}") + + +def export_world_point_cloud_video( + output_dir: Path, + gt_depth_m: np.ndarray, + generated_depth_m: np.ndarray, + gt_rgb_uint8: np.ndarray, + generated_rgb_uint8: np.ndarray, + gt_valid_mask: np.ndarray, + geometry: WorldGeometry, + config: dict[str, Any], + *, + fps: int, + depth_first_frame_conditioned: bool, +) -> dict[str, float]: + """Save world point grids, per-frame PLY files, rendered videos, and metrics.""" + + stride = int(config.get("point_stride", 2)) + min_depth_m = float(config.get("min_depth_m", 0.05)) + max_depth_m = float(config.get("max_depth_m", 5.0)) + gt_points, gt_valid = depth_video_to_world_grid( + gt_depth_m, geometry.intrinsics, geometry.camera_to_reference, + stride=stride, min_depth_m=min_depth_m, max_depth_m=max_depth_m, + valid_mask=gt_valid_mask, + ) + generated_points, generated_valid = depth_video_to_world_grid( + generated_depth_m, geometry.intrinsics, geometry.camera_to_reference, + stride=stride, min_depth_m=min_depth_m, max_depth_m=max_depth_m, + ) + correspondence_valid = gt_valid & generated_valid + metrics = calculate_world_point_cloud_metrics( + gt_points, generated_points, correspondence_valid, + depth_first_frame_conditioned=depth_first_frame_conditioned, + ) + gt_colors = np.asarray(gt_rgb_uint8, dtype=np.uint8)[:, ::stride, ::stride] + generated_colors = np.asarray(generated_rgb_uint8, dtype=np.uint8)[:, ::stride, ::stride] + output_dir.mkdir(parents=True, exist_ok=True) + + if bool(config.get("export_npz", True)): + np.savez_compressed( + output_dir / "gt_world_point_cloud_video.npz", + points=gt_points, + colors=gt_colors, + valid=gt_valid, + source_indices=geometry.source_indices, + ) + np.savez_compressed( + output_dir / "generated_world_point_cloud_video.npz", + points=generated_points, + colors=generated_colors, + valid=generated_valid, + source_indices=geometry.source_indices, + ) + if bool(config.get("export_ply", True)): + for frame_index in range(len(gt_points)): + write_binary_ply( + output_dir / "ply" / "gt" / f"frame_{frame_index:03d}.ply", + gt_points[frame_index][gt_valid[frame_index]], + gt_colors[frame_index][gt_valid[frame_index]], + ) + write_binary_ply( + output_dir / "ply" / "generated" / f"frame_{frame_index:03d}.ply", + generated_points[frame_index][generated_valid[frame_index]], + generated_colors[frame_index][generated_valid[frame_index]], + ) + + render_view = str(config.get("render_view", "xy")) + view_axes = {"xy": (0, 1), "xz": (0, 2), "yz": (1, 2)} + if render_view not in view_axes: + raise ValueError(f"render_view must be one of {sorted(view_axes)}, got {render_view}") + if bool(config.get("render_video", True)): + axes = view_axes[render_view] + size = int(config.get("render_size", 512)) + if size < 64: + raise ValueError("point-cloud render_size must be >= 64") + bounds = _point_bounds([gt_points, generated_points], axes) + gt_frames, generated_frames, comparison_frames = [], [], [] + for frame_index in range(len(gt_points)): + gt_frame = _render_cloud( + gt_points[frame_index], gt_colors[frame_index], bounds, axes, size, + f"GT WORLD POINTS | {render_view.upper()} | f={frame_index:02d}", + ) + generated_frame = _render_cloud( + generated_points[frame_index], generated_colors[frame_index], bounds, + axes, size, + f"GEN WORLD POINTS | {render_view.upper()} | f={frame_index:02d}", + ) + gt_frames.append(gt_frame) + generated_frames.append(generated_frame) + comparison_frames.append(np.concatenate((gt_frame, generated_frame), axis=1)) + _write_rgb_video(output_dir / "gt_world_point_cloud.mp4", np.stack(gt_frames), fps) + _write_rgb_video( + output_dir / "generated_world_point_cloud.mp4", + np.stack(generated_frames), fps, + ) + _write_rgb_video( + output_dir / "gt_generated_world_point_cloud_side_by_side.mp4", + np.stack(comparison_frames), fps, + ) + + manifest = { + "format_version": 1, + "coordinate_frame": str(config.get("coordinate_frame", "first_camera_frame")), + "derivation": "metric_depth_transformed_to_first_camera_frame", + "pose_assumption": geometry.pose_assumption, + "source_metadata_path_basename": geometry.source_metadata_path.name, + "pose_vector_layout": str(config.get("pose_vector_layout", "txyz_qwxyz")), + "intrinsics": geometry.intrinsics.tolist(), + "camera_to_reference": geometry.camera_to_reference.tolist(), + "source_indices": geometry.source_indices.tolist(), + "point_stride": stride, + "min_depth_m": min_depth_m, + "max_depth_m": max_depth_m, + "metrics": metrics, + } + (output_dir / "geometry.json").write_text( + json.dumps(manifest, indent=2), encoding="utf-8" + ) + return metrics + + +def normalize_world_point_video( + world_points: np.ndarray, + valid_mask: np.ndarray, + xyz_min: list[float] | tuple[float, float, float] | np.ndarray, + xyz_max: list[float] | tuple[float, float, float] | np.ndarray, + *, + invalid_fill_reference: list[float] | tuple[float, float, float] | np.ndarray = (0.0, 0.0, 0.0), +) -> tuple[np.ndarray, float]: + """Map dense world XYZ [T,H,W,3] to frozen-Wan-VAE input [3,T,H,W].""" + + points = np.asarray(world_points, dtype=np.float32) + valid = np.asarray(valid_mask, dtype=bool) + low = np.asarray(xyz_min, dtype=np.float32) + high = np.asarray(xyz_max, dtype=np.float32) + fill = np.asarray(invalid_fill_reference, dtype=np.float32) + if points.ndim != 4 or points.shape[-1] != 3 or valid.shape != points.shape[:-1]: + raise ValueError(f"Expected points [T,H,W,3] and valid [T,H,W], got {points.shape}, {valid.shape}") + if low.shape != (3,) or high.shape != (3,) or fill.shape != (3,): + raise ValueError("xyz_min, xyz_max, and invalid_fill_reference must each contain 3 values") + if np.any(high <= low) or np.any(fill < low) or np.any(fill > high): + raise ValueError("Invalid world XYZ normalization bounds/fill") + finite_valid = valid & np.isfinite(points).all(axis=-1) + filled = np.broadcast_to(fill, points.shape).copy() + filled[finite_valid] = points[finite_valid] + outside = finite_valid & ( + ((filled < low) | (filled > high)).any(axis=-1) + ) + clipped_fraction = float(np.sum(outside) / max(np.sum(finite_valid), 1)) + normalized = 2.0 * (np.clip(filled, low, high) - low) / (high - low) - 1.0 + return normalized.transpose(3, 0, 1, 2).astype(np.float32), clipped_fraction + + +def denormalize_world_point_video( + vae_video: np.ndarray, + xyz_min: list[float] | tuple[float, float, float] | np.ndarray, + xyz_max: list[float] | tuple[float, float, float] | np.ndarray, +) -> np.ndarray: + """Map decoded point video [3,T,H,W] from [-1,1] back to [T,H,W,3] meters.""" + + video = np.asarray(vae_video, dtype=np.float32) + low = np.asarray(xyz_min, dtype=np.float32) + high = np.asarray(xyz_max, dtype=np.float32) + if video.ndim != 4 or video.shape[0] != 3: + raise ValueError(f"Expected decoded point video [3,T,H,W], got {video.shape}") + if low.shape != (3,) or high.shape != (3,) or np.any(high <= low): + raise ValueError("Invalid world XYZ normalization bounds") + normalized = np.moveaxis(np.clip(video, -1.0, 1.0), 0, -1) + return ((normalized + 1.0) * 0.5 * (high - low) + low).astype(np.float32) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/utils/dc_utils.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/utils/dc_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1380956cae365a2b107b09015538211962a0ea3a --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/utils/dc_utils.py @@ -0,0 +1,86 @@ +# This file is originally from DepthCrafter/depthcrafter/utils.py at main · Tencent/DepthCrafter +# SPDX-License-Identifier: MIT License license +# +# This file may have been modified by ByteDance Ltd. and/or its affiliates on [date of modification] +# Original file is released under [ MIT License license], with the full license text available at [https://github.com/Tencent/DepthCrafter?tab=License-1-ov-file]. +import numpy as np +import matplotlib.cm as cm +import imageio +try: + from decord import VideoReader, cpu + DECORD_AVAILABLE = True +except: + import cv2 + DECORD_AVAILABLE = False + +def ensure_even(value): + return value if value % 2 == 0 else value + 1 + +def read_video_frames(video_path, process_length, target_fps=-1, max_res=-1): + if DECORD_AVAILABLE: + vid = VideoReader(video_path, ctx=cpu(0)) + original_height, original_width = vid.get_batch([0]).shape[1:3] + height = original_height + width = original_width + if max_res > 0 and max(height, width) > max_res: + scale = max_res / max(original_height, original_width) + height = ensure_even(round(original_height * scale)) + width = ensure_even(round(original_width * scale)) + + vid = VideoReader(video_path, ctx=cpu(0), width=width, height=height) + + fps = vid.get_avg_fps() if target_fps == -1 else target_fps + stride = round(vid.get_avg_fps() / fps) + stride = max(stride, 1) + frames_idx = list(range(0, len(vid), stride)) + if process_length != -1 and process_length < len(frames_idx): + frames_idx = frames_idx[:process_length] + frames = vid.get_batch(frames_idx).asnumpy() + else: + cap = cv2.VideoCapture(video_path) + original_fps = cap.get(cv2.CAP_PROP_FPS) + original_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + original_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + + if max_res > 0 and max(original_height, original_width) > max_res: + scale = max_res / max(original_height, original_width) + height = round(original_height * scale) + width = round(original_width * scale) + + fps = original_fps if target_fps < 0 else target_fps + + stride = max(round(original_fps / fps), 1) + + frames = [] + frame_count = 0 + while cap.isOpened(): + ret, frame = cap.read() + if not ret or (process_length > 0 and frame_count >= process_length): + break + if frame_count % stride == 0: + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # Convert BGR to RGB + if max_res > 0 and max(original_height, original_width) > max_res: + frame = cv2.resize(frame, (width, height)) # Resize frame + frames.append(frame) + frame_count += 1 + cap.release() + frames = np.stack(frames, axis=0) + + return frames, fps + + +def save_video(frames, output_video_path, fps=10, is_depths=False, grayscale=False): + writer = imageio.get_writer(output_video_path, fps=fps, macro_block_size=1, codec='libx264', ffmpeg_params=['-crf', '18']) + if is_depths: + colormap = np.array(cm.get_cmap("inferno").colors) + d_min, d_max = frames.min(), frames.max() + for i in range(frames.shape[0]): + depth = frames[i] + depth_norm = ((depth - d_min) / (d_max - d_min) * 255).astype(np.uint8) + depth_vis = (colormap[depth_norm] * 255).astype(np.uint8) if not grayscale else depth_norm + writer.append_data(depth_vis) + else: + for i in range(frames.shape[0]): + writer.append_data(frames[i]) + + writer.close() diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/utils/util.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/utils/util.py new file mode 100644 index 0000000000000000000000000000000000000000..75ff80a841f2e30cb182a17c7a0e9e76ec062025 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/utils/util.py @@ -0,0 +1,74 @@ +# Copyright (2025) Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +def compute_scale_and_shift(prediction, target, mask, scale_only=False): + if scale_only: + return compute_scale(prediction, target, mask), 0 + else: + return compute_scale_and_shift_full(prediction, target, mask) + + +def compute_scale(prediction, target, mask): + # system matrix: A = [[a_00, a_01], [a_10, a_11]] + prediction = prediction.astype(np.float32) + target = target.astype(np.float32) + mask = mask.astype(np.float32) + + a_00 = np.sum(mask * prediction * prediction) + a_01 = np.sum(mask * prediction) + a_11 = np.sum(mask) + + # right hand side: b = [b_0, b_1] + b_0 = np.sum(mask * prediction * target) + + x_0 = b_0 / (a_00 + 1e-6) + + return x_0 + +def compute_scale_and_shift_full(prediction, target, mask): + # system matrix: A = [[a_00, a_01], [a_10, a_11]] + prediction = prediction.astype(np.float32) + target = target.astype(np.float32) + mask = mask.astype(np.float32) + + a_00 = np.sum(mask * prediction * prediction) + a_01 = np.sum(mask * prediction) + a_11 = np.sum(mask) + + b_0 = np.sum(mask * prediction * target) + b_1 = np.sum(mask * target) + + x_0 = 1 + x_1 = 0 + + det = a_00 * a_11 - a_01 * a_01 + + if det != 0: + x_0 = (a_11 * b_0 - a_01 * b_1) / det + x_1 = (-a_01 * b_0 + a_00 * b_1) / det + + return x_0, x_1 + + +def get_interpolate_frames(frame_list_pre, frame_list_post): + assert len(frame_list_pre) == len(frame_list_post) + min_w = 0.0 + max_w = 1.0 + step = (max_w - min_w) / (len(frame_list_pre)-1) + post_w_list = [min_w] + [i * step for i in range(1,len(frame_list_pre)-1)] + [max_w] + interpolated_frames = [] + for i in range(len(frame_list_pre)): + interpolated_frames.append(frame_list_pre[i] * (1-post_w_list[i]) + frame_list_post[i] * post_w_list[i]) + return interpolated_frames \ No newline at end of file diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2.py new file mode 100644 index 0000000000000000000000000000000000000000..83d250818c721c6df3b30d3f4352945527701615 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2.py @@ -0,0 +1,415 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the Apache License, Version 2.0 +# found in the LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/main/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py + +from functools import partial +import math +import logging +from typing import Sequence, Tuple, Union, Callable + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from torch.nn.init import trunc_normal_ + +from .dinov2_layers import Mlp, PatchEmbed, SwiGLUFFNFused, MemEffAttention, NestedTensorBlock as Block + + +logger = logging.getLogger("dinov2") + + +def named_apply(fn: Callable, module: nn.Module, name="", depth_first=True, include_root=False) -> nn.Module: + if not depth_first and include_root: + fn(module=module, name=name) + for child_name, child_module in module.named_children(): + child_name = ".".join((name, child_name)) if name else child_name + named_apply(fn=fn, module=child_module, name=child_name, depth_first=depth_first, include_root=True) + if depth_first and include_root: + fn(module=module, name=name) + return module + + +class BlockChunk(nn.ModuleList): + def forward(self, x): + for b in self: + x = b(x) + return x + + +class DinoVisionTransformer(nn.Module): + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4.0, + qkv_bias=True, + ffn_bias=True, + proj_bias=True, + drop_path_rate=0.0, + drop_path_uniform=False, + init_values=None, # for layerscale: None or 0 => no layerscale + embed_layer=PatchEmbed, + act_layer=nn.GELU, + block_fn=Block, + ffn_layer="mlp", + block_chunks=1, + num_register_tokens=0, + interpolate_antialias=False, + interpolate_offset=0.1, + ): + """ + Args: + img_size (int, tuple): input image size + patch_size (int, tuple): patch size + in_chans (int): number of input channels + embed_dim (int): embedding dimension + depth (int): depth of transformer + num_heads (int): number of attention heads + mlp_ratio (int): ratio of mlp hidden dim to embedding dim + qkv_bias (bool): enable bias for qkv if True + proj_bias (bool): enable bias for proj in attn if True + ffn_bias (bool): enable bias for ffn if True + drop_path_rate (float): stochastic depth rate + drop_path_uniform (bool): apply uniform drop rate across blocks + weight_init (str): weight init scheme + init_values (float): layer-scale init values + embed_layer (nn.Module): patch embedding layer + act_layer (nn.Module): MLP activation layer + block_fn (nn.Module): transformer block class + ffn_layer (str): "mlp", "swiglu", "swiglufused" or "identity" + block_chunks: (int) split block sequence into block_chunks units for FSDP wrap + num_register_tokens: (int) number of extra cls tokens (so-called "registers") + interpolate_antialias: (str) flag to apply anti-aliasing when interpolating positional embeddings + interpolate_offset: (float) work-around offset to apply when interpolating positional embeddings + """ + super().__init__() + norm_layer = partial(nn.LayerNorm, eps=1e-6) + + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + self.num_tokens = 1 + self.n_blocks = depth + self.num_heads = num_heads + self.patch_size = patch_size + self.num_register_tokens = num_register_tokens + self.interpolate_antialias = interpolate_antialias + self.interpolate_offset = interpolate_offset + + self.patch_embed = embed_layer(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) + assert num_register_tokens >= 0 + self.register_tokens = ( + nn.Parameter(torch.zeros(1, num_register_tokens, embed_dim)) if num_register_tokens else None + ) + + if drop_path_uniform is True: + dpr = [drop_path_rate] * depth + else: + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + + if ffn_layer == "mlp": + logger.info("using MLP layer as FFN") + ffn_layer = Mlp + elif ffn_layer == "swiglufused" or ffn_layer == "swiglu": + logger.info("using SwiGLU layer as FFN") + ffn_layer = SwiGLUFFNFused + elif ffn_layer == "identity": + logger.info("using Identity layer as FFN") + + def f(*args, **kwargs): + return nn.Identity() + + ffn_layer = f + else: + raise NotImplementedError + + blocks_list = [ + block_fn( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + proj_bias=proj_bias, + ffn_bias=ffn_bias, + drop_path=dpr[i], + norm_layer=norm_layer, + act_layer=act_layer, + ffn_layer=ffn_layer, + init_values=init_values, + ) + for i in range(depth) + ] + if block_chunks > 0: + self.chunked_blocks = True + chunked_blocks = [] + chunksize = depth // block_chunks + for i in range(0, depth, chunksize): + # this is to keep the block index consistent if we chunk the block list + chunked_blocks.append([nn.Identity()] * i + blocks_list[i : i + chunksize]) + self.blocks = nn.ModuleList([BlockChunk(p) for p in chunked_blocks]) + else: + self.chunked_blocks = False + self.blocks = nn.ModuleList(blocks_list) + + self.norm = norm_layer(embed_dim) + self.head = nn.Identity() + + self.mask_token = nn.Parameter(torch.zeros(1, embed_dim)) + + self.init_weights() + + def init_weights(self): + trunc_normal_(self.pos_embed, std=0.02) + nn.init.normal_(self.cls_token, std=1e-6) + if self.register_tokens is not None: + nn.init.normal_(self.register_tokens, std=1e-6) + named_apply(init_weights_vit_timm, self) + + def interpolate_pos_encoding(self, x, w, h): + previous_dtype = x.dtype + npatch = x.shape[1] - 1 + N = self.pos_embed.shape[1] - 1 + if npatch == N and w == h: + return self.pos_embed + pos_embed = self.pos_embed.float() + class_pos_embed = pos_embed[:, 0] + patch_pos_embed = pos_embed[:, 1:] + dim = x.shape[-1] + w0 = w // self.patch_size + h0 = h // self.patch_size + # we add a small number to avoid floating point error in the interpolation + # see discussion at https://github.com/facebookresearch/dino/issues/8 + # DINOv2 with register modify the interpolate_offset from 0.1 to 0.0 + w0, h0 = w0 + self.interpolate_offset, h0 + self.interpolate_offset + # w0, h0 = w0 + 0.1, h0 + 0.1 + + sqrt_N = math.sqrt(N) + sx, sy = float(w0) / sqrt_N, float(h0) / sqrt_N + patch_pos_embed = nn.functional.interpolate( + patch_pos_embed.reshape(1, int(sqrt_N), int(sqrt_N), dim).permute(0, 3, 1, 2), + scale_factor=(sx, sy), + # (int(w0), int(h0)), # to solve the upsampling shape issue + mode="bicubic", + antialias=self.interpolate_antialias + ) + + assert int(w0) == patch_pos_embed.shape[-2] + assert int(h0) == patch_pos_embed.shape[-1] + patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) + return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1).to(previous_dtype) + + def prepare_tokens_with_masks(self, x, masks=None): + B, nc, w, h = x.shape + x = self.patch_embed(x) + if masks is not None: + x = torch.where(masks.unsqueeze(-1), self.mask_token.to(x.dtype).unsqueeze(0), x) + + x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1) + x = x + self.interpolate_pos_encoding(x, w, h) + + if self.register_tokens is not None: + x = torch.cat( + ( + x[:, :1], + self.register_tokens.expand(x.shape[0], -1, -1), + x[:, 1:], + ), + dim=1, + ) + + return x + + def forward_features_list(self, x_list, masks_list): + x = [self.prepare_tokens_with_masks(x, masks) for x, masks in zip(x_list, masks_list)] + for blk in self.blocks: + x = blk(x) + + all_x = x + output = [] + for x, masks in zip(all_x, masks_list): + x_norm = self.norm(x) + output.append( + { + "x_norm_clstoken": x_norm[:, 0], + "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1], + "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :], + "x_prenorm": x, + "masks": masks, + } + ) + return output + + def forward_features(self, x, masks=None): + if isinstance(x, list): + return self.forward_features_list(x, masks) + + x = self.prepare_tokens_with_masks(x, masks) + + for blk in self.blocks: + x = blk(x) + + x_norm = self.norm(x) + return { + "x_norm_clstoken": x_norm[:, 0], + "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1], + "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :], + "x_prenorm": x, + "masks": masks, + } + + def _get_intermediate_layers_not_chunked(self, x, n=1): + x = self.prepare_tokens_with_masks(x) + # If n is an int, take the n last blocks. If it's a list, take them + output, total_block_len = [], len(self.blocks) + blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n + for i, blk in enumerate(self.blocks): + x = blk(x) + if i in blocks_to_take: + output.append(x) + assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found" + return output + + def _get_intermediate_layers_chunked(self, x, n=1): + x = self.prepare_tokens_with_masks(x) + output, i, total_block_len = [], 0, len(self.blocks[-1]) + # If n is an int, take the n last blocks. If it's a list, take them + blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n + for block_chunk in self.blocks: + for blk in block_chunk[i:]: # Passing the nn.Identity() + x = blk(x) + if i in blocks_to_take: + output.append(x) + i += 1 + assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found" + return output + + def get_intermediate_layers( + self, + x: torch.Tensor, + n: Union[int, Sequence] = 1, # Layers or n last layers to take + reshape: bool = False, + return_class_token: bool = False, + norm=True + ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]: + if self.chunked_blocks: + outputs = self._get_intermediate_layers_chunked(x, n) + else: + outputs = self._get_intermediate_layers_not_chunked(x, n) + if norm: + outputs = [self.norm(out) for out in outputs] + class_tokens = [out[:, 0] for out in outputs] + outputs = [out[:, 1 + self.num_register_tokens:] for out in outputs] + if reshape: + B, _, w, h = x.shape + outputs = [ + out.reshape(B, w // self.patch_size, h // self.patch_size, -1).permute(0, 3, 1, 2).contiguous() + for out in outputs + ] + if return_class_token: + return tuple(zip(outputs, class_tokens)) + return tuple(outputs) + + def forward(self, *args, is_training=False, **kwargs): + ret = self.forward_features(*args, **kwargs) + if is_training: + return ret + else: + return self.head(ret["x_norm_clstoken"]) + + +def init_weights_vit_timm(module: nn.Module, name: str = ""): + """ViT weight initialization, original timm impl (for reproducibility)""" + if isinstance(module, nn.Linear): + trunc_normal_(module.weight, std=0.02) + if module.bias is not None: + nn.init.zeros_(module.bias) + + +def vit_small(patch_size=16, num_register_tokens=0, **kwargs): + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=384, + depth=12, + num_heads=6, + mlp_ratio=4, + block_fn=partial(Block, attn_class=MemEffAttention), + num_register_tokens=num_register_tokens, + **kwargs, + ) + return model + + +def vit_base(patch_size=16, num_register_tokens=0, **kwargs): + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4, + block_fn=partial(Block, attn_class=MemEffAttention), + num_register_tokens=num_register_tokens, + **kwargs, + ) + return model + + +def vit_large(patch_size=16, num_register_tokens=0, **kwargs): + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=1024, + depth=24, + num_heads=16, + mlp_ratio=4, + block_fn=partial(Block, attn_class=MemEffAttention), + num_register_tokens=num_register_tokens, + **kwargs, + ) + return model + + +def vit_giant2(patch_size=16, num_register_tokens=0, **kwargs): + """ + Close to ViT-giant, with embed-dim 1536 and 24 heads => embed-dim per head 64 + """ + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=1536, + depth=40, + num_heads=24, + mlp_ratio=4, + block_fn=partial(Block, attn_class=MemEffAttention), + num_register_tokens=num_register_tokens, + **kwargs, + ) + return model + + +def DINOv2(model_name): + model_zoo = { + "vits": vit_small, + "vitb": vit_base, + "vitl": vit_large, + "vitg": vit_giant2 + } + + return model_zoo[model_name]( + img_size=518, + patch_size=14, + init_values=1.0, + ffn_layer="mlp" if model_name != "vitg" else "swiglufused", + block_chunks=0, + num_register_tokens=0, + interpolate_antialias=False, + interpolate_offset=0.1 + ) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/__init__.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8120f4bc83066cb3f825ce32daa3b437f88486f1 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from .mlp import Mlp +from .patch_embed import PatchEmbed +from .swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused +from .block import NestedTensorBlock +from .attention import MemEffAttention diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/attention.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..815a2bf53dbec496f6a184ed7d03bcecb7124262 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/attention.py @@ -0,0 +1,83 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py + +import logging + +from torch import Tensor +from torch import nn + + +logger = logging.getLogger("dinov2") + + +try: + from xformers.ops import memory_efficient_attention, unbind, fmha + + XFORMERS_AVAILABLE = True +except ImportError: + logger.warning("xFormers not available") + XFORMERS_AVAILABLE = False + + +class Attention(nn.Module): + def __init__( + self, + dim: int, + num_heads: int = 8, + qkv_bias: bool = False, + proj_bias: bool = True, + attn_drop: float = 0.0, + proj_drop: float = 0.0, + ) -> None: + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim, bias=proj_bias) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x: Tensor) -> Tensor: + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + + q, k, v = qkv[0] * self.scale, qkv[1], qkv[2] + attn = q @ k.transpose(-2, -1) + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class MemEffAttention(Attention): + def forward(self, x: Tensor, attn_bias=None) -> Tensor: + if not XFORMERS_AVAILABLE: + assert attn_bias is None, "xFormers is required for nested tensors usage" + return super().forward(x) + + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads) + + q, k, v = unbind(qkv, 2) + + x = memory_efficient_attention(q, k, v, attn_bias=attn_bias) + x = x.reshape([B, N, C]) + + x = self.proj(x) + x = self.proj_drop(x) + return x + + \ No newline at end of file diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/block.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/block.py new file mode 100644 index 0000000000000000000000000000000000000000..25488f57cc0ad3c692f86b62555f6668e2a66db1 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/block.py @@ -0,0 +1,252 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py + +import logging +from typing import Callable, List, Any, Tuple, Dict + +import torch +from torch import nn, Tensor + +from .attention import Attention, MemEffAttention +from .drop_path import DropPath +from .layer_scale import LayerScale +from .mlp import Mlp + + +logger = logging.getLogger("dinov2") + + +try: + from xformers.ops import fmha + from xformers.ops import scaled_index_add, index_select_cat + + XFORMERS_AVAILABLE = True +except ImportError: + logger.warning("xFormers not available") + XFORMERS_AVAILABLE = False + + +class Block(nn.Module): + def __init__( + self, + dim: int, + num_heads: int, + mlp_ratio: float = 4.0, + qkv_bias: bool = False, + proj_bias: bool = True, + ffn_bias: bool = True, + drop: float = 0.0, + attn_drop: float = 0.0, + init_values=None, + drop_path: float = 0.0, + act_layer: Callable[..., nn.Module] = nn.GELU, + norm_layer: Callable[..., nn.Module] = nn.LayerNorm, + attn_class: Callable[..., nn.Module] = Attention, + ffn_layer: Callable[..., nn.Module] = Mlp, + ) -> None: + super().__init__() + # print(f"biases: qkv: {qkv_bias}, proj: {proj_bias}, ffn: {ffn_bias}") + self.norm1 = norm_layer(dim) + self.attn = attn_class( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + proj_bias=proj_bias, + attn_drop=attn_drop, + proj_drop=drop, + ) + self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity() + self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = ffn_layer( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop, + bias=ffn_bias, + ) + self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity() + self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + self.sample_drop_ratio = drop_path + + def forward(self, x: Tensor) -> Tensor: + def attn_residual_func(x: Tensor) -> Tensor: + return self.ls1(self.attn(self.norm1(x))) + + def ffn_residual_func(x: Tensor) -> Tensor: + return self.ls2(self.mlp(self.norm2(x))) + + if self.training and self.sample_drop_ratio > 0.1: + # the overhead is compensated only for a drop path rate larger than 0.1 + x = drop_add_residual_stochastic_depth( + x, + residual_func=attn_residual_func, + sample_drop_ratio=self.sample_drop_ratio, + ) + x = drop_add_residual_stochastic_depth( + x, + residual_func=ffn_residual_func, + sample_drop_ratio=self.sample_drop_ratio, + ) + elif self.training and self.sample_drop_ratio > 0.0: + x = x + self.drop_path1(attn_residual_func(x)) + x = x + self.drop_path1(ffn_residual_func(x)) # FIXME: drop_path2 + else: + x = x + attn_residual_func(x) + x = x + ffn_residual_func(x) + return x + + +def drop_add_residual_stochastic_depth( + x: Tensor, + residual_func: Callable[[Tensor], Tensor], + sample_drop_ratio: float = 0.0, +) -> Tensor: + # 1) extract subset using permutation + b, n, d = x.shape + sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1) + brange = (torch.randperm(b, device=x.device))[:sample_subset_size] + x_subset = x[brange] + + # 2) apply residual_func to get residual + residual = residual_func(x_subset) + + x_flat = x.flatten(1) + residual = residual.flatten(1) + + residual_scale_factor = b / sample_subset_size + + # 3) add the residual + x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor) + return x_plus_residual.view_as(x) + + +def get_branges_scales(x, sample_drop_ratio=0.0): + b, n, d = x.shape + sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1) + brange = (torch.randperm(b, device=x.device))[:sample_subset_size] + residual_scale_factor = b / sample_subset_size + return brange, residual_scale_factor + + +def add_residual(x, brange, residual, residual_scale_factor, scaling_vector=None): + if scaling_vector is None: + x_flat = x.flatten(1) + residual = residual.flatten(1) + x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor) + else: + x_plus_residual = scaled_index_add( + x, brange, residual.to(dtype=x.dtype), scaling=scaling_vector, alpha=residual_scale_factor + ) + return x_plus_residual + + +attn_bias_cache: Dict[Tuple, Any] = {} + + +def get_attn_bias_and_cat(x_list, branges=None): + """ + this will perform the index select, cat the tensors, and provide the attn_bias from cache + """ + batch_sizes = [b.shape[0] for b in branges] if branges is not None else [x.shape[0] for x in x_list] + all_shapes = tuple((b, x.shape[1]) for b, x in zip(batch_sizes, x_list)) + if all_shapes not in attn_bias_cache.keys(): + seqlens = [] + for b, x in zip(batch_sizes, x_list): + for _ in range(b): + seqlens.append(x.shape[1]) + attn_bias = fmha.BlockDiagonalMask.from_seqlens(seqlens) + attn_bias._batch_sizes = batch_sizes + attn_bias_cache[all_shapes] = attn_bias + + if branges is not None: + cat_tensors = index_select_cat([x.flatten(1) for x in x_list], branges).view(1, -1, x_list[0].shape[-1]) + else: + tensors_bs1 = tuple(x.reshape([1, -1, *x.shape[2:]]) for x in x_list) + cat_tensors = torch.cat(tensors_bs1, dim=1) + + return attn_bias_cache[all_shapes], cat_tensors + + +def drop_add_residual_stochastic_depth_list( + x_list: List[Tensor], + residual_func: Callable[[Tensor, Any], Tensor], + sample_drop_ratio: float = 0.0, + scaling_vector=None, +) -> Tensor: + # 1) generate random set of indices for dropping samples in the batch + branges_scales = [get_branges_scales(x, sample_drop_ratio=sample_drop_ratio) for x in x_list] + branges = [s[0] for s in branges_scales] + residual_scale_factors = [s[1] for s in branges_scales] + + # 2) get attention bias and index+concat the tensors + attn_bias, x_cat = get_attn_bias_and_cat(x_list, branges) + + # 3) apply residual_func to get residual, and split the result + residual_list = attn_bias.split(residual_func(x_cat, attn_bias=attn_bias)) # type: ignore + + outputs = [] + for x, brange, residual, residual_scale_factor in zip(x_list, branges, residual_list, residual_scale_factors): + outputs.append(add_residual(x, brange, residual, residual_scale_factor, scaling_vector).view_as(x)) + return outputs + + +class NestedTensorBlock(Block): + def forward_nested(self, x_list: List[Tensor]) -> List[Tensor]: + """ + x_list contains a list of tensors to nest together and run + """ + assert isinstance(self.attn, MemEffAttention) + + if self.training and self.sample_drop_ratio > 0.0: + + def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor: + return self.attn(self.norm1(x), attn_bias=attn_bias) + + def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor: + return self.mlp(self.norm2(x)) + + x_list = drop_add_residual_stochastic_depth_list( + x_list, + residual_func=attn_residual_func, + sample_drop_ratio=self.sample_drop_ratio, + scaling_vector=self.ls1.gamma if isinstance(self.ls1, LayerScale) else None, + ) + x_list = drop_add_residual_stochastic_depth_list( + x_list, + residual_func=ffn_residual_func, + sample_drop_ratio=self.sample_drop_ratio, + scaling_vector=self.ls2.gamma if isinstance(self.ls1, LayerScale) else None, + ) + return x_list + else: + + def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor: + return self.ls1(self.attn(self.norm1(x), attn_bias=attn_bias)) + + def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor: + return self.ls2(self.mlp(self.norm2(x))) + + attn_bias, x = get_attn_bias_and_cat(x_list) + x = x + attn_residual_func(x, attn_bias=attn_bias) + x = x + ffn_residual_func(x) + return attn_bias.split(x) + + def forward(self, x_or_x_list): + if isinstance(x_or_x_list, Tensor): + return super().forward(x_or_x_list) + elif isinstance(x_or_x_list, list): + assert XFORMERS_AVAILABLE, "Please install xFormers for nested tensors usage" + return self.forward_nested(x_or_x_list) + else: + raise AssertionError diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/drop_path.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/drop_path.py new file mode 100644 index 0000000000000000000000000000000000000000..af05625984dd14682cc96a63bf0c97bab1f123b1 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/drop_path.py @@ -0,0 +1,35 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py + + +from torch import nn + + +def drop_path(x, drop_prob: float = 0.0, training: bool = False): + if drop_prob == 0.0 or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0: + random_tensor.div_(keep_prob) + output = x * random_tensor + return output + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/layer_scale.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/layer_scale.py new file mode 100644 index 0000000000000000000000000000000000000000..ca5daa52bd81d3581adeb2198ea5b7dba2a3aea1 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/layer_scale.py @@ -0,0 +1,28 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110 + +from typing import Union + +import torch +from torch import Tensor +from torch import nn + + +class LayerScale(nn.Module): + def __init__( + self, + dim: int, + init_values: Union[float, Tensor] = 1e-5, + inplace: bool = False, + ) -> None: + super().__init__() + self.inplace = inplace + self.gamma = nn.Parameter(init_values * torch.ones(dim)) + + def forward(self, x: Tensor) -> Tensor: + return x.mul_(self.gamma) if self.inplace else x * self.gamma diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/mlp.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/mlp.py new file mode 100644 index 0000000000000000000000000000000000000000..5e4b315f972f9a9f54aef1e4ef4e81b52976f018 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/mlp.py @@ -0,0 +1,41 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py + + +from typing import Callable, Optional + +from torch import Tensor, nn + + +class Mlp(nn.Module): + def __init__( + self, + in_features: int, + hidden_features: Optional[int] = None, + out_features: Optional[int] = None, + act_layer: Callable[..., nn.Module] = nn.GELU, + drop: float = 0.0, + bias: bool = True, + ) -> None: + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features, bias=bias) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features, bias=bias) + self.drop = nn.Dropout(drop) + + def forward(self, x: Tensor) -> Tensor: + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/patch_embed.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/patch_embed.py new file mode 100644 index 0000000000000000000000000000000000000000..574abe41175568d700a389b8b96d1ba554914779 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/patch_embed.py @@ -0,0 +1,89 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py + +from typing import Callable, Optional, Tuple, Union + +from torch import Tensor +import torch.nn as nn + + +def make_2tuple(x): + if isinstance(x, tuple): + assert len(x) == 2 + return x + + assert isinstance(x, int) + return (x, x) + + +class PatchEmbed(nn.Module): + """ + 2D image to patch embedding: (B,C,H,W) -> (B,N,D) + + Args: + img_size: Image size. + patch_size: Patch token size. + in_chans: Number of input image channels. + embed_dim: Number of linear projection output channels. + norm_layer: Normalization layer. + """ + + def __init__( + self, + img_size: Union[int, Tuple[int, int]] = 224, + patch_size: Union[int, Tuple[int, int]] = 16, + in_chans: int = 3, + embed_dim: int = 768, + norm_layer: Optional[Callable] = None, + flatten_embedding: bool = True, + ) -> None: + super().__init__() + + image_HW = make_2tuple(img_size) + patch_HW = make_2tuple(patch_size) + patch_grid_size = ( + image_HW[0] // patch_HW[0], + image_HW[1] // patch_HW[1], + ) + + self.img_size = image_HW + self.patch_size = patch_HW + self.patches_resolution = patch_grid_size + self.num_patches = patch_grid_size[0] * patch_grid_size[1] + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.flatten_embedding = flatten_embedding + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x: Tensor) -> Tensor: + _, _, H, W = x.shape + patch_H, patch_W = self.patch_size + + assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}" + assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}" + + x = self.proj(x) # B C H W + H, W = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) # B HW C + x = self.norm(x) + if not self.flatten_embedding: + x = x.reshape(-1, H, W, self.embed_dim) # B H W C + return x + + def flops(self) -> float: + Ho, Wo = self.patches_resolution + flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1]) + if self.norm is not None: + flops += Ho * Wo * self.embed_dim + return flops diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/swiglu_ffn.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/swiglu_ffn.py new file mode 100644 index 0000000000000000000000000000000000000000..b3324b266fb0a50ccf8c3a0ede2ae10ac4dfa03e --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dinov2_layers/swiglu_ffn.py @@ -0,0 +1,63 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Callable, Optional + +from torch import Tensor, nn +import torch.nn.functional as F + + +class SwiGLUFFN(nn.Module): + def __init__( + self, + in_features: int, + hidden_features: Optional[int] = None, + out_features: Optional[int] = None, + act_layer: Callable[..., nn.Module] = None, + drop: float = 0.0, + bias: bool = True, + ) -> None: + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias) + self.w3 = nn.Linear(hidden_features, out_features, bias=bias) + + def forward(self, x: Tensor) -> Tensor: + x12 = self.w12(x) + x1, x2 = x12.chunk(2, dim=-1) + hidden = F.silu(x1) * x2 + return self.w3(hidden) + + +try: + from xformers.ops import SwiGLU + + XFORMERS_AVAILABLE = True +except ImportError: + SwiGLU = SwiGLUFFN + XFORMERS_AVAILABLE = False + + +class SwiGLUFFNFused(SwiGLU): + def __init__( + self, + in_features: int, + hidden_features: Optional[int] = None, + out_features: Optional[int] = None, + act_layer: Callable[..., nn.Module] = None, + drop: float = 0.0, + bias: bool = True, + ) -> None: + out_features = out_features or in_features + hidden_features = hidden_features or in_features + hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8 + super().__init__( + in_features=in_features, + hidden_features=hidden_features, + out_features=out_features, + bias=bias, + ) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dpt.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dpt.py new file mode 100644 index 0000000000000000000000000000000000000000..8c43a1447799cc3550e7e5bba71577f7c5aec9e6 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dpt.py @@ -0,0 +1,160 @@ +# Copyright (2025) Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .util.blocks import FeatureFusionBlock, _make_scratch + + +def _make_fusion_block(features, use_bn, size=None): + return FeatureFusionBlock( + features, + nn.ReLU(False), + deconv=False, + bn=use_bn, + expand=False, + align_corners=True, + size=size, + ) + + +class ConvBlock(nn.Module): + def __init__(self, in_feature, out_feature): + super().__init__() + + self.conv_block = nn.Sequential( + nn.Conv2d(in_feature, out_feature, kernel_size=3, stride=1, padding=1), + nn.BatchNorm2d(out_feature), + nn.ReLU(True) + ) + + def forward(self, x): + return self.conv_block(x) + + +class DPTHead(nn.Module): + def __init__( + self, + in_channels, + features=256, + use_bn=False, + out_channels=[256, 512, 1024, 1024], + use_clstoken=False + ): + super(DPTHead, self).__init__() + + self.use_clstoken = use_clstoken + + self.projects = nn.ModuleList([ + nn.Conv2d( + in_channels=in_channels, + out_channels=out_channel, + kernel_size=1, + stride=1, + padding=0, + ) for out_channel in out_channels + ]) + + self.resize_layers = nn.ModuleList([ + nn.ConvTranspose2d( + in_channels=out_channels[0], + out_channels=out_channels[0], + kernel_size=4, + stride=4, + padding=0), + nn.ConvTranspose2d( + in_channels=out_channels[1], + out_channels=out_channels[1], + kernel_size=2, + stride=2, + padding=0), + nn.Identity(), + nn.Conv2d( + in_channels=out_channels[3], + out_channels=out_channels[3], + kernel_size=3, + stride=2, + padding=1) + ]) + + if use_clstoken: + self.readout_projects = nn.ModuleList() + for _ in range(len(self.projects)): + self.readout_projects.append( + nn.Sequential( + nn.Linear(2 * in_channels, in_channels), + nn.GELU())) + + self.scratch = _make_scratch( + out_channels, + features, + groups=1, + expand=False, + ) + + self.scratch.stem_transpose = None + + self.scratch.refinenet1 = _make_fusion_block(features, use_bn) + self.scratch.refinenet2 = _make_fusion_block(features, use_bn) + self.scratch.refinenet3 = _make_fusion_block(features, use_bn) + self.scratch.refinenet4 = _make_fusion_block(features, use_bn) + + head_features_1 = features + head_features_2 = 32 + + self.scratch.output_conv1 = nn.Conv2d(head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1) + self.scratch.output_conv2 = nn.Sequential( + nn.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1), + nn.ReLU(True), + nn.Conv2d(head_features_2, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True), + nn.Identity(), + ) + + def forward(self, out_features, patch_h, patch_w): + out = [] + for i, x in enumerate(out_features): + if self.use_clstoken: + x, cls_token = x[0], x[1] + readout = cls_token.unsqueeze(1).expand_as(x) + x = self.readout_projects[i](torch.cat((x, readout), -1)) + else: + x = x[0] + + x = x.permute(0, 2, 1).reshape((x.shape[0], x.shape[-1], patch_h, patch_w)) + + x = self.projects[i](x) + x = self.resize_layers[i](x) + + out.append(x) + + layer_1, layer_2, layer_3, layer_4 = out + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:]) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn, size=layer_2_rn.shape[2:]) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn, size=layer_1_rn.shape[2:]) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv1(path_1) + out = F.interpolate(out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True) + out = self.scratch.output_conv2(out) + + return out + \ No newline at end of file diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dpt_temporal.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dpt_temporal.py new file mode 100644 index 0000000000000000000000000000000000000000..d629a1127887ea561fadb5fc21aeae26313e24c6 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/dpt_temporal.py @@ -0,0 +1,125 @@ +# Copyright (2025) Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn.functional as F +import torch.nn as nn +from .dpt import DPTHead +from .motion_module.motion_module import TemporalModule +from easydict import EasyDict + + +class DPTHeadTemporal(DPTHead): + def __init__(self, + in_channels, + features=256, + use_bn=False, + out_channels=[256, 512, 1024, 1024], + use_clstoken=False, + num_frames=32, + pe='ape' + ): + super().__init__(in_channels, features, use_bn, out_channels, use_clstoken) + + assert num_frames > 0 + motion_module_kwargs = EasyDict(num_attention_heads = 8, + num_transformer_block = 1, + num_attention_blocks = 2, + temporal_max_len = num_frames, + zero_initialize = True, + pos_embedding_type = pe) + + self.motion_modules = nn.ModuleList([ + TemporalModule(in_channels=out_channels[2], + **motion_module_kwargs), + TemporalModule(in_channels=out_channels[3], + **motion_module_kwargs), + TemporalModule(in_channels=features, + **motion_module_kwargs), + TemporalModule(in_channels=features, + **motion_module_kwargs) + ]) + + def forward(self, out_features, patch_h, patch_w, frame_length, micro_batch_size=4, cached_hidden_state_list=None): + out = [] + for i, x in enumerate(out_features): + if self.use_clstoken: + x, cls_token = x[0], x[1] + readout = cls_token.unsqueeze(1).expand_as(x) + x = self.readout_projects[i](torch.cat((x, readout), -1)) + else: + x = x[0] + + x = x.permute(0, 2, 1).reshape((x.shape[0], x.shape[-1], patch_h, patch_w)).contiguous() + + B, T = x.shape[0] // frame_length, frame_length + x = self.projects[i](x) + x = self.resize_layers[i](x) + + out.append(x) + + layer_1, layer_2, layer_3, layer_4 = out + + B, T = layer_1.shape[0] // frame_length, frame_length + if cached_hidden_state_list is not None: + N = len(cached_hidden_state_list) // len(self.motion_modules) + else: + N = 0 + + layer_3, h0 = self.motion_modules[0](layer_3.unflatten(0, (B, T)).permute(0, 2, 1, 3, 4), None, None, cached_hidden_state_list[0:N] if N else None) + layer_3 = layer_3.permute(0, 2, 1, 3, 4).flatten(0, 1) + layer_4, h1 = self.motion_modules[1](layer_4.unflatten(0, (B, T)).permute(0, 2, 1, 3, 4), None, None, cached_hidden_state_list[N:2*N] if N else None) + layer_4 = layer_4.permute(0, 2, 1, 3, 4).flatten(0, 1) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:]) + path_4, h2 = self.motion_modules[2](path_4.unflatten(0, (B, T)).permute(0, 2, 1, 3, 4), None, None, cached_hidden_state_list[2*N:3*N] if N else None) + path_4 = path_4.permute(0, 2, 1, 3, 4).flatten(0, 1) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn, size=layer_2_rn.shape[2:]) + path_3, h3 = self.motion_modules[3](path_3.unflatten(0, (B, T)).permute(0, 2, 1, 3, 4), None, None, cached_hidden_state_list[3*N:] if N else None) + path_3 = path_3.permute(0, 2, 1, 3, 4).flatten(0, 1) + + batch_size = layer_1_rn.shape[0] + if batch_size <= micro_batch_size or batch_size % micro_batch_size != 0: + path_2 = self.scratch.refinenet2(path_3, layer_2_rn, size=layer_1_rn.shape[2:]) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv1(path_1) + out = F.interpolate( + out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True + ) + ori_type = out.dtype + with torch.autocast(device_type="cuda", enabled=False): + out = self.scratch.output_conv2(out.float()) + + output = out.to(ori_type) + else: + ret = [] + for i in range(0, batch_size, micro_batch_size): + path_2 = self.scratch.refinenet2(path_3[i:i + micro_batch_size], layer_2_rn[i:i + micro_batch_size], size=layer_1_rn[i:i + micro_batch_size].shape[2:]) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn[i:i + micro_batch_size]) + out = self.scratch.output_conv1(path_1) + out = F.interpolate( + out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True + ) + ori_type = out.dtype + with torch.autocast(device_type="cuda", enabled=False): + out = self.scratch.output_conv2(out.float()) + ret.append(out.to(ori_type)) + output = torch.cat(ret, dim=0) + + return output, h0 + h1 + h2 + h3 diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/motion_module/attention.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/motion_module/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..41f551ba17e95c008f3ad1db5475c342855dd2a4 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/motion_module/attention.py @@ -0,0 +1,429 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn + +try: + import xformers + import xformers.ops + + XFORMERS_AVAILABLE = True +except ImportError: + print("xFormers not available") + XFORMERS_AVAILABLE = False + + +class CrossAttention(nn.Module): + r""" + A cross attention layer. + + Parameters: + query_dim (`int`): The number of channels in the query. + cross_attention_dim (`int`, *optional*): + The number of channels in the encoder_hidden_states. If not given, defaults to `query_dim`. + heads (`int`, *optional*, defaults to 8): The number of heads to use for multi-head attention. + dim_head (`int`, *optional*, defaults to 64): The number of channels in each head. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + bias (`bool`, *optional*, defaults to False): + Set to `True` for the query, key, and value linear layers to contain a bias parameter. + """ + + def __init__( + self, + query_dim: int, + cross_attention_dim: Optional[int] = None, + heads: int = 8, + dim_head: int = 64, + dropout: float = 0.0, + bias=False, + upcast_attention: bool = False, + upcast_softmax: bool = False, + added_kv_proj_dim: Optional[int] = None, + norm_num_groups: Optional[int] = None, + ): + super().__init__() + inner_dim = dim_head * heads + cross_attention_dim = cross_attention_dim if cross_attention_dim is not None else query_dim + self.upcast_attention = upcast_attention + self.upcast_softmax = upcast_softmax + self.upcast_efficient_attention = False + + self.scale = dim_head**-0.5 + + self.heads = heads + # for slice_size > 0 the attention score computation + # is split across the batch axis to save memory + # You can set slice_size with `set_attention_slice` + self.sliceable_head_dim = heads + self._slice_size = None + self._use_memory_efficient_attention_xformers = False + self.added_kv_proj_dim = added_kv_proj_dim + + if norm_num_groups is not None: + self.group_norm = nn.GroupNorm(num_channels=inner_dim, num_groups=norm_num_groups, eps=1e-5, affine=True) + else: + self.group_norm = None + + self.to_q = nn.Linear(query_dim, inner_dim, bias=bias) + self.to_k = nn.Linear(cross_attention_dim, inner_dim, bias=bias) + self.to_v = nn.Linear(cross_attention_dim, inner_dim, bias=bias) + + if self.added_kv_proj_dim is not None: + self.add_k_proj = nn.Linear(added_kv_proj_dim, cross_attention_dim) + self.add_v_proj = nn.Linear(added_kv_proj_dim, cross_attention_dim) + + self.to_out = nn.ModuleList([]) + self.to_out.append(nn.Linear(inner_dim, query_dim)) + self.to_out.append(nn.Dropout(dropout)) + + def reshape_heads_to_batch_dim(self, tensor): + batch_size, seq_len, dim = tensor.shape + head_size = self.heads + tensor = tensor.reshape(batch_size, seq_len, head_size, dim // head_size).contiguous() + tensor = tensor.permute(0, 2, 1, 3).reshape(batch_size * head_size, seq_len, dim // head_size).contiguous() + return tensor + + def reshape_heads_to_4d(self, tensor): + batch_size, seq_len, dim = tensor.shape + head_size = self.heads + tensor = tensor.reshape(batch_size, seq_len, head_size, dim // head_size).contiguous() + return tensor + + def reshape_batch_dim_to_heads(self, tensor): + batch_size, seq_len, dim = tensor.shape + head_size = self.heads + tensor = tensor.reshape(batch_size // head_size, head_size, seq_len, dim).contiguous() + tensor = tensor.permute(0, 2, 1, 3).reshape(batch_size // head_size, seq_len, dim * head_size).contiguous() + return tensor + + def reshape_4d_to_heads(self, tensor): + batch_size, seq_len, head_size, dim = tensor.shape + head_size = self.heads + tensor = tensor.reshape(batch_size, seq_len, dim * head_size).contiguous() + return tensor + + def set_attention_slice(self, slice_size): + if slice_size is not None and slice_size > self.sliceable_head_dim: + raise ValueError(f"slice_size {slice_size} has to be smaller or equal to {self.sliceable_head_dim}.") + + self._slice_size = slice_size + + def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None): + batch_size, sequence_length, _ = hidden_states.shape + + encoder_hidden_states = encoder_hidden_states + + if self.group_norm is not None: + hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = self.to_q(hidden_states) + dim = query.shape[-1] + query = self.reshape_heads_to_batch_dim(query) + + if self.added_kv_proj_dim is not None: + key = self.to_k(hidden_states) + value = self.to_v(hidden_states) + encoder_hidden_states_key_proj = self.add_k_proj(encoder_hidden_states) + encoder_hidden_states_value_proj = self.add_v_proj(encoder_hidden_states) + + key = self.reshape_heads_to_batch_dim(key) + value = self.reshape_heads_to_batch_dim(value) + encoder_hidden_states_key_proj = self.reshape_heads_to_batch_dim(encoder_hidden_states_key_proj) + encoder_hidden_states_value_proj = self.reshape_heads_to_batch_dim(encoder_hidden_states_value_proj) + + key = torch.concat([encoder_hidden_states_key_proj, key], dim=1) + value = torch.concat([encoder_hidden_states_value_proj, value], dim=1) + else: + encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states + key = self.to_k(encoder_hidden_states) + value = self.to_v(encoder_hidden_states) + + key = self.reshape_heads_to_batch_dim(key) + value = self.reshape_heads_to_batch_dim(value) + + if attention_mask is not None: + if attention_mask.shape[-1] != query.shape[1]: + target_length = query.shape[1] + attention_mask = F.pad(attention_mask, (0, target_length), value=0.0) + attention_mask = attention_mask.repeat_interleave(self.heads, dim=0) + + # attention, what we cannot get enough of + if XFORMERS_AVAILABLE and self._use_memory_efficient_attention_xformers: + hidden_states = self._memory_efficient_attention_xformers(query, key, value, attention_mask) + # Some versions of xformers return output in fp32, cast it back to the dtype of the input + hidden_states = hidden_states.to(query.dtype) + else: + if self._slice_size is None or query.shape[0] // self._slice_size == 1: + hidden_states = self._attention(query, key, value, attention_mask) + else: + hidden_states = self._sliced_attention(query, key, value, sequence_length, dim, attention_mask) + + # linear proj + hidden_states = self.to_out[0](hidden_states) + + # dropout + hidden_states = self.to_out[1](hidden_states) + return hidden_states + + def _attention(self, query, key, value, attention_mask=None): + if self.upcast_attention: + query = query.float() + key = key.float() + + attention_scores = torch.baddbmm( + torch.empty(query.shape[0], query.shape[1], key.shape[1], dtype=query.dtype, device=query.device), + query, + key.transpose(-1, -2), + beta=0, + alpha=self.scale, + ) + + if attention_mask is not None: + attention_scores = attention_scores + attention_mask + + if self.upcast_softmax: + attention_scores = attention_scores.float() + + attention_probs = attention_scores.softmax(dim=-1) + + # cast back to the original dtype + attention_probs = attention_probs.to(value.dtype) + + # compute attention output + hidden_states = torch.bmm(attention_probs, value) + + # reshape hidden_states + hidden_states = self.reshape_batch_dim_to_heads(hidden_states) + return hidden_states + + def _sliced_attention(self, query, key, value, sequence_length, dim, attention_mask): + batch_size_attention = query.shape[0] + hidden_states = torch.zeros( + (batch_size_attention, sequence_length, dim // self.heads), device=query.device, dtype=query.dtype + ) + slice_size = self._slice_size if self._slice_size is not None else hidden_states.shape[0] + for i in range(hidden_states.shape[0] // slice_size): + start_idx = i * slice_size + end_idx = (i + 1) * slice_size + + query_slice = query[start_idx:end_idx] + key_slice = key[start_idx:end_idx] + + if self.upcast_attention: + query_slice = query_slice.float() + key_slice = key_slice.float() + + attn_slice = torch.baddbmm( + torch.empty(slice_size, query.shape[1], key.shape[1], dtype=query_slice.dtype, device=query.device), + query_slice, + key_slice.transpose(-1, -2), + beta=0, + alpha=self.scale, + ) + + if attention_mask is not None: + attn_slice = attn_slice + attention_mask[start_idx:end_idx] + + if self.upcast_softmax: + attn_slice = attn_slice.float() + + attn_slice = attn_slice.softmax(dim=-1) + + # cast back to the original dtype + attn_slice = attn_slice.to(value.dtype) + attn_slice = torch.bmm(attn_slice, value[start_idx:end_idx]) + + hidden_states[start_idx:end_idx] = attn_slice + + # reshape hidden_states + hidden_states = self.reshape_batch_dim_to_heads(hidden_states) + return hidden_states + + def _memory_efficient_attention_xformers(self, query, key, value, attention_mask): + if self.upcast_efficient_attention: + org_dtype = query.dtype + query = query.float() + key = key.float() + value = value.float() + if attention_mask is not None: + attention_mask = attention_mask.float() + hidden_states = self._memory_efficient_attention_split(query, key, value, attention_mask) + + if self.upcast_efficient_attention: + hidden_states = hidden_states.to(org_dtype) + + hidden_states = self.reshape_4d_to_heads(hidden_states) + return hidden_states + + # print("Errror: no xformers") + # raise NotImplementedError + + def _memory_efficient_attention_split(self, query, key, value, attention_mask): + batch_size = query.shape[0] + max_batch_size = 65535 + num_batches = (batch_size + max_batch_size - 1) // max_batch_size + results = [] + for i in range(num_batches): + start_idx = i * max_batch_size + end_idx = min((i + 1) * max_batch_size, batch_size) + query_batch = query[start_idx:end_idx] + key_batch = key[start_idx:end_idx] + value_batch = value[start_idx:end_idx] + if attention_mask is not None: + attention_mask_batch = attention_mask[start_idx:end_idx] + else: + attention_mask_batch = None + result = xformers.ops.memory_efficient_attention(query_batch, key_batch, value_batch, attn_bias=attention_mask_batch) + results.append(result) + full_result = torch.cat(results, dim=0) + return full_result + + +class FeedForward(nn.Module): + r""" + A feed-forward layer. + + Parameters: + dim (`int`): The number of channels in the input. + dim_out (`int`, *optional*): The number of channels in the output. If not given, defaults to `dim`. + mult (`int`, *optional*, defaults to 4): The multiplier to use for the hidden dimension. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + """ + + def __init__( + self, + dim: int, + dim_out: Optional[int] = None, + mult: int = 4, + dropout: float = 0.0, + activation_fn: str = "geglu", + ): + super().__init__() + inner_dim = int(dim * mult) + dim_out = dim_out if dim_out is not None else dim + + if activation_fn == "gelu": + act_fn = GELU(dim, inner_dim) + elif activation_fn == "geglu": + act_fn = GEGLU(dim, inner_dim) + elif activation_fn == "geglu-approximate": + act_fn = ApproximateGELU(dim, inner_dim) + + self.net = nn.ModuleList([]) + # project in + self.net.append(act_fn) + # project dropout + self.net.append(nn.Dropout(dropout)) + # project out + self.net.append(nn.Linear(inner_dim, dim_out)) + + def forward(self, hidden_states): + for module in self.net: + hidden_states = module(hidden_states) + return hidden_states + + +class GELU(nn.Module): + r""" + GELU activation function + """ + + def __init__(self, dim_in: int, dim_out: int): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out) + + def gelu(self, gate): + if gate.device.type != "mps": + return F.gelu(gate) + # mps: gelu is not implemented for float16 + return F.gelu(gate.to(dtype=torch.float32)).to(dtype=gate.dtype) + + def forward(self, hidden_states): + hidden_states = self.proj(hidden_states) + hidden_states = self.gelu(hidden_states) + return hidden_states + + +# feedforward +class GEGLU(nn.Module): + r""" + A variant of the gated linear unit activation function from https://arxiv.org/abs/2002.05202. + + Parameters: + dim_in (`int`): The number of channels in the input. + dim_out (`int`): The number of channels in the output. + """ + + def __init__(self, dim_in: int, dim_out: int): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out * 2) + + def gelu(self, gate): + if gate.device.type != "mps": + return F.gelu(gate) + # mps: gelu is not implemented for float16 + return F.gelu(gate.to(dtype=torch.float32)).to(dtype=gate.dtype) + + def forward(self, hidden_states): + hidden_states, gate = self.proj(hidden_states).chunk(2, dim=-1) + return hidden_states * self.gelu(gate) + + +class ApproximateGELU(nn.Module): + """ + The approximate form of Gaussian Error Linear Unit (GELU) + + For more details, see section 2: https://arxiv.org/abs/1606.08415 + """ + + def __init__(self, dim_in: int, dim_out: int): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out) + + def forward(self, x): + x = self.proj(x) + return x * torch.sigmoid(1.702 * x) + + +def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0): + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)) + t = torch.arange(end, device=freqs.device, dtype=torch.float32) + freqs = torch.outer(t, freqs) + freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex64 + return freqs_cis + + +def reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor): + ndim = x.ndim + assert 0 <= 1 < ndim + assert freqs_cis.shape == (x.shape[1], x.shape[-1]) + shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] + return freqs_cis.view(*shape) + + +def apply_rotary_emb( + xq: torch.Tensor, + xk: torch.Tensor, + freqs_cis: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2).contiguous()) + xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2).contiguous()) + freqs_cis = reshape_for_broadcast(freqs_cis, xq_) + xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(2) + xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(2) + return xq_out.type_as(xq), xk_out.type_as(xk) diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/motion_module/motion_module.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/motion_module/motion_module.py new file mode 100644 index 0000000000000000000000000000000000000000..d4d81a3e00dbbeb071cbccbf3bc1603178132f5f --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/motion_module/motion_module.py @@ -0,0 +1,321 @@ +# This file is originally from AnimateDiff/animatediff/models/motion_module.py at main · guoyww/AnimateDiff +# SPDX-License-Identifier: Apache-2.0 license +# +# This file may have been modified by ByteDance Ltd. and/or its affiliates on [date of modification] +# Original file was released under [ Apache-2.0 license], with the full license text available at [https://github.com/guoyww/AnimateDiff?tab=Apache-2.0-1-ov-file#readme]. +import torch +import torch.nn.functional as F +from torch import nn + +from .attention import CrossAttention, FeedForward, apply_rotary_emb, precompute_freqs_cis + +from einops import rearrange, repeat +import math + +try: + import xformers + import xformers.ops + + XFORMERS_AVAILABLE = True +except ImportError: + print("xFormers not available") + XFORMERS_AVAILABLE = False + + +def zero_module(module): + # Zero out the parameters of a module and return it. + for p in module.parameters(): + p.detach().zero_() + return module + + +class TemporalModule(nn.Module): + def __init__( + self, + in_channels, + num_attention_heads = 8, + num_transformer_block = 2, + num_attention_blocks = 2, + norm_num_groups = 32, + temporal_max_len = 32, + zero_initialize = True, + pos_embedding_type = "ape", + ): + super().__init__() + + self.temporal_transformer = TemporalTransformer3DModel( + in_channels=in_channels, + num_attention_heads=num_attention_heads, + attention_head_dim=in_channels // num_attention_heads, + num_layers=num_transformer_block, + num_attention_blocks=num_attention_blocks, + norm_num_groups=norm_num_groups, + temporal_max_len=temporal_max_len, + pos_embedding_type=pos_embedding_type, + ) + + if zero_initialize: + self.temporal_transformer.proj_out = zero_module(self.temporal_transformer.proj_out) + + def forward(self, input_tensor, encoder_hidden_states, attention_mask=None, cached_hidden_state_list=None): + hidden_states = input_tensor + hidden_states, output_hidden_state_list = self.temporal_transformer(hidden_states, encoder_hidden_states, attention_mask, cached_hidden_state_list) + + output = hidden_states + return output, output_hidden_state_list # list of hidden states + + +class TemporalTransformer3DModel(nn.Module): + def __init__( + self, + in_channels, + num_attention_heads, + attention_head_dim, + num_layers, + num_attention_blocks = 2, + norm_num_groups = 32, + temporal_max_len = 32, + pos_embedding_type = "ape", + ): + super().__init__() + + inner_dim = num_attention_heads * attention_head_dim + + self.norm = torch.nn.GroupNorm(num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True) + self.proj_in = nn.Linear(in_channels, inner_dim) + + self.transformer_blocks = nn.ModuleList( + [ + TemporalTransformerBlock( + dim=inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + num_attention_blocks=num_attention_blocks, + temporal_max_len=temporal_max_len, + pos_embedding_type=pos_embedding_type, + ) + for d in range(num_layers) + ] + ) + self.proj_out = nn.Linear(inner_dim, in_channels) + + def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, cached_hidden_state_list=None): + assert hidden_states.dim() == 5, f"Expected hidden_states to have ndim=5, but got ndim={hidden_states.dim()}." + output_hidden_state_list = [] + + video_length = hidden_states.shape[2] + hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w") + + batch, channel, height, width = hidden_states.shape + residual = hidden_states + + hidden_states = self.norm(hidden_states) + inner_dim = hidden_states.shape[1] + hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * width, inner_dim).contiguous() + hidden_states = self.proj_in(hidden_states) + + # Transformer Blocks + if cached_hidden_state_list is not None: + n = len(cached_hidden_state_list) // len(self.transformer_blocks) + else: + n = 0 + for i, block in enumerate(self.transformer_blocks): + hidden_states, hidden_state_list = block(hidden_states, encoder_hidden_states=encoder_hidden_states, video_length=video_length, attention_mask=attention_mask, + cached_hidden_state_list=cached_hidden_state_list[i*n:(i+1)*n] if n else None) + output_hidden_state_list.extend(hidden_state_list) + + # output + hidden_states = self.proj_out(hidden_states) + hidden_states = hidden_states.reshape(batch, height, width, inner_dim).permute(0, 3, 1, 2).contiguous() + + output = hidden_states + residual + output = rearrange(output, "(b f) c h w -> b c f h w", f=video_length) + + return output, output_hidden_state_list + + +class TemporalTransformerBlock(nn.Module): + def __init__( + self, + dim, + num_attention_heads, + attention_head_dim, + num_attention_blocks = 2, + temporal_max_len = 32, + pos_embedding_type = "ape", + ): + super().__init__() + + self.attention_blocks = nn.ModuleList( + [ + TemporalAttention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + temporal_max_len=temporal_max_len, + pos_embedding_type=pos_embedding_type, + ) + for i in range(num_attention_blocks) + ] + ) + self.norms = nn.ModuleList( + [ + nn.LayerNorm(dim) + for i in range(num_attention_blocks) + ] + ) + + self.ff = FeedForward(dim, dropout=0.0, activation_fn="geglu") + self.ff_norm = nn.LayerNorm(dim) + + + def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, video_length=None, cached_hidden_state_list=None): + output_hidden_state_list = [] + for i, (attention_block, norm) in enumerate(zip(self.attention_blocks, self.norms)): + norm_hidden_states = norm(hidden_states) + residual_hidden_states, output_hidden_states = attention_block( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + video_length=video_length, + attention_mask=attention_mask, + cached_hidden_states=cached_hidden_state_list[i] if cached_hidden_state_list is not None else None, + ) + hidden_states = residual_hidden_states + hidden_states + output_hidden_state_list.append(output_hidden_states) + + hidden_states = self.ff(self.ff_norm(hidden_states)) + hidden_states + + output = hidden_states + return output, output_hidden_state_list + + +class PositionalEncoding(nn.Module): + def __init__( + self, + d_model, + dropout = 0., + max_len = 32 + ): + super().__init__() + self.dropout = nn.Dropout(p=dropout) + position = torch.arange(max_len).unsqueeze(1) + div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model)) + pe = torch.zeros(1, max_len, d_model) + pe[0, :, 0::2] = torch.sin(position * div_term) + pe[0, :, 1::2] = torch.cos(position * div_term) + self.register_buffer('pe', pe) + + def forward(self, x): + x = x + self.pe[:, :x.size(1)].to(x.dtype) + return self.dropout(x) + +class TemporalAttention(CrossAttention): + def __init__( + self, + temporal_max_len = 32, + pos_embedding_type = "ape", + *args, **kwargs + ): + super().__init__(*args, **kwargs) + + self.pos_embedding_type = pos_embedding_type + self._use_memory_efficient_attention_xformers = True + + self.pos_encoder = None + self.freqs_cis = None + if self.pos_embedding_type == "ape": + self.pos_encoder = PositionalEncoding( + kwargs["query_dim"], + dropout=0., + max_len=temporal_max_len + ) + + elif self.pos_embedding_type == "rope": + self.freqs_cis = precompute_freqs_cis( + kwargs["query_dim"], + temporal_max_len + ) + + else: + raise NotImplementedError + + def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, video_length=None, cached_hidden_states=None): + # TODO: support cache for these + assert encoder_hidden_states is None + assert attention_mask is None + + d = hidden_states.shape[1] + d_in = 0 + if cached_hidden_states is None: + hidden_states = rearrange(hidden_states, "(b f) d c -> (b d) f c", f=video_length) + input_hidden_states = hidden_states # (bxd) f c + else: + hidden_states = rearrange(hidden_states, "(b f) d c -> (b d) f c", f=1) + input_hidden_states = hidden_states + d_in = cached_hidden_states.shape[1] + hidden_states = torch.cat([cached_hidden_states, hidden_states], dim=1) + + if self.pos_encoder is not None: + hidden_states = self.pos_encoder(hidden_states) + + encoder_hidden_states = repeat(encoder_hidden_states, "b n c -> (b d) n c", d=d) if encoder_hidden_states is not None else encoder_hidden_states + + if self.group_norm is not None: + hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = self.to_q(hidden_states[:, d_in:, ...]) + dim = query.shape[-1] + + if self.added_kv_proj_dim is not None: + raise NotImplementedError + + encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states + key = self.to_k(encoder_hidden_states) + value = self.to_v(encoder_hidden_states) + + if self.freqs_cis is not None: + seq_len = query.shape[1] + freqs_cis = self.freqs_cis[:seq_len].to(query.device) + query, key = apply_rotary_emb(query, key, freqs_cis) + + if attention_mask is not None: + if attention_mask.shape[-1] != query.shape[1]: + target_length = query.shape[1] + attention_mask = F.pad(attention_mask, (0, target_length), value=0.0) + attention_mask = attention_mask.repeat_interleave(self.heads, dim=0) + + + use_memory_efficient = XFORMERS_AVAILABLE and self._use_memory_efficient_attention_xformers + if use_memory_efficient and (dim // self.heads) % 8 != 0: + # print('Warning: the dim {} cannot be divided by 8. Fall into normal attention'.format(dim // self.heads)) + use_memory_efficient = False + + # attention, what we cannot get enough of + if use_memory_efficient: + query = self.reshape_heads_to_4d(query) + key = self.reshape_heads_to_4d(key) + value = self.reshape_heads_to_4d(value) + + hidden_states = self._memory_efficient_attention_xformers(query, key, value, attention_mask) + # Some versions of xformers return output in fp32, cast it back to the dtype of the input + hidden_states = hidden_states.to(query.dtype) + else: + query = self.reshape_heads_to_batch_dim(query) + key = self.reshape_heads_to_batch_dim(key) + value = self.reshape_heads_to_batch_dim(value) + + if self._slice_size is None or query.shape[0] // self._slice_size == 1: + hidden_states = self._attention(query, key, value, attention_mask) + else: + raise NotImplementedError + # hidden_states = self._sliced_attention(query, key, value, sequence_length, dim, attention_mask) + + # linear proj + hidden_states = self.to_out[0](hidden_states) + + # dropout + hidden_states = self.to_out[1](hidden_states) + + hidden_states = rearrange(hidden_states, "(b d) f c -> (b f) d c", d=d) + + return hidden_states, input_hidden_states diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/util/blocks.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/util/blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..0be16c053d8ae65bf33c5221351add16ecd6fb15 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/util/blocks.py @@ -0,0 +1,162 @@ +import torch.nn as nn + + +def _make_scratch(in_shape, out_shape, groups=1, expand=False): + scratch = nn.Module() + + out_shape1 = out_shape + out_shape2 = out_shape + out_shape3 = out_shape + if len(in_shape) >= 4: + out_shape4 = out_shape + + if expand: + out_shape1 = out_shape + out_shape2 = out_shape * 2 + out_shape3 = out_shape * 4 + if len(in_shape) >= 4: + out_shape4 = out_shape * 8 + + scratch.layer1_rn = nn.Conv2d( + in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + scratch.layer2_rn = nn.Conv2d( + in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + scratch.layer3_rn = nn.Conv2d( + in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + if len(in_shape) >= 4: + scratch.layer4_rn = nn.Conv2d( + in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + + return scratch + + +class ResidualConvUnit(nn.Module): + """Residual convolution module.""" + + def __init__(self, features, activation, bn): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.bn = bn + + self.groups = 1 + + self.conv1 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups + ) + + self.conv2 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups + ) + + if self.bn is True: + self.bn1 = nn.BatchNorm2d(features) + self.bn2 = nn.BatchNorm2d(features) + + self.activation = activation + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + + out = self.activation(x) + out = self.conv1(out) + if self.bn is True: + out = self.bn1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.bn is True: + out = self.bn2(out) + + if self.groups > 1: + out = self.conv_merge(out) + + return self.skip_add.add(out, x) + + +class FeatureFusionBlock(nn.Module): + """Feature fusion block.""" + + def __init__( + self, + features, + activation, + deconv=False, + bn=False, + expand=False, + align_corners=True, + size=None, + ): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.deconv = deconv + self.align_corners = align_corners + + self.groups = 1 + + self.expand = expand + out_features = features + if self.expand is True: + out_features = features // 2 + + self.out_conv = nn.Conv2d( + features, out_features, kernel_size=1, stride=1, padding=0, bias=True, groups=1 + ) + + self.resConfUnit1 = ResidualConvUnit(features, activation, bn) + self.resConfUnit2 = ResidualConvUnit(features, activation, bn) + + self.skip_add = nn.quantized.FloatFunctional() + + self.size = size + + def forward(self, *xs, size=None): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + res = self.resConfUnit1(xs[1]) + output = self.skip_add.add(output, res) + + output = self.resConfUnit2(output) + + if (size is None) and (self.size is None): + modifier = {"scale_factor": 2} + elif size is None: + modifier = {"size": self.size} + else: + modifier = {"size": size} + + output = nn.functional.interpolate( + output.contiguous(), **modifier, mode="bilinear", align_corners=self.align_corners + ) + + output = self.out_conv(output) + + return output diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/util/transform.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/util/transform.py new file mode 100644 index 0000000000000000000000000000000000000000..b14aacd44ea086b01725a9ca68bb49eadcf37d73 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/util/transform.py @@ -0,0 +1,158 @@ +import numpy as np +import cv2 + + +class Resize(object): + """Resize sample to given size (width, height). + """ + + def __init__( + self, + width, + height, + resize_target=True, + keep_aspect_ratio=False, + ensure_multiple_of=1, + resize_method="lower_bound", + image_interpolation_method=cv2.INTER_AREA, + ): + """Init. + + Args: + width (int): desired output width + height (int): desired output height + resize_target (bool, optional): + True: Resize the full sample (image, mask, target). + False: Resize image only. + Defaults to True. + keep_aspect_ratio (bool, optional): + True: Keep the aspect ratio of the input sample. + Output sample might not have the given width and height, and + resize behaviour depends on the parameter 'resize_method'. + Defaults to False. + ensure_multiple_of (int, optional): + Output width and height is constrained to be multiple of this parameter. + Defaults to 1. + resize_method (str, optional): + "lower_bound": Output will be at least as large as the given size. + "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.) + "minimal": Scale as least as possible. (Output size might be smaller than given size.) + Defaults to "lower_bound". + """ + self.__width = width + self.__height = height + + self.__resize_target = resize_target + self.__keep_aspect_ratio = keep_aspect_ratio + self.__multiple_of = ensure_multiple_of + self.__resize_method = resize_method + self.__image_interpolation_method = image_interpolation_method + + def constrain_to_multiple_of(self, x, min_val=0, max_val=None): + y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if max_val is not None and y > max_val: + y = (np.floor(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if y < min_val: + y = (np.ceil(x / self.__multiple_of) * self.__multiple_of).astype(int) + + return y + + def get_size(self, width, height): + # determine new height and width + scale_height = self.__height / height + scale_width = self.__width / width + + if self.__keep_aspect_ratio: + if self.__resize_method == "lower_bound": + # scale such that output size is lower bound + if scale_width > scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "upper_bound": + # scale such that output size is upper bound + if scale_width < scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "minimal": + # scale as least as possbile + if abs(1 - scale_width) < abs(1 - scale_height): + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + else: + raise ValueError(f"resize_method {self.__resize_method} not implemented") + + if self.__resize_method == "lower_bound": + new_height = self.constrain_to_multiple_of(scale_height * height, min_val=self.__height) + new_width = self.constrain_to_multiple_of(scale_width * width, min_val=self.__width) + elif self.__resize_method == "upper_bound": + new_height = self.constrain_to_multiple_of(scale_height * height, max_val=self.__height) + new_width = self.constrain_to_multiple_of(scale_width * width, max_val=self.__width) + elif self.__resize_method == "minimal": + new_height = self.constrain_to_multiple_of(scale_height * height) + new_width = self.constrain_to_multiple_of(scale_width * width) + else: + raise ValueError(f"resize_method {self.__resize_method} not implemented") + + return (new_width, new_height) + + def __call__(self, sample): + width, height = self.get_size(sample["image"].shape[1], sample["image"].shape[0]) + + # resize sample + sample["image"] = cv2.resize(sample["image"], (width, height), interpolation=self.__image_interpolation_method) + + if self.__resize_target: + if "depth" in sample: + sample["depth"] = cv2.resize(sample["depth"], (width, height), interpolation=cv2.INTER_NEAREST) + + if "mask" in sample: + sample["mask"] = cv2.resize(sample["mask"].astype(np.float32), (width, height), interpolation=cv2.INTER_NEAREST) + + return sample + + +class NormalizeImage(object): + """Normlize image by given mean and std. + """ + + def __init__(self, mean, std): + self.__mean = mean + self.__std = std + + def __call__(self, sample): + sample["image"] = (sample["image"] - self.__mean) / self.__std + + return sample + + +class PrepareForNet(object): + """Prepare sample for usage as network input. + """ + + def __init__(self): + pass + + def __call__(self, sample): + image = np.transpose(sample["image"], (2, 0, 1)) + sample["image"] = np.ascontiguousarray(image).astype(np.float32) + + if "depth" in sample: + depth = sample["depth"].astype(np.float32) + sample["depth"] = np.ascontiguousarray(depth) + + if "mask" in sample: + sample["mask"] = sample["mask"].astype(np.float32) + sample["mask"] = np.ascontiguousarray(sample["mask"]) + + return sample \ No newline at end of file diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/video_depth.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/video_depth.py new file mode 100644 index 0000000000000000000000000000000000000000..5598d302afbefba2a6ed33bdaa1bce84260ef9c0 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/video_depth.py @@ -0,0 +1,163 @@ +# Copyright (2025) Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn.functional as F +import torch.nn as nn +from torchvision.transforms import Compose +import cv2 +from tqdm import tqdm +import numpy as np +import gc + +from .dinov2 import DINOv2 +from .dpt_temporal import DPTHeadTemporal +from .util.transform import Resize, NormalizeImage, PrepareForNet + +from utils.util import compute_scale_and_shift, get_interpolate_frames + +# infer settings, do not change +INFER_LEN = 32 +OVERLAP = 10 +KEYFRAMES = [0,12,24,25,26,27,28,29,30,31] +INTERP_LEN = 8 + +class VideoDepthAnything(nn.Module): + def __init__( + self, + encoder='vitl', + features=256, + out_channels=[256, 512, 1024, 1024], + use_bn=False, + use_clstoken=False, + num_frames=32, + pe='ape', + metric=False, + ): + super(VideoDepthAnything, self).__init__() + + self.intermediate_layer_idx = { + 'vits': [2, 5, 8, 11], + "vitb": [2, 5, 8, 11], + 'vitl': [4, 11, 17, 23] + } + + self.encoder = encoder + self.pretrained = DINOv2(model_name=encoder) + + self.head = DPTHeadTemporal(self.pretrained.embed_dim, features, use_bn, out_channels=out_channels, use_clstoken=use_clstoken, num_frames=num_frames, pe=pe) + self.metric = metric + + def forward(self, x): + B, T, C, H, W = x.shape + patch_h, patch_w = H // 14, W // 14 + features = self.pretrained.get_intermediate_layers(x.flatten(0,1), self.intermediate_layer_idx[self.encoder], return_class_token=True) + depth = self.head(features, patch_h, patch_w, T)[0] + depth = F.interpolate(depth, size=(H, W), mode="bilinear", align_corners=True) + depth = F.relu(depth) + return depth.squeeze(1).unflatten(0, (B, T)) # return shape [B, T, H, W] + + def infer_video_depth(self, frames, target_fps, input_size=518, device='cuda', fp32=False): + frame_height, frame_width = frames[0].shape[:2] + ratio = max(frame_height, frame_width) / min(frame_height, frame_width) + if ratio > 1.78: # we recommend to process video with ratio smaller than 16:9 due to memory limitation + input_size = int(input_size * 1.777 / ratio) + input_size = round(input_size / 14) * 14 + + transform = Compose([ + Resize( + width=input_size, + height=input_size, + resize_target=False, + keep_aspect_ratio=True, + ensure_multiple_of=14, + resize_method='lower_bound', + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + PrepareForNet(), + ]) + + frame_list = [frames[i] for i in range(frames.shape[0])] + frame_step = INFER_LEN - OVERLAP + org_video_len = len(frame_list) + append_frame_len = (frame_step - (org_video_len % frame_step)) % frame_step + (INFER_LEN - frame_step) + frame_list = frame_list + [frame_list[-1].copy()] * append_frame_len + + depth_list = [] + pre_input = None + for frame_id in tqdm(range(0, org_video_len, frame_step)): + cur_list = [] + for i in range(INFER_LEN): + cur_list.append(torch.from_numpy(transform({'image': frame_list[frame_id+i].astype(np.float32) / 255.0})['image']).unsqueeze(0).unsqueeze(0)) + cur_input = torch.cat(cur_list, dim=1).to(device) + if pre_input is not None: + cur_input[:, :OVERLAP, ...] = pre_input[:, KEYFRAMES, ...] + + with torch.no_grad(): + with torch.autocast(device_type=device, enabled=(not fp32)): + depth = self.forward(cur_input) # depth shape: [1, T, H, W] + + depth = depth.to(cur_input.dtype) + depth = F.interpolate(depth.flatten(0,1).unsqueeze(1), size=(frame_height, frame_width), mode='bilinear', align_corners=True) + depth_list += [depth[i][0].cpu().numpy() for i in range(depth.shape[0])] + + pre_input = cur_input + + del frame_list + gc.collect() + + depth_list_aligned = [] + ref_align = [] + align_len = OVERLAP - INTERP_LEN + kf_align_list = KEYFRAMES[:align_len] + + for frame_id in range(0, len(depth_list), INFER_LEN): + if len(depth_list_aligned) == 0: + depth_list_aligned += depth_list[:INFER_LEN] + for kf_id in kf_align_list: + ref_align.append(depth_list[frame_id+kf_id]) + else: + curr_align = [] + for i in range(len(kf_align_list)): + curr_align.append(depth_list[frame_id+i]) + + if self.metric: + scale, shift = 1.0, 0.0 + else: + scale, shift = compute_scale_and_shift(np.concatenate(curr_align), + np.concatenate(ref_align), + np.concatenate(np.ones_like(ref_align)==1)) + + pre_depth_list = depth_list_aligned[-INTERP_LEN:] + post_depth_list = depth_list[frame_id+align_len:frame_id+OVERLAP] + for i in range(len(post_depth_list)): + post_depth_list[i] = post_depth_list[i] * scale + shift + post_depth_list[i][post_depth_list[i]<0] = 0 + depth_list_aligned[-INTERP_LEN:] = get_interpolate_frames(pre_depth_list, post_depth_list) + + for i in range(OVERLAP, INFER_LEN): + new_depth = depth_list[frame_id+i] * scale + shift + new_depth[new_depth<0] = 0 + depth_list_aligned.append(new_depth) + + ref_align = ref_align[:1] + for kf_id in kf_align_list[1:]: + new_depth = depth_list[frame_id+kf_id] * scale + shift + new_depth[new_depth<0] = 0 + ref_align.append(new_depth) + + depth_list = depth_list_aligned + + return np.stack(depth_list[:org_video_len], axis=0), target_fps + diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/video_depth_stream.py b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/video_depth_stream.py new file mode 100644 index 0000000000000000000000000000000000000000..c56847a10cc07f04d1551b315ca74b83ee101635 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/source/vda/video_depth_anything/video_depth_stream.py @@ -0,0 +1,161 @@ +# Copyright (2025) Bytedance Ltd. and/or its affiliates + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn.functional as F +import torch.nn as nn +from torchvision.transforms import Compose +import cv2 +import numpy as np + +from .dinov2 import DINOv2 +from .dpt_temporal import DPTHeadTemporal +from .util.transform import Resize, NormalizeImage, PrepareForNet + +from utils.util import compute_scale_and_shift, get_interpolate_frames + +# infer settings, do not change +INFER_LEN = 32 +OVERLAP = 10 +INTERP_LEN = 8 + +class VideoDepthAnything(nn.Module): + def __init__( + self, + encoder='vitl', + features=256, + out_channels=[256, 512, 1024, 1024], + use_bn=False, + use_clstoken=False, + num_frames=32, + pe='ape' + ): + super(VideoDepthAnything, self).__init__() + + self.intermediate_layer_idx = { + 'vits': [2, 5, 8, 11], + "vitb": [2, 5, 8, 11], + 'vitl': [4, 11, 17, 23] + } + + self.encoder = encoder + self.pretrained = DINOv2(model_name=encoder) + + self.head = DPTHeadTemporal(self.pretrained.embed_dim, features, use_bn, out_channels=out_channels, use_clstoken=use_clstoken, num_frames=num_frames, pe=pe) + self.transform = None + self.frame_id_list = [] + self.frame_cache_list = [] + self.gap = (INFER_LEN - OVERLAP) * 2 - 1 - (OVERLAP - INTERP_LEN) + assert self.gap == 41 + self.id = -1 + + def forward(self, x): + return self.forward_depth(self.forward_features(x), x.shape)[0] + + def forward_features(self, x): + features = self.pretrained.get_intermediate_layers(x.flatten(0,1), self.intermediate_layer_idx[self.encoder], return_class_token=True) + return features + + def forward_depth(self, features, x_shape, cached_hidden_state_list=None): + B, T, C, H, W = x_shape + patch_h, patch_w = H // 14, W // 14 + depth, cur_cached_hidden_state_list = self.head(features, patch_h, patch_w, T, cached_hidden_state_list=cached_hidden_state_list) + depth = F.interpolate(depth, size=(H, W), mode="bilinear", align_corners=True) + depth = F.relu(depth) + return depth.squeeze(1).unflatten(0, (B, T)), cur_cached_hidden_state_list # return shape [B, T, H, W] + + def infer_video_depth_one(self, frame, input_size=518, device='cuda', fp32=False): + self.id += 1 + + if self.transform is None: # first frame + # Initialize the transform + frame_height, frame_width = frame.shape[:2] + self.frame_height = frame_height + self.frame_width = frame_width + ratio = max(frame_height, frame_width) / min(frame_height, frame_width) + if ratio > 1.78: # we recommend to process video with ratio smaller than 16:9 due to memory limitation + input_size = int(input_size * 1.777 / ratio) + input_size = round(input_size / 14) * 14 + + self.transform = Compose([ + Resize( + width=input_size, + height=input_size, + resize_target=False, + keep_aspect_ratio=True, + ensure_multiple_of=14, + resize_method='lower_bound', + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + PrepareForNet(), + ]) + + # Inference the first frame + cur_list = [torch.from_numpy(self.transform({'image': frame.astype(np.float32) / 255.0})['image']).unsqueeze(0).unsqueeze(0)] + cur_input = torch.cat(cur_list, dim=1).to(device) + + with torch.no_grad(): + with torch.autocast(device_type=device, enabled=(not fp32)): + cur_feature = self.forward_features(cur_input) + x_shape = cur_input.shape + depth, cached_hidden_state_list = self.forward_depth(cur_feature, x_shape) + + depth = depth.to(cur_input.dtype) + depth = F.interpolate(depth.flatten(0,1).unsqueeze(1), size=(frame_height, frame_width), mode='bilinear', align_corners=True) + + # Copy multiple cache to simulate the windows + self.frame_cache_list = [cached_hidden_state_list] * INFER_LEN + self.frame_id_list.extend([0] * (INFER_LEN - 1)) + + new_depth = depth[0][0].cpu().numpy() + else: + frame_height, frame_width = frame.shape[:2] + assert frame_height == self.frame_height + assert frame_width == self.frame_width + + # infer feature + cur_input = torch.from_numpy(self.transform({'image': frame.astype(np.float32) / 255.0})['image']).unsqueeze(0).unsqueeze(0).to(device) + with torch.no_grad(): + with torch.autocast(device_type=device, enabled=(not fp32)): + cur_feature = self.forward_features(cur_input) + x_shape = cur_input.shape + + cur_list = self.frame_cache_list[0:2] + self.frame_cache_list[-INFER_LEN+3:] + ''' + cur_id = self.frame_id_list[0:2] + self.frame_id_list[-INFER_LEN+3:] + print(f"cur_id: {cur_id}") + ''' + assert len(cur_list) == INFER_LEN - 1 + cur_cache = [torch.cat([h[i] for h in cur_list], dim=1) for i in range(len(cur_list[0]))] + + # infer depth + with torch.no_grad(): + with torch.autocast(device_type=device, enabled=(not fp32)): + depth, new_cache = self.forward_depth(cur_feature, x_shape, cached_hidden_state_list=cur_cache) + + depth = depth.to(cur_input.dtype) + depth = F.interpolate(depth.flatten(0,1).unsqueeze(1), size=(frame_height, frame_width), mode='bilinear', align_corners=True) + depth_list = [depth[i][0].cpu().numpy() for i in range(depth.shape[0])] + + new_depth = depth_list[-1] + + self.frame_cache_list.append(new_cache) + + # adjust the sliding window + self.frame_id_list.append(self.id) + if self.id + INFER_LEN > self.gap + 1: + del self.frame_id_list[1] + del self.frame_cache_list[1] + + return new_depth diff --git a/spacedreamer_stack_cups_clean40_30ep/runtime/spacedreamer_galexea.py b/spacedreamer_stack_cups_clean40_30ep/runtime/spacedreamer_galexea.py new file mode 100644 index 0000000000000000000000000000000000000000..9bd9b989a5fcd25b960235a94c8d7cf9f23ef1cc --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/runtime/spacedreamer_galexea.py @@ -0,0 +1,262 @@ +"""Portable adapter for the five released SpaceDreamer v3 Galaxea checkpoints. + +The source/ tree is the byte-identical training snapshot. This adapter provides +the infer_action API; RgbGeometryActionMoT itself has no legacy FastWAM API. +""" +from __future__ import annotations +import copy +import hashlib +import json +from pathlib import Path +import sys +import threading + +import cv2 +import numpy as np +from PIL import Image +import torch +from torch import nn + +ROOT = Path(__file__).resolve().parent +for _path in (ROOT / 'source/model_old/src', ROOT / 'source/multimodal'): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +PREFIX = "A video recorded from a robot's point of view executing the following instruction: " +VDA_SHA = 'f6f58576b9680a112f2428d4f39aff92656d3ae85745b0164675ace8d5b1fade' + + +def sha256(path): + h = hashlib.sha256() + with Path(path).open('rb') as f: + for block in iter(lambda: f.read(8 << 20), b''): + h.update(block) + return h.hexdigest() + + +def normalize(values, bounds): + """Exact training dataset q01/q99 transform, including constant channels.""" + lo, hi = (np.asarray(bounds[k], np.float32) for k in ('q01', 'q99')) + active = ~np.isclose(hi, lo) + out = np.zeros_like(values, dtype=np.float32) + out[..., active] = 2 * (values[..., active] - lo[active]) / (hi[active] - lo[active]) - 1 + return np.clip(out, -1, 1) + + +def denormalize(values, bounds): + lo, hi = (np.asarray(bounds[k], np.float32) for k in ('q01', 'q99')) + return (np.clip(values, -1, 1) + 1) * .5 * (hi - lo) + lo + + +class JointNormalizer(nn.Module): + """Match the four registered buffers in the original training model.""" + def __init__(self, norm): + super().__init__() + for feature, key in [('state', 'state'), ('action', 'actions')]: + for bound in ('q01', 'q99'): + self.register_buffer(feature + '_' + bound, + torch.tensor(norm[key][bound], dtype=torch.float32)) + + +def build_mot(config, norm, *, meta=True): + """Same constructor sequence as frozen training_reference/model.py.""" + from contextlib import nullcontext + from fastwam.models.wan22.wan_video_dit import WanVideoDiT, precompute_freqs_cis_3d, precompute_freqs_cis + from robodojo_rgb_geometry_action_model import _build_geometry_action_mot + from wan_rgb_depth_point_initialization import upgrade_wan_dit_for_rgb_depth_point, enable_point_first_frame_timestep_adapter + mc = copy.deepcopy(config['model']) + with torch.device('meta') if meta else nullcontext(): + video = WanVideoDiT(**mc['dit']) + upgrade_wan_dit_for_rgb_depth_point(video) + enable_point_first_frame_timestep_adapter(video, point_bottleneck_dim=256) + model = _build_geometry_action_mot( + video, mc['action_dit'], modalities=('rgb', 'depth', 'point'), + random_seed=config['seed'], action_state_dim=14, + mot_checkpoint_mixed_attn=False, cross_modal_attention='bidirectional') + model.add_module('action_normalizer', JointNormalizer(norm)) + video.freqs = precompute_freqs_cis_3d(mc['dit']['attn_head_dim']) + model.action_expert.freqs = precompute_freqs_cis(mc['action_dit']['attn_head_dim'], end=1024) + model.condition_depth_frame0 = True + return model + + +def prepare_head_rgb(head_rgb, source_hw=(512, 640)): + """Training's 224x224 OpenPI letterbox, before the 256x256 model resize.""" + image = np.asarray(head_rgb) + if image.dtype != np.uint8 or image.ndim != 3 or image.shape[2] != 3: + raise ValueError('head_rgb must be RGB uint8 [H,W,3], not BGR or float') + sh, sw = source_hw + ratio = max(sw / 224, sh / 224) + rh, rw = int(sh / ratio), int(sw / ratio) + top, left = int((224 - rh) / 2), int((224 - rw) / 2) + valid = np.zeros((224, 224), bool) + valid[top:top + rh, left:left + rw] = True + if image.shape[:2] == tuple(source_hw): + resized = np.asarray(Image.fromarray(image).resize((rw, rh), Image.Resampling.BILINEAR)) + padded = np.zeros((224, 224, 3), np.uint8) + padded[top:top + rh, left:left + rw] = resized + elif image.shape[:2] == (224, 224): + # Caller certifies this is the same letterbox, not an arbitrary square crop. + padded = image.copy() + else: + raise ValueError(f'Expected raw {tuple(source_hw)} or already letterboxed (224,224); got {image.shape[:2]}') + return padded, valid + + +def observation_pixels(head_rgb, depth_m, *, source_hw=(512, 640)): + image, valid = prepare_head_rgb(head_rgb, source_hw) + depth = np.asarray(depth_m) + if depth.shape != (224, 224) or not np.issubdtype(depth.dtype, np.floating): + raise ValueError('depth_m must be floating metric z-depth on the 224x224 letterboxed head image') + if not np.isfinite(depth).all(): + raise ValueError('depth_m contains nonfinite values') + # Original causal depth cache stores float16 meter values, then loads float32. + depth = np.where(valid & (depth > 0), np.minimum(depth, 65000.), 0).astype(np.float16).astype(np.float32) + depth = cv2.resize(depth, (256, 256), interpolation=cv2.INTER_NEAREST) + valid = cv2.resize(valid.astype(np.uint8), (256, 256), interpolation=cv2.INTER_NEAREST).astype(bool) + ok = valid & (depth >= .05) & (depth <= 5.) + depth = np.where(ok, depth, 5.) + rgb = cv2.resize(image, (256, 256), interpolation=cv2.INTER_LINEAR) + rgb = torch.from_numpy(rgb.copy()).permute(2, 0, 1).float().div(255).mul(2).sub(1)[None, :, None] + depth = torch.from_numpy((2 * np.clip(depth, 0, 5) / 5 - 1).astype(np.float32))[None, None, None].repeat(1, 3, 1, 1, 1) + return rgb, depth + + +class SpaceDreamerGalaxea: + """One task, one episode stream; serial calls only. No robot I/O is performed.""" + def __init__(self, task_dir, assets_dir=None, *, device='cuda:0', verify_checkpoint=True): + self.task_dir = Path(task_dir).resolve() + self.assets = Path(assets_dir).resolve() if assets_dir else self.task_dir.parent / 'assets' + self.device = torch.device(device) + self.dtype = torch.bfloat16 + self._lock = threading.Lock() + self.config = json.loads((self.task_dir / 'training_config.json').read_text()) + self.norm = json.loads((self.task_dir / 'reference_norm_stats.json').read_text())['norm_stats'] + self.metadata = json.loads((self.task_dir / 'checkpoint_metadata.json').read_text()) + if (self.config['action_horizon'], self.config['action_dim'], self.config['resolution']) != (32, 14, 256): + raise ValueError('Expected this release\'s 32x14,256px joint-position contract') + weights = self.task_dir / 'final_weights.pt' + if verify_checkpoint and sha256(weights) != self.metadata['sha256']: + raise ValueError('Checkpoint SHA256 does not match task metadata') + self.model = build_mot(self.config, self.norm, meta=True) + payload = torch.load(weights, map_location='cpu', mmap=True, weights_only=True) + self.model.load_state_dict(payload['state_dict'], strict=True, assign=True) + self.model = self.model.to(device=self.device, dtype=self.dtype).eval().requires_grad_(False) + del payload + self.vae = self.vda = self.text_encoder = self.tokenizer = None + self.cached_text = torch.load(self.task_dir / 'text_context.pt', weights_only=True, map_location='cpu') + if self.cached_text['prompt'] != PREFIX + self.config['prompt']: + raise ValueError('Task text cache does not match its original instruction') + + def _load_vae(self): + if self.vae is None: + from train_joint_rgb_depth import load_vae + self.vae = load_vae(self.assets / 'Wan2.2-TI2V-5B', self.dtype, self.device) + torch.nn.utils.convert_conv3d_weight_memory_format(self.vae, torch.channels_last_3d) + + @torch.inference_mode() + def text_context(self, instruction=None): + instruction = self.config['prompt'] if instruction is None else instruction + if instruction == self.config['prompt']: + return (self.cached_text['context'][None].to(self.device, dtype=self.dtype), + self.cached_text['mask'][None].to(self.device, dtype=torch.bool)) + if self.text_encoder is None: + from train_joint_rgb_depth import load_text_encoder_and_tokenizer + self.text_encoder, self.tokenizer = load_text_encoder_and_tokenizer( + self.assets / 'Wan2.2-TI2V-5B', self.dtype, self.device, 128) + ids, mask = self.tokenizer([PREFIX + instruction], return_mask=True, add_special_tokens=True) + ids, mask = ids.to(self.device), mask.to(self.device).bool() + context = self.text_encoder(ids, mask) + context[~mask] = 0 + # Exact training cache contract: zero padded context with all 128 keys visible. + return context, torch.ones((1, 128), device=self.device, dtype=torch.bool) + + @torch.inference_mode() + def update_depth(self, head_rgb): + """Feed successive head frames to causal VDA; call for each camera frame.""" + if self.vda is None: + sys.path.insert(0, str(ROOT / 'source/vda')) + from video_depth_anything.video_depth_stream import VideoDepthAnything + path = self.assets / 'metric_video_depth_anything_vitb.pth' + if sha256(path) != VDA_SHA: + raise ValueError('Metric VDA checkpoint SHA256 mismatch') + self.vda = VideoDepthAnything(encoder='vitb', features=128, out_channels=[96, 192, 384, 768]) + self.vda.load_state_dict(torch.load(path, map_location='cpu', weights_only=True), strict=True) + self.vda = self.vda.to(self.device).eval().requires_grad_(False) + self.reset_episode() + image, _ = prepare_head_rgb(head_rgb, tuple(self.config['source_video_hw'])) + result = self.vda.infer_video_depth_one(image, input_size=518, + device=str(self.device), fp32=self.device.type == 'cpu') + depth = np.asarray(result, dtype=np.float32) + if depth.shape != (224, 224) or not np.isfinite(depth).all(): + raise ValueError('VDA returned invalid metric head depth') + return depth + + def reset_episode(self): + """Reset VDA temporal cache once at a new episode, not at every replan.""" + if self.vda is not None: + self.vda.transform = None + self.vda.frame_id_list = [] + self.vda.frame_cache_list = [] + self.vda.id = -1 + + @torch.inference_mode() + def infer_action(self, head_rgb, state, *, depth_m=None, instruction=None, + num_inference_steps=10, seed=None, return_latents=False): + """Return float32 [32,14] absolute joint/gripper commands in original units. + + state: float [14], left joints6,left gripper,right joints6,right gripper. + depth_m: metric VDA output [224,224]; when omitted, advances VDA once. + RGB/depth frame0 are fixed; point frame0 and future modalities are sampled. + This method never sends commands to a robot. + """ + if not self._lock.acquire(blocking=False): + raise RuntimeError('One policy instance supports one serial episode stream') + try: + s = np.asarray(state, dtype=np.float32) + if s.shape != (14,) or not np.isfinite(s).all(): + raise ValueError('state must be finite [14] in original physical units') + if not isinstance(num_inference_steps, int) or num_inference_steps < 1: + raise ValueError('num_inference_steps must be positive') + if depth_m is None: + depth_m = self.update_depth(head_rgb) + rgb, depth = observation_pixels(head_rgb, depth_m, + source_hw=tuple(self.config['source_video_hw'])) + self._load_vae() + rgb_z = self.vae.single_encode(rgb.to(self.device, dtype=self.dtype), device=self.device) + depth_z = self.vae.single_encode(depth.to(self.device, dtype=self.dtype), device=self.device) + context, mask = self.text_context(instruction) + initial_state = torch.from_numpy(normalize(s, self.norm['state']))[None].to(self.device, dtype=self.dtype) + return self.infer_action_latents(rgb_z, depth_z, initial_state, context, mask, + num_inference_steps=num_inference_steps, seed=seed, return_latents=return_latents) + finally: + self._lock.release() + + @torch.inference_mode() + def infer_action_latents(self, rgb_z, depth_z, normalized_state, context, context_mask, + *, num_inference_steps=10, seed=None, return_latents=False): + """Low-level API: normalized state and VAE latents already on model device.""" + from joint_rgb_geometry_action_diffusion import sample_joint_geometry_and_action + from fastwam.models.wan22.schedulers.scheduler_continuous import WanContinuousFlowMatchScheduler + if tuple(rgb_z.shape) != (1, 48, 1, 16, 16) or tuple(depth_z.shape) != tuple(rgb_z.shape): + raise ValueError('Expected one-frame RGB/depth VAE latents [1,48,1,16,16]') + if tuple(normalized_state.shape) != (1, 14): + raise ValueError('Expected normalized state [1,14]') + if tuple(context.shape) != (1, 128, 4096) or tuple(context_mask.shape) != (1, 128): + raise ValueError('Expected context [1,128,4096] and mask [1,128]') + generator = None if seed is None else torch.Generator(device=self.device).manual_seed(seed) + vs = WanContinuousFlowMatchScheduler(num_train_timesteps=1000, shift=self.config['video_shift']) + acs = WanContinuousFlowMatchScheduler(num_train_timesteps=1000, shift=self.config['action_shift']) + result = sample_joint_geometry_and_action( + self.model, vs, acs, modalities=('rgb', 'depth', 'point'), + latent_shape=(1, 144, 9, 16, 16), action_shape=(1, 32, 14), + context=context, context_mask=context_mask, initial_state=normalized_state, + rgb_condition_latent=rgb_z, depth_condition_latent=depth_z, + num_inference_steps=num_inference_steps, generator=generator) + normalized = result['action'][0].float().cpu().numpy() + if not np.isfinite(normalized).all(): + raise FloatingPointError('Model produced nonfinite actions') + actions = denormalize(normalized, self.norm['actions']).astype(np.float32) + if actions.shape != (32, 14) or not np.isfinite(actions).all(): + raise FloatingPointError('Invalid physical action chunk') + return (actions, result) if return_latents else actions diff --git a/spacedreamer_stack_cups_clean40_30ep/text_context.pt b/spacedreamer_stack_cups_clean40_30ep/text_context.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d8efd63c412ba13be9a434dd63efd78d9454123 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/text_context.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49cf3918423e55ed12776ac91546bd4efb4142fd4645309b34b8526953a7cc92 +size 1050977 diff --git a/spacedreamer_stack_cups_clean40_30ep/training_complete.json b/spacedreamer_stack_cups_clean40_30ep/training_complete.json new file mode 100644 index 0000000000000000000000000000000000000000..98a50f452f9d530e78d86e8a03f881767ae49273 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_complete.json @@ -0,0 +1,8 @@ +{ + "epochs": 30, + "global_step": 4350, + "weights": "/workspace/group_share/rc-wbc/shouqx@xiaopeng.com/real_robot_training/stack_cups_new67_clean40_refitnorm_30ep_16h200_r2_20261114/runs/stack_cups/final_weights.pt", + "action_horizon": 32, + "video_frames": 33, + "config_fingerprint": "e788eb28e909fc8800dcb61a148650ab93c87c4636b997dc6eadff62a44956eb" +} diff --git a/spacedreamer_stack_cups_clean40_30ep/training_config.json b/spacedreamer_stack_cups_clean40_30ep/training_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a020fd92792b0984ab4b7c874a69cf044eeb9d38 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_config.json @@ -0,0 +1,180 @@ +{ + "task": "stack_cups", + "reference_job": "bifrost-2026102616140000-shouqx", + "reference_root": "/dataset_rc_o1_p1/qshou@xiaopeng.com/dataset/stack_cups_spacedreamer_vda_148_20261105/merged148", + "input_dataset": "/dataset_rc_o1_p1/qshou@xiaopeng.com/dataset/stack_cups_spacedreamer_vda_148_20261105/encoded_spacedreamer33_pad_v2", + "source_episodes": 40, + "segments": 41, + "frames": 37273, + "prompt": "Stack the cups.", + "fps": 20, + "video_frames": 33, + "action_horizon": 32, + "action_dim": 14, + "action_source": "absolute_original_motion_target_position", + "joint_order": "left_arm6,left_gripper,right_arm6,right_gripper", + "epochs": 30, + "seed": 0, + "global_batch": 256, + "world_size": 16, + "micro_batch": 16, + "accumulation": 1, + "workers": 2, + "resolution": 256, + "learning_rate": 5e-05, + "weight_decay": 1e-10, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "grad_clip": 1.0, + "warmup_steps": 500, + "lr_decay_steps": 10000, + "lr_floor": 5e-06, + "checkpoint_every": 1000, + "checkpoint_keep": 2, + "pretrained_checkpoint": "/dataset_rc_o1_p1/qshou@xiaopeng.com/training_runs/SpaceDreamer-v3-agibot60-egoverse20-robodojo-full20-32gpu-gbs512-1ep-20260919T072555Z/checkpoints/joint_action_mot_full_dit_018126.pt", + "pretrained_sha256": "fd1d665394b271c189132378fa2f8439589e0dc81310e72a43d89e7474ed3649", + "pretrained_model_dir": "/workspace-wl/shouqx@xiaopeng.com/model/Wan2.2-TI2V-5B", + "pretrained_config_sha256": "5b9a739842e11610888bb8cadae28004a8bceecbe42bdf4135f5ba3a446a5d62", + "model": { + "joint_rgb_depth": true, + "rgb_channels": 48, + "depth_channels": 48, + "share_rgb_depth_vae": true, + "freeze_vae": true, + "freeze_text_encoder": true, + "dtype": "bfloat16", + "dit": { + "hidden_dim": 3072, + "in_dim": 48, + "ffn_dim": 14336, + "out_dim": 48, + "text_dim": 4096, + "freq_dim": 256, + "eps": 1e-06, + "patch_size": [ + 1, + 2, + 2 + ], + "num_heads": 24, + "attn_head_dim": 128, + "num_layers": 30, + "has_image_input": false, + "has_image_pos_emb": false, + "has_ref_conv": false, + "add_control_adapter": false, + "in_dim_control_adapter": 24, + "seperated_timestep": true, + "require_vae_embedding": false, + "require_clip_embedding": false, + "fuse_vae_embedding_in_latents": true, + "action_conditioned": false, + "action_dim": 7, + "action_group_causal_mask_mode": "causal", + "video_attention_mask_mode": "bidirectional", + "use_gradient_checkpointing": true + }, + "joint_rgb_depth_point": true, + "point_channels": 48, + "share_rgb_depth_point_vae": true, + "joint_rgb_depth_point_action_mot": true, + "action_initialization": "random", + "action_random_seed": 20260724, + "action_state_dim": 14, + "action_state_conditioning": "initial_state_context_token", + "camera_dim": 6, + "mot_checkpoint_mixed_attn": false, + "cross_modal_attention": "bidirectional", + "action_loss_to_video_stop_gradient": false, + "action_dit": { + "action_dim": 14, + "hidden_dim": 576, + "ffn_dim": 2304, + "text_dim": 4096, + "freq_dim": 256, + "eps": 1e-06, + "num_heads": 24, + "attn_head_dim": 128, + "num_layers": 30, + "use_gradient_checkpointing": true + } + }, + "video_shift": 5.0, + "action_shift": 1.0, + "loss_weights": { + "rgb": 1.0, + "depth": 1.0, + "point": 1.0, + "action": 1.0 + }, + "depth_model": "metric_video_depth_anything_vitb", + "depth_mode": "streaming_causal", + "vda_checkpoint": "/workspace-wl/shouqx@xiaopeng.com/wam/third-party/Video-Depth-Anything/checkpoints/metric_video_depth_anything_vitb.pth", + "vda_sha256": "f6f58576b9680a112f2428d4f39aff92656d3ae85745b0164675ace8d5b1fade", + "vda_input_size": 518, + "point_frame": "per_frame_camera_optical", + "source_video_hw": [ + 512, + 640 + ], + "prepared_root": "/dataset_rc_o1_p1/qshou@xiaopeng.com/dataset/stack_cups_spacedreamer_vda_148_20261105/merged148/prepared", + "output_dir": "/workspace/group_share/rc-wbc/shouqx@xiaopeng.com/real_robot_training/stack_cups_new67_clean40_refitnorm_30ep_16h200_r2_20261114/runs/stack_cups", + "steps_per_epoch": 145, + "total_steps": 4350, + "reference_spatial_augmentation": "shared head 95% crop and +/-5 degree rotation", + "reference_color_augmentation": { + "brightness": 0.3, + "contrast": 0.4, + "saturation": 0.5 + }, + "pretrained_transfer": "all video+ActionDiT backbone weights; fresh joint action I/O and state encoder", + "normalizer_policy": "q01/q99 refitted using ONLY all37273 unique rows of the approved40 new episodes; clipped [-1,1]", + "action_tail": "repeat last same-segment command; all 32 slots in loss", + "video_tail": "repeat last same-segment image; padded future latent steps masked", + "source_identity": { + "row_index.json": "94e5dbff3a9cc7f86fdb08b9afc1d065f43e531c7f76bb52bbadc60e84f5041f", + "actions.npy": "804a05b4fe4d5b679359b9b79914a9a98b8f60a43d654c1f67b285a272fae93b", + "states.npy": "f12a286dc672aa66af449545fccfbd7cf4be76c69be9dd94d99e67d39930a0b5", + "reference_pi05_norm_stats.json": "c6f16375ffd71a28d107134bbe68e729ac7a82bfc8e8976b3f5f2d05e77fbb46", + "reference_rows.arrow": { + "bytes": 12697643298, + "mtime_ns": 1793874538756398057, + "recorded_sha256": "aa8fa61535f5e6ba40a60288d92804802e67f269e2bd0e59e56d36716fa83c2e", + "fresh_full_rehash": false + } + }, + "source_fingerprint": "f625a390fa7228673d43d5a4751e7f3eb5e8eb5cee7528d6f834092857322d3f", + "calibration_manifest": "/dataset_rc_o1_p1/qshou@xiaopeng.com/dataset/stack_cups_spacedreamer_vda_148_20261105/calibrations.json", + "vae_sha256": "20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36", + "calibration_manifest_sha256": "bf0a0bb8c85a7254dab64f34962ebf81b153d448205da89e5f09f523900070ad", + "vae_encode_batch": 8, + "vae_conv_memory_format": "channels_last_3d", + "action_dimension_loss_weights": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 1.0 + ], + "run_intent": "Fresh SpaceDreamer-v3 pretrained018126 fine-tune using only 40 clean episodes from Nov03/04 new67; 30 epochs on 2x8 H200; right arm joint loss2x, grippers1x.", + "data_selection": { + "manifest": "/workspace/group_share/rc-wbc/shouqx@xiaopeng.com/real_robot_training/stack_cups_new67_clean40_refitnorm_30ep_16h200_r2_20261114/metadata/data_selection.json", + "manifest_sha256": "01a8ce6a58563ccce347dcf6306a605f2419d1109f5a9a6d8c0ef00a2859eb80", + "row_indices": "/workspace/group_share/rc-wbc/shouqx@xiaopeng.com/real_robot_training/stack_cups_new67_clean40_refitnorm_30ep_16h200_r2_20261114/metadata/selected_rows.npy", + "row_indices_sha256": "5c655de69bd19c8b2f55d2c87a72d62522ab2f3800efb27732402e77c983fcc4" + }, + "normalizer_override": "/workspace/group_share/rc-wbc/shouqx@xiaopeng.com/real_robot_training/stack_cups_new67_clean40_refitnorm_30ep_16h200_r2_20261114/metadata/selected40_norm_stats.json", + "normalizer_override_sha256": "10f598cd5148e9ac85fa2b7b8c3b2f2c503127fcb307f33ad3e5277f8a9ff6f8" +} diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/audit.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/audit.py new file mode 100644 index 0000000000000000000000000000000000000000..825912b41b30699444c86b52f0a501c279c480c6 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/audit.py @@ -0,0 +1,42 @@ +"""Check actual reference identities, actual pretrained tensor coverage, and cache readiness.""" +import argparse +from pathlib import Path +import json + +from common import ROOT, TASKS, fingerprint, load_config, read, write +from data import SourceRows + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--pretrained-structure', action='store_true') + args = p.parse_args() + result = [] + for task in TASKS: + c = load_config(ROOT / 'configs' / (task + '.json')) + source = SourceRows(c) + identity = source.identity() + if fingerprint(identity) != c['source_fingerprint']: + raise ValueError('Reference data changed') + prepared = Path(c['prepared_root']) / 'manifest.json' + record = dict(task=task, reference_job=c['reference_job'], frames=len(source), segments=len(source.segments), + optimizer_steps=c['total_steps'], prepared=prepared.exists(), + training_complete=(Path(c['output_dir']) / 'COMPLETE.json').exists()) + if prepared.exists(): + m = read(prepared) + if not m['complete'] or m['source_fingerprint'] != c['source_fingerprint'] or m['frames'] != len(source): + raise ValueError('Invalid prepared-data manifest') + result.append(record) + report = dict(tasks=result, epochs=20, frames_per_clip=33, action_horizon=32, fps=20, global_batch=256) + if args.pretrained_structure: + from model import build_model + model, audit = build_model(load_config(ROOT / 'configs/stack_cups.json')) + report['actual_pretrained_transfer'] = audit + report['actual_parameter_count'] = sum(p.numel() for p in model.parameters()) + report['pretrained_tensor_values'] = 'mmap assigned without conversion; fresh joint interface only' + write(ROOT / 'reports/implementation_audit.json', report) + print(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/build_smoke_sample.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/build_smoke_sample.py new file mode 100644 index 0000000000000000000000000000000000000000..f0f7a1fa815b3b8a83055a708519799bcc9b200d --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/build_smoke_sample.py @@ -0,0 +1,46 @@ +"""Build one real RGB/VDA/XYZ/action window without declaring the full cache complete.""" +import argparse +from pathlib import Path + +import cv2 +import numpy as np +import torch + +from common import load_config +from data import SourceRows, normalize, point_video + +p = argparse.ArgumentParser() +p.add_argument('--config', type=Path, required=True) +p.add_argument('--output', type=Path, required=True) +args = p.parse_args() +c = load_config(args.config) +s = SourceRows(c) +s.open() +if s.segments[0]['length'] < 33: + raise ValueError('First source segment is too short for preflight') +folder = Path(c['prepared_root']) / 'smoke_segment_000000' +d = np.asarray(np.load(folder / 'depth_m.npy'), np.float32) +with np.load(folder / 'calibration.npz') as calib: + rays, valid = calib['rays'], calib['valid'] +if d.shape[0] != 33: + raise ValueError('Preflight VDA window must contain 33 real frames') +rgb = np.stack([cv2.resize(s.rgb(0, i), (256, 256), interpolation=cv2.INTER_LINEAR) for i in range(33)]) +d = np.stack([cv2.resize(x, (256, 256), interpolation=cv2.INTER_NEAREST) for x in d]) +rays = cv2.resize(rays, (256, 256), interpolation=cv2.INTER_NEAREST) +valid = cv2.resize(valid.astype(np.uint8), (256, 256), interpolation=cv2.INTER_NEAREST).astype(bool) +point, ok = point_video(d, rays, valid) +depth = np.repeat((2 * np.where(ok, np.clip(d, 0, 5), 5) / 5 - 1)[None], 3, axis=0) +context = torch.load(s.assets / 'context.pt', map_location='cpu', weights_only=True) +sample = dict(rgb=torch.from_numpy(rgb).permute(3, 0, 1, 2).float() / 127.5 - 1, + depth=torch.from_numpy(depth), point=torch.from_numpy(point), + action=torch.from_numpy(normalize(np.asarray(s._actions[:32]), s.norm['actions'])), + initial_state=torch.from_numpy(normalize(np.asarray(s._states[0]), s.norm['state'])), + image_is_pad=torch.zeros(33, dtype=torch.bool), action_is_pad=torch.zeros(32, dtype=torch.bool), + context=context['context'], context_mask=context['mask'].bool(), row=0) +for value in sample.values(): + if isinstance(value, torch.Tensor) and not torch.isfinite(value).all(): + raise ValueError('Nonfinite real preflight sample') +args.output.parent.mkdir(parents=True, exist_ok=True) +torch.save(dict(sample=sample, source_fingerprint=c['source_fingerprint'], + purpose='single_real_window_repeated_only_for_engine_preflight'), args.output) +print(dict(output=str(args.output), real_video_frames=33, action_horizon=32), flush=True) diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cache_calibrations.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cache_calibrations.py new file mode 100644 index 0000000000000000000000000000000000000000..41e44829538e86fae55c7b860125ffc4d6be6940 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cache_calibrations.py @@ -0,0 +1,47 @@ +"""CPU-only complete inventory and calibration verification for every selected episode.""" +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +import time + +from calibration import index_mcaps, read_camera_info +from common import ROOT, TASKS, read, sha, write + + +def main(): + began = time.monotonic() + index = index_mcaps() + configs = {task: read(ROOT / 'configs' / (task + '.json')) for task in TASKS} + required = {} + for task, config in configs.items(): + rows = read(Path(config['reference_root']) / 'assets/row_index.json')['segments'] + keys = sorted({row['source_episode_key'] for row in rows}) + if len(keys) != config['source_episodes']: + raise ValueError(f'Wrong source episode population: {task}') + for key in keys: + if key not in index: + raise FileNotFoundError(f'No actual raw MCAP for {key}') + required[key] = index[key] + print(dict(required_episodes=len(required), indexed_mcaps=len(index), status='ALL_RAW_PATHS_RESOLVED'), flush=True) + episodes = {} + with ThreadPoolExecutor(max_workers=8) as pool: + futures = {pool.submit(read_camera_info, key, path): key for key, path in required.items()} + for future in as_completed(futures): + key = futures[future] + episodes[key] = future.result() + if len(episodes) % 40 == 0: + print(dict(checked=len(episodes), total=len(required), seconds=time.monotonic() - began), flush=True) + destination = ROOT / 'metadata/calibrations.json' + write(destination, dict(complete=True, episodes=dict(sorted(episodes.items())), + selected_episodes=len(required), seconds=time.monotonic() - began)) + digest = sha(destination) + for task, config in configs.items(): + config['calibration_manifest_sha256'] = digest + write(ROOT / 'configs' / (task + '.json'), config) + write(ROOT / 'reports/all_raw_calibrations_PASS.json', dict(status='PASS_ALL_SELECTED_RAW_EPISODES', + selected_episodes=len(episodes), tasks={task: c['source_episodes'] for task, c in configs.items()}, + manifest=str(destination), sha256=digest, seconds=time.monotonic() - began)) + print(dict(status='ALL_CAMERA_CALIBRATIONS_VERIFIED', episodes=len(episodes), sha256=digest), flush=True) + + +if __name__ == '__main__': + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/calibration.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/calibration.py new file mode 100644 index 0000000000000000000000000000000000000000..624b41944820b811f23818eb9bdd847cbbb902ff --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/calibration.py @@ -0,0 +1,71 @@ +"""Resolve actual MCAP layouts and seal CPU-read camera calibrations before GPU submission.""" +from __future__ import annotations + +from functools import lru_cache +import math +from pathlib import Path +import subprocess + +from common import read, sha + +RAW_ROOT = Path('/dataset_rc_o1_p1/qshou@xiaopeng.com/Real_Robot_Dataset/Galexea_Dataset') +CAMERA_TOPIC = '/calib/head_left/camera_info' + + +def index_mcaps(root=RAW_ROOT): + paths = subprocess.check_output(['rg', '--files', '-g', '*.mcap', str(root)], text=True).splitlines() + result = {} + for name in paths: + path = Path(name) + task = path.relative_to(root).parts[0] + episode = path.stem if path.stem.endswith('_RAW') else path.parent.name + key = task + '/' + episode + if key in result: + raise ValueError(f'Ambiguous raw episode: {key}') + result[key] = path + return result + + +def read_camera_info(key, path): + from mcap.reader import make_reader + from mcap_ros2.decoder import DecoderFactory + stat = path.stat() + with path.open('rb') as stream: + if stream.read(8) != b'\x89MCAP0\r\n': + raise ValueError(f'Invalid raw MCAP header: {path}') + stream.seek(0) + reader = make_reader(stream, decoder_factories=[DecoderFactory()]) + summary = reader.get_summary() + if summary is None or CAMERA_TOPIC not in {c.topic for c in summary.channels.values()}: + raise ValueError(f'Missing indexed head calibration topic: {path}') + for _, _, message, msg in reader.iter_decoded_messages(topics=[CAMERA_TOPIC]): + if msg.width <= 0 or msg.height <= 0: + continue + k = list(msg.k) + d = list(msg.d) + if len(k) != 9 or min(k[0], k[4]) <= 0 or not all(math.isfinite(x) for x in k + d): + raise ValueError(f'Invalid camera intrinsics: {path}') + if msg.distortion_model not in ('plumb_bob', 'rational_polynomial', 'equidistant', ''): + raise ValueError(f'Unsupported distortion model: {path}') + return dict(episode_key=key, width=msg.width, height=msg.height, k=k, d=d, + distortion_model=msg.distortion_model, r=list(msg.r), p=list(msg.p), + raw_mcap=str(path), raw_mcap_bytes=stat.st_size, raw_mcap_mtime_ns=stat.st_mtime_ns, + source=CAMERA_TOPIC, calibration_log_time_ns=message.log_time) + raise ValueError(f'No head calibration messages: {path}') + + +@lru_cache(maxsize=4) +def load_calibrations(path, expected_sha256): + if sha(path) != expected_sha256: + raise ValueError('CPU calibration manifest changed') + manifest = read(path) + if manifest.get('complete') is not True: + raise ValueError('Camera calibration manifest is incomplete') + return manifest['episodes'] + + +def camera_calibration(key, config): + records = load_calibrations(config['calibration_manifest'], config['calibration_manifest_sha256']) + if key not in records: + raise KeyError(f'Missing prevalidated calibration: {key}') + return records[key] diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/check_global_batch.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/check_global_batch.py new file mode 100644 index 0000000000000000000000000000000000000000..27797028cf8cbc1f8b4e4e00c40567d289fea151 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/check_global_batch.py @@ -0,0 +1,59 @@ +"""A known SGD update proves all 256 samples contribute on this actual runtime.""" +import argparse +from pathlib import Path + +import torch + +from common import load_config, read, write +from data import ExactBatchSampler +from distributed import make_accelerator + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--config', type=Path, required=True) + p.add_argument('--reports', type=Path, required=True) + args = p.parse_args() + c = load_config(args.config) + torch.set_num_threads(2) + # An unclipped SGD update exposes gradient scale and omitted microbatches; + # Adam's scale normalization would conceal those errors on its first step. + accelerator = make_accelerator(c, grad_clip=10000.) + if accelerator.num_processes != 8: + raise RuntimeError('The analytical acceptance requires the actual eight ranks') + model = torch.nn.Linear(32, 1, bias=False) + with torch.no_grad(): + model.weight.fill_(1.) + optimizer = torch.optim.SGD(model.parameters(), lr=.01) + model, optimizer = accelerator.prepare(model, optimizer) + for rows in ExactBatchSampler(256, accelerator.process_index, 8, c['micro_batch'], c['accumulation']): + x = (((torch.tensor(rows, device=accelerator.device) // 16 + 1).float() / 16.)[:, None] + .repeat(1, 32)).bfloat16() + with accelerator.accumulate(model): + with accelerator.autocast(): + loss = model(x).float().square().mean() + accelerator.backward(loss) + optimizer.step() + optimizer.zero_grad(set_to_none=True) + weights = accelerator.unwrap_model(model).weight.detach().float().cpu() + # For v=1/16..16/16, each repeated 16 times: d E[(32v)^2]/dw_j=23.375. + expected_gradient = 23.375 + expected_parameter = 1. - .01 * expected_gradient + error = float((weights - expected_parameter).abs().max()) + if not torch.isfinite(weights).all() or error >= .01: + raise RuntimeError(f'GBS256 analytical update failed: error={error}, weights={weights}') + result = dict(status='PASS_ANALYTICAL_GLOBAL_BATCH_UPDATE', rank=accelerator.process_index, + world_size=8, micro_batch=c['micro_batch'], accumulation=c['accumulation'], global_batch=256, + expected_gradient_per_weight=expected_gradient, expected_parameter=expected_parameter, + actual_parameter=float(weights[0, 0]), max_abs_error=error) + write(args.reports / f'rank_{accelerator.process_index}.json', result) + accelerator.wait_for_everyone() + if accelerator.is_main_process: + evidence = [read(args.reports / f'rank_{rank}.json') for rank in range(8)] + write(args.reports / 'PASS.json', dict(status='PASS_ANALYTICAL_GLOBAL_BATCH_UPDATE', ranks=evidence)) + print(result, flush=True) + accelerator.wait_for_everyone() + + +if __name__ == '__main__': + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cluster_entry.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cluster_entry.py new file mode 100644 index 0000000000000000000000000000000000000000..af28e5688ac44a93256b04eb599a0251d4999a60 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cluster_entry.py @@ -0,0 +1,88 @@ +"""Two-node fresh fine-tune; all staging is local and restricted to selected segments.""" +from pathlib import Path +import fcntl,json,os,shutil,socket,subprocess,time +from concurrent.futures import ThreadPoolExecutor +from common import ROOT,WORKSPACE,load_config,read,sha,write +from staging import verified_copy + +def main(): + c=load_config(ROOT/'configs/stack_cups.json');job=os.environ['BIFROST_JOB_NAME'];node=int(os.environ['NODE_RANK']);assert node in (0,1) + reports=ROOT/'reports'/job/f'node{node}';reports.mkdir(parents=True,exist_ok=True) + status={'job':job,'node':node,'host':socket.gethostname(),'world_size':16,'selected_episodes':40,'epochs':30} + def stage(name,**kw): + status.update(stage=name,updated_unix=time.time(),**kw);write(reports/'status.json',status);print(json.dumps(status),flush=True) + def run(cmd,log,env): + with (reports/log).open('a') as stream:subprocess.run(cmd,env=env,stdout=stream,stderr=subprocess.STDOUT,check=True) + with (reports/'pipeline.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + try: + stage('VERIFY_IMMUTABLE_INPUTS') + for p,d in read(ROOT/'reports/submission_inputs.json').items(): + if sha(ROOT/p)!=d:raise RuntimeError('Frozen input changed: '+p) + if shutil.disk_usage(ROOT).free < 240*2**30:raise RuntimeError('Insufficient shared checkpoint capacity') + stage('STAGE_PORTABLE_RUNTIME') + with Path('/code/sd67-runtime.lock').open('a') as runtime_lock: + fcntl.flock(runtime_lock,fcntl.LOCK_EX) + for archive,executable in [('pico-baseline-runtime-v1.tar.zst','/code/ironflow-pico-runtime-v1/gr00t/bin/python'),('fastwam-runtime-v1.tar.zst','/code/fastwam-runtime-v1/bin/python')]: + if not Path(executable).exists():subprocess.run(['bash',str(WORKSPACE/'iron_flow/scripts/extract_runtime_archive.sh'),str(WORKSPACE/'iron_flow/runtime'/archive),'/code'],check=True) + py='/code/fastwam-runtime-v1/bin/python';local=Path('/code')/('sd67clean-'+job);local.mkdir(exist_ok=False) + if shutil.disk_usage(local).free<160*2**30:raise RuntimeError('Insufficient local staging disk') + env={k:v for k,v in os.environ.items() if k not in ('PYTHONPATH','LD_PRELOAD','WORLD_SIZE','RANK','LOCAL_RANK','CUDA_VISIBLE_DEVICES','ACCELERATE_USE_FSDP')} + deps=local/'deps' + run([py,'-m','pip','install','--quiet','--no-deps','--index-url','https://mirrors.aliyun.com/pypi/simple/','--target',str(deps),'accelerate==1.14.0','mcap==1.3.0','mcap-ros2-support==0.5.6','zstandard==0.25.0','lz4==4.4.5','easydict==1.13'],'runtime.log',env) + env.update(PYTHONPATH=str(deps)+':'+str(ROOT/'code'),PYTHONNOUSERSITE='1',PYTHONDONTWRITEBYTECODE='1',PYTHONUNBUFFERED='1',CUDA_VISIBLE_DEVICES='0,1,2,3,4,5,6,7',OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='1',TOKENIZERS_PARALLELISM='false',HF_HUB_OFFLINE='1',TRANSFORMERS_OFFLINE='1',WANDB_MODE='offline',ACCELERATE_USE_DEEPSPEED='true',SD_GALEXEA_OWNED_STORAGE_BUDGET='0',TORCH_EXTENSIONS_DIR=str(local/'torch_extensions'),TRITON_CACHE_DIR=str(local/'triton')) + (local/'triton').mkdir();stage('CHECK_ALLOCATED_EIGHT_H200_PER_NODE') + run([py,str(ROOT/'code/cuda_probe.py'),'--report',str(reports/'cuda.json')],'cuda.log',env) + stage('STAGE_VERIFIED_REFERENCE_AND_PRETRAINED') + src=Path(c['reference_root']);assets=local/'assets';plan=read(src/'training_plan.json') + def copy_asset(item): + name,info=item;verified_copy(src/'assets'/name,assets/name,info['sha256']) + with ThreadPoolExecutor(max_workers=3) as pool:list(pool.map(copy_asset,plan['assets'].items())) + verified_copy(src/'assets/context.pt',assets/'context.pt',sha(src/'assets/context.pt')) + verified_copy(c['pretrained_checkpoint'],local/'pretrained_v3.pt',c['pretrained_sha256']) + verified_copy(Path(c['pretrained_model_dir'])/'Wan2.2_VAE.pth',local/'models/Wan2.2_VAE.pth',c['vae_sha256']) + selection=read(c['data_selection']['manifest']);geometry=local/'geometry';prepared=Path(c['prepared_root']) + stage('REUSE_SELECTED_CAUSAL_GEOMETRY',segments=len(selection['selected_segments'])) + manifest=read(prepared/'manifest.json') + def copy_geometry(rec): + sid=rec['original_segment'];name=f'segment_{sid:06d}';old=prepared/name;receipt=read(old/'ready.json') + if sha(old/'ready.json')!=manifest['segment_receipt_sha256'][sid] or receipt['frames']!=rec['length']:raise ValueError('Geometry segment identity changed') + for filename,digest in [('depth_m.npy',receipt['depth_sha256']),('calibration.npz',receipt['calibration_sha256']),('ready.json',sha(old/'ready.json'))]:verified_copy(old/filename,geometry/name/filename,digest) + with ThreadPoolExecutor(max_workers=4) as pool:list(pool.map(copy_geometry,selection['selected_segments'])) + verified_copy(prepared/'manifest.json',geometry/'manifest.json',sha(prepared/'manifest.json')) + env.update(SD_GALEXEA_LOCAL_ASSETS=str(assets),SD_GALEXEA_LOCAL_BASE=str(local/'pretrained_v3.pt'),SD_GALEXEA_LOCAL_MODEL_DIR=str(local/'models'),SD_GALEXEA_LOCAL_PREPARED=str(geometry)) + stage('DECODE_ONLY_SELECTED_HEAD_SEGMENTS') + run([py,str(ROOT/'code/decode_selected.py'),'--config',str(ROOT/'configs/stack_cups.json'),'--report',str(reports/'decode.json')],'decode.log',env) + stage('WAIT_BOTH_NODES_READY') + write(reports/'READY.json',dict(job=job,node=node,config_sha256=sha(ROOT/'configs/stack_cups.json'),host=socket.gethostname())) + deadline=time.monotonic()+2400 + while True: + ready=[ROOT/'reports'/job/f'node{i}'/'READY.json' for i in range(2)] + if all(x.exists() for x in ready): + if all(read(x)['job']==job and read(x)['config_sha256']==sha(ROOT/'configs/stack_cups.json') for x in ready):break + raise RuntimeError('Peer readiness identity mismatch') + if time.monotonic()>deadline:raise TimeoutError('Peer node did not finish staging') + time.sleep(5) + master=os.environ['MASTER_ADDR'];port=os.environ.get('MASTER_PORT','29500') + if master in ('127.0.0.1','localhost','0.0.0.0'):raise ValueError('Multi-node rendezvous cannot use loopback') + stage('FORMAL_TRAINING',master=master,port=port,target_steps=c['total_steps']) + cmd=[py,'-m','torch.distributed.run','--nnodes=2','--nproc_per_node=8',f'--node_rank={node}',f'--master_addr={master}',f'--master_port={port}','--rdzv_conf=timeout=1800',str(ROOT/'code/train.py'),'--config',str(ROOT/'configs/stack_cups.json')] + run(cmd,'training.log',env) + if node==0: + out=Path(c['output_dir']);complete=read(out/'COMPLETE.json');assert (complete['epochs'],complete['global_step'])==(30,c['total_steps']) + rows=[json.loads(x) for x in (out/'metrics.jsonl').read_text().splitlines()];assert [x['step'] for x in rows]==list(range(1,c['total_steps']+1)) + from storage_budget import checkpoint_durable + state=out/'states'/f"step_{c['total_steps']:08d}";durable=checkpoint_durable(state,16);weight=out/'final_weights.pt' + if sha(out/'reference_norm_stats.json')!=c['normalizer_override_sha256']:raise RuntimeError('Exported inference statistics differ from training') + import torch + payload=torch.load(weight,map_location='cpu',weights_only=True,mmap=True);norm=read(c['normalizer_override'])['norm_stats'] + for field,key in [('state','state'),('action','actions')]: + for bound in ('q01','q99'): + actual=payload['state_dict'][f'action_normalizer.{field}_{bound}'].float();expected=torch.tensor(norm[key][bound],dtype=torch.float32) + if not torch.equal(actual,expected):raise RuntimeError('Checkpoint normalizer buffer mismatch') + del payload + result=dict(status='PASS_FINAL_30_EPOCH_EXPORT',epochs=30,global_step=c['total_steps'],selected_episodes=40,frames=c['frames'],world_size=16,weights=str(weight),weights_sha256=sha(weight),native_state=str(state),durability=durable) + write(ROOT/'reports/final_export_verified.json',result) + stage('COMPLETE') + except BaseException as e:stage('FAILED',error=repr(e));raise +if __name__=='__main__':main() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/common.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/common.py new file mode 100644 index 0000000000000000000000000000000000000000..8bdc0f412af259a7dfd26ecaeaca6560b083201c --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/common.py @@ -0,0 +1,94 @@ +"""Shared configuration and immutable source identities for the five real tasks.""" +from __future__ import annotations + +import hashlib +import json +import math +import os +from pathlib import Path +import sys +from functools import lru_cache + +ROOT = Path(__file__).resolve().parents[1] +REFERENCE = Path('/workspace/group_share/rc-wbc/shouqx@xiaopeng.com/recovery_20261026_quota/fastwam_r3') +WORKSPACE = Path('/workspace-wl/shouqx@xiaopeng.com') +ORIGINAL_CODE = WORKSPACE / 'wam/model_wan_rgb_depth_point_cloud_action_mot_bidirectional_nostopgrad_first_frame_pi05_flow_fastwam_sampling' +BACKBONE_CODE = ROOT / 'source/multimodal' +BASE = Path('/dataset_rc_o1_p1/qshou@xiaopeng.com/training_runs/SpaceDreamer-v3-agibot60-egoverse20-robodojo-full20-32gpu-gbs512-1ep-20260919T072555Z/checkpoints/joint_action_mot_full_dit_018126.pt') +BASE_SHA256 = 'fd1d665394b271c189132378fa2f8439589e0dc81310e72a43d89e7474ed3649' +TASKS = {'stack_cups': 'bifrost-2026102616140000-shouqx', 'tennis_ball_stopping_left_arm': 'bifrost-2026102616141601-shouqx', 'tennis_ball_stopping_right_arm': 'bifrost-2026102616140801-shouqx', 'zipper_pulling': 'bifrost-2026102616142400-shouqx'} +TASK_CONTRACTS = {'stack_cups': {'reference_job': 'bifrost-2026102616140000-shouqx', 'frames': 59355, 'segments': 81, 'source_episodes': 81}, 'tennis_ball_stopping_left_arm': {'reference_job': 'bifrost-2026102616141601-shouqx', 'frames': 55257, 'segments': 101, 'source_episodes': 101}, 'tennis_ball_stopping_right_arm': {'reference_job': 'bifrost-2026102616140801-shouqx', 'frames': 49386, 'segments': 99, 'source_episodes': 99}, 'zipper_pulling': {'reference_job': 'bifrost-2026102616142400-shouqx', 'frames': 139947, 'segments': 1309, 'source_episodes': 207}} + + +def read(path): + return json.loads(Path(path).read_text()) + + +def write(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f'.{path.name}.{os.getpid()}.tmp') + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n') + temporary.replace(path) + + +def sha(path): + h = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(8 << 20), b''): + h.update(block) + return h.hexdigest() + + +def fingerprint(value): + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def frozen_inputs(): + files = list((ROOT / 'code').glob('*.py')) + list((ROOT / 'configs').glob('*.json')) + files += list((ROOT / 'tests').glob('*.py')) + list((ROOT / 'source').rglob('*.py')) + list(ROOT.glob('*.sh')) + files += [f for f in (ROOT / 'metadata').rglob('*') if f.is_file()] + return {str(f.relative_to(ROOT)): sha(f) for f in files} + + +@lru_cache(maxsize=1) +def add_backbone(): + if not BACKBONE_CODE.is_dir(): + raise FileNotFoundError('Run code/setup.py to freeze the model source first') + for p in [ROOT / 'source/model_old/src', BACKBONE_CODE]: + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + + +def geometry_fingerprint(config): + return fingerprint({k: config[k] for k in ('source_fingerprint', 'vda_sha256', 'vda_input_size', + 'depth_mode', 'calibration_manifest_sha256', 'source_video_hw', 'point_frame', 'resolution')}) + + +def load_config(path): + c=read(path) + if (c['task'],c['fps'],c['video_frames'],c['action_horizon'],c['epochs']) != ('stack_cups',20,33,32,30):raise ValueError('Wrong fresh new67 clean training contract') + if (c['source_episodes'],c['segments'],c['frames']) != (40,41,37273):raise ValueError('Wrong approved clean population') + if (c['world_size'],c['micro_batch'],c['accumulation'],c['global_batch']) != (16,16,1,256):raise ValueError('Expected direct 16x16 GBS256') + if c['steps_per_epoch']!=c['frames']//256 or c['total_steps']!=30*c['steps_per_epoch']:raise ValueError('Incorrect epoch budget') + if 'continuation' in c or c['pretrained_checkpoint']!=str(BASE) or c['pretrained_sha256']!=BASE_SHA256:raise ValueError('Must initialize from original pretrained v3') + if c['action_dimension_loss_weights'] != [1.]*7+[2.]*6+[1.]:raise ValueError('Wrong right arm loss weighting') + if sha(c['normalizer_override'])!=c['normalizer_override_sha256']:raise ValueError('Selected40 normalizer changed') + selection=c['data_selection'] + if sha(selection['manifest'])!=selection['manifest_sha256'] or sha(selection['row_indices'])!=selection['row_indices_sha256']:raise ValueError('Selection inputs changed') + m=read(selection['manifest']) + if len(m['included_episode_keys'])!=40 or len(m['excluded_episodes'])!=27 or m['source_dates']!={'20261103':47,'20261104':20}:raise ValueError('Wrong source population') + if any(x['max_interior_gap_s']>.2 for x in m['included_episodes']):raise ValueError('Bad head capture episode retained') + if any(x['source_date'] not in ('20261103','20261104') for x in m['included_episodes']):raise ValueError('Old episodes included') + if not 1 <= c['vae_encode_batch'] <= c['micro_batch'] or c['vae_conv_memory_format']!='channels_last_3d':raise ValueError('Invalid VAE batching') + if c['depth_mode']!='streaming_causal' or c['point_frame']!='per_frame_camera_optical':raise ValueError('Geometry recipe drift') + if Path(c['output_dir']).resolve()!=(ROOT/'runs/stack_cups').resolve():raise ValueError('Output must be fresh owned scope') + return c + + +def lr_factor(step): + """Exact 500-step warmup and 10k-step cosine used by the reference jobs.""" + if step < 500: + return 1. / 501. + (1. - 1. / 501.) * step / 500. + progress = min(max((step - 500.) / 9500., 0.), 1.) + return .1 + .9 * .5 * (1. + math.cos(math.pi * progress)) diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cpu_bootstrap.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cpu_bootstrap.py new file mode 100644 index 0000000000000000000000000000000000000000..97b6b93e3b5a8d128e7b75bed4aa7151fcf72194 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cpu_bootstrap.py @@ -0,0 +1,9 @@ +"""CPU-only preflight launcher with an existing private optional dependency path.""" +from pathlib import Path +import sys + +sys.path.append(str(Path("/workspace-wl/shouqx@xiaopeng.com/wam/outputs/paper_data_pipeline_20261026T031341Z/.deps"))) + +if __name__ == "__main__": + from cpu_preflight import main + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cpu_preflight.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cpu_preflight.py new file mode 100644 index 0000000000000000000000000000000000000000..403e0e953248a2ceb0615ce12d1607a81b27f9b0 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cpu_preflight.py @@ -0,0 +1,45 @@ +from pathlib import Path +import json,py_compile,tempfile,hashlib +import numpy as np,torch +from common import ROOT,load_config,read,write,fingerprint,frozen_inputs,geometry_fingerprint,sha,add_backbone +from selection_data import SelectedDataset +from data import ExactBatchSampler + +def main(): + torch.set_num_threads(2) + c=load_config(ROOT/'configs/stack_cups.json');selection=read(c['data_selection']['manifest']);rows=np.load(c['data_selection']['row_indices']);assert rows.min()>=59355 + base=Path(c['reference_root']);index=read(base/'assets/row_index.json');assert len(selection['included_episode_keys'])==40 + for s in selection['selected_segments']: + original=index['segments'][s['original_segment']];assert all(original[k]==s[k] for k in original) + assert original['source_episode_key'] in selection['included_episode_keys'] + geometry=read(Path(c['prepared_root'])/'manifest.json');assert geometry['geometry_fingerprint']==geometry_fingerprint(c) + for s in selection['selected_segments']: + folder=Path(c['prepared_root'])/f"segment_{s['original_segment']:06d}";receipt=read(folder/'ready.json') + assert receipt['frames']==s['length'] and receipt['source_fingerprint']==c['source_fingerprint'] + assert sha(folder/'ready.json')==geometry['segment_receipt_sha256'][s['original_segment']] + assert (folder/'depth_m.npy').is_file() and (folder/'calibration.npz').is_file() + d=SelectedDataset(c,training=True);assert len(d)==37273 + assert d.base.source.norm==read(c['normalizer_override'])['norm_stats'] + rawbase=Path(c['reference_root'])/'assets' + for key,file in [('state','states.npy'),('actions','actions.npy')]: + raw=np.asarray(np.load(rawbase/file,mmap_mode='r')[rows],np.float64) + assert np.allclose(np.quantile(raw,.01,axis=0),d.base.source.norm[key]['q01']) + assert np.allclose(np.quantile(raw,.99,axis=0),d.base.source.norm[key]['q99']) + sample=d[0] + assert all(sample[k].shape==(3,33,256,256) and torch.isfinite(sample[k]).all() for k in ('rgb','depth','point')) + assert sample['action'].shape==(32,14) and sample['initial_state'].shape==(14,) + ranks=[list(ExactBatchSampler(len(d),r,16,16,1,epoch=0,seed=0)) for r in range(16)] + assert all(len(x)==145 for x in ranks) + used=np.concatenate([np.asarray(x).reshape(-1) for x in ranks]);assert len(np.unique(used))==len(used)==145*256 + for step in (0,144):assert len(set(i for rank in ranks for i in rank[step]))==256 + add_backbone() + from joint_rgb_depth_point_action_fastwam_diffusion import _weighted_action_mse + from joint_rgb_action_diffusion import _action_mse + x=torch.ones(1,3,14,requires_grad=True);target=torch.zeros_like(x);pad=torch.tensor([[False,False,True]]) + l=_weighted_action_mse(x,target,pad,c['action_dimension_loss_weights']);assert torch.allclose(l,_action_mse(x,target,pad));l.sum().backward() + assert torch.allclose(x.grad[0,0,7:13],2*x.grad[0,0,:6]);assert torch.allclose(x.grad[0,0,13],x.grad[0,0,6]);assert x.grad[0,2].abs().sum()==0 + with tempfile.TemporaryDirectory() as tmp: + for p in (ROOT/'code').glob('*.py'):py_compile.compile(str(p),cfile=str(Path(tmp)/(p.name+'c')),doraise=True) + frozen=frozen_inputs();write(ROOT/'reports/submission_inputs.json',frozen);write(ROOT/'reports/cpu_preflight_PASS.json',dict(status='PASS',tested_files=frozen,episodes=40,segments=41,frames=37273,world_size=16,micro_batch=16,global_batch=256,total_steps=4350,raw_capture_time_filter='PASS',selected_population_and_no_old_rows='PASS',actual_selected_dataset_sample='PASS',all16_sampler_partition='PASS',right_arm_gradient2x_grippers1x='PASS',selected40_normalizer_fit_and_data_use='PASS',normalizer_sha256=c['normalizer_override_sha256'],fresh_pretrained_checkpoint=c['pretrained_checkpoint'],fresh_pretrained_sha256=c['pretrained_sha256'])) + print(json.dumps(dict(status='PASS',episodes=40,frames=37273,total_steps=4350,world_size=16,files=len(frozen))),flush=True) +if __name__=='__main__':main() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cuda_probe.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cuda_probe.py new file mode 100644 index 0000000000000000000000000000000000000000..91322486f83803829c8d27755782b3fa4a5b528a --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/cuda_probe.py @@ -0,0 +1,56 @@ +"""Validate the eight-card H200-queue allocation without assuming its driver brand.""" +import argparse +import os +import socket +import subprocess + +import torch + +from common import write + +def validate_inventory(cards): + if len(cards) != 8: + raise RuntimeError(f'Expected eight assigned GPUs, got {len(cards)}') + if len({c['uuid'] for c in cards}) != 8: + raise RuntimeError('Assigned GPU UUIDs must be distinct') + for card in cards: + # Successful reference jobs use this L20X driver alias in the H200 queue. + if 'H200' not in card['name'] and card['name'] != 'NVIDIA L20X': + raise RuntimeError(f"Unexpected H200-queue runtime device: {card['name']}") + if card['memory_bytes'] < 130 * 2**30: + raise RuntimeError('Assigned GPU has less than 130 GiB memory') + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--report', required=True) + args = p.parse_args() + lines = subprocess.check_output(['nvidia-smi', '--query-gpu=index,uuid', '--format=csv,noheader'], text=True).splitlines() + expected = {int(line.split(',')[0]): line.split(',')[1].strip().removeprefix('GPU-') for line in lines} + if torch.cuda.device_count() != 8: + raise RuntimeError('Formal training requires exactly eight visible GPUs') + rows = [] + for rank in range(8): + prop = torch.cuda.get_device_properties(rank) + uuid = str(prop.uuid).removeprefix('GPU-') + if uuid != expected[rank]: + raise RuntimeError(f'CUDA/NVML UUID mismatch on GPU {rank}') + rows.append(dict(index=rank, name=prop.name, uuid=uuid, memory_bytes=prop.total_memory, + compute_capability=list(torch.cuda.get_device_capability(rank)))) + validate_inventory(rows) + for rank, row in enumerate(rows): + with torch.cuda.device(rank): + for dtype in (torch.float32, torch.bfloat16): + x = torch.ones(1024, 1024, device=f'cuda:{rank}', dtype=dtype) + y = x @ x + torch.cuda.synchronize(rank) + if not torch.isfinite(y).all() or float(y[0, 0]) != 1024: + raise RuntimeError(f'{dtype} operation failed on GPU {rank}') + row[str(dtype)] = 'PASS' + del x, y + write(args.report, dict(job=os.environ.get('BIFROST_JOB_NAME'), host=socket.gethostname(), gpus=rows, + status='ACTUAL_EIGHT_H200_QUEUE_UUID_FP32_BF16_PASS', torch_version=torch.__version__, cuda_version=torch.version.cuda)) + + +if __name__ == '__main__': + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/data.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/data.py new file mode 100644 index 0000000000000000000000000000000000000000..1e926861ae2fe2d04686922d411cf1bfbf07903a --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/data.py @@ -0,0 +1,296 @@ +"""Head-only clips from exactly the reference Arrow rows; no cross-gap windows.""" +from __future__ import annotations + +import bisect +from collections import OrderedDict +import io +import os +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image +import torch +from torch.utils.data import Dataset, Sampler + +from common import fingerprint, geometry_fingerprint, read, sha + + +def window_indices(start, end, horizon=32, frames=33): + a = int(start) + np.arange(horizon, dtype=np.int64) + v = int(start) + np.arange(frames, dtype=np.int64) + return np.minimum(a, end), np.minimum(v, end), v > end + + +def normalize(values, bounds): + lo, hi = (np.asarray(bounds[k], np.float32) for k in ('q01', 'q99')) + active = ~np.isclose(hi, lo) + out = np.zeros_like(values, dtype=np.float32) + out[..., active] = 2 * (values[..., active] - lo[active]) / (hi[active] - lo[active]) - 1 + return np.clip(out, -1, 1) + + +def denormalize(values, bounds): + lo, hi = (np.asarray(bounds[k], np.float32) for k in ('q01', 'q99')) + return (np.clip(values, -1, 1) + 1) * .5 * (hi - lo) + lo + + +def pixel_calibration(calib, source_hw, padded_hw=(224, 224)): + """Calibration resolution -> source video -> exact PIL resize-with-pad pixels. + + Both resize stages use pixel-center transforms. Integer resized dimensions + and asymmetric padding reproduce OpenPI's actual resize implementation. + """ + sh, sw = source_hw + ph, pw = padded_hw + ratio = max(sw / pw, sh / ph) + rh, rw = int(sh / ratio), int(sw / ratio) + top, left = int((ph - rh) / 2), int((pw - rw) / 2) + sx, sy = rw / calib['width'], rh / calib['height'] + affine = np.array([[sx, 0, left + (sx - 1) / 2], + [0, sy, top + (sy - 1) / 2], [0, 0, 1]], np.float64) + k = affine @ np.asarray(calib['k'], np.float64).reshape(3, 3) + yy, xx = np.mgrid[:ph, :pw] + pixels = np.stack((xx, yy), -1).astype(np.float64).reshape(-1, 1, 2) + distortion = np.asarray(calib['d'], np.float64) + if calib['distortion_model'] == 'equidistant': + normalized = cv2.fisheye.undistortPoints(pixels, k, distortion.reshape(4, 1)) + elif calib['distortion_model'] in ('plumb_bob', 'rational_polynomial', ''): + normalized = cv2.undistortPoints(pixels, k, distortion if distortion.size else None) + else: + raise ValueError(f"Unsupported distortion model: {calib['distortion_model']}") + rays = np.concatenate((normalized.reshape(ph, pw, 2), np.ones((ph, pw, 1))), -1).astype(np.float32) + valid = (yy >= top) & (yy < top + rh) & (xx >= left) & (xx < left + rw) + return k, rays, valid + + +def point_video(depth, rays, valid): + """Z-depth in meters -> dense optical XYZ, using the pretraining XYZ bounds.""" + ok = valid[None] & np.isfinite(depth) & (depth >= .05) & (depth <= 5.) + xyz = depth[..., None] * rays[None] + xyz[~ok] = [0., -.5, 3.] + lo = np.array([-5., -5.5, 0.], np.float32) + hi = np.array([5., 4.5, 6.], np.float32) + result = 2 * (np.clip(xyz, lo, hi) - lo) / (hi - lo) - 1 + return result.transpose(3, 0, 1, 2).astype(np.float32), ok + + +def spatial_augmentation(rgb, depth, rays, valid, rng): + """One crop+rotation per clip, shared by RGB, depth, rays and validity. + + Reindexing calibrated rays rather than rotating XYZ axes also supports + distorted cameras and image homographies with non-triangular transformed K. + """ + h, w = valid.shape + ch, cw = int(h * .95), int(w * .95) + top, left = int(rng.integers(h - ch + 1)), int(rng.integers(w - cw + 1)) + sx, sy = w / cw, h / ch + crop = np.array([[sx, 0, -left * sx + (sx - 1) / 2], + [0, sy, -top * sy + (sy - 1) / 2], [0, 0, 1]], np.float64) + rotate = np.eye(3) + rotate[:2] = cv2.getRotationMatrix2D(((w - 1) / 2, (h - 1) / 2), float(rng.uniform(-5, 5)), 1.) + matrix = rotate @ crop + warp = lambda x, interpolation: cv2.warpPerspective(x, matrix, (w, h), flags=interpolation, borderMode=cv2.BORDER_CONSTANT) + rgb = np.stack([warp(x, cv2.INTER_LINEAR) for x in rgb]) + depth = np.stack([warp(x, cv2.INTER_NEAREST) for x in depth]) + rays = warp(rays, cv2.INTER_NEAREST) + valid = warp(valid.astype(np.uint8), cv2.INTER_NEAREST).astype(bool) + return rgb, depth, rays, valid + + +class SourceRows: + def __init__(self, config): + self.config = config + self.assets = Path(os.environ.get('SD_GALEXEA_LOCAL_ASSETS', str(Path(config['reference_root']) / 'assets'))) + self.index = read(self.assets / 'row_index.json') + self.segments = self.index['segments'] + self.offsets = [s['offset'] for s in self.segments] + self.norm = read(config.get('normalizer_override', self.assets / 'reference_pi05_norm_stats.json'))['norm_stats'] + self._reader = self._mapping = self._actions = self._states = None + self._decoded = OrderedDict() + self._batches = OrderedDict() + + def __len__(self): + return self.index['frames'] + + def __getstate__(self): + state = dict(self.__dict__) + for key in ('_reader', '_mapping', '_actions', '_states'): + state[key] = None + state['_batches'] = OrderedDict() + state['_decoded'] = OrderedDict() + return state + + def open(self): + if self._reader is None: + import pyarrow as pa + import pyarrow.ipc as ipc + self._mapping = pa.memory_map(str(self.assets / 'reference_rows.arrow'), 'r') + self._reader = ipc.open_file(self._mapping) + self._actions = np.load(self.assets / 'actions.npy', mmap_mode='r') + self._states = np.load(self.assets / 'states.npy', mmap_mode='r') + + def segment_for(self, row): + if not 0 <= row < len(self): + raise IndexError(row) + return bisect.bisect_right(self.offsets, row) - 1 + + def batch(self, segment): + self.open() + if segment not in self._batches: + if len(self._batches) >= 2: + self._batches.popitem(last=False) + self._batches[segment] = self._reader.get_batch(segment) + return self._batches[segment] + + def rgb(self, segment, row): + decoded = self.assets / 'head_segments' / f'segment_{segment:06d}.npy' + if decoded.exists(): + if segment not in self._decoded: + if len(self._decoded) >= 2: + self._decoded.popitem(last=False) + self._decoded[segment] = np.load(decoded, mmap_mode='r') + return np.asarray(self._decoded[segment][int(row) - self.offsets[segment]]).copy() + b = self.batch(segment) + cell = b.column(b.schema.get_field_index('observation.images.head_left'))[int(row) - self.offsets[segment]].as_py() + with Image.open(io.BytesIO(cell['bytes'])) as im: + return np.asarray(im.convert('RGB')).copy() + + def decode_segment(self, segment): + """Decode only head PNGs once on the allocated node's private NVMe.""" + if 'SD_GALEXEA_LOCAL_ASSETS' not in os.environ: + raise RuntimeError('Decoded image cache must live in private node-local assets') + target = self.assets / 'head_segments' / f'segment_{segment:06d}.npy' + record = self.segments[segment] + if target.exists(): + if np.load(target, mmap_mode='r').shape != (record['length'], 224, 224, 3): + raise ValueError('Decoded head cache shape mismatch') + return + target.parent.mkdir(exist_ok=True) + temporary = target.with_suffix('.partial.npy') + pixels = np.lib.format.open_memmap(temporary, mode='w+', dtype=np.uint8, shape=(record['length'], 224, 224, 3)) + for row in range(record['length']): + pixels[row] = self.rgb(segment, record['offset'] + row) + pixels.flush() + del pixels + temporary.replace(target) + + def identity(self): + """Rehash small labels/index/norm; large Arrow is bound by size+mtime+recorded hash.""" + plan = read(Path(self.config['reference_root']) / 'training_plan.json') + identities = {name: sha(self.assets / name) for name in ('row_index.json', 'actions.npy', 'states.npy', 'reference_pi05_norm_stats.json')} + for name, actual in identities.items(): + if actual != plan['assets'][name]['sha256']: + raise ValueError(f'Reference asset hash mismatch: {name}') + p = self.assets / 'reference_rows.arrow' + identities['reference_rows.arrow'] = {'bytes': p.stat().st_size, 'mtime_ns': p.stat().st_mtime_ns, + 'recorded_sha256': plan['assets']['reference_rows.arrow']['sha256'], 'fresh_full_rehash': False} + if p.stat().st_size != plan['assets']['reference_rows.arrow']['bytes']: + raise ValueError('Reference Arrow size changed') + return identities + + +class GalexeaDataset(Dataset): + def __init__(self, config, training=True): + self.config = config + self.source = SourceRows(config) + self.training = training + self.epoch = 0 + self.prepared = Path(config['prepared_root']) + manifest = read(self.prepared / 'manifest.json') + self.source_fingerprint = fingerprint(self.source.identity()) + if manifest['source_fingerprint'] != self.source_fingerprint: + raise ValueError('Prepared geometry belongs to different source rows') + if manifest.get('geometry_fingerprint') != geometry_fingerprint(config): + raise ValueError('Prepared geometry has different camera/VDA configuration') + if manifest['segments'] != len(self.source.segments) or manifest['frames'] != len(self.source): + raise ValueError('Incomplete prepared manifest') + self.context = torch.load(self.source.assets / 'context.pt', weights_only=True, map_location='cpu') + if not self.context['prompt'].endswith(config['prompt']): + raise ValueError('Text context prompt mismatch') + self._depth = OrderedDict() + + def __len__(self): + return len(self.source) + + def __getstate__(self): + state = dict(self.__dict__) + state['_depth'] = OrderedDict() + return state + + def geometry(self, segment): + if segment not in self._depth: + folder = self.prepared / f'segment_{segment:06d}' + receipt = read(folder / 'ready.json') + if receipt['source_fingerprint'] != self.source_fingerprint or receipt['frames'] != self.source.segments[segment]['length']: + raise ValueError(f'Geometry cache identity/length mismatch: {folder}') + if receipt['depth_mode'] != 'streaming_causal' or receipt['vda_sha256'] != self.config['vda_sha256']: + raise ValueError('Unverified, noncausal, or wrong-model depth cache') + if receipt.get('geometry_fingerprint') != geometry_fingerprint(self.config): + raise ValueError('Segment geometry fingerprint mismatch') + if len(self._depth) >= 2: + self._depth.popitem(last=False) + with np.load(folder / 'calibration.npz') as c: + rays, valid = c['rays'].copy(), c['valid'].copy() + self._depth[segment] = (np.load(folder / 'depth_m.npy', mmap_mode='r'), rays, valid) + return self._depth[segment] + + def __getitem__(self, row): + source = self.source + source.open() + seg = source.segment_for(row) + record = source.segments[seg] + end = record['offset'] + record['length'] - 1 + actions, video, padded = window_indices(row, end) + rgb = np.stack([source.rgb(seg, i) for i in video]) + d, rays, valid = self.geometry(seg) + depth = np.asarray(d[video - record['offset']], np.float32) + if self.training: + rng = np.random.default_rng(np.random.SeedSequence([self.config['seed'], self.epoch, int(row)])) + rgb, depth, rays, valid = spatial_augmentation(rgb, depth, rays, valid, rng) + resolution = self.config['resolution'] + rgb = np.stack([cv2.resize(x, (resolution, resolution), interpolation=cv2.INTER_LINEAR) for x in rgb]) + depth = np.stack([cv2.resize(x, (resolution, resolution), interpolation=cv2.INTER_NEAREST) for x in depth]) + rays = cv2.resize(rays, (resolution, resolution), interpolation=cv2.INTER_NEAREST) + valid = cv2.resize(valid.astype(np.uint8), (resolution, resolution), interpolation=cv2.INTER_NEAREST).astype(bool) + point, ok = point_video(depth, rays, valid) + depth = np.where(ok, depth, 5.) + depth = np.repeat((2 * np.clip(depth, 0, 5) / 5 - 1)[None], 3, axis=0).astype(np.float32) + rgb = torch.from_numpy(rgb).permute(0, 3, 1, 2).float() / 255 + if self.training: + from torchvision.transforms import ColorJitter + with torch.random.fork_rng(devices=[]): + torch.manual_seed(int(row) + 1_000_003 * self.epoch + self.config['seed']) + rgb = ColorJitter(brightness=.3, contrast=.4, saturation=.5)(rgb) + return {'rgb': rgb.permute(1, 0, 2, 3).contiguous() * 2 - 1, + 'depth': torch.from_numpy(depth), 'point': torch.from_numpy(point), + 'action': torch.from_numpy(normalize(np.asarray(source._actions[actions]), source.norm['actions'])), + 'initial_state': torch.from_numpy(normalize(np.asarray(source._states[row]), source.norm['state'])), + 'image_is_pad': torch.from_numpy(padded), + # Reference jobs repeat the last action and train on every slot. + 'action_is_pad': torch.zeros(32, dtype=torch.bool), + 'context': self.context['context'], 'context_mask': self.context['mask'].bool(), + 'row': int(row)} + + +class ExactBatchSampler(Sampler): + """Shuffle all starts; drop only the final incomplete GLOBAL batch, like reference.""" + def __init__(self, size, rank, world, micro, accumulation, epoch=0, start_step=0, seed=0): + self.size, self.rank, self.world = size, rank, world + self.micro, self.accumulation = micro, accumulation + self.epoch, self.start_step, self.seed = epoch, start_step, seed + self.global_batch = world * micro * accumulation + if not 0 <= rank < world or start_step < 0 or start_step > size // self.global_batch: + raise ValueError('Invalid rank or resume position') + + def __len__(self): + return (self.size // self.global_batch - self.start_step) * self.accumulation + + def __iter__(self): + g = torch.Generator().manual_seed(self.seed + self.epoch) + n = self.size // self.global_batch + permutation = torch.randperm(self.size, generator=g)[:n * self.global_batch] + ids = permutation.reshape(n, self.accumulation, self.world, self.micro) + for step in range(self.start_step, n): + for microstep in range(self.accumulation): + yield ids[step, microstep, self.rank].tolist() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/decode_selected.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/decode_selected.py new file mode 100644 index 0000000000000000000000000000000000000000..396a85267535641cbf86cc04b1a935a095be14d9 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/decode_selected.py @@ -0,0 +1,9 @@ +import argparse +from concurrent.futures import ThreadPoolExecutor +from common import load_config,read,write +from data import SourceRows +p=argparse.ArgumentParser();p.add_argument('--config',required=True);p.add_argument('--report',required=True);a=p.parse_args();c=load_config(a.config);selection=read(c['data_selection']['manifest']) +def decode(rec): + s=SourceRows(c);s.decode_segment(rec['original_segment']);return rec['length'] +with ThreadPoolExecutor(max_workers=4) as pool:frames=sum(pool.map(decode,selection['selected_segments'])) +assert frames==c['frames'];write(a.report,dict(status='PASS_ONLY_SELECTED_40_EPISODES_DECODED',frames=frames,segments=len(selection['selected_segments']))) diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/distributed.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..0807d11befbcbfb1bbb370375601a1513708af1d --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/distributed.py @@ -0,0 +1,17 @@ +"""One distributed-engine configuration for the analytical probe and full model.""" + + +def make_accelerator(config, *, grad_clip=None): + from accelerate import Accelerator, DeepSpeedPlugin + c = config + zero = DeepSpeedPlugin(hf_ds_config={ + 'train_batch_size': c['global_batch'], 'train_micro_batch_size_per_gpu': c['micro_batch'], + 'gradient_accumulation_steps': c['accumulation'], + 'gradient_clipping': c['grad_clip'] if grad_clip is None else grad_clip, + 'bf16': {'enabled': True}, 'steps_per_print': 1000000, + 'zero_optimization': {'stage': 2, 'overlap_comm': True, 'contiguous_gradients': True, + 'reduce_scatter': True, 'allgather_partitions': True, + 'allgather_bucket_size': 200000000, 'reduce_bucket_size': 200000000}, + 'zero_allow_untested_optimizer': True}) + return Accelerator(mixed_precision='bf16', gradient_accumulation_steps=c['accumulation'], + deepspeed_plugin=zero, step_scheduler_with_optimizer=False) diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/model.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/model.py new file mode 100644 index 0000000000000000000000000000000000000000..e9fd79d642f74a097e190775b5a47bb42b8085fa --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/model.py @@ -0,0 +1,135 @@ +"""SpaceDreamer-v3 transfer with explicit EE-to-joint embodiment adaptation.""" +from __future__ import annotations + +from contextlib import nullcontext +import copy +import os +from pathlib import Path + +import torch +from torch import nn + +from common import add_backbone, read + +CAMERA_ONLY = { + 'structured_type_embedding', 'camera_encoder.bias', 'camera_encoder.weight', + 'camera_head.bias', 'camera_head.weight', 'camera_normalizer.count', + 'camera_normalizer.fitted', 'camera_normalizer.mean', 'camera_normalizer.std', +} +SOURCE_NORMALIZER = {'action_normalizer.' + k for k in ('mean', 'std', 'count', 'fitted')} +NEW_INTERFACE_PREFIXES = ('mot.mixtures.action.action_encoder.', 'mot.mixtures.action.head.', 'action_state_encoder.') + + +class JointNormalizer(nn.Module): + def __init__(self, norm): + super().__init__() + for feature, key in [('state', 'state'), ('action', 'actions')]: + for bound in ('q01', 'q99'): + self.register_buffer(feature + '_' + bound, torch.tensor(norm[key][bound], dtype=torch.float32)) + + +def construct(config, *, tiny=False, meta=False): + add_backbone() + from fastwam.models.wan22.wan_video_dit import WanVideoDiT, precompute_freqs_cis_3d, precompute_freqs_cis + from robodojo_rgb_geometry_action_model import _build_geometry_action_mot + from wan_rgb_depth_point_initialization import upgrade_wan_dit_for_rgb_depth_point, enable_point_first_frame_timestep_adapter + model_config = copy.deepcopy(config['model']) + if tiny: + model_config['dit'].update(hidden_dim=48, ffn_dim=96, num_layers=2, num_heads=2, + attn_head_dim=24, text_dim=32, freq_dim=16, + use_gradient_checkpointing=False) + model_config['action_dit'].update(hidden_dim=32, ffn_dim=64, num_layers=2, num_heads=2, + attn_head_dim=24, text_dim=32, freq_dim=16, + use_gradient_checkpointing=False) + context = torch.device('meta') if meta else nullcontext() + with context: + video = WanVideoDiT(**model_config['dit']) + upgrade_wan_dit_for_rgb_depth_point(video) + enable_point_first_frame_timestep_adapter(video, point_bottleneck_dim=8 if tiny else 256) + model = _build_geometry_action_mot(video, model_config['action_dit'], modalities=('rgb', 'depth', 'point'), + random_seed=config['seed'], action_state_dim=14, mot_checkpoint_mixed_attn=False, + cross_modal_attention='bidirectional') + # RoPE tensors are not registered buffers and need explicit real-device construction. + video.freqs = precompute_freqs_cis_3d(model_config['dit']['attn_head_dim']) + model.action_expert.freqs = precompute_freqs_cis(model_config['action_dit']['attn_head_dim'], end=1024) + model.condition_depth_frame0 = True + norm = read(config.get('normalizer_override', Path(config['reference_root']) / 'assets/reference_pi05_norm_stats.json'))['norm_stats'] + model.add_module('action_normalizer', JointNormalizer(norm)) + return model + + +def transfer_weights(model, payload, *, seed=0): + source = payload['state_dict'] + metadata = payload['metadata'] + expected_video = dict(joint_rgb_depth_point=True, joint_input_channels=144, + rgb_latent_channels=48, depth_latent_channels=48, point_latent_channels=48, + rgb_frame0_conditioned=True, depth_frame0_conditioned=True, point_frame0_conditioned=False) + if any(metadata['video'].get(k) != v for k, v in expected_video.items()): + raise ValueError('Source checkpoint video contract is not SpaceDreamer-v3') + if metadata['mot']['action_dim'] != 14 or metadata['mot']['cross_modal_attention'] != 'bidirectional': + raise ValueError('Source action expert contract mismatch') + target = model.state_dict() + fresh = {k for k in target if k.startswith(NEW_INTERFACE_PREFIXES) or k.startswith('action_normalizer.')} + source_only = set(source) - set(target) + if source_only != CAMERA_ONLY | SOURCE_NORMALIZER: + raise ValueError(f'Unexpected source-only tensors: {sorted(source_only)}') + needed = set(target) - fresh + if needed - set(source): + raise ValueError(f'Missing pretrained backbone keys: {sorted(needed - set(source))}') + for k in needed: + if source[k].shape != target[k].shape: + raise ValueError(f'Pretrained shape mismatch: {k}') + result = model.load_state_dict({k: source[k] for k in needed}, strict=False, assign=True) + if set(result.missing_keys) != fresh or result.unexpected_keys: + raise RuntimeError(f'Unexpected partial load: {result}') + # Different physical meanings: source EE xyz/so3log -> target joint position. + # Adapt only these interfaces; every video and ActionDiT backbone tensor transfers. + with torch.random.fork_rng(devices=[]): + torch.manual_seed(seed) + for module in (model.action_expert.action_encoder, model.action_expert.head, model.action_state_encoder): + if any(p.is_meta for p in module.parameters()): + module.to_empty(device='cpu') + for layer in module.modules(): + if isinstance(layer, nn.Linear): + layer.reset_parameters() + if any(p.is_meta for p in model.parameters()): + raise RuntimeError('Some model parameters were not materialized') + return dict(loaded_backbone_tensors=len(needed), fresh_interface_tensors=sorted(fresh), + omitted_source_tensors=sorted(source_only), pretrained_action_semantics='absolute_ee_xyz_so3log_grip', + target_action_semantics='absolute_joint_position_grip', optimizer_restored=False) + + +def build_model(config): + model = construct(config, meta=True) + checkpoint = torch.load(os.environ.get('SD_GALEXEA_LOCAL_BASE', config['pretrained_checkpoint']), weights_only=True, mmap=True, map_location='cpu') + audit = transfer_weights(model, checkpoint, seed=config['seed']) + return model, audit + + +@torch.no_grad() +def encode_batch(vae, batch, device, config): + from vae_runtime import encode_video + latents = {} + for modality in ('rgb', 'depth', 'point'): + pixels = batch[modality].to(device=device, dtype=torch.bfloat16, non_blocking=True) + z = encode_video(vae, pixels, device, config['vae_encode_batch']) + if tuple(z.shape[1:]) != (48, 9, 16, 16): + raise ValueError(f'Unexpected {modality} VAE latent shape: {z.shape}') + latents['z_' + modality] = z + return latents + + +def loss(model, batch, latents, video_scheduler, action_scheduler, config, device): + add_backbone() + from joint_rgb_depth_point_action_fastwam_diffusion import joint_rgb_depth_point_action_fastwam_loss + values = {k: batch[k].to(device, dtype=torch.bfloat16, non_blocking=True) + for k in ('action', 'initial_state', 'context')} + if values['action'].shape[1:] != (32, 14): + raise ValueError('Action chunk must be [B,32,14]') + return joint_rgb_depth_point_action_fastwam_loss( + model, video_scheduler, action_scheduler, **latents, **values, + context_mask=batch['context_mask'].to(device, dtype=torch.bool), + image_is_pad=batch['image_is_pad'].to(device), action_is_pad=batch['action_is_pad'].to(device), + lambda_rgb=config['loss_weights']['rgb'], lambda_depth=config['loss_weights']['depth'], + lambda_point=config['loss_weights']['point'], lambda_action=config['loss_weights']['action'], + condition_depth_frame0=True, action_dimension_loss_weights=config['action_dimension_loss_weights']) diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/output_guard.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/output_guard.py new file mode 100644 index 0000000000000000000000000000000000000000..4ca6ae8755bd9e233f287c48b926bee576852b02 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/output_guard.py @@ -0,0 +1,38 @@ +"""Only rank zero claims a fresh output; all ranks receive the same decision.""" +import json +import os +from pathlib import Path + + +def claim_output(path, payload): + path = Path(path) + path.mkdir(parents=True, exist_ok=True) + try: + fd = os.open(path / 'STARTED.json', os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + except FileExistsError as error: + raise RuntimeError('An existing training output requires explicit --resume') from error + with os.fdopen(fd, 'w') as stream: + json.dump(payload, stream, indent=2) + stream.write('\n') + stream.flush() + os.fsync(stream.fileno()) + + +def initialize_output(accelerator, path, payload, resume=None): + import torch.distributed as dist + result = [None] + if accelerator.is_main_process: + try: + if resume is None: + claim_output(path, payload) + else: + marker = Path(path) / 'STARTED.json' + if marker.exists() and json.loads(marker.read_text())['config_fingerprint'] != payload['config_fingerprint']: + raise ValueError('Resume output configuration changed') + except Exception as error: + result[0] = f'{type(error).__name__}: {error}' + if dist.is_initialized(): + dist.broadcast_object_list(result, src=0, device=accelerator.device) + if result[0] is not None: + raise RuntimeError(result[0]) + accelerator.wait_for_everyone() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/prepare.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/prepare.py new file mode 100644 index 0000000000000000000000000000000000000000..df23b3639cf10720e73b80278e15bbcbe5c18d5f --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/prepare.py @@ -0,0 +1,154 @@ +"""Causal metric VDA depth, calibrated rays, and all-segment completion receipts.""" +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import sys + +import numpy as np +import torch + +from common import ROOT, fingerprint, geometry_fingerprint, load_config, read, sha, write +from data import SourceRows, pixel_calibration +from calibration import camera_calibration + + +def load_vda(config, device): + sys.path.insert(0, str(ROOT / 'source/vda')) + from video_depth_anything.video_depth_stream import VideoDepthAnything + if sha(config['vda_checkpoint']) != config['vda_sha256']: + raise ValueError('VDA weights changed') + model = VideoDepthAnything(encoder='vitb', features=128, out_channels=[96, 192, 384, 768]) + model.load_state_dict(torch.load(config['vda_checkpoint'], map_location='cpu', weights_only=True), strict=True) + return model.to(device).eval().requires_grad_(False) + + +def reset_stream(model): + model.transform = None + model.frame_id_list = [] + model.frame_cache_list = [] + model.id = -1 + + +def prepare_segment(config, source, index, model, device, source_identity, *, frame_limit=None): + segment = source.segments[index] + n = segment['length'] + smoke = frame_limit is not None + folder = Path(config['prepared_root']) / (f'smoke_segment_{index:06d}' if smoke else f'segment_{index:06d}') + folder.mkdir(parents=True, exist_ok=True) + ready = folder / 'ready.json' + identity = fingerprint(source_identity) + if ready.exists() and not smoke: + receipt = read(ready) + if (receipt['source_fingerprint'], receipt['vda_sha256'], receipt['frames'], receipt['depth_mode']) != (identity, config['vda_sha256'], n, 'streaming_causal'): + raise ValueError(f'Existing geometry has different provenance: {folder}') + if receipt.get('geometry_fingerprint') != geometry_fingerprint(config): + raise ValueError('Prepared geometry calibration/configuration changed') + return receipt + # A per-segment lock permits distributed preprocessing shards without partial readers. + lock = folder / 'preparing.lock' + descriptor = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) + os.write(descriptor, f'{os.getpid()}\n'.encode()) + os.close(descriptor) + try: + n = min(n, frame_limit) if smoke else n + image = source.rgb(index, segment['offset']) + calib = camera_calibration(segment['source_episode_key'], config) + k, rays, valid = pixel_calibration(calib, config['source_video_hw'], image.shape[:2]) + write(folder / 'calibration.json', {**calib, 'reference_video_hw': config['source_video_hw'], + 'reference_padded_hw': list(image.shape[:2]), 'processed_K': k.tolist(), + 'point_frame': config['point_frame']}) + np.savez(folder / 'calibration.npz', k=k, rays=rays, valid=valid) + temporary = folder / 'depth_m.partial.npy' + depth = np.lib.format.open_memmap(temporary, mode='w+', dtype=np.float16, shape=(n, *image.shape[:2])) + reset_stream(model) + positive = 0 + for row in range(n): + image = source.rgb(index, segment['offset'] + row) + with torch.inference_mode(): + prediction = model.infer_video_depth_one(image, input_size=config['vda_input_size'], + device=str(device), fp32=device.type == 'cpu') + prediction = np.asarray(prediction, np.float32) + if prediction.shape != valid.shape or not np.isfinite(prediction).all(): + raise FloatingPointError(f'Invalid VDA result in segment {index}, row {row}') + mask = valid & (prediction > 0) + positive += int(mask.sum()) + # Keep metric values, never use a visualization colormap as supervision. + depth[row] = np.where(mask, np.minimum(prediction, 65000.), 0).astype(np.float16) + if row % 200 == 0: + print(dict(segment=index, frame=row, frames=n), flush=True) + depth.flush() + del depth + if positive < .01 * n * int(valid.sum()): + raise ValueError('Too few valid metric-depth pixels') + destination = folder / 'depth_m.npy' + temporary.replace(destination) + receipt = dict(frames=n, source_fingerprint=identity, vda_sha256=config['vda_sha256'], + geometry_fingerprint=geometry_fingerprint(config), + vda_input_size=config['vda_input_size'], depth_mode='streaming_causal', depth_unit='meter', + dtype='float16', camera_source='head_left', point_frame=config['point_frame'], + depth_sha256=sha(destination), calibration_sha256=sha(folder / 'calibration.npz'), + complete=not smoke, positive_pixels=positive) + write(ready, receipt) + return receipt + finally: + lock.unlink(missing_ok=True) + + +def finalize(config, source): + prepared = Path(config['prepared_root']) + receipts = [] + for i, segment in enumerate(source.segments): + file = prepared / f'segment_{i:06d}/ready.json' + if not file.is_file(): + return False + r = read(file) + if not r['complete'] or r['frames'] != segment['length'] or r['source_fingerprint'] != config['source_fingerprint'] or r['vda_sha256'] != config['vda_sha256']: + raise ValueError(f'Invalid prepared segment receipt: {file}') + if r.get('geometry_fingerprint') != geometry_fingerprint(config): + raise ValueError('Prepared segment geometry configuration mismatch') + receipts.append(sha(file)) + write(prepared / 'manifest.json', dict(source_fingerprint=config['source_fingerprint'], + geometry_fingerprint=geometry_fingerprint(config), + vda_sha256=config['vda_sha256'], segments=len(receipts), frames=len(source), + segment_receipt_sha256=receipts, complete=True)) + return True + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--config', type=Path, required=True) + p.add_argument('--device', default='cuda') + p.add_argument('--shard-index', type=int, default=0) + p.add_argument('--num-shards', type=int, default=1) + p.add_argument('--segment', type=int) + p.add_argument('--smoke-frames', type=int) + p.add_argument('--finalize-only', action='store_true') + p.add_argument('--decode-head', action='store_true') + args = p.parse_args() + if not 0 <= args.shard_index < args.num_shards: + p.error('Invalid shard index/count') + config = load_config(args.config) + source = SourceRows(config) + identity = source.identity() + if fingerprint(identity) != config['source_fingerprint']: + raise ValueError('Reference identity changed') + if args.finalize_only: + if not finalize(config, source): + raise RuntimeError('Some segments are not prepared') + return + device = torch.device(args.device) + model = load_vda(config, device) + indices = [args.segment] if args.segment is not None else range(args.shard_index, len(source.segments), args.num_shards) + for index in indices: + if args.decode_head: + source.decode_segment(index) + receipt = prepare_segment(config, source, index, model, device, identity, frame_limit=args.smoke_frames) + print(dict(segment=index, receipt=receipt), flush=True) + if args.smoke_frames is None: + print(dict(all_segments_ready=finalize(config, source)), flush=True) + + +if __name__ == '__main__': + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/selection_data.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/selection_data.py new file mode 100644 index 0000000000000000000000000000000000000000..511c1a45c3c74aca97b96cece9438c9badef16e1 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/selection_data.py @@ -0,0 +1,23 @@ +from pathlib import Path +import numpy as np +from torch.utils.data import Dataset +from data import GalexeaDataset +from common import sha,read +class SelectedDataset(Dataset): + def __init__(self,config,training=True): + selection=config['data_selection'] + if sha(selection['row_indices'])!=selection['row_indices_sha256']:raise ValueError('Selected row map hash mismatch') + self.rows=np.load(selection['row_indices'],allow_pickle=False) + if self.rows.ndim!=1 or self.rows.dtype!=np.int64 or len(self.rows)!=config['frames'] or len(np.unique(self.rows))!=len(self.rows):raise ValueError('Invalid selected row map') + self.base=GalexeaDataset(config,training=training) + if self.rows.min()<0 or self.rows.max()>=len(self.base):raise ValueError('Selected row outside original source') + allowed=set(read(selection['manifest'])['included_episode_keys']) + for i in np.unique([self.base.source.segment_for(int(x)) for x in self.rows]): + rec=self.base.source.segments[int(i)] + if rec['source_episode_key'] not in allowed:raise ValueError('Disallowed source episode in sampler') + @property + def epoch(self):return self.base.epoch + @epoch.setter + def epoch(self,value):self.base.epoch=value + def __len__(self):return len(self.rows) + def __getitem__(self,index):return self.base[int(self.rows[index])] diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/staging.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/staging.py new file mode 100644 index 0000000000000000000000000000000000000000..cfc7450239f4689a7a99c2bcadc19f99a8921b01 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/staging.py @@ -0,0 +1,56 @@ +"""Parallel, bounded immutable-file copies with preserved source identity.""" +from concurrent.futures import ThreadPoolExecutor +import os +from pathlib import Path +import shutil +import time + +from common import sha + + +def verified_copy(source, destination, expected): + source, destination = Path(source), Path(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + before = source.stat() + if destination.exists() and destination.stat().st_size == before.st_size and sha(destination) == expected: + shutil.copystat(source, destination) + return + temporary = destination.with_name(destination.name + '.copying') + infd = os.open(source, os.O_RDONLY) + outfd = os.open(temporary, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600) + parts = min(8, max(1, (before.st_size + (64 << 20) - 1) // (64 << 20))) + def part(index): + offset, end = before.st_size * index // parts, before.st_size * (index + 1) // parts + while offset < end: + for attempt in range(5): + try: + block = os.pread(infd, min(8 << 20, end - offset), offset) + break + except OSError as error: + if error.errno not in (5, 11, 116, 121, 523) or attempt == 4: + raise + time.sleep(min(2**attempt, 8)) + if not block: + raise OSError(f'Immutable source ended early: {source}') + written = 0 + while written < len(block): + n = os.pwrite(outfd, block[written:], offset + written) + if n <= 0: + raise OSError('Local staging write made no progress') + written += n + offset += len(block) + try: + os.ftruncate(outfd, before.st_size) + with ThreadPoolExecutor(max_workers=parts) as pool: + list(pool.map(part, range(parts))) + os.fsync(outfd) + finally: + os.close(infd) + os.close(outfd) + after = source.stat() + if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns): + raise ValueError('Staging source changed') + if sha(temporary) != expected: + raise ValueError(f'Local copy SHA256 mismatch: {source}') + shutil.copystat(source, temporary) + temporary.replace(destination) diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/storage_budget.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/storage_budget.py new file mode 100644 index 0000000000000000000000000000000000000000..a6ca6ad80fa646279d2a238c1f31171655b373c3 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/storage_budget.py @@ -0,0 +1,237 @@ +"""Own dense NAS capacity, exchanged for checkpoint bytes and reclaimed after pruning. + +Training overwrites a bounded set of checkpoints. Unlike append-only evaluation, +the reservation is replenished only against the measured current owned payload. +No other task's reservation, payload or historical headroom is borrowed. +""" +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +import fcntl, hashlib, json, os, stat, threading, time, uuid, zipfile + +GIB=2**30 +TASK_BUDGET=280*GIB +CHECKPOINT_ALLOWANCE=80*GIB +FINAL_ALLOWANCE=20*GIB + +def save(path,obj): + path=Path(path);tmp=path.with_name('.'+path.name+'.'+uuid.uuid4().hex+'.tmp') + with tmp.open('w') as f: + json.dump(obj,f,indent=2);f.write('\n');f.flush();os.fsync(f.fileno()) + tmp.replace(path) + +def allocated(root): + total=0;seen=set() + if not root.exists():return 0 + for folder,dirs,files in os.walk(root,followlinks=False): + if any((Path(folder)/d).is_symlink() for d in dirs): + raise RuntimeError('Unexpected directory link in owned training output: '+folder) + for name in files: + p=Path(folder)/name;s=p.lstat() + if stat.S_ISLNK(s.st_mode):raise RuntimeError('Unexpected link in owned training output: '+str(p)) + if stat.S_ISREG(s.st_mode) and (s.st_dev,s.st_ino) not in seen: + seen.add((s.st_dev,s.st_ino));total+=s.st_blocks*512 + return total + +class Budget: + def __init__(self,output,reservation,budget=TASK_BUDGET): + self.output=Path(output);self.root=Path(reservation);self.budget=budget + self.blob=self.root/'capacity.bin';self.owner=self.root/'owner.json' + self.retired=self.root/'retired_capacity' + + @classmethod + def for_config(cls,config): + from common import ROOT + return cls(config['output_dir'],ROOT/'reservations'/config['task']) + + def marker(self,owner): + raw=json.dumps(owner,sort_keys=True).encode() + return b'SD_TRAIN_OWNED_DENSE_BUDGET_V1\n'+hashlib.sha256(raw).hexdigest().encode()+b'\n' + + @contextmanager + def lock(self): + self.root.mkdir(parents=True,exist_ok=True) + with (self.root/'owner.lock').open('a') as f: + fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB) + yield + + def _identity(self): + owner=json.loads(self.owner.read_text()) + assert owner['output']==str(self.output) and owner['reservation']==str(self.root) + assert owner['budget_bytes']==self.budget + assert not self.blob.is_symlink() and self.blob.resolve()==self.blob + s=self.blob.stat();assert s.st_nlink==1 and s.st_dev==self.output.parent.stat().st_dev + # The creator's st_dev/inode are provenance, not a portable NAS identity. + # Another allocated host mounts this same path under different device IDs. + # Bind ownership to the exact path, UUID-bearing owner and embedded digest; + # require the current output and reserve to share the current filesystem. + with self.blob.open('rb') as f:assert f.read(len(self.marker(owner)))==self.marker(owner) + return owner + + def _resize_dense(self,wanted,*,initial=False): + wanted=(wanted//4096)*4096 + assert wanted>=4096 and wanted<=self.budget + fd=os.open(self.blob,os.O_RDWR|os.O_NOFOLLOW) + pending=self.root/'resize_inflight.json' + try: + s=os.fstat(fd);old=s.st_size + # Some NAS clients report st_blocks from logical length even for holes. + # A prior incomplete growth therefore needs actual writes again, regardless + # of apparent allocation. Never convert a truncated partial into a PASS. + force_dense=pending.exists() or (initial and not (self.root/'PREPARED.json').exists()) + if old>wanted: + os.ftruncate(fd,wanted);os.fsync(fd) + if force_dense or old=old else 0) + save(pending,dict(owner_id=json.loads(self.owner.read_text())['identity'], + from_bytes=old,to_bytes=wanted,write_from=start,started_unix=time.time())) + os.ftruncate(fd,wanted) + block=bytes(8*2**20) + stop=threading.Event() + def put(offset): + data=memoryview(block)[:min(len(block),wanted-offset)] + while data: + n=os.pwrite(fd,data,offset) + if n<=0:raise OSError('No progress writing owned quota reservation') + offset+=n;data=data[n:] + def lane(worker): + try: + for offset in range(start+worker*len(block),wanted,8*len(block)): + if stop.is_set():return + put(offset) + except BaseException: + stop.set();raise + with ThreadPoolExecutor(max_workers=8) as pool: + for _ in pool.map(lane,range(8)):pass + os.fsync(fd) + owner=json.loads(self.owner.read_text()) + os.pwrite(fd,self.marker(owner),0);os.fsync(fd) + s=os.fstat(fd) + assert s.st_size==wanted and s.st_blocks*512>=wanted,'Sparse reservation is not quota proof' + assert os.pread(fd,len(self.marker(owner)),0)==self.marker(owner) + if pending.exists(): + save(self.root/'last_dense_write.json',dict(owner_id=owner['identity'], + fsynced_bytes=wanted,completed_unix=time.time())) + pending.unlink() + finally:os.close(fd) + return wanted + + def prepare(self): + with self.lock(): + self.output.parent.mkdir(parents=True,exist_ok=True) + if not self.owner.exists(): + assert not self.blob.exists() + fd=os.open(self.blob,os.O_RDWR|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) + s=os.fstat(fd) + owner=dict(output=str(self.output),reservation=str(self.root),budget_bytes=self.budget, + identity=uuid.uuid4().hex,inode=s.st_ino,device=s.st_dev,created_unix=time.time()) + save(self.owner,owner);os.pwrite(fd,self.marker(owner),0);os.fsync(fd);os.close(fd) + owner=self._identity();used=allocated(self.output) + assert used==0,'Initial reservation may not claim an already-started output' + size=self._resize_dense(self.budget,initial=True) + proof=dict(status='PASS_ACTUAL_DENSE_CAPACITY',owner=owner,output_bytes=used, + reserve_bytes=size,checked_unix=time.time()) + save(self.root/'PREPARED.json',proof);return proof + + def retire(self,path): + """Keep already allocated obsolete bytes as this task's reserve, with no copy. + + Only after a newer checkpoint is durable may pruning hand its old state + here. The bytes cease being a resumable checkpoint and are consumed before + the blank reserve. This avoids rewriting tens of GiB of zeros after pruning. + """ + path=Path(path) + with self.lock(): + owner=self._identity() + assert path.is_relative_to(self.output) and path!=self.output + assert path.is_dir() and not path.is_symlink() and path.resolve()==path + size=allocated(path);assert path.stat().st_dev==self.blob.stat().st_dev + self.retired.mkdir(exist_ok=True) + target=self.retired/(path.name+'-'+uuid.uuid4().hex) + journal=self.root/('retire-'+target.name+'.json') + save(journal,dict(status='INTENT_OWN_CAPACITY_RETIREMENT',owner_id=owner['identity'], + source=str(path),destination=str(target),allocated_bytes=size,unix=time.time())) + path.rename(target) + # A retired state must never advertise exact resume readiness. + for complete in target.rglob('COMPLETE.json'): + complete.rename(complete.with_name('RETIRED_COMPLETE_PROVENANCE.json')) + save(journal,dict(status='COMPLETE_OWN_CAPACITY_RETIREMENT',owner_id=owner['identity'], + source=str(path),destination=str(target),allocated_bytes=size,unix=time.time())) + + def _shrink_retired(self,target_bytes): + held=allocated(self.retired) + if held<=target_bytes:return held + for p in sorted(self.retired.rglob('*')): + if not p.is_file():continue + assert not p.is_symlink() and p.resolve().is_relative_to(self.retired) + s=p.stat();assert s.st_nlink==1 + if held-s.st_blocks*512>=target_bytes: + p.unlink();held-=s.st_blocks*512 + else: + # Truncate an owned, explicitly retired file to release only the + # required blocks. It is no longer a resumable checkpoint. + release=((held-target_bytes+4095)//4096)*4096 + with p.open('r+b') as f: + f.truncate(max(0,s.st_size-release));f.flush();os.fsync(f.fileno()) + held=allocated(self.retired) + if held<=target_bytes:break + assert held<=target_bytes + return held + + def rebalance(self,allowance=0,phase='AFTER_WRITE'): + with self.lock(): + owner=self._identity() + assert json.loads((self.root/'PREPARED.json').read_text())['status']=='PASS_ACTUAL_DENSE_CAPACITY' + used=allocated(self.output) + assert used+allowance+40960,str(p) + with p.open('rb') as f:os.fsync(f.fileno()) + return dict(status='PASS_CLOSED_NATIVE_STATE_FILES_FSYNC_AND_ZIP_STRUCTURE', + world_size=world_size,allocated_bytes=allocated(folder),checked_unix=time.time()) + +def rank0_call(accelerator,function): + import torch.distributed as dist + result=[None] + if accelerator.is_main_process: + try:function() + except Exception as e: + result[0]=repr(e);print('[storage-budget-fatal] '+result[0],flush=True) + if dist.is_initialized():dist.broadcast_object_list(result,src=0,device=accelerator.device) + if result[0] is not None:raise RuntimeError('Owned storage operation failed: '+result[0]) + +if __name__=='__main__': + import argparse + p=argparse.ArgumentParser();p.add_argument('--config',required=True);a=p.parse_args() + config=json.loads(Path(a.config).read_text()) + print(json.dumps(Budget.for_config(config).prepare(),indent=2),flush=True) diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/submit.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/submit.py new file mode 100644 index 0000000000000000000000000000000000000000..ca192d97a0fafa87662e9020731bbe3fa44b6f91 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/submit.py @@ -0,0 +1,36 @@ +"""Idempotent two-node / sixteen-H200 job submission.""" +from pathlib import Path +import fcntl,json,re,subprocess,time,shutil,requests +from common import ROOT,load_config,read,write,frozen_inputs +from xbigdata.fuyao.console.constant import const +from xbigdata.fuyao.console.utils.helper import get_api_host,get_session_id +SITE='fuyao_sh_n2';QUEUE='rc-wm-h200';LABEL='sdv3-new67-clean40-normfit-right2x-30ep-16h200-r2-20261114' +def api(method,path,**kwargs): + r=requests.request(method,get_api_host(SITE)+path,headers={const.API_KEY_HEADER:get_session_id(),const.CLUSTER_HEADER:SITE},timeout=(10,45),**kwargs);r.raise_for_status();return r.json() +def actual(job): + data=api('POST','/api/2.0/runs/search',json={'run_view_type':'ALL','max_results':10,'filter':f"attr.site='{SITE}' AND fuyao.job_name='{job}'"});rows=[r for r in data.get('runs',[]) if r.get('fuyao_job',{}).get('job_name')==job];assert len(rows)==1 + rid=rows[0]['info']['run_uuid'];j=api('GET','/api/2.0/runs/get',params={'run_uuid':rid})['run']['fuyao_job'];return dict(run_id=rid,**{k:j.get(k) for k in ['job_name','label','state','partition','gpu_type','priority','gpus_per_node','node_count','node_list']}) +def duplicate_check(): + query={'run_view_type':'ACTIVE_ONLY','max_results':100,'filter':f"attr.site='{SITE}' AND attribute.user_id='shouqx@xiaopeng.com'",'order_by':['fuyao.submitted_at DESC']};rows=[] + for i in range(30): + data=api('POST','/api/2.0/runs/search',json=query);rows.extend(data.get('runs',[]));token=data.get('next_page_token') + if not token:break + query['page_token']=token + else:raise RuntimeError('Incomplete owner job listing') + matches=[r['fuyao_job']['job_name'] for r in rows if r.get('fuyao_job',{}).get('label')==LABEL];assert not matches,'Equivalent active job: '+repr(matches) +def main(): + reports=ROOT/'reports' + with (reports/'submission.lock').open('a') as f: + fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB) + if (reports/'submission.json').exists():print(json.dumps(actual(read(reports/'submission.json')['job_name'])));return + if (reports/'deploy_intent.json').exists():raise RuntimeError('Reconcile existing deployment intent before retry') + c=load_config(ROOT/'configs/stack_cups.json');proof=read(reports/'cpu_preflight_PASS.json');inputs=frozen_inputs();assert proof['status']=='PASS' and proof['tested_files']==inputs and read(reports/'submission_inputs.json')==inputs + duplicate_check();assert shutil.disk_usage(ROOT).free>=240*2**30 + workspace=ROOT/'submission_workspace';(workspace/'launch.sh').write_text('#!/usr/bin/env bash\nset -euo pipefail\nexport PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1\nexec python3 '+str(ROOT/'code/cluster_entry.py')+'\n') + command=['fuyao','deploy','--docker-image','infra-registry-vpc.cn-wulanchabu.cr.aliyuncs.com/data-infra/fuyao:shouqx-260618-0659','--project','rc-wm','--experiment','shouqx_exp','--site',SITE,'--queue',QUEUE,'--gpu-type','H200','--nodes','2','--gpus-per-node','8','--cpus-per-gpu','16','--time-limit','48h','--priority','normal','--label',LABEL,'--description','Fresh SpaceDreamer-v3 pretrained018126 fine-tune: ONLY40 clean trajectories from Nov03/04 new67, whole episodes with interior head capture gap >0.2s excluded. 37273 rows/41 segments,30epochs/4350updates,16H200=2x8,GBS256=16x16. Right arm6 joint loss2x, both grippers1x; original causal metric VDA/depth/point; action/state q01/q99 refitted on ONLY selected40 rows to avoid old81 range clipping. Exact selected-row sampler, no old81 data, no task checkpoint continuation. Final weights and native16-rank state retained.','--sdk','true','--disable-fault-tolerance','--ignore-artifact-size','--envs','NVIDIA_DRIVER_CAPABILITIES=all','-y','--','bash launch.sh'] + write(reports/'deploy_intent.json',dict(command=command,workspace=str(workspace),unix=time.time())) + with (reports/'deploy.log').open('x') as stream:r=subprocess.run(command,cwd=workspace,input='n\n',text=True,stdout=stream,stderr=subprocess.STDOUT,timeout=900) + names=sorted(set(re.findall(r'bifrost-[0-9a-z]+-shouqx',(reports/'deploy.log').read_text())));write(reports/'deploy_exit.json',dict(exit_code=r.returncode,jobs=names));assert r.returncode==0 and len(names)==1 + write(reports/'submission_handle.json',dict(job_name=names[0]));state=actual(names[0]);assert (state['partition'],state['gpu_type'],state['gpus_per_node'],state['node_count'],state['priority'].lower())==(QUEUE,'H200',8,2,'normal'),state + write(reports/'submission.json',dict(**state,epochs=30,total_steps=4350,episodes=40,frames=37273,output_dir=c['output_dir'],submitted_unix=time.time()));print(json.dumps(read(reports/'submission.json')),flush=True) +if __name__=='__main__':main() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/submit20.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/submit20.py new file mode 100644 index 0000000000000000000000000000000000000000..9630df216c29f755a336089ed9c01c7240741f00 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/submit20.py @@ -0,0 +1,139 @@ +"""Exactly-once Normal 8-H200 submission for the four independent real-task 20-epoch runs.""" +from pathlib import Path +import argparse +import fcntl +import json +import re +import subprocess +import time +import requests + +from common import ROOT, TASKS, frozen_inputs, load_config, read, write +from storage_budget import Budget +from xbigdata.fuyao.console.constant import const +from xbigdata.fuyao.console.utils.helper import get_api_host, get_session_id + +SITE = "fuyao_sh_n2" +LABEL_PREFIX = "sdv3-galexea-full20ep-8h200-20261108-" +QUEUE = "rc-wm-h200" +ACTIVE = {"JOB_RUNNING", "JOB_PENDING", "JOB_INITIALIZING", "JOB_RECEIVED", "JOB_SUSPENDED"} + + +def api(method, endpoint, **kwargs): + result = requests.request( + method, get_api_host(SITE) + endpoint, + headers={const.API_KEY_HEADER: get_session_id(), const.CLUSTER_HEADER: SITE}, + timeout=(10, 35), **kwargs, + ) + result.raise_for_status() + return result.json() + + +def actual(job): + result = api("POST", "/api/2.0/runs/search", json={ + "run_view_type": "ALL", "max_results": 10, + "filter": f"attr.site='{SITE}' AND fuyao.job_name='{job}'", + }) + rows = [r for r in result.get("runs", []) if r.get("fuyao_job", {}).get("job_name") == job] + assert len(rows) == 1 + rid = rows[0]["info"]["run_uuid"] + payload = api("GET", "/api/2.0/runs/get", params={"run_uuid": rid}) + j = payload["run"]["fuyao_job"] + assert j["job_name"] == job + return dict(run_id=rid, **{k: j.get(k) for k in ( + "job_name", "state", "priority", "partition", "gpu_type", + "gpus_per_node", "node_count", "node_list", "end_at", "label", + )}) + + +def duplicate_check(task): + query = { + "run_view_type": "ACTIVE_ONLY", "max_results": 100, + "filter": f"attr.site='{SITE}' AND attribute.user_id='shouqx@xiaopeng.com'", + "order_by": ["fuyao.submitted_at DESC"], + } + rows = [] + for _ in range(20): + data = api("POST", "/api/2.0/runs/search", json=query) + rows.extend(data.get("runs", [])) + page = data.get("next_page_token") + if not page: + break + query["page_token"] = page + else: + raise RuntimeError("Active-job pagination did not finish") + matches = [r["fuyao_job"]["job_name"] for r in rows + if r.get("fuyao_job", {}).get("state") in ACTIVE + and task.replace("_", "-") in r["fuyao_job"].get("label", "") + and "20ep" in r["fuyao_job"].get("label", "")] + assert not matches, "Equivalent active training exists: " + repr(matches) + return dict(owner_runs_observed=len(rows), equivalent_active=matches) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--task', choices=TASKS, required=True) + args = parser.parse_args() + task = args.task + REPORTS = ROOT / 'reports' / task + LABEL = LABEL_PREFIX + task.replace('_', '-') + REPORTS.mkdir(exist_ok=True) + with (REPORTS / "submission.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + receipt = REPORTS / "submission.json" + if receipt.exists(): + print(json.dumps(actual(read(receipt)["job_name"])), flush=True) + return + if (REPORTS / "submission_handle.json").exists() or (REPORTS / "deploy_intent.json").exists(): + raise RuntimeError("Reconcile the retained deployment handle/intent; do not deploy twice") + config = load_config(ROOT / "configs" / (task + ".json")) + inputs = frozen_inputs() + proof = read(ROOT / "reports/cpu_preflight_PASS.json") + assert proof["status"] == "PASS" and proof["tested_files"] == inputs + assert proof["tasks"] == [dict(task=t, **{k: load_config(ROOT/'configs'/(t+'.json'))[k] + for k in ('frames', 'segments', 'source_episodes')}, + optimizer_steps=load_config(ROOT/'configs'/(t+'.json'))['total_steps']) for t in TASKS] + duplicate = duplicate_check(task) + capacity = Budget.for_config(config).rebalance(phase="BEFORE_GPU_SUBMISSION") + write(ROOT / "reports/submission_inputs.json", inputs) + workspace = ROOT / "submission_workspace" / task + workspace.mkdir(exist_ok=True) + (workspace / "launch.sh").write_text( + "#!/usr/bin/env bash\nset -euo pipefail\n" + "export PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1\n" + "exec python3 " + str(ROOT / "code/cluster_entry.py") + " --task " + task + "\n" + ) + command = [ + "fuyao", "deploy", "--docker-image", + "infra-registry-vpc.cn-wulanchabu.cr.aliyuncs.com/data-infra/fuyao:shouqx-260618-0659", + "--project", "rc-wm", "--experiment", "shouqx_exp", "--site", SITE, + "--queue", QUEUE, "--gpu-type", "H200", "--nodes", "1", + "--gpus-per-node", "8", "--cpus-per-gpu", "16", + "--time-limit", "48h", "--priority", "normal", "--label", LABEL, + "--description", + f"SpaceDreamer v3 {task}: full {config['source_episodes']} episodes/{config['segments']} segments/{config['frames']} rows; fresh fine-tune pretrained018126,20epochs/{config['total_steps']}updates,GBS256=8x32,20Hz,33RGB-depth-XYZframes/32absolute_joint_actions. Same seed0,optimizer10k schedule,normalizers and augmentations as50ep reference. Own output and280GiB dense budget;all8 BF16/native train-save-reload gates. Keep final20ep weights AND full resume checkpoint; verify export before COMPLETE.", + "--sdk", "true", "--disable-fault-tolerance", "--ignore-artifact-size", + "--envs", "NVIDIA_DRIVER_CAPABILITIES=all", "-y", "--", "bash launch.sh", + ] + write(REPORTS / "deploy_intent.json", dict( + command=command, workspace=str(workspace), unix=time.time(), + duplicate_audit=duplicate, storage=capacity, automatic_retry=False, + )) + with (REPORTS / "deploy.log").open("x") as stream: + result = subprocess.run(command, cwd=workspace, input="n\n", text=True, + stdout=stream, stderr=subprocess.STDOUT, timeout=900) + text = (REPORTS / "deploy.log").read_text() + names = sorted(set(re.findall(r"bifrost-[0-9a-z]+-shouqx", text))) + write(REPORTS / "deploy_exit.json", dict(returncode=result.returncode, candidate_jobs=names)) + assert result.returncode == 0 and len(names) == 1 + write(REPORTS / "submission_handle.json", dict(job_name=names[0], unix=time.time())) + row = actual(names[0]) + assert row["partition"] == QUEUE and row["gpus_per_node"] == 8 and row["node_count"] == 1 + assert row["gpu_type"] == "H200" and row["priority"].lower() == "normal" + write(receipt, dict(**row, epochs=20, total_steps=config["total_steps"], task=task, + output_dir=config["output_dir"], submitted_unix=time.time())) + print(json.dumps(read(receipt)), flush=True) + + +if __name__ == "__main__": + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/train.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/train.py new file mode 100644 index 0000000000000000000000000000000000000000..5c431ffdc774bd0086cc040d7d24faf47a1d5140 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/train.py @@ -0,0 +1,260 @@ +"""Independent 20-epoch real-task fine-tune, using native four-output flow matching.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import time + +import torch +from torch.utils.data import DataLoader + +from common import ROOT, add_backbone, fingerprint, load_config, lr_factor, read, write +from data import ExactBatchSampler +from selection_data import SelectedDataset as GalexeaDataset +from model import build_model, encode_batch, loss +from output_guard import initialize_output + + +def resume_probe(optimizer, scheduler, device): + digest = lambda tensor: hashlib.sha256(tensor.detach().cpu().contiguous().numpy().tobytes()).hexdigest() + inner = getattr(optimizer, 'optimizer', optimizer) + masters = getattr(inner, 'single_partition_of_fp32_groups', []) + probes = [] + for tensor in masters: + flat = tensor.detach().reshape(-1) + probes.append(dict(size=flat.numel(), sha256=digest(torch.cat((flat[:16], flat[-16:])).float()))) + return dict(lr=optimizer.param_groups[0]['lr'], + scheduler_epoch=getattr(scheduler, 'scheduler', scheduler).last_epoch, + cpu_rng_sha256=digest(torch.random.get_rng_state()), + cuda_rng_sha256=digest(torch.cuda.get_rng_state(device)), fp32_master_probes=probes) + + +class RepeatedRealSmokeDataset(torch.utils.data.Dataset): + """One real aligned window repeated only for the preflight engine test.""" + def __init__(self, path, config): + payload = torch.load(path, map_location='cpu', weights_only=True) + if payload['source_fingerprint'] != config['source_fingerprint']: + raise ValueError('Preflight sample has different reference data') + self.sample = payload['sample'] + self.size = config['frames'] + for key in ('rgb', 'depth', 'point'): + if self.sample[key].shape != (3, 33, 256, 256): + raise ValueError('Preflight sample video shape mismatch') + if self.sample['action'].shape != (32, 14): + raise ValueError('Preflight sample action shape mismatch') + + def __len__(self): + return self.size + + def __getitem__(self, row): + return self.sample + + +def prune_states(root, keep, retire=None): + checkpoints = sorted(p.parent for p in root.glob('step_*/COMPLETE.json')) + for folder in checkpoints[:-keep]: + if folder.parent.resolve() != root.resolve() or not folder.name.startswith('step_'): + raise RuntimeError('Unexpected checkpoint deletion target') + if retire is None: + shutil.rmtree(folder) + else: + retire(folder) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--config', type=Path, required=True) + parser.add_argument('--resume', type=Path) + parser.add_argument('--smoke-steps', type=int) + parser.add_argument('--smoke-output', type=Path) + parser.add_argument('--smoke-sample', type=Path) + parser.add_argument('--verify-base-sha256', action='store_true') + args = parser.parse_args() + c = load_config(args.config) + if args.smoke_steps is not None and (args.smoke_steps <= 0 or args.smoke_output is None): + parser.error('A smoke run requires positive --smoke-steps and separate --smoke-output') + if args.smoke_sample and not args.smoke_steps: + parser.error('Repeated preflight samples are forbidden for formal training') + out = args.smoke_output if args.smoke_steps else Path(c['output_dir']) + if out.resolve() == Path(c['pretrained_checkpoint']).parent.resolve(): + raise ValueError('Output overlaps pretrained weights') + if args.verify_base_sha256 and int(os.environ.get('RANK', '0')) == 0: + from common import sha + if sha(c['pretrained_checkpoint']) != c['pretrained_sha256']: + raise ValueError('Pretrained weight hash mismatch') + add_backbone() + from accelerate.utils import set_seed + from distributed import make_accelerator + from fastwam.models.wan22.schedulers.scheduler_continuous import WanContinuousFlowMatchScheduler + from train_joint_rgb_depth import load_vae + from vae_runtime import configure_vae + set_seed(c['seed']) + # MoT calls transformer sub-blocks directly, so block-level FSDP wrappers + # cannot safely gather their parameters. Use the reference jobs' ZeRO-2. + accelerator = make_accelerator(c) + if accelerator.num_processes != c['world_size']: + raise ValueError(f"Expected {c['world_size']} training ranks, got {accelerator.num_processes}") + if accelerator.device.type != 'cuda': + raise RuntimeError('Full training requires CUDA') + config_identity = fingerprint(c) + initialize_output(accelerator, out, dict(config_fingerprint=config_identity, + job=os.environ.get('BIFROST_JOB_NAME'), time=time.time()), resume=args.resume) + if accelerator.is_main_process: + print('[startup] output directory claimed; building dataset and pretrained model', flush=True) + dataset = RepeatedRealSmokeDataset(args.smoke_sample, c) if args.smoke_sample else GalexeaDataset(c, training=True) + model, audit = build_model(c) + torch.set_num_threads(2) + torch.backends.cuda.matmul.allow_tf32 = True + optimizer = torch.optim.AdamW(model.parameters(), lr=c['learning_rate'], betas=tuple(c['betas']), + eps=c['eps'], weight_decay=c['weight_decay']) + scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_factor) + # Loader is deliberately NOT prepared: ExactBatchSampler already partitions ranks. + model, optimizer, scheduler = accelerator.prepare(model, optimizer, scheduler) + if accelerator.is_main_process: + print('[startup] native distributed engine ready; loading frozen VAE', flush=True) + vae = configure_vae(load_vae(Path(os.environ.get('SD_GALEXEA_LOCAL_MODEL_DIR', c['pretrained_model_dir'])), + torch.bfloat16, accelerator.device), c) + video_scheduler = WanContinuousFlowMatchScheduler(1000, c['video_shift']) + action_scheduler = WanContinuousFlowMatchScheduler(1000, c['action_shift']) + start_epoch = start_step = global_step = 0 + if args.resume: + state = read(args.resume / 'COMPLETE.json') + if state['config_fingerprint'] != config_identity: + raise ValueError('Resume configuration changed') + accelerator.load_state(str(args.resume)) + start_epoch, start_step, global_step = state['next_epoch'], state['next_step'], state['global_step'] + probe_path = args.resume / f'probe_rank_{accelerator.process_index}.json' + actual = resume_probe(optimizer, scheduler, accelerator.device) + if not probe_path.exists() or read(probe_path) != actual: + raise RuntimeError(f'Optimizer/scheduler/RNG restore mismatch on rank {accelerator.process_index}') + write(out / f'resume_rank_{accelerator.process_index}.json', dict(status='PASS_EXACT_RESTORE_PROBE', + rank=accelerator.process_index, restored_step=global_step, probe=actual)) + if accelerator.is_main_process: + out.mkdir(parents=True, exist_ok=True) + write(out / 'config.json', c) + write(out / 'initialization.json', audit) + shutil.copyfile(c['normalizer_override'], out / 'reference_norm_stats.json') + print('[startup] VAE ready; starting optimizer steps', flush=True) + accelerator.wait_for_everyone() + limit = min(c['total_steps'], global_step + args.smoke_steps) if args.smoke_steps else c['total_steps'] + from storage_budget import Budget, CHECKPOINT_ALLOWANCE, FINAL_ALLOWANCE, checkpoint_durable, rank0_call + storage = Budget.for_config(c) if os.environ.get('SD_GALEXEA_OWNED_STORAGE_BUDGET') == '1' else None + if storage is not None: + rank0_call(accelerator, lambda: storage.rebalance(phase='BEFORE_OPTIMIZER_STEPS')) + began = time.monotonic() + optimizer.zero_grad(set_to_none=True) + + def save_state(next_epoch, next_step): + folder = out / 'states' / f'step_{global_step:08d}' + if storage is not None: + rank0_call(accelerator, lambda: storage.rebalance(CHECKPOINT_ALLOWANCE, phase='BEFORE_CHECKPOINT')) + accelerator.save_state(str(folder), safe_serialization=True) + write(folder / f'probe_rank_{accelerator.process_index}.json', resume_probe(optimizer, scheduler, accelerator.device)) + accelerator.wait_for_everyone() + def finish_checkpoint(): + proof = checkpoint_durable(folder, accelerator.num_processes) + if proof['allocated_bytes'] > CHECKPOINT_ALLOWANCE: + raise RuntimeError('Native state exceeds its measured storage allowance') + write(folder / 'COMPLETE.json', dict(config_fingerprint=config_identity, + global_step=global_step, next_epoch=next_epoch, next_step=next_step, + durability=proof)) + with (folder / 'COMPLETE.json').open('rb') as stream: + os.fsync(stream.fileno()) + prune_states(out / 'states', 1 if args.smoke_steps else c['checkpoint_keep'], + retire=storage.retire if storage is not None else None) + if storage is not None: + storage.rebalance(phase='AFTER_DURABLE_CHECKPOINT_AND_PRUNE') + rank0_call(accelerator, finish_checkpoint) + accelerator.wait_for_everyone() + + last_position = (start_epoch, start_step) + for epoch in range(start_epoch, c['epochs']): + dataset.epoch = epoch + resume_step = start_step if epoch == start_epoch else 0 + sampler = ExactBatchSampler(len(dataset), accelerator.process_index, accelerator.num_processes, + c['micro_batch'], c['accumulation'], epoch, resume_step, c['seed']) + loader = DataLoader(dataset, batch_sampler=sampler, num_workers=c['workers'], pin_memory=True, + persistent_workers=False, **({'prefetch_factor': 1} if c['workers'] else {})) + epoch_step = resume_step + sums = {} + for batch in loader: + with accelerator.accumulate(model): + latents = encode_batch(vae, batch, accelerator.device, c) + with accelerator.autocast(): + total, metrics = loss(model, batch, latents, video_scheduler, action_scheduler, c, accelerator.device) + if not torch.isfinite(total).all(): + raise FloatingPointError('Nonfinite four-output training loss') + accelerator.backward(total) + from joint_rgb_depth_point_action_fastwam_diffusion import output_gradient_evidence + point_grad = output_gradient_evidence().get('point_prediction', 0.) + sums['point_prediction_grad_norm'] = sums.get('point_prediction_grad_norm', 0.) + torch.tensor( + point_grad / c['accumulation'], device=accelerator.device) + for key, value in metrics.items(): + sums[key] = sums.get(key, 0.) + value.detach().float() / c['accumulation'] + if accelerator.sync_gradients: + grad = accelerator.clip_grad_norm_(model.parameters(), c['grad_clip']) + if not torch.isfinite(torch.as_tensor(grad)).all(): + raise FloatingPointError('Nonfinite training gradients') + sums['grad_norm'] = torch.as_tensor(grad, device=accelerator.device).detach().float() + optimizer.step() + optimizer.zero_grad(set_to_none=True) + if accelerator.sync_gradients: + scheduler.step() + global_step += 1 + epoch_step += 1 + actual_lr = optimizer.param_groups[0]['lr'] + expected_lr = c['learning_rate'] * lr_factor(global_step) + if abs(actual_lr - expected_lr) > 1e-12: + raise RuntimeError('Actual optimizer LR differs from reference schedule') + metrics_mean = {k: float(accelerator.reduce(v, reduction='mean')) for k, v in sums.items()} + if metrics_mean['point_prediction_grad_norm'] <= 0 or metrics_mean['grad_norm'] <= 0: + raise RuntimeError('Point prediction or model has no training gradient') + sums = {} + next_epoch, next_step = (epoch + 1, 0) if epoch_step == c['steps_per_epoch'] else (epoch, epoch_step) + last_position = (next_epoch, next_step) + if accelerator.is_main_process: + record = dict(step=global_step, epoch=epoch + epoch_step / c['steps_per_epoch'], + lr=actual_lr, seconds=time.monotonic() - began, **metrics_mean) + with (out / 'metrics.jsonl').open('a') as stream: + stream.write(json.dumps(record) + '\n') + if global_step % 10 == 0 or global_step <= 3 or args.smoke_steps: + print(record, flush=True) + if global_step % c['checkpoint_every'] == 0: + save_state(*last_position) + if global_step >= limit: + break + if global_step >= limit: + break + if global_step % c['checkpoint_every']: + save_state(*last_position) + if global_step != limit: + raise RuntimeError(f'Incomplete training: {global_step} != {limit}') + if not args.smoke_steps: + # All ranks participate in full-state gathering; only rank zero writes. + full_state = accelerator.get_state_dict(model) + if storage is not None: + rank0_call(accelerator, lambda: storage.rebalance(FINAL_ALLOWANCE, phase='BEFORE_FINAL_EXPORT')) + def finish_weights(): + path = out / 'final_weights.pt' + temporary = path.with_suffix('.partial') + torch.save(dict(state_dict=full_state, config=c, initialization=audit), temporary) + with temporary.open('rb') as stream: + os.fsync(stream.fileno()) + temporary.replace(path) + write(out / 'COMPLETE.json', dict(epochs=c['epochs'], global_step=global_step, weights=str(path), + action_horizon=32, video_frames=33, config_fingerprint=config_identity)) + if storage is not None: + storage.rebalance(phase='AFTER_FINAL_EXPORT') + rank0_call(accelerator, finish_weights) + elif accelerator.is_main_process: + write(out / 'SMOKE_COMPLETE.json', dict(optimizer_steps=args.smoke_steps, global_step=global_step, + formal_training=False)) + accelerator.wait_for_everyone() + + +if __name__ == '__main__': + main() diff --git a/spacedreamer_stack_cups_clean40_30ep/training_reference/code/vae_runtime.py b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/vae_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..4021bfad8948296a4210853b5b2937ddf3a2fe26 --- /dev/null +++ b/spacedreamer_stack_cups_clean40_30ep/training_reference/code/vae_runtime.py @@ -0,0 +1,20 @@ +"""Frozen VAE tensor-core layout; independent clips share only the batch axis.""" +import torch + + +def configure_vae(vae, config): + if config['vae_conv_memory_format'] != 'channels_last_3d': + raise ValueError('Expected the validated frozen-VAE convolution layout') + if not 1 <= config['vae_encode_batch'] <= config['micro_batch']: + raise ValueError('Invalid independent VAE encode batch') + # Conv3d in NCDHW was the measured bottleneck. Values and dtype are preserved; + # NDHWC weights select the efficient cuDNN BF16 convolution implementation. + torch.nn.utils.convert_conv3d_weight_memory_format(vae, torch.channels_last_3d) + return vae + + +@torch.no_grad() +def encode_video(vae, pixels, device, batch_size): + # single_encode resets the causal feature cache for each batch. VAE layers + # normalize within each sample, so independent windows never share history. + return torch.cat([vae.single_encode(part, device=device) for part in pixels.split(batch_size)], dim=0)