qshou-coder commited on
Commit
3092a4e
·
verified ·
1 Parent(s): 64a7a5f

Add final SpaceDreamer stack_cups clean40 30-epoch checkpoint and inference package

Browse files

Final step4350 from bifrost-2026111502493001-shouqx. Original unchanged final_weights.pt, exact selected40 FP32 statistics, matching frozen source and inference adapter. The optimizer completed30epochs; platform failure was traced to a postflight BF16/FP32 comparison false positive. Changes confined to spacedreamer_stack_cups_clean40_30ep.

This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. spacedreamer_stack_cups_clean40_30ep/CPU_validation.json +23 -0
  2. spacedreamer_stack_cups_clean40_30ep/README.md +100 -0
  3. spacedreamer_stack_cups_clean40_30ep/checkpoint_metadata.json +18 -0
  4. spacedreamer_stack_cups_clean40_30ep/final_weights.pt +3 -0
  5. spacedreamer_stack_cups_clean40_30ep/initialization.json +35 -0
  6. spacedreamer_stack_cups_clean40_30ep/manifest.json +928 -0
  7. spacedreamer_stack_cups_clean40_30ep/postflight_normalizer_audit.json +64 -0
  8. spacedreamer_stack_cups_clean40_30ep/reference_norm_stats.json +136 -0
  9. spacedreamer_stack_cups_clean40_30ep/runtime/requirements.txt +22 -0
  10. spacedreamer_stack_cups_clean40_30ep/runtime/run_inference.py +29 -0
  11. spacedreamer_stack_cups_clean40_30ep/runtime/runtime_packages.json +161 -0
  12. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/__init__.py +1 -0
  13. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/__init__.py +0 -0
  14. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/dataset_utils.py +109 -0
  15. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/base_lerobot_dataset.py +412 -0
  16. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/constants.py +54 -0
  17. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py +179 -0
  18. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py +903 -0
  19. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py +527 -0
  20. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py +1304 -0
  21. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/base_processor.py +263 -0
  22. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/fastwam_processor.py +304 -0
  23. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/robot_video_dataset.py +279 -0
  24. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/action_state_merger.py +78 -0
  25. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/image.py +25 -0
  26. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/misc.py +20 -0
  27. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/relative_action.py +104 -0
  28. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/rotation.py +133 -0
  29. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/normalizer.py +291 -0
  30. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/rotation.py +621 -0
  31. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/__init__.py +1 -0
  32. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/__init__.py +0 -0
  33. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/action_dit.py +337 -0
  34. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/attention_masks.py +82 -0
  35. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam.py +1122 -0
  36. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_idm.py +452 -0
  37. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_joint.py +229 -0
  38. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/__init__.py +15 -0
  39. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/gradient.py +25 -0
  40. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/io.py +207 -0
  41. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/loader.py +221 -0
  42. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/state_dict_converters.py +90 -0
  43. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/mot.py +563 -0
  44. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/schedulers/scheduler_continuous.py +88 -0
  45. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan22.py +405 -0
  46. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_dit.py +673 -0
  47. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_text_encoder.py +342 -0
  48. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_vae.py +1384 -0
  49. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/runtime.py +436 -0
  50. spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/trainer.py +791 -0
spacedreamer_stack_cups_clean40_30ep/CPU_validation.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "PASS_CPU_STRICT_LOAD_EXACT_SELECTED40_NORMALIZATION_PROMPT_AND_FROZEN_SOURCE",
3
+ "checkpoint_step": 4350,
4
+ "epochs": 30,
5
+ "state_dict_keys": 1661,
6
+ "strict_load_missing_keys": [],
7
+ "strict_load_unexpected_keys": [],
8
+ "all_parameters_materialized": true,
9
+ "selected40_FP32_statistics_file_identical": true,
10
+ "training_and_adapter_normalize_decode_bitwise_equal": true,
11
+ "all4_quantized_buffers_correct": true,
12
+ "context_shape": [
13
+ 128,
14
+ 4096
15
+ ],
16
+ "context_prompt_matches": true,
17
+ "shared_dependencies_at_revision": "64a7a5fad88d69ed041b00d233fc1d4374e4bf8e",
18
+ "shared_dependency_sha256": {
19
+ "spacedreamer_20epochs/assets/Wan2.2-TI2V-5B/Wan2.2_VAE.pth": "20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36",
20
+ "spacedreamer_20epochs/assets/metric_video_depth_anything_vitb.pth": "f6f58576b9680a112f2428d4f39aff92656d3ae85745b0164675ace8d5b1fade"
21
+ },
22
+ "scope": "CPU checkpoint reconstruction and packaging; no new GPU sampling or robot success claimed."
23
+ }
spacedreamer_stack_cups_clean40_30ep/README.md ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SpaceDreamer stack_cups — clean40,30 epochs
2
+
3
+ Final original `final_weights.pt` from **bifrost-2026111502493001-shouqx**: **30 epochs,4350 optimizer steps**,trained on the approved40 demonstrations from the new67 November3/4 stack_cups recordings (41 continuous segments,37273 unique rows). Fresh initialization from SpaceDreamer-v3 pretrained018126;16H200,global batch256. Right-arm6joint loss weight2;left joints and both grippers1. This is the new selected40-refit-normalization run.
4
+
5
+ The model is **RgbGeometryActionMoT**,joint RGB/Depth/dense-point-map/Action prediction. It predicts **32×14 absolute joint/gripper commands**,not the RGB-only FastWAM40-step model. All final model parameters are contained in this weight file; distributed optimizer/RNG training states are retained on the original training storage and are not part of this inference release.
6
+
7
+ ## Files
8
+
9
+ - `final_weights.pt`: unchanged final training export (`state_dict`,`config`,`initialization`).
10
+ - `training_config.json`: original training config; historical absolute dataset/output paths are provenance and are not needed by the inference adapter.
11
+ - `reference_norm_stats.json`: **exact FP32 statistics refit on selected40 only**. Use these for state normalization and action decoding.
12
+ - `text_context.pt`: exact cached128-token training instruction,`Stack the cups.`
13
+ - `checkpoint_metadata.json`: training step,weight size and SHA-256.
14
+ - `runtime/spacedreamer_galexea.py`: inference adapter exposing `infer_action()` through the original joint sampler.
15
+ - `runtime/run_inference.py`: saved-observation prediction CLI,with no robot I/O.
16
+ - `runtime/source/`: matching frozen training source;`training_reference/code/`: exact training recipe.
17
+ - `postflight_normalizer_audit.json`: independent explanation of the platform failure after training.
18
+ - `CPU_validation.json`,`manifest.json`: packaging checks and exact file digests.
19
+
20
+ ## Download this model and shared dependencies
21
+
22
+ VAE,VDA and optional text encoder already reside in this repository under `spacedreamer_20epochs/assets`. They are reused rather than duplicated in this new folder. The standard cached instruction needs VAE and,if depth is not provided,VDA. UMT5/tokenizer are needed only for a different instruction.
23
+
24
+ ```python
25
+ from huggingface_hub import HfApi,snapshot_download
26
+ repo="qshou-coder/SpaceDreamer_Galexea"
27
+ prefix="spacedreamer_stack_cups_clean40_30ep"
28
+ shared="spacedreamer_20epochs/assets"
29
+ revision=HfApi().model_info(repo).sha # save/pin this revision
30
+ snapshot_download(
31
+ repo_id=repo,revision=revision,local_dir="SpaceDreamer_Galexea",
32
+ allow_patterns=[
33
+ f"{prefix}/**",
34
+ f"{shared}/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
35
+ f"{shared}/metric_video_depth_anything_vitb.pth",
36
+ f"{shared}/calibrations.json",
37
+ ],
38
+ )
39
+ # For custom instructions,also download the existing shared UMT5 weights/tokenizer.
40
+ ```
41
+
42
+ Known shared hashes:
43
+
44
+ - VAE: `20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36`.
45
+ - Metric VDA: `f6f58576b9680a112f2428d4f39aff92656d3ae85745b0164675ace8d5b1fade`.
46
+
47
+ The calibration manifest is historical training-camera metadata. No training datasets are required for prediction. Deployment camera geometry must correspond to the trained image/depth convention.
48
+
49
+ ## Runtime and inference
50
+
51
+ `runtime/runtime_packages.json` records observed package versions. The checked runtime uses Python3.12,PyTorch2.9.0 and torchvision0.24.0. Install the CUDA build appropriate to your machine and the dependencies in `runtime/requirements.txt` in an isolated environment. The checkpoint uses BF16. No Wan diffusion backbone checkpoint is needed beyond this trained export and its VAE.
52
+
53
+ From the downloaded repository root:
54
+
55
+ ```bash
56
+ export PYTHONPATH="$PWD/spacedreamer_stack_cups_clean40_30ep/runtime${PYTHONPATH:+:$PYTHONPATH}"
57
+ python spacedreamer_stack_cups_clean40_30ep/runtime/run_inference.py \
58
+ --task-dir spacedreamer_stack_cups_clean40_30ep \
59
+ --assets-dir spacedreamer_20epochs/assets \
60
+ --observation observation.npz \
61
+ --output predicted_actions.npy \
62
+ --steps 10 --seed 0
63
+ ```
64
+
65
+ `observation.npz` contains your own `head_rgb` RGB,uint8 image and `state` float14;optionally `depth_m`. No training-camera images are published in this folder.
66
+
67
+ ```python
68
+ from spacedreamer_galexea import SpaceDreamerGalaxea
69
+ policy=SpaceDreamerGalaxea(
70
+ "spacedreamer_stack_cups_clean40_30ep",
71
+ "spacedreamer_20epochs/assets",
72
+ device="cuda:0",
73
+ )
74
+ policy.reset_episode() # once at the start of a new episode
75
+ commands=policy.infer_action(
76
+ head_rgb,state,depth_m=depth_m,
77
+ num_inference_steps=10,seed=None,
78
+ )
79
+ assert commands.shape==(32,14)
80
+ ```
81
+
82
+ Ten sampling steps is the adapter's default,not a claim that it is optimal for this new checkpoint. If `depth_m` is omitted,the adapter advances causal metric VDA using the head image. Feed successive camera frames through `update_depth` to maintain its temporal history when replanning less often;reset the stream once per episode. The adapter performs the original joint multimodal sampling and returns physical actions;it does not implement a new action-only algorithm or execute robot commands.
83
+
84
+ ## Exact input/output contract
85
+
86
+ - One **head-left RGB view**,not FastWAM's concatenated three-camera layout.
87
+ - Accepted image: raw RGB,uint8 **512×640**,or the exact training letterboxed **224×224** image. Arbitrary square resizing of a raw camera frame is not equivalent. The adapter applies the matching letterbox and model resize to256×256.
88
+ - `depth_m`,if supplied: finite metric z-depth,float224×224,aligned with that letterboxed RGB. The adapter applies the training float16 cache quantization,nearest resize and0.05–5m validity convention.
89
+ - State order: left_arm6,left_gripper,right_arm6,right_gripper. Joint values in radians;grippers keep original continuous raw units approximately0–100.
90
+ - State normalization and output decoding use the external **reference_norm_stats.json in FP32**,including native clipping;do not substitute the old81/148-demo statistics or rounded checkpoint buffer values.
91
+ - Returned array is float32 **32×14**,already denormalized **absolute original motion_target.position**. Do not add current state or inverse-normalize again. At the20Hz training rate,32 commands span approximately1.6 seconds. Execution prefix/replanning frequency is controlled separately.
92
+ - Current RGB/depth frame0 condition the model;point frame0 and future RGB/depth/point/action latents are sampled jointly.
93
+
94
+ ## Training completion and postflight issue
95
+
96
+ All4350 optimizer steps completed;all logged loss/gradient values were finite. The final100-step mean action loss was0.0086685. The platform then marked the job JOB_FAILED because an export checker required bitwise equality between BF16-saved normalizer buffers converted back toFP32 and the original FP32 statistics.
97
+
98
+ The independent audit found all four stored buffers **exactly equal to the correct selected40 statistics cast to BF16**,and the external inference statistics file byte-identical to the training normalizer. The failure is a postflight comparison false positive,not missing training steps or a switched normalizer. The weight file is uploaded unchanged. The inference adapter explicitly uses the original FP32 JSON for normalization/decoding.
99
+
100
+ Packaging/CPU model reconstruction checks are documented in CPU_validation.json. This upload does not claim new GPU rollout acceptance or a real-robot success rate for this checkpoint.
spacedreamer_stack_cups_clean40_30ep/checkpoint_metadata.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": "stack_cups",
3
+ "training_job": "bifrost-2026111502493001-shouqx",
4
+ "epochs": 30,
5
+ "global_step": 4350,
6
+ "checkpoint": "final_weights.pt",
7
+ "bytes": 11030029192,
8
+ "sha256": "b9c898a58d03a4b863d3ec9cc5b3b9c87e7ac00427a27478e61a17cbf269a6e7",
9
+ "action_dim": 14,
10
+ "action_horizon": 32,
11
+ "video_frames": 33,
12
+ "action_source": "absolute_original_motion_target_position",
13
+ "joint_order": "left_arm6,left_gripper,right_arm6,right_gripper",
14
+ "initialization": "Fresh fine-tune from original pretrained018126 on approved40 of new67 demos; selected40-refit normalization; right6joint loss2x,grippers1x.",
15
+ "optimizer_training_complete": true,
16
+ "platform_state": "JOB_FAILED",
17
+ "platform_failure_reason": "Postflight BF16-buffer versus FP32-statistics bitwise comparison false positive, independently audited; no missing optimizer steps."
18
+ }
spacedreamer_stack_cups_clean40_30ep/final_weights.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9c898a58d03a4b863d3ec9cc5b3b9c87e7ac00427a27478e61a17cbf269a6e7
3
+ size 11030029192
spacedreamer_stack_cups_clean40_30ep/initialization.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "loaded_backbone_tensors": 1649,
3
+ "fresh_interface_tensors": [
4
+ "action_normalizer.action_q01",
5
+ "action_normalizer.action_q99",
6
+ "action_normalizer.state_q01",
7
+ "action_normalizer.state_q99",
8
+ "action_state_encoder.0.bias",
9
+ "action_state_encoder.0.weight",
10
+ "action_state_encoder.2.bias",
11
+ "action_state_encoder.2.weight",
12
+ "mot.mixtures.action.action_encoder.bias",
13
+ "mot.mixtures.action.action_encoder.weight",
14
+ "mot.mixtures.action.head.bias",
15
+ "mot.mixtures.action.head.weight"
16
+ ],
17
+ "omitted_source_tensors": [
18
+ "action_normalizer.count",
19
+ "action_normalizer.fitted",
20
+ "action_normalizer.mean",
21
+ "action_normalizer.std",
22
+ "camera_encoder.bias",
23
+ "camera_encoder.weight",
24
+ "camera_head.bias",
25
+ "camera_head.weight",
26
+ "camera_normalizer.count",
27
+ "camera_normalizer.fitted",
28
+ "camera_normalizer.mean",
29
+ "camera_normalizer.std",
30
+ "structured_type_embedding"
31
+ ],
32
+ "pretrained_action_semantics": "absolute_ee_xyz_so3log_grip",
33
+ "target_action_semantics": "absolute_joint_position_grip",
34
+ "optimizer_restored": false
35
+ }
spacedreamer_stack_cups_clean40_30ep/manifest.json ADDED
@@ -0,0 +1,928 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "VALIDATED_ORIGINAL_FINAL30EP_RELEASE",
3
+ "training_job": "bifrost-2026111502493001-shouqx",
4
+ "step": 4350,
5
+ "epochs": 30,
6
+ "files": [
7
+ {
8
+ "path": "CPU_validation.json",
9
+ "bytes": 1008,
10
+ "sha256": "40f3bee5664f3afba5b173ea57168944b7f6c7362118cf311365906c3fafb0cb"
11
+ },
12
+ {
13
+ "path": "README.md",
14
+ "bytes": 7458,
15
+ "sha256": "f75d0fe3b967edfbc5956a70466836a93edeb8266d801fa71cbd0e48b348ed3b"
16
+ },
17
+ {
18
+ "path": "checkpoint_metadata.json",
19
+ "bytes": 856,
20
+ "sha256": "3709afdfb089858776b4c16d5430c53b6a31e31c4f6a419ef34591dceff750d3"
21
+ },
22
+ {
23
+ "path": "final_weights.pt",
24
+ "bytes": 11030029192,
25
+ "sha256": "b9c898a58d03a4b863d3ec9cc5b3b9c87e7ac00427a27478e61a17cbf269a6e7"
26
+ },
27
+ {
28
+ "path": "initialization.json",
29
+ "bytes": 1104,
30
+ "sha256": "e7d9654086d1758bc509f6b47877423cabc6314e3c32439443a54b2568a7d7a6"
31
+ },
32
+ {
33
+ "path": "postflight_normalizer_audit.json",
34
+ "bytes": 2771,
35
+ "sha256": "e02b02ca0501d9ab43a11baec6e7c0400e2ab27154722c510325d529f11fc954"
36
+ },
37
+ {
38
+ "path": "reference_norm_stats.json",
39
+ "bytes": 3360,
40
+ "sha256": "10f598cd5148e9ac85fa2b7b8c3b2f2c503127fcb307f33ad3e5277f8a9ff6f8"
41
+ },
42
+ {
43
+ "path": "runtime/requirements.txt",
44
+ "bytes": 456,
45
+ "sha256": "93ed58aca470a8fec133f87b19e479f28744f29c848a0e8e26cd74a729afa16b"
46
+ },
47
+ {
48
+ "path": "runtime/run_inference.py",
49
+ "bytes": 1427,
50
+ "sha256": "86c4c73bd13919001de9991b15883a2c7e6c4aff2f59a0c15aae51bb0b208e53"
51
+ },
52
+ {
53
+ "path": "runtime/runtime_packages.json",
54
+ "bytes": 4403,
55
+ "sha256": "54120c1011271310e5babd300edcbff2b4ccab01fb1a3f84ed8f28f54b0f94e3"
56
+ },
57
+ {
58
+ "path": "runtime/source/model_old/src/fastwam/__init__.py",
59
+ "bytes": 23,
60
+ "sha256": "c4149c5ff0f9abd1517360daf761e692feb8570eb7a0caa2dfde52d4412c3e23"
61
+ },
62
+ {
63
+ "path": "runtime/source/model_old/src/fastwam/datasets/__init__.py",
64
+ "bytes": 0,
65
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
66
+ },
67
+ {
68
+ "path": "runtime/source/model_old/src/fastwam/datasets/dataset_utils.py",
69
+ "bytes": 3735,
70
+ "sha256": "3bc17590049c7cb54061acf8d731b07892e7f530c5d7eeac2613fa77a285a392"
71
+ },
72
+ {
73
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/base_lerobot_dataset.py",
74
+ "bytes": 18557,
75
+ "sha256": "9a0690381f0ce0058bb0d526ea4fd4982939a4dbe293dabfc83d5abc086f0512"
76
+ },
77
+ {
78
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/constants.py",
79
+ "bytes": 1915,
80
+ "sha256": "11ac9260341e1560c5eb0dd7acb79ed63c74180b8ab4113c27622ff213db417f"
81
+ },
82
+ {
83
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py",
84
+ "bytes": 7067,
85
+ "sha256": "7d7daab219332b5abdfd5ec67ccaf3502e8eb583a17a4e482c3f3d4f7a691e89"
86
+ },
87
+ {
88
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py",
89
+ "bytes": 33439,
90
+ "sha256": "40f1b94e6ceb50765fb8dfa3d498887c2a0fc2eb49f09770c5f36190e7110c98"
91
+ },
92
+ {
93
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py",
94
+ "bytes": 19421,
95
+ "sha256": "cb647a1e955c3279e39ce8d7af7a37ea7432bbd71534c183f9ceebc8557dd89f"
96
+ },
97
+ {
98
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py",
99
+ "bytes": 55587,
100
+ "sha256": "6abe9eccf17077eee478805b574c25980cedf7a4f673a3dee1cb7184db9126f8"
101
+ },
102
+ {
103
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/processors/base_processor.py",
104
+ "bytes": 10811,
105
+ "sha256": "1f47bc85fee11e3d16714b2f1e3a2dc874f4609a1706a788617156d34b48b58e"
106
+ },
107
+ {
108
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/processors/fastwam_processor.py",
109
+ "bytes": 13316,
110
+ "sha256": "d71bd147c6594ee9d1a50c04d22fa69ccb914d601c4ad816233496f5b583fda5"
111
+ },
112
+ {
113
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/robot_video_dataset.py",
114
+ "bytes": 12358,
115
+ "sha256": "a30583e16418c2fd848e1588f42a28889fdc7b3b9dbaeccffbedbfe7fca0b081"
116
+ },
117
+ {
118
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/action_state_merger.py",
119
+ "bytes": 2522,
120
+ "sha256": "669d6991415fc5eca44b32b129d15fee1e06e2f17d59d90350c252598033b427"
121
+ },
122
+ {
123
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/image.py",
124
+ "bytes": 725,
125
+ "sha256": "4d7ab61d0844918525244db653831f0f632730046b2714a6163ff57c441a16fa"
126
+ },
127
+ {
128
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/misc.py",
129
+ "bytes": 422,
130
+ "sha256": "8b505bd9e3a54ab2db7efcd64f0929d2e0303f5db1123de1074fd01fa9887e2b"
131
+ },
132
+ {
133
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/relative_action.py",
134
+ "bytes": 4032,
135
+ "sha256": "e555e0558e948c4eea3d1119e274fa5d923af30862eec4380fb8528f4a319c47"
136
+ },
137
+ {
138
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/rotation.py",
139
+ "bytes": 5603,
140
+ "sha256": "0de34b252d60b956cda07c0c19baebbd3c8a28916face700c6664c5815e11e91"
141
+ },
142
+ {
143
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/utils/normalizer.py",
144
+ "bytes": 11274,
145
+ "sha256": "5ac89a646cd6395e98feaa832f8fdf7c7e00857080af3d9c15710e243c43ad43"
146
+ },
147
+ {
148
+ "path": "runtime/source/model_old/src/fastwam/datasets/lerobot/utils/rotation.py",
149
+ "bytes": 21110,
150
+ "sha256": "6169162f98faf02a3fe5201e2b8a1069fe299840d036ee7cad8af5fa8f561542"
151
+ },
152
+ {
153
+ "path": "runtime/source/model_old/src/fastwam/models/__init__.py",
154
+ "bytes": 35,
155
+ "sha256": "ae4c5733f4bbf7caf31df2ae6f789516f6a10cd84a321f7d0375a7034253ac6d"
156
+ },
157
+ {
158
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/__init__.py",
159
+ "bytes": 0,
160
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
161
+ },
162
+ {
163
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/action_dit.py",
164
+ "bytes": 13347,
165
+ "sha256": "1301d9224149de43bb701f620a5d41858ecc63c6b19a573ec32edd45a3bdb0a2"
166
+ },
167
+ {
168
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/attention_masks.py",
169
+ "bytes": 2846,
170
+ "sha256": "5c84fd14dfba35086c75fbd14244609d54e52f482a658ca4f02ddd63800bbb31"
171
+ },
172
+ {
173
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/fastwam.py",
174
+ "bytes": 48807,
175
+ "sha256": "36a61b652944a89dfd59dbba94ce206a664e40ed27868ca6d7bdbf0642e3bd0a"
176
+ },
177
+ {
178
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/fastwam_idm.py",
179
+ "bytes": 20427,
180
+ "sha256": "aae332d17645a6744aafd85edde8e8d7dacbb8b110de3988015ca330146c3c65"
181
+ },
182
+ {
183
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/fastwam_joint.py",
184
+ "bytes": 9795,
185
+ "sha256": "10937e5102839c8e6103aab828e3355fab6015299bdf66e303b056703dc79bb8"
186
+ },
187
+ {
188
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/__init__.py",
189
+ "bytes": 415,
190
+ "sha256": "f26116de229557a37a4812dc4ee75971f5f9e8b80240786ea4d0e5b0ff703cbf"
191
+ },
192
+ {
193
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/gradient.py",
194
+ "bytes": 565,
195
+ "sha256": "ba5d8f7272eb029dc6cd2849ca99b70f6ad5abb838d21c818beb0590620dc793"
196
+ },
197
+ {
198
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/io.py",
199
+ "bytes": 7751,
200
+ "sha256": "1c6c1bc8a975b47092912833194fce25e2e3ac886fbab72d73af3ff5a00b3c62"
201
+ },
202
+ {
203
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/loader.py",
204
+ "bytes": 8279,
205
+ "sha256": "792f66bd0609c1365256aa24d43301340f81260c1651d89b04d2ae8224a08726"
206
+ },
207
+ {
208
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/state_dict_converters.py",
209
+ "bytes": 5226,
210
+ "sha256": "81d213373e5056a8d8bf31768b9b04164138ea78d0b306e316334841499f4bf9"
211
+ },
212
+ {
213
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/mot.py",
214
+ "bytes": 22491,
215
+ "sha256": "fe0dce22413e18ff2e9cb249f7f9714f25d828b214f1a19e73feee029d29944b"
216
+ },
217
+ {
218
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/schedulers/scheduler_continuous.py",
219
+ "bytes": 4068,
220
+ "sha256": "2ececce284f7b9fe876965fbf5e11ca7b69caaf693979266b0f63062a589f181"
221
+ },
222
+ {
223
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/wan22.py",
224
+ "bytes": 16945,
225
+ "sha256": "b72998999b4b2c5096773092339b8ca81e9e7d96f398b81c57d3d22d14972ac4"
226
+ },
227
+ {
228
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/wan_video_dit.py",
229
+ "bytes": 31351,
230
+ "sha256": "c254ac7f2a43da59f103864140bb84474f5588290f5fb17a6ed426d0b64e02c9"
231
+ },
232
+ {
233
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/wan_video_text_encoder.py",
234
+ "bytes": 11459,
235
+ "sha256": "aedf7f131556a39b99a273deec3a85d67f41b5367cef41f310f33a807ae60a87"
236
+ },
237
+ {
238
+ "path": "runtime/source/model_old/src/fastwam/models/wan22/wan_video_vae.py",
239
+ "bytes": 50058,
240
+ "sha256": "31b3fb9f844a76f5db5af4854e14802aae640b85b61d0de03a4b53b52515abb7"
241
+ },
242
+ {
243
+ "path": "runtime/source/model_old/src/fastwam/runtime.py",
244
+ "bytes": 18262,
245
+ "sha256": "e19cd2abf94f3af9f65a231158406b3d61887136a603bd3a954335d3623319b0"
246
+ },
247
+ {
248
+ "path": "runtime/source/model_old/src/fastwam/trainer.py",
249
+ "bytes": 35849,
250
+ "sha256": "551a81bb01adcbbe9c1bc753d397b3e53df1b9d80cc28041cef56c3a1f32607c"
251
+ },
252
+ {
253
+ "path": "runtime/source/model_old/src/fastwam/utils/__init__.py",
254
+ "bytes": 96,
255
+ "sha256": "2893c05b99b17162941c70a41859c2808f8ae390be367a26cac7d5ab93ee197b"
256
+ },
257
+ {
258
+ "path": "runtime/source/model_old/src/fastwam/utils/config_resolvers.py",
259
+ "bytes": 2526,
260
+ "sha256": "01c48cbd42b86e636b70ccc3e193a81147162ba890777102b2c61086fa415340"
261
+ },
262
+ {
263
+ "path": "runtime/source/model_old/src/fastwam/utils/fs.py",
264
+ "bytes": 76,
265
+ "sha256": "69e6df236b43360428a613686c8f37c531c0d16a3a2c5b06a725c46b49d657c5"
266
+ },
267
+ {
268
+ "path": "runtime/source/model_old/src/fastwam/utils/logging_config.py",
269
+ "bytes": 4679,
270
+ "sha256": "cadc4f6ad487c3f37171a6defad36d23d9ab8bf262f73a461cd9ee9b03b757e7"
271
+ },
272
+ {
273
+ "path": "runtime/source/model_old/src/fastwam/utils/misc.py",
274
+ "bytes": 1223,
275
+ "sha256": "05aa51247936a8d3faac27f90b78b65ff83a688259bed55d114b6a617ced532d"
276
+ },
277
+ {
278
+ "path": "runtime/source/model_old/src/fastwam/utils/pytorch_utils.py",
279
+ "bytes": 5049,
280
+ "sha256": "ffc483e7fb435cd654326750eaffdbfc9243837c3140dcc74e45b24abb751a33"
281
+ },
282
+ {
283
+ "path": "runtime/source/model_old/src/fastwam/utils/samplers.py",
284
+ "bytes": 1339,
285
+ "sha256": "4d3bd9b2d498f4ad8f2cf9765d60c4bf5368fa6fa2a7d42d0db6210f66899652"
286
+ },
287
+ {
288
+ "path": "runtime/source/model_old/src/fastwam/utils/video_io.py",
289
+ "bytes": 819,
290
+ "sha256": "9e9898b021a44e87aa1b4e6d13e3755a9869a2e27a7eb53c72a6802808d5bd80"
291
+ },
292
+ {
293
+ "path": "runtime/source/model_old/src/fastwam/utils/video_metrics.py",
294
+ "bytes": 3446,
295
+ "sha256": "aaa37e458cc9e4bb79a17923b5ff5900c97b4309f7abd50db44cf76e9143ffd6"
296
+ },
297
+ {
298
+ "path": "runtime/source/multimodal/audit_robodojo_warmup_reproduction.py",
299
+ "bytes": 14938,
300
+ "sha256": "ab2d694fc3b00d3047296eac4a196ee7a30180fa7abed2547bf1f68b781a2405"
301
+ },
302
+ {
303
+ "path": "runtime/source/multimodal/build_piper_hdf5_index.py",
304
+ "bytes": 1287,
305
+ "sha256": "c6fa001c3363d4306e85a5eab2fe4e847f18d69c899b068259e8a35f090e1b9a"
306
+ },
307
+ {
308
+ "path": "runtime/source/multimodal/canonical_spacedreamer_data.py",
309
+ "bytes": 39499,
310
+ "sha256": "0b5f5c718c882f24012c3ae8a2070ffb53e386aea53a1ee60d8eb0a361906920"
311
+ },
312
+ {
313
+ "path": "runtime/source/multimodal/controlled_directional_mot.py",
314
+ "bytes": 6970,
315
+ "sha256": "8f1c53f5a7570ed251057274136a733faa4ef653b46a7728ad00539670ab4503"
316
+ },
317
+ {
318
+ "path": "runtime/source/multimodal/depth_preprocessing.py",
319
+ "bytes": 3254,
320
+ "sha256": "4fe296aee8e22ee56e488bf6fcbf6279e7f1ac385b286ae3b7f705d92940d482"
321
+ },
322
+ {
323
+ "path": "runtime/source/multimodal/evaluate_spacedreamer_validation.py",
324
+ "bytes": 2147,
325
+ "sha256": "ec76991435cbe39bec9e3ea506810cabbf466a129a9ad4544bed906b4b7fe550"
326
+ },
327
+ {
328
+ "path": "runtime/source/multimodal/finetune_galexea_rgb_action.py",
329
+ "bytes": 28110,
330
+ "sha256": "dc29a32aa0dc5afad4273f30e690016b3a65e0a385d4aa751b17e6db7c8d2d2a"
331
+ },
332
+ {
333
+ "path": "runtime/source/multimodal/galexea_mcap.py",
334
+ "bytes": 18359,
335
+ "sha256": "2f2e60c5a81bd44a9c9d0272af79ebb2b9dd8004b12fbddffb2a7c8d4e84ace1"
336
+ },
337
+ {
338
+ "path": "runtime/source/multimodal/joint_rgb_action_diffusion.py",
339
+ "bytes": 7935,
340
+ "sha256": "e71d7210e7bbf20e3807215e6b9feacc48f7c2f31b496e7d4e5a4af76f7323eb"
341
+ },
342
+ {
343
+ "path": "runtime/source/multimodal/joint_rgb_action_diffusion_shared_timestep.py",
344
+ "bytes": 5510,
345
+ "sha256": "cfcf934e2c77d4d69a6a7280e0dcf264a55a42e885b99612cb89bf721a35a96f"
346
+ },
347
+ {
348
+ "path": "runtime/source/multimodal/joint_rgb_depth_action_fastwam_diffusion.py",
349
+ "bytes": 4807,
350
+ "sha256": "8bc87f96c4ed0d6076f9c5be3367e290bb6863c3e9b9f5146539971c7b243541"
351
+ },
352
+ {
353
+ "path": "runtime/source/multimodal/joint_rgb_depth_point_action_diffusion.py",
354
+ "bytes": 10866,
355
+ "sha256": "0874a83ebfe1f4452fb8eedace7293d750fd1aa5814fd93ee6bd68c9e12e453b"
356
+ },
357
+ {
358
+ "path": "runtime/source/multimodal/joint_rgb_depth_point_action_fastwam_diffusion.py",
359
+ "bytes": 7018,
360
+ "sha256": "78d7d14c323178994391414e6cf7b102e97adddfb79266d4e66385a8e473d165"
361
+ },
362
+ {
363
+ "path": "runtime/source/multimodal/joint_rgb_geometry_action_diffusion.py",
364
+ "bytes": 9126,
365
+ "sha256": "e94cf04caac709aa806bec600e38db472dae896b122ae23be8d9d35ef24fad56"
366
+ },
367
+ {
368
+ "path": "runtime/source/multimodal/online_egoverse_action.py",
369
+ "bytes": 27776,
370
+ "sha256": "c484a0e11953ba78a154bb38c170ba890d8ef427a0ec515148312466ee966c16"
371
+ },
372
+ {
373
+ "path": "runtime/source/multimodal/optimizer_step_scheduler.py",
374
+ "bytes": 5408,
375
+ "sha256": "84b09ff87013725d120bf2cd5357a71752d3e406724b57fdffe054de262698c7"
376
+ },
377
+ {
378
+ "path": "runtime/source/multimodal/piper_hdf5.py",
379
+ "bytes": 23659,
380
+ "sha256": "544d6e39836339de5c69a22756df198f09cce52adcfdafab97e0d63e4e95d2a0"
381
+ },
382
+ {
383
+ "path": "runtime/source/multimodal/post_training_test.py",
384
+ "bytes": 29116,
385
+ "sha256": "5a41fd8f38ec3baf6a0e69619d074fffa8acb30aaeed22c26612d0576120cbb8"
386
+ },
387
+ {
388
+ "path": "runtime/source/multimodal/post_training_test_action_mot.py",
389
+ "bytes": 21288,
390
+ "sha256": "ba1718fddc337b11e9e7f7c4d7e68241cfe87db6cd5123cfcfd9ee7aa7df7cf1"
391
+ },
392
+ {
393
+ "path": "runtime/source/multimodal/post_training_test_rgb_depth_point.py",
394
+ "bytes": 22379,
395
+ "sha256": "418af7e62aa7ae6115538a01364c0ec5ee7647321c7bea6701e820637385445e"
396
+ },
397
+ {
398
+ "path": "runtime/source/multimodal/precompute_robodojo_exhaustive_rgb_latents.py",
399
+ "bytes": 14671,
400
+ "sha256": "943e83702c1c5eb1ee224d691d14c53aa87f8084c8824304aff91bbbf8d8d28a"
401
+ },
402
+ {
403
+ "path": "runtime/source/multimodal/precompute_robodojo_geometry_latents.py",
404
+ "bytes": 10570,
405
+ "sha256": "27b2c7f5c27b865921f4403ac68e861d2cd02e4f928bb1e96400eea675d0163b"
406
+ },
407
+ {
408
+ "path": "runtime/source/multimodal/precompute_robodojo_native_depth_latents_fastwam.py",
409
+ "bytes": 12125,
410
+ "sha256": "1596c08fd81ca38ef8d4dcc11dec36a7b9df17c57282b057571f44176b7e9ca5"
411
+ },
412
+ {
413
+ "path": "runtime/source/multimodal/precompute_robodojo_native_point_latents_fastwam.py",
414
+ "bytes": 14509,
415
+ "sha256": "ff70aa582dc179104058a451fc4796d0b502394f8b30fe9026e0bd31666bb42e"
416
+ },
417
+ {
418
+ "path": "runtime/source/multimodal/preflight_robodojo_ego_first_cam.py",
419
+ "bytes": 6302,
420
+ "sha256": "bae3d9a15a791cee000c85daca94c0c977895ff88be0da835c5f24361d615d53"
421
+ },
422
+ {
423
+ "path": "runtime/source/multimodal/prepare_point_cloud_cache.py",
424
+ "bytes": 7450,
425
+ "sha256": "b6c3bcacc0ae1d912ae1370525d841896b4f6acba51d2ca53e0cab1a561fddde"
426
+ },
427
+ {
428
+ "path": "runtime/source/multimodal/rgb_action_model_factory.py",
429
+ "bytes": 3791,
430
+ "sha256": "285364b62bf5f9ca488f003328150aacd95a7fd4a98e990965cfe2cad9f2965a"
431
+ },
432
+ {
433
+ "path": "runtime/source/multimodal/rgb_depth_point_action_model_factory.py",
434
+ "bytes": 7916,
435
+ "sha256": "1ac6d134c3f94f4b04604d6b9a35cc4a98bafff728361affc5b0c69211e71ba3"
436
+ },
437
+ {
438
+ "path": "runtime/source/multimodal/robodojo_ego_first_cam.py",
439
+ "bytes": 11763,
440
+ "sha256": "9543f5f29ea78db8c0763450c08fee8bd52a9fc1e9433826e161338b95b3a4f1"
441
+ },
442
+ {
443
+ "path": "runtime/source/multimodal/robodojo_geometry_data.py",
444
+ "bytes": 12794,
445
+ "sha256": "37221e841d324cc8713a142515591cd375c09ce01d7cb896be5f18fd75e5332e"
446
+ },
447
+ {
448
+ "path": "runtime/source/multimodal/robodojo_lerobot_v30.py",
449
+ "bytes": 17380,
450
+ "sha256": "4bfc2df11d6786bc78b5fa1452904f6cca66e1203b3a8e8524d81fca20ee9563"
451
+ },
452
+ {
453
+ "path": "runtime/source/multimodal/robodojo_native_depth_action_model.py",
454
+ "bytes": 15497,
455
+ "sha256": "21ac2f0d5aabcde94fc2961db747449fd4d08e27ad7067b47daf460a813be40e"
456
+ },
457
+ {
458
+ "path": "runtime/source/multimodal/robodojo_native_depth_data.py",
459
+ "bytes": 15063,
460
+ "sha256": "4e982175ceb1211f08386a339a973eb0d377c2a4ac1ff368738857a484820584"
461
+ },
462
+ {
463
+ "path": "runtime/source/multimodal/robodojo_native_depth_point_action_model.py",
464
+ "bytes": 18092,
465
+ "sha256": "eec08ebb5ada5a278916bf6f414fe82e5749d5c5159f7d2e1c576b476f76e6fe"
466
+ },
467
+ {
468
+ "path": "runtime/source/multimodal/robodojo_native_depth_point_data.py",
469
+ "bytes": 12559,
470
+ "sha256": "16895df58500abc5b9926b37a6f5face6f9d6e7d1f8fda15331efee165a5dc28"
471
+ },
472
+ {
473
+ "path": "runtime/source/multimodal/robodojo_rgb_geometry_action_model.py",
474
+ "bytes": 7859,
475
+ "sha256": "5a496d410f42f206542e5111d1ac38003b646b0f50e55dfb78a2ee3e50bc5bc0"
476
+ },
477
+ {
478
+ "path": "runtime/source/multimodal/robodojo_v2_checkpoint_contract.py",
479
+ "bytes": 6497,
480
+ "sha256": "f361f24c019c0e04989208457cc0b92f47dcb4567499902a1f62ac6af5afc89c"
481
+ },
482
+ {
483
+ "path": "runtime/source/multimodal/scripts/audit_piper_pretrained018126.py",
484
+ "bytes": 3894,
485
+ "sha256": "5011329e2f3c0c7962c6c31df7fe0a946042dfaded19a08b51209e32dbbd9ab2"
486
+ },
487
+ {
488
+ "path": "runtime/source/multimodal/scripts/audit_piper_pretrained018126_target_norm.py",
489
+ "bytes": 2207,
490
+ "sha256": "dd817495b883b408eb53a0f04ba298cc87f4084cbbdd8a0fa53582a46cbaf17b"
491
+ },
492
+ {
493
+ "path": "runtime/source/multimodal/scripts/audit_robodojo_full35_training.py",
494
+ "bytes": 4532,
495
+ "sha256": "1f8dd936fb44781533ed3db581b82017d27d99a3b2147f79300af9b8bf283864"
496
+ },
497
+ {
498
+ "path": "runtime/source/multimodal/scripts/audit_robodojo_full_normalizer_noaug_configs.py",
499
+ "bytes": 6127,
500
+ "sha256": "58a4bc39a9d59c1add35f44d8b3931117ba23be8f61bfb70163205a17d368917"
501
+ },
502
+ {
503
+ "path": "runtime/source/multimodal/scripts/audit_source_normalizer_noaug_configs.py",
504
+ "bytes": 6930,
505
+ "sha256": "cb72a0c6361b40340382625022ef93f978e3b1bcc9938a5a447861ab0c3a55b5"
506
+ },
507
+ {
508
+ "path": "runtime/source/multimodal/scripts/audit_spacedreamer_v3_robodojo_srs_resume5000.py",
509
+ "bytes": 2683,
510
+ "sha256": "9bf8dd49834503c92664df78d3a33e7b08244454d275595780550e69d719c53c"
511
+ },
512
+ {
513
+ "path": "runtime/source/multimodal/scripts/audit_spacedreamer_v3_robodojo_srs_trail1.py",
514
+ "bytes": 3388,
515
+ "sha256": "c96e68f282fe17da9ae1d84d21b11eecb2dc8e5fb80e50b028c6eaa6169df3c3"
516
+ },
517
+ {
518
+ "path": "runtime/source/multimodal/scripts/audit_target_normalizer_noaug_configs.py",
519
+ "bytes": 2826,
520
+ "sha256": "4b817fffe274c9e139ba79c1d07ec6ec50af4d92027cfe0a85c2fb39b9a47dc7"
521
+ },
522
+ {
523
+ "path": "runtime/source/multimodal/scripts/build_egoverse_canonical_motion_cache.py",
524
+ "bytes": 12355,
525
+ "sha256": "a34371ef4c32998343a19646977450d5cad51f2514a010cc382579fd6c49697d"
526
+ },
527
+ {
528
+ "path": "runtime/source/multimodal/scripts/build_spacedreamer_canonical_mix.py",
529
+ "bytes": 27136,
530
+ "sha256": "c8769ed42f9e36e62c734bfe960efcc57fe8657fc46af5852a9598ed067568b2"
531
+ },
532
+ {
533
+ "path": "runtime/source/multimodal/scripts/build_spacedreamer_indexed_canonical_manifest.py",
534
+ "bytes": 12295,
535
+ "sha256": "a06fbeda5afc37f141cf97bfc4686613ed11e3fd86db112a7d7e6ba6e58b6a76"
536
+ },
537
+ {
538
+ "path": "runtime/source/multimodal/scripts/evaluate_robodojo_train_action_generation.py",
539
+ "bytes": 23899,
540
+ "sha256": "cc950de9e1efae04e87af46600b8beee9742a2689267416cf929e718580b4ed4"
541
+ },
542
+ {
543
+ "path": "runtime/source/multimodal/scripts/merge_spacedreamer_canonical_parts.py",
544
+ "bytes": 6149,
545
+ "sha256": "d43aa61829542c19efe07950dc52802e6c3086706fbaa1e81dcb19f821c63bd1"
546
+ },
547
+ {
548
+ "path": "runtime/source/multimodal/scripts/prepare_robodojo_full35_base_cache.py",
549
+ "bytes": 1389,
550
+ "sha256": "a446bedc5b09d769761a87e572247c3df4fe7ade950d6ec8c3c3893e4496ae35"
551
+ },
552
+ {
553
+ "path": "runtime/source/multimodal/scripts/run_spacedreamer_canonical_full_build.py",
554
+ "bytes": 4433,
555
+ "sha256": "5fe85a4edeaec5572c7bac49c2541c58b86393e34db5b8d2b66aeba4841c6d28"
556
+ },
557
+ {
558
+ "path": "runtime/source/multimodal/scripts/smoke_robodojo_geometry_cached_dataset.py",
559
+ "bytes": 3709,
560
+ "sha256": "fba36fe8f2bf0d8802f5f4013867bd4a651771222daf07bfb42361aeeb32e909"
561
+ },
562
+ {
563
+ "path": "runtime/source/multimodal/scripts/validate_spacedreamer_canonical_mix.py",
564
+ "bytes": 12597,
565
+ "sha256": "1397bd66a8ff17f59eb2668aa115e34ace8c44b05782d7805994923f01e84e88"
566
+ },
567
+ {
568
+ "path": "runtime/source/multimodal/scripts/validate_spacedreamer_indexed_canonical.py",
569
+ "bytes": 17200,
570
+ "sha256": "1efcc093f42978254a1fbc6c26487261ff9bec25c38adcd8fff218af59d03ba4"
571
+ },
572
+ {
573
+ "path": "runtime/source/multimodal/scripts/verify_robodojo_geometry_experiment.py",
574
+ "bytes": 7763,
575
+ "sha256": "ac10edb1850a3229f52c7c472bde0584a9d212095dd37ea98c87454729c4fb11"
576
+ },
577
+ {
578
+ "path": "runtime/source/multimodal/scripts/visualize_depth_preprocessing.py",
579
+ "bytes": 6814,
580
+ "sha256": "9f20bbef948916866cdc2a2ac3a5010e549ba0787ed350b71435adc2aa66c28a"
581
+ },
582
+ {
583
+ "path": "runtime/source/multimodal/spacedreamer_ego_dataset.py",
584
+ "bytes": 38628,
585
+ "sha256": "633e1c7de3799c0d05329bfe8edf97b1dced4541ed6cec3cea1afd02dc80726c"
586
+ },
587
+ {
588
+ "path": "runtime/source/multimodal/train_galexea_rgb_state_joint_action_ablation.py",
589
+ "bytes": 28500,
590
+ "sha256": "ed1e3cef681348472320ac04a9bb7f7e1f26a203e25f2673cd40239187c84d3f"
591
+ },
592
+ {
593
+ "path": "runtime/source/multimodal/train_joint_rgb_depth.py",
594
+ "bytes": 67200,
595
+ "sha256": "53b1f71eda541f36499b7be813b805068aa22245693cb9fea0e6c302fa0bebd8"
596
+ },
597
+ {
598
+ "path": "runtime/source/multimodal/train_joint_rgb_depth_point.py",
599
+ "bytes": 40421,
600
+ "sha256": "012bcd542e7e43bc99b7606d76dc2af9a1816e9630246fcfa779b1d0c6a833fe"
601
+ },
602
+ {
603
+ "path": "runtime/source/multimodal/train_joint_rgb_depth_point_action.py",
604
+ "bytes": 35857,
605
+ "sha256": "ddbcabac68c212ad3aab2afd1e825ce3de3120be32b14ac791670b2d8b66bf0d"
606
+ },
607
+ {
608
+ "path": "runtime/source/multimodal/train_piper_rgb_state_joint_action_quantile_4way.py",
609
+ "bytes": 27604,
610
+ "sha256": "76353a68caeb954a3be02359a75c626072e93e617cb2c2b079363d72786ec0d9"
611
+ },
612
+ {
613
+ "path": "runtime/source/multimodal/train_robodojo_rgb_geometry_action_absolute_ee.py",
614
+ "bytes": 12293,
615
+ "sha256": "ced9745b4ab2bca6b3bea5508f433247106af98dc708cd08bfe7a9400bbc19ce"
616
+ },
617
+ {
618
+ "path": "runtime/source/multimodal/train_robodojo_rgb_native_depth_point_state_joint_action_fastwam.py",
619
+ "bytes": 13798,
620
+ "sha256": "9e2519a5dbe5a960a5cdedfa247630a6dfd28bef72fecfed0b4c4bbc3cbc18be"
621
+ },
622
+ {
623
+ "path": "runtime/source/multimodal/train_robodojo_rgb_native_depth_state_joint_action_fastwam.py",
624
+ "bytes": 17614,
625
+ "sha256": "32fddebea5accffc5be727e61d7f7c4865a75522c61d2293bca1b503ebe4dd7f"
626
+ },
627
+ {
628
+ "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_4way.py",
629
+ "bytes": 80135,
630
+ "sha256": "26b1ed4fc9c8c6b079a8b212aae78287a6b36f8bb35c3ed171d8bfa511eaed39"
631
+ },
632
+ {
633
+ "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_ablation.py",
634
+ "bytes": 42461,
635
+ "sha256": "485b33691060a00f85beae7e180ac11f30881982406f2698622e0ed91a93969e"
636
+ },
637
+ {
638
+ "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_4way.py",
639
+ "bytes": 7861,
640
+ "sha256": "e069cd1b6198e7c591ba179d5f26cf0b02bad98133cb5b8cf26ae558acc9caa0"
641
+ },
642
+ {
643
+ "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way.py",
644
+ "bytes": 10551,
645
+ "sha256": "465b849da066d562e12b43a31557f74eb3faac71669ef52f87ac90a185185b8d"
646
+ },
647
+ {
648
+ "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_quantile_shared_timestep_4way.py",
649
+ "bytes": 3857,
650
+ "sha256": "119231049ec988daf4fbaa4b4db7f8013d84fd47e23a8bf0596131b4e2911143"
651
+ },
652
+ {
653
+ "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_ego_first_cam_meanstd_4way.py",
654
+ "bytes": 26471,
655
+ "sha256": "600eca1cd377a5399f1659260a798e548f368f1788566d2a5467ca9d66ed168e"
656
+ },
657
+ {
658
+ "path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_episode_anchor_4way.py",
659
+ "bytes": 7408,
660
+ "sha256": "d90b516158b2781c3b913098c6d488b05db1e30714269330ffce6daf8b55fd6d"
661
+ },
662
+ {
663
+ "path": "runtime/source/multimodal/train_spacedreamer_ego.py",
664
+ "bytes": 68897,
665
+ "sha256": "c8f87e0a82f564a07dc58a09d4cb6220fc850d15654662307abf6cd55f8c0140"
666
+ },
667
+ {
668
+ "path": "runtime/source/multimodal/wan_rgb_action_mot.py",
669
+ "bytes": 10421,
670
+ "sha256": "a33437c1dbeed4dcaa6cf70e5505bd6ddc89a2324e11d81f5e18bff56ae115e7"
671
+ },
672
+ {
673
+ "path": "runtime/source/multimodal/wan_rgb_depth_initialization.py",
674
+ "bytes": 24878,
675
+ "sha256": "025d5c2cef3ddfced4ef0f3f4b25644cf68618e5e75be19f4eb20be40be79df3"
676
+ },
677
+ {
678
+ "path": "runtime/source/multimodal/wan_rgb_depth_point_action_mot.py",
679
+ "bytes": 17358,
680
+ "sha256": "238447d95a5b07d876e4ddf235077ac40cd8a945af8b27f4147811c771868557"
681
+ },
682
+ {
683
+ "path": "runtime/source/multimodal/wan_rgb_depth_point_initialization.py",
684
+ "bytes": 13728,
685
+ "sha256": "724f8ef02bb42754d2cc3fa3e2ed0e507e2a0451f96e37e2bb72a00da847ba7d"
686
+ },
687
+ {
688
+ "path": "runtime/source/multimodal/world_point_cloud.py",
689
+ "bytes": 25523,
690
+ "sha256": "3323ae5e1779126182e5a27ffcd39336dac5feb3d4db4f3ab572e5483272180b"
691
+ },
692
+ {
693
+ "path": "runtime/source/vda/utils/dc_utils.py",
694
+ "bytes": 3675,
695
+ "sha256": "45103ac573780e51ae54bbc3a010e391003c0e8bbd02746cb5f493d60d4e7907"
696
+ },
697
+ {
698
+ "path": "runtime/source/vda/utils/util.py",
699
+ "bytes": 2449,
700
+ "sha256": "f02cf85f5a9543ce0f832a4fde1492076026650c8eb25c6a8cc4315bf67a8ed2"
701
+ },
702
+ {
703
+ "path": "runtime/source/vda/video_depth_anything/dinov2.py",
704
+ "bytes": 15178,
705
+ "sha256": "d22df8d0da44c8525ebed898fd9c33b3acfcda919a84d577cc2a08bfd3d50ac1"
706
+ },
707
+ {
708
+ "path": "runtime/source/vda/video_depth_anything/dinov2_layers/__init__.py",
709
+ "bytes": 382,
710
+ "sha256": "e3c28c7b775165b6844000eb69f44d104d6c3803ac7da5d59f6ac3ecfb46d9d8"
711
+ },
712
+ {
713
+ "path": "runtime/source/vda/video_depth_anything/dinov2_layers/attention.py",
714
+ "bytes": 2343,
715
+ "sha256": "0d2410ce8b8b4499a6b0cdab939ae2bceab8971efd8660d999b0f3088f28cbf7"
716
+ },
717
+ {
718
+ "path": "runtime/source/vda/video_depth_anything/dinov2_layers/block.py",
719
+ "bytes": 9332,
720
+ "sha256": "093266ea08084c7cce7b8ddd779ecfaaa13055735faa8a7d68b5924f385b395b"
721
+ },
722
+ {
723
+ "path": "runtime/source/vda/video_depth_anything/dinov2_layers/drop_path.py",
724
+ "bytes": 1160,
725
+ "sha256": "949100aac4d9c19c9d65319dbd362886c5d46a72d7586ba4fd500e9f1d51570c"
726
+ },
727
+ {
728
+ "path": "runtime/source/vda/video_depth_anything/dinov2_layers/layer_scale.py",
729
+ "bytes": 823,
730
+ "sha256": "356817b5d1390737128cbcc03a5333fd3060b73beae478772282ccb851d0ef85"
731
+ },
732
+ {
733
+ "path": "runtime/source/vda/video_depth_anything/dinov2_layers/mlp.py",
734
+ "bytes": 1272,
735
+ "sha256": "c18c5b3f021408597a473bb710070ec3fb3b39f5a2e42e88ae83ced425a608b5"
736
+ },
737
+ {
738
+ "path": "runtime/source/vda/video_depth_anything/dinov2_layers/patch_embed.py",
739
+ "bytes": 2832,
740
+ "sha256": "b3f82e19926c9cfb63f10136aca5e8a723bde1ad62dae1d04f90d405f9439baf"
741
+ },
742
+ {
743
+ "path": "runtime/source/vda/video_depth_anything/dinov2_layers/swiglu_ffn.py",
744
+ "bytes": 1859,
745
+ "sha256": "df81d9901240214edc1831ce1b77558dcd6227b29a2f000fc5f546df55d2d09a"
746
+ },
747
+ {
748
+ "path": "runtime/source/vda/video_depth_anything/dpt.py",
749
+ "bytes": 5467,
750
+ "sha256": "4751fac16206892889d74fd7f564f275e0efa3a903172c9350a1b55ed850c7d8"
751
+ },
752
+ {
753
+ "path": "runtime/source/vda/video_depth_anything/dpt_temporal.py",
754
+ "bytes": 5909,
755
+ "sha256": "b80c448f4fe66d0ca7092be887c0ea3925a4ce7c03cffa0083480e7e2f55805d"
756
+ },
757
+ {
758
+ "path": "runtime/source/vda/video_depth_anything/motion_module/attention.py",
759
+ "bytes": 16719,
760
+ "sha256": "15e4049641f04f73027ea520ada3ba2c81196cea1339f0d6a6063d944df61c8b"
761
+ },
762
+ {
763
+ "path": "runtime/source/vda/video_depth_anything/motion_module/motion_module.py",
764
+ "bytes": 12744,
765
+ "sha256": "73d6a9cee40681532ae2603dd426405e840dc5c592a313005fa367f6fe4c69e8"
766
+ },
767
+ {
768
+ "path": "runtime/source/vda/video_depth_anything/util/blocks.py",
769
+ "bytes": 4064,
770
+ "sha256": "8a9fde84bddbfa2498b5f5ef06895ddc941c1fc12c5919e9891fad481997b7dd"
771
+ },
772
+ {
773
+ "path": "runtime/source/vda/video_depth_anything/util/transform.py",
774
+ "bytes": 6075,
775
+ "sha256": "a97df1dbb687457a40c387fdc3482edee7ddc5c142c7d2735ca6235a8ae2885f"
776
+ },
777
+ {
778
+ "path": "runtime/source/vda/video_depth_anything/video_depth.py",
779
+ "bytes": 6712,
780
+ "sha256": "208e882d60a41434b3a0d9935a7025b01cce0756c17b3d889c6db0c4c945eb29"
781
+ },
782
+ {
783
+ "path": "runtime/source/vda/video_depth_anything/video_depth_stream.py",
784
+ "bytes": 6833,
785
+ "sha256": "23ebab70525f318cf0bfeba38dfcac3f1c345ba50aaa1bb5c6dbae671b0caa6d"
786
+ },
787
+ {
788
+ "path": "runtime/spacedreamer_galexea.py",
789
+ "bytes": 14685,
790
+ "sha256": "d694e516d3f6baca2a49df8086c95b5438147091c9eb27bc684f16ffd319942f"
791
+ },
792
+ {
793
+ "path": "text_context.pt",
794
+ "bytes": 1050977,
795
+ "sha256": "49cf3918423e55ed12776ac91546bd4efb4142fd4645309b34b8526953a7cc92"
796
+ },
797
+ {
798
+ "path": "training_complete.json",
799
+ "bytes": 358,
800
+ "sha256": "92e46d8dd3370060bacf464880b0fcec4b712629c83e06b0203cfec1baccb07a"
801
+ },
802
+ {
803
+ "path": "training_config.json",
804
+ "bytes": 7131,
805
+ "sha256": "a94d0f7e603f5f905ee694c7bed920693bb0b32db77869f80878855fe8311718"
806
+ },
807
+ {
808
+ "path": "training_reference/code/audit.py",
809
+ "bytes": 1956,
810
+ "sha256": "110519fba757e41fb284c9d8853e416f4b9312fe8d33d7d79519fded2a41cc83"
811
+ },
812
+ {
813
+ "path": "training_reference/code/build_smoke_sample.py",
814
+ "bytes": 2493,
815
+ "sha256": "aa2976e044bab50de47268fff191dc73122308ce4d3188169a2cf06f046e6263"
816
+ },
817
+ {
818
+ "path": "training_reference/code/cache_calibrations.py",
819
+ "bytes": 2398,
820
+ "sha256": "65c51b36d239cbbfd4f9dea9d40280f445bcc276bb5164ea9a71806fb1f36e19"
821
+ },
822
+ {
823
+ "path": "training_reference/code/calibration.py",
824
+ "bytes": 3092,
825
+ "sha256": "39f4cfc0ab2a841a2e9c44f04af149aee9b2b52db4c42e6feab2b54b75d4c0ee"
826
+ },
827
+ {
828
+ "path": "training_reference/code/check_global_batch.py",
829
+ "bytes": 2872,
830
+ "sha256": "de59449caa64e50eef846768454895d4d904daf6339f797be80ea959fbd23e6a"
831
+ },
832
+ {
833
+ "path": "training_reference/code/cluster_entry.py",
834
+ "bytes": 7841,
835
+ "sha256": "d42963927ce7637ebd5ad2bee0102b595dff41b564aa318c81781de833d7cd98"
836
+ },
837
+ {
838
+ "path": "training_reference/code/common.py",
839
+ "bytes": 5682,
840
+ "sha256": "54bdfd54a3dca23bfa7073d8d85f011b5009bd81623f0dac7d5a8c0ca561134d"
841
+ },
842
+ {
843
+ "path": "training_reference/code/cpu_bootstrap.py",
844
+ "bytes": 315,
845
+ "sha256": "59ffd83c32036d4bd3a83b01f480058e51f3f5a04838500c85a197afe7d0ae50"
846
+ },
847
+ {
848
+ "path": "training_reference/code/cpu_preflight.py",
849
+ "bytes": 3879,
850
+ "sha256": "53455315a2aa2141293e111e3cbd67b5bc9b600e5df3c69938c03c791713a5b3"
851
+ },
852
+ {
853
+ "path": "training_reference/code/cuda_probe.py",
854
+ "bytes": 2547,
855
+ "sha256": "01949bf05a92ba781aa65f929d5a6bd7bb7972231524b89e3186ee18f14f12c2"
856
+ },
857
+ {
858
+ "path": "training_reference/code/data.py",
859
+ "bytes": 14912,
860
+ "sha256": "697b7dd046b054e2db0c2e61f8350b02b3a973458ac9670edb295f3903c156d6"
861
+ },
862
+ {
863
+ "path": "training_reference/code/decode_selected.py",
864
+ "bytes": 695,
865
+ "sha256": "6754d0f88170e5e87dd2ebe733c350c405debd3492801d71693ce31038bc15f8"
866
+ },
867
+ {
868
+ "path": "training_reference/code/distributed.py",
869
+ "bytes": 1052,
870
+ "sha256": "cdafe58a0101d1adb508b44d06f1dfbf4c52d8fc73ba66cbcdbf7fd3d0dfa843"
871
+ },
872
+ {
873
+ "path": "training_reference/code/model.py",
874
+ "bytes": 7629,
875
+ "sha256": "99bafc6873be3cca2d016df41c30c7741841701518256617038f36972969ef3e"
876
+ },
877
+ {
878
+ "path": "training_reference/code/output_guard.py",
879
+ "bytes": 1453,
880
+ "sha256": "e9c1c6192dc71c478c7d612d544e238d8995d5b45bbf38145f3dd66ae41d4969"
881
+ },
882
+ {
883
+ "path": "training_reference/code/prepare.py",
884
+ "bytes": 7614,
885
+ "sha256": "7e63fdae5a8b765a206c5c2196d507c96770fbdb73b9356cc1b9faa8438df088"
886
+ },
887
+ {
888
+ "path": "training_reference/code/selection_data.py",
889
+ "bytes": 1297,
890
+ "sha256": "175ed3df31c6e13ed2276078e888e8605485d1aff26282a861a6bbbe4c879be3"
891
+ },
892
+ {
893
+ "path": "training_reference/code/staging.py",
894
+ "bytes": 2327,
895
+ "sha256": "6d8f94c6e24db6373fd1129fdf7ee0387c03b9505b37c77f0c421b56c3cf7521"
896
+ },
897
+ {
898
+ "path": "training_reference/code/storage_budget.py",
899
+ "bytes": 12441,
900
+ "sha256": "e7ed733b8ccce3480d59f46c54293cf469b34269e2bc301203ebb71a00b8bb2c"
901
+ },
902
+ {
903
+ "path": "training_reference/code/submit.py",
904
+ "bytes": 4656,
905
+ "sha256": "70dd48052e9b5b8ecea25605811bba0f9e20d441e28b664fb3af860f787910c2"
906
+ },
907
+ {
908
+ "path": "training_reference/code/submit20.py",
909
+ "bytes": 6964,
910
+ "sha256": "41763dfa7b49b466462fca56c2bf4d27dea2ee06c6287c47bfbebaf12ed4ca4a"
911
+ },
912
+ {
913
+ "path": "training_reference/code/train.py",
914
+ "bytes": 14828,
915
+ "sha256": "28a320e6fa8a737ce71e880d5d1103246b1e054975c0db9723ed9a0170ed0788"
916
+ },
917
+ {
918
+ "path": "training_reference/code/vae_runtime.py",
919
+ "bytes": 1005,
920
+ "sha256": "ab129de4f7d5aff6271176b6305d00ba78931b4510595a41d0f2c28cd86192fe"
921
+ }
922
+ ],
923
+ "total_bytes": 11033164188,
924
+ "shared_dependencies": {
925
+ "spacedreamer_20epochs/assets/Wan2.2-TI2V-5B/Wan2.2_VAE.pth": "20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36",
926
+ "spacedreamer_20epochs/assets/metric_video_depth_anything_vitb.pth": "f6f58576b9680a112f2428d4f39aff92656d3ae85745b0164675ace8d5b1fade"
927
+ }
928
+ }
spacedreamer_stack_cups_clean40_30ep/postflight_normalizer_audit.json ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "TRAINING_COMPLETE_POSTFLIGHT_BF16_COMPARISON_FALSE_POSITIVE_CONFIRMED",
3
+ "platform_state": "JOB_FAILED",
4
+ "job": "bifrost-2026111502493001-shouqx",
5
+ "epochs": 30,
6
+ "optimizer_steps": 4350,
7
+ "rows_of_training_metrics": 4350,
8
+ "all_loss_and_grad_metrics_finite": true,
9
+ "last_metrics": {
10
+ "step": 4350,
11
+ "epoch": 30.0,
12
+ "lr": 3.4098125841150466e-05,
13
+ "seconds": 38622.264301493764,
14
+ "point_prediction_grad_norm": 0.00035875628236681223,
15
+ "loss_rgb": 0.045066364109516144,
16
+ "loss_depth": 0.03409406542778015,
17
+ "loss_point": 0.043846286833286285,
18
+ "loss_action": 0.008644891902804375,
19
+ "video_timestep_mean": 800.71875,
20
+ "action_timestep_mean": 461.4375,
21
+ "prediction_rgb_norm": 1707.3424072265625,
22
+ "prediction_depth_norm": 2257.48876953125,
23
+ "prediction_point_norm": 1777.875732421875,
24
+ "prediction_action_norm": 107.14266967773438,
25
+ "grad_norm": 0.3622361421585083
26
+ },
27
+ "last100_mean_loss_action": 0.008668523519299925,
28
+ "weights": "/workspace/group_share/rc-wbc/shouqx@xiaopeng.com/real_robot_training/stack_cups_new67_clean40_refitnorm_30ep_16h200_r2_20261114/runs/stack_cups/final_weights.pt",
29
+ "weights_bytes": 11030029192,
30
+ "normalizer_sha256": "10f598cd5148e9ac85fa2b7b8c3b2f2c503127fcb307f33ad3e5277f8a9ff6f8",
31
+ "checks": [
32
+ {
33
+ "buffer": "state_q01",
34
+ "dtype": "torch.bfloat16",
35
+ "matches_exact_expected_after_dtype_conversion": true,
36
+ "max_joint_rounding_rad": 0.0037899017333984375,
37
+ "max_gripper_rounding_raw": 0.023340225219726562
38
+ },
39
+ {
40
+ "buffer": "state_q99",
41
+ "dtype": "torch.bfloat16",
42
+ "matches_exact_expected_after_dtype_conversion": true,
43
+ "max_joint_rounding_rad": 0.005093097686767578,
44
+ "max_gripper_rounding_raw": 0.19965362548828125
45
+ },
46
+ {
47
+ "buffer": "action_q01",
48
+ "dtype": "torch.bfloat16",
49
+ "matches_exact_expected_after_dtype_conversion": true,
50
+ "max_joint_rounding_rad": 0.0017759203910827637,
51
+ "max_gripper_rounding_raw": 0.0
52
+ },
53
+ {
54
+ "buffer": "action_q99",
55
+ "dtype": "torch.bfloat16",
56
+ "matches_exact_expected_after_dtype_conversion": true,
57
+ "max_joint_rounding_rad": 0.00553131103515625,
58
+ "max_gripper_rounding_raw": 0.19965362548828125
59
+ }
60
+ ],
61
+ "root_cause": "Export verifier converted quantized BF16 buffers back to FP32 and required bitwise equality to original FP32 statistics. All4 buffers exactly match correctly quantized selected40 statistics. Data pipeline normalizes from the original JSON, not these stored buffers.",
62
+ "inference_note": "Use accompanying reference_norm_stats.json for exact FP32 normalization. No new GPU inference acceptance is claimed by this metadata/buffer audit.",
63
+ "checked_utc": "2026-11-15T06:13:04Z"
64
+ }
spacedreamer_stack_cups_clean40_30ep/reference_norm_stats.json ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "norm_stats": {
3
+ "state": {
4
+ "mean": [
5
+ -0.13387035604186223,
6
+ 0.9388066769062815,
7
+ -0.299460975359783,
8
+ -0.312899377843042,
9
+ 0.022158233386334263,
10
+ -0.013364365686538105,
11
+ 87.67583665493922,
12
+ 0.12845319557789786,
13
+ 1.0475177045852482,
14
+ -0.3682720079020069,
15
+ -0.4035744942019698,
16
+ 0.09048382540451419,
17
+ 0.05871849529736292,
18
+ 88.37490834410208
19
+ ],
20
+ "std": [
21
+ 0.23867227111243974,
22
+ 1.0532982660446137,
23
+ 0.4497836674554094,
24
+ 0.521777337868213,
25
+ 0.0756900378886052,
26
+ 0.06476722031329171,
27
+ 26.71132088139677,
28
+ 0.1772370447709506,
29
+ 1.0844598923871354,
30
+ 0.49809485962894984,
31
+ 0.5703715013499049,
32
+ 0.1773866395459158,
33
+ 0.07316617681887644,
34
+ 23.618158731215697
35
+ ],
36
+ "q01": [
37
+ -0.7902127504348755,
38
+ 0.21595744788646698,
39
+ -1.4180850982666016,
40
+ -1.5263829231262207,
41
+ -0.26163404583930966,
42
+ -0.22297872602939606,
43
+ 12.665559768676758,
44
+ -0.04957446828484535,
45
+ 0.19914893805980682,
46
+ -1.4697531938552857,
47
+ -1.5238298177719116,
48
+ -0.19659574329853058,
49
+ -0.07021276652812958,
50
+ 18.101660156250002
51
+ ],
52
+ "q99": [
53
+ 0.0274468082934618,
54
+ 2.8855319023132324,
55
+ -0.004680851008743048,
56
+ 0.15954042673110888,
57
+ 0.2176595777273178,
58
+ 0.07531914860010147,
59
+ 100.07527160644531,
60
+ 0.5876595973968506,
61
+ 2.826382875442505,
62
+ -0.003617021255195141,
63
+ 0.0663829818367958,
64
+ 0.8282978534698486,
65
+ 0.2704255282878876,
66
+ 101.19965362548828
67
+ ]
68
+ },
69
+ "actions": {
70
+ "mean": [
71
+ -0.14292097344549323,
72
+ 0.7355806430838492,
73
+ -0.29974345955300896,
74
+ -0.3301010056178284,
75
+ -0.0026388765341016007,
76
+ -0.03810476927142731,
77
+ 87.42233316834316,
78
+ 0.11110324934059856,
79
+ 0.8414215713411128,
80
+ -0.3732627181630525,
81
+ -0.42123973672921317,
82
+ 0.06530478815310313,
83
+ 0.03338203608640254,
84
+ 83.2737688514841
85
+ ],
86
+ "std": [
87
+ 0.2332689717409583,
88
+ 1.0724814487217418,
89
+ 0.4633568221251692,
90
+ 0.5291241592842331,
91
+ 0.07604966254227942,
92
+ 0.06502434652450503,
93
+ 32.40528197209235,
94
+ 0.17417586584814845,
95
+ 1.10393937977348,
96
+ 0.514695628729752,
97
+ 0.5803439554075464,
98
+ 0.17785820538290803,
99
+ 0.07406519902851658,
100
+ 36.22980289874895
101
+ ],
102
+ "q01": [
103
+ -0.7869321703910828,
104
+ -0.0,
105
+ -1.4373400211334229,
106
+ -1.5707999467849731,
107
+ -0.2991262674331665,
108
+ -0.24697090685367584,
109
+ 0.0,
110
+ -0.07516505569219589,
111
+ -0.0,
112
+ -1.4986991882324219,
113
+ -1.5707999467849731,
114
+ -0.22396118938922882,
115
+ -0.09664078801870346,
116
+ 0.0
117
+ ],
118
+ "q99": [
119
+ 0.012271846644580364,
120
+ 2.723245506286619,
121
+ 0.0,
122
+ 0.15800002217292786,
123
+ 0.19174760580062866,
124
+ 0.05368932709097862,
125
+ 100.0,
126
+ 0.5599029660224915,
127
+ 2.6507186889648438,
128
+ 0.0,
129
+ 0.06442718952894211,
130
+ 0.8053399324417114,
131
+ 0.24697090685367584,
132
+ 101.19965362548828
133
+ ]
134
+ }
135
+ }
136
+ }
spacedreamer_stack_cups_clean40_30ep/runtime/requirements.txt ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # CPU packaging validation versions; choose the matching PyTorch CUDA wheel index.
2
+ torch==2.9.0+cu128
3
+ torchvision==0.24.0+cu128
4
+ numpy==1.26.4
5
+ opencv-python-headless==4.11.0.86
6
+ Pillow==12.2.0
7
+ accelerate==1.13.0
8
+ einops==0.8.1
9
+ safetensors==0.7.0
10
+ transformers==4.57.3
11
+ ftfy==6.3.1
12
+ sentencepiece==0.2.1
13
+ PyYAML==6.0.3
14
+ h5py==3.14.0
15
+ zarr==2.18.7
16
+ numcodecs==0.15.1
17
+ fasteners==0.20
18
+ asciitree==0.3.3
19
+ zstandard==0.25.0
20
+ lz4==4.4.5
21
+ easydict==1.13
22
+ huggingface-hub==0.36.2
spacedreamer_stack_cups_clean40_30ep/runtime/run_inference.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Predict and save an action chunk from one NPZ observation; no robot I/O."""
2
+ from pathlib import Path
3
+ import argparse
4
+ import numpy as np
5
+ from spacedreamer_galexea import SpaceDreamerGalaxea
6
+
7
+ def main():
8
+ p = argparse.ArgumentParser(description=__doc__)
9
+ p.add_argument('--task-dir', type=Path, required=True)
10
+ p.add_argument('--assets-dir', type=Path)
11
+ p.add_argument('--observation', type=Path, required=True,
12
+ help='NPZ containing head_rgb uint8[512,640,3] or letterboxed[224,224,3], state[14], optional depth_m[224,224]')
13
+ p.add_argument('--output', type=Path, required=True)
14
+ p.add_argument('--steps', type=int, default=10)
15
+ p.add_argument('--seed', type=int, default=0)
16
+ p.add_argument('--device', default='cuda:0')
17
+ a = p.parse_args()
18
+ with np.load(a.observation, allow_pickle=False) as obs:
19
+ rgb, state = obs['head_rgb'], obs['state']
20
+ depth = obs['depth_m'] if 'depth_m' in obs else None
21
+ policy = SpaceDreamerGalaxea(a.task_dir, a.assets_dir, device=a.device)
22
+ actions = policy.infer_action(rgb, state, depth_m=depth,
23
+ num_inference_steps=a.steps, seed=a.seed)
24
+ a.output.parent.mkdir(parents=True, exist_ok=True)
25
+ np.save(a.output, actions, allow_pickle=False)
26
+ print(f'Saved {actions.shape} {actions.dtype} absolute commands to {a.output}; no robot commands sent.')
27
+
28
+ if __name__ == '__main__':
29
+ main()
spacedreamer_stack_cups_clean40_30ep/runtime/runtime_packages.json ADDED
@@ -0,0 +1,161 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "python": "3.12.14",
3
+ "context": "Local CPU packaging validation runtime; remote training pod package dump is not claimed.",
4
+ "packages": {
5
+ "absl-py": "2.4.0",
6
+ "accelerate": "1.13.0",
7
+ "aiohappyeyeballs": "2.6.1",
8
+ "aiohttp": "3.14.0",
9
+ "aiosignal": "1.4.0",
10
+ "albucore": "0.0.17",
11
+ "albumentations": "1.4.18",
12
+ "annotated-types": "0.7.0",
13
+ "antlr4-python3-runtime": "4.9.3",
14
+ "asciitree": "0.3.3",
15
+ "attrs": "25.4.0",
16
+ "av": "15.1.0",
17
+ "boto3": "1.35.99",
18
+ "botocore": "1.35.99",
19
+ "certifi": "2026.7.22",
20
+ "cffi": "2.0.0",
21
+ "charset-normalizer": "3.5.2",
22
+ "click": "8.1.8",
23
+ "cloudpickle": "3.1.2",
24
+ "contourpy": "1.3.2",
25
+ "cryptography": "48.0.0",
26
+ "cycler": "0.12.1",
27
+ "datasets": "3.6.0",
28
+ "deepspeed": "0.17.6",
29
+ "diffusers": "0.35.1",
30
+ "dill": "0.3.8",
31
+ "dm-tree": "0.1.9",
32
+ "docstring-parser": "0.17.0",
33
+ "easydict": "1.13",
34
+ "einops": "0.8.1",
35
+ "eval-type-backport": "0.3.1",
36
+ "farama-notifications": "0.0.4",
37
+ "fasteners": "0.20",
38
+ "fastwam": "0.1.0",
39
+ "filelock": "4.0.9",
40
+ "flash-attn": "2.8.3",
41
+ "fonttools": "4.62.1",
42
+ "frozenlist": "1.8.0",
43
+ "fsspec": "2025.3.0",
44
+ "ftfy": "6.3.1",
45
+ "gitdb": "4.0.12",
46
+ "gitpython": "3.1.50",
47
+ "gr00t": "0.1.0",
48
+ "gymnasium": "1.2.2",
49
+ "h5py": "3.14.0",
50
+ "hf-xet": "1.4.2",
51
+ "hjson": "3.1.0",
52
+ "huggingface-hub": "0.36.2",
53
+ "hydra-core": "1.3.2",
54
+ "idna": "3.20",
55
+ "imageio": "2.37.3",
56
+ "imageio-ffmpeg": "0.6.0",
57
+ "importlib-metadata": "8.7.1",
58
+ "inquirerpy": "0.3.4",
59
+ "jinja2": "3.1.6",
60
+ "jmespath": "1.1.0",
61
+ "jsonlines": "4.0.0",
62
+ "kiwisolver": "1.5.0",
63
+ "lazy-loader": "0.5",
64
+ "lmdb": "1.7.5",
65
+ "lz4": "4.4.5",
66
+ "markdown-it-py": "4.0.0",
67
+ "markupsafe": "3.0.3",
68
+ "matplotlib": "3.10.1",
69
+ "mdurl": "0.1.2",
70
+ "ml-dtypes": "0.5.4",
71
+ "modelscope": "1.34.0",
72
+ "mpmath": "1.3.0",
73
+ "msgpack": "1.1.0",
74
+ "msgpack-numpy": "0.4.8",
75
+ "multidict": "6.7.1",
76
+ "multiprocess": "0.70.16",
77
+ "networkx": "3.4.2",
78
+ "ninja": "1.13.0",
79
+ "numcodecs": "0.15.1",
80
+ "numpy": "1.26.4",
81
+ "nvidia-cublas-cu12": "12.8.4.1",
82
+ "nvidia-cuda-cupti-cu12": "12.8.90",
83
+ "nvidia-cuda-nvrtc-cu12": "12.8.93",
84
+ "nvidia-cuda-runtime-cu12": "12.8.90",
85
+ "nvidia-cudnn-cu12": "9.10.2.21",
86
+ "nvidia-cufft-cu12": "11.3.3.83",
87
+ "nvidia-cufile-cu12": "1.13.1.3",
88
+ "nvidia-curand-cu12": "10.3.9.90",
89
+ "nvidia-cusolver-cu12": "11.7.3.90",
90
+ "nvidia-cusparse-cu12": "12.5.8.93",
91
+ "nvidia-cusparselt-cu12": "0.7.1",
92
+ "nvidia-nccl-cu12": "2.27.5",
93
+ "nvidia-nvjitlink-cu12": "12.8.93",
94
+ "nvidia-nvshmem-cu12": "3.3.20",
95
+ "nvidia-nvtx-cu12": "12.8.90",
96
+ "omegaconf": "2.3.1",
97
+ "onnx": "1.20.1",
98
+ "onnx-ir": "0.2.1",
99
+ "onnxscript": "0.7.0",
100
+ "opencv-python-headless": "4.11.0.86",
101
+ "packaging": "26.3",
102
+ "pandas": "2.2.3",
103
+ "peft": "0.17.1",
104
+ "pfzy": "0.3.4",
105
+ "pillow": "12.2.0",
106
+ "pip": "26.2.1",
107
+ "platformdirs": "4.9.4",
108
+ "prompt-toolkit": "3.0.52",
109
+ "propcache": "0.4.1",
110
+ "protobuf": "6.33.5",
111
+ "psutil": "7.2.2",
112
+ "py-cpuinfo": "9.0.0",
113
+ "pyarrow": "23.0.1",
114
+ "pycparser": "3.0",
115
+ "pydantic": "2.12.5",
116
+ "pydantic-core": "2.41.5",
117
+ "pygments": "2.20.0",
118
+ "pyparsing": "3.3.2",
119
+ "python-dateutil": "2.9.0.post0",
120
+ "pytz": "2026.1.post1",
121
+ "pyyaml": "6.0.3",
122
+ "pyzmq": "27.0.1",
123
+ "regex": "2026.2.28",
124
+ "requests": "2.34.2",
125
+ "rich": "14.3.3",
126
+ "s3transfer": "0.10.4",
127
+ "safetensors": "0.7.0",
128
+ "scikit-image": "0.25.2",
129
+ "scipy": "1.15.3",
130
+ "sentencepiece": "0.2.1",
131
+ "sentry-sdk": "2.55.0",
132
+ "setuptools": "84.0.0",
133
+ "shtab": "1.8.0",
134
+ "six": "1.17.0",
135
+ "smmap": "5.0.3",
136
+ "sympy": "1.14.0",
137
+ "termcolor": "3.2.0",
138
+ "tifffile": "2025.5.10",
139
+ "tokenizers": "0.22.2",
140
+ "torch": "2.9.0+cu128",
141
+ "torchcodec": "0.8.0",
142
+ "torchvision": "0.24.0+cu128",
143
+ "tqdm": "4.70.1",
144
+ "transformers": "4.57.3",
145
+ "triton": "3.5.0",
146
+ "typeguard": "4.5.1",
147
+ "typing-extensions": "4.15.0",
148
+ "typing-inspection": "0.4.2",
149
+ "tyro": "0.9.17",
150
+ "tzdata": "2025.3",
151
+ "urllib3": "2.8.0",
152
+ "wandb": "0.23.0",
153
+ "wcwidth": "0.6.0",
154
+ "wrapt": "2.1.2",
155
+ "xxhash": "3.6.0",
156
+ "yarl": "1.23.0",
157
+ "zarr": "2.18.7",
158
+ "zipp": "3.23.0",
159
+ "zstandard": "0.25.0"
160
+ }
161
+ }
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """FastWAM package."""
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/__init__.py ADDED
File without changes
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/dataset_utils.py ADDED
@@ -0,0 +1,109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ from typing import Optional
3
+
4
+ import torch
5
+ import torchvision.transforms.functional as transforms_F
6
+ from PIL import Image
7
+
8
+
9
+ def obtain_image_size(data: torch.Tensor | Image.Image) -> tuple[int, int]:
10
+ r"""Return spatial size from a PIL image or image/video tensor.
11
+
12
+ Args:
13
+ data (torch.Tensor | Image.Image): Input image or video tensor.
14
+ Returns:
15
+ width (int): Input width.
16
+ height (int): Input height.
17
+ """
18
+
19
+ if isinstance(data, Image.Image):
20
+ width, height = data.size
21
+ elif isinstance(data, torch.Tensor):
22
+ height, width = data.size()[-2:]
23
+ else:
24
+ raise ValueError("data to random crop should be PIL Image or tensor")
25
+
26
+ return width, height
27
+
28
+ class ResizeSmallestSideAspectPreserving:
29
+ def __init__(self, args: Optional[dict] = None) -> None:
30
+ self.args = args
31
+
32
+ def __call__(self, video: torch.Tensor | Image.Image) -> torch.Tensor | Image.Image:
33
+ r"""Resize while preserving aspect ratio.
34
+
35
+ The output is scaled so both spatial dimensions are at least the
36
+ requested target size.
37
+
38
+ Args:
39
+ video (torch.Tensor | Image.Image): Input image or video tensor.
40
+ Returns:
41
+ torch.Tensor | Image.Image: Resized image or video tensor.
42
+ """
43
+
44
+ assert self.args is not None, "Please specify args in augmentations"
45
+
46
+ img_w, img_h = self.args["img_w"], self.args["img_h"]
47
+
48
+ orig_w, orig_h = obtain_image_size(video)
49
+ scaling_ratio = max((img_w / orig_w), (img_h / orig_h))
50
+ target_size = (int(scaling_ratio * orig_h + 0.5), int(scaling_ratio * orig_w + 0.5))
51
+
52
+ assert (
53
+ target_size[0] >= img_h and target_size[1] >= img_w
54
+ ), f"Resize error. orig {(orig_w, orig_h)} desire {(img_w, img_h)} compute {target_size}"
55
+
56
+ return transforms_F.resize(
57
+ video,
58
+ size=target_size, # type: ignore
59
+ interpolation=self.args.get("interpolation", transforms_F.InterpolationMode.BICUBIC),
60
+ antialias=True,
61
+ )
62
+
63
+
64
+ class CenterCrop:
65
+ def __init__(self, args: Optional[dict] = None) -> None:
66
+ self.args = args
67
+
68
+ def __call__(self, video: torch.Tensor | Image.Image) -> torch.Tensor | Image.Image:
69
+ r"""Center crop to the requested spatial size.
70
+
71
+ Args:
72
+ video (torch.Tensor | Image.Image): Input image or video tensor.
73
+ Returns:
74
+ torch.Tensor | Image.Image: Center cropped image or video tensor.
75
+ """
76
+ assert (
77
+ (self.args is not None) and ("img_w" in self.args) and ("img_h" in self.args)
78
+ ), "Please specify size in args"
79
+
80
+ img_w, img_h = self.args["img_w"], self.args["img_h"]
81
+ return transforms_F.center_crop(video, [img_h, img_w])
82
+
83
+
84
+ class Normalize:
85
+ def __init__(self, args: Optional[dict] = None) -> None:
86
+ self.args = args
87
+
88
+ def __call__(self, video: torch.Tensor | Image.Image) -> torch.Tensor:
89
+ r"""Convert to tensor if needed and normalize by mean/std.
90
+
91
+ Args:
92
+ video (torch.Tensor | Image.Image): Input image or video tensor.
93
+ Returns:
94
+ torch.Tensor: Normalized image or video tensor.
95
+ """
96
+ assert self.args is not None, "Please specify args"
97
+
98
+ mean = self.args["mean"]
99
+ std = self.args["std"]
100
+
101
+ if isinstance(video, torch.Tensor):
102
+ data = video.to(dtype=torch.float32)
103
+ if video.dtype == torch.uint8:
104
+ data = data / 255.0
105
+ data = data.to(dtype=torch.get_default_dtype())
106
+ else:
107
+ data = transforms_F.to_tensor(video) # division by 255 is applied in to_tensor()
108
+
109
+ return transforms_F.normalize(tensor=data, mean=mean, std=std)
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/base_lerobot_dataset.py ADDED
@@ -0,0 +1,412 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import numpy as np
3
+ from pathlib import Path
4
+ from typing import List, Literal, Dict, Optional, Any, DefaultDict
5
+ from tqdm import tqdm
6
+ from .lerobot.lerobot_dataset import LeRobotDatasetMetadata, MultiLeRobotDataset
7
+
8
+ from concurrent.futures import ThreadPoolExecutor, as_completed
9
+ import traceback
10
+ from fastwam.utils.logging_config import get_logger
11
+ from .processors.base_processor import BaseProcessor
12
+
13
+ logger = get_logger(__name__)
14
+
15
+ MAX_GETITEM_ATTEMPT = 5
16
+
17
+ class BaseLerobotDataset(torch.utils.data.Dataset):
18
+ def __init__(
19
+ self,
20
+ dataset_dirs: List[str],
21
+
22
+ # shapes
23
+ shape_meta: Dict[str, Any],
24
+ action_size: int = 1,
25
+ past_action_size: int = 0, # Excludes the current frame
26
+ obs_size: int = 1, # should be
27
+ past_obs_size: int = 0,
28
+
29
+ # train vs val
30
+ val_set_proportion: float = 0.05,
31
+ is_training_set: bool = False,
32
+ seed: int = 42,
33
+
34
+ # sampling
35
+ global_sample_stride: int = 1,
36
+ ):
37
+ assert len(dataset_dirs) > 0, "At least one dataset directory is required"
38
+ assert past_action_size == 0
39
+ assert past_obs_size == 0
40
+ assert action_size == obs_size - 1, "In this dataset, action_size should be obs_size - 1"
41
+
42
+ self.dataset_dirs = dataset_dirs
43
+ self.shape_meta = shape_meta
44
+ self.action_size = action_size
45
+ self.past_action_size = past_action_size
46
+ self.obs_size = obs_size
47
+ self.processor = None # Will be set externally
48
+ metas = []
49
+ for ds_dir in dataset_dirs:
50
+ ds_root = Path(ds_dir)
51
+ repo_id = ds_dir
52
+ meta = LeRobotDatasetMetadata(repo_id=repo_id, root=ds_root)
53
+ metas.append(meta)
54
+
55
+ fps_list = [m.fps for m in metas]
56
+ assert len(set(fps_list)) == 1, f"All dataset_dirs must have the same fps, got {fps_list}"
57
+ fps = fps_list[0]
58
+
59
+ self.global_sample_stride = global_sample_stride
60
+
61
+ self.val_set_proportion = val_set_proportion
62
+ self.is_training_set = is_training_set
63
+
64
+ self.image_meta = shape_meta["images"]
65
+ self.state_meta = shape_meta["state"]
66
+ self.action_meta = shape_meta["action"]
67
+
68
+ delta_timestamps = {}
69
+ for meta in self.image_meta:
70
+ key = meta["key"]
71
+ meta["lerobot_key"] = f"observation.images.{key}" if key != "default" else "observation.images"
72
+ delta_timestamps[meta["lerobot_key"]] = [
73
+ (t * global_sample_stride) / fps for t in range(-past_obs_size, -past_obs_size + obs_size)
74
+ ]
75
+
76
+ for meta in self.state_meta:
77
+ key = meta["key"]
78
+ meta["lerobot_key"] = f"observation.state.{key}" if key != "default" else "observation.state"
79
+ delta_timestamps[meta["lerobot_key"]] = [
80
+ (t * global_sample_stride) / fps for t in range(-past_obs_size, -past_obs_size + obs_size)
81
+ ]
82
+
83
+ for meta in self.action_meta:
84
+ key = meta["key"]
85
+ meta["lerobot_key"] = f"action.{key}" if key != "default" else "action"
86
+ delta_timestamps[meta["lerobot_key"]] = [(t * global_sample_stride) / fps for t in range(-past_action_size, -past_action_size + action_size)]
87
+
88
+ episodes = {}
89
+ if val_set_proportion < 1e-6:
90
+ for meta in metas:
91
+ episodes.update({meta.repo_id: list(range(meta.total_episodes))})
92
+ else:
93
+ for meta in metas:
94
+ split_idx = int(meta.total_episodes * (1 - val_set_proportion))
95
+ # random shuffle episode indices before splitting
96
+ episode_indices = list(range(meta.total_episodes))
97
+ rng = np.random.default_rng(seed)
98
+ rng.shuffle(episode_indices)
99
+ if self.is_training_set:
100
+ episodes.update({meta.repo_id: [episode_indices[i] for i in range(split_idx)]})
101
+ else:
102
+ episodes.update({meta.repo_id: [episode_indices[i] for i in range(split_idx, meta.total_episodes)]})
103
+
104
+ self.multi_dataset = MultiLeRobotDataset(
105
+ dataset_dirs=self.dataset_dirs,
106
+ episodes=episodes,
107
+ delta_timestamps=delta_timestamps,
108
+ )
109
+
110
+ # HACK: lerobot 3.0 will fix this
111
+ episode_data_index = []
112
+ end_index = 0
113
+ for dataset in self.multi_dataset._datasets:
114
+ multi_episode_data_index = {
115
+ "from": dataset.episode_data_index["from"] + end_index,
116
+ "to": dataset.episode_data_index["to"] + end_index,
117
+ }
118
+ episode_data_index.append(multi_episode_data_index)
119
+ end_index = multi_episode_data_index["to"][-1]
120
+
121
+ self.episode_data_index = {
122
+ "from": torch.cat([dataset["from"] for dataset in episode_data_index]),
123
+ "to": torch.cat([dataset["to"] for dataset in episode_data_index]),
124
+ }
125
+
126
+ def _get_action(self, meta, lerobot_sample) -> torch.Tensor:
127
+ key, lerobot_key, raw_shape = meta["key"], meta["lerobot_key"], meta["raw_shape"]
128
+ action: torch.Tensor = lerobot_sample[lerobot_key] # [T, action_dim]
129
+ if action.ndim == 1: # for shape of 1, like gripper
130
+ action = action.unsqueeze(-1)
131
+ assert action.shape[-1] == raw_shape, f"Action '{key}' shape {action.shape[-1]} mismatch with meta {raw_shape}."
132
+ return action
133
+
134
+ def _get_state(self, meta, lerobot_sample) -> torch.Tensor:
135
+ key, lerobot_key, raw_shape = meta["key"], meta["lerobot_key"], meta["raw_shape"]
136
+ state: torch.Tensor = lerobot_sample[lerobot_key]
137
+ if state.ndim == 1: # for shape of 1, like gripper
138
+ state = state.unsqueeze(-1)
139
+ # state = state[..., :-1, :] # use state_{t} as observation_t
140
+ assert state.shape[-1] == raw_shape, f"State '{key}' shape {state.shape[-1]} mismatch with meta {raw_shape}."
141
+ return state
142
+
143
+ def _get_image(self, meta, lerobot_sample) -> torch.Tensor:
144
+ key, lerobot_key, raw_shape = meta["key"], meta["lerobot_key"], meta["raw_shape"]
145
+ image: torch.Tensor = lerobot_sample[lerobot_key]
146
+ if image.ndim == 3: # time dim will lost when obs_size is 1
147
+ image = image.unsqueeze(0)
148
+ image = (image * 255).to(torch.uint8) # (1, 3, H, W)
149
+ # For config simplication
150
+ # assert image.shape[1:] == raw_shape, f"Image '{key}' shape {image.shape[1:]} mismatch with {raw_shape}."
151
+ return image
152
+
153
+ def _split_lerobot_sample(self, lerobot_sample) -> Dict[str, Any]:
154
+ return lerobot_sample
155
+
156
+ def _get_episode_data(self, episode_idx):
157
+ lerobot_sample = self.multi_dataset.get_episode_data(episode_idx)
158
+ lerobot_sample = self._split_lerobot_sample(lerobot_sample)
159
+ state, action = {}, {}
160
+ for meta in self.state_meta:
161
+ s = self._get_state(meta, lerobot_sample)
162
+ state[meta["key"]] = s.unsqueeze(1).float()
163
+ for meta in self.action_meta:
164
+ a = self._get_action(meta, lerobot_sample)
165
+ a = sliding_window_with_replication(a, self.action_size)
166
+ action[meta["key"]] = a.float()
167
+ return {"action": action, "state": state}
168
+
169
+ def _set_return_images(self, flag: bool):
170
+ self.return_images = flag
171
+ self.multi_dataset.set_during_training(flag)
172
+
173
+ def __len__(self):
174
+ return self.multi_dataset.num_frames
175
+
176
+ def _get_additional_data(self, sample, lerobot_sample):
177
+ return sample
178
+
179
+ def __getitem__(self, idx):
180
+ if idx >= len(self):
181
+ raise IndexError(f"Index {idx} out of bounds {len(self)}.")
182
+
183
+ # Retry with random indices until we successfully load a frame.
184
+ sample_idx = idx
185
+ attempt = 0
186
+ last_exception: Optional[Exception] = None
187
+ while attempt < MAX_GETITEM_ATTEMPT:
188
+ try:
189
+ lerobot_sample = self.multi_dataset[sample_idx]
190
+ lerobot_sample = self._split_lerobot_sample(lerobot_sample)
191
+ break
192
+ except Exception as err:
193
+ attempt += 1
194
+ last_exception = err
195
+ logger.warning(
196
+ f"Error loading sample {sample_idx} (attempt {attempt}). "
197
+ "Retrying with a random index. "
198
+ f"Error: {err}"
199
+ )
200
+ sample_idx = np.random.randint(len(self))
201
+ print(traceback.format_exc())
202
+ else:
203
+ raise RuntimeError(
204
+ f"Failed to load a valid sample after {MAX_GETITEM_ATTEMPT} attempts "
205
+ f"for index {idx}."
206
+ ) from last_exception
207
+
208
+ # Get data from lerobot, organized in nested dict
209
+ sample = {
210
+ "idx": sample_idx,
211
+ "task": lerobot_sample["task"],
212
+ "action": {},
213
+ "state": {},
214
+ "images": {},
215
+ }
216
+ for meta in self.state_meta:
217
+ sample["state"][meta["key"]] = self._get_state(meta, lerobot_sample)
218
+
219
+ for meta in self.action_meta:
220
+ sample["action"][meta["key"]] = self._get_action(meta, lerobot_sample)
221
+
222
+ for meta in self.image_meta:
223
+ sample["images"][meta["key"]] = self._get_image(meta, lerobot_sample)
224
+
225
+ sample["action_is_pad"] = lerobot_sample[f"{self.action_meta[0]['lerobot_key']}_is_pad"]
226
+ sample["state_is_pad"] = lerobot_sample[f"{self.state_meta[0]['lerobot_key']}_is_pad"]
227
+ sample["image_is_pad"] = lerobot_sample[f"{self.image_meta[0]['lerobot_key']}_is_pad"]
228
+
229
+ sample = self._get_additional_data(sample, lerobot_sample)
230
+
231
+ for key in lerobot_sample:
232
+ if key not in sample and "observation" not in key and "action" not in key:
233
+ sample[key] = lerobot_sample[key]
234
+
235
+ # Preprocess the sample using the processor
236
+ # for quick data loading
237
+ if self.processor is not None:
238
+ sample = self.processor.preprocess(sample)
239
+
240
+ return sample
241
+
242
+ def set_processor(self, processor: BaseProcessor):
243
+ """Set processor instance from external initialization."""
244
+ self.processor = processor
245
+ if self.is_training_set:
246
+ self.processor.train()
247
+ else:
248
+ self.processor.eval()
249
+ return self
250
+
251
+ def get_dataset_stats(self, preprocessor: BaseProcessor):
252
+ state_min = DefaultDict(list)
253
+ state_max = DefaultDict(list)
254
+ state_mean = DefaultDict(list)
255
+ state_var = DefaultDict(list)
256
+ state_q01 = DefaultDict(list)
257
+ state_q99 = DefaultDict(list)
258
+
259
+ action_min = DefaultDict(list)
260
+ action_max = DefaultDict(list)
261
+ action_mean = DefaultDict(list)
262
+ action_var = DefaultDict(list)
263
+ action_q01 = DefaultDict(list)
264
+ action_q99 = DefaultDict(list)
265
+
266
+ episodes_num = self.multi_dataset.num_episodes
267
+
268
+ def process_episode(episode_idx):
269
+ batch = self._get_episode_data(episode_idx)
270
+ batch = preprocessor.action_state_transform(batch)
271
+ return batch
272
+
273
+ multi_thread = True
274
+ if not multi_thread:
275
+ for episode_idx in tqdm(range(episodes_num), desc="Iterating dataset to get normalization"):
276
+ batch = process_episode(episode_idx)
277
+ for meta in self.state_meta:
278
+ key = meta["key"]
279
+ cur_state: torch.Tensor = batch["state"][key] # (B, T, dim)
280
+ state_min[key].append(cur_state.amin(0))
281
+ state_max[key].append(cur_state.amax(0))
282
+ state_mean[key].append(cur_state.mean(0))
283
+ state_var[key].append(cur_state.var(0))
284
+ state_q01[key].append(torch.quantile(cur_state, 0.01, dim=0, keepdim=False))
285
+ state_q99[key].append(torch.quantile(cur_state, 0.99, dim=0, keepdim=False))
286
+ for meta in self.action_meta:
287
+ key = meta["key"]
288
+ cur_action: torch.Tensor = batch["action"][key] # (B, T, dim)
289
+ action_min[key].append(cur_action.amin(0))
290
+ action_max[key].append(cur_action.amax(0))
291
+ action_mean[key].append(cur_action.mean(0))
292
+ action_var[key].append(cur_action.var(0))
293
+ action_q01[key].append(torch.quantile(cur_action, 0.01, dim=0, keepdim=False))
294
+ action_q99[key].append(torch.quantile(cur_action, 0.99, dim=0, keepdim=False))
295
+
296
+ else:
297
+ with ThreadPoolExecutor() as executor:
298
+ futures = [executor.submit(process_episode, num) for num in range(episodes_num)]
299
+
300
+ for future in tqdm(as_completed(futures), total=episodes_num, desc="Iterating dataset to get normalization"):
301
+ try:
302
+ batch = future.result()
303
+ for meta in self.state_meta:
304
+ key = meta["key"]
305
+ cur_state: torch.Tensor = batch["state"][key] # (B, T, dim)
306
+ state_min[key].append(cur_state.amin(0))
307
+ state_max[key].append(cur_state.amax(0))
308
+ state_mean[key].append(cur_state.mean(0))
309
+ state_var[key].append(cur_state.var(0))
310
+ state_q01[key].append(torch.quantile(cur_state, 0.01, dim=0, keepdim=False))
311
+ state_q99[key].append(torch.quantile(cur_state, 0.99, dim=0, keepdim=False))
312
+
313
+ for meta in self.action_meta:
314
+ key = meta["key"]
315
+ cur_action: torch.Tensor = batch["action"][key] # (B, T, dim)
316
+ action_min[key].append(cur_action.amin(0))
317
+ action_max[key].append(cur_action.amax(0))
318
+ action_mean[key].append(cur_action.mean(0))
319
+ action_var[key].append(cur_action.var(0))
320
+ action_q01[key].append(torch.quantile(cur_action, 0.01, dim=0, keepdim=False))
321
+ action_q99[key].append(torch.quantile(cur_action, 0.99, dim=0, keepdim=False))
322
+
323
+ except Exception as e:
324
+ logger.error(f"Error processing episode: {e}")
325
+ print(traceback.format_exc())
326
+ raise e
327
+
328
+ # assume that each minibatch has equal number of samples
329
+ def get_mean_std(means, vars):
330
+ means = torch.stack(means)
331
+ vars = torch.stack(vars)
332
+ stepwise_mean = means.mean(0)
333
+ stepwise_std = (vars + (means - stepwise_mean) ** 2).mean(0).sqrt()
334
+ global_mean = means.mean((0, 1))
335
+ global_std = (vars + (means - global_mean) ** 2).mean((0, 1)).sqrt()
336
+ return stepwise_mean, stepwise_std, global_mean, global_std
337
+
338
+ stats = {"state": DefaultDict(dict), "action": DefaultDict(dict), "num_episodes": episodes_num, "num_transition": self.multi_dataset.num_frames}
339
+ for meta in self.state_meta:
340
+ key = meta["key"]
341
+ stats["state"][key]["stepwise_min"] = torch.stack(state_min[key]).amin(0)
342
+ stats["state"][key]["stepwise_max"] = torch.stack(state_max[key]).amax(0)
343
+ stats["state"][key]["global_min"] = stats["state"][key]["stepwise_min"].amin(0)
344
+ stats["state"][key]["global_max"] = stats["state"][key]["stepwise_max"].amax(0)
345
+ stats["state"][key]["stepwise_q01"] = torch.stack(state_q01[key]).amin(0)
346
+ stats["state"][key]["stepwise_q99"] = torch.stack(state_q99[key]).amax(0)
347
+ stats["state"][key]["global_q01"] = stats["state"][key]["stepwise_q01"].amin(0)
348
+ stats["state"][key]["global_q99"] = stats["state"][key]["stepwise_q99"].amax(0)
349
+ (
350
+ stats["state"][key]["stepwise_mean"],
351
+ stats["state"][key]["stepwise_std"],
352
+ stats["state"][key]["global_mean"],
353
+ stats["state"][key]["global_std"],
354
+ ) = get_mean_std(state_mean[key], state_var[key])
355
+
356
+ for meta in self.action_meta:
357
+ key = meta["key"]
358
+ stats["action"][key]["stepwise_min"] = torch.stack(action_min[key]).amin(0)
359
+ stats["action"][key]["stepwise_max"] = torch.stack(action_max[key]).amax(0)
360
+ stats["action"][key]["global_min"] = stats["action"][key]["stepwise_min"].amin(0)
361
+ stats["action"][key]["global_max"] = stats["action"][key]["stepwise_max"].amax(0)
362
+ stats["action"][key]["stepwise_q01"] = torch.stack(action_q01[key]).amin(0)
363
+ stats["action"][key]["stepwise_q99"] = torch.stack(action_q99[key]).amax(0)
364
+ stats["action"][key]["global_q01"] = stats["action"][key]["stepwise_q01"].amin(0)
365
+ stats["action"][key]["global_q99"] = stats["action"][key]["stepwise_q99"].amax(0)
366
+ (
367
+ stats["action"][key]["stepwise_mean"],
368
+ stats["action"][key]["stepwise_std"],
369
+ stats["action"][key]["global_mean"],
370
+ stats["action"][key]["global_std"],
371
+ ) = get_mean_std(action_mean[key], action_var[key])
372
+
373
+ return stats
374
+
375
+
376
+ def sliding_window_with_replication(x: torch.Tensor, window_size: int) -> torch.Tensor:
377
+ """
378
+ Construct a sliding-window tensor from the input tensor x (shape: [N, D]).
379
+ The output shape is [N, window_size, D].
380
+
381
+ For each starting index i:
382
+ out[i, j, :] =
383
+ x[i + j, :] if i + j < N
384
+ x[-1, :] otherwise (replicate the last row when out of bounds)
385
+
386
+ Args:
387
+ x (torch.Tensor): Input tensor of shape [N, D]
388
+ window_size (int): Size of the sliding window
389
+
390
+ Returns:
391
+ torch.Tensor: Tensor of shape [N, window_size, D]
392
+ """
393
+ assert x.dim() == 2
394
+ assert window_size > 0
395
+
396
+ N, D = x.shape
397
+
398
+ # shape [N, window_size]
399
+ # indices[i, j] = i + j
400
+ i_indices = torch.arange(N).unsqueeze(1) # [N, 1]
401
+ j_indices = torch.arange(window_size).unsqueeze(0) # [1, window_size]
402
+ indices = i_indices + j_indices # [N, window_size]
403
+
404
+ # N-1
405
+ # torch.clamp [0, N-1]
406
+ clamped_indices = torch.clamp(indices, min=0, max=N - 1)
407
+
408
+ # clamped_indices [N, window_size],x [N, D]
409
+ # out[i, j, :] = x[clamped_indices[i, j], :]
410
+ out = x[clamped_indices] # [N, window_size, D]
411
+
412
+ return out
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/constants.py ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2024 The HuggingFace Inc. team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # keys
15
+ import os
16
+ from pathlib import Path
17
+
18
+ from huggingface_hub.constants import HF_HOME
19
+
20
+ OBS_ENV_STATE = "observation.environment_state"
21
+ OBS_STATE = "observation.state"
22
+ OBS_IMAGE = "observation.image"
23
+ OBS_IMAGES = "observation.images"
24
+ ACTION = "action"
25
+ REWARD = "next.reward"
26
+
27
+ ROBOTS = "robots"
28
+ ROBOT_TYPE = "robot_type"
29
+ TELEOPERATORS = "teleoperators"
30
+
31
+ # files & directories
32
+ CHECKPOINTS_DIR = "checkpoints"
33
+ LAST_CHECKPOINT_LINK = "last"
34
+ PRETRAINED_MODEL_DIR = "pretrained_model"
35
+ TRAINING_STATE_DIR = "training_state"
36
+ RNG_STATE = "rng_state.safetensors"
37
+ TRAINING_STEP = "training_step.json"
38
+ OPTIMIZER_STATE = "optimizer_state.safetensors"
39
+ OPTIMIZER_PARAM_GROUPS = "optimizer_param_groups.json"
40
+ SCHEDULER_STATE = "scheduler_state.json"
41
+
42
+ if "LEROBOT_HOME" in os.environ:
43
+ raise ValueError(
44
+ f"You have a 'LEROBOT_HOME' environment variable set to '{os.getenv('LEROBOT_HOME')}'.\n"
45
+ "'LEROBOT_HOME' is deprecated, please use 'HF_LEROBOT_HOME' instead."
46
+ )
47
+
48
+ # cache dir
49
+ default_cache_path = Path(HF_HOME) / "lerobot"
50
+ HF_LEROBOT_HOME = Path(os.getenv("HF_LEROBOT_HOME", default_cache_path)).expanduser()
51
+
52
+ # calibration dir
53
+ default_calibration_path = HF_LEROBOT_HOME / "calibration"
54
+ HF_LEROBOT_CALIBRATION = Path(os.getenv("HF_LEROBOT_CALIBRATION", default_calibration_path)).expanduser()
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright 2024 The HuggingFace Inc. team. All rights reserved.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ import numpy as np
17
+
18
+ from .utils import load_image_as_numpy
19
+
20
+
21
+ def estimate_num_samples(
22
+ dataset_len: int, min_num_samples: int = 100, max_num_samples: int = 10_000, power: float = 0.75
23
+ ) -> int:
24
+ """Heuristic to estimate the number of samples based on dataset size.
25
+ The power controls the sample growth relative to dataset size.
26
+ Lower the power for less number of samples.
27
+
28
+ For default arguments, we have:
29
+ - from 1 to ~500, num_samples=100
30
+ - at 1000, num_samples=177
31
+ - at 2000, num_samples=299
32
+ - at 5000, num_samples=594
33
+ - at 10000, num_samples=1000
34
+ - at 20000, num_samples=1681
35
+ """
36
+ if dataset_len < min_num_samples:
37
+ min_num_samples = dataset_len
38
+ return max(min_num_samples, min(int(dataset_len**power), max_num_samples))
39
+
40
+
41
+ def sample_indices(data_len: int) -> list[int]:
42
+ num_samples = estimate_num_samples(data_len)
43
+ return np.round(np.linspace(0, data_len - 1, num_samples)).astype(int).tolist()
44
+
45
+
46
+ def auto_downsample_height_width(img: np.ndarray, target_size: int = 150, max_size_threshold: int = 300):
47
+ _, height, width = img.shape
48
+
49
+ if max(width, height) < max_size_threshold:
50
+ # no downsampling needed
51
+ return img
52
+
53
+ downsample_factor = int(width / target_size) if width > height else int(height / target_size)
54
+ return img[:, ::downsample_factor, ::downsample_factor]
55
+
56
+
57
+ def sample_images(image_paths: list[str]) -> np.ndarray:
58
+ sampled_indices = sample_indices(len(image_paths))
59
+
60
+ images = None
61
+ for i, idx in enumerate(sampled_indices):
62
+ path = image_paths[idx]
63
+ # we load as uint8 to reduce memory usage
64
+ img = load_image_as_numpy(path, dtype=np.uint8, channel_first=True)
65
+ img = auto_downsample_height_width(img)
66
+
67
+ if images is None:
68
+ images = np.empty((len(sampled_indices), *img.shape), dtype=np.uint8)
69
+
70
+ images[i] = img
71
+
72
+ return images
73
+
74
+
75
+ def get_feature_stats(array: np.ndarray, axis: tuple, keepdims: bool) -> dict[str, np.ndarray]:
76
+ return {
77
+ "min": np.min(array, axis=axis, keepdims=keepdims),
78
+ "max": np.max(array, axis=axis, keepdims=keepdims),
79
+ "mean": np.mean(array, axis=axis, keepdims=keepdims),
80
+ "std": np.std(array, axis=axis, keepdims=keepdims),
81
+ "count": np.array([len(array)]),
82
+ }
83
+
84
+
85
+ def compute_episode_stats(episode_data: dict[str, list[str] | np.ndarray], features: dict, is_compute_episode_stats_image : bool) -> dict:
86
+ ep_stats = {}
87
+ for key, data in episode_data.items():
88
+ if features[key]["dtype"] == "string":
89
+ continue # HACK: we should receive np.arrays of strings
90
+ elif features[key]["dtype"] in ["image", "video"]:
91
+ if is_compute_episode_stats_image:
92
+ ep_ft_array = sample_images(data) # data is a list of image paths
93
+ axes_to_reduce = (0, 2, 3) # keep channel dim
94
+ keepdims = True
95
+ else:
96
+ continue
97
+ else:
98
+ ep_ft_array = data # data is already a np.ndarray
99
+ axes_to_reduce = 0 # compute stats over the first axis
100
+ keepdims = data.ndim == 1 # keep as np.array
101
+
102
+ ep_stats[key] = get_feature_stats(ep_ft_array, axis=axes_to_reduce, keepdims=keepdims)
103
+
104
+ # finally, we normalize and remove batch dim for images
105
+ if features[key]["dtype"] in ["image", "video"]:
106
+ ep_stats[key] = {
107
+ k: v if k == "count" else np.squeeze(v / 255.0, axis=0) for k, v in ep_stats[key].items()
108
+ }
109
+
110
+ return ep_stats
111
+
112
+
113
+ def _assert_type_and_shape(stats_list: list[dict[str, dict]]):
114
+ for i in range(len(stats_list)):
115
+ for fkey in stats_list[i]:
116
+ for k, v in stats_list[i][fkey].items():
117
+ if not isinstance(v, np.ndarray):
118
+ raise ValueError(
119
+ f"Stats must be composed of numpy array, but key '{k}' of feature '{fkey}' is of type '{type(v)}' instead."
120
+ )
121
+ if v.ndim == 0:
122
+ raise ValueError("Number of dimensions must be at least 1, and is 0 instead.")
123
+ if k == "count" and v.shape != (1,):
124
+ raise ValueError(f"Shape of 'count' must be (1), but is {v.shape} instead.")
125
+ if "image" in fkey and k != "count" and v.shape != (3, 1, 1):
126
+ raise ValueError(f"Shape of '{k}' must be (3,1,1), but is {v.shape} instead.")
127
+
128
+
129
+ def aggregate_feature_stats(stats_ft_list: list[dict[str, dict]]) -> dict[str, dict[str, np.ndarray]]:
130
+ """Aggregates stats for a single feature."""
131
+ means = np.stack([s["mean"] for s in stats_ft_list])
132
+ variances = np.stack([s["std"] ** 2 for s in stats_ft_list])
133
+ counts = np.stack([s["count"] for s in stats_ft_list])
134
+ total_count = counts.sum(axis=0)
135
+
136
+ # Prepare weighted mean by matching number of dimensions
137
+ while counts.ndim < means.ndim:
138
+ counts = np.expand_dims(counts, axis=-1)
139
+
140
+ # Compute the weighted mean
141
+ weighted_means = means * counts
142
+ total_mean = weighted_means.sum(axis=0) / total_count
143
+
144
+ # Compute the variance using the parallel algorithm
145
+ delta_means = means - total_mean
146
+ weighted_variances = (variances + delta_means**2) * counts
147
+ total_variance = weighted_variances.sum(axis=0) / total_count
148
+
149
+ return {
150
+ "min": np.min(np.stack([s["min"] for s in stats_ft_list]), axis=0),
151
+ "max": np.max(np.stack([s["max"] for s in stats_ft_list]), axis=0),
152
+ "mean": total_mean,
153
+ "std": np.sqrt(total_variance),
154
+ "count": total_count,
155
+ }
156
+
157
+
158
+ def aggregate_stats(stats_list: list[dict[str, dict]]) -> dict[str, dict[str, np.ndarray]]:
159
+ """Aggregate stats from multiple compute_stats outputs into a single set of stats.
160
+
161
+ The final stats will have the union of all data keys from each of the stats dicts.
162
+
163
+ For instance:
164
+ - new_min = min(min_dataset_0, min_dataset_1, ...)
165
+ - new_max = max(max_dataset_0, max_dataset_1, ...)
166
+ - new_mean = (mean of all data, weighted by counts)
167
+ - new_std = (std of all data)
168
+ """
169
+
170
+ _assert_type_and_shape(stats_list)
171
+
172
+ data_keys = {key for stats in stats_list for key in stats}
173
+ aggregated_stats = {key: {} for key in data_keys}
174
+
175
+ for key in data_keys:
176
+ stats_with_key = [stats[key] for stats in stats_list if key in stats]
177
+ aggregated_stats[key] = aggregate_feature_stats(stats_with_key)
178
+
179
+ return aggregated_stats
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py ADDED
@@ -0,0 +1,903 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright 2024 The HuggingFace Inc. team. All rights reserved.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ import contextlib
17
+ import importlib.resources
18
+ import io
19
+ import json
20
+ import logging
21
+ from collections.abc import Iterator
22
+ from itertools import accumulate
23
+ from pathlib import Path
24
+ from pprint import pformat
25
+ from types import SimpleNamespace
26
+ from typing import Any
27
+ import array
28
+
29
+ import datasets
30
+ import jsonlines
31
+ import numpy as np
32
+ import packaging.version
33
+ import torch
34
+ from datasets.table import embed_table_storage
35
+ from huggingface_hub import DatasetCard, DatasetCardData, HfApi
36
+ from huggingface_hub.errors import RevisionNotFoundError
37
+ from PIL import Image as PILImage
38
+ from torchvision import transforms
39
+
40
+ from functools import partial
41
+
42
+ # from lerobot.configs.types import DictLike, FeatureType, PolicyFeature
43
+ # from lerobot.datasets.backward_compatibility import (
44
+ # V21_MESSAGE,
45
+ # BackwardCompatibilityError,
46
+ # ForwardCompatibilityError,
47
+ # )
48
+ # from lerobot.robots import Robot
49
+ # from lerobot.utils.utils import is_valid_numpy_dtype_string
50
+
51
+ DEFAULT_CHUNK_SIZE = 1000 # Max number of episodes per chunk
52
+
53
+ INFO_PATH = "meta/info.json"
54
+ EPISODES_PATH = "meta/episodes.jsonl"
55
+ STATS_PATH = "meta/stats.json"
56
+ EPISODES_STATS_PATH = "meta/episodes_stats.jsonl"
57
+ TASKS_PATH = "meta/tasks.jsonl"
58
+
59
+ ANNOTATION_PATHS = {
60
+ "subtask": "annotations/subtask_annotations.jsonl",
61
+ "scene": "annotations/scene_annotations.jsonl",
62
+ "gripper_mode": "annotations/gripper_mode_annotation.jsonl",
63
+ "gripper_activity": "annotations/gripper_activity_annotation.jsonl",
64
+ "eef_velocity": "annotations/eef_velocity_annotation.jsonl",
65
+ "eef_acc_mag": "annotations/eef_acc_mag_annotation.jsonl",
66
+ "eef_direction": "annotations/eef_direction_annotation.jsonl",
67
+ }
68
+
69
+ DEFAULT_VIDEO_PATH = "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4"
70
+ DEFAULT_PARQUET_PATH = "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet"
71
+ DEFAULT_IMAGE_PATH = "images/{image_key}/episode_{episode_index:06d}/frame_{frame_index:06d}.jpeg"
72
+
73
+ DATASET_CARD_TEMPLATE = """
74
+ ---
75
+ # Metadata will go there
76
+ ---
77
+ This dataset was created using [LeRobot](https://github.com/huggingface/lerobot).
78
+
79
+ ## {}
80
+
81
+ """
82
+
83
+ DEFAULT_FEATURES = {
84
+ "timestamp": {"dtype": "float32", "shape": (1,), "names": None},
85
+ "frame_index": {"dtype": "int64", "shape": (1,), "names": None},
86
+ "episode_index": {"dtype": "int64", "shape": (1,), "names": None},
87
+ "index": {"dtype": "int64", "shape": (1,), "names": None},
88
+ "coarse_task_index": {"dtype": "int64", "shape": (1,), "names": None},
89
+ "task_index": {"dtype": "int64", "shape": (1,), "names": None},
90
+ "coarse_quality_index": {"dtype": "int64", "shape": (1,), "names": None},
91
+ "quality_index": {"dtype": "int64", "shape": (1,), "names": None},
92
+ }
93
+
94
+
95
+ def flatten_dict(d: dict, parent_key: str = "", sep: str = "/") -> dict:
96
+ """Flatten a nested dictionary structure by collapsing nested keys into one key with a separator.
97
+
98
+ For example:
99
+ ```
100
+ >>> dct = {"a": {"b": 1, "c": {"d": 2}}, "e": 3}`
101
+ >>> print(flatten_dict(dct))
102
+ {"a/b": 1, "a/c/d": 2, "e": 3}
103
+ """
104
+ items = []
105
+ for k, v in d.items():
106
+ new_key = f"{parent_key}{sep}{k}" if parent_key else k
107
+ if isinstance(v, dict):
108
+ items.extend(flatten_dict(v, new_key, sep=sep).items())
109
+ else:
110
+ items.append((new_key, v))
111
+ return dict(items)
112
+
113
+
114
+ def unflatten_dict(d: dict, sep: str = "/") -> dict:
115
+ outdict = {}
116
+ for key, value in d.items():
117
+ parts = key.split(sep)
118
+ d = outdict
119
+ for part in parts[:-1]:
120
+ if part not in d:
121
+ d[part] = {}
122
+ d = d[part]
123
+ d[parts[-1]] = value
124
+ return outdict
125
+
126
+
127
+ # def get_nested_item(obj: DictLike, flattened_key: str, sep: str = "/") -> Any:
128
+ # split_keys = flattened_key.split(sep)
129
+ # getter = obj[split_keys[0]]
130
+ # if len(split_keys) == 1:
131
+ # return getter
132
+
133
+ # for key in split_keys[1:]:
134
+ # getter = getter[key]
135
+
136
+ # return getter
137
+
138
+
139
+ def serialize_dict(stats: dict[str, torch.Tensor | np.ndarray | dict]) -> dict:
140
+ serialized_dict = {}
141
+ for key, value in flatten_dict(stats).items():
142
+ if isinstance(value, (torch.Tensor, np.ndarray)):
143
+ serialized_dict[key] = value.tolist()
144
+ elif isinstance(value, np.generic):
145
+ serialized_dict[key] = value.item()
146
+ elif isinstance(value, (int, float)):
147
+ serialized_dict[key] = value
148
+ else:
149
+ raise NotImplementedError(f"The value '{value}' of type '{type(value)}' is not supported.")
150
+ return unflatten_dict(serialized_dict)
151
+
152
+
153
+ def embed_images(dataset: datasets.Dataset) -> datasets.Dataset:
154
+ # Embed image bytes into the table before saving to parquet
155
+ format = dataset.format
156
+ dataset = dataset.with_format("arrow")
157
+ dataset = dataset.map(embed_table_storage, batched=False)
158
+ dataset = dataset.with_format(**format)
159
+ return dataset
160
+
161
+
162
+ def load_json(fpath: Path) -> Any:
163
+ with open(fpath) as f:
164
+ return json.load(f)
165
+
166
+
167
+ def write_json(data: dict, fpath: Path) -> None:
168
+ fpath.parent.mkdir(exist_ok=True, parents=True)
169
+ with open(fpath, "w") as f:
170
+ json.dump(data, f, indent=4, ensure_ascii=False)
171
+
172
+
173
+ def load_jsonlines(fpath: Path) -> list[Any]:
174
+ # allow \t and \n
175
+ loose_loader = partial(json.loads, strict=False)
176
+ with jsonlines.open(fpath, "r", loads=loose_loader) as reader:
177
+ return list(reader)
178
+
179
+
180
+ def write_jsonlines(data: dict, fpath: Path) -> None:
181
+ fpath.parent.mkdir(exist_ok=True, parents=True)
182
+ with jsonlines.open(fpath, "w") as writer:
183
+ writer.write_all(data)
184
+
185
+
186
+ def append_jsonlines(data: dict, fpath: Path) -> None:
187
+ fpath.parent.mkdir(exist_ok=True, parents=True)
188
+ with jsonlines.open(fpath, "a") as writer:
189
+ writer.write(data)
190
+
191
+
192
+ def write_info(info: dict, local_dir: Path):
193
+ write_json(info, local_dir / INFO_PATH)
194
+
195
+
196
+ def load_info(local_dir: Path) -> dict:
197
+ info = load_json(local_dir / INFO_PATH)
198
+ for ft in info["features"].values():
199
+ ft["shape"] = tuple(ft["shape"])
200
+ return info
201
+
202
+
203
+ def write_stats(stats: dict, local_dir: Path):
204
+ serialized_stats = serialize_dict(stats)
205
+ write_json(serialized_stats, local_dir / STATS_PATH)
206
+
207
+
208
+ def cast_stats_to_numpy(stats) -> dict[str, dict[str, np.ndarray]]:
209
+ stats = {key: np.array(value) for key, value in flatten_dict(stats).items()}
210
+ return unflatten_dict(stats)
211
+
212
+
213
+ def load_stats(local_dir: Path) -> dict[str, dict[str, np.ndarray]]:
214
+ if not (local_dir / STATS_PATH).exists():
215
+ return None
216
+ stats = load_json(local_dir / STATS_PATH)
217
+ return cast_stats_to_numpy(stats)
218
+
219
+
220
+ def write_task(task_index: int, task: dict, local_dir: Path):
221
+ task_dict = {
222
+ "task_index": task_index,
223
+ "task": task,
224
+ }
225
+ append_jsonlines(task_dict, local_dir / TASKS_PATH)
226
+
227
+
228
+ def load_tasks(local_dir: Path) -> tuple[dict, dict]:
229
+ tasks = load_jsonlines(local_dir / TASKS_PATH)
230
+ tasks = {item["task_index"]: item["task"] for item in sorted(tasks, key=lambda x: x["task_index"])}
231
+ task_to_task_index = {task: task_index for task_index, task in tasks.items()}
232
+ return tasks, task_to_task_index
233
+
234
+ def load_annotations(local_dir: Path) -> dict[str, dict[int, str]]:
235
+ annotations = {}
236
+ for key, path in ANNOTATION_PATHS.items():
237
+ anno = load_jsonlines(local_dir / path)
238
+ anno = {item[f"{key}_index"]: item[key] for item in sorted(anno, key=lambda x: x[f"{key}_index"])}
239
+ annotations[key] = anno
240
+ return annotations
241
+
242
+ def write_episode(episode: dict, local_dir: Path):
243
+ append_jsonlines(episode, local_dir / EPISODES_PATH)
244
+
245
+
246
+ def load_episodes(local_dir: Path) -> dict:
247
+ episodes = load_jsonlines(local_dir / EPISODES_PATH)
248
+ return {item["episode_index"]: item for item in sorted(episodes, key=lambda x: x["episode_index"])}
249
+
250
+
251
+ def write_episode_stats(episode_index: int, episode_stats: dict, local_dir: Path):
252
+ # We wrap episode_stats in a dictionary since `episode_stats["episode_index"]`
253
+ # is a dictionary of stats and not an integer.
254
+ episode_stats = {"episode_index": episode_index, "stats": serialize_dict(episode_stats)}
255
+ append_jsonlines(episode_stats, local_dir / EPISODES_STATS_PATH)
256
+
257
+
258
+ def load_episodes_stats(local_dir: Path) -> dict:
259
+ episodes_stats = load_jsonlines(local_dir / EPISODES_STATS_PATH)
260
+ return {
261
+ item["episode_index"]: cast_stats_to_numpy(item["stats"])
262
+ for item in sorted(episodes_stats, key=lambda x: x["episode_index"])
263
+ }
264
+
265
+
266
+ def backward_compatible_episodes_stats(
267
+ stats: dict[str, dict[str, np.ndarray]], episodes: list[int]
268
+ ) -> dict[str, dict[str, np.ndarray]]:
269
+ return dict.fromkeys(episodes, stats)
270
+
271
+
272
+ def load_image_as_numpy(
273
+ fpath: str | Path, dtype: np.dtype = np.float32, channel_first: bool = True
274
+ ) -> np.ndarray:
275
+ img = PILImage.open(fpath).convert("RGB")
276
+ img_array = np.array(img, dtype=dtype)
277
+ if channel_first: # (H, W, C) -> (C, H, W)
278
+ img_array = np.transpose(img_array, (2, 0, 1))
279
+ if np.issubdtype(dtype, np.floating):
280
+ img_array /= 255.0
281
+ return img_array
282
+
283
+
284
+ def hf_transform_to_torch(items_dict: dict[torch.Tensor | None]):
285
+ """Get a transform function that convert items from Hugging Face dataset (pyarrow)
286
+ to torch tensors. Importantly, images are converted from PIL, which corresponds to
287
+ a channel last representation (h w c) of uint8 type, to a torch image representation
288
+ with channel first (c h w) of float32 type in range [0,1].
289
+ """
290
+ for key in items_dict:
291
+ first_item = items_dict[key][0]
292
+ if type(first_item) == dict and 'bytes' in first_item and first_item['bytes'] is not None:
293
+ to_pil = lambda img: PILImage.open(io.BytesIO(img['bytes']))
294
+ items_dict[key] = [to_pil(img) for img in items_dict[key]]
295
+ first_item = items_dict[key][0]
296
+ if isinstance(first_item, PILImage.Image):
297
+ to_tensor = transforms.ToTensor()
298
+ items_dict[key] = [to_tensor(img) for img in items_dict[key]]
299
+ elif first_item is None:
300
+ pass
301
+ else:
302
+ items_dict[key] = [x if isinstance(x, str) else torch.tensor(x) for x in items_dict[key]]
303
+ return items_dict
304
+
305
+
306
+ def is_valid_version(version: str) -> bool:
307
+ try:
308
+ packaging.version.parse(version)
309
+ return True
310
+ except packaging.version.InvalidVersion:
311
+ return False
312
+
313
+
314
+ # def check_version_compatibility(
315
+ # repo_id: str,
316
+ # version_to_check: str | packaging.version.Version,
317
+ # current_version: str | packaging.version.Version,
318
+ # enforce_breaking_major: bool = True,
319
+ # ) -> None:
320
+ # v_check = (
321
+ # packaging.version.parse(version_to_check)
322
+ # if not isinstance(version_to_check, packaging.version.Version)
323
+ # else version_to_check
324
+ # )
325
+ # v_current = (
326
+ # packaging.version.parse(current_version)
327
+ # if not isinstance(current_version, packaging.version.Version)
328
+ # else current_version
329
+ # )
330
+ # if v_check.major < v_current.major and enforce_breaking_major:
331
+ # raise BackwardCompatibilityError(repo_id, v_check)
332
+ # elif v_check.minor < v_current.minor:
333
+ # logging.warning(V21_MESSAGE.format(repo_id=repo_id, version=v_check))
334
+
335
+
336
+ def get_repo_versions(repo_id: str) -> list[packaging.version.Version]:
337
+ """Returns available valid versions (branches and tags) on given repo."""
338
+ api = HfApi()
339
+ repo_refs = api.list_repo_refs(repo_id, repo_type="dataset")
340
+ repo_refs = [b.name for b in repo_refs.branches + repo_refs.tags]
341
+ repo_versions = []
342
+ for ref in repo_refs:
343
+ with contextlib.suppress(packaging.version.InvalidVersion):
344
+ repo_versions.append(packaging.version.parse(ref))
345
+
346
+ return repo_versions
347
+
348
+
349
+ # def get_safe_version(repo_id: str, version: str | packaging.version.Version) -> str:
350
+ # """
351
+ # Returns the version if available on repo or the latest compatible one.
352
+ # Otherwise, will throw a `CompatibilityError`.
353
+ # """
354
+ # target_version = (
355
+ # packaging.version.parse(version) if not isinstance(version, packaging.version.Version) else version
356
+ # )
357
+ # hub_versions = get_repo_versions(repo_id)
358
+
359
+ # if not hub_versions:
360
+ # raise RevisionNotFoundError(
361
+ # f"""Your dataset must be tagged with a codebase version.
362
+ # Assuming _version_ is the codebase_version value in the info.json, you can run this:
363
+ # ```python
364
+ # from huggingface_hub import HfApi
365
+
366
+ # hub_api = HfApi()
367
+ # hub_api.create_tag("{repo_id}", tag="_version_", repo_type="dataset")
368
+ # ```
369
+ # """
370
+ # )
371
+
372
+ # if target_version in hub_versions:
373
+ # return f"v{target_version}"
374
+
375
+ # compatibles = [
376
+ # v for v in hub_versions if v.major == target_version.major and v.minor <= target_version.minor
377
+ # ]
378
+ # if compatibles:
379
+ # return_version = max(compatibles)
380
+ # if return_version < target_version:
381
+ # logging.warning(f"Revision {version} for {repo_id} not found, using version v{return_version}")
382
+ # return f"v{return_version}"
383
+
384
+ # lower_major = [v for v in hub_versions if v.major < target_version.major]
385
+ # if lower_major:
386
+ # raise BackwardCompatibilityError(repo_id, max(lower_major))
387
+
388
+ # upper_versions = [v for v in hub_versions if v > target_version]
389
+ # assert len(upper_versions) > 0
390
+ # raise ForwardCompatibilityError(repo_id, min(upper_versions))
391
+
392
+
393
+ def get_hf_features_from_features(features: dict) -> datasets.Features:
394
+ hf_features = {}
395
+ for key, ft in features.items():
396
+ if ft["dtype"] == "video":
397
+ continue
398
+ elif ft["dtype"] == "image":
399
+ hf_features[key] = datasets.Image()
400
+ elif ft["shape"] == (1,):
401
+ hf_features[key] = datasets.Value(dtype=ft["dtype"])
402
+ elif len(ft["shape"]) == 1:
403
+ hf_features[key] = datasets.Sequence(
404
+ length=ft["shape"][0], feature=datasets.Value(dtype=ft["dtype"])
405
+ )
406
+ elif len(ft["shape"]) == 2:
407
+ hf_features[key] = datasets.Array2D(shape=ft["shape"], dtype=ft["dtype"])
408
+ elif len(ft["shape"]) == 3:
409
+ hf_features[key] = datasets.Array3D(shape=ft["shape"], dtype=ft["dtype"])
410
+ elif len(ft["shape"]) == 4:
411
+ hf_features[key] = datasets.Array4D(shape=ft["shape"], dtype=ft["dtype"])
412
+ elif len(ft["shape"]) == 5:
413
+ hf_features[key] = datasets.Array5D(shape=ft["shape"], dtype=ft["dtype"])
414
+ else:
415
+ raise ValueError(f"Corresponding feature is not valid: {ft}")
416
+
417
+ return datasets.Features(hf_features)
418
+
419
+
420
+ def _validate_feature_names(features: dict[str, dict]) -> None:
421
+ invalid_features = {name: ft for name, ft in features.items() if "/" in name}
422
+ if invalid_features:
423
+ raise ValueError(f"Feature names should not contain '/'. Found '/' in '{invalid_features}'.")
424
+
425
+
426
+ def hw_to_dataset_features(
427
+ hw_features: dict[str, type | tuple], prefix: str, use_video: bool = True
428
+ ) -> dict[str, dict]:
429
+ features = {}
430
+ joint_fts = {key: ftype for key, ftype in hw_features.items() if ftype is float}
431
+ cam_fts = {key: shape for key, shape in hw_features.items() if isinstance(shape, tuple)}
432
+
433
+ if joint_fts and prefix == "action":
434
+ features[prefix] = {
435
+ "dtype": "float32",
436
+ "shape": (len(joint_fts),),
437
+ "names": list(joint_fts),
438
+ }
439
+
440
+ if joint_fts and prefix == "observation":
441
+ features[f"{prefix}.state"] = {
442
+ "dtype": "float32",
443
+ "shape": (len(joint_fts),),
444
+ "names": list(joint_fts),
445
+ }
446
+
447
+ for key, shape in cam_fts.items():
448
+ features[f"{prefix}.images.{key}"] = {
449
+ "dtype": "video" if use_video else "image",
450
+ "shape": shape,
451
+ "names": ["height", "width", "channels"],
452
+ }
453
+
454
+ _validate_feature_names(features)
455
+ return features
456
+
457
+
458
+ def build_dataset_frame(
459
+ ds_features: dict[str, dict], values: dict[str, Any], prefix: str
460
+ ) -> dict[str, np.ndarray]:
461
+ frame = {}
462
+ for key, ft in ds_features.items():
463
+ if key in DEFAULT_FEATURES or not key.startswith(prefix):
464
+ continue
465
+ elif ft["dtype"] == "float32" and len(ft["shape"]) == 1:
466
+ frame[key] = np.array([values[name] for name in ft["names"]], dtype=np.float32)
467
+ elif ft["dtype"] in ["image", "video"]:
468
+ frame[key] = values[key.removeprefix(f"{prefix}.images.")]
469
+
470
+ return frame
471
+
472
+
473
+ # def get_features_from_robot(robot: Robot, use_videos: bool = True) -> dict:
474
+ # camera_ft = {}
475
+ # if robot.cameras:
476
+ # camera_ft = {
477
+ # key: {"dtype": "video" if use_videos else "image", **ft}
478
+ # for key, ft in robot.camera_features.items()
479
+ # }
480
+ # return {**robot.motor_features, **camera_ft, **DEFAULT_FEATURES}
481
+
482
+
483
+ # def dataset_to_policy_features(features: dict[str, dict]) -> dict[str, PolicyFeature]:
484
+ # # TODO(aliberts): Implement "type" in dataset features and simplify this
485
+ # policy_features = {}
486
+ # for key, ft in features.items():
487
+ # shape = ft["shape"]
488
+ # if ft["dtype"] in ["image", "video"]:
489
+ # type = FeatureType.VISUAL
490
+ # if len(shape) != 3:
491
+ # raise ValueError(f"Number of dimensions of {key} != 3 (shape={shape})")
492
+
493
+ # names = ft["names"]
494
+ # # Backward compatibility for "channel" which is an error introduced in LeRobotDataset v2.0 for ported datasets.
495
+ # if names[2] in ["channel", "channels"]: # (h, w, c) -> (c, h, w)
496
+ # shape = (shape[2], shape[0], shape[1])
497
+ # elif key == "observation.environment_state":
498
+ # type = FeatureType.ENV
499
+ # elif key.startswith("observation"):
500
+ # type = FeatureType.STATE
501
+ # elif key.startswith("action"):
502
+ # type = FeatureType.ACTION
503
+ # else:
504
+ # continue
505
+
506
+ # policy_features[key] = PolicyFeature(
507
+ # type=type,
508
+ # shape=shape,
509
+ # )
510
+
511
+ # return policy_features
512
+
513
+
514
+ def create_empty_dataset_info(
515
+ codebase_version: str,
516
+ fps: int,
517
+ features: dict,
518
+ use_videos: bool,
519
+ robot_type: str | None = None,
520
+ ) -> dict:
521
+ return {
522
+ "codebase_version": codebase_version,
523
+ "robot_type": robot_type,
524
+ "total_episodes": 0,
525
+ "total_frames": 0,
526
+ "total_tasks": 0,
527
+ "total_videos": 0,
528
+ "total_chunks": 0,
529
+ "chunks_size": DEFAULT_CHUNK_SIZE,
530
+ "fps": fps,
531
+ "splits": {},
532
+ "data_path": DEFAULT_PARQUET_PATH,
533
+ "video_path": DEFAULT_VIDEO_PATH if use_videos else None,
534
+ "features": features,
535
+ }
536
+
537
+
538
+ def get_episode_data_index(
539
+ episode_dicts: dict[dict], episodes: list[int] | None = None
540
+ ) -> dict[str, torch.Tensor]:
541
+ episode_lengths = {ep_idx: ep_dict["length"] for ep_idx, ep_dict in episode_dicts.items()}
542
+ if episodes is not None:
543
+ episode_lengths = {ep_idx: episode_lengths[ep_idx] for ep_idx in episodes}
544
+
545
+ cumulative_lengths = list(accumulate(episode_lengths.values()))
546
+ return {
547
+ "from": torch.LongTensor([0] + cumulative_lengths[:-1]),
548
+ "to": torch.LongTensor(cumulative_lengths),
549
+ }
550
+
551
+
552
+ def check_timestamps_sync(
553
+ timestamps: np.ndarray,
554
+ episode_indices: np.ndarray,
555
+ episode_data_index: dict[str, np.ndarray],
556
+ fps: int,
557
+ tolerance_s: float,
558
+ raise_value_error: bool = True,
559
+ ) -> bool:
560
+ """
561
+ This check is to make sure that each timestamp is separated from the next by (1/fps) +/- tolerance
562
+ to account for possible numerical error.
563
+
564
+ Args:
565
+ timestamps (np.ndarray): Array of timestamps in seconds.
566
+ episode_indices (np.ndarray): Array indicating the episode index for each timestamp.
567
+ episode_data_index (dict[str, np.ndarray]): A dictionary that includes 'to',
568
+ which identifies indices for the end of each episode.
569
+ fps (int): Frames per second. Used to check the expected difference between consecutive timestamps.
570
+ tolerance_s (float): Allowed deviation from the expected (1/fps) difference.
571
+ raise_value_error (bool): Whether to raise a ValueError if the check fails.
572
+
573
+ Returns:
574
+ bool: True if all checked timestamp differences lie within tolerance, False otherwise.
575
+
576
+ Raises:
577
+ ValueError: If the check fails and `raise_value_error` is True.
578
+ """
579
+ if timestamps.shape != episode_indices.shape:
580
+ raise ValueError(
581
+ "timestamps and episode_indices should have the same shape. "
582
+ f"Found {timestamps.shape=} and {episode_indices.shape=}."
583
+ )
584
+
585
+ # Consecutive differences
586
+ diffs = np.diff(timestamps)
587
+ within_tolerance = np.abs(diffs - (1.0 / fps)) <= tolerance_s
588
+
589
+ # Mask to ignore differences at the boundaries between episodes
590
+ mask = np.ones(len(diffs), dtype=bool)
591
+ ignored_diffs = episode_data_index["to"][:-1] - 1 # indices at the end of each episode
592
+ mask[ignored_diffs] = False
593
+ filtered_within_tolerance = within_tolerance[mask]
594
+
595
+ # Check if all remaining diffs are within tolerance
596
+ if not np.all(filtered_within_tolerance):
597
+ # Track original indices before masking
598
+ original_indices = np.arange(len(diffs))
599
+ filtered_indices = original_indices[mask]
600
+ outside_tolerance_filtered_indices = np.nonzero(~filtered_within_tolerance)[0]
601
+ outside_tolerance_indices = filtered_indices[outside_tolerance_filtered_indices]
602
+
603
+ outside_tolerances = []
604
+ for idx in outside_tolerance_indices:
605
+ entry = {
606
+ "timestamps": [timestamps[idx], timestamps[idx + 1]],
607
+ "diff": diffs[idx],
608
+ "episode_index": episode_indices[idx].item()
609
+ if hasattr(episode_indices[idx], "item")
610
+ else episode_indices[idx],
611
+ }
612
+ outside_tolerances.append(entry)
613
+
614
+ if raise_value_error:
615
+ raise ValueError(
616
+ f"""One or several timestamps unexpectedly violate the tolerance inside episode range.
617
+ This might be due to synchronization issues during data collection.
618
+ \n{pformat(outside_tolerances)}"""
619
+ )
620
+ return False
621
+
622
+ return True
623
+
624
+
625
+ def check_delta_timestamps(
626
+ delta_timestamps: dict[str, list[float]], fps: int, tolerance_s: float, raise_value_error: bool = True
627
+ ) -> bool:
628
+ """This will check if all the values in delta_timestamps are multiples of 1/fps +/- tolerance.
629
+ This is to ensure that these delta_timestamps added to any timestamp from a dataset will themselves be
630
+ actual timestamps from the dataset.
631
+ """
632
+ outside_tolerance = {}
633
+ for key, delta_ts in delta_timestamps.items():
634
+ within_tolerance = [abs(ts * fps - round(ts * fps)) / fps <= tolerance_s for ts in delta_ts]
635
+ if not all(within_tolerance):
636
+ outside_tolerance[key] = [
637
+ ts for ts, is_within in zip(delta_ts, within_tolerance, strict=True) if not is_within
638
+ ]
639
+
640
+ if len(outside_tolerance) > 0:
641
+ if raise_value_error:
642
+ raise ValueError(
643
+ f"""
644
+ The following delta_timestamps are found outside of tolerance range.
645
+ Please make sure they are multiples of 1/{fps} +/- tolerance and adjust
646
+ their values accordingly.
647
+ \n{pformat(outside_tolerance)}
648
+ """
649
+ )
650
+ return False
651
+
652
+ return True
653
+
654
+
655
+ def get_delta_indices(delta_timestamps: dict[str, list[float]], fps: int) -> dict[str, list[int]]:
656
+ delta_indices = {}
657
+ for key, delta_ts in delta_timestamps.items():
658
+ delta_indices[key] = [round(d * fps) for d in delta_ts]
659
+
660
+ return delta_indices
661
+
662
+
663
+ def cycle(iterable):
664
+ """The equivalent of itertools.cycle, but safe for Pytorch dataloaders.
665
+
666
+ See https://github.com/pytorch/pytorch/issues/23900 for information on why itertools.cycle is not safe.
667
+ """
668
+ iterator = iter(iterable)
669
+ while True:
670
+ try:
671
+ yield next(iterator)
672
+ except StopIteration:
673
+ iterator = iter(iterable)
674
+
675
+
676
+ def create_branch(repo_id, *, branch: str, repo_type: str | None = None) -> None:
677
+ """Create a branch on a existing Hugging Face repo. Delete the branch if it already
678
+ exists before creating it.
679
+ """
680
+ api = HfApi()
681
+
682
+ branches = api.list_repo_refs(repo_id, repo_type=repo_type).branches
683
+ refs = [branch.ref for branch in branches]
684
+ ref = f"refs/heads/{branch}"
685
+ if ref in refs:
686
+ api.delete_branch(repo_id, repo_type=repo_type, branch=branch)
687
+
688
+ api.create_branch(repo_id, repo_type=repo_type, branch=branch)
689
+
690
+
691
+ def create_lerobot_dataset_card(
692
+ tags: list | None = None,
693
+ dataset_info: dict | None = None,
694
+ **kwargs,
695
+ ) -> DatasetCard:
696
+ """
697
+ Keyword arguments will be used to replace values in src/lerobot/datasets/card_template.md.
698
+ Note: If specified, license must be one of https://huggingface.co/docs/hub/repositories-licenses.
699
+ """
700
+ card_tags = ["LeRobot"]
701
+
702
+ if tags:
703
+ card_tags += tags
704
+ if dataset_info:
705
+ dataset_structure = "[meta/info.json](meta/info.json):\n"
706
+ dataset_structure += f"```json\n{json.dumps(dataset_info, indent=4)}\n```\n"
707
+ kwargs = {**kwargs, "dataset_structure": dataset_structure}
708
+ card_data = DatasetCardData(
709
+ license=kwargs.get("license"),
710
+ tags=card_tags,
711
+ task_categories=["robotics"],
712
+ configs=[
713
+ {
714
+ "config_name": "default",
715
+ "data_files": "data/*/*.parquet",
716
+ }
717
+ ],
718
+ )
719
+
720
+ card_template = (importlib.resources.files("lerobot.datasets") / "card_template.md").read_text()
721
+
722
+ return DatasetCard.from_template(
723
+ card_data=card_data,
724
+ template_str=card_template,
725
+ **kwargs,
726
+ )
727
+
728
+
729
+ class IterableNamespace(SimpleNamespace):
730
+ """
731
+ A namespace object that supports both dictionary-like iteration and dot notation access.
732
+ Automatically converts nested dictionaries into IterableNamespaces.
733
+
734
+ This class extends SimpleNamespace to provide:
735
+ - Dictionary-style iteration over keys
736
+ - Access to items via both dot notation (obj.key) and brackets (obj["key"])
737
+ - Dictionary-like methods: items(), keys(), values()
738
+ - Recursive conversion of nested dictionaries
739
+
740
+ Args:
741
+ dictionary: Optional dictionary to initialize the namespace
742
+ **kwargs: Additional keyword arguments passed to SimpleNamespace
743
+
744
+ Examples:
745
+ >>> data = {"name": "Alice", "details": {"age": 25}}
746
+ >>> ns = IterableNamespace(data)
747
+ >>> ns.name
748
+ 'Alice'
749
+ >>> ns.details.age
750
+ 25
751
+ >>> list(ns.keys())
752
+ ['name', 'details']
753
+ >>> for key, value in ns.items():
754
+ ... print(f"{key}: {value}")
755
+ name: Alice
756
+ details: IterableNamespace(age=25)
757
+ """
758
+
759
+ def __init__(self, dictionary: dict[str, Any] = None, **kwargs):
760
+ super().__init__(**kwargs)
761
+ if dictionary is not None:
762
+ for key, value in dictionary.items():
763
+ if isinstance(value, dict):
764
+ setattr(self, key, IterableNamespace(value))
765
+ else:
766
+ setattr(self, key, value)
767
+
768
+ def __iter__(self) -> Iterator[str]:
769
+ return iter(vars(self))
770
+
771
+ def __getitem__(self, key: str) -> Any:
772
+ return vars(self)[key]
773
+
774
+ def items(self):
775
+ return vars(self).items()
776
+
777
+ def values(self):
778
+ return vars(self).values()
779
+
780
+ def keys(self):
781
+ return vars(self).keys()
782
+
783
+
784
+ def validate_frame(frame: dict, features: dict):
785
+ expected_features = set(features) - set(DEFAULT_FEATURES)
786
+ actual_features = set(frame)
787
+
788
+ error_message = validate_features_presence(actual_features, expected_features)
789
+
790
+ common_features = actual_features & expected_features
791
+ for name in common_features - {"task"}:
792
+ error_message += validate_feature_dtype_and_shape(name, features[name], frame[name])
793
+
794
+ if error_message:
795
+ raise ValueError(error_message)
796
+
797
+
798
+ def validate_features_presence(actual_features: set[str], expected_features: set[str]):
799
+ error_message = ""
800
+ missing_features = expected_features - actual_features
801
+ extra_features = actual_features - expected_features
802
+
803
+ if missing_features or extra_features:
804
+ error_message += "Feature mismatch in `frame` dictionary:\n"
805
+ if missing_features:
806
+ error_message += f"Missing features: {missing_features}\n"
807
+ if extra_features:
808
+ error_message += f"Extra features: {extra_features}\n"
809
+
810
+ return error_message
811
+
812
+ def is_valid_numpy_dtype_string(dtype_str: str) -> bool:
813
+ """
814
+ Return True if a given string can be converted to a numpy dtype.
815
+ """
816
+ try:
817
+ # Attempt to convert the string to a numpy dtype
818
+ np.dtype(dtype_str)
819
+ return True
820
+ except TypeError:
821
+ # If a TypeError is raised, the string is not a valid dtype
822
+ return False
823
+ def validate_feature_dtype_and_shape(name: str, feature: dict, value: np.ndarray | PILImage.Image | str | bytes):
824
+ if isinstance(value, bytes) or isinstance(value, array.array): # ROS 1 and 2
825
+ #TODO fix bytes
826
+ return ""
827
+ expected_dtype = feature["dtype"]
828
+ expected_shape = feature["shape"]
829
+ if is_valid_numpy_dtype_string(expected_dtype):
830
+ return validate_feature_numpy_array(name, expected_dtype, expected_shape, value)
831
+ elif expected_dtype in ["image", "video"]:
832
+ return validate_feature_image_or_video(name, expected_shape, value)
833
+ elif expected_dtype == "string":
834
+ return validate_feature_string(name, value)
835
+ else:
836
+ raise NotImplementedError(f"The feature dtype '{expected_dtype}' is not implemented yet.")
837
+
838
+
839
+ def validate_feature_numpy_array(
840
+ name: str, expected_dtype: str, expected_shape: list[int], value: np.ndarray
841
+ ):
842
+ error_message = ""
843
+ if isinstance(value, np.ndarray):
844
+ actual_dtype = value.dtype
845
+ actual_shape = value.shape
846
+
847
+ if actual_dtype != np.dtype(expected_dtype):
848
+ error_message += f"The feature '{name}' of dtype '{actual_dtype}' is not of the expected dtype '{expected_dtype}'.\n"
849
+
850
+ if actual_shape != expected_shape:
851
+ error_message += f"The feature '{name}' of shape '{actual_shape}' does not have the expected shape '{expected_shape}'.\n"
852
+ else:
853
+ error_message += f"The feature '{name}' is not a 'np.ndarray'. Expected type is '{expected_dtype}', but type '{type(value)}' provided instead.\n"
854
+
855
+ return error_message
856
+
857
+
858
+ def validate_feature_image_or_video(name: str, expected_shape: list[str], value: np.ndarray | PILImage.Image):
859
+ # Note: The check of pixels range ([0,1] for float and [0,255] for uint8) is done by the image writer threads.
860
+ error_message = ""
861
+ if isinstance(value, np.ndarray):
862
+ actual_shape = value.shape
863
+ c, h, w = expected_shape
864
+ if len(actual_shape) != 3 or (actual_shape != (c, h, w) and actual_shape != (h, w, c)):
865
+ error_message += f"The feature '{name}' of shape '{actual_shape}' does not have the expected shape '{(c, h, w)}' or '{(h, w, c)}'.\n"
866
+ elif isinstance(value, PILImage.Image):
867
+ pass
868
+ else:
869
+ error_message += f"The feature '{name}' is expected to be of type 'PIL.Image' or 'np.ndarray' channel first or channel last, but type '{type(value)}' provided instead.\n"
870
+
871
+ return error_message
872
+
873
+
874
+ def validate_feature_string(name: str, value: str):
875
+ if not isinstance(value, str):
876
+ return f"The feature '{name}' is expected to be of type 'str', but type '{type(value)}' provided instead.\n"
877
+ return ""
878
+
879
+
880
+ def validate_episode_buffer(episode_buffer: dict, total_episodes: int, features: dict):
881
+ if "size" not in episode_buffer:
882
+ raise ValueError("size key not found in episode_buffer")
883
+
884
+ if "task" not in episode_buffer:
885
+ raise ValueError("task key not found in episode_buffer")
886
+
887
+ if episode_buffer["episode_index"] != total_episodes:
888
+ # TODO(aliberts): Add option to use existing episode_index
889
+ raise NotImplementedError(
890
+ "You might have manually provided the episode_buffer with an episode_index that doesn't "
891
+ "match the total number of episodes already in the dataset. This is not supported for now."
892
+ )
893
+
894
+ if episode_buffer["size"] == 0:
895
+ raise ValueError("You must add one or several frames with `add_frame` before calling `add_episode`.")
896
+
897
+ buffer_keys = set(episode_buffer.keys()) - {"task", "size"}
898
+ if not buffer_keys == set(features):
899
+ raise ValueError(
900
+ f"Features from `episode_buffer` don't match the ones in `features`."
901
+ f"In episode_buffer not in features: {buffer_keys - set(features)}"
902
+ f"In features not in episode_buffer: {set(features) - buffer_keys}"
903
+ )
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py ADDED
@@ -0,0 +1,527 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright 2024 The HuggingFace Inc. team. All rights reserved.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ import glob
17
+ import importlib
18
+ import logging
19
+ import warnings
20
+ from dataclasses import dataclass, field
21
+ from pathlib import Path
22
+ from typing import Any, ClassVar
23
+
24
+ import av
25
+ import pyarrow as pa
26
+ import torch
27
+ import torchvision
28
+ from datasets.features.features import register_feature
29
+ from PIL import Image
30
+
31
+
32
+ def get_safe_default_codec():
33
+ if importlib.util.find_spec("torchcodec"):
34
+ return "torchcodec"
35
+ else:
36
+ logging.warning(
37
+ "'torchcodec' is not available in your platform, falling back to 'pyav' as a default decoder"
38
+ )
39
+ return "pyav"
40
+
41
+
42
+ def decode_video_frames(
43
+ video_path: Path | str,
44
+ timestamps: list[float],
45
+ tolerance_s: float,
46
+ backend: str | None = None,
47
+ ) -> torch.Tensor:
48
+ """
49
+ Decodes video frames using the specified backend.
50
+
51
+ Args:
52
+ video_path (Path): Path to the video file.
53
+ timestamps (list[float]): List of timestamps to extract frames.
54
+ tolerance_s (float): Allowed deviation in seconds for frame retrieval.
55
+ backend (str, optional): Backend to use for decoding. Defaults to "torchcodec" when available in the platform; otherwise, defaults to "pyav"..
56
+
57
+ Returns:
58
+ torch.Tensor: Decoded frames.
59
+
60
+ Currently supports torchcodec on cpu and pyav.
61
+ """
62
+ if backend is None:
63
+ backend = get_safe_default_codec()
64
+ if backend == "torchcodec":
65
+ try:
66
+ return decode_video_frames_torchcodec(video_path, timestamps, tolerance_s)
67
+ except Exception as err:
68
+ warnings.warn(
69
+ f"torchcodec video decode failed ({type(err).__name__}: {err}); falling back to torchvision/pyav."
70
+ )
71
+ return decode_video_frames_torchvision(video_path, timestamps, tolerance_s, backend="pyav")
72
+ elif backend in ["pyav", "video_reader"]:
73
+ return decode_video_frames_torchvision(video_path, timestamps, tolerance_s, backend)
74
+ else:
75
+ raise ValueError(f"Unsupported video backend: {backend}")
76
+
77
+
78
+ def decode_video_frames_torchvision(
79
+ video_path: Path | str,
80
+ timestamps: list[float],
81
+ tolerance_s: float,
82
+ backend: str = "pyav",
83
+ log_loaded_timestamps: bool = False,
84
+ ) -> torch.Tensor:
85
+ """Loads frames associated to the requested timestamps of a video
86
+
87
+ The backend can be either "pyav" (default) or "video_reader".
88
+ "video_reader" requires installing torchvision from source, see:
89
+ https://github.com/pytorch/vision/blob/main/torchvision/csrc/io/decoder/gpu/README.rst
90
+ (note that you need to compile against ffmpeg<4.3)
91
+
92
+ While both use cpu, "video_reader" is supposedly faster than "pyav" but requires additional setup.
93
+ For more info on video decoding, see `benchmark/video/README.md`
94
+
95
+ See torchvision doc for more info on these two backends:
96
+ https://pytorch.org/vision/0.18/index.html?highlight=backend#torchvision.set_video_backend
97
+
98
+ Note: Video benefits from inter-frame compression. Instead of storing every frame individually,
99
+ the encoder stores a reference frame (or a key frame) and subsequent frames as differences relative to
100
+ that key frame. As a consequence, to access a requested frame, we need to load the preceding key frame,
101
+ and all subsequent frames until reaching the requested frame. The number of key frames in a video
102
+ can be adjusted during encoding to take into account decoding time and video size in bytes.
103
+ """
104
+ video_path = str(video_path)
105
+
106
+ # set backend
107
+ keyframes_only = False
108
+ torchvision.set_video_backend(backend)
109
+ if backend == "pyav":
110
+ keyframes_only = True # pyav doesn't support accurate seek
111
+
112
+ # set a video stream reader
113
+ # TODO(rcadene): also load audio stream at the same time
114
+ reader = torchvision.io.VideoReader(video_path, "video")
115
+
116
+ # set the first and last requested timestamps
117
+ # Note: previous timestamps are usually loaded, since we need to access the previous key frame
118
+ first_ts = min(timestamps)
119
+ last_ts = max(timestamps)
120
+
121
+ # access closest key frame of the first requested frame
122
+ # Note: closest key frame timestamp is usually smaller than `first_ts` (e.g. key frame can be the first frame of the video)
123
+ # for details on what `seek` is doing see: https://pyav.basswood-io.com/docs/stable/api/container.html?highlight=inputcontainer#av.container.InputContainer.seek
124
+ reader.seek(first_ts, keyframes_only=keyframes_only)
125
+
126
+ # load all frames until last requested frame
127
+ loaded_frames = []
128
+ loaded_ts = []
129
+ for frame in reader:
130
+ current_ts = frame["pts"]
131
+ if log_loaded_timestamps:
132
+ logging.info(f"frame loaded at timestamp={current_ts:.4f}")
133
+ loaded_frames.append(frame["data"])
134
+ loaded_ts.append(current_ts)
135
+ if current_ts >= last_ts:
136
+ break
137
+
138
+ if backend == "pyav":
139
+ reader.container.close()
140
+
141
+ reader = None
142
+
143
+ # Use float32 for timestamp distance computation (torch.cdist doesn't support bfloat16).
144
+ query_ts = torch.tensor(timestamps, dtype=torch.float32)
145
+ loaded_ts = torch.tensor(loaded_ts, dtype=torch.float32)
146
+
147
+ # compute distances between each query timestamp and timestamps of all loaded frames
148
+ dist = torch.cdist(query_ts[:, None], loaded_ts[:, None], p=1)
149
+ min_, argmin_ = dist.min(1)
150
+
151
+ is_within_tol = min_ < tolerance_s
152
+ assert is_within_tol.all(), (
153
+ f"One or several query timestamps unexpectedly violate the tolerance ({min_[~is_within_tol]} > {tolerance_s=})."
154
+ "It means that the closest frame that can be loaded from the video is too far away in time."
155
+ "This might be due to synchronization issues with timestamps during data collection."
156
+ "To be safe, we advise to ignore this item during training."
157
+ f"\nqueried timestamps: {query_ts}"
158
+ f"\nloaded timestamps: {loaded_ts}"
159
+ f"\nvideo: {video_path}"
160
+ f"\nbackend: {backend}"
161
+ )
162
+
163
+ # get closest frames to the query timestamps
164
+ closest_frames = torch.stack([loaded_frames[idx] for idx in argmin_])
165
+ closest_ts = loaded_ts[argmin_]
166
+
167
+ if log_loaded_timestamps:
168
+ logging.info(f"{closest_ts=}")
169
+
170
+ # convert to the pytorch format which is float32 in [0,1] range (and channel first)
171
+ closest_frames = closest_frames.type(torch.float32) / 255
172
+
173
+ assert len(timestamps) == len(closest_frames)
174
+ return closest_frames
175
+
176
+
177
+ def decode_video_frames_torchcodec(
178
+ video_path: Path | str,
179
+ timestamps: list[float],
180
+ tolerance_s: float,
181
+ device: str = "cpu",
182
+ log_loaded_timestamps: bool = False,
183
+ ) -> torch.Tensor:
184
+ """Loads frames associated with the requested timestamps of a video using torchcodec.
185
+
186
+ Note: Setting device="cuda" outside the main process, e.g. in data loader workers, will lead to CUDA initialization errors.
187
+
188
+ Note: Video benefits from inter-frame compression. Instead of storing every frame individually,
189
+ the encoder stores a reference frame (or a key frame) and subsequent frames as differences relative to
190
+ that key frame. As a consequence, to access a requested frame, we need to load the preceding key frame,
191
+ and all subsequent frames until reaching the requested frame. The number of key frames in a video
192
+ can be adjusted during encoding to take into account decoding time and video size in bytes.
193
+ """
194
+
195
+ if importlib.util.find_spec("torchcodec"):
196
+ from torchcodec.decoders import VideoDecoder
197
+ else:
198
+ raise ImportError("torchcodec is required but not available.")
199
+
200
+ # initialize video decoder
201
+ decoder = VideoDecoder(video_path, device=device, seek_mode="approximate")
202
+ loaded_frames = []
203
+ loaded_ts = []
204
+ # get metadata for frame information
205
+ metadata = decoder.metadata
206
+ average_fps = metadata.average_fps
207
+
208
+ # convert timestamps to frame indices
209
+ frame_indices = [round(ts * average_fps) for ts in timestamps]
210
+
211
+ # retrieve frames based on indices
212
+ frames_batch = decoder.get_frames_at(indices=frame_indices)
213
+
214
+ for frame, pts in zip(frames_batch.data, frames_batch.pts_seconds, strict=False):
215
+ loaded_frames.append(frame)
216
+ loaded_ts.append(pts.item())
217
+ if log_loaded_timestamps:
218
+ logging.info(f"Frame loaded at timestamp={pts:.4f}")
219
+
220
+ # Use float32 for timestamp distance computation (torch.cdist doesn't support bfloat16).
221
+ query_ts = torch.tensor(timestamps, dtype=torch.float32)
222
+ loaded_ts = torch.tensor(loaded_ts, dtype=torch.float32)
223
+
224
+ # compute distances between each query timestamp and loaded timestamps
225
+ dist = torch.cdist(query_ts[:, None], loaded_ts[:, None], p=1)
226
+ min_, argmin_ = dist.min(1)
227
+
228
+ is_within_tol = min_ < tolerance_s
229
+ assert is_within_tol.all(), (
230
+ f"One or several query timestamps unexpectedly violate the tolerance ({min_[~is_within_tol]} > {tolerance_s=})."
231
+ "It means that the closest frame that can be loaded from the video is too far away in time."
232
+ "This might be due to synchronization issues with timestamps during data collection."
233
+ "To be safe, we advise to ignore this item during training."
234
+ f"\nqueried timestamps: {query_ts}"
235
+ f"\nloaded timestamps: {loaded_ts}"
236
+ f"\nvideo: {video_path}"
237
+ )
238
+
239
+ # get closest frames to the query timestamps
240
+ closest_frames = torch.stack([loaded_frames[idx] for idx in argmin_])
241
+ closest_ts = loaded_ts[argmin_]
242
+
243
+ if log_loaded_timestamps:
244
+ logging.info(f"{closest_ts=}")
245
+
246
+ # convert to float32 in [0,1] range (channel first)
247
+ closest_frames = closest_frames.type(torch.float32) / 255
248
+
249
+ assert len(timestamps) == len(closest_frames)
250
+ return closest_frames
251
+
252
+ def encode_video_frames(
253
+ imgs_dir: Path | str,
254
+ video_path: Path | str,
255
+ fps: int,
256
+ vcodec: str = "libsvtav1",
257
+ pix_fmt: str = "yuv420p",
258
+ g: int | None = 2,
259
+ crf: int | None = 30,
260
+ fast_decode: int = 0,
261
+ log_level: int | None = av.logging.ERROR,
262
+ overwrite: bool = False,
263
+ ) -> None:
264
+ """More info on ffmpeg arguments tuning on `benchmark/video/README.md`"""
265
+ # Check encoder availability
266
+ if vcodec == "h264_nvenc" :
267
+ return encode_video_frames_ffmpeg(
268
+ imgs_dir, video_path, fps, pix_fmt = pix_fmt, overwrite=overwrite
269
+ )
270
+
271
+ if vcodec not in ["h264", "hevc", "libsvtav1"]:
272
+ raise ValueError(f"Unsupported video codec: {vcodec}. Supported codecs are: h264, hevc, libsvtav1.")
273
+
274
+ video_path = Path(video_path)
275
+ imgs_dir = Path(imgs_dir)
276
+
277
+ video_path.parent.mkdir(parents=True, exist_ok=overwrite)
278
+
279
+ # Encoders/pixel formats incompatibility check
280
+ if (vcodec == "libsvtav1" or vcodec == "hevc") and pix_fmt == "yuv444p":
281
+ logging.warning(
282
+ f"Incompatible pixel format 'yuv444p' for codec {vcodec}, auto-selecting format 'yuv420p'"
283
+ )
284
+ pix_fmt = "yuv420p"
285
+
286
+ # Get input frames
287
+ template = "frame_" + ("[0-9]" * 6) + ".jpeg"
288
+ input_list = sorted(
289
+ glob.glob(str(imgs_dir / template)), key=lambda x: int(x.split("_")[-1].split(".")[0])
290
+ )
291
+
292
+ # Define video output frame size (assuming all input frames are the same size)
293
+ if len(input_list) == 0:
294
+ raise FileNotFoundError(f"No images found in {imgs_dir}.")
295
+ dummy_image = Image.open(input_list[0])
296
+ width, height = dummy_image.size
297
+
298
+ # Define video codec options
299
+ video_options = {}
300
+
301
+ if g is not None:
302
+ video_options["g"] = str(g)
303
+
304
+ if crf is not None:
305
+ video_options["crf"] = str(crf)
306
+
307
+ if fast_decode:
308
+ key = "svtav1-params" if vcodec == "libsvtav1" else "tune"
309
+ value = f"fast-decode={fast_decode}" if vcodec == "libsvtav1" else "fastdecode"
310
+ video_options[key] = value
311
+
312
+ # Set logging level
313
+ if log_level is not None:
314
+ # "While less efficient, it is generally preferable to modify logging with Python’s logging"
315
+ logging.getLogger("libav").setLevel(log_level)
316
+
317
+ # Create and open output file (overwrite by default)
318
+ with av.open(str(video_path), "w") as output:
319
+ output_stream = output.add_stream(vcodec, fps, options=video_options)
320
+ output_stream.pix_fmt = pix_fmt
321
+ output_stream.width = width
322
+ output_stream.height = height
323
+
324
+ # Loop through input frames and encode them
325
+ for input_data in input_list:
326
+ input_image = Image.open(input_data).convert("RGB")
327
+ input_frame = av.VideoFrame.from_image(input_image)
328
+ packet = output_stream.encode(input_frame)
329
+ if packet:
330
+ output.mux(packet)
331
+
332
+ # Flush the encoder
333
+ packet = output_stream.encode()
334
+ if packet:
335
+ output.mux(packet)
336
+
337
+ # Reset logging level
338
+ if log_level is not None:
339
+ av.logging.restore_default_callback()
340
+
341
+ if not video_path.exists():
342
+ raise OSError(f"Video encoding did not work. File not found: {video_path}.")
343
+
344
+ # GPU speed up video encoding
345
+ def encode_video_frames_ffmpeg(
346
+ imgs_dir: Path | str,
347
+ video_path: Path | str,
348
+ fps: int,
349
+ vcodec: str = "h264_nvenc",
350
+ pix_fmt: str = "yuv420p",
351
+ g: int | None = 4,
352
+ crf: int | None = 30,
353
+ overwrite: bool = False,
354
+ ) -> None:
355
+ import subprocess
356
+ import shutil
357
+
358
+ video_path = Path(video_path)
359
+ imgs_dir = Path(imgs_dir)
360
+
361
+ video_path.parent.mkdir(parents=True, exist_ok=True)
362
+
363
+ # Check if ffmpeg is available
364
+ ffmpeg_path = shutil.which("ffmpeg")
365
+ if ffmpeg_path is None:
366
+ raise RuntimeError("ffmpeg is not installed or not in system PATH")
367
+
368
+ # Build ffmpeg command
369
+ cmd = [
370
+ ffmpeg_path,
371
+ "-y" if overwrite else "-n",
372
+ "-framerate", str(fps),
373
+ "-pattern_type", "sequence",
374
+ "-i", str(imgs_dir / "frame_%06d.jpeg"),
375
+ "-vcodec", vcodec,
376
+ "-pix_fmt", pix_fmt,
377
+ ]
378
+
379
+ if g is not None:
380
+ cmd.extend(["-g", str(g)])
381
+
382
+ if crf is not None:
383
+ cmd.extend(["-crf", str(crf)])
384
+
385
+ cmd.append(str(video_path))
386
+
387
+ try:
388
+ result = subprocess.run(
389
+ cmd,
390
+ capture_output=True,
391
+ text=True,
392
+ check=True
393
+ )
394
+ except subprocess.CalledProcessError as e:
395
+ error_msg = f"FFmpeg failed:\nCommand: {' '.join(cmd)}\nStdout: {e.stdout}\nStderr: {e.stderr}"
396
+ logging.error(error_msg)
397
+ logging.error(video_path)
398
+ logging.error(vcodec)
399
+ print(f"ffmpeg {' '.join(cmd)}")
400
+ raise RuntimeError(error_msg)
401
+ except Exception as e:
402
+ error_msg = f"Failed to run FFmpeg command: {' '.join(cmd)}\nError: {str(e)}"
403
+ logging.error(error_msg)
404
+ raise RuntimeError(error_msg)
405
+
406
+
407
+ @dataclass
408
+ class VideoFrame:
409
+ # TODO(rcadene, lhoestq): move to Hugging Face `datasets` repo
410
+ """
411
+ Provides a type for a dataset containing video frames.
412
+
413
+ Example:
414
+
415
+ ```python
416
+ data_dict = [{"image": {"path": "videos/episode_0.mp4", "timestamp": 0.3}}]
417
+ features = {"image": VideoFrame()}
418
+ Dataset.from_dict(data_dict, features=Features(features))
419
+ ```
420
+ """
421
+
422
+ pa_type: ClassVar[Any] = pa.struct({"path": pa.string(), "timestamp": pa.float32()})
423
+ _type: str = field(default="VideoFrame", init=False, repr=False)
424
+
425
+ def __call__(self):
426
+ return self.pa_type
427
+
428
+
429
+ with warnings.catch_warnings():
430
+ warnings.filterwarnings(
431
+ "ignore",
432
+ "'register_feature' is experimental and might be subject to breaking changes in the future.",
433
+ category=UserWarning,
434
+ )
435
+ # to make VideoFrame available in HuggingFace `datasets`
436
+ register_feature(VideoFrame, "VideoFrame")
437
+
438
+
439
+ def get_audio_info(video_path: Path | str) -> dict:
440
+ # Set logging level
441
+ logging.getLogger("libav").setLevel(av.logging.ERROR)
442
+
443
+ # Getting audio stream information
444
+ audio_info = {}
445
+ with av.open(str(video_path), "r") as audio_file:
446
+ try:
447
+ audio_stream = audio_file.streams.audio[0]
448
+ except IndexError:
449
+ # Reset logging level
450
+ av.logging.restore_default_callback()
451
+ return {"has_audio": False}
452
+
453
+ audio_info["audio.channels"] = audio_stream.channels
454
+ audio_info["audio.codec"] = audio_stream.codec.canonical_name
455
+ # In an ideal loseless case : bit depth x sample rate x channels = bit rate.
456
+ # In an actual compressed case, the bit rate is set according to the compression level : the lower the bit rate, the more compression is applied.
457
+ audio_info["audio.bit_rate"] = audio_stream.bit_rate
458
+ audio_info["audio.sample_rate"] = audio_stream.sample_rate # Number of samples per second
459
+ # In an ideal loseless case : fixed number of bits per sample.
460
+ # In an actual compressed case : variable number of bits per sample (often reduced to match a given depth rate).
461
+ audio_info["audio.bit_depth"] = audio_stream.format.bits
462
+ audio_info["audio.channel_layout"] = audio_stream.layout.name
463
+ audio_info["has_audio"] = True
464
+
465
+ # Reset logging level
466
+ av.logging.restore_default_callback()
467
+
468
+ return audio_info
469
+
470
+
471
+ def get_video_info(video_path: Path | str) -> dict:
472
+ # Set logging level
473
+ logging.getLogger("libav").setLevel(av.logging.ERROR)
474
+
475
+ # Getting video stream information
476
+ video_info = {}
477
+ with av.open(str(video_path), "r") as video_file:
478
+ try:
479
+ video_stream = video_file.streams.video[0]
480
+ except IndexError:
481
+ # Reset logging level
482
+ av.logging.restore_default_callback()
483
+ return {}
484
+
485
+ video_info["video.height"] = video_stream.height
486
+ video_info["video.width"] = video_stream.width
487
+ video_info["video.codec"] = video_stream.codec.canonical_name
488
+ video_info["video.pix_fmt"] = video_stream.pix_fmt
489
+ video_info["video.is_depth_map"] = False
490
+
491
+ # Calculate fps from r_frame_rate
492
+ video_info["video.fps"] = int(video_stream.base_rate)
493
+
494
+ pixel_channels = get_video_pixel_channels(video_stream.pix_fmt)
495
+ video_info["video.channels"] = pixel_channels
496
+
497
+ # Reset logging level
498
+ av.logging.restore_default_callback()
499
+
500
+ # Adding audio stream information
501
+ video_info.update(**get_audio_info(video_path))
502
+
503
+ return video_info
504
+
505
+
506
+ def get_video_pixel_channels(pix_fmt: str) -> int:
507
+ if "gray" in pix_fmt or "depth" in pix_fmt or "monochrome" in pix_fmt:
508
+ return 1
509
+ elif "rgba" in pix_fmt or "yuva" in pix_fmt:
510
+ return 4
511
+ elif "rgb" in pix_fmt or "yuv" in pix_fmt:
512
+ return 3
513
+ else:
514
+ raise ValueError("Unknown format")
515
+
516
+
517
+ def get_image_pixel_channels(image: Image):
518
+ if image.mode == "L":
519
+ return 1 # Grayscale
520
+ elif image.mode == "LA":
521
+ return 2 # Grayscale + Alpha
522
+ elif image.mode == "RGB":
523
+ return 3 # RGB
524
+ elif image.mode == "RGBA":
525
+ return 4 # RGBA
526
+ else:
527
+ raise ValueError("Unknown format")
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py ADDED
@@ -0,0 +1,1304 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright 2024 The HuggingFace Inc. team. All rights reserved.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ import contextlib
17
+ import logging
18
+ import shutil
19
+ from pathlib import Path
20
+ from typing import Callable, List, Literal
21
+ import warnings
22
+
23
+ import datasets
24
+ import numpy as np
25
+ import packaging.version
26
+ import PIL.Image
27
+ import torch
28
+ import torch.utils
29
+ import pyarrow.parquet as pq
30
+ from datasets import concatenate_datasets, load_dataset
31
+ from huggingface_hub import HfApi, snapshot_download
32
+ from huggingface_hub.constants import REPOCARD_NAME
33
+ from huggingface_hub.errors import RevisionNotFoundError
34
+
35
+ # adapte to own path
36
+ from ..constants import HF_LEROBOT_HOME
37
+ from .datasets.compute_stats import aggregate_stats, compute_episode_stats
38
+ from .datasets.utils import (
39
+ DEFAULT_FEATURES,
40
+ DEFAULT_IMAGE_PATH,
41
+ INFO_PATH,
42
+ TASKS_PATH,
43
+ _validate_feature_names,
44
+ append_jsonlines,
45
+ backward_compatible_episodes_stats,
46
+ check_delta_timestamps,
47
+ check_timestamps_sync,
48
+ # check_version_compatibility,
49
+ create_empty_dataset_info,
50
+ create_lerobot_dataset_card,
51
+ embed_images,
52
+ get_delta_indices,
53
+ get_episode_data_index,
54
+ get_hf_features_from_features,
55
+ # get_safe_version,
56
+ hf_transform_to_torch,
57
+ is_valid_version,
58
+ load_episodes,
59
+ load_episodes_stats,
60
+ load_info,
61
+ load_stats,
62
+ load_tasks,
63
+ load_annotations,
64
+ validate_episode_buffer,
65
+ validate_frame,
66
+ write_episode,
67
+ write_episode_stats,
68
+ write_info,
69
+ write_json,
70
+ )
71
+ from .datasets.video_utils import (
72
+ VideoFrame,
73
+ decode_video_frames,
74
+ encode_video_frames,
75
+ get_safe_default_codec,
76
+ get_video_info,
77
+ )
78
+ import traceback
79
+
80
+ CODEBASE_VERSION = "v2.1"
81
+
82
+
83
+ class LeRobotDatasetMetadata:
84
+ def __init__(
85
+ self,
86
+ repo_id: str,
87
+ root: str | Path | None = None,
88
+ revision: str | None = None,
89
+ force_cache_sync: bool = False,
90
+ ):
91
+ self.repo_id = repo_id
92
+ self.revision = revision if revision else CODEBASE_VERSION
93
+ self.root = Path(root) if root is not None else HF_LEROBOT_HOME / repo_id
94
+
95
+ try:
96
+ if force_cache_sync:
97
+ raise FileNotFoundError
98
+ self.load_metadata()
99
+ except (FileNotFoundError, NotADirectoryError):
100
+ # if is_valid_version(self.revision):
101
+ # self.revision = get_safe_version(self.repo_id, self.revision)
102
+
103
+ (self.root / "meta").mkdir(exist_ok=True, parents=True)
104
+ self.pull_from_repo(allow_patterns="meta/")
105
+ self.load_metadata()
106
+
107
+ def load_metadata(self):
108
+ self.info = load_info(self.root)
109
+ # TODO add new check
110
+ # check_version_compatibility(self.repo_id, self._version, CODEBASE_VERSION)
111
+ self.tasks, self.task_to_task_index = load_tasks(self.root)
112
+ if (self.root / "annotations").exists():
113
+ self.annotations = load_annotations(self.root)
114
+ self.episodes = load_episodes(self.root)
115
+ if self._version < packaging.version.parse("v2.1"):
116
+ self.stats = load_stats(self.root)
117
+ self.episodes_stats = backward_compatible_episodes_stats(self.stats, self.episodes)
118
+ else:
119
+ self.episodes_stats = load_episodes_stats(self.root)
120
+ self.stats = aggregate_stats(list(self.episodes_stats.values()))
121
+
122
+ def pull_from_repo(
123
+ self,
124
+ allow_patterns: list[str] | str | None = None,
125
+ ignore_patterns: list[str] | str | None = None,
126
+ ) -> None:
127
+ snapshot_download(
128
+ self.repo_id,
129
+ repo_type="dataset",
130
+ revision=self.revision,
131
+ local_dir=self.root,
132
+ allow_patterns=allow_patterns,
133
+ ignore_patterns=ignore_patterns,
134
+ )
135
+
136
+ @property
137
+ def _version(self) -> packaging.version.Version:
138
+ """Codebase version used to create this dataset."""
139
+ return packaging.version.parse(self.info["codebase_version"])
140
+
141
+ def get_data_file_path(self, ep_index: int) -> Path:
142
+ ep_chunk = self.get_episode_chunk(ep_index)
143
+ fpath = self.data_path.format(episode_chunk=ep_chunk, episode_index=ep_index)
144
+ return Path(fpath)
145
+
146
+ def get_video_file_path(self, ep_index: int, vid_key: str) -> Path:
147
+ ep_chunk = self.get_episode_chunk(ep_index)
148
+ fpath = self.video_path.format(episode_chunk=ep_chunk, video_key=vid_key, episode_index=ep_index)
149
+ return Path(fpath)
150
+
151
+ def get_episode_chunk(self, ep_index: int) -> int:
152
+ return ep_index // self.chunks_size
153
+
154
+ @property
155
+ def data_path(self) -> str:
156
+ """Formattable string for the parquet files."""
157
+ return self.info["data_path"]
158
+
159
+ @property
160
+ def video_path(self) -> str | None:
161
+ """Formattable string for the video files."""
162
+ return self.info["video_path"]
163
+
164
+ @property
165
+ def robot_type(self) -> str | None:
166
+ """Robot type used in recording this dataset."""
167
+ return self.info["robot_type"]
168
+
169
+ @property
170
+ def fps(self) -> int:
171
+ """Frames per second used during data collection."""
172
+ return self.info["fps"]
173
+
174
+ @property
175
+ def features(self) -> dict[str, dict]:
176
+ """All features contained in the dataset."""
177
+ return self.info["features"]
178
+
179
+ @property
180
+ def image_keys(self) -> list[str]:
181
+ """Keys to access visual modalities stored as images."""
182
+ return [key for key, ft in self.features.items() if ft["dtype"] == "image"]
183
+
184
+ @property
185
+ def video_keys(self) -> list[str]:
186
+ """Keys to access visual modalities stored as videos."""
187
+ return [key for key, ft in self.features.items() if ft["dtype"] == "video"]
188
+
189
+ @property
190
+ def camera_keys(self) -> list[str]:
191
+ """Keys to access visual modalities (regardless of their storage method)."""
192
+ return [key for key, ft in self.features.items() if ft["dtype"] in ["video", "image"]]
193
+
194
+ @property
195
+ def names(self) -> dict[str, list | dict]:
196
+ """Names of the various dimensions of vector modalities."""
197
+ return {key: ft["names"] for key, ft in self.features.items()}
198
+
199
+ @property
200
+ def shapes(self) -> dict:
201
+ """Shapes for the different features."""
202
+ return {key: tuple(ft["shape"]) for key, ft in self.features.items()}
203
+
204
+ @property
205
+ def total_episodes(self) -> int:
206
+ """Total number of episodes available."""
207
+ return self.info["total_episodes"]
208
+
209
+ @property
210
+ def total_frames(self) -> int:
211
+ """Total number of frames saved in this dataset."""
212
+ return self.info["total_frames"]
213
+
214
+ @property
215
+ def total_tasks(self) -> int:
216
+ """Total number of different tasks performed in this dataset."""
217
+ return self.info["total_tasks"]
218
+
219
+ @property
220
+ def total_chunks(self) -> int:
221
+ """Total number of chunks (groups of episodes)."""
222
+ return self.info["total_chunks"]
223
+
224
+ @property
225
+ def chunks_size(self) -> int:
226
+ """Max number of episodes per chunk."""
227
+ return self.info["chunks_size"]
228
+
229
+ def get_task_index(self, task: str) -> int | None:
230
+ """
231
+ Given a task in natural language, returns its task_index if the task already exists in the dataset,
232
+ otherwise return None.
233
+ """
234
+ return self.task_to_task_index.get(task, None)
235
+
236
+ def add_task(self, task: str):
237
+ """
238
+ Given a task in natural language, add it to the dictionary of tasks.
239
+ """
240
+ if task in self.task_to_task_index:
241
+ raise ValueError(f"The task '{task}' already exists and can't be added twice.")
242
+
243
+ task_index = self.info["total_tasks"]
244
+ self.task_to_task_index[task] = task_index
245
+ self.tasks[task_index] = task
246
+ self.info["total_tasks"] += 1
247
+
248
+ task_dict = {
249
+ "task_index": task_index,
250
+ "task": task,
251
+ }
252
+ append_jsonlines(task_dict, self.root / TASKS_PATH)
253
+
254
+ def save_episode(
255
+ self,
256
+ episode_index: int,
257
+ episode_length: int,
258
+ episode_tasks: list[str],
259
+ episode_stats: dict[str, dict],
260
+ raw_file_name: str | None = None,
261
+ ) -> None:
262
+ self.info["total_episodes"] += 1
263
+ self.info["total_frames"] += episode_length
264
+
265
+ chunk = self.get_episode_chunk(episode_index)
266
+ if chunk >= self.total_chunks:
267
+ self.info["total_chunks"] += 1
268
+
269
+ self.info["splits"] = {"train": f"0:{self.info['total_episodes']}"}
270
+ self.info["total_videos"] += len(self.video_keys)
271
+ if len(self.video_keys) > 0:
272
+ self.update_video_info()
273
+
274
+ write_info(self.info, self.root)
275
+
276
+ episode_dict = {
277
+ "episode_index": episode_index,
278
+ "tasks": episode_tasks,
279
+ "length": episode_length,
280
+ }
281
+ if raw_file_name is not None:
282
+ episode_dict["raw_file_name"] = raw_file_name
283
+
284
+ self.episodes[episode_index] = episode_dict
285
+ write_episode(episode_dict, self.root)
286
+
287
+ self.episodes_stats[episode_index] = episode_stats
288
+ self.stats = aggregate_stats([self.stats, episode_stats]) if self.stats else episode_stats
289
+ write_episode_stats(episode_index, episode_stats, self.root)
290
+
291
+ def update_video_info(self) -> None:
292
+ """
293
+ Warning: this function writes info from first episode videos, implicitly assuming that all videos have
294
+ been encoded the same way. Also, this means it assumes the first episode exists.
295
+ """
296
+ for key in self.video_keys:
297
+ if not self.features[key].get("info", None):
298
+ video_path = self.root / self.get_video_file_path(ep_index=0, vid_key=key)
299
+ self.info["features"][key]["info"] = get_video_info(video_path)
300
+
301
+ def __repr__(self):
302
+ feature_keys = list(self.features)
303
+ return (
304
+ f"{self.__class__.__name__}({{\n"
305
+ f" Repository ID: '{self.repo_id}',\n"
306
+ f" Total episodes: '{self.total_episodes}',\n"
307
+ f" Total frames: '{self.total_frames}',\n"
308
+ f" Features: '{feature_keys}',\n"
309
+ "})',\n"
310
+ )
311
+
312
+ @classmethod
313
+ def create(
314
+ cls,
315
+ repo_id: str,
316
+ fps: int,
317
+ features: dict,
318
+ robot_type: str | None = None,
319
+ root: str | Path | None = None,
320
+ use_videos: bool = True,
321
+ ) -> "LeRobotDatasetMetadata":
322
+ """Creates metadata for a LeRobotDataset."""
323
+ obj = cls.__new__(cls)
324
+ obj.repo_id = repo_id
325
+ obj.root = Path(root) if root is not None else HF_LEROBOT_HOME / repo_id
326
+
327
+ obj.root.mkdir(parents=True, exist_ok=False)
328
+
329
+ # TODO(aliberts, rcadene): implement sanity check for features
330
+ features = {**features, **DEFAULT_FEATURES}
331
+ _validate_feature_names(features)
332
+
333
+ obj.tasks, obj.task_to_task_index = {}, {}
334
+ obj.episodes_stats, obj.stats, obj.episodes = {}, {}, {}
335
+ obj.info = create_empty_dataset_info(CODEBASE_VERSION, fps, features, use_videos, robot_type)
336
+ if len(obj.video_keys) > 0 and not use_videos:
337
+ raise ValueError()
338
+ write_json(obj.info, obj.root / INFO_PATH)
339
+ obj.revision = None
340
+ return obj
341
+
342
+
343
+ class LeRobotDataset(torch.utils.data.Dataset):
344
+ def __init__(
345
+ self,
346
+ repo_id: str,
347
+ root: str | Path | None = None,
348
+ episodes: list[int] | None = None,
349
+ image_transforms: Callable | None = None,
350
+ delta_timestamps: dict[list[float]] | None = None,
351
+ tolerance_s: float = 1e-4,
352
+ revision: str | None = None,
353
+ force_cache_sync: bool = False,
354
+ download_videos: bool = True,
355
+ video_backend: str | None = None,
356
+ video_codec: Literal["h264", "hevc", "libsvtav1", "h264_nvenc"] = "libsvtav1",
357
+ is_compute_episode_stats_image: bool = True,
358
+ ):
359
+ """
360
+ 2 modes are available for instantiating this class, depending on 2 different use cases:
361
+
362
+ 1. Your dataset already exists:
363
+ - On your local disk in the 'root' folder. This is typically the case when you recorded your
364
+ dataset locally and you may or may not have pushed it to the hub yet. Instantiating this class
365
+ with 'root' will load your dataset directly from disk. This can happen while you're offline (no
366
+ internet connection).
367
+
368
+ - On the Hugging Face Hub at the address https://huggingface.co/datasets/{repo_id} and not on
369
+ your local disk in the 'root' folder. Instantiating this class with this 'repo_id' will download
370
+ the dataset from that address and load it, pending your dataset is compliant with
371
+ codebase_version v2.0. If your dataset has been created before this new format, you will be
372
+ prompted to convert it using our conversion script from v1.6 to v2.0, which you can find at
373
+ lerobot/datasets/v2/convert_dataset_v1_to_v2.py.
374
+
375
+
376
+ 2. Your dataset doesn't already exists (either on local disk or on the Hub): you can create an empty
377
+ LeRobotDataset with the 'create' classmethod. This can be used for recording a dataset or port an
378
+ existing dataset to the LeRobotDataset format.
379
+
380
+
381
+ In terms of files, LeRobotDataset encapsulates 3 main things:
382
+ - metadata:
383
+ - info contains various information about the dataset like shapes, keys, fps etc.
384
+ - stats stores the dataset statistics of the different modalities for normalization
385
+ - tasks contains the prompts for each task of the dataset, which can be used for
386
+ task-conditioned training.
387
+ - hf_dataset (from datasets.Dataset), which will read any values from parquet files.
388
+ - videos (optional) from which frames are loaded to be synchronous with data from parquet files.
389
+
390
+ A typical LeRobotDataset looks like this from its root path:
391
+ .
392
+ ├── data
393
+ │ ├── chunk-000
394
+ │ │ ├── episode_000000.parquet
395
+ │ │ ├── episode_000001.parquet
396
+ │ │ ├── episode_000002.parquet
397
+ │ │ └── ...
398
+ │ ├── chunk-001
399
+ │ │ ├── episode_001000.parquet
400
+ │ │ ├── episode_001001.parquet
401
+ │ │ ├── episode_001002.parquet
402
+ │ │ └── ...
403
+ │ └── ...
404
+ ├── meta
405
+ │ ├── episodes.jsonl
406
+ │ ├── info.json
407
+ │ ├── stats.json
408
+ │ └── tasks.jsonl
409
+ └── videos
410
+ ├── chunk-000
411
+ │ ├── observation.images.laptop
412
+ │ │ ├── episode_000000.mp4
413
+ │ │ ├── episode_000001.mp4
414
+ │ │ ├── episode_000002.mp4
415
+ │ │ └── ...
416
+ │ ├── observation.images.phone
417
+ │ │ ├── episode_000000.mp4
418
+ │ │ ├── episode_000001.mp4
419
+ │ │ ├── episode_000002.mp4
420
+ │ │ └── ...
421
+ ├── chunk-001
422
+ └── ...
423
+
424
+ Note that this file-based structure is designed to be as versatile as possible. The files are split by
425
+ episodes which allows a more granular control over which episodes one wants to use and download. The
426
+ structure of the dataset is entirely described in the info.json file, which can be easily downloaded
427
+ or viewed directly on the hub before downloading any actual data. The type of files used are very
428
+ simple and do not need complex tools to be read, it only uses .parquet, .json and .mp4 files (and .md
429
+ for the README).
430
+
431
+ Args:
432
+ repo_id (str): This is the repo id that will be used to fetch the dataset. Locally, the dataset
433
+ will be stored under root/repo_id.
434
+ root (Path | None, optional): Local directory to use for downloading/writing files. You can also
435
+ set the LEROBOT_HOME environment variable to point to a different location. Defaults to
436
+ '~/.cache/huggingface/lerobot'.
437
+ episodes (list[int] | None, optional): If specified, this will only load episodes specified by
438
+ their episode_index in this list. Defaults to None.
439
+ image_transforms (Callable | None, optional): You can pass standard v2 image transforms from
440
+ torchvision.transforms.v2 here which will be applied to visual modalities (whether they come
441
+ from videos or images). Defaults to None.
442
+ delta_timestamps (dict[list[float]] | None, optional): _description_. Defaults to None.
443
+ tolerance_s (float, optional): Tolerance in seconds used to ensure data timestamps are actually in
444
+ sync with the fps value. It is used at the init of the dataset to make sure that each
445
+ timestamps is separated to the next by 1/fps +/- tolerance_s. This also applies to frames
446
+ decoded from video files. It is also used to check that `delta_timestamps` (when provided) are
447
+ multiples of 1/fps. Defaults to 1e-4.
448
+ revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a
449
+ commit hash. Defaults to current codebase version tag.
450
+ sync_cache_first (bool, optional): Flag to sync and refresh local files first. If True and files
451
+ are already present in the local cache, this will be faster. However, files loaded might not
452
+ be in sync with the version on the hub, especially if you specified 'revision'. Defaults to
453
+ False.
454
+ download_videos (bool, optional): Flag to download the videos. Note that when set to True but the
455
+ video files are already present on local disk, they won't be downloaded again. Defaults to
456
+ True.
457
+ video_backend (str | None, optional): Video backend to use for decoding videos. Defaults to torchcodec when available int the platform; otherwise, defaults to 'pyav'.
458
+ You can also use the 'pyav' decoder used by Torchvision, which used to be the default option, or 'video_reader' which is another decoder of Torchvision.
459
+ """
460
+ super().__init__()
461
+ self.repo_id = repo_id
462
+ self.root = Path(root) if root else HF_LEROBOT_HOME / repo_id
463
+ self.image_transforms = image_transforms
464
+ self.delta_timestamps = delta_timestamps
465
+ self.episodes = episodes
466
+ self.tolerance_s = tolerance_s
467
+ self.revision = revision if revision else CODEBASE_VERSION
468
+ self.video_backend = video_backend if video_backend else get_safe_default_codec()
469
+ self.video_codec = video_codec
470
+ self.is_compute_episode_stats_image = is_compute_episode_stats_image
471
+ self.delta_indices = None
472
+ self.during_training = True
473
+
474
+ # Unused attributes
475
+ self.image_writer = None
476
+ self.episode_buffer = None
477
+
478
+ self.root.mkdir(exist_ok=True, parents=True)
479
+
480
+ # Load metadata
481
+ self.meta = LeRobotDatasetMetadata(
482
+ self.repo_id, self.root, self.revision, force_cache_sync=force_cache_sync
483
+ )
484
+ if self.episodes is not None and self.meta._version >= packaging.version.parse("v2.1"):
485
+ episodes_stats = [self.meta.episodes_stats[ep_idx] for ep_idx in self.episodes]
486
+ self.stats = aggregate_stats(episodes_stats)
487
+
488
+ # Load actual data
489
+ try:
490
+ if force_cache_sync:
491
+ raise FileNotFoundError
492
+ assert all((self.root / fpath).is_file() for fpath in self.get_episodes_file_paths())
493
+ self.hf_dataset = self.load_hf_dataset()
494
+ except (AssertionError, FileNotFoundError, NotADirectoryError):
495
+ # self.revision = get_safe_version(self.repo_id, self.revision)
496
+ self.revision = CODEBASE_VERSION
497
+ self.download_episodes(download_videos)
498
+ self.hf_dataset = self.load_hf_dataset()
499
+
500
+ self.episode_data_index = get_episode_data_index(self.meta.episodes, self.episodes)
501
+
502
+ # Check timestamps
503
+ timestamps = torch.stack(self.hf_dataset["timestamp"]).numpy()
504
+ episode_indices = torch.stack(self.hf_dataset["episode_index"]).numpy()
505
+ ep_data_index_np = {k: t.numpy() for k, t in self.episode_data_index.items()}
506
+ # check_timestamps_sync(timestamps, episode_indices, ep_data_index_np, self.fps, self.tolerance_s)
507
+
508
+ # Setup delta_indices
509
+ if self.delta_timestamps is not None:
510
+ # check_delta_timestamps(self.delta_timestamps, self.fps, self.tolerance_s)
511
+ self.delta_indices = get_delta_indices(self.delta_timestamps, self.fps)
512
+
513
+ def push_to_hub(
514
+ self,
515
+ branch: str | None = None,
516
+ tags: list | None = None,
517
+ license: str | None = "apache-2.0",
518
+ tag_version: bool = True,
519
+ push_videos: bool = True,
520
+ private: bool = False,
521
+ allow_patterns: list[str] | str | None = None,
522
+ upload_large_folder: bool = False,
523
+ **card_kwargs,
524
+ ) -> None:
525
+ ignore_patterns = ["images/"]
526
+ if not push_videos:
527
+ ignore_patterns.append("videos/")
528
+
529
+ hub_api = HfApi()
530
+ hub_api.create_repo(
531
+ repo_id=self.repo_id,
532
+ private=private,
533
+ repo_type="dataset",
534
+ exist_ok=True,
535
+ )
536
+ if branch:
537
+ hub_api.create_branch(
538
+ repo_id=self.repo_id,
539
+ branch=branch,
540
+ revision=self.revision,
541
+ repo_type="dataset",
542
+ exist_ok=True,
543
+ )
544
+
545
+ upload_kwargs = {
546
+ "repo_id": self.repo_id,
547
+ "folder_path": self.root,
548
+ "repo_type": "dataset",
549
+ "revision": branch,
550
+ "allow_patterns": allow_patterns,
551
+ "ignore_patterns": ignore_patterns,
552
+ }
553
+ if upload_large_folder:
554
+ hub_api.upload_large_folder(**upload_kwargs)
555
+ else:
556
+ hub_api.upload_folder(**upload_kwargs)
557
+
558
+ if not hub_api.file_exists(self.repo_id, REPOCARD_NAME, repo_type="dataset", revision=branch):
559
+ card = create_lerobot_dataset_card(
560
+ tags=tags, dataset_info=self.meta.info, license=license, **card_kwargs
561
+ )
562
+ card.push_to_hub(repo_id=self.repo_id, repo_type="dataset", revision=branch)
563
+
564
+ if tag_version:
565
+ with contextlib.suppress(RevisionNotFoundError):
566
+ hub_api.delete_tag(self.repo_id, tag=CODEBASE_VERSION, repo_type="dataset")
567
+ hub_api.create_tag(self.repo_id, tag=CODEBASE_VERSION, revision=branch, repo_type="dataset")
568
+
569
+ def pull_from_repo(
570
+ self,
571
+ allow_patterns: list[str] | str | None = None,
572
+ ignore_patterns: list[str] | str | None = None,
573
+ ) -> None:
574
+ snapshot_download(
575
+ self.repo_id,
576
+ repo_type="dataset",
577
+ revision=self.revision,
578
+ local_dir=self.root,
579
+ allow_patterns=allow_patterns,
580
+ ignore_patterns=ignore_patterns,
581
+ )
582
+
583
+ def download_episodes(self, download_videos: bool = True) -> None:
584
+ """Downloads the dataset from the given 'repo_id' at the provided version. If 'episodes' is given, this
585
+ will only download those episodes (selected by their episode_index). If 'episodes' is None, the whole
586
+ dataset will be downloaded. Thanks to the behavior of snapshot_download, if the files are already present
587
+ in 'local_dir', they won't be downloaded again.
588
+ """
589
+ # TODO(rcadene, aliberts): implement faster transfer
590
+ # https://huggingface.co/docs/huggingface_hub/en/guides/download#faster-downloads
591
+ files = None
592
+ ignore_patterns = None if download_videos else "videos/"
593
+ if self.episodes is not None:
594
+ files = self.get_episodes_file_paths()
595
+
596
+ self.pull_from_repo(allow_patterns=files, ignore_patterns=ignore_patterns)
597
+
598
+ def get_episodes_file_paths(self) -> list[Path]:
599
+ episodes = self.episodes if self.episodes is not None else list(range(self.meta.total_episodes))
600
+ fpaths = [str(self.meta.get_data_file_path(ep_idx)) for ep_idx in episodes]
601
+ if len(self.meta.video_keys) > 0:
602
+ video_files = [
603
+ str(self.meta.get_video_file_path(ep_idx, vid_key))
604
+ for vid_key in self.meta.video_keys
605
+ for ep_idx in episodes
606
+ ]
607
+ fpaths += video_files
608
+
609
+ return fpaths
610
+
611
+ def load_hf_dataset(self) -> datasets.Dataset:
612
+ """hf_dataset contains all the observations, states, actions, rewards, etc."""
613
+ if self.episodes is None:
614
+ path = str(self.root / "data")
615
+ hf_dataset = load_dataset("parquet", data_dir=path, split="train")
616
+ else:
617
+ files = [str(self.root / self.meta.get_data_file_path(ep_idx)) for ep_idx in self.episodes]
618
+ hf_dataset = load_dataset("parquet", data_files=files, split="train")
619
+
620
+ # TODO(aliberts): hf_dataset.set_format("torch")
621
+ hf_dataset.set_transform(hf_transform_to_torch)
622
+ return hf_dataset
623
+
624
+ def create_hf_dataset(self) -> datasets.Dataset:
625
+ features = get_hf_features_from_features(self.features)
626
+ ft_dict = {col: [] for col in features}
627
+ hf_dataset = datasets.Dataset.from_dict(ft_dict, features=features, split="train")
628
+
629
+ # TODO(aliberts): hf_dataset.set_format("torch")
630
+ hf_dataset.set_transform(hf_transform_to_torch)
631
+ return hf_dataset
632
+
633
+ @property
634
+ def fps(self) -> int:
635
+ """Frames per second used during data collection."""
636
+ return self.meta.fps
637
+
638
+ @property
639
+ def num_frames(self) -> int:
640
+ """Number of frames in selected episodes."""
641
+ return len(self.hf_dataset) if self.hf_dataset is not None else self.meta.total_frames
642
+
643
+ @property
644
+ def num_episodes(self) -> int:
645
+ """Number of episodes selected."""
646
+ return len(self.episodes) if self.episodes is not None else self.meta.total_episodes
647
+
648
+ @property
649
+ def features(self) -> dict[str, dict]:
650
+ return self.meta.features
651
+
652
+ @property
653
+ def hf_features(self) -> datasets.Features:
654
+ """Features of the hf_dataset."""
655
+ if self.hf_dataset is not None:
656
+ return self.hf_dataset.features
657
+ else:
658
+ return get_hf_features_from_features(self.features)
659
+
660
+ def _get_query_indices(self, idx: int, ep_idx: int) -> tuple[dict[str, list[int | bool]]]:
661
+ ep_start = self.episode_data_index["from"][ep_idx]
662
+ ep_end = self.episode_data_index["to"][ep_idx]
663
+ query_indices = {
664
+ key: [max(ep_start.item(), min(ep_end.item() - 1, idx + delta)) for delta in delta_idx]
665
+ for key, delta_idx in self.delta_indices.items()
666
+ }
667
+ padding = { # Pad values outside of current episode range
668
+ f"{key}_is_pad": torch.BoolTensor(
669
+ [(idx + delta < ep_start.item()) | (idx + delta >= ep_end.item()) for delta in delta_idx]
670
+ )
671
+ for key, delta_idx in self.delta_indices.items()
672
+ }
673
+ return query_indices, padding
674
+
675
+ def _get_query_timestamps(
676
+ self,
677
+ current_ts: float,
678
+ query_indices: dict[str, list[int]] | None = None,
679
+ ) -> dict[str, list[float]]:
680
+ query_timestamps = {}
681
+ for key in self.meta.video_keys:
682
+ if query_indices is not None and key in query_indices:
683
+ timestamps = self.hf_dataset.select(query_indices[key])["timestamp"]
684
+ query_timestamps[key] = torch.stack(timestamps).tolist()
685
+ else:
686
+ query_timestamps[key] = [current_ts]
687
+
688
+ return query_timestamps
689
+
690
+ def _query_hf_dataset(self, query_indices: dict[str, list[int]]) -> dict:
691
+ return {
692
+ key: torch.stack(self.hf_dataset.select(q_idx)[key])
693
+ for key, q_idx in query_indices.items()
694
+ if key not in self.meta.video_keys
695
+ }
696
+
697
+ def _query_hf_dataset_fast(self, query_indices: dict[str, list[int]]) -> dict:
698
+ result = {}
699
+ processed_indices = set()
700
+ index_to_selected = {}
701
+ for key, q_idx in query_indices.items():
702
+ if key not in self.meta.video_keys :
703
+ if 'images' in key and not self.during_training:
704
+ continue
705
+ q_idx_tuple = tuple(q_idx)
706
+ if q_idx_tuple not in processed_indices:
707
+ selected_data = self.hf_dataset.select(q_idx)
708
+ index_to_selected[q_idx_tuple] = selected_data
709
+ processed_indices.add(q_idx_tuple)
710
+ else:
711
+ selected_data = index_to_selected[q_idx_tuple]
712
+ result[key] = torch.stack(selected_data[key])
713
+ return result
714
+
715
+ # no videos
716
+ def get_episode_data(self, episode_id: int) -> dict:
717
+ ep_start = self.episode_data_index["from"][episode_id].item()
718
+ ep_end = self.episode_data_index["to"][episode_id].item()
719
+ q_idx = list(range(ep_start, ep_end))
720
+ # selected_data = self.hf_dataset.select(q_idx)
721
+ selected_data = self.hf_dataset[q_idx]
722
+ res_keys = self.meta.features.keys() - set(self.meta.video_keys)
723
+ res = {key : torch.stack(selected_data[key]) for key in res_keys}
724
+ return res
725
+
726
+ def _query_videos(self, query_timestamps: dict[str, list[float]], ep_idx: int) -> dict[str, torch.Tensor]:
727
+ """Note: When using data workers (e.g. DataLoader with num_workers>0), do not call this function
728
+ in the main process (e.g. by using a second Dataloader with num_workers=0). It will result in a
729
+ Segmentation Fault. This probably happens because a memory reference to the video loader is created in
730
+ the main process and a subprocess fails to access it.
731
+ """
732
+ item = {}
733
+ for vid_key, query_ts in query_timestamps.items():
734
+ video_path = self.root / self.meta.get_video_file_path(ep_idx, vid_key)
735
+ frames = decode_video_frames(video_path, query_ts, self.tolerance_s, self.video_backend)
736
+ item[vid_key] = frames.squeeze(0)
737
+
738
+ return item
739
+
740
+ def _add_padding_keys(self, item: dict, padding: dict[str, list[bool]]) -> dict:
741
+ for key, val in padding.items():
742
+ item[key] = torch.BoolTensor(val)
743
+ return item
744
+
745
+ def __len__(self):
746
+ return self.num_frames
747
+
748
+ def __getitem__(self, idx) -> dict:
749
+ item = self.hf_dataset[idx]
750
+ ep_idx = item["episode_index"].item()
751
+
752
+ query_indices = None
753
+ if self.delta_indices is not None:
754
+ current_ep_idx = self.episodes.index(ep_idx) if self.episodes is not None else ep_idx
755
+ query_indices, padding = self._get_query_indices(idx, current_ep_idx)
756
+ query_result = self._query_hf_dataset_fast(query_indices)
757
+ item = {**item, **padding}
758
+ for key, val in query_result.items():
759
+ item[key] = val
760
+
761
+ if len(self.meta.video_keys) > 0 and self.during_training:
762
+ current_ts = item["timestamp"].item()
763
+ query_timestamps = self._get_query_timestamps(current_ts, query_indices)
764
+ video_frames = self._query_videos(query_timestamps, ep_idx)
765
+ item = {**video_frames, **item}
766
+
767
+ if self.image_transforms is not None:
768
+ image_keys = self.meta.camera_keys
769
+ for cam in image_keys:
770
+ item[cam] = self.image_transforms(item[cam])
771
+
772
+ # Add task as a string
773
+ task_idx = item["task_index"].item()
774
+ item["task"] = self.meta.tasks[task_idx]
775
+ if "coarse_task_index" in item:
776
+ coarse_task_index = item["coarse_task_index"].item()
777
+ item["coarse_task"] = self.meta.tasks[coarse_task_index]
778
+
779
+ if "operating_hand_index" in item:
780
+ operating_hand_index = item["operating_hand_index"].item()
781
+ item["operating_hand"] = self.meta.tasks[operating_hand_index]
782
+
783
+ if "subtask_annotation" in item and hasattr(self.meta, "annotations"):
784
+ index = item["subtask_annotation"][0].item()
785
+ item["subtask"] = self.meta.annotations["subtask"][index]
786
+
787
+ if "atomic_task_index" in item and item["atomic_task_index"] is not None:
788
+ atomic_task_index = item["atomic_task_index"].item()
789
+ item["subtask"] = self.meta.tasks[int(atomic_task_index)]
790
+
791
+ return item
792
+
793
+ def __repr__(self):
794
+ feature_keys = list(self.features)
795
+ return (
796
+ f"{self.__class__.__name__}({{\n"
797
+ f" Repository ID: '{self.repo_id}',\n"
798
+ f" Number of selected episodes: '{self.num_episodes}',\n"
799
+ f" Number of selected samples: '{self.num_frames}',\n"
800
+ f" Features: '{feature_keys}',\n"
801
+ "})',\n"
802
+ )
803
+
804
+ def create_episode_buffer(self, episode_index: int | None = None) -> dict:
805
+ current_ep_idx = self.meta.total_episodes if episode_index is None else episode_index
806
+ ep_buffer = {}
807
+ # size and task are special cases that are not in self.features
808
+ ep_buffer["size"] = 0
809
+ ep_buffer["task"] = []
810
+ for key in self.features:
811
+ ep_buffer[key] = current_ep_idx if key == "episode_index" else []
812
+ return ep_buffer
813
+
814
+ def _get_image_file_path(self, episode_index: int, image_key: str, frame_index: int) -> Path:
815
+ fpath = DEFAULT_IMAGE_PATH.format(
816
+ image_key=image_key, episode_index=episode_index, frame_index=frame_index
817
+ )
818
+ return self.root / fpath
819
+
820
+ # def _save_image(self, image: torch.Tensor | np.ndarray | PIL.Image.Image | bytes, fpath: Path) -> None:
821
+ # if self.image_writer is None:
822
+ # if isinstance(image, torch.Tensor):
823
+ # image = image.cpu().numpy()
824
+ # write_image(image, fpath)
825
+ # else:
826
+ # self.image_writer.save_image(image=image, fpath=fpath)
827
+
828
+ def add_frame(self, frame: dict, task: List[str], timestamp: float | None = None) -> None:
829
+ """
830
+ This function only adds the frame to the episode_buffer. Apart from images — which are written in a
831
+ temporary directory — nothing is written to disk. To save those frames, the 'save_episode()' method
832
+ then needs to be called.
833
+ """
834
+ # Convert torch to numpy if needed
835
+ assert len(task) == 4, "Task frame must be of two elements"
836
+ for name in frame:
837
+ if isinstance(frame[name], torch.Tensor):
838
+ frame[name] = frame[name].numpy()
839
+
840
+ validate_frame(frame, self.features)
841
+
842
+ if self.episode_buffer is None:
843
+ self.episode_buffer = self.create_episode_buffer()
844
+
845
+ # Automatically add frame_index and timestamp to episode buffer
846
+ frame_index = self.episode_buffer["size"]
847
+ if timestamp is None:
848
+ timestamp = frame_index / self.fps
849
+ self.episode_buffer["frame_index"].append(frame_index)
850
+ self.episode_buffer["timestamp"].append(timestamp)
851
+ self.episode_buffer["task"].append(task)
852
+
853
+ # Add frame features to episode_buffer
854
+ for key in frame:
855
+ if key not in self.features:
856
+ raise ValueError(
857
+ f"An element of the frame is not in the features. '{key}' not in '{self.features.keys()}'."
858
+ )
859
+
860
+ if self.features[key]["dtype"] in ["image", "video"]:
861
+ img_path = self._get_image_file_path(
862
+ episode_index=self.episode_buffer["episode_index"], image_key=key, frame_index=frame_index
863
+ )
864
+ if frame_index == 0:
865
+ img_path.parent.mkdir(parents=True, exist_ok=True)
866
+ # self._save_image(frame[key], img_path)
867
+ self.episode_buffer[key].append(str(img_path))
868
+ else:
869
+ self.episode_buffer[key].append(frame[key])
870
+
871
+ self.episode_buffer["size"] += 1
872
+
873
+ def save_episode(self, episode_data: dict | None = None, raw_file_name: str | None = None) -> None:
874
+ """
875
+ This will save to disk the current episode in self.episode_buffer.
876
+
877
+ Args:
878
+ episode_data (dict | None, optional): Dict containing the episode data to save. If None, this will
879
+ save the current episode in self.episode_buffer, which is filled with 'add_frame'. Defaults to
880
+ None.
881
+ """
882
+ if not episode_data:
883
+ episode_buffer = self.episode_buffer
884
+
885
+ validate_episode_buffer(episode_buffer, self.meta.total_episodes, self.features)
886
+
887
+ # size and task are special cases that won't be added to hf_dataset
888
+ episode_length = episode_buffer.pop("size")
889
+ tasks = episode_buffer.pop("task")
890
+ episode_tasks = list(set([item for sublist in tasks for item in sublist]))
891
+ episode_index = episode_buffer["episode_index"]
892
+
893
+ episode_buffer["index"] = np.arange(self.meta.total_frames, self.meta.total_frames + episode_length)
894
+ episode_buffer["episode_index"] = np.full((episode_length,), episode_index)
895
+
896
+ # Add new tasks to the tasks dictionary
897
+ for task in episode_tasks:
898
+ task_index = self.meta.get_task_index(task)
899
+ if task_index is None:
900
+ self.meta.add_task(task)
901
+
902
+ # Given tasks in natural language, find their corresponding task indices
903
+ episode_buffer["coarse_task_index"] = np.array([self.meta.get_task_index(task[0]) for task in tasks])
904
+ episode_buffer["task_index"] = np.array([self.meta.get_task_index(task[1]) for task in tasks])
905
+ episode_buffer["coarse_quality_index"] = np.array([self.meta.get_task_index(task[2]) for task in tasks])
906
+ episode_buffer["quality_index"] = np.array([self.meta.get_task_index(task[3]) for task in tasks])
907
+
908
+ for key, ft in self.features.items():
909
+ # index, episode_index, task_index are already processed above, and image and video
910
+ # are processed separately by storing image path and frame info as meta data
911
+ if key in ["index", "episode_index", "coarse_task_index", "task_index", "coarse_quality_index", "quality_index"] or ft["dtype"] in ["image", "video"]:
912
+ continue
913
+ episode_buffer[key] = np.stack(episode_buffer[key])
914
+
915
+ self._wait_image_writer()
916
+ self._save_episode_table(episode_buffer, episode_index)
917
+ ep_stats = compute_episode_stats(episode_buffer, self.features, self.is_compute_episode_stats_image)
918
+
919
+ if len(self.meta.video_keys) > 0:
920
+ video_paths = self.encode_episode_videos(episode_index)
921
+ for key in self.meta.video_keys:
922
+ episode_buffer[key] = video_paths[key]
923
+
924
+ # `meta.save_episode` be executed after encoding the videos
925
+ self.meta.save_episode(episode_index, episode_length, episode_tasks, ep_stats, raw_file_name)
926
+
927
+ ep_data_index = get_episode_data_index(self.meta.episodes, [episode_index])
928
+ ep_data_index_np = {k: t.numpy() for k, t in ep_data_index.items()}
929
+ check_timestamps_sync(
930
+ episode_buffer["timestamp"],
931
+ episode_buffer["episode_index"],
932
+ ep_data_index_np,
933
+ self.fps,
934
+ self.tolerance_s,
935
+ )
936
+
937
+ video_files = list(self.root.rglob("*.mp4"))
938
+ assert len(video_files) == self.num_episodes * len(self.meta.video_keys)
939
+
940
+ parquet_files = list(self.root.rglob("*.parquet"))
941
+ assert len(parquet_files) == self.num_episodes
942
+
943
+ # delete images
944
+ img_dir = self.root / "images"
945
+ if img_dir.is_dir():
946
+ shutil.rmtree(self.root / "images")
947
+
948
+ if not episode_data: # Reset the buffer
949
+ self.episode_buffer = self.create_episode_buffer()
950
+
951
+ def _save_episode_table(self, episode_buffer: dict, episode_index: int) -> None:
952
+ episode_dict = {key: episode_buffer[key] for key in self.hf_features}
953
+ ep_dataset = datasets.Dataset.from_dict(episode_dict, features=self.hf_features, split="train")
954
+ ep_dataset = embed_images(ep_dataset)
955
+ self.hf_dataset = concatenate_datasets([self.hf_dataset, ep_dataset])
956
+ self.hf_dataset.set_transform(hf_transform_to_torch)
957
+ ep_data_path = self.root / self.meta.get_data_file_path(ep_index=episode_index)
958
+ ep_data_path.parent.mkdir(parents=True, exist_ok=True)
959
+ ep_dataset.to_parquet(ep_data_path)
960
+
961
+ def clear_episode_buffer(self) -> None:
962
+ episode_index = self.episode_buffer["episode_index"]
963
+ if self.image_writer is not None:
964
+ for cam_key in self.meta.camera_keys:
965
+ img_dir = self._get_image_file_path(
966
+ episode_index=episode_index, image_key=cam_key, frame_index=0
967
+ ).parent
968
+ if img_dir.is_dir():
969
+ shutil.rmtree(img_dir)
970
+
971
+ # Reset the buffer
972
+ self.episode_buffer = self.create_episode_buffer()
973
+
974
+ # def start_image_writer(self, num_processes: int = 0, num_threads: int = 4) -> None:
975
+ # if isinstance(self.image_writer, AsyncImageWriter):
976
+ # logging.warning(
977
+ # "You are starting a new AsyncImageWriter that is replacing an already existing one in the dataset."
978
+ # )
979
+
980
+ # self.image_writer = AsyncImageWriter(
981
+ # num_processes=num_processes,
982
+ # num_threads=num_threads,
983
+ # )
984
+
985
+ def stop_image_writer(self) -> None:
986
+ """
987
+ Whenever wrapping this dataset inside a parallelized DataLoader, this needs to be called first to
988
+ remove the image_writer in order for the LeRobotDataset object to be picklable and parallelized.
989
+ """
990
+ if self.image_writer is not None:
991
+ self.image_writer.stop()
992
+ self.image_writer = None
993
+
994
+ def _wait_image_writer(self) -> None:
995
+ """Wait for asynchronous image writer to finish."""
996
+ if self.image_writer is not None:
997
+ self.image_writer.wait_until_done()
998
+
999
+ def encode_videos(self) -> None:
1000
+ """
1001
+ Use ffmpeg to convert frames stored as png into mp4 videos.
1002
+ Note: `encode_video_frames` is a blocking call. Making it asynchronous shouldn't speedup encoding,
1003
+ since video encoding with ffmpeg is already using multithreading.
1004
+ """
1005
+ for ep_idx in range(self.meta.total_episodes):
1006
+ self.encode_episode_videos(ep_idx)
1007
+
1008
+ def encode_episode_videos(self, episode_index: int) -> dict:
1009
+ """
1010
+ Use ffmpeg to convert frames stored as png into mp4 videos.
1011
+ Note: `encode_video_frames` is a blocking call. Making it asynchronous shouldn't speedup encoding,
1012
+ since video encoding with ffmpeg is already using multithreading.
1013
+ """
1014
+ video_paths = {}
1015
+ for key in self.meta.video_keys:
1016
+ video_path = self.root / self.meta.get_video_file_path(episode_index, key)
1017
+ video_paths[key] = str(video_path)
1018
+ if video_path.is_file():
1019
+ # Skip if video is already encoded. Could be the case when resuming data recording.
1020
+ continue
1021
+ img_dir = self._get_image_file_path(
1022
+ episode_index=episode_index, image_key=key, frame_index=0
1023
+ ).parent
1024
+ encode_video_frames(img_dir, video_path, self.fps, overwrite=True, vcodec=self.video_codec)
1025
+
1026
+ return video_paths
1027
+
1028
+ @classmethod
1029
+ def create(
1030
+ cls,
1031
+ repo_id: str,
1032
+ fps: int,
1033
+ features: dict,
1034
+ root: str | Path | None = None,
1035
+ robot_type: str | None = None,
1036
+ use_videos: bool = True,
1037
+ tolerance_s: float = 1e-4,
1038
+ image_writer_processes: int = 0,
1039
+ image_writer_threads: int = 0,
1040
+ video_backend: str | None = None,
1041
+ video_codec: Literal["h264", "hevc", "libsvtav1", "h264_nvenc"] = "libsvtav1",
1042
+ is_compute_episode_stats_image = True,
1043
+ ) -> "LeRobotDataset":
1044
+ """Create a LeRobot Dataset from scratch in order to record data."""
1045
+ obj = cls.__new__(cls)
1046
+ obj.meta = LeRobotDatasetMetadata.create(
1047
+ repo_id=repo_id,
1048
+ fps=fps,
1049
+ robot_type=robot_type,
1050
+ features=features,
1051
+ root=root,
1052
+ use_videos=use_videos,
1053
+ )
1054
+ obj.repo_id = obj.meta.repo_id
1055
+ obj.root = obj.meta.root
1056
+ obj.revision = None
1057
+ obj.tolerance_s = tolerance_s
1058
+ obj.image_writer = None
1059
+
1060
+ # if image_writer_processes or image_writer_threads:
1061
+ # obj.start_image_writer(image_writer_processes, image_writer_threads)
1062
+
1063
+ # TODO(aliberts, rcadene, alexander-soare): Merge this with OnlineBuffer/DataBuffer
1064
+ obj.episode_buffer = obj.create_episode_buffer()
1065
+
1066
+ obj.episodes = None
1067
+ obj.hf_dataset = obj.create_hf_dataset()
1068
+ obj.image_transforms = None
1069
+ obj.delta_timestamps = None
1070
+ obj.delta_indices = None
1071
+ obj.episode_data_index = None
1072
+ obj.video_backend = video_backend if video_backend is not None else get_safe_default_codec()
1073
+ obj.video_codec = video_codec
1074
+ obj.is_compute_episode_stats_image = is_compute_episode_stats_image
1075
+ return obj
1076
+
1077
+
1078
+ class MultiLeRobotDataset(torch.utils.data.Dataset):
1079
+ """A dataset consisting of multiple underlying `LeRobotDataset`s.
1080
+
1081
+ The underlying `LeRobotDataset`s are effectively concatenated, and this class adopts much of the API
1082
+ structure of `LeRobotDataset`.
1083
+ """
1084
+
1085
+ def __init__(
1086
+ self,
1087
+ dataset_dirs: list[str],
1088
+ episodes: dict | None = None,
1089
+ image_transforms: Callable | None = None,
1090
+ delta_timestamps: dict[list[float]] | None = None,
1091
+ tolerances_s: dict | None = None,
1092
+ download_videos: bool = True,
1093
+ video_backend: str | None = None,
1094
+ ):
1095
+ super().__init__()
1096
+ self.dataset_dirs = dataset_dirs
1097
+ ds_roots = [Path(ds_dir) for ds_dir in dataset_dirs]
1098
+ ds_names = [ds_dir for ds_dir in dataset_dirs]
1099
+ self.ds_names = ds_names
1100
+ self.ds_roots = ds_roots
1101
+ self.tolerances_s = tolerances_s if tolerances_s else dict.fromkeys(ds_names, 0.0001)
1102
+ # Construct the underlying datasets passing everything but `transform` and `delta_timestamps` which
1103
+ # are handled by this class.
1104
+ self._datasets = []
1105
+ for ds_root, ds_name in zip(ds_roots, ds_names, strict=True):
1106
+ try:
1107
+ _dataset = LeRobotDataset(
1108
+ ds_name,
1109
+ root=ds_root,
1110
+ episodes=episodes[ds_name] if episodes else None,
1111
+ image_transforms=image_transforms,
1112
+ delta_timestamps=delta_timestamps,
1113
+ tolerance_s=self.tolerances_s[ds_name],
1114
+ download_videos=download_videos,
1115
+ video_backend=video_backend,
1116
+ )
1117
+ self._datasets.append(_dataset)
1118
+ except Exception as e:
1119
+ # logging.error(e)
1120
+ logging.error(f"Exception while process ds_root: {ds_root}, ds_name: {ds_name}")
1121
+ traceback.print_exc()
1122
+ raise e
1123
+
1124
+ # Disable any data keys that are not common across all of the datasets. Note: we may relax this
1125
+ # restriction in future iterations of this class. For now, this is necessary at least for being able
1126
+ # to use PyTorch's default DataLoader collate function.
1127
+ self.disabled_features = set()
1128
+ intersection_features = set(self._datasets[0].features)
1129
+ for ds in self._datasets:
1130
+ intersection_features.intersection_update(ds.features)
1131
+ if len(intersection_features) == 0:
1132
+ raise RuntimeError(
1133
+ "Multiple datasets were provided but they had no keys common to all of them. "
1134
+ "The multi-dataset functionality currently only keeps common keys."
1135
+ )
1136
+ for ds_name, ds in zip(self.ds_names, self._datasets, strict=True):
1137
+ # Disable warning of empty extra keys do
1138
+ extra_keys = set(ds.features).difference(intersection_features)
1139
+ if extra_keys:
1140
+ logging.warning(
1141
+ f"keys {extra_keys} of {ds_name} were disabled as they are not contained in all the "
1142
+ "other datasets."
1143
+ )
1144
+ self.disabled_features.update(extra_keys)
1145
+
1146
+ self.image_transforms = image_transforms
1147
+ self.delta_timestamps = delta_timestamps
1148
+ # TODO(rcadene, aliberts): We should not perform this aggregation for datasets
1149
+ # with multiple robots of different ranges. Instead we should have one normalization
1150
+ # per robot.
1151
+ self.stats = aggregate_stats([dataset.meta.stats for dataset in self._datasets])
1152
+
1153
+ def set_during_training(self, during_training: bool):
1154
+ for dataset in self._datasets:
1155
+ dataset.during_training = during_training
1156
+
1157
+ @property
1158
+ def repo_id_to_index(self):
1159
+ """Return a mapping from dataset repo_id to a dataset index automatically created by this class.
1160
+
1161
+ This index is incorporated as a data key in the dictionary returned by `__getitem__`.
1162
+ """
1163
+ return {repo_id: i for i, repo_id in enumerate(self.ds_names)}
1164
+
1165
+ @property
1166
+ def repo_index_to_id(self):
1167
+ """Return the inverse mapping if repo_id_to_index."""
1168
+ return {v: k for k, v in self.repo_id_to_index}
1169
+
1170
+ @property
1171
+ def fps(self) -> int:
1172
+ """Frames per second used during data collection.
1173
+
1174
+ NOTE: Fow now, this relies on a check in __init__ to make sure all sub-datasets have the same info.
1175
+ """
1176
+ return self._datasets[0].meta.info["fps"]
1177
+
1178
+ @property
1179
+ def video(self) -> bool:
1180
+ """Returns True if this dataset loads video frames from mp4 files.
1181
+
1182
+ Returns False if it only loads images from png files.
1183
+
1184
+ NOTE: Fow now, this relies on a check in __init__ to make sure all sub-datasets have the same info.
1185
+ """
1186
+ return self._datasets[0].meta.info.get("video", False)
1187
+
1188
+ @property
1189
+ def features(self) -> datasets.Features:
1190
+ features = {}
1191
+ for dataset in self._datasets:
1192
+ features.update({k: v for k, v in dataset.hf_features.items() if k not in self.disabled_features})
1193
+ return features
1194
+
1195
+ @property
1196
+ def camera_keys(self) -> list[str]:
1197
+ """Keys to access image and video stream from cameras."""
1198
+ keys = []
1199
+ for key, feats in self.features.items():
1200
+ if isinstance(feats, (datasets.Image, VideoFrame)):
1201
+ keys.append(key)
1202
+ return keys
1203
+
1204
+ @property
1205
+ def video_frame_keys(self) -> list[str]:
1206
+ """Keys to access video frames that requires to be decoded into images.
1207
+
1208
+ Note: It is empty if the dataset contains images only,
1209
+ or equal to `self.cameras` if the dataset contains videos only,
1210
+ or can even be a subset of `self.cameras` in a case of a mixed image/video dataset.
1211
+ """
1212
+ video_frame_keys = []
1213
+ for key, feats in self.features.items():
1214
+ if isinstance(feats, VideoFrame):
1215
+ video_frame_keys.append(key)
1216
+ return video_frame_keys
1217
+
1218
+ @property
1219
+ def num_frames(self) -> int:
1220
+ """Number of samples/frames."""
1221
+ return sum(d.num_frames for d in self._datasets)
1222
+
1223
+ @property
1224
+ def num_episodes(self) -> int:
1225
+ """Number of episodes."""
1226
+ return sum(d.num_episodes for d in self._datasets)
1227
+
1228
+ @property
1229
+ def tolerance_s(self) -> float:
1230
+ """Tolerance in seconds used to discard loaded frames when their timestamps
1231
+ are not close enough from the requested frames. It is only used when `delta_timestamps`
1232
+ is provided or when loading video frames from mp4 files.
1233
+ """
1234
+ # 1e-4 to account for possible numerical error
1235
+ return 1 / self.fps - 1e-4
1236
+
1237
+ def get_episode_data(self, episode_idx: int) -> dict:
1238
+ for dataset in self._datasets:
1239
+ if episode_idx < dataset.num_episodes:
1240
+ file = str(dataset.root / dataset.meta.get_data_file_path(dataset.episodes[episode_idx]))
1241
+ table = pq.read_table(str(file))
1242
+
1243
+ result_dict = {}
1244
+ for col_name in table.column_names:
1245
+ col = table[col_name]
1246
+ try:
1247
+ np_arr = col.to_numpy(zero_copy_only=True)
1248
+ except Exception:
1249
+ raw = col.to_numpy()
1250
+ np_arr = np.stack(raw) if raw.dtype == object else raw
1251
+ with warnings.catch_warnings():
1252
+ warnings.filterwarnings(
1253
+ "ignore",
1254
+ message="The given NumPy array is not writable",
1255
+ category=UserWarning,
1256
+ )
1257
+ # deal with string in parquet file
1258
+ if np_arr.dtype == 'O':
1259
+ result_dict[col_name] = np_arr
1260
+ else:
1261
+ result_dict[col_name] = torch.from_numpy(np_arr)
1262
+ return result_dict
1263
+ else:
1264
+ episode_idx -= dataset.num_episodes
1265
+ raise IndexError(f"Episode index {episode_idx} out of bounds.")
1266
+
1267
+ def __len__(self):
1268
+ return self.num_frames
1269
+
1270
+ def __getitem__(self, idx: int) -> dict[str, torch.Tensor]:
1271
+ if idx >= len(self):
1272
+ raise IndexError(f"Index {idx} out of bounds.")
1273
+ # Determine which dataset to get an item from based on the index.
1274
+ start_idx = 0
1275
+ dataset_idx = 0
1276
+ for dataset in self._datasets:
1277
+ if idx >= start_idx + dataset.num_frames:
1278
+ start_idx += dataset.num_frames
1279
+ dataset_idx += 1
1280
+ continue
1281
+ break
1282
+ else:
1283
+ raise AssertionError("We expect the loop to break out as long as the index is within bounds.")
1284
+ item = self._datasets[dataset_idx][idx - start_idx]
1285
+ item["dataset_index"] = torch.tensor(dataset_idx)
1286
+ for data_key in self.disabled_features:
1287
+ if data_key in item:
1288
+ del item[data_key]
1289
+
1290
+ return item
1291
+
1292
+ def __repr__(self):
1293
+ return (
1294
+ f"{self.__class__.__name__}(\n"
1295
+ f" Dataset Names: '{self.ds_names}',\n"
1296
+ f" Number of Samples: {self.num_frames},\n"
1297
+ f" Number of Episodes: {self.num_episodes},\n"
1298
+ f" Type: {'video (.mp4)' if self.video else 'image (.png)'},\n"
1299
+ f" Recorded Frames per Second: {self.fps},\n"
1300
+ f" Camera Keys: {self.camera_keys},\n"
1301
+ f" Video Frame Keys: {self.video_frame_keys if self.video else 'N/A'},\n"
1302
+ f" Transformations: {self.image_transforms},\n"
1303
+ f")"
1304
+ )
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/base_processor.py ADDED
@@ -0,0 +1,263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from abc import ABC, abstractmethod
2
+ from typing import Dict, Any, Optional, List, Literal
3
+
4
+ import torch
5
+ import numpy as np
6
+ from copy import deepcopy
7
+ from ..utils.normalizer import LinearNormalizer, NormMode
8
+ from fastwam.utils.pytorch_utils import dict_apply
9
+
10
+
11
+ class BaseProcessor(ABC):
12
+ def __init__(
13
+ self,
14
+ # keys
15
+ shape_meta: Dict[str, Any],
16
+ num_obs_steps: int,
17
+ num_output_cameras: int,
18
+ action_output_dim: int,
19
+ proprio_output_dim: int,
20
+
21
+ action_state_transforms: Optional[List[Any]],
22
+
23
+ # action & state normalization
24
+ use_stepwise_action_norm: bool,
25
+ norm_default_mode: NormMode,
26
+ norm_exception_mode: Dict[str, Dict[str, NormMode]],
27
+
28
+ action_state_merger,
29
+
30
+ # image transform
31
+ train_transforms: Dict[str, List[Any]] | None,
32
+ val_transforms: Dict[str, List[Any]] | None,
33
+
34
+ # instruction transform
35
+ drop_high_level_prob: float,
36
+ use_zh_instruction: bool,
37
+
38
+ tokenizer: Any
39
+ ):
40
+ self.shape_meta = shape_meta
41
+ self.num_obs_steps = num_obs_steps
42
+ self.num_output_cameras = num_output_cameras
43
+ self.action_output_dim = action_output_dim
44
+ self.proprio_output_dim = proprio_output_dim
45
+
46
+ self.drop_high_level_prob = drop_high_level_prob
47
+ self.use_zh_instruction = use_zh_instruction
48
+
49
+ # image
50
+ self.train_transforms = train_transforms
51
+ self.val_transforms = val_transforms
52
+
53
+ self._is_train = None
54
+
55
+ self.action_state_transforms = action_state_transforms
56
+ self.action_state_merger = action_state_merger
57
+ self.action_state_merger.set_shape_meta(self.shape_meta)
58
+
59
+ self.use_stepwise_action_norm = use_stepwise_action_norm
60
+ self.norm_default_mode = norm_default_mode
61
+ self.norm_exception_mode = norm_exception_mode
62
+ self._normalizer = None
63
+
64
+ self.tokenizer = tokenizer
65
+
66
+ @property
67
+ def is_train(self):
68
+ if self._is_train is None:
69
+ raise ValueError("is_train has not been set. Please call train() and eval() first.")
70
+ return self._is_train
71
+
72
+ @property
73
+ def normalizer(self) -> LinearNormalizer:
74
+ if self._normalizer is None:
75
+ raise ValueError("normalizer has not been set. Please call set_normalizer_from_stats() first.")
76
+ return self._normalizer
77
+
78
+ def train(self):
79
+ self._is_train = True
80
+ return self
81
+
82
+ def eval(self):
83
+ self._is_train = False
84
+ return self
85
+
86
+ def set_normalizer_from_stats(self, dataset_stats: Dict[str, Any] = None):
87
+ self._normalizer = LinearNormalizer(
88
+ use_stepwise_action_norm=self.use_stepwise_action_norm,
89
+ shape_meta=self.shape_meta,
90
+ default_mode=self.norm_default_mode,
91
+ exception_mode=self.norm_exception_mode,
92
+ stats=dataset_stats,
93
+ )
94
+
95
+ def augment_instruction(self, data: Dict[str, str] | List[str]) -> List[str]:
96
+ """
97
+ Args:
98
+ data: Dict[str, str] | List[str], lerobot sample in raw mcap
99
+
100
+ Returns:
101
+ List[str], processed instructions
102
+ """
103
+ # if single instruction, convert to list
104
+ if "coarse_task" in data:
105
+ high_level_instruction = data["coarse_task"]
106
+ else:
107
+ high_level_instruction = ""
108
+ if "task" not in data:
109
+ return f"[high] {high_level_instruction}"
110
+
111
+ low_level_instruction = data["task"]
112
+ # Galaxea lerobot use @ to split Chinese and English instruction
113
+ if "@" in low_level_instruction:
114
+ zh, eng = low_level_instruction.split("@")
115
+ low_level_instruction = zh if self.use_zh_instruction else eng
116
+
117
+ if np.random.rand() < self.drop_high_level_prob:
118
+ instruction = f"[Low]: {low_level_instruction}"
119
+ else:
120
+ instruction = f"[High]: {high_level_instruction}, [Low]: {low_level_instruction}"
121
+
122
+ return instruction
123
+
124
+ def action_state_transform(self, batch):
125
+ if "action" in batch:
126
+ for meta in self.shape_meta["action"]:
127
+ k, meta_shape = meta["key"], meta["raw_shape"]
128
+ actual_shape = batch["action"][k].shape[-1]
129
+ assert actual_shape == meta_shape, \
130
+ f"Action key {k} actual raw shape {actual_shape} mismatch with meta raw shape {meta_shape}."
131
+
132
+ for meta in self.shape_meta["state"]:
133
+ k, meta_shape = meta["key"], meta["raw_shape"]
134
+ actual_shape = batch["state"][k].shape[-1]
135
+ assert actual_shape == meta_shape, \
136
+ f"State key {k} actual raw shape {actual_shape} mismatch with meta raw shape{meta_shape}."
137
+
138
+ if self.action_state_transforms is not None:
139
+ for trans in self.action_state_transforms:
140
+ batch = trans.forward(batch)
141
+
142
+ if "action" in batch:
143
+ for meta in self.shape_meta["action"]:
144
+ k, meta_shape = meta["key"], meta["shape"]
145
+ actual_shape = batch["action"][k].shape[-1]
146
+ assert actual_shape == meta_shape, \
147
+ f"Action key {k} actual transformed shape {actual_shape} mismatch with meta shape {meta_shape}."
148
+
149
+ for meta in self.shape_meta["state"]:
150
+ k, meta_shape = meta["key"], meta["shape"]
151
+ actual_shape = batch["state"][k].shape[-1]
152
+ assert actual_shape == meta_shape, \
153
+ f"State key {k} actual transformed shape {actual_shape} mismatch with meta raw shape {meta_shape}."
154
+
155
+ return batch
156
+
157
+ def preprocess(self, data: Dict[str, Any]) -> Dict[str, Any]:
158
+ """
159
+ Preprocess the data for the policy model.
160
+
161
+ Args:
162
+ Data: Dict[str, Any], lerobot sample in raw mcap obtained from dataset __getitem__:
163
+ - "action": Optional, Dict[str, torch.Tensor] -> [action_horizon, action_dim]
164
+ - "state": Dict[str, torch.Tensor] -> [num_obs_steps, state_dim]
165
+ - "images": Dict[str, torch.Tensor] -> [num_obs_steps, C, H, W]
166
+ - "action_is_pad": Optional, torch.Tensor -> [action_horizon,]
167
+ - "state_is_pad": torch.Tensor -> [num_obs_steps,]
168
+ - "image_is_pad": torch.Tensor -> [num_obs_steps,]
169
+ - "idx": int, sample index
170
+
171
+ Returns:
172
+ Sample: Dict[str, Any], which can collated:
173
+ - "input_ids": torch.Tensor -> [max_image_text_tokens,]
174
+ - "attention_mask": torch.Tensor -> [max_image_text_tokens,]
175
+ - "pixel_values": torch.Tensor -> [num_input_cameras, C, H, W]
176
+ - "image_is_pad": torch.Tensor -> [num_obs_steps,]
177
+ - "proprio": torch.Tensor -> [num_obs_steps, proprio_dim]
178
+ - "state_is_pad": torch.Tensor -> [num_obs_steps,]
179
+ - "action": Optional, torch.Tensor -> [action_horizon, action_dim]
180
+ - "action_is_pad": Optional, torch.Tensor -> [action_horizon,]
181
+ - "gt_action: Optional, deepcopy of input action for open loop eval, which is left untouched
182
+ - "idx": int, sample index
183
+ """
184
+ sample = {}
185
+ # 1. instruction
186
+ sample["instruction"] = self.augment_instruction(data)
187
+ sample["image_is_pad"] = data["image_is_pad"]
188
+
189
+ # 2. image
190
+ processed_images = []
191
+ for meta in self.shape_meta["images"]:
192
+ key, shape = meta["key"], meta["shape"]
193
+ image = data["images"][key] # [num_obs_steps, C, H, W]
194
+ assert image.ndim == 4, f"Expected 4 dimensions (num_obs_steps, C, H, W), got shape {image.shape}"
195
+
196
+ # Apply transforms efficiently on the merged batch
197
+ transforms = self.train_transforms if self.is_train else self.val_transforms
198
+ for trans in transforms[key]:
199
+ image = trans(image)
200
+
201
+ meta_shape = [self.num_obs_steps] + shape
202
+ assert image.shape == meta_shape, \
203
+ f"Expected shape {meta_shape}, got {image.shape} after transforms for key {key}"
204
+
205
+ processed_images.append(image)
206
+
207
+ pixel_values = torch.cat(processed_images, dim=0) # [num_input_cameras, C, H, W]
208
+ if self.num_output_cameras > pixel_values.shape[0]:
209
+ out = torch.zeros((self.num_output_cameras,) + pixel_values.shape[1:], device=pixel_values.device, dtype=pixel_values.dtype)
210
+ out[0: pixel_values.shape[0]] = pixel_values
211
+ sample["pixel_values"] = out
212
+ else:
213
+ sample["pixel_values"] = pixel_values
214
+
215
+ # Copy action before transform for open-loop evaluation,
216
+ # disabled for training dataset as it may cause collating key problem.
217
+ if not self.is_train and "action" in data:
218
+ sample["gt_action"] = deepcopy(data["action"])
219
+
220
+ # 3. action & state
221
+ data = self.action_state_transform(data)
222
+ data = self.normalizer.forward(data)
223
+ data = self.action_state_merger.forward(data)
224
+
225
+ if "action" in data:
226
+ sample["action"] = data["action"] # [action_horizon, action_dim]
227
+ sample["action_is_pad"] = data["action_is_pad"] # [action_horizon,]
228
+ sample["action_dim_is_pad"] = data["action_dim_is_pad"] # [action_dim,]
229
+ assert sample["action"].shape[-1] == self.action_output_dim
230
+
231
+ # TODO: rename all "state" into "proprio"
232
+ sample["proprio"] = data["state"] # [num_obs_steps, proprio_dim]
233
+ sample["proprio_is_pad"] = data["state_is_pad"] # [num_obs_steps,]
234
+ sample["proprio_dim_is_pad"] = data["state_dim_is_pad"] # [proprio_dim,]
235
+ assert sample["proprio"].shape[-1] == self.proprio_output_dim
236
+
237
+ sample["idx"] = data["idx"]
238
+
239
+ sample = self.tokenizer(sample)
240
+
241
+ return sample
242
+
243
+ def postprocess(self, data: Dict[str, Any]) -> Dict[str, Any]:
244
+ """
245
+ Postprocess the data for the policy model.
246
+
247
+ Args:
248
+ data: Dict[str, Any], lerobot sample in raw mcap
249
+
250
+ Returns:
251
+ data: Dict[str, Any], processed data including unnormalized action
252
+ """
253
+ assert "action" in data, "Action is required in postprocess"
254
+ data["state"] = data.pop("proprio")
255
+ data = self.action_state_merger.backward(data)
256
+ data = self.normalizer.backward(data)
257
+ if self.action_state_transforms is not None:
258
+ for trans in reversed(self.action_state_transforms):
259
+ data = trans.backward(data)
260
+
261
+ start_obs_step = self.num_obs_steps - 1
262
+ data["action"] = dict_apply(data["action"], lambda x: x[:, start_obs_step:, :])
263
+ return data
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/fastwam_processor.py ADDED
@@ -0,0 +1,304 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from abc import ABC, abstractmethod
2
+ from typing import Dict, Any, Optional, List, Literal
3
+
4
+ import torch
5
+ import numpy as np
6
+ from copy import deepcopy
7
+ from ..utils.normalizer import LinearNormalizer, NormMode
8
+ from fastwam.utils.pytorch_utils import dict_apply
9
+ from fastwam.utils.logging_config import get_logger
10
+ from .base_processor import BaseProcessor
11
+
12
+ logger = get_logger(__name__)
13
+
14
+ class FastWAMProcessor(BaseProcessor):
15
+ def __init__(
16
+ self,
17
+ # keys
18
+ shape_meta: Dict[str, Any],
19
+ num_obs_steps: int,
20
+ num_output_cameras: int,
21
+ action_output_dim: int,
22
+ proprio_output_dim: int,
23
+
24
+ action_state_transforms: Optional[List[Any]],
25
+
26
+ # action & state normalization
27
+ use_stepwise_action_norm: bool,
28
+ norm_default_mode: NormMode,
29
+ norm_exception_mode: Dict[str, Dict[str, NormMode]],
30
+
31
+ action_state_merger,
32
+
33
+ # image transform
34
+ train_transforms: Dict[str, List[Any]] | None,
35
+ val_transforms: Dict[str, List[Any]] | None,
36
+
37
+ # instruction transform
38
+ drop_high_level_prob: float = 1.0,
39
+ use_zh_instruction: bool = False,
40
+
41
+ tokenizer: Optional[Any] = None,
42
+ delta_action_dim_mask: Optional[Dict[str, List[bool]]] = None,
43
+ ):
44
+ self.shape_meta = shape_meta
45
+ self.num_obs_steps = num_obs_steps
46
+ self.num_output_cameras = num_output_cameras
47
+ self.action_output_dim = action_output_dim
48
+ self.proprio_output_dim = proprio_output_dim
49
+
50
+ self.drop_high_level_prob = drop_high_level_prob
51
+ self.use_zh_instruction = use_zh_instruction
52
+
53
+ # image
54
+ self.train_transforms = train_transforms
55
+ self.val_transforms = val_transforms
56
+
57
+ self._is_train = None
58
+
59
+ self.action_state_transforms = action_state_transforms
60
+ self.action_state_merger = action_state_merger
61
+ self.action_state_merger.set_shape_meta(self.shape_meta)
62
+
63
+ self.use_stepwise_action_norm = use_stepwise_action_norm
64
+ self.norm_default_mode = norm_default_mode
65
+ self.norm_exception_mode = norm_exception_mode
66
+ self._normalizer = None
67
+
68
+ self.tokenizer = tokenizer
69
+ if delta_action_dim_mask is None:
70
+ self.delta_action_dim_mask = None
71
+ else:
72
+ action_meta = self.shape_meta["action"]
73
+ expected_keys = [m["key"] for m in action_meta]
74
+ provided_keys = list(delta_action_dim_mask.keys())
75
+ if set(provided_keys) != set(expected_keys):
76
+ raise ValueError(
77
+ f"`delta_action_dim_mask` keys mismatch. Expected {expected_keys}, got {provided_keys}."
78
+ )
79
+
80
+ self.delta_action_dim_mask = {}
81
+ for meta in action_meta:
82
+ key = meta["key"]
83
+ expected_dim = meta["shape"]
84
+ mask = delta_action_dim_mask[key]
85
+ if len(mask) != expected_dim:
86
+ raise ValueError(
87
+ f"`delta_action_dim_mask[{key}]` length must be {expected_dim}, got {len(mask)}."
88
+ )
89
+ self.delta_action_dim_mask[key] = torch.as_tensor(mask, dtype=torch.bool)
90
+
91
+ @property
92
+ def is_train(self):
93
+ if self._is_train is None:
94
+ raise ValueError("is_train has not been set. Please call train() and eval() first.")
95
+ return self._is_train
96
+
97
+ @property
98
+ def normalizer(self) -> LinearNormalizer:
99
+ if self._normalizer is None:
100
+ raise ValueError("normalizer has not been set. Please call set_normalizer_from_stats() first.")
101
+ return self._normalizer
102
+
103
+ def train(self):
104
+ self._is_train = True
105
+ return self
106
+
107
+ def eval(self):
108
+ self._is_train = False
109
+ return self
110
+
111
+ def set_normalizer_from_stats(self, dataset_stats: Dict[str, Any] = None):
112
+ self._normalizer = LinearNormalizer(
113
+ use_stepwise_action_norm=self.use_stepwise_action_norm,
114
+ shape_meta=self.shape_meta,
115
+ default_mode=self.norm_default_mode,
116
+ exception_mode=self.norm_exception_mode,
117
+ stats=dataset_stats,
118
+ )
119
+
120
+ def augment_instruction(self, data: Dict[str, str] | List[str]) -> List[str]:
121
+ """
122
+ Args:
123
+ data: Dict[str, str] | List[str], lerobot sample in raw mcap
124
+
125
+ Returns:
126
+ List[str], processed instructions
127
+ """
128
+ # if single instruction, convert to list
129
+ if "coarse_task" in data:
130
+ high_level_instruction = data["coarse_task"]
131
+ else:
132
+ high_level_instruction = ""
133
+ if "task" not in data:
134
+ return f"[high] {high_level_instruction}"
135
+
136
+ low_level_instruction = data["task"]
137
+ # Galaxea lerobot use @ to split Chinese and English instruction
138
+ if "@" in low_level_instruction:
139
+ zh, eng = low_level_instruction.split("@")
140
+ low_level_instruction = zh if self.use_zh_instruction else eng
141
+
142
+ if np.random.rand() < self.drop_high_level_prob:
143
+ instruction = f"{low_level_instruction}"
144
+ else:
145
+ instruction = f"[High]: {high_level_instruction}, [Low]: {low_level_instruction}"
146
+
147
+ return instruction
148
+
149
+ def action_state_transform(self, batch):
150
+ if "action" in batch:
151
+ for meta in self.shape_meta["action"]:
152
+ k, meta_shape = meta["key"], meta["raw_shape"]
153
+ actual_shape = batch["action"][k].shape[-1]
154
+ assert actual_shape == meta_shape, \
155
+ f"Action key {k} actual raw shape {actual_shape} mismatch with meta raw shape {meta_shape}."
156
+
157
+ for meta in self.shape_meta["state"]:
158
+ k, meta_shape = meta["key"], meta["raw_shape"]
159
+ actual_shape = batch["state"][k].shape[-1]
160
+ assert actual_shape == meta_shape, \
161
+ f"State key {k} actual raw shape {actual_shape} mismatch with meta raw shape {meta_shape}."
162
+
163
+ if self.action_state_transforms is not None:
164
+ for trans in self.action_state_transforms:
165
+ batch = trans.forward(batch)
166
+
167
+ if "action" in batch:
168
+ for meta in self.shape_meta["action"]:
169
+ k, meta_shape = meta["key"], meta["shape"]
170
+ actual_shape = batch["action"][k].shape[-1]
171
+ assert actual_shape == meta_shape, \
172
+ f"Action key {k} actual transformed shape {actual_shape} mismatch with meta shape {meta_shape}."
173
+
174
+ for meta in self.shape_meta["state"]:
175
+ k, meta_shape = meta["key"], meta["shape"]
176
+ actual_shape = batch["state"][k].shape[-1]
177
+ assert actual_shape == meta_shape, \
178
+ f"State key {k} actual transformed shape {actual_shape} mismatch with meta raw shape {meta_shape}."
179
+
180
+ return batch
181
+
182
+ def preprocess(self, data: Dict[str, Any]) -> Dict[str, Any]:
183
+ """
184
+ Preprocess the data for the policy model.
185
+
186
+ Args:
187
+ Data: Dict[str, Any], lerobot sample in raw mcap obtained from dataset __getitem__:
188
+ - "action": Optional, Dict[str, torch.Tensor] -> [action_horizon, action_dim]
189
+ - "state": Dict[str, torch.Tensor] -> [num_obs_steps, state_dim]
190
+ - "images": Dict[str, torch.Tensor] -> [num_obs_steps, C, H, W]
191
+ - "action_is_pad": Optional, torch.Tensor -> [action_horizon,]
192
+ - "state_is_pad": torch.Tensor -> [num_obs_steps,]
193
+ - "image_is_pad": torch.Tensor -> [num_obs_steps,]
194
+ - "idx": int, sample index
195
+
196
+ Returns:
197
+ Sample: Dict[str, Any], which can collated:
198
+ - "input_ids": torch.Tensor -> [max_image_text_tokens,]
199
+ - "attention_mask": torch.Tensor -> [max_image_text_tokens,]
200
+ - "pixel_values": torch.Tensor -> [num_input_cameras, C, H, W]
201
+ - "image_is_pad": torch.Tensor -> [num_obs_steps,]
202
+ - "proprio": torch.Tensor -> [num_obs_steps, proprio_dim]
203
+ - "state_is_pad": torch.Tensor -> [num_obs_steps,]
204
+ - "action": Optional, torch.Tensor -> [action_horizon, action_dim]
205
+ - "action_is_pad": Optional, torch.Tensor -> [action_horizon,]
206
+ - "gt_action: Optional, deepcopy of input action for open loop eval, which is left untouched
207
+ - "idx": int, sample index
208
+ """
209
+ sample = {}
210
+ # 1. instruction
211
+ sample["instruction"] = self.augment_instruction(data)
212
+ sample["image_is_pad"] = data["image_is_pad"]
213
+
214
+ # 2. image
215
+ processed_images = []
216
+ for meta in self.shape_meta["images"]:
217
+ key, shape = meta["key"], meta["shape"]
218
+ image = data["images"][key] # [num_obs_steps, C, H, W]
219
+ assert image.ndim == 4, f"Expected 4 dimensions (num_obs_steps, C, H, W), got shape {image.shape}"
220
+
221
+ # Apply transforms efficiently on the merged batch
222
+ transforms = self.train_transforms if self.is_train else self.val_transforms
223
+ current_transforms = transforms[key] if isinstance(transforms, dict) else transforms
224
+ for trans in current_transforms:
225
+ image = trans(image)
226
+
227
+ meta_shape = [self.num_obs_steps] + shape
228
+ assert image.shape == meta_shape, \
229
+ f"Expected shape {meta_shape}, got {image.shape} after transforms for key {key}"
230
+
231
+ processed_images.append(image)
232
+ pixel_values = torch.stack(processed_images, dim=0) # [num_input_cameras, T, C, H, W]
233
+
234
+ if self.num_output_cameras > pixel_values.shape[0]:
235
+ out = torch.zeros((self.num_output_cameras,) + pixel_values.shape[1:], device=pixel_values.device, dtype=pixel_values.dtype)
236
+ out[0: pixel_values.shape[0]] = pixel_values
237
+ sample["pixel_values"] = out
238
+ elif self.num_output_cameras < pixel_values.shape[0]:
239
+ logger.warning(f"num_output_cameras {self.num_output_cameras} is less than the number of cameras in data {pixel_values.shape[0]}, "
240
+ f"truncating the input to the first {self.num_output_cameras} cameras.")
241
+ sample["pixel_values"] = pixel_values[:self.num_output_cameras]
242
+ else:
243
+ sample["pixel_values"] = pixel_values
244
+
245
+ # Copy action before transform for open-loop evaluation,
246
+ # disabled for training dataset as it may cause collating key problem.
247
+ if not self.is_train and "action" in data:
248
+ sample["gt_action"] = deepcopy(data["action"])
249
+
250
+ # 3. action & state
251
+ if "action" in data and self.delta_action_dim_mask is not None:
252
+ action_is_pad = torch.as_tensor(data["action_is_pad"], dtype=torch.bool)
253
+ if bool(action_is_pad.any().item()):
254
+ for key, dim_mask in self.delta_action_dim_mask.items():
255
+ cur_action = data["action"][key]
256
+ cur_action_is_pad = action_is_pad.to(device=cur_action.device)
257
+ cur_dim_mask = dim_mask.to(device=cur_action.device)
258
+ pad_delta_mask = cur_action_is_pad.unsqueeze(1) & cur_dim_mask.unsqueeze(0)
259
+ cur_action[pad_delta_mask] = 0.0
260
+ data = self.action_state_transform(data)
261
+ data = self.normalizer.forward(data)
262
+ data = self.action_state_merger.forward(data)
263
+
264
+ if "action" in data:
265
+ sample["action"] = data["action"] # [action_horizon, action_dim]
266
+ sample["action_is_pad"] = data["action_is_pad"] # [action_horizon,]
267
+ sample["action_dim_is_pad"] = data["action_dim_is_pad"] # [action_dim,]
268
+ assert sample["action"].shape[-1] == self.action_output_dim
269
+ # sample["action"][sample["action_is_pad"], :-1] = 0.0 # NOTE: we assume use delta_eef_pose + gripper, so pad action is 0
270
+
271
+
272
+ # TODO: rename all "state" into "proprio"
273
+ sample["proprio"] = data["state"] # [num_obs_steps, proprio_dim]
274
+ sample["proprio_is_pad"] = data["state_is_pad"] # [num_obs_steps,]
275
+ sample["proprio_dim_is_pad"] = data["state_dim_is_pad"] # [proprio_dim,]
276
+ assert sample["proprio"].shape[-1] == self.proprio_output_dim
277
+
278
+ sample["idx"] = data["idx"]
279
+
280
+ # sample = self.tokenizer(sample)
281
+
282
+ return sample
283
+
284
+ def postprocess(self, data: Dict[str, Any]) -> Dict[str, Any]:
285
+ """
286
+ Postprocess the data for the policy model.
287
+
288
+ Args:
289
+ data: Dict[str, Any], lerobot sample in raw mcap
290
+
291
+ Returns:
292
+ data: Dict[str, Any], processed data including unnormalized action
293
+ """
294
+ assert "action" in data, "Action is required in postprocess"
295
+ data["state"] = data.pop("proprio")
296
+ data = self.action_state_merger.backward(data)
297
+ data = self.normalizer.backward(data)
298
+ if self.action_state_transforms is not None:
299
+ for trans in reversed(self.action_state_transforms):
300
+ data = trans.backward(data)
301
+
302
+ start_obs_step = self.num_obs_steps - 1
303
+ data["action"] = dict_apply(data["action"], lambda x: x[:, start_obs_step:, :])
304
+ return data
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/robot_video_dataset.py ADDED
@@ -0,0 +1,279 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import hashlib
2
+ import os
3
+ from typing import Optional
4
+ import time
5
+ import numpy as np
6
+ import traceback
7
+ import torch
8
+ import torchvision.transforms.functional as transforms_F
9
+ from contextlib import contextmanager
10
+
11
+ from omegaconf import DictConfig, OmegaConf
12
+
13
+ from hydra.utils import instantiate
14
+ from .base_lerobot_dataset import BaseLerobotDataset
15
+ from .utils.normalizer import save_dataset_stats_to_json, load_dataset_stats_from_json
16
+ from ..dataset_utils import ResizeSmallestSideAspectPreserving, CenterCrop, Normalize
17
+ from fastwam.utils.logging_config import get_logger
18
+ from fastwam.utils import misc, pytorch_utils
19
+ from accelerate import PartialState
20
+ logger = get_logger(__name__)
21
+
22
+
23
+ DEFAULT_PROMPT = "A video recorded from a robot's point of view executing the following instruction: {task}"
24
+
25
+ class RobotVideoDataset(torch.utils.data.Dataset):
26
+ def __init__(
27
+ self,
28
+ dataset_dirs,
29
+ shape_meta,
30
+ num_frames=33,
31
+ video_size=[384, 640],
32
+ camera_key=None,
33
+ processor=None,
34
+ text_embedding_cache_dir=None,
35
+ context_len=128,
36
+ pretrained_norm_stats=None,
37
+ val_set_proportion=0.05,
38
+ is_training_set=False,
39
+ global_sample_stride=1,
40
+ action_video_freq_ratio: int = 1,
41
+ skip_padding_as_possible: bool = False,
42
+ max_padding_retry: int = 3,
43
+ concat_multi_camera: str = "horizontal", # "horizontal", "vertical", "robotwin", or None
44
+ override_instruction: Optional[str] = None, # whether to hardcode a specific instruction for all samples, for debugging
45
+ ):
46
+ self.lerobot_dataset = BaseLerobotDataset(
47
+ dataset_dirs=dataset_dirs,
48
+ shape_meta=OmegaConf.to_container(shape_meta, resolve=True),
49
+ obs_size=num_frames,
50
+ action_size=num_frames - 1,
51
+ val_set_proportion=val_set_proportion,
52
+ is_training_set=is_training_set,
53
+ global_sample_stride=global_sample_stride,
54
+ )
55
+
56
+ self.num_frames = num_frames
57
+ self.action_video_freq_ratio = action_video_freq_ratio
58
+
59
+ assert (num_frames - 1) % self.action_video_freq_ratio == 0, \
60
+ f"num_frames-1 must be divisible by action_video_freq_ratio, got {num_frames - 1} and {self.action_video_freq_ratio}"
61
+ assert ((num_frames - 1) // self.action_video_freq_ratio) % 4 == 0, \
62
+ f"video frames must be divisible by 4 for tokenization, got {(num_frames - 1) // self.action_video_freq_ratio}"
63
+ self.video_sample_indices = list(range(0, num_frames, self.action_video_freq_ratio))
64
+
65
+ self.camera_key = camera_key
66
+ self.lerobot_dataset._set_return_images(True)
67
+
68
+ self.video_size = video_size
69
+ self.text_embedding_cache_dir = text_embedding_cache_dir
70
+ self.context_len = context_len
71
+ self.skip_padding_as_possible = skip_padding_as_possible
72
+ self.max_padding_retry = max_padding_retry
73
+ self.concat_multi_camera = concat_multi_camera
74
+ self.override_instruction = override_instruction
75
+
76
+ self.resize_transform = ResizeSmallestSideAspectPreserving(
77
+ args={"img_w": self.video_size[1], "img_h": self.video_size[0]},
78
+ )
79
+ self.crop_transform = CenterCrop(
80
+ args={"img_w": self.video_size[1], "img_h": self.video_size[0]},
81
+ )
82
+ self.normalize_transform = Normalize(
83
+ args={"mean": 0.5, "std": 0.5},
84
+ )
85
+ if processor is not None:
86
+ if isinstance(processor, DictConfig):
87
+ processor = instantiate(processor)
88
+ if not pretrained_norm_stats:
89
+ if not is_training_set:
90
+ raise ValueError("pretrained_norm_stats must be provided for validation/test sets since we don't want to calculate stats on them.")
91
+ if PartialState().is_main_process:
92
+ logger.info("Calculating dataset stats for normalization...")
93
+ dataset_stats = self.lerobot_dataset.get_dataset_stats(processor)
94
+ work_dir = misc.get_work_dir()
95
+ save_dataset_stats_to_json(dataset_stats, os.path.join(work_dir, "dataset_stats.json"))
96
+ else:
97
+ dataset_stats = None
98
+ if torch.distributed.is_available() and torch.distributed.is_initialized():
99
+ obj_list = [dataset_stats]
100
+ torch.distributed.broadcast_object_list(obj_list, src=0)
101
+ dataset_stats = obj_list[0]
102
+ else:
103
+ dataset_stats = load_dataset_stats_from_json(pretrained_norm_stats)
104
+ logger.info(f"Using dataset stats: {pretrained_norm_stats}")
105
+ if PartialState().is_main_process:
106
+ work_dir = misc.get_work_dir()
107
+ save_dataset_stats_to_json(dataset_stats, os.path.join(work_dir, "dataset_stats.json"))
108
+
109
+ processor.set_normalizer_from_stats(dataset_stats)
110
+ self.lerobot_dataset.set_processor(processor)
111
+
112
+ def __len__(self):
113
+ return len(self.lerobot_dataset)
114
+
115
+ def _get(self, idx):
116
+ sample_idx = idx
117
+ sample = None
118
+ for attempt in range(self.max_padding_retry + 1):
119
+ sample = self.lerobot_dataset[sample_idx]
120
+
121
+ if not self.skip_padding_as_possible:
122
+ break
123
+
124
+ action_is_pad = sample["action_is_pad"]
125
+ image_is_pad = sample["image_is_pad"]
126
+ proprio_is_pad = sample["proprio_is_pad"]
127
+ has_pad = False
128
+ if bool(action_is_pad.any().item()):
129
+ has_pad = True
130
+ if bool(image_is_pad.any().item()):
131
+ has_pad = True
132
+ if bool(proprio_is_pad.any().item()):
133
+ has_pad = True
134
+
135
+ if not has_pad or attempt >= self.max_padding_retry:
136
+ break
137
+
138
+ sample_idx = np.random.randint(len(self.lerobot_dataset))
139
+
140
+ image_is_pad = sample["image_is_pad"]
141
+
142
+ video = sample["pixel_values"] # [T, C, H, W] or [num_cameras, T, C, H, W]
143
+ num_cameras = 1
144
+ if video.ndim == 5:
145
+ video = video[:, self.video_sample_indices, :, :, :] # [num_cameras, T_video, C, H, W]
146
+ num_cameras, T_video, C, H, W = video.shape
147
+ else:
148
+ assert video.ndim == 4, f"Expected video to have shape [T, C, H, W], but got {video.shape}"
149
+ video = video[self.video_sample_indices, :, :, :] # [T_video, C, H, W]
150
+ T_video, C, H, W = video.shape
151
+ image_is_pad = image_is_pad[self.video_sample_indices]
152
+
153
+ video = video.view(num_cameras, T_video, C, H, W) # [num_cameras, T_video, C, H, W]
154
+ if self.concat_multi_camera == "robotwin":
155
+ if num_cameras != 3:
156
+ raise ValueError(
157
+ f"`concat_multi_camera='robotwin'` requires exactly 3 cameras, got {num_cameras}"
158
+ )
159
+ cam_top = transforms_F.resize(
160
+ video[0],
161
+ size=[256, 320],
162
+ interpolation=transforms_F.InterpolationMode.BILINEAR,
163
+ antialias=True,
164
+ ) # [T_video, C, 256, 320]
165
+ cam_left = transforms_F.resize(
166
+ video[1],
167
+ size=[128, 160],
168
+ interpolation=transforms_F.InterpolationMode.BILINEAR,
169
+ antialias=True,
170
+ ) # [T_video, C, 128, 160]
171
+ cam_right = transforms_F.resize(
172
+ video[2],
173
+ size=[128, 160],
174
+ interpolation=transforms_F.InterpolationMode.BILINEAR,
175
+ antialias=True,
176
+ ) # [T_video, C, 128, 160]
177
+ bottom = torch.cat([cam_left, cam_right], dim=-1) # [T_video, C, 128, 320]
178
+ video = torch.cat([cam_top, bottom], dim=-2) # [T_video, C, 384, 320]
179
+ elif num_cameras > 1:
180
+ if self.concat_multi_camera == "horizontal":
181
+ video = torch.cat([video[i] for i in range(num_cameras)], dim=-1) # [T_video, C, H, num_cameras*W]
182
+ elif self.concat_multi_camera == "vertical":
183
+ video = torch.cat([video[i] for i in range(num_cameras)], dim=-2) # [T_video, C, num_cameras*H, W]
184
+ else:
185
+ raise ValueError(
186
+ f"Invalid concat_multi_camera: {self.concat_multi_camera}. "
187
+ "Expected one of: horizontal, vertical, robotwin."
188
+ )
189
+ else:
190
+ video = video.squeeze(0) # [T_video, C, H, W]
191
+
192
+ # final resize and normalization
193
+ video = self.resize_transform(video)
194
+ video = self.crop_transform(video)
195
+ video = self.normalize_transform(video) # [T_video, C, H, W]
196
+
197
+ video = video.permute(1, 0, 2, 3) # [C, T_video, H, W], range [-1, 1]
198
+
199
+ # Proxy (from lerobot):
200
+ # action: [num_frames-1, action_dim] # start from t0, except the last frame
201
+ # proprio: [num_frames, proprio_dim] # start from t0 to the last frame, aligned with video frames
202
+ action = sample["action"] # [T-1, action_dim]
203
+ proprio = sample["proprio"][:-1, :] # [T-1, state_dim], to align with action
204
+ if video.shape[1] <= 1:
205
+ raise ValueError(f"`video` must have at least 2 frames, got shape {tuple(video.shape)}")
206
+ if action.shape[0] % (video.shape[1] - 1) != 0:
207
+ raise ValueError(
208
+ f"`action` horizon must be divisible by `video` transitions, got {action.shape[0]} and {video.shape[1] - 1}"
209
+ )
210
+
211
+ task = sample["instruction"]
212
+
213
+ # FIXME
214
+ if self.override_instruction is not None:
215
+ task = self.override_instruction
216
+ instruction = DEFAULT_PROMPT.format(task=task)
217
+
218
+ context, context_mask = self._get_cached_text_context(instruction)
219
+ # NOTE: to keep consistent with wan2.2's behavior
220
+ context[~context_mask] = 0.0
221
+ context_mask = torch.ones_like(context_mask)
222
+
223
+ data = {
224
+ "video": video,
225
+ "action": action,
226
+ "proprio": proprio,
227
+ "prompt": instruction,
228
+ "context": context,
229
+ "context_mask": context_mask,
230
+ "image_is_pad": image_is_pad,
231
+ "action_is_pad": sample["action_is_pad"],
232
+ "proprio_is_pad": sample["proprio_is_pad"],
233
+ }
234
+ return data
235
+
236
+ def _get_cached_text_context(self, prompt: str):
237
+ if self.text_embedding_cache_dir is None:
238
+ raise ValueError("text_embedding_cache_dir is not set.")
239
+ cache_dir = self.text_embedding_cache_dir
240
+ os.makedirs(cache_dir, exist_ok=True)
241
+ hashed = hashlib.sha256(prompt.encode("utf-8")).hexdigest()
242
+ cache_path = os.path.join(cache_dir, f"{hashed}.t5_len{self.context_len}.wan22ti2v5b.pt")
243
+ if not os.path.exists(cache_path):
244
+ raise FileNotFoundError(
245
+ f"Missing text embedding cache: {cache_path}. "
246
+ "Run scripts/precompute_text_embeds.py first."
247
+ )
248
+ payload = torch.load(cache_path, map_location="cpu")
249
+ context = payload["context"]
250
+ context_mask = payload["mask"].bool()
251
+ if context.ndim != 2:
252
+ raise ValueError(
253
+ f"Cached `context` must be 2D [L, D], got shape {tuple(context.shape)} in {cache_path}"
254
+ )
255
+ if context_mask.ndim != 1:
256
+ raise ValueError(
257
+ f"Cached `mask` must be 1D [L], got shape {tuple(context_mask.shape)} in {cache_path}"
258
+ )
259
+ if context.shape[0] != self.context_len:
260
+ raise ValueError(
261
+ f"Cached context_len mismatch: expected {self.context_len}, got {context.shape[0]} in {cache_path}"
262
+ )
263
+ if context_mask.shape[0] != self.context_len:
264
+ raise ValueError(
265
+ f"Cached mask_len mismatch: expected {self.context_len}, got {context_mask.shape[0]} in {cache_path}"
266
+ )
267
+
268
+ return context, context_mask
269
+
270
+ def __getitem__(self, idx):
271
+ try:
272
+ data = self._get(idx)
273
+ except Exception as e:
274
+ print(f"Error processing sample idx {idx}: {e}. Returning a random sample instead.")
275
+ # trace back
276
+ print(traceback.format_exc())
277
+ random_idx = np.random.randint(len(self))
278
+ data = self._get(random_idx)
279
+ return data
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/action_state_merger.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Dict
2
+
3
+ import torch
4
+ from torch.nn.functional import pad
5
+
6
+
7
+ class ConcatLeftAlign:
8
+ def __init__(
9
+ self,
10
+ action_target_dim: int | None = None,
11
+ state_target_dim: int | None = None
12
+ ):
13
+ self.action_target_dim = action_target_dim
14
+ self.state_target_dim = state_target_dim
15
+
16
+ def set_shape_meta(self, shape_meta):
17
+ self.action_meta = shape_meta["action"]
18
+ self.state_meta = shape_meta["state"]
19
+
20
+ def forward(self, batch):
21
+ if "action" in batch:
22
+ batch["action"] = self._concat(batch["action"], self.action_meta)
23
+ batch["action"], batch["action_dim_is_pad"] = self._pad(batch["action"], self.action_target_dim)
24
+
25
+ batch["state"] = self._concat(batch["state"], self.state_meta)
26
+ batch["state"], batch["state_dim_is_pad"] = self._pad(batch["state"], self.state_target_dim)
27
+
28
+ return batch
29
+
30
+ def backward(self, batch):
31
+ if self.state_target_dim is not None:
32
+ assert batch["state"].shape[-1] == self.state_target_dim
33
+ batch["state"] = self._crop(batch["state"], self.state_meta)
34
+ batch["state"] = self._split(batch["state"], self.state_meta)
35
+
36
+ if self.action_target_dim is not None:
37
+ assert batch["action"].shape[-1] == self.action_target_dim
38
+ batch["action"] = self._crop(batch["action"], self.action_meta)
39
+ batch["action"] = self._split(batch["action"], self.action_meta)
40
+
41
+ return batch
42
+
43
+ @staticmethod
44
+ def _pad(x: torch.Tensor, dim: int):
45
+ if dim is None:
46
+ dim = x.shape[-1]
47
+
48
+ assert x.ndim == 2 and x.shape[-1] <= dim
49
+ pad_dim = dim - x.shape[-1]
50
+ x_padded = pad(x, (0, pad_dim))
51
+ mask = torch.zeros_like(x[0]).bool()
52
+ mask = pad(mask, (0, pad_dim), value=True)
53
+ return x_padded, mask
54
+
55
+ @staticmethod
56
+ def _crop(x: torch.Tensor, meta: int):
57
+ assert x.ndim == 3
58
+ dim = sum([m["shape"] for m in meta])
59
+ x = x[:, :, :dim]
60
+ return x
61
+
62
+ @staticmethod
63
+ def _concat(x: Dict[str, torch.Tensor], meta: Dict[str, Dict]):
64
+ x = torch.cat([x[m["key"]] for m in meta], dim=-1)
65
+ assert x.ndim == 2
66
+ return x
67
+
68
+ @staticmethod
69
+ def _split(x: torch.Tensor, meta: Dict[str, Dict]):
70
+ assert x.ndim == 3
71
+ y = {}
72
+ idx = 0
73
+ for m in meta:
74
+ key, dim = m["key"], m["shape"]
75
+ y[key] = x[:, :, idx: idx + dim]
76
+ idx += dim
77
+
78
+ return y
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/image.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import torchvision.transforms as TF
4
+
5
+
6
+ class ToTensor(nn.Module):
7
+ def __init__(self):
8
+ super().__init__()
9
+
10
+ def forward(self, x: torch.Tensor):
11
+ assert x.dtype == torch.uint8
12
+ x = x.to(torch.float32) / 255.0
13
+ return x
14
+
15
+ class Pad(nn.Module):
16
+ def __init__(self, padding, fill=0, padding_mode='constant'):
17
+ super().__init__()
18
+ self.padding = padding
19
+ self.fill = fill
20
+ self.padding_mode = padding_mode
21
+ self.pad = TF.Pad(padding=tuple(padding), fill=fill, padding_mode=padding_mode)
22
+
23
+ def forward(self, x: torch.Tensor):
24
+ assert x.ndim == 4, "Can only pad tensor of 4 dims."
25
+ return self.pad(x)
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/misc.py ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import List
2
+
3
+ import torch
4
+
5
+
6
+ class WrapStateAngle:
7
+ def __init__(self, keys: List[str]):
8
+ self.keys = keys
9
+
10
+ @staticmethod
11
+ def _wrap(x):
12
+ return torch.atan2(torch.sin(x), torch.cos(x))
13
+
14
+ def forward(self, batch):
15
+ for k in self.keys:
16
+ batch["state"][k] = self._wrap(batch["state"][k])
17
+ return batch
18
+
19
+ def backward(self, batch):
20
+ return batch
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/relative_action.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import List, Dict
2
+
3
+ import torch
4
+ from .rotation import quaternion_to_matrix, matrix_to_quaternion
5
+
6
+ # pose: position and quaternion in x, y, z, i, j, k, r
7
+ # mat: homogeneous transformation matrix in 4×4
8
+ # pos: position in x, y, z
9
+ # quat: quaternion in i, j, k, r
10
+
11
+
12
+ class RelativePoseTransform:
13
+ def __init__(self, keys: List[str]):
14
+ self.keys = keys
15
+
16
+ def forward(self, batch: Dict):
17
+ # for close-loop eval, "action" may not be in batch
18
+ if "action" not in batch:
19
+ return batch
20
+
21
+ for k in self.keys:
22
+ action = batch["action"][k]
23
+ state = batch["state"][k]
24
+ batch["action"][k] = self._forward(action, state[..., -1:, :])
25
+
26
+ return batch
27
+
28
+ def backward(self, batch: Dict):
29
+ for k in self.keys:
30
+ action = batch["action"][k]
31
+ state = batch["state"][k]
32
+ batch["action"][k] = self._backward(action, state[..., -1:, :])
33
+
34
+ return batch
35
+
36
+ def _forward(self, pose: torch.Tensor, base_pose: torch.Tensor):
37
+ # pose & base_pose: position and quaternion in x, y, z, i, j, k, r
38
+ assert pose.shape[-1] == 7, f"Pose shape must be (..., 7), but got {pose.shape}"
39
+ assert base_pose.shape[-1] == 7, f"Base pose shape must be (..., 7), but got {base_pose.shape}"
40
+ pose_matrix = self._pose_to_matrix(pose)
41
+ base_pose_matrix = self._pose_to_matrix(base_pose)
42
+ pose_matrix = self._absolute_to_relative(pose_matrix, base_pose_matrix)
43
+ pose = self._matrix_to_pose(pose_matrix)
44
+ return pose
45
+
46
+ def _backward(self, pose: torch.Tensor, base_pose: torch.Tensor):
47
+ # pose & base_pose: position and quaternion in x, y, z, i, j, k, r
48
+ assert pose.shape[-1] == 7, f"Pose shape must be (..., 7), but got {pose.shape}"
49
+ assert base_pose.shape[-1] == 7, f"Base pose shape must be (..., 7), but got {base_pose.shape}"
50
+ pose_matrix = self._pose_to_matrix(pose)
51
+ base_pose_matrix = self._pose_to_matrix(base_pose)
52
+ pose_matrix = self._relative_to_absolute(pose_matrix, base_pose_matrix)
53
+ pose = self._matrix_to_pose(pose_matrix)
54
+ return pose
55
+
56
+ @staticmethod
57
+ def _pose_to_matrix(pose: torch.Tensor):
58
+ position = pose[..., 0: 3]
59
+ quaternion = pose[..., [6, 3, 4, 5]] # (i j k r) to (r i j k)
60
+ rotation = quaternion_to_matrix(quaternion)
61
+ matrix = torch.zeros(pose.shape[:-1] + (4, 4), dtype=pose.dtype, device=pose.device)
62
+ matrix[..., 0: 3, 0: 3] = rotation
63
+ matrix[..., 0: 3, 3] = position
64
+ matrix[..., 3, 3] = 1
65
+ return matrix
66
+
67
+ @staticmethod
68
+ def _matrix_to_pose(matrix: torch.Tensor):
69
+ position = matrix[..., 0: 3, 3] / matrix[..., 3, 3][..., None]
70
+ rotation = matrix[..., 0: 3, 0: 3]
71
+ quaternion = matrix_to_quaternion(rotation)
72
+ quaternion = quaternion[..., [1, 2, 3, 0]] # (r i j k) to (i j k r)
73
+ pose = torch.cat([position, quaternion], dim=-1)
74
+ return pose
75
+ @staticmethod
76
+ def _absolute_to_relative(pose_matrix: torch.Tensor, base_pose_matrix: torch.Tensor):
77
+ return torch.linalg.inv(base_pose_matrix) @ pose_matrix
78
+
79
+ @staticmethod
80
+ def _relative_to_absolute(pose_matrix: torch.Tensor, base_pose_matrix: torch.Tensor):
81
+ return base_pose_matrix @ pose_matrix
82
+
83
+
84
+ class RelativeJointTransform:
85
+ def __init__(self, keys: List[str]):
86
+ self.keys = keys
87
+
88
+ def forward(self, batch: Dict):
89
+ # for close-loop eval, "action" may not be in batch
90
+ if "action" not in batch:
91
+ return batch
92
+
93
+ for k in self.keys:
94
+ # NOTE: fixed to the first frame
95
+ batch["action"][k] = batch["action"][k] - batch["state"][k][..., :1, :]
96
+
97
+ return batch
98
+
99
+ def backward(self, batch: Dict):
100
+ for k in self.keys:
101
+ # NOTE: fixed to the first frame
102
+ batch["action"][k] = batch["action"][k] + batch["state"][k][..., :1, :]
103
+
104
+ return batch
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/rotation.py ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the BSD-style license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ from typing import Dict, List, Tuple
8
+ import torch
9
+
10
+ from ..utils.rotation import (
11
+ quaternion_to_matrix,
12
+ matrix_to_quaternion,
13
+ matrix_to_rotation_6d,
14
+ rotation_6d_to_matrix,
15
+ matrix_to_rotation_9d,
16
+ rotation_9d_to_matrix,
17
+ quaternion_to_axis_angle,
18
+ axis_angle_to_quaternion,
19
+ axis_angle_to_matrix,
20
+ matrix_to_axis_angle,
21
+ euler_angles_to_matrix,
22
+ matrix_to_euler_angles,
23
+ )
24
+
25
+ class PoseRotationTransform:
26
+ def __init__(self, rotation_type: str, category_keys: Dict[str, List[str]]):
27
+ self.rotation_type = rotation_type
28
+ self.category_keys = category_keys
29
+ self.src_type, self.dst_type = self._parse_rotation_type(rotation_type)
30
+
31
+ def forward(self, batch):
32
+ for cat, ks in self.category_keys.items():
33
+ if cat == "action" and "action" not in batch:
34
+ continue
35
+
36
+ for k in ks:
37
+ batch[cat][k] = self._forward(batch[cat][k])
38
+
39
+ return batch
40
+
41
+ def backward(self, batch):
42
+ for cat, ks in self.category_keys.items():
43
+ for k in ks:
44
+ batch[cat][k] = self._backward(batch[cat][k])
45
+
46
+ return batch
47
+
48
+
49
+ def _forward(self, pose):
50
+ assert pose.shape[-1] >= 6
51
+ if self.src_type == self.dst_type:
52
+ return pose
53
+ position, rotation, others = self._split_pose(pose, self.src_type)
54
+ matrix = self._rotation_to_matrix(rotation, self.src_type)
55
+ rotation_out = self._matrix_to_rotation(matrix, self.dst_type)
56
+ return torch.cat([position, rotation_out, others], axis=-1)
57
+
58
+ def _backward(self, pose: torch.Tensor):
59
+ if self.src_type == self.dst_type:
60
+ return pose
61
+ position, rotation, others = self._split_pose(pose, self.dst_type)
62
+ matrix = self._rotation_to_matrix(rotation, self.dst_type)
63
+ rotation_out = self._matrix_to_rotation(matrix, self.src_type)
64
+ return torch.cat([position, rotation_out, others], axis=-1)
65
+
66
+ def add_noise(self, pose: torch.Tensor, std_position=0.05, std_angle=0.05):
67
+ position, rotation, others = self._split_pose(pose, self.src_type)
68
+ matrix = self._rotation_to_matrix(rotation, self.src_type)
69
+ axis_angles = matrix_to_axis_angle(matrix)
70
+ position = position + std_position * torch.randn_like(position)
71
+ axis_angles = axis_angles + std_angle * torch.randn_like(axis_angles)
72
+ matrix = axis_angle_to_matrix(axis_angles)
73
+ rotation_out = self._matrix_to_rotation(matrix, self.src_type)
74
+ return torch.cat([position, rotation_out, others], axis=-1)
75
+
76
+ def _parse_rotation_type(self, rotation_type: str) -> Tuple[str, str]:
77
+ if "_to_" in rotation_type:
78
+ src, dst = rotation_type.split("_to_", 1)
79
+ else:
80
+ src, dst = "quaternion", rotation_type
81
+ return src, dst
82
+
83
+ def _split_pose(self, pose: torch.Tensor, rotation_type: str) -> Tuple[torch.Tensor, torch.Tensor]:
84
+ rotation_dim = self._rotation_dim(rotation_type)
85
+ position = pose[..., 0:3]
86
+ rotation = pose[..., 3: 3 + rotation_dim]
87
+ others = pose[..., 3 + rotation_dim:] if pose.shape[-1] > 3 + rotation_dim else torch.empty_like(position[..., :0])
88
+ assert rotation.shape[-1] == rotation_dim, f"Expected {rotation_dim} dims for {rotation_type}"
89
+ return position, rotation, others
90
+
91
+ def _rotation_dim(self, rotation_type: str) -> int:
92
+ if rotation_type == "quaternion":
93
+ return 4
94
+ if rotation_type == "axis_angle":
95
+ return 3
96
+ if rotation_type in ("rotation_6d",):
97
+ return 6
98
+ if rotation_type in ("rotation_9d", "matrix"):
99
+ return 9
100
+ if rotation_type.startswith("euler_"):
101
+ return 3
102
+ raise ValueError(f"Unsupported rotation type '{rotation_type}'")
103
+
104
+ def _rotation_to_matrix(self, rotation: torch.Tensor, rotation_type: str) -> torch.Tensor:
105
+ if rotation_type == "quaternion":
106
+ quaternion = rotation[..., [3, 0, 1, 2]]
107
+ return quaternion_to_matrix(quaternion)
108
+ if rotation_type == "axis_angle":
109
+ return axis_angle_to_matrix(rotation)
110
+ if rotation_type == "rotation_6d":
111
+ return rotation_6d_to_matrix(rotation)
112
+ if rotation_type in ("rotation_9d", "matrix"):
113
+ return rotation_9d_to_matrix(rotation)
114
+ if rotation_type.startswith("euler_"):
115
+ convention = rotation_type.split("_", 1)[1].upper()
116
+ return euler_angles_to_matrix(rotation, convention)
117
+ raise ValueError(f"Unsupported rotation type '{rotation_type}'")
118
+
119
+ def _matrix_to_rotation(self, matrix: torch.Tensor, rotation_type: str) -> torch.Tensor:
120
+ if rotation_type == "quaternion":
121
+ quaternion = matrix_to_quaternion(matrix)
122
+ return quaternion[..., [1, 2, 3, 0]]
123
+ if rotation_type == "axis_angle":
124
+ return matrix_to_axis_angle(matrix)
125
+ if rotation_type == "rotation_6d":
126
+ return matrix_to_rotation_6d(matrix)
127
+ if rotation_type in ("rotation_9d", "matrix"):
128
+ return matrix_to_rotation_9d(matrix)
129
+ if rotation_type.startswith("euler_"):
130
+ convention = rotation_type.split("_", 1)[1].upper()
131
+ return matrix_to_euler_angles(matrix, convention)
132
+ raise ValueError(f"Unsupported rotation type '{rotation_type}'")
133
+
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/normalizer.py ADDED
@@ -0,0 +1,291 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Literal, Dict, Annotated, Union, Any, List, Tuple, Optional
2
+ import torch
3
+ import json
4
+ from collections import defaultdict
5
+ import numpy as np
6
+ from omegaconf import DictConfig, OmegaConf
7
+ import hashlib
8
+ from pathlib import Path
9
+ from git import Repo
10
+ from fastwam.utils.logging_config import get_logger
11
+
12
+ from fastwam.utils.pytorch_utils import dict_apply
13
+
14
+ logger = get_logger(__name__)
15
+
16
+ ConstConstStr = Annotated[str, "format: 'const_min/const_max', where const_min and const_max give the constant range"]
17
+ NormMode = Union[Literal["min/max", "q01/q99", "z-score"], ConstConstStr]
18
+
19
+ class LinearNormalizer:
20
+ def __init__(
21
+ self,
22
+ shape_meta,
23
+ use_stepwise_action_norm,
24
+ default_mode: NormMode,
25
+ exception_mode: Dict[str, Dict[str, NormMode]],
26
+ stats: Dict[str, Dict[str, Dict[str, torch.Tensor]]]
27
+ ):
28
+ super().__init__()
29
+ self.normalizers = {"action": {}, "state": {}}
30
+ self.stats = stats
31
+
32
+ for meta in shape_meta["action"]:
33
+ key = meta["key"]
34
+
35
+ if use_stepwise_action_norm:
36
+ cur_stats = {k.removeprefix("stepwise_"): v for k, v in stats["action"][key].items() if k.startswith("stepwise_")}
37
+ else:
38
+ cur_stats = {k.removeprefix("global_"): v for k, v in stats["action"][key].items() if k.startswith("global_")}
39
+
40
+ if exception_mode is not None and "action" in exception_mode and key in exception_mode["action"]:
41
+ cur_mode = exception_mode["action"][key]
42
+ else:
43
+ cur_mode = default_mode
44
+
45
+ self.normalizers["action"][key] = SingleFieldLinearNormalizer(
46
+ stats=cur_stats,
47
+ mode=cur_mode,
48
+ )
49
+
50
+ for meta in shape_meta["state"]:
51
+ key = meta["key"]
52
+ cur_stats = {k.removeprefix("global_"): v for k, v in stats["state"][key].items() if k.startswith("global_")}
53
+
54
+ if exception_mode is not None and "state" in exception_mode and key in exception_mode["state"]:
55
+ cur_mode = exception_mode["state"][key]
56
+ else:
57
+ cur_mode = default_mode
58
+
59
+ self.normalizers["state"][key] = SingleFieldLinearNormalizer(
60
+ stats=cur_stats,
61
+ mode=cur_mode,
62
+ )
63
+
64
+ def get_stats(self):
65
+ stats = {
66
+ "action": {key: norm.get_stats() for key, norm in self.normalizers["action"].items()},
67
+ "state": {key: norm.get_stats() for key, norm in self.normalizers["state"].items()}
68
+ }
69
+ return stats
70
+
71
+ def forward(self, batch: Dict[str, Dict[str, torch.Tensor]]) -> torch.Tensor:
72
+ if "action" in batch:
73
+ for key, norm in self.normalizers["action"].items():
74
+ batch["action"][key] = norm.forward(batch["action"][key])
75
+
76
+ for key, norm in self.normalizers["state"].items():
77
+ batch["state"][key] = norm.forward(batch["state"][key])
78
+
79
+ return batch
80
+
81
+ def backward(self, batch: Dict[str, Dict[str, torch.Tensor]]) -> torch.Tensor:
82
+ for key, norm in self.normalizers["action"].items():
83
+ batch["action"][key] = norm.backward(batch["action"][key])
84
+
85
+ for key, norm in self.normalizers["state"].items():
86
+ batch["state"][key] = norm.backward(batch["state"][key])
87
+
88
+ return batch
89
+
90
+
91
+ class SingleFieldLinearNormalizer:
92
+ std_reg = 1e-8
93
+ range_tol = 1e-4
94
+ output_max = 1.0
95
+ output_min = -1.0
96
+ def __init__(self, stats, mode: NormMode="min/max"):
97
+ self.stats = stats
98
+ self.mode = mode
99
+
100
+ if mode == "z-score":
101
+ input_mean, input_std = stats["mean"], stats["std"]
102
+ scale = 1.0 / (input_std + self.std_reg)
103
+ offset = - input_mean / (input_std + self.std_reg)
104
+ else:
105
+ if mode == "min/max":
106
+ input_min, input_max = stats["min"], stats["max"]
107
+ elif mode == "q01/q99":
108
+ input_min, input_max = stats["q01"], stats["q99"]
109
+ else:
110
+ # parse const_min/const_max
111
+ input_min, input_max = map(float, mode.split("/"))
112
+ input_min = torch.full_like(stats["min"], input_min)
113
+ input_max = torch.full_like(stats["max"], input_max)
114
+
115
+ input_range = input_max - input_min
116
+ ignore_dim = input_range < self.range_tol
117
+ input_range[ignore_dim] = self.output_max - self.output_min
118
+ scale = (self.output_max - self.output_min) / input_range
119
+ offset = self.output_min - scale * input_min
120
+ offset[ignore_dim] = (self.output_max + self.output_min) / 2 - input_min[ignore_dim]
121
+
122
+ self.scale = scale
123
+ self.offset = offset
124
+ def get_stats(self):
125
+ return self.stats
126
+
127
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
128
+ x = x * self.scale + self.offset
129
+ x = torch.clamp(x, -5.0, 5.0)
130
+ return x
131
+ def backward(self, x: torch.Tensor) -> torch.Tensor:
132
+ x = (x - self.offset) / self.scale
133
+ return x
134
+
135
+ def save_dataset_stats_to_json(dataset_stats: dict, file_path: str):
136
+
137
+ def convert_tensor(obj):
138
+ if isinstance(obj, torch.Tensor):
139
+ return obj.detach().cpu().numpy().tolist()
140
+ elif isinstance(obj, (defaultdict, dict)):
141
+ return {k: convert_tensor(v) for k, v in dict(obj).items()}
142
+ elif isinstance(obj, (list, tuple)):
143
+ return [convert_tensor(item) for item in obj]
144
+ elif isinstance(obj, (int, float, str, bool, type(None))):
145
+ return obj
146
+ else:
147
+ return str(obj)
148
+
149
+ serializable_stats = convert_tensor(dataset_stats)
150
+
151
+ with open(file_path, 'w', encoding='utf-8') as f:
152
+ json.dump(serializable_stats, f, ensure_ascii=False, indent=2)
153
+
154
+ def load_dataset_stats_from_json(file_path: str,
155
+ try_convert_tensor: bool = True) -> Dict[str, Any]:
156
+
157
+ def is_numeric_list(obj):
158
+ if isinstance(obj, list):
159
+ if not obj:
160
+ return True
161
+ first = obj[0]
162
+ if isinstance(first, (int, float)):
163
+ return all(isinstance(x, (int, float)) for x in obj)
164
+ elif isinstance(first, list):
165
+ return all(is_numeric_list(item) for item in obj)
166
+ else:
167
+ return False
168
+ return False
169
+
170
+ def convert_back_to_tensor(obj):
171
+ if isinstance(obj, dict):
172
+ return {k: convert_back_to_tensor(v) for k, v in obj.items()}
173
+ elif isinstance(obj, list):
174
+ if is_numeric_list(obj):
175
+ try:
176
+ arr = np.array(obj)
177
+ return torch.from_numpy(arr)
178
+ except Exception:
179
+ return [convert_back_to_tensor(item) for item in obj]
180
+ else:
181
+ return [convert_back_to_tensor(item) for item in obj]
182
+ else:
183
+ return obj
184
+
185
+ with open(file_path, 'r', encoding='utf-8') as f:
186
+ data = json.load(f)
187
+
188
+ if try_convert_tensor:
189
+ data = convert_back_to_tensor(data)
190
+
191
+ data = dict_apply(
192
+ data,
193
+ lambda x: x.to(torch.float32) if isinstance(x, torch.Tensor) else x,
194
+ )
195
+
196
+ return data
197
+
198
+
199
+ def search_dataset_stats_cache_json(cache_dir: str | Path, data_config: DictConfig) -> Tuple[bool, str | None]:
200
+ if isinstance(cache_dir, str):
201
+ cache_dir = Path(cache_dir)
202
+ cache_dir.mkdir(parents=True, exist_ok=True)
203
+
204
+ def get_git_hash() -> Optional[str]:
205
+ repo = Repo(__file__, search_parent_directories=True)
206
+ return repo.head.commit.hexsha
207
+
208
+ def to_plain(value: Any) -> Any:
209
+ if OmegaConf.is_config(value):
210
+ return OmegaConf.to_container(value, resolve=True)
211
+ return value
212
+
213
+ def normalize_str_list(value: Any) -> List[str]:
214
+ value = to_plain(value)
215
+ if value is None:
216
+ return []
217
+ if isinstance(value, str):
218
+ value = [value]
219
+ return [str(item) for item in value if item is not None]
220
+
221
+ def normalize_transforms(value: Any) -> Any:
222
+ value = to_plain(value)
223
+ if isinstance(value, dict):
224
+ return [value]
225
+ return value
226
+
227
+ def normalize_dataset_dirs(cfg: DictConfig) -> Any:
228
+ dataset_cfg = cfg.get("dataset")
229
+ if dataset_cfg is None:
230
+ return None
231
+ embodiment_datasets = dataset_cfg.get("embodiment_datasets")
232
+ if embodiment_datasets is not None:
233
+ emb_dirs: Dict[str, List[str]] = {}
234
+ for emb, emb_cfg in embodiment_datasets.items():
235
+ dataset_groups = emb_cfg.get("dataset_groups")
236
+ if dataset_groups is None:
237
+ emb_dirs[emb] = []
238
+ continue
239
+ dirs: List[str] = []
240
+ for group in dataset_groups:
241
+ group_dirs = group.get("dataset_dirs")
242
+ if group_dirs is None:
243
+ continue
244
+ dirs.extend(normalize_str_list(group_dirs))
245
+ emb_dirs[emb] = sorted(dirs)
246
+ return emb_dirs
247
+
248
+ dataset_dirs = dataset_cfg.get("dataset_dirs")
249
+ return sorted(normalize_str_list(dataset_dirs))
250
+
251
+ def normalize_action_state_transforms(cfg: DictConfig) -> Any:
252
+ processor_cfg = cfg.get("processor")
253
+ if processor_cfg is None:
254
+ return None
255
+ embodiment_processors = processor_cfg.get("embodiment_processors")
256
+ if embodiment_processors is not None:
257
+ emb_transforms: Dict[str, Any] = {}
258
+ for emb, emb_cfg in embodiment_processors.items():
259
+ transforms = emb_cfg.get("action_state_transforms")
260
+ emb_transforms[emb] = normalize_transforms(transforms)
261
+ return emb_transforms
262
+
263
+ transforms = processor_cfg.get("action_state_transforms")
264
+ return normalize_transforms(transforms)
265
+
266
+ signature = {
267
+ "action_size": data_config.dataset.action_size,
268
+ "dataset_dirs": normalize_dataset_dirs(data_config),
269
+ "action_state_transforms": normalize_action_state_transforms(data_config),
270
+ }
271
+ signature_json = json.dumps(signature, sort_keys=True, separators=(",", ":"))
272
+ dataset_hash = hashlib.sha256(signature_json.encode("utf-8"), usedforsecurity=False).hexdigest()
273
+
274
+ git_hash = get_git_hash()
275
+ precise_name = f"dataset_stats_{dataset_hash}_{git_hash}.json"
276
+ precise = cache_dir / precise_name
277
+ if precise.exists():
278
+ logger.info(f"Found dataset stats cache with precisely matching dataset and git hash: {precise_name}.")
279
+ return True, str(precise)
280
+
281
+ candidates = sorted(cache_dir.glob(f"dataset_stats_{dataset_hash}_*.json"))
282
+ if not candidates:
283
+ logger.info(f"No dataset stats cache found for dataset hash {dataset_hash}")
284
+ return False, str(precise) # return precise cache path for saving cache
285
+
286
+ picked = candidates[0]
287
+ prefix = f"dataset_stats_{dataset_hash}_"
288
+ picked_git_hash = picked.name[len(prefix):-5]
289
+ assert picked_git_hash != git_hash
290
+ logger.warning(f"Found substitute dataset stats cache {picked.name} which mismatch current git hash {git_hash}.")
291
+ return True, str(picked)
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/rotation.py ADDED
@@ -0,0 +1,621 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ from typing import Optional, Union, Literal, List
3
+ import torch
4
+ import torch.nn.functional as F
5
+
6
+ Device = Union[str, torch.device]
7
+
8
+
9
+ # [Important] Below is adapted from pytorch3d, which uses (r, i, j, k) quat order
10
+
11
+ """
12
+ The transformation matrices returned from the functions in this file assume
13
+ the points on which the transformation will be applied are column vectors.
14
+ i.e. the R matrix is structured as
15
+
16
+ R = [
17
+ [Rxx, Rxy, Rxz],
18
+ [Ryx, Ryy, Ryz],
19
+ [Rzx, Rzy, Rzz],
20
+ ] # (3, 3)
21
+
22
+ This matrix can be applied to column vectors by post multiplication
23
+ by the points e.g.
24
+
25
+ points = [[0], [1], [2]] # (3 x 1) xyz coordinates of a point
26
+ transformed_points = R * points
27
+
28
+ To apply the same matrix to points which are row vectors, the R matrix
29
+ can be transposed and pre multiplied by the points:
30
+
31
+ e.g.
32
+ points = [[0, 1, 2]] # (1 x 3) xyz coordinates of a point
33
+ transformed_points = points * R.transpose(1, 0)
34
+ """
35
+
36
+
37
+ def quaternion_to_matrix(quaternions: torch.Tensor) -> torch.Tensor:
38
+ """
39
+ Convert rotations given as quaternions to rotation matrices.
40
+
41
+ Args:
42
+ quaternions: quaternions with real part first,
43
+ as tensor of shape (..., 4).
44
+
45
+ Returns:
46
+ Rotation matrices as tensor of shape (..., 3, 3).
47
+ """
48
+ r, i, j, k = torch.unbind(quaternions, -1)
49
+ # pyre-fixme[58]: `/` is not supported for operand types `float` and `Tensor`.
50
+ two_s = 2.0 / (quaternions * quaternions).sum(-1)
51
+
52
+ o = torch.stack(
53
+ (
54
+ 1 - two_s * (j * j + k * k),
55
+ two_s * (i * j - k * r),
56
+ two_s * (i * k + j * r),
57
+ two_s * (i * j + k * r),
58
+ 1 - two_s * (i * i + k * k),
59
+ two_s * (j * k - i * r),
60
+ two_s * (i * k - j * r),
61
+ two_s * (j * k + i * r),
62
+ 1 - two_s * (i * i + j * j),
63
+ ),
64
+ -1,
65
+ )
66
+ return o.reshape(quaternions.shape[:-1] + (3, 3))
67
+
68
+
69
+ def _copysign(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
70
+ """
71
+ Return a tensor where each element has the absolute value taken from the,
72
+ corresponding element of a, with sign taken from the corresponding
73
+ element of b. This is like the standard copysign floating-point operation,
74
+ but is not careful about negative 0 and NaN.
75
+
76
+ Args:
77
+ a: source tensor.
78
+ b: tensor whose signs will be used, of the same shape as a.
79
+
80
+ Returns:
81
+ Tensor of the same shape as a with the signs of b.
82
+ """
83
+ signs_differ = (a < 0) != (b < 0)
84
+ return torch.where(signs_differ, -a, a)
85
+
86
+ @torch.jit.script
87
+ def _sqrt_positive_part(x: torch.Tensor) -> torch.Tensor:
88
+ """
89
+ Returns torch.sqrt(torch.max(0, x))
90
+ but with a zero subgradient where x is 0.
91
+ """
92
+ ret = torch.zeros_like(x)
93
+ positive_mask = x > 0
94
+ ret[positive_mask] = torch.sqrt(x[positive_mask])
95
+ return ret
96
+
97
+
98
+ def matrix_to_quaternion(matrix: torch.Tensor) -> torch.Tensor:
99
+ """
100
+ Convert rotations given as rotation matrices to quaternions.
101
+
102
+ Args:
103
+ matrix: Rotation matrices as tensor of shape (..., 3, 3).
104
+
105
+ Returns:
106
+ quaternions with real part first, as tensor of shape (..., 4).
107
+ """
108
+ if matrix.size(-1) != 3 or matrix.size(-2) != 3:
109
+ raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.")
110
+
111
+ batch_dim = matrix.shape[:-2]
112
+ m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.unbind(
113
+ matrix.reshape(batch_dim + (9,)), dim=-1
114
+ )
115
+
116
+ q_abs = _sqrt_positive_part(
117
+ torch.stack(
118
+ [
119
+ 1.0 + m00 + m11 + m22,
120
+ 1.0 + m00 - m11 - m22,
121
+ 1.0 - m00 + m11 - m22,
122
+ 1.0 - m00 - m11 + m22,
123
+ ],
124
+ dim=-1,
125
+ )
126
+ )
127
+
128
+ # we produce the desired quaternion multiplied by each of r, i, j, k
129
+ quat_by_rijk = torch.stack(
130
+ [
131
+ # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and
132
+ # `int`.
133
+ torch.stack([q_abs[..., 0] ** 2, m21 - m12, m02 - m20, m10 - m01], dim=-1),
134
+ # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and
135
+ # `int`.
136
+ torch.stack([m21 - m12, q_abs[..., 1] ** 2, m10 + m01, m02 + m20], dim=-1),
137
+ # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and
138
+ # `int`.
139
+ torch.stack([m02 - m20, m10 + m01, q_abs[..., 2] ** 2, m12 + m21], dim=-1),
140
+ # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and
141
+ # `int`.
142
+ torch.stack([m10 - m01, m20 + m02, m21 + m12, q_abs[..., 3] ** 2], dim=-1),
143
+ ],
144
+ dim=-2,
145
+ )
146
+
147
+ # We floor here at 0.1 but the exact level is not important; if q_abs is small,
148
+ # the candidate won't be picked.
149
+ flr = torch.tensor(0.1).to(dtype=q_abs.dtype, device=q_abs.device)
150
+ quat_candidates = quat_by_rijk / (2.0 * q_abs[..., None].max(flr))
151
+
152
+ # if not for numerical problems, quat_candidates[i] should be same (up to a sign),
153
+ # forall i; we pick the best-conditioned one (with the largest denominator)
154
+ out = quat_candidates[
155
+ F.one_hot(q_abs.argmax(dim=-1), num_classes=4) > 0.5, :
156
+ ].reshape(batch_dim + (4,))
157
+ return standardize_quaternion(out)
158
+
159
+
160
+ def _axis_angle_rotation(axis: str, angle: torch.Tensor) -> torch.Tensor:
161
+ """
162
+ Return the rotation matrices for one of the rotations about an axis
163
+ of which Euler angles describe, for each value of the angle given.
164
+
165
+ Args:
166
+ axis: Axis label "X" or "Y or "Z".
167
+ angle: any shape tensor of Euler angles in radians
168
+
169
+ Returns:
170
+ Rotation matrices as tensor of shape (..., 3, 3).
171
+ """
172
+
173
+ cos = torch.cos(angle)
174
+ sin = torch.sin(angle)
175
+ one = torch.ones_like(angle)
176
+ zero = torch.zeros_like(angle)
177
+
178
+ if axis == "X":
179
+ R_flat = (one, zero, zero, zero, cos, -sin, zero, sin, cos)
180
+ elif axis == "Y":
181
+ R_flat = (cos, zero, sin, zero, one, zero, -sin, zero, cos)
182
+ elif axis == "Z":
183
+ R_flat = (cos, -sin, zero, sin, cos, zero, zero, zero, one)
184
+ else:
185
+ raise ValueError("letter must be either X, Y or Z.")
186
+
187
+ return torch.stack(R_flat, -1).reshape(angle.shape + (3, 3))
188
+
189
+
190
+ def euler_angles_to_matrix(euler_angles: torch.Tensor, convention: str) -> torch.Tensor:
191
+ """
192
+ Convert rotations given as Euler angles in radians to rotation matrices.
193
+
194
+ Args:
195
+ euler_angles: Euler angles in radians as tensor of shape (..., 3).
196
+ convention: Convention string of three uppercase letters from
197
+ {"X", "Y", and "Z"}.
198
+
199
+ Returns:
200
+ Rotation matrices as tensor of shape (..., 3, 3).
201
+ """
202
+ if euler_angles.dim() == 0 or euler_angles.shape[-1] != 3:
203
+ raise ValueError("Invalid input euler angles.")
204
+ if len(convention) != 3:
205
+ raise ValueError("Convention must have 3 letters.")
206
+ if convention[1] in (convention[0], convention[2]):
207
+ raise ValueError(f"Invalid convention {convention}.")
208
+ for letter in convention:
209
+ if letter not in ("X", "Y", "Z"):
210
+ raise ValueError(f"Invalid letter {letter} in convention string.")
211
+ matrices = [
212
+ _axis_angle_rotation(c, e)
213
+ for c, e in zip(convention, torch.unbind(euler_angles, -1))
214
+ ]
215
+ # return functools.reduce(torch.matmul, matrices)
216
+ return torch.matmul(torch.matmul(matrices[0], matrices[1]), matrices[2])
217
+
218
+
219
+ def _angle_from_tan(
220
+ axis: str, other_axis: str, data, horizontal: bool, tait_bryan: bool
221
+ ) -> torch.Tensor:
222
+ """
223
+ Extract the first or third Euler angle from the two members of
224
+ the matrix which are positive constant times its sine and cosine.
225
+
226
+ Args:
227
+ axis: Axis label "X" or "Y or "Z" for the angle we are finding.
228
+ other_axis: Axis label "X" or "Y or "Z" for the middle axis in the
229
+ convention.
230
+ data: Rotation matrices as tensor of shape (..., 3, 3).
231
+ horizontal: Whether we are looking for the angle for the third axis,
232
+ which means the relevant entries are in the same row of the
233
+ rotation matrix. If not, they are in the same column.
234
+ tait_bryan: Whether the first and third axes in the convention differ.
235
+
236
+ Returns:
237
+ Euler Angles in radians for each matrix in data as a tensor
238
+ of shape (...).
239
+ """
240
+
241
+ i1, i2 = {"X": (2, 1), "Y": (0, 2), "Z": (1, 0)}[axis]
242
+ if horizontal:
243
+ i2, i1 = i1, i2
244
+ even = (axis + other_axis) in ["XY", "YZ", "ZX"]
245
+ if horizontal == even:
246
+ return torch.atan2(data[..., i1], data[..., i2])
247
+ if tait_bryan:
248
+ return torch.atan2(-data[..., i2], data[..., i1])
249
+ return torch.atan2(data[..., i2], -data[..., i1])
250
+
251
+
252
+ def _index_from_letter(letter: str) -> int:
253
+ if letter == "X":
254
+ return 0
255
+ if letter == "Y":
256
+ return 1
257
+ if letter == "Z":
258
+ return 2
259
+ raise ValueError("letter must be either X, Y or Z.")
260
+
261
+
262
+ def matrix_to_euler_angles(matrix: torch.Tensor, convention: str) -> torch.Tensor:
263
+ """
264
+ Convert rotations given as rotation matrices to Euler angles in radians.
265
+
266
+ Args:
267
+ matrix: Rotation matrices as tensor of shape (..., 3, 3).
268
+ convention: Convention string of three uppercase letters.
269
+
270
+ Returns:
271
+ Euler angles in radians as tensor of shape (..., 3).
272
+ """
273
+ if len(convention) != 3:
274
+ raise ValueError("Convention must have 3 letters.")
275
+ if convention[1] in (convention[0], convention[2]):
276
+ raise ValueError(f"Invalid convention {convention}.")
277
+ for letter in convention:
278
+ if letter not in ("X", "Y", "Z"):
279
+ raise ValueError(f"Invalid letter {letter} in convention string.")
280
+ if matrix.size(-1) != 3 or matrix.size(-2) != 3:
281
+ raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.")
282
+ i0 = _index_from_letter(convention[0])
283
+ i2 = _index_from_letter(convention[2])
284
+ tait_bryan = i0 != i2
285
+ if tait_bryan:
286
+ central_angle = torch.asin(
287
+ matrix[..., i0, i2] * (-1.0 if i0 - i2 in [-1, 2] else 1.0)
288
+ )
289
+ else:
290
+ central_angle = torch.acos(matrix[..., i0, i0])
291
+
292
+ o = (
293
+ _angle_from_tan(
294
+ convention[0], convention[1], matrix[..., i2], False, tait_bryan
295
+ ),
296
+ central_angle,
297
+ _angle_from_tan(
298
+ convention[2], convention[1], matrix[..., i0, :], True, tait_bryan
299
+ ),
300
+ )
301
+ return torch.stack(o, -1)
302
+
303
+
304
+ def random_quaternions(
305
+ n: int, dtype: Optional[torch.dtype] = None, device: Optional[Device] = None
306
+ ) -> torch.Tensor:
307
+ """
308
+ Generate random quaternions representing rotations,
309
+ i.e. versors with nonnegative real part.
310
+
311
+ Args:
312
+ n: Number of quaternions in a batch to return.
313
+ dtype: Type to return.
314
+ device: Desired device of returned tensor. Default:
315
+ uses the current device for the default tensor type.
316
+
317
+ Returns:
318
+ Quaternions as tensor of shape (N, 4).
319
+ """
320
+ if isinstance(device, str):
321
+ device = torch.device(device)
322
+ o = torch.randn((n, 4), dtype=dtype, device=device)
323
+ s = (o * o).sum(1)
324
+ o = o / _copysign(torch.sqrt(s), o[:, 0])[:, None]
325
+ return o
326
+
327
+
328
+ def random_rotations(
329
+ n: int, dtype: Optional[torch.dtype] = None, device: Optional[Device] = None
330
+ ) -> torch.Tensor:
331
+ """
332
+ Generate random rotations as 3x3 rotation matrices.
333
+
334
+ Args:
335
+ n: Number of rotation matrices in a batch to return.
336
+ dtype: Type to return.
337
+ device: Device of returned tensor. Default: if None,
338
+ uses the current device for the default tensor type.
339
+
340
+ Returns:
341
+ Rotation matrices as tensor of shape (n, 3, 3).
342
+ """
343
+ quaternions = random_quaternions(n, dtype=dtype, device=device)
344
+ return quaternion_to_matrix(quaternions)
345
+
346
+
347
+ def random_rotation(
348
+ dtype: Optional[torch.dtype] = None, device: Optional[Device] = None
349
+ ) -> torch.Tensor:
350
+ """
351
+ Generate a single random 3x3 rotation matrix.
352
+
353
+ Args:
354
+ dtype: Type to return
355
+ device: Device of returned tensor. Default: if None,
356
+ uses the current device for the default tensor type
357
+
358
+ Returns:
359
+ Rotation matrix as tensor of shape (3, 3).
360
+ """
361
+ return random_rotations(1, dtype, device)[0]
362
+
363
+
364
+ def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor:
365
+ """
366
+ Convert a unit quaternion to a standard form: one in which the real
367
+ part is non negative.
368
+
369
+ Args:
370
+ quaternions: Quaternions with real part first,
371
+ as tensor of shape (..., 4).
372
+
373
+ Returns:
374
+ Standardized quaternions as tensor of shape (..., 4).
375
+ """
376
+ return torch.where(quaternions[..., 0:1] < 0, -quaternions, quaternions)
377
+
378
+
379
+ def quaternion_raw_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
380
+ """
381
+ Multiply two quaternions.
382
+ Usual torch rules for broadcasting apply.
383
+
384
+ Args:
385
+ a: Quaternions as tensor of shape (..., 4), real part first.
386
+ b: Quaternions as tensor of shape (..., 4), real part first.
387
+
388
+ Returns:
389
+ The product of a and b, a tensor of quaternions shape (..., 4).
390
+ """
391
+ aw, ax, ay, az = torch.unbind(a, -1)
392
+ bw, bx, by, bz = torch.unbind(b, -1)
393
+ ow = aw * bw - ax * bx - ay * by - az * bz
394
+ ox = aw * bx + ax * bw + ay * bz - az * by
395
+ oy = aw * by - ax * bz + ay * bw + az * bx
396
+ oz = aw * bz + ax * by - ay * bx + az * bw
397
+ return torch.stack((ow, ox, oy, oz), -1)
398
+
399
+
400
+ def quaternion_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
401
+ """
402
+ Multiply two quaternions representing rotations, returning the quaternion
403
+ representing their composition, i.e. the versor with nonnegative real part.
404
+ Usual torch rules for broadcasting apply.
405
+
406
+ Args:
407
+ a: Quaternions as tensor of shape (..., 4), real part first.
408
+ b: Quaternions as tensor of shape (..., 4), real part first.
409
+
410
+ Returns:
411
+ The product of a and b, a tensor of quaternions of shape (..., 4).
412
+ """
413
+ ab = quaternion_raw_multiply(a, b)
414
+ return standardize_quaternion(ab)
415
+
416
+
417
+ def quaternion_invert(quaternion: torch.Tensor) -> torch.Tensor:
418
+ """
419
+ Given a quaternion representing rotation, get the quaternion representing
420
+ its inverse.
421
+
422
+ Args:
423
+ quaternion: Quaternions as tensor of shape (..., 4), with real part
424
+ first, which must be versors (unit quaternions).
425
+
426
+ Returns:
427
+ The inverse, a tensor of quaternions of shape (..., 4).
428
+ """
429
+
430
+ scaling = torch.tensor([1, -1, -1, -1], device=quaternion.device)
431
+ return quaternion * scaling
432
+
433
+
434
+ def quaternion_apply(quaternion: torch.Tensor, point: torch.Tensor) -> torch.Tensor:
435
+ """
436
+ Apply the rotation given by a quaternion to a 3D point.
437
+ Usual torch rules for broadcasting apply.
438
+
439
+ Args:
440
+ quaternion: Tensor of quaternions, real part first, of shape (..., 4).
441
+ point: Tensor of 3D points of shape (..., 3).
442
+
443
+ Returns:
444
+ Tensor of rotated points of shape (..., 3).
445
+ """
446
+ if point.size(-1) != 3:
447
+ raise ValueError(f"Points are not in 3D, {point.shape}.")
448
+ real_parts = point.new_zeros(point.shape[:-1] + (1,))
449
+ point_as_quaternion = torch.cat((real_parts, point), -1)
450
+ out = quaternion_raw_multiply(
451
+ quaternion_raw_multiply(quaternion, point_as_quaternion),
452
+ quaternion_invert(quaternion),
453
+ )
454
+ return out[..., 1:]
455
+
456
+
457
+ def axis_angle_to_matrix(axis_angle: torch.Tensor) -> torch.Tensor:
458
+ """
459
+ Convert rotations given as axis/angle to rotation matrices.
460
+
461
+ Args:
462
+ axis_angle: Rotations given as a vector in axis angle form,
463
+ as a tensor of shape (..., 3), where the magnitude is
464
+ the angle turned anticlockwise in radians around the
465
+ vector's direction.
466
+
467
+ Returns:
468
+ Rotation matrices as tensor of shape (..., 3, 3).
469
+ """
470
+ return quaternion_to_matrix(axis_angle_to_quaternion(axis_angle))
471
+
472
+
473
+ def matrix_to_axis_angle(matrix: torch.Tensor) -> torch.Tensor:
474
+ """
475
+ Convert rotations given as rotation matrices to axis/angle.
476
+
477
+ Args:
478
+ matrix: Rotation matrices as tensor of shape (..., 3, 3).
479
+
480
+ Returns:
481
+ Rotations given as a vector in axis angle form, as a tensor
482
+ of shape (..., 3), where the magnitude is the angle
483
+ turned anticlockwise in radians around the vector's
484
+ direction.
485
+ """
486
+ return quaternion_to_axis_angle(matrix_to_quaternion(matrix))
487
+
488
+
489
+ def axis_angle_to_quaternion(axis_angle: torch.Tensor) -> torch.Tensor:
490
+ """
491
+ Convert rotations given as axis/angle to quaternions.
492
+
493
+ Args:
494
+ axis_angle: Rotations given as a vector in axis angle form,
495
+ as a tensor of shape (..., 3), where the magnitude is
496
+ the angle turned anticlockwise in radians around the
497
+ vector's direction.
498
+
499
+ Returns:
500
+ quaternions with real part first, as tensor of shape (..., 4).
501
+ """
502
+ angles = torch.norm(axis_angle, p=2, dim=-1, keepdim=True)
503
+ half_angles = angles * 0.5
504
+ eps = 1e-6
505
+ small_angles = angles.abs() < eps
506
+ sin_half_angles_over_angles = torch.empty_like(angles)
507
+ sin_half_angles_over_angles[~small_angles] = (
508
+ torch.sin(half_angles[~small_angles]) / angles[~small_angles]
509
+ )
510
+ # for x small, sin(x/2) is about x/2 - (x/2)^3/6
511
+ # so sin(x/2)/x is about 1/2 - (x*x)/48
512
+ sin_half_angles_over_angles[small_angles] = (
513
+ 0.5 - (angles[small_angles] * angles[small_angles]) / 48
514
+ )
515
+ quaternions = torch.cat(
516
+ [torch.cos(half_angles), axis_angle * sin_half_angles_over_angles], dim=-1
517
+ )
518
+ return quaternions
519
+
520
+
521
+ def quaternion_to_axis_angle(quaternions: torch.Tensor) -> torch.Tensor:
522
+ """
523
+ Convert rotations given as quaternions to axis/angle.
524
+
525
+ Args:
526
+ quaternions: quaternions with real part first,
527
+ as tensor of shape (..., 4).
528
+
529
+ Returns:
530
+ Rotations given as a vector in axis angle form, as a tensor
531
+ of shape (..., 3), where the magnitude is the angle
532
+ turned anticlockwise in radians around the vector's
533
+ direction.
534
+ """
535
+ norms = torch.norm(quaternions[..., 1:], p=2, dim=-1, keepdim=True)
536
+ half_angles = torch.atan2(norms, quaternions[..., :1])
537
+ angles = 2 * half_angles
538
+ eps = 1e-6
539
+ small_angles = angles.abs() < eps
540
+ sin_half_angles_over_angles = torch.empty_like(angles)
541
+ sin_half_angles_over_angles[~small_angles] = (
542
+ torch.sin(half_angles[~small_angles]) / angles[~small_angles]
543
+ )
544
+ # for x small, sin(x/2) is about x/2 - (x/2)^3/6
545
+ # so sin(x/2)/x is about 1/2 - (x*x)/48
546
+ sin_half_angles_over_angles[small_angles] = (
547
+ 0.5 - (angles[small_angles] * angles[small_angles]) / 48
548
+ )
549
+ return quaternions[..., 1:] / sin_half_angles_over_angles
550
+
551
+
552
+ def rotation_6d_to_matrix(d6: torch.Tensor) -> torch.Tensor:
553
+ """
554
+ Converts 6D rotation representation by Zhou et al. [1] to rotation matrix
555
+ using Gram--Schmidt orthogonalization per Section B of [1].
556
+ Args:
557
+ d6: 6D rotation representation, of size (*, 6)
558
+
559
+ Returns:
560
+ batch of rotation matrices of size (*, 3, 3)
561
+
562
+ [1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H.
563
+ On the Continuity of Rotation Representations in Neural Networks.
564
+ IEEE Conference on Computer Vision and Pattern Recognition, 2019.
565
+ Retrieved from http://arxiv.org/abs/1812.07035
566
+ """
567
+
568
+ a1, a2 = d6[..., :3], d6[..., 3:]
569
+ b1 = F.normalize(a1, dim=-1)
570
+ b2 = a2 - (b1 * a2).sum(-1, keepdim=True) * b1
571
+ b2 = F.normalize(b2, dim=-1)
572
+ b3 = torch.cross(b1, b2, dim=-1)
573
+ return torch.stack((b1, b2, b3), dim=-2)
574
+
575
+
576
+ def matrix_to_rotation_6d(matrix: torch.Tensor) -> torch.Tensor:
577
+ """
578
+ Converts rotation matrices to 6D rotation representation by Zhou et al. [1]
579
+ by dropping the last row. Note that 6D representation is not unique.
580
+ Args:
581
+ matrix: batch of rotation matrices of size (*, 3, 3)
582
+
583
+ Returns:
584
+ 6D rotation representation, of size (*, 6)
585
+
586
+ [1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H.
587
+ On the Continuity of Rotation Representations in Neural Networks.
588
+ IEEE Conference on Computer Vision and Pattern Recognition, 2019.
589
+ Retrieved from http://arxiv.org/abs/1812.07035
590
+ """
591
+ batch_dim = matrix.size()[:-2]
592
+ return matrix[..., :2, :].clone().reshape(batch_dim + (6,))
593
+
594
+
595
+ def rotation_9d_to_matrix(d9: torch.Tensor) -> torch.Tensor:
596
+ """"
597
+ Args:
598
+ d9: 9D rotaion representation of size (..., 9)
599
+
600
+ Returns:
601
+ batch of rotation matrices of size (..., 3, 3)
602
+ """
603
+ assert d9.shape[-1] == 9
604
+ matrices = d9.reshape(-1, 3, 3).permute(0, 2, 1)
605
+ u, s, vt = torch.linalg.svd(matrices) # torch.svd returns v instead of vt
606
+ det = torch.det(torch.matmul(u, vt))
607
+ vt[:, :, -1] *= det[:, None]
608
+ rot_matrices = torch.matmul(u, vt)
609
+ rot_matrices = rot_matrices.unflatten(0, d9.shape[:-1])
610
+ return rot_matrices
611
+
612
+
613
+ def matrix_to_rotation_9d(matrix: torch.Tensor) -> torch.Tensor:
614
+ """"
615
+ Args:
616
+ matrix: batch of rotation matrices of size (..., 3, 3)
617
+ Returns:
618
+ 9D rotaion representation of size (..., 9)
619
+ """
620
+ assert matrix.shape[-2:] == (3, 3)
621
+ return matrix.transpose(-2, -1).flatten(-2, -1)
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Model namespace for fastwam."""
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/__init__.py ADDED
File without changes
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/action_dit.py ADDED
@@ -0,0 +1,337 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import torch
3
+ import torch.nn as nn
4
+ from typing import Any, Dict, Optional
5
+
6
+ from fastwam.utils.logging_config import get_logger
7
+
8
+ from .helpers.gradient import gradient_checkpoint_forward
9
+ from .wan_video_dit import (
10
+ DiTBlock,
11
+ sinusoidal_embedding_1d,
12
+ precompute_freqs_cis,
13
+ )
14
+
15
+ logger = get_logger(__name__)
16
+
17
+
18
+ class ActionHead(nn.Module):
19
+ def __init__(self, hidden_dim: int, out_dim: int, eps: float):
20
+ super().__init__()
21
+ self.norm = nn.LayerNorm(hidden_dim, eps=eps, elementwise_affine=False)
22
+ self.proj = nn.Linear(hidden_dim, out_dim)
23
+ self.modulation = nn.Parameter(torch.randn(1, 2, hidden_dim) / hidden_dim**0.5)
24
+
25
+ def forward(self, x: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
26
+ shift, scale = (self.modulation.to(dtype=t.dtype, device=t.device) + t.unsqueeze(1)).chunk(2, dim=1)
27
+ shift = shift.squeeze(1)
28
+ scale = scale.squeeze(1)
29
+ return self.proj(self.norm(x) * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1))
30
+
31
+
32
+ class ActionDiT(nn.Module):
33
+ ACTION_BACKBONE_SKIP_PREFIXES = ("action_encoder.", "head.")
34
+ ACTION_BACKBONE_META_KEYS = (
35
+ "hidden_dim",
36
+ "ffn_dim",
37
+ "num_layers",
38
+ "num_heads",
39
+ "attn_head_dim",
40
+ "text_dim",
41
+ "freq_dim",
42
+ "eps",
43
+ )
44
+
45
+ def __init__(
46
+ self,
47
+ hidden_dim: int,
48
+ action_dim: int,
49
+ ffn_dim: int,
50
+ text_dim: int,
51
+ freq_dim: int,
52
+ eps: float,
53
+ num_heads: int,
54
+ attn_head_dim: int,
55
+ num_layers: int,
56
+ use_gradient_checkpointing: bool = False,
57
+ ):
58
+ super().__init__()
59
+ self.hidden_dim = hidden_dim
60
+ self.action_dim = action_dim
61
+ self.ffn_dim = ffn_dim
62
+ self.text_dim = text_dim
63
+ self.freq_dim = freq_dim
64
+ self.num_heads = num_heads
65
+ self.attn_head_dim = attn_head_dim
66
+
67
+ if num_heads <= 0:
68
+ raise ValueError(f"`num_heads` must be > 0, got {num_heads}")
69
+ if attn_head_dim <= 0:
70
+ raise ValueError(f"`attn_head_dim` must be > 0, got {attn_head_dim}")
71
+ if attn_head_dim % 2 != 0:
72
+ raise ValueError(f"`attn_head_dim` must be even for RoPE, got {attn_head_dim}")
73
+
74
+ self.action_encoder = nn.Linear(action_dim, hidden_dim)
75
+ self.text_embedding = nn.Sequential(
76
+ nn.Linear(text_dim, hidden_dim),
77
+ nn.GELU(approximate="tanh"),
78
+ nn.Linear(hidden_dim, hidden_dim),
79
+ )
80
+ self.time_embedding = nn.Sequential(
81
+ nn.Linear(freq_dim, hidden_dim),
82
+ nn.SiLU(),
83
+ nn.Linear(hidden_dim, hidden_dim),
84
+ )
85
+ self.time_projection = nn.Sequential(nn.SiLU(), nn.Linear(hidden_dim, hidden_dim * 6))
86
+ self.blocks = nn.ModuleList(
87
+ [
88
+ DiTBlock(
89
+ hidden_dim=hidden_dim,
90
+ attn_head_dim=attn_head_dim,
91
+ num_heads=num_heads,
92
+ ffn_dim=ffn_dim,
93
+ eps=eps,
94
+ )
95
+ for _ in range(num_layers)
96
+ ]
97
+ )
98
+ self.head = nn.Linear(hidden_dim, action_dim)
99
+ self.freqs = precompute_freqs_cis(attn_head_dim, end=1024)
100
+
101
+ self.use_gradient_checkpointing = use_gradient_checkpointing
102
+
103
+ @classmethod
104
+ def backbone_key_set(cls, keys) -> set[str]:
105
+ return {
106
+ key
107
+ for key in keys
108
+ if not any(key.startswith(prefix) for prefix in cls.ACTION_BACKBONE_SKIP_PREFIXES)
109
+ }
110
+
111
+ @classmethod
112
+ def from_pretrained(
113
+ cls,
114
+ action_dit_config: dict[str, Any],
115
+ action_dit_pretrained_path: str | None = None,
116
+ skip_dit_load_from_pretrain: bool = False,
117
+ device: str = "cuda",
118
+ torch_dtype: torch.dtype = torch.bfloat16,
119
+ ) -> "ActionDiT":
120
+ if action_dit_config is None:
121
+ raise ValueError("`action_dit_config` is required for ActionDiT.from_pretrained().")
122
+ if skip_dit_load_from_pretrain:
123
+ logger.info(
124
+ "Skipping ActionDiT pretrained load (`skip_dit_load_from_pretrain=True`); "
125
+ "initializing action expert randomly and expecting checkpoint override."
126
+ )
127
+ return cls(**action_dit_config).to(device=device, dtype=torch_dtype)
128
+ if not action_dit_pretrained_path:
129
+ logger.info("No `action_dit_pretrained_path` provided, initializing ActionDiT with random weights.")
130
+ return cls(**action_dit_config).to(device=device, dtype=torch_dtype)
131
+ from pathlib import Path
132
+ p = Path(action_dit_pretrained_path)
133
+ if not p.is_absolute():
134
+ p = Path(__file__).resolve().parents[4] / p
135
+ action_dit_pretrained_path = str(p)
136
+ if not os.path.isfile(action_dit_pretrained_path):
137
+ raise FileNotFoundError(
138
+ f"`action_dit_pretrained_path` does not exist: {action_dit_pretrained_path}"
139
+ )
140
+
141
+ action_cfg = dict(action_dit_config)
142
+ action_expert = cls(**action_cfg).to(device=device, dtype=torch_dtype)
143
+ action_state = action_expert.state_dict()
144
+ expected_backbone_keys = cls.backbone_key_set(action_state.keys())
145
+
146
+ payload = torch.load(action_dit_pretrained_path, map_location="cpu")
147
+ if not isinstance(payload, dict):
148
+ raise ValueError(
149
+ f"Invalid action backbone payload type from {action_dit_pretrained_path}: {type(payload)}"
150
+ )
151
+
152
+ policy = payload.get("policy", {})
153
+ if policy:
154
+ logger.info(f"ActionDiT backbone payload policy: {policy}")
155
+
156
+ meta = payload.get("meta")
157
+ expected_meta = {
158
+ "hidden_dim": int(action_cfg["hidden_dim"]),
159
+ "ffn_dim": int(action_cfg["ffn_dim"]),
160
+ "num_layers": int(action_cfg["num_layers"]),
161
+ "num_heads": int(action_cfg["num_heads"]),
162
+ "attn_head_dim": int(action_cfg["attn_head_dim"]),
163
+ "text_dim": int(action_cfg["text_dim"]),
164
+ "freq_dim": int(action_cfg["freq_dim"]),
165
+ "eps": float(action_cfg["eps"]),
166
+ }
167
+ for key in cls.ACTION_BACKBONE_META_KEYS:
168
+ if key not in meta:
169
+ raise ValueError(f"`meta.{key}` missing in {action_dit_pretrained_path}")
170
+ expected_value = expected_meta[key]
171
+ got_value = meta[key]
172
+ if key == "eps":
173
+ if abs(float(got_value) - float(expected_value)) > 1e-12:
174
+ raise ValueError(
175
+ f"`meta.{key}` mismatch in {action_dit_pretrained_path}: "
176
+ f"expected {expected_value}, got {got_value}"
177
+ )
178
+ elif int(got_value) != int(expected_value):
179
+ raise ValueError(
180
+ f"`meta.{key}` mismatch in {action_dit_pretrained_path}: "
181
+ f"expected {expected_value}, got {got_value}"
182
+ )
183
+
184
+ backbone_state_dict = payload.get("backbone_state_dict")
185
+ if not isinstance(backbone_state_dict, dict):
186
+ raise ValueError(
187
+ f"`backbone_state_dict` must be a dict in {action_dit_pretrained_path}, "
188
+ f"got {type(backbone_state_dict)}"
189
+ )
190
+
191
+ provided_keys = set(backbone_state_dict.keys())
192
+ missing_keys = sorted(expected_backbone_keys - provided_keys)
193
+ unexpected_keys = sorted(provided_keys - expected_backbone_keys)
194
+ if missing_keys or unexpected_keys:
195
+ raise ValueError(
196
+ "Action backbone key mismatch in preprocessed payload. "
197
+ f"missing={missing_keys[:10]}{'...' if len(missing_keys) > 10 else ''}, "
198
+ f"unexpected={unexpected_keys[:10]}{'...' if len(unexpected_keys) > 10 else ''}"
199
+ )
200
+
201
+ merged_state = dict(action_state)
202
+ for key in expected_backbone_keys:
203
+ value = backbone_state_dict[key]
204
+ if not isinstance(value, torch.Tensor):
205
+ raise ValueError(
206
+ f"`backbone_state_dict[{key}]` must be torch.Tensor in {action_dit_pretrained_path}, "
207
+ f"got {type(value)}"
208
+ )
209
+ target = merged_state[key]
210
+ if tuple(value.shape) != tuple(target.shape):
211
+ raise ValueError(
212
+ f"Shape mismatch for `{key}` in {action_dit_pretrained_path}: "
213
+ f"expected {tuple(target.shape)}, got {tuple(value.shape)}"
214
+ )
215
+ merged_state[key] = value.to(device=target.device, dtype=target.dtype)
216
+
217
+ action_expert.load_state_dict(merged_state, strict=True)
218
+ logger.info(
219
+ "Loaded ActionDiT backbone from %s (keys=%d; random_kept_prefixes=%s).",
220
+ action_dit_pretrained_path,
221
+ len(expected_backbone_keys),
222
+ list(cls.ACTION_BACKBONE_SKIP_PREFIXES),
223
+ )
224
+ return action_expert.to(device=device, dtype=torch_dtype)
225
+
226
+ def pre_dit(
227
+ self,
228
+ action_tokens: torch.Tensor,
229
+ timestep: torch.Tensor,
230
+ context: torch.Tensor,
231
+ context_mask: Optional[torch.Tensor] = None,
232
+ ) -> Dict[str, Any]:
233
+ if action_tokens.ndim != 3:
234
+ raise ValueError(
235
+ f"`action_tokens` must be 3D [B, T, action_dim], got shape {tuple(action_tokens.shape)}"
236
+ )
237
+ if action_tokens.shape[2] != self.action_dim:
238
+ raise ValueError(
239
+ f"`action_tokens` last dim must be {self.action_dim}, got {action_tokens.shape[2]}"
240
+ )
241
+ if timestep.ndim != 1:
242
+ raise ValueError(f"`timestep` must be 1D [B] or [1], got shape {tuple(timestep.shape)}")
243
+ if context.ndim != 3:
244
+ raise ValueError(
245
+ f"`context` must be 3D [B, L, D], got shape {tuple(context.shape)}"
246
+ )
247
+
248
+ batch_size = action_tokens.shape[0]
249
+ if context.shape[0] != batch_size:
250
+ raise ValueError(
251
+ f"Batch mismatch between action tokens and text context: {batch_size} vs {context.shape[0]}"
252
+ )
253
+ if timestep.shape[0] not in (1, batch_size):
254
+ raise ValueError(
255
+ f"`timestep` length must be 1 or batch_size({batch_size}), got {timestep.shape[0]}"
256
+ )
257
+ if timestep.shape[0] == 1 and batch_size > 1:
258
+ if self.training:
259
+ raise ValueError("During training, action timestep length must match batch_size.")
260
+ timestep = timestep.expand(batch_size)
261
+
262
+ if context_mask is None:
263
+ context_mask = torch.ones(
264
+ (batch_size, context.shape[1]), dtype=torch.bool, device=context.device
265
+ )
266
+ else:
267
+ if context_mask.ndim != 2:
268
+ raise ValueError(f"`context_mask` must be 2D [B, L], got shape {tuple(context_mask.shape)}")
269
+ if context_mask.shape[0] != batch_size or context_mask.shape[1] != context.shape[1]:
270
+ raise ValueError(
271
+ f"`context_mask` shape must match `context` shape [B, L], got {tuple(context_mask.shape)} vs {tuple(context.shape)}"
272
+ )
273
+
274
+ seq_len = action_tokens.shape[1]
275
+ if seq_len > self.freqs.shape[0]:
276
+ raise ValueError(
277
+ f"Action token length {seq_len} exceeds RoPE cache {self.freqs.shape[0]}."
278
+ )
279
+
280
+ t = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timestep))
281
+ t_mod = self.time_projection(t).unflatten(1, (6, self.hidden_dim))
282
+
283
+ tokens = self.action_encoder(action_tokens)
284
+ context_emb = self.text_embedding(context)
285
+ context_attn_mask = context_mask.unsqueeze(1).expand(-1, seq_len, -1)
286
+ freqs = self.freqs[:seq_len].view(seq_len, 1, -1).to(tokens.device)
287
+
288
+ return {
289
+ "tokens": tokens,
290
+ "freqs": freqs,
291
+ "t": t,
292
+ "t_mod": t_mod,
293
+ "context": context_emb,
294
+ "context_mask": context_attn_mask,
295
+ "meta": {
296
+ "batch_size": batch_size,
297
+ "seq_len": seq_len,
298
+ },
299
+ }
300
+
301
+ def post_dit(self, tokens: torch.Tensor, pre_state: Dict[str, Any]) -> torch.Tensor:
302
+ return self.head(tokens)
303
+
304
+ def forward(
305
+ self,
306
+ action_tokens: torch.Tensor,
307
+ timestep: torch.Tensor,
308
+ context: torch.Tensor,
309
+ context_mask: Optional[torch.Tensor] = None,
310
+ ) -> torch.Tensor:
311
+ pre_state = self.pre_dit(
312
+ action_tokens=action_tokens,
313
+ timestep=timestep,
314
+ context=context,
315
+ context_mask=context_mask,
316
+ )
317
+ x = pre_state["tokens"]
318
+ context = pre_state["context"]
319
+ t_mod = pre_state["t_mod"]
320
+ freqs = pre_state["freqs"]
321
+ context_mask = pre_state["context_mask"]
322
+
323
+ for block in self.blocks:
324
+ if self.use_gradient_checkpointing:
325
+ x = gradient_checkpoint_forward(
326
+ block,
327
+ self.use_gradient_checkpointing,
328
+ x,
329
+ context,
330
+ t_mod,
331
+ freqs,
332
+ context_mask=context_mask,
333
+ )
334
+ else:
335
+ x = block(x, context, t_mod, freqs, context_mask=context_mask)
336
+
337
+ return self.post_dit(x, pre_state)
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/attention_masks.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Structured attention masks for video/action joint denoising.
2
+
3
+ Boolean masks follow PyTorch SDPA semantics: ``True`` means that the query is
4
+ allowed to attend to the key. Rows are queries and columns are keys.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import torch
10
+
11
+
12
+ def build_frame_causal_video_mask(
13
+ video_seq_len: int,
14
+ video_tokens_per_frame: int,
15
+ *,
16
+ device: torch.device | str | None = None,
17
+ ) -> torch.Tensor:
18
+ """Build block-causal video attention with bidirectional intra-frame attention.
19
+
20
+ Every token in frame ``i`` can see every token in frames ``0..i``. Thus
21
+ spatial tokens inside a frame are bidirectional, while information cannot
22
+ travel directly from a future frame to a past frame through this sub-mask.
23
+ """
24
+ if video_seq_len <= 0:
25
+ raise ValueError(f"`video_seq_len` must be > 0, got {video_seq_len}")
26
+ if video_tokens_per_frame <= 0:
27
+ raise ValueError(
28
+ "`video_tokens_per_frame` must be > 0, "
29
+ f"got {video_tokens_per_frame}"
30
+ )
31
+ if video_seq_len % video_tokens_per_frame != 0:
32
+ raise ValueError(
33
+ "`video_seq_len` must be divisible by `video_tokens_per_frame`, "
34
+ f"got {video_seq_len} and {video_tokens_per_frame}"
35
+ )
36
+
37
+ num_frames = video_seq_len // video_tokens_per_frame
38
+ frame_ids = torch.arange(num_frames, device=device).repeat_interleave(
39
+ video_tokens_per_frame
40
+ )
41
+ query_frame = frame_ids[:, None]
42
+ key_frame = frame_ids[None, :]
43
+ return key_frame <= query_frame
44
+
45
+
46
+ def build_joint_denoise_attention_mask(
47
+ video_seq_len: int,
48
+ action_seq_len: int,
49
+ video_tokens_per_frame: int,
50
+ *,
51
+ device: torch.device | str | None = None,
52
+ ) -> torch.Tensor:
53
+ """Build shared-self-attention visibility for joint video/action denoising.
54
+
55
+ The four query/key quadrants are:
56
+
57
+ * video -> video: frame causal, intra-frame bidirectional;
58
+ * video -> action: fully visible;
59
+ * action -> video: fully visible;
60
+ * action -> action: bidirectional.
61
+
62
+ This differs from upstream ``FastWAMJoint``, whose released mask permits
63
+ action -> video but blocks video -> action.
64
+
65
+ Full cross-modal visibility can indirectly carry future-video information
66
+ through action tokens in deeper layers. Consequently only the direct
67
+ video/video sub-mask is causal; the whole multimodal graph is not strictly
68
+ causal.
69
+ """
70
+ if action_seq_len <= 0:
71
+ raise ValueError(f"`action_seq_len` must be > 0, got {action_seq_len}")
72
+
73
+ total_seq_len = video_seq_len + action_seq_len
74
+ mask = torch.ones(
75
+ (total_seq_len, total_seq_len), dtype=torch.bool, device=device
76
+ )
77
+ mask[:video_seq_len, :video_seq_len] = build_frame_causal_video_mask(
78
+ video_seq_len,
79
+ video_tokens_per_frame,
80
+ device=device,
81
+ )
82
+ return mask
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam.py ADDED
@@ -0,0 +1,1122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, Optional, Sequence, Union
2
+
3
+ import torch
4
+ import torch.nn as nn
5
+ import torch.nn.functional as F
6
+ from PIL import Image
7
+
8
+ from fastwam.utils.logging_config import get_logger
9
+
10
+ from .action_dit import ActionDiT
11
+ from .helpers.loader import load_wan22_ti2v_5b_components
12
+ from .mot import MoT
13
+ from .schedulers.scheduler_continuous import WanContinuousFlowMatchScheduler
14
+
15
+ logger = get_logger(__name__)
16
+
17
+
18
+ class FastWAM(torch.nn.Module):
19
+ """MoT world model with video/action experts."""
20
+
21
+ def __init__(
22
+ self,
23
+ video_expert,
24
+ action_expert: ActionDiT,
25
+ mot: MoT,
26
+ vae,
27
+ text_encoder=None,
28
+ tokenizer=None,
29
+ text_dim: Optional[int] = None,
30
+ proprio_dim: Optional[int] = None,
31
+ device: str = "cpu",
32
+ torch_dtype: torch.dtype = torch.float32,
33
+ video_train_shift: float = 5.0,
34
+ video_infer_shift: float = 5.0,
35
+ video_num_train_timesteps: int = 1000,
36
+ action_train_shift: float = 5.0,
37
+ action_infer_shift: float = 5.0,
38
+ action_num_train_timesteps: int = 1000,
39
+ loss_lambda_video: float = 1.0,
40
+ loss_lambda_action: float = 1.0,
41
+ ):
42
+ super().__init__()
43
+ self.video_expert = video_expert
44
+ self.action_expert = action_expert
45
+ self.mot = mot
46
+ # Keep trainer compatibility: optimizer and freeze logic use `model.dit`.
47
+ self.dit = self.mot
48
+
49
+ self.vae = vae
50
+ self.text_encoder = text_encoder
51
+ self.tokenizer = tokenizer
52
+ if text_dim is None:
53
+ if self.text_encoder is None:
54
+ raise ValueError("`text_dim` is required when `text_encoder` is not loaded.")
55
+ text_dim = int(self.text_encoder.dim)
56
+ self.text_dim = int(text_dim)
57
+ self.proprio_dim = None if proprio_dim is None else int(proprio_dim)
58
+ if self.proprio_dim is not None:
59
+ self.proprio_encoder = nn.Linear(self.proprio_dim, self.text_dim).to(torch_dtype)
60
+ else:
61
+ self.proprio_encoder = None
62
+
63
+ self.train_video_scheduler = WanContinuousFlowMatchScheduler(
64
+ num_train_timesteps=video_num_train_timesteps,
65
+ shift=video_train_shift,
66
+ )
67
+ self.infer_video_scheduler = WanContinuousFlowMatchScheduler(
68
+ num_train_timesteps=video_num_train_timesteps,
69
+ shift=video_infer_shift,
70
+ )
71
+ self.train_action_scheduler = WanContinuousFlowMatchScheduler(
72
+ num_train_timesteps=action_num_train_timesteps,
73
+ shift=action_train_shift,
74
+ )
75
+ self.infer_action_scheduler = WanContinuousFlowMatchScheduler(
76
+ num_train_timesteps=action_num_train_timesteps,
77
+ shift=action_infer_shift,
78
+ )
79
+ # Optional aliases for consistency with Wan22Core naming.
80
+ self.train_scheduler = self.train_video_scheduler
81
+ self.infer_scheduler = self.infer_video_scheduler
82
+
83
+ self.device = torch.device(device)
84
+ self.torch_dtype = torch_dtype
85
+ self.loss_lambda_video = float(loss_lambda_video)
86
+ self.loss_lambda_action = float(loss_lambda_action)
87
+
88
+ self.to(self.device)
89
+
90
+ @classmethod
91
+ def from_wan22_pretrained(
92
+ cls,
93
+ device: str = "cuda",
94
+ torch_dtype: torch.dtype = torch.bfloat16,
95
+ model_id: str = "Wan-AI/Wan2.2-TI2V-5B",
96
+ tokenizer_model_id: str = "Wan-AI/Wan2.1-T2V-1.3B",
97
+ tokenizer_max_len: int = 512,
98
+ load_text_encoder: bool = True,
99
+ proprio_dim: Optional[int] = None,
100
+ redirect_common_files: bool = True,
101
+ video_dit_config: dict[str, Any] | None = None,
102
+ action_dit_config: dict[str, Any] | None = None,
103
+ action_dit_pretrained_path: str | None = None,
104
+ skip_dit_load_from_pretrain: bool = False,
105
+ mot_checkpoint_mixed_attn: bool = True,
106
+ video_train_shift: float = 5.0,
107
+ video_infer_shift: float = 5.0,
108
+ video_num_train_timesteps: int = 1000,
109
+ action_train_shift: float = 5.0,
110
+ action_infer_shift: float = 5.0,
111
+ action_num_train_timesteps: int = 1000,
112
+ loss_lambda_video: float = 1.0,
113
+ loss_lambda_action: float = 1.0,
114
+ ):
115
+ if video_dit_config is None:
116
+ raise ValueError("`video_dit_config` is required for FastWAM.from_wan22_pretrained().")
117
+ if "text_dim" not in video_dit_config:
118
+ raise ValueError("`video_dit_config['text_dim']` is required for FastWAM.")
119
+
120
+ components = load_wan22_ti2v_5b_components(
121
+ device=device,
122
+ torch_dtype=torch_dtype,
123
+ model_id=model_id,
124
+ tokenizer_model_id=tokenizer_model_id,
125
+ tokenizer_max_len=tokenizer_max_len,
126
+ redirect_common_files=redirect_common_files,
127
+ dit_config=video_dit_config,
128
+ skip_dit_load_from_pretrain=skip_dit_load_from_pretrain,
129
+ load_text_encoder=load_text_encoder,
130
+ )
131
+
132
+ video_expert = components.dit
133
+ action_expert = ActionDiT.from_pretrained(
134
+ action_dit_config=action_dit_config,
135
+ action_dit_pretrained_path=action_dit_pretrained_path,
136
+ skip_dit_load_from_pretrain=skip_dit_load_from_pretrain,
137
+ device=device,
138
+ torch_dtype=torch_dtype,
139
+ )
140
+ if int(action_expert.num_heads) != int(video_expert.num_heads):
141
+ raise ValueError("ActionDiT `num_heads` must match video expert for MoT mixed attention.")
142
+ if int(action_expert.attn_head_dim) != int(video_expert.attn_head_dim):
143
+ raise ValueError("ActionDiT `attn_head_dim` must match video expert for MoT mixed attention.")
144
+ if int(len(action_expert.blocks)) != int(len(video_expert.blocks)):
145
+ raise ValueError("ActionDiT `num_layers` must match video expert.")
146
+
147
+ mot = MoT(
148
+ mixtures={"video": video_expert, "action": action_expert},
149
+ mot_checkpoint_mixed_attn=mot_checkpoint_mixed_attn,
150
+ )
151
+
152
+ model = cls(
153
+ video_expert=video_expert,
154
+ action_expert=action_expert,
155
+ mot=mot,
156
+ vae=components.vae,
157
+ text_encoder=components.text_encoder,
158
+ tokenizer=components.tokenizer,
159
+ text_dim=int(video_dit_config["text_dim"]),
160
+ proprio_dim=proprio_dim,
161
+ device=device,
162
+ torch_dtype=torch_dtype,
163
+ video_train_shift=video_train_shift,
164
+ video_infer_shift=video_infer_shift,
165
+ video_num_train_timesteps=video_num_train_timesteps,
166
+ action_train_shift=action_train_shift,
167
+ action_infer_shift=action_infer_shift,
168
+ action_num_train_timesteps=action_num_train_timesteps,
169
+ loss_lambda_video=loss_lambda_video,
170
+ loss_lambda_action=loss_lambda_action,
171
+ )
172
+ model.model_paths = {
173
+ "video_dit": components.dit_path,
174
+ "vae": components.vae_path,
175
+ "text_encoder": components.text_encoder_path,
176
+ "tokenizer": components.tokenizer_path,
177
+ "action_dit_backbone": (
178
+ "SKIPPED_PRETRAIN" if skip_dit_load_from_pretrain else action_dit_pretrained_path
179
+ ),
180
+ }
181
+ return model
182
+
183
+ def to(self, *args, **kwargs):
184
+ super().to(*args, **kwargs)
185
+ self.mot.to(*args, **kwargs)
186
+ if self.text_encoder is not None:
187
+ self.text_encoder.to(*args, **kwargs)
188
+ self.vae.to(*args, **kwargs)
189
+ return self
190
+
191
+ @staticmethod
192
+ def _check_resize_height_width(height, width, num_frames):
193
+ if height % 16 != 0:
194
+ height = (height + 15) // 16 * 16
195
+ if width % 16 != 0:
196
+ width = (width + 15) // 16 * 16
197
+ if num_frames % 4 != 1:
198
+ num_frames = (num_frames + 3) // 4 * 4 + 1
199
+ return height, width, num_frames
200
+
201
+ @torch.no_grad()
202
+ def encode_prompt(self, prompt: Union[str, Sequence[str]]):
203
+ if self.text_encoder is None or self.tokenizer is None:
204
+ raise ValueError(
205
+ "Prompt encoding requires loaded text encoder/tokenizer. "
206
+ "Set `load_text_encoder=true` or provide precomputed `context/context_mask`."
207
+ )
208
+ ids, mask = self.tokenizer(prompt, return_mask=True, add_special_tokens=True)
209
+ ids = ids.to(self.device)
210
+ mask = mask.to(self.device, dtype=torch.bool)
211
+ prompt_emb = self.text_encoder(ids, mask)
212
+ # FIXME: original implementation's zero padding is visible in cross-attn.
213
+ seq_lens = mask.gt(0).sum(dim=1).long()
214
+ for i, v in enumerate(seq_lens):
215
+ prompt_emb[i, v:] = 0
216
+ mask = torch.ones_like(mask)
217
+ return prompt_emb.to(device=self.device), mask
218
+
219
+ def _append_proprio_to_context(
220
+ self,
221
+ context: torch.Tensor,
222
+ context_mask: torch.Tensor,
223
+ proprio: Optional[torch.Tensor],
224
+ ) -> tuple[torch.Tensor, torch.Tensor]:
225
+ if self.proprio_encoder is None or proprio is None:
226
+ return context, context_mask
227
+ if proprio.ndim != 2:
228
+ raise ValueError(f"`proprio` must be 2D [B, D], got shape {tuple(proprio.shape)}")
229
+ if self.proprio_dim is None or proprio.shape[1] != self.proprio_dim:
230
+ raise ValueError(
231
+ f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}"
232
+ )
233
+ proprio_token = self.proprio_encoder(
234
+ proprio.to(device=self.device, dtype=context.dtype).unsqueeze(1)
235
+ ).to(dtype=context.dtype) # [B, 1, D]
236
+ proprio_mask = torch.ones((context_mask.shape[0], 1), dtype=torch.bool, device=context_mask.device)
237
+ return (
238
+ torch.cat([context, proprio_token], dim=1),
239
+ torch.cat([context_mask, proprio_mask], dim=1),
240
+ )
241
+
242
+ @torch.no_grad()
243
+ def _encode_video_latents(self, video_tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)):
244
+ z = self.vae.encode(
245
+ video_tensor,
246
+ device=self.device,
247
+ tiled=tiled,
248
+ tile_size=tile_size,
249
+ tile_stride=tile_stride,
250
+ )
251
+ return z
252
+
253
+ @torch.no_grad()
254
+ def _encode_input_image_latents_tensor(self, input_image: torch.Tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)):
255
+ if input_image.ndim == 3:
256
+ input_image = input_image.unsqueeze(0)
257
+ if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
258
+ raise ValueError(
259
+ f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
260
+ )
261
+ image = input_image.to(device=self.device)[0].unsqueeze(1)
262
+ z = self.vae.encode([image], device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride)
263
+ if isinstance(z, list):
264
+ z = z[0].unsqueeze(0)
265
+ return z
266
+
267
+ def _decode_latents(self, latents, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)):
268
+ video_tensor = self.vae.decode(latents, device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride)
269
+ video_tensor = video_tensor.squeeze(0).detach().float().clamp(-1, 1)
270
+ video_tensor = ((video_tensor + 1.0) * 127.5).to(torch.uint8).cpu()
271
+ frames = []
272
+ for t in range(video_tensor.shape[1]):
273
+ frame = video_tensor[:, t].permute(1, 2, 0).numpy()
274
+ frames.append(Image.fromarray(frame))
275
+ return frames
276
+
277
+ def build_inputs(self, sample, tiled: bool = False):
278
+ video = sample["video"]
279
+ if "context" not in sample or "context_mask" not in sample:
280
+ raise ValueError(
281
+ "FastWAM training requires `sample['context']` and `sample['context_mask']`."
282
+ )
283
+ context = sample["context"]
284
+ context_mask = sample["context_mask"]
285
+ proprio = sample.get("proprio", None)
286
+ if video.ndim != 5:
287
+ raise ValueError(f"`sample['video']` must be 5D [B, 3, T, H, W], got shape {tuple(video.shape)}")
288
+ if video.shape[1] != 3:
289
+ raise ValueError(f"`sample['video']` channel dimension must be 3, got shape {tuple(video.shape)}")
290
+
291
+ batch_size, _, num_frames, height, width = video.shape
292
+ if height % 16 != 0 or width % 16 != 0:
293
+ raise ValueError(
294
+ f"Video spatial dims must be multiples of 16, got H={height}, W={width}"
295
+ )
296
+ if num_frames % 4 != 1:
297
+ raise ValueError(f"Video T must satisfy T % 4 == 1, got T={num_frames}")
298
+ if num_frames <= 1:
299
+ raise ValueError(f"Video T must be > 1 for action-conditioned training, got T={num_frames}")
300
+
301
+ if "action" not in sample:
302
+ raise ValueError("`sample['action']` is required for FastWAM training.")
303
+
304
+ action = sample["action"]
305
+ if action.ndim != 3:
306
+ raise ValueError(f"`sample['action']` must be 3D [B, T, a_dim], got shape {tuple(action.shape)}")
307
+ action_horizon = int(action.shape[1])
308
+ if action_horizon % (num_frames - 1) != 0:
309
+ raise ValueError(
310
+ f"`sample['action']` temporal dimension must be divisible by video transitions ({num_frames - 1}), got {action_horizon}"
311
+ )
312
+
313
+ action_is_pad = sample.get("action_is_pad", None)
314
+ if action_is_pad is not None:
315
+ if action_is_pad.ndim != 2:
316
+ raise ValueError(
317
+ f"`sample['action_is_pad']` must be 2D [B, T], got shape {tuple(action_is_pad.shape)}"
318
+ )
319
+ if action_is_pad.shape[0] != batch_size or action_is_pad.shape[1] != action_horizon:
320
+ raise ValueError(
321
+ "`sample['action_is_pad']` shape mismatch: "
322
+ f"got {tuple(action_is_pad.shape)} vs expected ({batch_size}, {action_horizon})"
323
+ )
324
+
325
+ image_is_pad = sample.get("image_is_pad", None)
326
+ if image_is_pad is not None:
327
+ if image_is_pad.ndim != 2:
328
+ raise ValueError(
329
+ f"`sample['image_is_pad']` must be 2D [B, T], got shape {tuple(image_is_pad.shape)}"
330
+ )
331
+ if image_is_pad.shape[0] != batch_size or image_is_pad.shape[1] != num_frames:
332
+ raise ValueError(
333
+ "`sample['image_is_pad']` shape mismatch: "
334
+ f"got {tuple(image_is_pad.shape)} vs expected ({batch_size}, {num_frames})"
335
+ )
336
+
337
+ input_video = video.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
338
+ input_latents = self._encode_video_latents(input_video, tiled=tiled)
339
+
340
+ first_frame_latents = None
341
+ fuse_flag = False
342
+ if getattr(self.video_expert, "fuse_vae_embedding_in_latents", False):
343
+ first_frame_latents = input_latents[:, :, 0:1]
344
+ fuse_flag = True
345
+
346
+ if context.ndim != 3 or context_mask.ndim != 2:
347
+ raise ValueError(
348
+ f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}"
349
+ )
350
+ context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
351
+ context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True)
352
+ if self.proprio_encoder is not None:
353
+ if proprio is None:
354
+ raise ValueError("`sample['proprio']` is required when `proprio_dim` is enabled.")
355
+ if proprio.ndim != 3:
356
+ raise ValueError(f"`sample['proprio']` must be 3D [B, T, d], got shape {tuple(proprio.shape)}")
357
+ if proprio.shape[2] != self.proprio_dim:
358
+ raise ValueError(
359
+ f"`sample['proprio']` last dim must be {self.proprio_dim}, got {proprio.shape[2]}"
360
+ )
361
+ proprio = proprio[:, 0, :] # [B, D]
362
+ context, context_mask = self._append_proprio_to_context(
363
+ context=context,
364
+ context_mask=context_mask,
365
+ proprio=proprio.to(device=self.device, dtype=self.torch_dtype),
366
+ )
367
+ action = action.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
368
+
369
+ if action_is_pad is not None:
370
+ action_is_pad = action_is_pad.to(device=self.device, dtype=torch.bool, non_blocking=True)
371
+ if image_is_pad is not None:
372
+ image_is_pad = image_is_pad.to(device=self.device, dtype=torch.bool, non_blocking=True)
373
+
374
+ return {
375
+ "context": context,
376
+ "context_mask": context_mask,
377
+ "input_latents": input_latents,
378
+ "first_frame_latents": first_frame_latents,
379
+ "fuse_vae_embedding_in_latents": fuse_flag,
380
+ "action": action,
381
+ "action_is_pad": action_is_pad,
382
+ "image_is_pad": image_is_pad,
383
+ }
384
+
385
+ @torch.no_grad()
386
+ def _build_mot_attention_mask(
387
+ self,
388
+ video_seq_len: int,
389
+ action_seq_len: int,
390
+ video_tokens_per_frame: int,
391
+ device: torch.device,
392
+ ) -> torch.Tensor:
393
+ total_seq_len = video_seq_len + action_seq_len
394
+ mask = torch.zeros((total_seq_len, total_seq_len), dtype=torch.bool, device=device)
395
+
396
+ # video -> video
397
+ mask[:video_seq_len, :video_seq_len] = self.video_expert.build_video_to_video_mask(
398
+ video_seq_len=video_seq_len,
399
+ video_tokens_per_frame=video_tokens_per_frame,
400
+ device=device,
401
+ )
402
+ # action -> action
403
+ mask[video_seq_len:, video_seq_len:] = True
404
+ # action -> first-frame video only
405
+ first_frame_tokens = min(video_tokens_per_frame, video_seq_len)
406
+ mask[video_seq_len:, :first_frame_tokens] = True
407
+ return mask
408
+
409
+ def _compute_video_loss_per_sample(
410
+ self,
411
+ pred_video: torch.Tensor,
412
+ target_video: torch.Tensor,
413
+ image_is_pad: Optional[torch.Tensor],
414
+ include_initial_video_step: bool,
415
+ ) -> torch.Tensor:
416
+ video_loss_token = F.mse_loss(pred_video.float(), target_video.float(), reduction="none").mean(dim=(1, 3, 4))
417
+ if image_is_pad is None:
418
+ return video_loss_token.mean(dim=1)
419
+
420
+ temporal_factor = int(self.vae.temporal_downsample_factor)
421
+ if temporal_factor <= 0:
422
+ raise ValueError(f"`vae.temporal_downsample_factor` must be positive, got {temporal_factor}.")
423
+ if image_is_pad.shape[1] < 1:
424
+ raise ValueError("`image_is_pad` must contain at least one frame.")
425
+ if (image_is_pad.shape[1] - 1) % temporal_factor != 0:
426
+ raise ValueError(
427
+ "Cannot align `image_is_pad` with video latent steps: "
428
+ f"num_frames={image_is_pad.shape[1]}, temporal_downsample_factor={temporal_factor}."
429
+ )
430
+
431
+ tail_is_pad = image_is_pad[:, 1:]
432
+ latent_tail_is_pad = tail_is_pad.view(image_is_pad.shape[0], -1, temporal_factor).all(dim=2)
433
+ if include_initial_video_step:
434
+ video_is_pad = torch.cat([image_is_pad[:, :1], latent_tail_is_pad], dim=1)
435
+ else:
436
+ video_is_pad = latent_tail_is_pad
437
+
438
+ if video_is_pad.shape[1] != video_loss_token.shape[1]:
439
+ raise ValueError(
440
+ "Video-loss mask shape mismatch: "
441
+ f"mask steps={video_is_pad.shape[1]}, loss steps={video_loss_token.shape[1]}."
442
+ )
443
+
444
+ valid = (~video_is_pad).to(device=video_loss_token.device, dtype=video_loss_token.dtype)
445
+ valid_sum = valid.sum(dim=1).clamp(min=1.0)
446
+ return (video_loss_token * valid).sum(dim=1) / valid_sum
447
+
448
+ def training_loss(self, sample, tiled: bool = False):
449
+ inputs = self.build_inputs(sample, tiled=tiled)
450
+ input_latents = inputs["input_latents"]
451
+ batch_size = input_latents.shape[0]
452
+ context = inputs["context"]
453
+ context_mask = inputs["context_mask"]
454
+ action = inputs["action"]
455
+ action_is_pad = inputs["action_is_pad"]
456
+ image_is_pad = inputs["image_is_pad"]
457
+
458
+ noise_video = torch.randn_like(input_latents)
459
+ timestep_video = self.train_video_scheduler.sample_training_t(
460
+ batch_size=batch_size,
461
+ device=self.device,
462
+ dtype=input_latents.dtype,
463
+ )
464
+ latents = self.train_video_scheduler.add_noise(input_latents, noise_video, timestep_video)
465
+ target_video = self.train_video_scheduler.training_target(input_latents, noise_video, timestep_video)
466
+
467
+ if inputs["first_frame_latents"] is not None:
468
+ latents[:, :, 0:1] = inputs["first_frame_latents"]
469
+
470
+ noise_action = torch.randn_like(action)
471
+ timestep_action = self.train_action_scheduler.sample_training_t(
472
+ batch_size=batch_size,
473
+ device=self.device,
474
+ dtype=action.dtype,
475
+ )
476
+ noisy_action = self.train_action_scheduler.add_noise(action, noise_action, timestep_action)
477
+ target_action = self.train_action_scheduler.training_target(action, noise_action, timestep_action)
478
+
479
+ video_pre = self.video_expert.pre_dit(
480
+ x=latents,
481
+ timestep=timestep_video,
482
+ context=context,
483
+ context_mask=context_mask,
484
+ action=action,
485
+ fuse_vae_embedding_in_latents=inputs["fuse_vae_embedding_in_latents"],
486
+ )
487
+
488
+ action_pre = self.action_expert.pre_dit(
489
+ action_tokens=noisy_action,
490
+ timestep=timestep_action,
491
+ context=context,
492
+ context_mask=context_mask,
493
+ )
494
+
495
+ video_tokens = video_pre["tokens"]
496
+ action_tokens = action_pre["tokens"]
497
+
498
+ attention_mask = self._build_mot_attention_mask(
499
+ video_seq_len=video_tokens.shape[1],
500
+ action_seq_len=action_tokens.shape[1],
501
+ video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]),
502
+ device=video_tokens.device,
503
+ )
504
+ tokens_out = self.mot(
505
+ embeds_all={
506
+ "video": video_tokens,
507
+ "action": action_tokens,
508
+ },
509
+ attention_mask=attention_mask,
510
+ freqs_all={
511
+ "video": video_pre["freqs"],
512
+ "action": action_pre["freqs"],
513
+ },
514
+ context_all={
515
+ "video": {
516
+ "context": video_pre["context"],
517
+ "mask": video_pre["context_mask"],
518
+ },
519
+ "action": {
520
+ "context": action_pre["context"],
521
+ "mask": action_pre["context_mask"],
522
+ },
523
+ },
524
+ t_mod_all={
525
+ "video": video_pre["t_mod"],
526
+ "action": action_pre["t_mod"],
527
+ },
528
+ )
529
+
530
+ pred_video = self.video_expert.post_dit(tokens_out["video"], video_pre)
531
+
532
+ pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre)
533
+
534
+ include_initial_video_step = inputs["first_frame_latents"] is None
535
+ if inputs["first_frame_latents"] is not None:
536
+ pred_video = pred_video[:, :, 1:]
537
+ target_video = target_video[:, :, 1:]
538
+
539
+ loss_video_per_sample = self._compute_video_loss_per_sample(
540
+ pred_video=pred_video,
541
+ target_video=target_video,
542
+ image_is_pad=image_is_pad,
543
+ include_initial_video_step=include_initial_video_step,
544
+ )
545
+ video_weight = self.train_video_scheduler.training_weight(timestep_video).to(
546
+ loss_video_per_sample.device, dtype=loss_video_per_sample.dtype
547
+ )
548
+ loss_video = (loss_video_per_sample * video_weight).mean()
549
+
550
+ action_loss_token = F.mse_loss(pred_action.float(), target_action.float(), reduction="none").mean(dim=2) # [B, T]
551
+ if action_is_pad is not None:
552
+ valid = (~action_is_pad).to(device=action_loss_token.device, dtype=action_loss_token.dtype)
553
+ valid_sum = valid.sum(dim=1).clamp(min=1.0)
554
+ action_loss_per_sample = (action_loss_token * valid).sum(dim=1) / valid_sum
555
+ else:
556
+ action_loss_per_sample = action_loss_token.mean(dim=1)
557
+
558
+ action_weight = self.train_action_scheduler.training_weight(timestep_action).to(
559
+ action_loss_per_sample.device, dtype=action_loss_per_sample.dtype
560
+ )
561
+ loss_action = (action_loss_per_sample * action_weight).mean()
562
+
563
+ loss_total = self.loss_lambda_video * loss_video + self.loss_lambda_action * loss_action
564
+ loss_dict = {
565
+ "loss_video": self.loss_lambda_video * float(loss_video.detach().item()),
566
+ "loss_action": self.loss_lambda_action * float(loss_action.detach().item()),
567
+ }
568
+ return loss_total, loss_dict
569
+
570
+ @torch.no_grad()
571
+ def _predict_joint_noise(
572
+ self,
573
+ latents_video: torch.Tensor,
574
+ latents_action: torch.Tensor,
575
+ timestep_video: torch.Tensor,
576
+ timestep_action: torch.Tensor,
577
+ context: torch.Tensor,
578
+ context_mask: torch.Tensor,
579
+ fuse_vae_embedding_in_latents: bool,
580
+ gt_action: Optional[torch.Tensor] = None,
581
+ ) -> tuple[torch.Tensor, torch.Tensor]:
582
+ video_pre = self.video_expert.pre_dit(
583
+ x=latents_video,
584
+ timestep=timestep_video,
585
+ context=context,
586
+ context_mask=context_mask,
587
+ action=gt_action,
588
+ fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents,
589
+ )
590
+ action_pre = self.action_expert.pre_dit(
591
+ action_tokens=latents_action,
592
+ timestep=timestep_action,
593
+ context=context,
594
+ context_mask=context_mask,
595
+ )
596
+
597
+ attention_mask = self._build_mot_attention_mask(
598
+ video_seq_len=video_pre["tokens"].shape[1],
599
+ action_seq_len=action_pre["tokens"].shape[1],
600
+ video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]),
601
+ device=video_pre["tokens"].device,
602
+ )
603
+
604
+ tokens_out = self.mot(
605
+ embeds_all={
606
+ "video": video_pre["tokens"],
607
+ "action": action_pre["tokens"],
608
+ },
609
+ attention_mask=attention_mask,
610
+ freqs_all={
611
+ "video": video_pre["freqs"],
612
+ "action": action_pre["freqs"],
613
+ },
614
+ context_all={
615
+ "video": {
616
+ "context": video_pre["context"],
617
+ "mask": video_pre["context_mask"],
618
+ },
619
+ "action": {
620
+ "context": action_pre["context"],
621
+ "mask": action_pre["context_mask"],
622
+ },
623
+ },
624
+ t_mod_all={
625
+ "video": video_pre["t_mod"],
626
+ "action": action_pre["t_mod"],
627
+ },
628
+ )
629
+
630
+ pred_video = self.video_expert.post_dit(tokens_out["video"], video_pre)
631
+ pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre)
632
+ return pred_video, pred_action
633
+
634
+ @torch.no_grad()
635
+ def _predict_action_noise(
636
+ self,
637
+ first_frame_latents: torch.Tensor,
638
+ latents_action: torch.Tensor,
639
+ timestep_action: torch.Tensor,
640
+ context: torch.Tensor,
641
+ context_mask: torch.Tensor,
642
+ fuse_vae_embedding_in_latents: bool,
643
+ ) -> torch.Tensor:
644
+ timestep_video = torch.zeros_like(timestep_action, dtype=first_frame_latents.dtype, device=self.device)
645
+ video_pre = self.video_expert.pre_dit(
646
+ x=first_frame_latents,
647
+ timestep=timestep_video,
648
+ context=context,
649
+ context_mask=context_mask,
650
+ action=None,
651
+ fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents,
652
+ )
653
+ action_pre = self.action_expert.pre_dit(
654
+ action_tokens=latents_action,
655
+ timestep=timestep_action,
656
+ context=context,
657
+ context_mask=context_mask,
658
+ )
659
+
660
+ attention_mask = self._build_mot_attention_mask(
661
+ video_seq_len=video_pre["tokens"].shape[1],
662
+ action_seq_len=action_pre["tokens"].shape[1],
663
+ video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]),
664
+ device=video_pre["tokens"].device,
665
+ )
666
+ tokens_out = self.mot(
667
+ embeds_all={
668
+ "video": video_pre["tokens"],
669
+ "action": action_pre["tokens"],
670
+ },
671
+ attention_mask=attention_mask,
672
+ freqs_all={
673
+ "video": video_pre["freqs"],
674
+ "action": action_pre["freqs"],
675
+ },
676
+ context_all={
677
+ "video": {
678
+ "context": video_pre["context"],
679
+ "mask": video_pre["context_mask"],
680
+ },
681
+ "action": {
682
+ "context": action_pre["context"],
683
+ "mask": action_pre["context_mask"],
684
+ },
685
+ },
686
+ t_mod_all={
687
+ "video": video_pre["t_mod"],
688
+ "action": action_pre["t_mod"],
689
+ },
690
+ )
691
+ pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre)
692
+ return pred_action
693
+
694
+ @torch.no_grad()
695
+ def _predict_action_noise_with_cache(
696
+ self,
697
+ latents_action: torch.Tensor,
698
+ timestep_action: torch.Tensor,
699
+ context: torch.Tensor,
700
+ context_mask: torch.Tensor,
701
+ video_kv_cache: list[dict[str, torch.Tensor]],
702
+ attention_mask: torch.Tensor,
703
+ video_seq_len: int,
704
+ ) -> torch.Tensor:
705
+ action_pre = self.action_expert.pre_dit(
706
+ action_tokens=latents_action,
707
+ timestep=timestep_action,
708
+ context=context,
709
+ context_mask=context_mask,
710
+ )
711
+ action_tokens = self.mot.forward_action_with_video_cache(
712
+ action_tokens=action_pre["tokens"],
713
+ action_freqs=action_pre["freqs"],
714
+ action_t_mod=action_pre["t_mod"],
715
+ action_context_payload={
716
+ "context": action_pre["context"],
717
+ "mask": action_pre["context_mask"],
718
+ },
719
+ video_kv_cache=video_kv_cache,
720
+ attention_mask=attention_mask,
721
+ video_seq_len=video_seq_len,
722
+ )
723
+ return self.action_expert.post_dit(action_tokens, action_pre)
724
+
725
+ @torch.no_grad()
726
+ def infer_joint(
727
+ self,
728
+ prompt: Optional[str],
729
+ input_image: torch.Tensor,
730
+ num_video_frames: int,
731
+ action_horizon: int,
732
+ action: Optional[torch.Tensor] = None, # NOTE: this is gt action for conditioning videos, not for action expert
733
+ proprio: Optional[torch.Tensor] = None,
734
+ context: Optional[torch.Tensor] = None,
735
+ context_mask: Optional[torch.Tensor] = None,
736
+ negative_prompt: Optional[str] = None,
737
+ text_cfg_scale: float = 1.0,
738
+ num_inference_steps: int = 20,
739
+ sigma_shift: Optional[float] = None,
740
+ seed: Optional[int] = None,
741
+ rand_device: str = "cpu",
742
+ tiled: bool = False,
743
+ test_action_with_infer_action: bool = True,
744
+ ) -> dict[str, Any]:
745
+ self.eval()
746
+ if test_action_with_infer_action:
747
+ if seed is None:
748
+ raise ValueError("`test_action_with_infer_action=True` requires non-null `seed`.")
749
+ action_only_out = self.infer_action(
750
+ prompt=prompt,
751
+ input_image=input_image.clone(),
752
+ action_horizon=action_horizon,
753
+ context=context.clone() if context is not None else None,
754
+ context_mask=context_mask.clone() if context_mask is not None else None,
755
+ num_inference_steps=num_inference_steps,
756
+ sigma_shift=sigma_shift,
757
+ seed=seed,
758
+ rand_device=rand_device,
759
+ tiled=tiled,
760
+ proprio=proprio.clone() if proprio is not None else None,
761
+ )["action"]
762
+
763
+ if input_image.ndim == 3:
764
+ input_image = input_image.unsqueeze(0)
765
+ if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
766
+ raise ValueError(
767
+ f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
768
+ )
769
+ _, _, height, width = input_image.shape
770
+ checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_video_frames)
771
+ if (checked_h, checked_w) != (height, width):
772
+ raise ValueError(
773
+ f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})"
774
+ )
775
+ if checked_t != num_video_frames:
776
+ raise ValueError(
777
+ f"`num_video_frames` must satisfy T % 4 == 1, got {num_video_frames}"
778
+ )
779
+ if action is not None:
780
+ if action.ndim == 2:
781
+ action = action.unsqueeze(0)
782
+ if action.ndim != 3 or action.shape[0] != 1 or action.shape[1] != action_horizon:
783
+ # NOTE: This enforces action condition to have the same shape as action horizon to predict, which may be unnecessary
784
+ raise ValueError(
785
+ f"`action` must have shape [1, T, a_dim] or [T, a_dim], got {tuple(action.shape)} with action_horizon={action_horizon}"
786
+ )
787
+ action = action.to(device=self.device, dtype=self.torch_dtype)
788
+ if proprio is not None:
789
+ if self.proprio_dim is None:
790
+ raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.")
791
+ if proprio.ndim == 1:
792
+ proprio = proprio.unsqueeze(0)
793
+ elif proprio.ndim == 2 and proprio.shape[0] == 1:
794
+ pass
795
+ else:
796
+ raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}")
797
+ if proprio.shape[1] != self.proprio_dim:
798
+ raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}")
799
+ proprio = proprio.to(device=self.device, dtype=self.torch_dtype)
800
+
801
+ latent_t = (num_video_frames - 1) // self.vae.temporal_downsample_factor + 1
802
+ latent_h = height // self.vae.upsampling_factor
803
+ latent_w = width // self.vae.upsampling_factor
804
+
805
+ video_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
806
+ action_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
807
+ latents_video = torch.randn(
808
+ (1, self.vae.model.z_dim, latent_t, latent_h, latent_w),
809
+ generator=video_generator,
810
+ device=rand_device,
811
+ dtype=torch.float32,
812
+ ).to(device=self.device, dtype=self.torch_dtype)
813
+ latents_action = torch.randn(
814
+ (1, action_horizon, self.action_expert.action_dim),
815
+ generator=action_generator,
816
+ device=rand_device,
817
+ dtype=torch.float32,
818
+ ).to(device=self.device, dtype=self.torch_dtype)
819
+
820
+ input_image = input_image.to(device=self.device, dtype=self.torch_dtype)
821
+ first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled)
822
+ latents_video[:, :, 0:1] = first_frame_latents.clone()
823
+ fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False))
824
+
825
+ use_prompt = prompt is not None
826
+ use_context = context is not None or context_mask is not None
827
+ if use_prompt and use_context:
828
+ raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.")
829
+ if not use_prompt and not use_context:
830
+ raise ValueError("Either `prompt` or both `context/context_mask` must be provided.")
831
+
832
+ if use_prompt:
833
+ context, context_mask = self.encode_prompt(prompt)
834
+ else:
835
+ if context is None or context_mask is None:
836
+ raise ValueError("`context` and `context_mask` must be both provided together.")
837
+ if context.ndim == 2:
838
+ context = context.unsqueeze(0)
839
+ if context_mask.ndim == 1:
840
+ context_mask = context_mask.unsqueeze(0)
841
+ if context.ndim != 3 or context_mask.ndim != 2:
842
+ raise ValueError(
843
+ f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}"
844
+ )
845
+ context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
846
+ context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True)
847
+ if proprio is not None:
848
+ context, context_mask = self._append_proprio_to_context(
849
+ context=context,
850
+ context_mask=context_mask,
851
+ proprio=proprio,
852
+ )
853
+
854
+ infer_timesteps_video, infer_deltas_video = self.infer_video_scheduler.build_inference_schedule(
855
+ num_inference_steps=num_inference_steps,
856
+ device=self.device,
857
+ dtype=latents_video.dtype,
858
+ shift_override=sigma_shift,
859
+ )
860
+ infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule(
861
+ num_inference_steps=num_inference_steps,
862
+ device=self.device,
863
+ dtype=latents_action.dtype,
864
+ shift_override=sigma_shift,
865
+ )
866
+ for step_t_video, step_delta_video, step_t_action, step_delta_action in zip(
867
+ infer_timesteps_video,
868
+ infer_deltas_video,
869
+ infer_timesteps_action,
870
+ infer_deltas_action,
871
+ ):
872
+ timestep_video = step_t_video.unsqueeze(0).to(dtype=latents_video.dtype, device=self.device)
873
+ timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device)
874
+
875
+ pred_video_posi, pred_action_posi = self._predict_joint_noise(
876
+ latents_video=latents_video,
877
+ latents_action=latents_action,
878
+ timestep_video=timestep_video,
879
+ timestep_action=timestep_action,
880
+ context=context,
881
+ context_mask=context_mask,
882
+ fuse_vae_embedding_in_latents=fuse_flag,
883
+ gt_action=action,
884
+ )
885
+ pred_video = pred_video_posi
886
+ pred_action = pred_action_posi
887
+
888
+ latents_video = self.infer_video_scheduler.step(pred_video, step_delta_video, latents_video)
889
+ latents_action = self.infer_action_scheduler.step(pred_action, step_delta_action, latents_action)
890
+ latents_video[:, :, 0:1] = first_frame_latents.clone()
891
+
892
+ action_out = latents_action[0].detach().to(device="cpu", dtype=torch.float32)
893
+ if test_action_with_infer_action:
894
+ if not torch.allclose(action_out, action_only_out, atol=1e-2, rtol=1e-2):
895
+ max_abs_diff = (action_out - action_only_out).abs().max().item()
896
+ logger.warning(
897
+ f"Action from infer_joint and infer_action differ with max abs diff {max_abs_diff:.6f}. "
898
+ )
899
+
900
+ return {
901
+ "video": self._decode_latents(latents_video, tiled=tiled),
902
+ "action": action_out,
903
+ }
904
+
905
+ @torch.no_grad()
906
+ def infer_action(
907
+ self,
908
+ prompt: Optional[str],
909
+ input_image: torch.Tensor,
910
+ action_horizon: int,
911
+ proprio: Optional[torch.Tensor] = None,
912
+ context: Optional[torch.Tensor] = None,
913
+ context_mask: Optional[torch.Tensor] = None,
914
+ negative_prompt: Optional[str] = None,
915
+ text_cfg_scale: float = 1.0,
916
+ num_inference_steps: int = 20,
917
+ sigma_shift: Optional[float] = None,
918
+ seed: Optional[int] = None,
919
+ rand_device: str = "cpu",
920
+ tiled: bool = False,
921
+ ) -> dict[str, Any]:
922
+ self.eval()
923
+ if str(getattr(self.video_expert, "video_attention_mask_mode", "")) != "first_frame_causal":
924
+ raise ValueError(
925
+ "`infer_action` requires `video_attention_mask_mode='first_frame_causal'`."
926
+ )
927
+
928
+ if input_image.ndim == 3:
929
+ input_image = input_image.unsqueeze(0)
930
+ if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
931
+ raise ValueError(
932
+ f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
933
+ )
934
+ _, _, height, width = input_image.shape
935
+ if height % 16 != 0 or width % 16 != 0:
936
+ raise ValueError(
937
+ f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})"
938
+ )
939
+ if proprio is not None:
940
+ if self.proprio_dim is None:
941
+ raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.")
942
+ if proprio.ndim == 1:
943
+ proprio = proprio.unsqueeze(0)
944
+ elif proprio.ndim == 2 and proprio.shape[0] == 1:
945
+ pass
946
+ else:
947
+ raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}")
948
+ if proprio.shape[1] != self.proprio_dim:
949
+ raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}")
950
+ proprio = proprio.to(device=self.device, dtype=self.torch_dtype)
951
+
952
+ generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
953
+ latents_action = torch.randn(
954
+ (1, action_horizon, self.action_expert.action_dim),
955
+ generator=generator,
956
+ device=rand_device,
957
+ dtype=torch.float32,
958
+ ).to(device=self.device, dtype=self.torch_dtype)
959
+
960
+ input_image = input_image.to(device=self.device, dtype=self.torch_dtype)
961
+ first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled)
962
+ fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False))
963
+
964
+ use_prompt = prompt is not None
965
+ use_context = context is not None or context_mask is not None
966
+ if use_prompt and use_context:
967
+ raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.")
968
+ if not use_prompt and not use_context:
969
+ raise ValueError("Either `prompt` or both `context/context_mask` must be provided.")
970
+
971
+ if use_prompt:
972
+ context, context_mask = self.encode_prompt(prompt)
973
+ else:
974
+ if context is None or context_mask is None:
975
+ raise ValueError("`context` and `context_mask` must be both provided together.")
976
+ if context.ndim == 2:
977
+ context = context.unsqueeze(0)
978
+ if context_mask.ndim == 1:
979
+ context_mask = context_mask.unsqueeze(0)
980
+ if context.ndim != 3 or context_mask.ndim != 2:
981
+ raise ValueError(
982
+ f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}"
983
+ )
984
+ context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
985
+ context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True)
986
+ if proprio is not None:
987
+ context, context_mask = self._append_proprio_to_context(
988
+ context=context,
989
+ context_mask=context_mask,
990
+ proprio=proprio,
991
+ )
992
+
993
+ timestep_video = torch.zeros(
994
+ (first_frame_latents.shape[0],),
995
+ dtype=first_frame_latents.dtype,
996
+ device=self.device,
997
+ )
998
+ video_pre = self.video_expert.pre_dit(
999
+ x=first_frame_latents,
1000
+ timestep=timestep_video,
1001
+ context=context,
1002
+ context_mask=context_mask,
1003
+ action=None,
1004
+ fuse_vae_embedding_in_latents=fuse_flag,
1005
+ )
1006
+ video_seq_len = int(video_pre["tokens"].shape[1])
1007
+ attention_mask = self._build_mot_attention_mask(
1008
+ video_seq_len=video_seq_len,
1009
+ action_seq_len=latents_action.shape[1],
1010
+ video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]),
1011
+ device=video_pre["tokens"].device,
1012
+ )
1013
+ video_kv_cache = self.mot.prefill_video_cache(
1014
+ video_tokens=video_pre["tokens"],
1015
+ video_freqs=video_pre["freqs"],
1016
+ video_t_mod=video_pre["t_mod"],
1017
+ video_context_payload={
1018
+ "context": video_pre["context"],
1019
+ "mask": video_pre["context_mask"],
1020
+ },
1021
+ video_attention_mask=attention_mask[:video_seq_len, :video_seq_len],
1022
+ )
1023
+
1024
+ infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule(
1025
+ num_inference_steps=num_inference_steps,
1026
+ device=self.device,
1027
+ dtype=latents_action.dtype,
1028
+ shift_override=sigma_shift,
1029
+ )
1030
+ for step_t_action, step_delta_action in zip(infer_timesteps_action, infer_deltas_action):
1031
+ timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device)
1032
+
1033
+ pred_action_posi = self._predict_action_noise_with_cache(
1034
+ latents_action=latents_action,
1035
+ timestep_action=timestep_action,
1036
+ context=context,
1037
+ context_mask=context_mask,
1038
+ video_kv_cache=video_kv_cache,
1039
+ attention_mask=attention_mask,
1040
+ video_seq_len=video_seq_len,
1041
+ )
1042
+ pred_action = pred_action_posi
1043
+
1044
+ latents_action = self.infer_action_scheduler.step(pred_action, step_delta_action, latents_action)
1045
+
1046
+ return {
1047
+ "action": latents_action[0].detach().to(device="cpu", dtype=torch.float32),
1048
+ }
1049
+
1050
+ @torch.no_grad()
1051
+ def infer(
1052
+ self,
1053
+ prompt: Optional[str],
1054
+ input_image: torch.Tensor,
1055
+ num_frames: int,
1056
+ action: Optional[torch.Tensor] = None,
1057
+ action_horizon: Optional[int] = None,
1058
+ proprio: Optional[torch.Tensor] = None,
1059
+ context: Optional[torch.Tensor] = None,
1060
+ context_mask: Optional[torch.Tensor] = None,
1061
+ negative_prompt: Optional[str] = None,
1062
+ text_cfg_scale: float = 5.0,
1063
+ action_cfg_scale: float = 1.0,
1064
+ num_inference_steps: int = 20,
1065
+ sigma_shift: Optional[float] = None,
1066
+ seed: Optional[int] = None,
1067
+ rand_device: str = "cpu",
1068
+ tiled: bool = False,
1069
+ ):
1070
+ return self.infer_joint(
1071
+ prompt=prompt,
1072
+ input_image=input_image,
1073
+ num_video_frames=num_frames,
1074
+ action_horizon=action_horizon,
1075
+ action=action,
1076
+ proprio=proprio,
1077
+ context=context,
1078
+ context_mask=context_mask,
1079
+ negative_prompt=negative_prompt,
1080
+ text_cfg_scale=text_cfg_scale,
1081
+ num_inference_steps=num_inference_steps,
1082
+ sigma_shift=sigma_shift,
1083
+ seed=seed,
1084
+ rand_device=rand_device,
1085
+ tiled=tiled,
1086
+ )
1087
+
1088
+ def save_checkpoint(self, path, optimizer=None, step=None):
1089
+ payload = {
1090
+ "mot": self.mot.state_dict(),
1091
+ "step": step,
1092
+ "torch_dtype": str(self.torch_dtype),
1093
+ }
1094
+ if self.proprio_encoder is not None:
1095
+ payload["proprio_encoder"] = self.proprio_encoder.state_dict()
1096
+ if optimizer is not None:
1097
+ payload["optimizer"] = optimizer.state_dict()
1098
+ torch.save(payload, path)
1099
+
1100
+ def load_checkpoint(self, path, optimizer=None):
1101
+ payload = torch.load(path, map_location="cpu")
1102
+ if "mot" in payload:
1103
+ self.mot.load_state_dict(payload["mot"], strict=False)
1104
+ elif "dit" in payload:
1105
+ logger.warning("Loading legacy `dit` checkpoint into video expert only.")
1106
+ self.video_expert.load_state_dict(payload["dit"], strict=False)
1107
+ else:
1108
+ raise ValueError(f"Checkpoint missing both `mot` and `dit` keys: {path}")
1109
+ if self.proprio_encoder is not None:
1110
+ if "proprio_encoder" in payload:
1111
+ self.proprio_encoder.load_state_dict(payload["proprio_encoder"], strict=True)
1112
+ else:
1113
+ logger.warning("Checkpoint has no `proprio_encoder` weights; keeping current `proprio_encoder` params.")
1114
+ elif "proprio_encoder" in payload:
1115
+ logger.warning("Checkpoint contains `proprio_encoder` weights but current model has `proprio_dim=None`; ignoring.")
1116
+
1117
+ if optimizer is not None and "optimizer" in payload:
1118
+ optimizer.load_state_dict(payload["optimizer"])
1119
+ return payload
1120
+
1121
+ def forward(self, *args, **kwargs):
1122
+ return self.training_loss(*args, **kwargs)
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_idm.py ADDED
@@ -0,0 +1,452 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, Optional
2
+
3
+ import torch
4
+ import torch.nn.functional as F
5
+
6
+ from fastwam.utils.logging_config import get_logger
7
+
8
+ from .fastwam_joint import FastWAMJoint
9
+
10
+ logger = get_logger(__name__)
11
+
12
+
13
+ class FastWAMIDM(FastWAMJoint):
14
+ """IDM variant with teacher-forcing video conditioning for action denoising."""
15
+
16
+ # Hardcoded probability: during training, cond-video is noised with this chance.
17
+ video_cond_noise_prob = 0.5
18
+
19
+ @torch.no_grad()
20
+ def _build_teacher_forcing_attention_mask(
21
+ self,
22
+ noisy_video_seq_len: int,
23
+ cond_video_seq_len: int,
24
+ action_seq_len: int,
25
+ noisy_video_tokens_per_frame: int,
26
+ cond_video_tokens_per_frame: int,
27
+ device: torch.device,
28
+ ) -> torch.Tensor:
29
+ if noisy_video_tokens_per_frame != cond_video_tokens_per_frame:
30
+ raise ValueError(
31
+ "Teacher-forcing requires identical `tokens_per_frame` for noisy and cond video branches, "
32
+ f"got {noisy_video_tokens_per_frame} and {cond_video_tokens_per_frame}."
33
+ )
34
+
35
+ noisy_end = noisy_video_seq_len
36
+ cond_end = noisy_video_seq_len + cond_video_seq_len
37
+ total_seq_len = cond_end + action_seq_len
38
+ mask = torch.zeros((total_seq_len, total_seq_len), dtype=torch.bool, device=device)
39
+
40
+ # noisy_video -> noisy_video
41
+ mask[:noisy_end, :noisy_end] = self.video_expert.build_video_to_video_mask(
42
+ video_seq_len=noisy_video_seq_len,
43
+ video_tokens_per_frame=noisy_video_tokens_per_frame,
44
+ device=device,
45
+ )
46
+ # cond_video -> cond_video
47
+ mask[noisy_end:cond_end, noisy_end:cond_end] = self.video_expert.build_video_to_video_mask(
48
+ video_seq_len=cond_video_seq_len,
49
+ video_tokens_per_frame=cond_video_tokens_per_frame,
50
+ device=device,
51
+ )
52
+ # action -> action
53
+ mask[cond_end:, cond_end:] = True
54
+ # action -> cond_video only
55
+ mask[cond_end:, noisy_end:cond_end] = True
56
+ return mask
57
+
58
+ def training_loss(self, sample, tiled: bool = False):
59
+ inputs = self.build_inputs(sample, tiled=tiled)
60
+ input_latents = inputs["input_latents"]
61
+ batch_size = input_latents.shape[0]
62
+ context = inputs["context"]
63
+ context_mask = inputs["context_mask"]
64
+ action = inputs["action"]
65
+ action_is_pad = inputs["action_is_pad"]
66
+ image_is_pad = inputs["image_is_pad"]
67
+ fuse_flag = inputs["fuse_vae_embedding_in_latents"]
68
+
69
+ # Branch A: noisy video (for video denoising target).
70
+ noise_video = torch.randn_like(input_latents)
71
+ timestep_video = self.train_video_scheduler.sample_training_t(
72
+ batch_size=batch_size,
73
+ device=self.device,
74
+ dtype=input_latents.dtype,
75
+ )
76
+ latents_noisy = self.train_video_scheduler.add_noise(input_latents, noise_video, timestep_video)
77
+ target_video = self.train_video_scheduler.training_target(input_latents, noise_video, timestep_video)
78
+ if inputs["first_frame_latents"] is not None:
79
+ latents_noisy[:, :, 0:1] = inputs["first_frame_latents"]
80
+
81
+ # Branch B: noisy action.
82
+ noise_action = torch.randn_like(action)
83
+ timestep_action = self.train_action_scheduler.sample_training_t(
84
+ batch_size=batch_size,
85
+ device=self.device,
86
+ dtype=action.dtype,
87
+ )
88
+ noisy_action = self.train_action_scheduler.add_noise(action, noise_action, timestep_action)
89
+ target_action = self.train_action_scheduler.training_target(action, noise_action, timestep_action)
90
+
91
+ # Branch C: teacher-forcing cond-video.
92
+ # Each sample is independently noised with probability `video_cond_noise_prob`.
93
+ cond_noise_mask = torch.rand((batch_size,), device=self.device) < float(self.video_cond_noise_prob)
94
+ timestep_video_cond = torch.zeros_like(timestep_video, dtype=input_latents.dtype, device=self.device)
95
+ latents_cond = input_latents
96
+ if bool(cond_noise_mask.any()):
97
+ timestep_video_cond_sampled = self.train_video_scheduler.sample_training_t(
98
+ batch_size=batch_size,
99
+ device=self.device,
100
+ dtype=input_latents.dtype,
101
+ )
102
+ timestep_video_cond = torch.where(cond_noise_mask, timestep_video_cond_sampled, timestep_video_cond)
103
+ noise_video_cond = torch.randn_like(input_latents)
104
+ latents_cond_noisy = self.train_video_scheduler.add_noise(
105
+ input_latents, noise_video_cond, timestep_video_cond_sampled
106
+ )
107
+ cond_noise_selector = cond_noise_mask.view(batch_size, 1, 1, 1, 1)
108
+ latents_cond = torch.where(cond_noise_selector, latents_cond_noisy, input_latents)
109
+ if inputs["first_frame_latents"] is not None:
110
+ latents_cond = latents_cond.clone()
111
+ latents_cond[:, :, 0:1] = inputs["first_frame_latents"]
112
+
113
+ video_pre_noisy = self.video_expert.pre_dit(
114
+ x=latents_noisy,
115
+ timestep=timestep_video,
116
+ context=context,
117
+ context_mask=context_mask,
118
+ action=None,
119
+ fuse_vae_embedding_in_latents=fuse_flag,
120
+ )
121
+ video_pre_cond = self.video_expert.pre_dit(
122
+ x=latents_cond,
123
+ timestep=timestep_video_cond,
124
+ context=context,
125
+ context_mask=context_mask,
126
+ action=None,
127
+ fuse_vae_embedding_in_latents=fuse_flag,
128
+ )
129
+ if video_pre_noisy["t_mod"].ndim != 4 or video_pre_cond["t_mod"].ndim != 4:
130
+ raise ValueError(
131
+ "Teacher-forcing requires token-wise `t_mod`; "
132
+ "ensure `seperated_timestep=true` and `fuse_vae_embedding_in_latents=true`."
133
+ )
134
+
135
+ action_pre = self.action_expert.pre_dit(
136
+ action_tokens=noisy_action,
137
+ timestep=timestep_action,
138
+ context=context,
139
+ context_mask=context_mask,
140
+ )
141
+
142
+ noisy_video_seq_len = int(video_pre_noisy["tokens"].shape[1])
143
+ cond_video_seq_len = int(video_pre_cond["tokens"].shape[1])
144
+ noisy_video_tokens_per_frame = int(video_pre_noisy["meta"]["tokens_per_frame"])
145
+ cond_video_tokens_per_frame = int(video_pre_cond["meta"]["tokens_per_frame"])
146
+
147
+ # Concatenate [noisy_video, cond_video] as the video expert sequence.
148
+ merged_video_tokens = torch.cat([video_pre_noisy["tokens"], video_pre_cond["tokens"]], dim=1)
149
+ merged_video_freqs = torch.cat([video_pre_noisy["freqs"], video_pre_cond["freqs"]], dim=0)
150
+ merged_video_t_mod = torch.cat([video_pre_noisy["t_mod"], video_pre_cond["t_mod"]], dim=1)
151
+ merged_video_context_mask = torch.cat([video_pre_noisy["context_mask"], video_pre_cond["context_mask"]], dim=1)
152
+
153
+ attention_mask = self._build_teacher_forcing_attention_mask(
154
+ noisy_video_seq_len=noisy_video_seq_len,
155
+ cond_video_seq_len=cond_video_seq_len,
156
+ action_seq_len=action_pre["tokens"].shape[1],
157
+ noisy_video_tokens_per_frame=noisy_video_tokens_per_frame,
158
+ cond_video_tokens_per_frame=cond_video_tokens_per_frame,
159
+ device=merged_video_tokens.device,
160
+ )
161
+
162
+ tokens_out = self.mot(
163
+ embeds_all={
164
+ "video": merged_video_tokens,
165
+ "action": action_pre["tokens"],
166
+ },
167
+ attention_mask=attention_mask,
168
+ freqs_all={
169
+ "video": merged_video_freqs,
170
+ "action": action_pre["freqs"],
171
+ },
172
+ context_all={
173
+ "video": {
174
+ "context": video_pre_noisy["context"],
175
+ "mask": merged_video_context_mask,
176
+ },
177
+ "action": {
178
+ "context": action_pre["context"],
179
+ "mask": action_pre["context_mask"],
180
+ },
181
+ },
182
+ t_mod_all={
183
+ "video": merged_video_t_mod,
184
+ "action": action_pre["t_mod"],
185
+ },
186
+ )
187
+
188
+ # Only the noisy-video half contributes to video denoising loss.
189
+ pred_video_tokens = tokens_out["video"][:, :noisy_video_seq_len]
190
+ pred_video = self.video_expert.post_dit(pred_video_tokens, video_pre_noisy)
191
+ pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre)
192
+
193
+ include_initial_video_step = inputs["first_frame_latents"] is None
194
+ if inputs["first_frame_latents"] is not None:
195
+ pred_video = pred_video[:, :, 1:]
196
+ target_video = target_video[:, :, 1:]
197
+
198
+ loss_video_per_sample = self._compute_video_loss_per_sample(
199
+ pred_video=pred_video,
200
+ target_video=target_video,
201
+ image_is_pad=image_is_pad,
202
+ include_initial_video_step=include_initial_video_step,
203
+ )
204
+ video_weight = self.train_video_scheduler.training_weight(timestep_video).to(
205
+ loss_video_per_sample.device, dtype=loss_video_per_sample.dtype
206
+ )
207
+ loss_video = (loss_video_per_sample * video_weight).mean()
208
+
209
+ action_loss_token = F.mse_loss(pred_action.float(), target_action.float(), reduction="none").mean(dim=2)
210
+ if action_is_pad is not None:
211
+ valid = (~action_is_pad).to(device=action_loss_token.device, dtype=action_loss_token.dtype)
212
+ valid_sum = valid.sum(dim=1).clamp(min=1.0)
213
+ action_loss_per_sample = (action_loss_token * valid).sum(dim=1) / valid_sum
214
+ else:
215
+ action_loss_per_sample = action_loss_token.mean(dim=1)
216
+
217
+ action_weight = self.train_action_scheduler.training_weight(timestep_action).to(
218
+ action_loss_per_sample.device, dtype=action_loss_per_sample.dtype
219
+ )
220
+ loss_action = (action_loss_per_sample * action_weight).mean()
221
+
222
+ loss_total = self.loss_lambda_video * loss_video + self.loss_lambda_action * loss_action
223
+ loss_dict = {
224
+ "loss_video": self.loss_lambda_video * float(loss_video.detach().item()),
225
+ "loss_action": self.loss_lambda_action * float(loss_action.detach().item()),
226
+ }
227
+ return loss_total, loss_dict
228
+
229
+ @torch.no_grad()
230
+ def infer_action(
231
+ self,
232
+ prompt: Optional[str],
233
+ input_image: torch.Tensor,
234
+ action_horizon: int,
235
+ num_video_frames: int,
236
+ proprio: Optional[torch.Tensor] = None,
237
+ context: Optional[torch.Tensor] = None,
238
+ context_mask: Optional[torch.Tensor] = None,
239
+ negative_prompt: Optional[str] = None,
240
+ text_cfg_scale: float = 1.0,
241
+ num_inference_steps: int = 20,
242
+ sigma_shift: Optional[float] = None,
243
+ seed: Optional[int] = None,
244
+ rand_device: str = "cpu",
245
+ tiled: bool = False,
246
+ ) -> dict[str, Any]:
247
+ # Reuse infer_joint pipeline and keep infer_action output contract.
248
+ out = self.infer_joint(
249
+ prompt=prompt,
250
+ input_image=input_image,
251
+ num_video_frames=num_video_frames,
252
+ action_horizon=action_horizon,
253
+ action=None,
254
+ proprio=proprio,
255
+ context=context,
256
+ context_mask=context_mask,
257
+ negative_prompt=negative_prompt,
258
+ text_cfg_scale=text_cfg_scale,
259
+ num_inference_steps=num_inference_steps,
260
+ sigma_shift=sigma_shift,
261
+ seed=seed,
262
+ rand_device=rand_device,
263
+ tiled=tiled,
264
+ test_action_with_infer_action=False,
265
+ )
266
+ return {"action": out["action"]}
267
+
268
+ @torch.no_grad()
269
+ def infer_joint(
270
+ self,
271
+ prompt: Optional[str],
272
+ input_image: torch.Tensor,
273
+ num_video_frames: int,
274
+ action_horizon: int,
275
+ action: Optional[torch.Tensor] = None,
276
+ proprio: Optional[torch.Tensor] = None,
277
+ context: Optional[torch.Tensor] = None,
278
+ context_mask: Optional[torch.Tensor] = None,
279
+ negative_prompt: Optional[str] = None,
280
+ text_cfg_scale: float = 1.0,
281
+ num_inference_steps: int = 20,
282
+ sigma_shift: Optional[float] = None,
283
+ seed: Optional[int] = None,
284
+ rand_device: str = "cpu",
285
+ tiled: bool = False,
286
+ test_action_with_infer_action: bool = True,
287
+ ) -> dict[str, Any]:
288
+ del negative_prompt, text_cfg_scale, test_action_with_infer_action
289
+ self.eval()
290
+
291
+ if action is not None:
292
+ logger.warning(
293
+ "`FastWAMIDM.infer_joint` ignores `action` input; "
294
+ "video is denoised in a standalone first stage."
295
+ )
296
+
297
+ if input_image.ndim == 3:
298
+ input_image = input_image.unsqueeze(0)
299
+ if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
300
+ raise ValueError(
301
+ f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
302
+ )
303
+ _, _, height, width = input_image.shape
304
+ checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_video_frames)
305
+ if (checked_h, checked_w) != (height, width):
306
+ raise ValueError(
307
+ f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})"
308
+ )
309
+ if checked_t != num_video_frames:
310
+ raise ValueError(
311
+ f"`num_video_frames` must satisfy T % 4 == 1, got {num_video_frames}"
312
+ )
313
+
314
+ if proprio is not None:
315
+ if self.proprio_dim is None:
316
+ raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.")
317
+ if proprio.ndim == 1:
318
+ proprio = proprio.unsqueeze(0)
319
+ elif proprio.ndim == 2 and proprio.shape[0] == 1:
320
+ pass
321
+ else:
322
+ raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}")
323
+ if proprio.shape[1] != self.proprio_dim:
324
+ raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}")
325
+ proprio = proprio.to(device=self.device, dtype=self.torch_dtype)
326
+
327
+ latent_t = (num_video_frames - 1) // self.vae.temporal_downsample_factor + 1
328
+ latent_h = height // self.vae.upsampling_factor
329
+ latent_w = width // self.vae.upsampling_factor
330
+
331
+ video_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
332
+ action_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
333
+ latents_video = torch.randn(
334
+ (1, self.vae.model.z_dim, latent_t, latent_h, latent_w),
335
+ generator=video_generator,
336
+ device=rand_device,
337
+ dtype=torch.float32,
338
+ ).to(device=self.device, dtype=self.torch_dtype)
339
+ latents_action = torch.randn(
340
+ (1, action_horizon, self.action_expert.action_dim),
341
+ generator=action_generator,
342
+ device=rand_device,
343
+ dtype=torch.float32,
344
+ ).to(device=self.device, dtype=self.torch_dtype)
345
+
346
+ input_image = input_image.to(device=self.device, dtype=self.torch_dtype)
347
+ first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled)
348
+ latents_video[:, :, 0:1] = first_frame_latents.clone()
349
+ fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False))
350
+
351
+ use_prompt = prompt is not None
352
+ use_context = context is not None or context_mask is not None
353
+ if use_prompt and use_context:
354
+ raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.")
355
+ if not use_prompt and not use_context:
356
+ raise ValueError("Either `prompt` or both `context/context_mask` must be provided.")
357
+
358
+ if use_prompt:
359
+ context, context_mask = self.encode_prompt(prompt)
360
+ else:
361
+ if context is None or context_mask is None:
362
+ raise ValueError("`context` and `context_mask` must be both provided together.")
363
+ if context.ndim == 2:
364
+ context = context.unsqueeze(0)
365
+ if context_mask.ndim == 1:
366
+ context_mask = context_mask.unsqueeze(0)
367
+ if context.ndim != 3 or context_mask.ndim != 2:
368
+ raise ValueError(
369
+ f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}"
370
+ )
371
+ context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
372
+ context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True)
373
+ if proprio is not None:
374
+ context, context_mask = self._append_proprio_to_context(
375
+ context=context,
376
+ context_mask=context_mask,
377
+ proprio=proprio,
378
+ )
379
+
380
+ # Stage 1: denoise video only.
381
+ infer_timesteps_video, infer_deltas_video = self.infer_video_scheduler.build_inference_schedule(
382
+ num_inference_steps=num_inference_steps,
383
+ device=self.device,
384
+ dtype=latents_video.dtype,
385
+ shift_override=sigma_shift,
386
+ )
387
+ for step_t_video, step_delta_video in zip(infer_timesteps_video, infer_deltas_video):
388
+ timestep_video = step_t_video.unsqueeze(0).to(dtype=latents_video.dtype, device=self.device)
389
+ pred_video = self.video_expert(
390
+ x=latents_video,
391
+ timestep=timestep_video,
392
+ context=context,
393
+ context_mask=context_mask,
394
+ action=None,
395
+ fuse_vae_embedding_in_latents=fuse_flag,
396
+ )
397
+ latents_video = self.infer_video_scheduler.step(pred_video, step_delta_video, latents_video)
398
+ latents_video[:, :, 0:1] = first_frame_latents.clone()
399
+
400
+ # Stage 2: freeze denoised video as cond and denoise action via video K/V cache.
401
+ timestep_video_cond = torch.zeros(
402
+ (latents_video.shape[0],), dtype=latents_video.dtype, device=self.device
403
+ )
404
+ video_pre_cond = self.video_expert.pre_dit(
405
+ x=latents_video,
406
+ timestep=timestep_video_cond,
407
+ context=context,
408
+ context_mask=context_mask,
409
+ action=None,
410
+ fuse_vae_embedding_in_latents=fuse_flag,
411
+ )
412
+ video_seq_len = int(video_pre_cond["tokens"].shape[1])
413
+ attention_mask = self._build_mot_attention_mask(
414
+ video_seq_len=video_seq_len,
415
+ action_seq_len=latents_action.shape[1],
416
+ video_tokens_per_frame=int(video_pre_cond["meta"]["tokens_per_frame"]),
417
+ device=video_pre_cond["tokens"].device,
418
+ )
419
+ video_kv_cache = self.mot.prefill_video_cache(
420
+ video_tokens=video_pre_cond["tokens"],
421
+ video_freqs=video_pre_cond["freqs"],
422
+ video_t_mod=video_pre_cond["t_mod"],
423
+ video_context_payload={
424
+ "context": video_pre_cond["context"],
425
+ "mask": video_pre_cond["context_mask"],
426
+ },
427
+ video_attention_mask=attention_mask[:video_seq_len, :video_seq_len],
428
+ )
429
+
430
+ infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule(
431
+ num_inference_steps=num_inference_steps,
432
+ device=self.device,
433
+ dtype=latents_action.dtype,
434
+ shift_override=sigma_shift,
435
+ )
436
+ for step_t_action, step_delta_action in zip(infer_timesteps_action, infer_deltas_action):
437
+ timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device)
438
+ pred_action = self._predict_action_noise_with_cache(
439
+ latents_action=latents_action,
440
+ timestep_action=timestep_action,
441
+ context=context,
442
+ context_mask=context_mask,
443
+ video_kv_cache=video_kv_cache,
444
+ attention_mask=attention_mask,
445
+ video_seq_len=video_seq_len,
446
+ )
447
+ latents_action = self.infer_action_scheduler.step(pred_action, step_delta_action, latents_action)
448
+
449
+ return {
450
+ "video": self._decode_latents(latents_video, tiled=tiled),
451
+ "action": latents_action[0].detach().to(device="cpu", dtype=torch.float32),
452
+ }
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_joint.py ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, Optional
2
+
3
+ import torch
4
+
5
+ from fastwam.utils.logging_config import get_logger
6
+
7
+ from .attention_masks import build_joint_denoise_attention_mask
8
+ from .fastwam import FastWAM
9
+
10
+ logger = get_logger(__name__)
11
+
12
+
13
+ class FastWAMJoint(FastWAM):
14
+ """MoT variant with bidirectional video/action joint denoising."""
15
+
16
+ @classmethod
17
+ def from_wan22_pretrained(cls, **kwargs):
18
+ video_dit_config = kwargs.get("video_dit_config", None)
19
+ if not isinstance(video_dit_config, dict):
20
+ raise ValueError(
21
+ "`video_dit_config` must be provided as dict for FastWAMJoint."
22
+ )
23
+ if bool(video_dit_config.get("action_conditioned", False)):
24
+ raise ValueError(
25
+ "FastWAMJoint requires `video_dit_config['action_conditioned']=false`."
26
+ )
27
+ return super().from_wan22_pretrained(**kwargs)
28
+
29
+ @torch.no_grad()
30
+ def _build_mot_attention_mask(
31
+ self,
32
+ video_seq_len: int,
33
+ action_seq_len: int,
34
+ video_tokens_per_frame: int,
35
+ device: torch.device,
36
+ ) -> torch.Tensor:
37
+ return build_joint_denoise_attention_mask(
38
+ video_seq_len=video_seq_len,
39
+ action_seq_len=action_seq_len,
40
+ video_tokens_per_frame=video_tokens_per_frame,
41
+ device=device,
42
+ )
43
+
44
+ @torch.no_grad()
45
+ def infer_joint(
46
+ self,
47
+ prompt: Optional[str],
48
+ input_image: torch.Tensor,
49
+ num_video_frames: int,
50
+ action_horizon: int,
51
+ action: Optional[torch.Tensor] = None,
52
+ proprio: Optional[torch.Tensor] = None,
53
+ context: Optional[torch.Tensor] = None,
54
+ context_mask: Optional[torch.Tensor] = None,
55
+ negative_prompt: Optional[str] = None,
56
+ text_cfg_scale: float = 1.0,
57
+ num_inference_steps: int = 20,
58
+ sigma_shift: Optional[float] = None,
59
+ seed: Optional[int] = None,
60
+ rand_device: str = "cpu",
61
+ tiled: bool = False,
62
+ test_action_with_infer_action: bool = True,
63
+ ) -> dict[str, Any]:
64
+ if test_action_with_infer_action:
65
+ logger.warning(
66
+ "`FastWAMJoint.infer_joint` ignores `test_action_with_infer_action=True` "
67
+ "and always runs with `test_action_with_infer_action=False`."
68
+ )
69
+ return super().infer_joint(
70
+ prompt=prompt,
71
+ input_image=input_image,
72
+ num_video_frames=num_video_frames,
73
+ action_horizon=action_horizon,
74
+ action=action,
75
+ proprio=proprio,
76
+ context=context,
77
+ context_mask=context_mask,
78
+ negative_prompt=negative_prompt,
79
+ text_cfg_scale=text_cfg_scale,
80
+ num_inference_steps=num_inference_steps,
81
+ sigma_shift=sigma_shift,
82
+ seed=seed,
83
+ rand_device=rand_device,
84
+ tiled=tiled,
85
+ test_action_with_infer_action=False,
86
+ )
87
+
88
+ @torch.no_grad()
89
+ def infer_action(
90
+ self,
91
+ prompt: Optional[str],
92
+ input_image: torch.Tensor,
93
+ action_horizon: int,
94
+ num_video_frames: int,
95
+ proprio: Optional[torch.Tensor] = None,
96
+ context: Optional[torch.Tensor] = None,
97
+ context_mask: Optional[torch.Tensor] = None,
98
+ negative_prompt: Optional[str] = None,
99
+ text_cfg_scale: float = 1.0,
100
+ num_inference_steps: int = 20,
101
+ sigma_shift: Optional[float] = None,
102
+ seed: Optional[int] = None,
103
+ rand_device: str = "cpu",
104
+ tiled: bool = False,
105
+ ) -> dict[str, Any]:
106
+ self.eval()
107
+
108
+ if input_image.ndim == 3:
109
+ input_image = input_image.unsqueeze(0)
110
+ if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
111
+ raise ValueError(
112
+ f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
113
+ )
114
+ _, _, height, width = input_image.shape
115
+ checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_video_frames)
116
+ if (checked_h, checked_w) != (height, width):
117
+ raise ValueError(
118
+ f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})"
119
+ )
120
+ if checked_t != num_video_frames:
121
+ raise ValueError(
122
+ f"`num_video_frames` must satisfy T % 4 == 1, got {num_video_frames}"
123
+ )
124
+
125
+ if proprio is not None:
126
+ if self.proprio_dim is None:
127
+ raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.")
128
+ if proprio.ndim == 1:
129
+ proprio = proprio.unsqueeze(0)
130
+ elif proprio.ndim == 2 and proprio.shape[0] == 1:
131
+ pass
132
+ else:
133
+ raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}")
134
+ if proprio.shape[1] != self.proprio_dim:
135
+ raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}")
136
+ proprio = proprio.to(device=self.device, dtype=self.torch_dtype)
137
+
138
+ latent_t = (num_video_frames - 1) // self.vae.temporal_downsample_factor + 1
139
+ latent_h = height // self.vae.upsampling_factor
140
+ latent_w = width // self.vae.upsampling_factor
141
+
142
+ video_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
143
+ action_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
144
+ latents_video = torch.randn(
145
+ (1, self.vae.model.z_dim, latent_t, latent_h, latent_w),
146
+ generator=video_generator,
147
+ device=rand_device,
148
+ dtype=torch.float32,
149
+ ).to(device=self.device, dtype=self.torch_dtype)
150
+ latents_action = torch.randn(
151
+ (1, action_horizon, self.action_expert.action_dim),
152
+ generator=action_generator,
153
+ device=rand_device,
154
+ dtype=torch.float32,
155
+ ).to(device=self.device, dtype=self.torch_dtype)
156
+
157
+ input_image = input_image.to(device=self.device, dtype=self.torch_dtype)
158
+ first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled)
159
+ latents_video[:, :, 0:1] = first_frame_latents.clone()
160
+ fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False))
161
+
162
+ use_prompt = prompt is not None
163
+ use_context = context is not None or context_mask is not None
164
+ if use_prompt and use_context:
165
+ raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.")
166
+ if not use_prompt and not use_context:
167
+ raise ValueError("Either `prompt` or both `context/context_mask` must be provided.")
168
+
169
+ if use_prompt:
170
+ context, context_mask = self.encode_prompt(prompt)
171
+ else:
172
+ if context is None or context_mask is None:
173
+ raise ValueError("`context` and `context_mask` must be both provided together.")
174
+ if context.ndim == 2:
175
+ context = context.unsqueeze(0)
176
+ if context_mask.ndim == 1:
177
+ context_mask = context_mask.unsqueeze(0)
178
+ if context.ndim != 3 or context_mask.ndim != 2:
179
+ raise ValueError(
180
+ f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}"
181
+ )
182
+ context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
183
+ context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True)
184
+ if proprio is not None:
185
+ context, context_mask = self._append_proprio_to_context(
186
+ context=context,
187
+ context_mask=context_mask,
188
+ proprio=proprio,
189
+ )
190
+
191
+ infer_timesteps_video, infer_deltas_video = self.infer_video_scheduler.build_inference_schedule(
192
+ num_inference_steps=num_inference_steps,
193
+ device=self.device,
194
+ dtype=latents_video.dtype,
195
+ shift_override=sigma_shift,
196
+ )
197
+ infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule(
198
+ num_inference_steps=num_inference_steps,
199
+ device=self.device,
200
+ dtype=latents_action.dtype,
201
+ shift_override=sigma_shift,
202
+ )
203
+ for step_t_video, step_delta_video, step_t_action, step_delta_action in zip(
204
+ infer_timesteps_video,
205
+ infer_deltas_video,
206
+ infer_timesteps_action,
207
+ infer_deltas_action,
208
+ ):
209
+ timestep_video = step_t_video.unsqueeze(0).to(dtype=latents_video.dtype, device=self.device)
210
+ timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device)
211
+
212
+ pred_video_posi, pred_action_posi = self._predict_joint_noise(
213
+ latents_video=latents_video,
214
+ latents_action=latents_action,
215
+ timestep_video=timestep_video,
216
+ timestep_action=timestep_action,
217
+ context=context,
218
+ context_mask=context_mask,
219
+ fuse_vae_embedding_in_latents=fuse_flag,
220
+ gt_action=None,
221
+ )
222
+
223
+ latents_video = self.infer_video_scheduler.step(pred_video_posi, step_delta_video, latents_video)
224
+ latents_action = self.infer_action_scheduler.step(pred_action_posi, step_delta_action, latents_action)
225
+ latents_video[:, :, 0:1] = first_frame_latents.clone()
226
+
227
+ return {
228
+ "action": latents_action[0].detach().to(device="cpu", dtype=torch.float32),
229
+ }
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/__init__.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from .io import ModelConfig, hash_model_file, load_state_dict
2
+ from .state_dict_converters import (
3
+ wan_video_dit_from_diffusers,
4
+ wan_video_dit_state_dict_converter,
5
+ wan_video_vae_state_dict_converter,
6
+ )
7
+
8
+ __all__ = [
9
+ "ModelConfig",
10
+ "hash_model_file",
11
+ "load_state_dict",
12
+ "wan_video_dit_from_diffusers",
13
+ "wan_video_dit_state_dict_converter",
14
+ "wan_video_vae_state_dict_converter",
15
+ ]
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/gradient.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+
4
+ def create_custom_forward(module):
5
+ def custom_forward(*inputs, **kwargs):
6
+ return module(*inputs, **kwargs)
7
+ return custom_forward
8
+
9
+
10
+ def gradient_checkpoint_forward(
11
+ model,
12
+ use_gradient_checkpointing,
13
+ *args,
14
+ **kwargs,
15
+ ):
16
+ if use_gradient_checkpointing:
17
+ model_output = torch.utils.checkpoint.checkpoint(
18
+ create_custom_forward(model),
19
+ *args,
20
+ **kwargs,
21
+ use_reentrant=False,
22
+ )
23
+ else:
24
+ model_output = model(*args, **kwargs)
25
+ return model_output
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/io.py ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import glob
2
+ import hashlib
3
+ import os
4
+ from dataclasses import dataclass
5
+ from typing import Dict, Optional, Union
6
+
7
+ import torch
8
+ from safetensors import safe_open
9
+
10
+
11
+ @dataclass
12
+ class ModelConfig:
13
+ path: Union[str, list[str], None] = None
14
+ model_id: Optional[str] = None
15
+ origin_file_pattern: Union[str, list[str], None] = None
16
+ download_source: Optional[str] = None
17
+ local_model_path: Optional[str] = None
18
+ skip_download: Optional[bool] = None
19
+ state_dict: Optional[Dict[str, torch.Tensor]] = None
20
+
21
+ def check_input(self):
22
+ if self.path is None and self.model_id is None:
23
+ raise ValueError("ModelConfig requires either `path` or (`model_id`, `origin_file_pattern`).")
24
+
25
+ def parse_original_file_pattern(self):
26
+ if self.origin_file_pattern in [None, "", "./"]:
27
+ return "*"
28
+ if isinstance(self.origin_file_pattern, list):
29
+ return self.origin_file_pattern
30
+ if self.origin_file_pattern.endswith("/"):
31
+ return self.origin_file_pattern + "*"
32
+ return self.origin_file_pattern
33
+
34
+ def parse_download_source(self):
35
+ if self.download_source is not None:
36
+ return self.download_source
37
+ env = os.environ.get("DIFFSYNTH_DOWNLOAD_SOURCE")
38
+ return env if env is not None else "modelscope"
39
+
40
+ def parse_skip_download(self):
41
+ if self.skip_download is not None:
42
+ return self.skip_download
43
+ env = os.environ.get("DIFFSYNTH_SKIP_DOWNLOAD")
44
+ if env is None:
45
+ return False
46
+ return env.lower() == "true"
47
+
48
+ def reset_local_model_path(self):
49
+ if os.environ.get("DIFFSYNTH_MODEL_BASE_PATH") is not None:
50
+ self.local_model_path = os.environ.get("DIFFSYNTH_MODEL_BASE_PATH")
51
+ elif self.local_model_path is None:
52
+ self.local_model_path = "./checkpoints/"
53
+
54
+ def require_downloading(self):
55
+ if self.path is not None:
56
+ return False
57
+ origin_file_pattern = self.parse_original_file_pattern()
58
+ local_root = os.path.join(self.local_model_path, self.model_id)
59
+ if isinstance(origin_file_pattern, list):
60
+ all_exist = True
61
+ for pattern in origin_file_pattern:
62
+ matches = glob.glob(os.path.join(local_root, pattern))
63
+ if len(matches) == 0:
64
+ all_exist = False
65
+ break
66
+ if all_exist:
67
+ return False
68
+ else:
69
+ if len(glob.glob(os.path.join(local_root, origin_file_pattern))) > 0:
70
+ return False
71
+ return not self.parse_skip_download()
72
+
73
+ def download(self):
74
+ origin_file_pattern = self.parse_original_file_pattern()
75
+ root = os.path.join(self.local_model_path, self.model_id)
76
+ downloaded_files = glob.glob(origin_file_pattern, root_dir=root)
77
+ download_source = self.parse_download_source().lower()
78
+ if download_source == "modelscope":
79
+ from modelscope import snapshot_download
80
+
81
+ snapshot_download(
82
+ self.model_id,
83
+ local_dir=root,
84
+ allow_file_pattern=origin_file_pattern,
85
+ ignore_file_pattern=downloaded_files,
86
+ local_files_only=False,
87
+ )
88
+ elif download_source == "huggingface":
89
+ from huggingface_hub import snapshot_download as hf_snapshot_download
90
+
91
+ hf_snapshot_download(
92
+ self.model_id,
93
+ local_dir=root,
94
+ allow_patterns=origin_file_pattern,
95
+ ignore_patterns=downloaded_files,
96
+ local_files_only=False,
97
+ )
98
+ else:
99
+ raise ValueError("`download_source` should be `modelscope` or `huggingface`.")
100
+
101
+ def download_if_necessary(self):
102
+ self.check_input()
103
+ self.reset_local_model_path()
104
+ if self.require_downloading():
105
+ self.download()
106
+ if self.path is None:
107
+ if self.origin_file_pattern in [None, "", "./"]:
108
+ self.path = os.path.join(self.local_model_path, self.model_id)
109
+ else:
110
+ matches = glob.glob(os.path.join(self.local_model_path, self.model_id, self.origin_file_pattern))
111
+ matches.sort()
112
+ self.path = matches
113
+ if isinstance(self.path, list) and len(self.path) == 1:
114
+ self.path = self.path[0]
115
+
116
+
117
+ def load_state_dict(file_path, torch_dtype=None, device="cpu"):
118
+ if isinstance(file_path, list):
119
+ state_dict = {}
120
+ for file_path_ in file_path:
121
+ state_dict.update(load_state_dict(file_path_, torch_dtype=torch_dtype, device=device))
122
+ return state_dict
123
+ if file_path.endswith(".safetensors"):
124
+ return load_state_dict_from_safetensors(file_path, torch_dtype=torch_dtype, device=device)
125
+ return load_state_dict_from_bin(file_path, torch_dtype=torch_dtype, device=device)
126
+
127
+
128
+ def load_state_dict_from_safetensors(file_path, torch_dtype=None, device="cpu"):
129
+ state_dict = {}
130
+ with safe_open(file_path, framework="pt", device=str(device)) as f:
131
+ for key in f.keys():
132
+ value = f.get_tensor(key)
133
+ if torch_dtype is not None:
134
+ value = value.to(torch_dtype)
135
+ state_dict[key] = value
136
+ return state_dict
137
+
138
+
139
+ def load_state_dict_from_bin(file_path, torch_dtype=None, device="cpu"):
140
+ state_dict = torch.load(file_path, map_location=device, weights_only=True)
141
+ if len(state_dict) == 1:
142
+ if "state_dict" in state_dict:
143
+ state_dict = state_dict["state_dict"]
144
+ elif "module" in state_dict:
145
+ state_dict = state_dict["module"]
146
+ elif "model_state" in state_dict:
147
+ state_dict = state_dict["model_state"]
148
+ if torch_dtype is not None:
149
+ for key in state_dict:
150
+ if isinstance(state_dict[key], torch.Tensor):
151
+ state_dict[key] = state_dict[key].to(torch_dtype)
152
+ return state_dict
153
+
154
+
155
+ def _load_keys_dict_from_safetensors(file_path):
156
+ keys_dict = {}
157
+ with safe_open(file_path, framework="pt", device="cpu") as f:
158
+ for key in f.keys():
159
+ keys_dict[key] = f.get_slice(key).get_shape()
160
+ return keys_dict
161
+
162
+
163
+ def _convert_state_dict_to_keys_dict(state_dict):
164
+ keys_dict = {}
165
+ for key, value in state_dict.items():
166
+ if isinstance(value, torch.Tensor):
167
+ keys_dict[key] = list(value.shape)
168
+ else:
169
+ keys_dict[key] = _convert_state_dict_to_keys_dict(value)
170
+ return keys_dict
171
+
172
+
173
+ def _load_keys_dict_from_bin(file_path):
174
+ state_dict = load_state_dict_from_bin(file_path)
175
+ return _convert_state_dict_to_keys_dict(state_dict)
176
+
177
+
178
+ def _load_keys_dict(file_path):
179
+ if isinstance(file_path, list):
180
+ merged = {}
181
+ for path in file_path:
182
+ merged.update(_load_keys_dict(path))
183
+ return merged
184
+ if file_path.endswith(".safetensors"):
185
+ return _load_keys_dict_from_safetensors(file_path)
186
+ return _load_keys_dict_from_bin(file_path)
187
+
188
+
189
+ def _convert_keys_dict_to_single_str(keys_dict, with_shape=True):
190
+ keys = []
191
+ for key, value in keys_dict.items():
192
+ if isinstance(key, str):
193
+ if isinstance(value, dict):
194
+ keys.append(key + "|" + _convert_keys_dict_to_single_str(value, with_shape=with_shape))
195
+ else:
196
+ if with_shape:
197
+ shape = "_".join(map(str, list(value)))
198
+ keys.append(key + ":" + shape)
199
+ keys.append(key)
200
+ keys.sort()
201
+ return ",".join(keys)
202
+
203
+
204
+ def hash_model_file(path, with_shape=True):
205
+ keys_dict = _load_keys_dict(path)
206
+ keys_str = _convert_keys_dict_to_single_str(keys_dict, with_shape=with_shape).encode("UTF-8")
207
+ return hashlib.md5(keys_str).hexdigest()
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/loader.py ADDED
@@ -0,0 +1,221 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ import inspect
3
+ from typing import Any
4
+
5
+ import torch
6
+ import time
7
+
8
+ from .io import ModelConfig, hash_model_file, load_state_dict
9
+ from .state_dict_converters import (
10
+ wan_video_vae_state_dict_converter,
11
+ )
12
+ from ..wan_video_dit import WanVideoDiT
13
+ from ..wan_video_text_encoder import HuggingfaceTokenizer, WanTextEncoder
14
+ from ..wan_video_vae import WanVideoVAE38
15
+ from fastwam.utils.logging_config import get_logger
16
+
17
+ logger = get_logger(__name__)
18
+ SKIPPED_PRETRAIN_SENTINEL = "SKIPPED_PRETRAIN"
19
+
20
+
21
+ @dataclass
22
+ class Wan22LoadedComponents:
23
+ dit: WanVideoDiT
24
+ vae: WanVideoVAE38
25
+ text_encoder: WanTextEncoder | None
26
+ tokenizer: HuggingfaceTokenizer | None
27
+ dit_path: str
28
+ vae_path: str
29
+ text_encoder_path: str | None
30
+ tokenizer_path: str | None
31
+
32
+
33
+ WAN22_MODEL_REGISTRY = [
34
+ {
35
+ # Example: ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth")
36
+ "model_hash": "9c8818c2cbea55eca56c7b447df170da",
37
+ "model_name": "wan_video_text_encoder",
38
+ "model_class": WanTextEncoder,
39
+ },
40
+ {
41
+ # Example: ModelConfig(model_id="Wan-AI/Wan2.2-TI2V-5B", origin_file_pattern="diffusion_pytorch_model*.safetensors")
42
+ "model_hash": "1f5ab7703c6fc803fdded85ff040c316",
43
+ "model_name": "wan_video_dit",
44
+ "model_class": WanVideoDiT,
45
+ },
46
+ {
47
+ # Example: ModelConfig(model_id="Wan-AI/Wan2.2-TI2V-5B", origin_file_pattern="Wan2.2_VAE.pth")
48
+ "model_hash": "e1de6c02cdac79f8b739f4d3698cd216",
49
+ "model_name": "wan_video_vae",
50
+ "model_class": WanVideoVAE38,
51
+ "state_dict_converter": wan_video_vae_state_dict_converter,
52
+ },
53
+ ]
54
+
55
+
56
+ def _validate_dit_config(dit_config: dict[str, Any]) -> dict[str, Any]:
57
+ if not isinstance(dit_config, dict):
58
+ raise ValueError(f"`dit_config` must be a dict, got {type(dit_config)}")
59
+
60
+ validated = dict(dit_config)
61
+
62
+ signature = inspect.signature(WanVideoDiT.__init__)
63
+ allowed_keys = set()
64
+ required_keys = set()
65
+ for name, param in signature.parameters.items():
66
+ if name == "self":
67
+ continue
68
+ allowed_keys.add(name)
69
+ if param.default is inspect.Signature.empty:
70
+ required_keys.add(name)
71
+
72
+ unknown_keys = sorted(set(validated) - allowed_keys)
73
+ if unknown_keys:
74
+ raise ValueError(
75
+ f"Unknown keys in `dit_config`: {unknown_keys}. "
76
+ f"Allowed keys: {sorted(allowed_keys)}"
77
+ )
78
+
79
+ missing_keys = sorted(required_keys - set(validated))
80
+ if missing_keys:
81
+ raise ValueError(
82
+ f"Missing required keys in `dit_config`: {missing_keys}. "
83
+ "Please specify all required WanVideoDiT constructor args."
84
+ )
85
+
86
+ return validated
87
+
88
+
89
+ def _load_registered_model(
90
+ path,
91
+ model_name: str,
92
+ torch_dtype: torch.dtype,
93
+ device: str,
94
+ model_kwargs_override: dict[str, Any] | None = None,
95
+ ):
96
+ model_hash = hash_model_file(path)
97
+
98
+ matched_config = None
99
+ for config in WAN22_MODEL_REGISTRY:
100
+ if config["model_hash"] == model_hash and config["model_name"] == model_name:
101
+ matched_config = config
102
+ break
103
+ if matched_config is None:
104
+ raise ValueError(
105
+ f"Cannot detect model type for {model_name}. File: {path}. "
106
+ f"Model hash: {model_hash}. This standalone package follows DiffSynth hash-based loading."
107
+ )
108
+
109
+ model_class = matched_config["model_class"]
110
+ model_kwargs = dict(matched_config.get("extra_kwargs", {}))
111
+ if model_kwargs_override is not None:
112
+ model_kwargs.update(model_kwargs_override)
113
+ state_dict_converter = matched_config.get("state_dict_converter")
114
+
115
+ model = model_class(**model_kwargs)
116
+ state_dict = load_state_dict(path, torch_dtype=torch_dtype, device="cpu")
117
+ if state_dict_converter is not None:
118
+ state_dict = state_dict_converter(state_dict)
119
+
120
+ model.load_state_dict(state_dict, strict=False)
121
+ model = model.to(device=device, dtype=torch_dtype)
122
+ return model
123
+
124
+
125
+ def _resolve_configs(model_id: str, tokenizer_model_id: str, redirect_common_files: bool = True):
126
+ dit_config = ModelConfig(model_id=model_id, origin_file_pattern="diffusion_pytorch_model*.safetensors")
127
+ text_config = ModelConfig(model_id=model_id, origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth")
128
+ vae_config = ModelConfig(model_id=model_id, origin_file_pattern="Wan2.2_VAE.pth")
129
+ tokenizer_config = ModelConfig(model_id=tokenizer_model_id, origin_file_pattern="google/umt5-xxl/")
130
+
131
+ if redirect_common_files:
132
+ redirect_dict = {
133
+ "models_t5_umt5-xxl-enc-bf16.pth": ("DiffSynth-Studio/Wan-Series-Converted-Safetensors", "models_t5_umt5-xxl-enc-bf16.safetensors"),
134
+ "Wan2.2_VAE.pth": ("DiffSynth-Studio/Wan-Series-Converted-Safetensors", "Wan2.2_VAE.safetensors"),
135
+ }
136
+ text_config.model_id, text_config.origin_file_pattern = redirect_dict[text_config.origin_file_pattern]
137
+ vae_config.model_id, vae_config.origin_file_pattern = redirect_dict[vae_config.origin_file_pattern]
138
+ return dit_config, text_config, vae_config, tokenizer_config
139
+
140
+
141
+ def load_wan22_ti2v_5b_components(
142
+ device: str = "cuda",
143
+ torch_dtype: torch.dtype = torch.bfloat16,
144
+ model_id: str = "Wan-AI/Wan2.2-TI2V-5B",
145
+ tokenizer_model_id: str = "Wan-AI/Wan2.1-T2V-1.3B",
146
+ tokenizer_max_len: int = 512,
147
+ redirect_common_files: bool = True,
148
+ dit_config: dict[str, Any] | None = None,
149
+ skip_dit_load_from_pretrain: bool = False,
150
+ load_text_encoder: bool = True,
151
+ ):
152
+ logger.info("Loading Wan2.2-TI2V-5B components...")
153
+ start = time.time()
154
+
155
+ if dit_config is None:
156
+ raise ValueError("`dit_config` is required for Wan2.2-TI2V-5B loading.")
157
+ validated_dit_config = _validate_dit_config(dit_config)
158
+
159
+ dit_model_config, text_config, vae_config, tokenizer_config = _resolve_configs(
160
+ model_id=model_id,
161
+ tokenizer_model_id=tokenizer_model_id,
162
+ redirect_common_files=redirect_common_files,
163
+ )
164
+
165
+ vae_config.download_if_necessary()
166
+ if load_text_encoder:
167
+ text_config.download_if_necessary()
168
+ tokenizer_config.download_if_necessary()
169
+
170
+ if skip_dit_load_from_pretrain:
171
+ logger.info(
172
+ "Skipping pretrained video DiT load (`skip_dit_load_from_pretrain=True`); "
173
+ "initializing video expert randomly and expecting checkpoint override."
174
+ )
175
+ dit: WanVideoDiT = WanVideoDiT(**validated_dit_config).to(device=device, dtype=torch_dtype)
176
+ dit_path = SKIPPED_PRETRAIN_SENTINEL
177
+ else:
178
+ dit_model_config.download_if_necessary()
179
+ dit = _load_registered_model(
180
+ dit_model_config.path,
181
+ "wan_video_dit",
182
+ torch_dtype=torch_dtype,
183
+ device=device,
184
+ model_kwargs_override=validated_dit_config,
185
+ )
186
+ dit_path = str(dit_model_config.path)
187
+ text_encoder: WanTextEncoder | None = None
188
+ tokenizer: HuggingfaceTokenizer | None = None
189
+ text_encoder_path: str | None = None
190
+ tokenizer_path: str | None = None
191
+ if load_text_encoder:
192
+ text_encoder = _load_registered_model(
193
+ text_config.path,
194
+ "wan_video_text_encoder",
195
+ torch_dtype=torch_dtype,
196
+ device=device,
197
+ )
198
+ tokenizer = HuggingfaceTokenizer(
199
+ name=tokenizer_config.path,
200
+ seq_len=int(tokenizer_max_len),
201
+ clean="whitespace",
202
+ )
203
+ text_encoder_path = str(text_config.path)
204
+ tokenizer_path = str(tokenizer_config.path)
205
+ else:
206
+ logger.info(
207
+ "Skipping pretrained text encoder/tokenizer load (`load_text_encoder=False`); "
208
+ "training must provide cached `context/context_mask`."
209
+ )
210
+ vae: WanVideoVAE38 = _load_registered_model(vae_config.path, "wan_video_vae", torch_dtype=torch_dtype, device=device)
211
+ logger.info("Finished loading Wan2.2-TI2V-5B components in %.2f seconds.", time.time() - start)
212
+ return Wan22LoadedComponents(
213
+ dit=dit,
214
+ vae=vae,
215
+ text_encoder=text_encoder,
216
+ tokenizer=tokenizer,
217
+ dit_path=dit_path,
218
+ vae_path=str(vae_config.path),
219
+ text_encoder_path=text_encoder_path,
220
+ tokenizer_path=tokenizer_path,
221
+ )
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/state_dict_converters.py ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ def wan_video_vae_state_dict_converter(state_dict):
2
+ converted = {}
3
+ if "model_state" in state_dict:
4
+ state_dict = state_dict["model_state"]
5
+ for name, value in state_dict.items():
6
+ converted[f"model.{name}"] = value
7
+ return converted
8
+
9
+
10
+ def wan_video_dit_from_diffusers(state_dict):
11
+ rename_dict = {
12
+ "blocks.0.attn1.norm_k.weight": "blocks.0.self_attn.norm_k.weight",
13
+ "blocks.0.attn1.norm_q.weight": "blocks.0.self_attn.norm_q.weight",
14
+ "blocks.0.attn1.to_k.bias": "blocks.0.self_attn.k.bias",
15
+ "blocks.0.attn1.to_k.weight": "blocks.0.self_attn.k.weight",
16
+ "blocks.0.attn1.to_out.0.bias": "blocks.0.self_attn.o.bias",
17
+ "blocks.0.attn1.to_out.0.weight": "blocks.0.self_attn.o.weight",
18
+ "blocks.0.attn1.to_q.bias": "blocks.0.self_attn.q.bias",
19
+ "blocks.0.attn1.to_q.weight": "blocks.0.self_attn.q.weight",
20
+ "blocks.0.attn1.to_v.bias": "blocks.0.self_attn.v.bias",
21
+ "blocks.0.attn1.to_v.weight": "blocks.0.self_attn.v.weight",
22
+ "blocks.0.attn2.norm_k.weight": "blocks.0.cross_attn.norm_k.weight",
23
+ "blocks.0.attn2.norm_q.weight": "blocks.0.cross_attn.norm_q.weight",
24
+ "blocks.0.attn2.to_k.bias": "blocks.0.cross_attn.k.bias",
25
+ "blocks.0.attn2.to_k.weight": "blocks.0.cross_attn.k.weight",
26
+ "blocks.0.attn2.to_out.0.bias": "blocks.0.cross_attn.o.bias",
27
+ "blocks.0.attn2.to_out.0.weight": "blocks.0.cross_attn.o.weight",
28
+ "blocks.0.attn2.to_q.bias": "blocks.0.cross_attn.q.bias",
29
+ "blocks.0.attn2.to_q.weight": "blocks.0.cross_attn.q.weight",
30
+ "blocks.0.attn2.to_v.bias": "blocks.0.cross_attn.v.bias",
31
+ "blocks.0.attn2.to_v.weight": "blocks.0.cross_attn.v.weight",
32
+ "blocks.0.attn2.add_k_proj.bias": "blocks.0.cross_attn.k_img.bias",
33
+ "blocks.0.attn2.add_k_proj.weight": "blocks.0.cross_attn.k_img.weight",
34
+ "blocks.0.attn2.add_v_proj.bias": "blocks.0.cross_attn.v_img.bias",
35
+ "blocks.0.attn2.add_v_proj.weight": "blocks.0.cross_attn.v_img.weight",
36
+ "blocks.0.attn2.norm_added_k.weight": "blocks.0.cross_attn.norm_k_img.weight",
37
+ "blocks.0.ffn.net.0.proj.bias": "blocks.0.ffn.0.bias",
38
+ "blocks.0.ffn.net.0.proj.weight": "blocks.0.ffn.0.weight",
39
+ "blocks.0.ffn.net.2.bias": "blocks.0.ffn.2.bias",
40
+ "blocks.0.ffn.net.2.weight": "blocks.0.ffn.2.weight",
41
+ "blocks.0.norm2.bias": "blocks.0.norm3.bias",
42
+ "blocks.0.norm2.weight": "blocks.0.norm3.weight",
43
+ "blocks.0.scale_shift_table": "blocks.0.modulation",
44
+ "condition_embedder.text_embedder.linear_1.bias": "text_embedding.0.bias",
45
+ "condition_embedder.text_embedder.linear_1.weight": "text_embedding.0.weight",
46
+ "condition_embedder.text_embedder.linear_2.bias": "text_embedding.2.bias",
47
+ "condition_embedder.text_embedder.linear_2.weight": "text_embedding.2.weight",
48
+ "condition_embedder.time_embedder.linear_1.bias": "time_embedding.0.bias",
49
+ "condition_embedder.time_embedder.linear_1.weight": "time_embedding.0.weight",
50
+ "condition_embedder.time_embedder.linear_2.bias": "time_embedding.2.bias",
51
+ "condition_embedder.time_embedder.linear_2.weight": "time_embedding.2.weight",
52
+ "condition_embedder.time_proj.bias": "time_projection.1.bias",
53
+ "condition_embedder.time_proj.weight": "time_projection.1.weight",
54
+ "condition_embedder.image_embedder.ff.net.0.proj.bias": "img_emb.proj.1.bias",
55
+ "condition_embedder.image_embedder.ff.net.0.proj.weight": "img_emb.proj.1.weight",
56
+ "condition_embedder.image_embedder.ff.net.2.bias": "img_emb.proj.3.bias",
57
+ "condition_embedder.image_embedder.ff.net.2.weight": "img_emb.proj.3.weight",
58
+ "condition_embedder.image_embedder.norm1.bias": "img_emb.proj.0.bias",
59
+ "condition_embedder.image_embedder.norm1.weight": "img_emb.proj.0.weight",
60
+ "condition_embedder.image_embedder.norm2.bias": "img_emb.proj.4.bias",
61
+ "condition_embedder.image_embedder.norm2.weight": "img_emb.proj.4.weight",
62
+ "patch_embedding.bias": "patch_embedding.bias",
63
+ "patch_embedding.weight": "patch_embedding.weight",
64
+ "scale_shift_table": "head.modulation",
65
+ "proj_out.bias": "head.head.bias",
66
+ "proj_out.weight": "head.head.weight",
67
+ }
68
+ converted = {}
69
+ for name, value in state_dict.items():
70
+ if name in rename_dict:
71
+ converted[rename_dict[name]] = value
72
+ else:
73
+ probe_name = ".".join(name.split(".")[:1] + ["0"] + name.split(".")[2:])
74
+ if probe_name in rename_dict:
75
+ mapped = rename_dict[probe_name]
76
+ mapped = ".".join(mapped.split(".")[:1] + [name.split(".")[1]] + mapped.split(".")[2:])
77
+ converted[mapped] = value
78
+ return converted
79
+
80
+
81
+ def wan_video_dit_state_dict_converter(state_dict):
82
+ converted = {}
83
+ for name, value in state_dict.items():
84
+ if name.startswith("vace"):
85
+ continue
86
+ if name.split(".")[0] in ["pose_patch_embedding", "face_adapter", "face_encoder", "motion_encoder"]:
87
+ continue
88
+ key = name[6:] if name.startswith("model.") else name
89
+ converted[key] = value
90
+ return converted
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/mot.py ADDED
@@ -0,0 +1,563 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import Dict, Optional
4
+
5
+ import torch
6
+ import torch.nn as nn
7
+
8
+ from .wan_video_dit import flash_attention, modulate, rope_apply
9
+ from fastwam.utils.logging_config import get_logger
10
+
11
+ logger = get_logger(__name__)
12
+
13
+
14
+ class MoT(nn.Module):
15
+ def __init__(
16
+ self,
17
+ mixtures: Dict[str, nn.Module],
18
+ mot_checkpoint_mixed_attn: bool = True,
19
+ ):
20
+ super().__init__()
21
+ if not mixtures:
22
+ raise ValueError("`mixtures` cannot be empty.")
23
+ if "video" not in mixtures or "action" not in mixtures:
24
+ raise ValueError("`mixtures` must include both 'video' and 'action' experts.")
25
+
26
+ self.mixtures = nn.ModuleDict(mixtures)
27
+ self.expert_order = list(self.mixtures.keys())
28
+ self.mot_checkpoint_mixed_attn = mot_checkpoint_mixed_attn
29
+ # Opt-in performance switches. Defaults preserve the historical path.
30
+ self.omit_all_true_attention_mask = False
31
+ self.rope_compute_dtype = torch.float64
32
+ if mot_checkpoint_mixed_attn:
33
+ logger.info("Using gradient checkpointing for mixture attention. This will save memory but use more computation.")
34
+
35
+ first_expert = self.mixtures[self.expert_order[0]]
36
+ self.num_layers = len(first_expert.blocks)
37
+ self.num_heads = first_expert.num_heads
38
+ self.attn_head_dim = first_expert.attn_head_dim
39
+
40
+ for name in self.expert_order[1:]:
41
+ expert = self.mixtures[name]
42
+ if len(expert.blocks) != self.num_layers:
43
+ raise ValueError(
44
+ f"All experts must have same number of layers; got {self.num_layers} and {len(expert.blocks)}"
45
+ )
46
+ if expert.num_heads != self.num_heads:
47
+ raise ValueError(
48
+ f"All experts must have same num_heads; got {self.num_heads} and {expert.num_heads}"
49
+ )
50
+ if expert.attn_head_dim != self.attn_head_dim:
51
+ raise ValueError(
52
+ "All experts must have same attn_head_dim; "
53
+ f"got {self.attn_head_dim} and {expert.attn_head_dim}"
54
+ )
55
+
56
+ logger.info(f"Initialized MoT with experts: {self.expert_order}, num_layers={self.num_layers}")
57
+ for name in self.expert_order:
58
+ expert = self.mixtures[name]
59
+ logger.info(f" Expert '{name}': num_params={sum(p.numel() for p in expert.parameters()) / 1e9:.2f} B")
60
+
61
+ @staticmethod
62
+ def _split_modulation(block, t_mod: torch.Tensor):
63
+ has_seq = len(t_mod.shape) == 4
64
+ chunk_dim = 2 if has_seq else 1
65
+
66
+ base_mod = block.modulation.to(dtype=t_mod.dtype, device=t_mod.device)
67
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (base_mod + t_mod).chunk(6, dim=chunk_dim)
68
+ if has_seq:
69
+ # means t_mod has separate modulation for each token, otherwise same modulation for all tokens in the block
70
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
71
+ shift_msa.squeeze(2),
72
+ scale_msa.squeeze(2),
73
+ gate_msa.squeeze(2),
74
+ shift_mlp.squeeze(2),
75
+ scale_mlp.squeeze(2),
76
+ gate_mlp.squeeze(2),
77
+ )
78
+ return shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp
79
+
80
+ def _mixed_attention(
81
+ self,
82
+ q_cat: torch.Tensor,
83
+ k_cat: torch.Tensor,
84
+ v_cat: torch.Tensor,
85
+ attention_mask: Optional[torch.Tensor],
86
+ ) -> torch.Tensor:
87
+ attn_mask = (
88
+ None
89
+ if attention_mask is None or self.omit_all_true_attention_mask
90
+ else attention_mask.to(device=q_cat.device)
91
+ )
92
+
93
+ def _forward(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
94
+ return flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=attn_mask)
95
+
96
+ if self.mot_checkpoint_mixed_attn and self.training:
97
+ return torch.utils.checkpoint.checkpoint(
98
+ _forward,
99
+ q_cat,
100
+ k_cat,
101
+ v_cat,
102
+ use_reentrant=False,
103
+ )
104
+ return _forward(q_cat, k_cat, v_cat)
105
+
106
+ @staticmethod
107
+ def _apply_expert_post_block(
108
+ block,
109
+ residual_x: torch.Tensor,
110
+ mixed_attn_out: torch.Tensor,
111
+ gate_msa: torch.Tensor,
112
+ shift_mlp: torch.Tensor,
113
+ scale_mlp: torch.Tensor,
114
+ gate_mlp: torch.Tensor,
115
+ context_payload: Optional[dict],
116
+ ) -> torch.Tensor:
117
+ x = block.gate(residual_x, gate_msa, block.self_attn.o(mixed_attn_out))
118
+
119
+ if context_payload is not None:
120
+ context = context_payload.get("context")
121
+ if context is not None:
122
+ context_mask = context_payload.get("mask")
123
+ if context_mask is not None and context_mask.dim() == 3:
124
+ context_mask = context_mask.unsqueeze(1)
125
+ x = x + block.cross_attn(block.norm3(x), context, ctx_mask=context_mask)
126
+
127
+ mlp_input = modulate(block.norm2(x), shift_mlp, scale_mlp)
128
+ x = block.gate(x, gate_mlp, block.ffn(mlp_input))
129
+ return x
130
+
131
+ def _build_expert_attention_io(
132
+ self,
133
+ expert,
134
+ block,
135
+ x: torch.Tensor,
136
+ freqs: torch.Tensor,
137
+ t_mod: torch.Tensor,
138
+ ) -> tuple[
139
+ torch.Tensor,
140
+ torch.Tensor,
141
+ torch.Tensor,
142
+ torch.Tensor,
143
+ torch.Tensor,
144
+ torch.Tensor,
145
+ torch.Tensor,
146
+ torch.Tensor,
147
+ bool,
148
+ ]:
149
+ """Build per-expert attention tensors and post-block states.
150
+
151
+ Args:
152
+ expert: Expert module that owns this `block`; only used to read
153
+ `use_gradient_checkpointing`.
154
+ block: Transformer block for current layer (`expert.blocks[layer_idx]`).
155
+ x: Current expert tokens, shape [B, S, D].
156
+ freqs: RoPE frequencies aligned with token sequence, shape [S, 1, rope_dim].
157
+ t_mod: Time modulation tensor for this expert/layer.
158
+
159
+ Returns:
160
+ q: Query after q-proj, RMSNorm, and RoPE, shape [B, S, H*Dh].
161
+ k: Key after k-proj, RMSNorm, and RoPE, shape [B, S, H*Dh].
162
+ v: Value after v-proj, shape [B, S, H*Dh].
163
+ residual_x: Original input `x` for residual path in post block.
164
+ gate_msa: Gating tensor for self-attention residual branch.
165
+ shift_mlp: Shift tensor for MLP modulation.
166
+ scale_mlp: Scale tensor for MLP modulation.
167
+ gate_mlp: Gating tensor for MLP residual branch.
168
+ use_gradient_checkpointing: Whether this expert enables checkpointing.
169
+ """
170
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self._split_modulation(block, t_mod)
171
+ attn_input = modulate(block.norm1(x), shift_msa, scale_msa)
172
+
173
+ q = block.self_attn.norm_q(block.self_attn.q(attn_input))
174
+ k = block.self_attn.norm_k(block.self_attn.k(attn_input))
175
+ v = block.self_attn.v(attn_input)
176
+
177
+ q = rope_apply(q, freqs, block.num_heads, self.rope_compute_dtype)
178
+ k = rope_apply(k, freqs, block.num_heads, self.rope_compute_dtype)
179
+
180
+ use_gradient_checkpointing = bool(getattr(expert, "use_gradient_checkpointing", False))
181
+ return (
182
+ q,
183
+ k,
184
+ v,
185
+ x,
186
+ gate_msa,
187
+ shift_mlp,
188
+ scale_mlp,
189
+ gate_mlp,
190
+ use_gradient_checkpointing,
191
+ )
192
+
193
+ def _apply_post_with_optional_checkpoint(
194
+ self,
195
+ block,
196
+ residual_x: torch.Tensor,
197
+ gate_msa: torch.Tensor,
198
+ shift_mlp: torch.Tensor,
199
+ scale_mlp: torch.Tensor,
200
+ gate_mlp: torch.Tensor,
201
+ use_gradient_checkpointing: bool,
202
+ mixed_slice: torch.Tensor,
203
+ context_payload: Optional[dict],
204
+ ) -> torch.Tensor:
205
+ """Apply post-attention computations, with optional checkpointing.
206
+
207
+ Args:
208
+ block: Transformer block for current layer.
209
+ residual_x: Residual input tokens before attention update, shape [B, S, D].
210
+ gate_msa: Gating tensor used after mixed self-attention.
211
+ shift_mlp: Shift tensor for MLP input modulation.
212
+ scale_mlp: Scale tensor for MLP input modulation.
213
+ gate_mlp: Gating tensor used after MLP.
214
+ use_gradient_checkpointing: If True and training, checkpoint this post block.
215
+ mixed_slice: Mixed-attention output for this expert, shape [B, S, H*Dh].
216
+ context_payload: Optional dict for cross-attention.
217
+ - `context`: encoder states [B, L, D]
218
+ - `mask`: attention mask [B, S, L] or [B, 1, S, L]
219
+
220
+ Returns:
221
+ Updated expert tokens after self-attn residual, optional cross-attn, and MLP.
222
+ """
223
+ def _post_fn(
224
+ _mixed_slice: torch.Tensor,
225
+ _x: torch.Tensor,
226
+ _gate_msa: torch.Tensor,
227
+ _shift_mlp: torch.Tensor,
228
+ _scale_mlp: torch.Tensor,
229
+ _gate_mlp: torch.Tensor,
230
+ _block=block,
231
+ _context_payload=context_payload,
232
+ ) -> torch.Tensor:
233
+ return self._apply_expert_post_block(
234
+ block=_block,
235
+ residual_x=_x,
236
+ mixed_attn_out=_mixed_slice,
237
+ gate_msa=_gate_msa,
238
+ shift_mlp=_shift_mlp,
239
+ scale_mlp=_scale_mlp,
240
+ gate_mlp=_gate_mlp,
241
+ context_payload=_context_payload,
242
+ )
243
+
244
+ if use_gradient_checkpointing and self.training:
245
+ return torch.utils.checkpoint.checkpoint(
246
+ _post_fn,
247
+ mixed_slice,
248
+ residual_x,
249
+ gate_msa,
250
+ shift_mlp,
251
+ scale_mlp,
252
+ gate_mlp,
253
+ use_reentrant=False,
254
+ )
255
+ return _post_fn(
256
+ mixed_slice,
257
+ residual_x,
258
+ gate_msa,
259
+ shift_mlp,
260
+ scale_mlp,
261
+ gate_mlp,
262
+ )
263
+
264
+ def prefill_video_cache(
265
+ self,
266
+ video_tokens: torch.Tensor,
267
+ video_freqs: torch.Tensor,
268
+ video_t_mod: torch.Tensor,
269
+ video_context_payload: Optional[dict],
270
+ video_attention_mask: torch.Tensor,
271
+ ) -> list[dict[str, torch.Tensor]]:
272
+ """Prefill video branch once and cache per-layer K/V for action denoising.
273
+
274
+ Args:
275
+ video_tokens: Video tokens before layer 0, shape [B, Sv, D].
276
+ video_freqs: Video RoPE frequencies, shape [Sv, 1, rope_dim].
277
+ video_t_mod: Video time modulation tensor.
278
+ video_context_payload: Optional dict for video cross-attention.
279
+ - `context`: encoder states [B, L, D]
280
+ - `mask`: attention mask [B, Sv, L] or [B, 1, Sv, L]
281
+ video_attention_mask: Video self-attention mask, shape [Sv, Sv].
282
+
283
+ Returns:
284
+ Layer-wise cache list with length `num_layers`.
285
+ Each entry contains:
286
+ - `k`: video key tensor [B, Sv, H*Dh]
287
+ - `v`: video value tensor [B, Sv, H*Dh]
288
+ """
289
+ if "video" not in self.mixtures:
290
+ raise ValueError("MoT requires `video` expert for `prefill_video_cache`.")
291
+ if video_attention_mask.ndim != 2:
292
+ raise ValueError(
293
+ f"`video_attention_mask` must be 2D [S,S], got shape {tuple(video_attention_mask.shape)}"
294
+ )
295
+ if video_attention_mask.shape[0] != video_attention_mask.shape[1]:
296
+ raise ValueError(
297
+ f"`video_attention_mask` must be square, got shape {tuple(video_attention_mask.shape)}"
298
+ )
299
+ if video_attention_mask.shape[0] != video_tokens.shape[1]:
300
+ raise ValueError(
301
+ "`video_attention_mask` seq length mismatch: "
302
+ f"mask={video_attention_mask.shape[0]} vs tokens={video_tokens.shape[1]}"
303
+ )
304
+
305
+ expert = self.mixtures["video"]
306
+ x = video_tokens
307
+ kv_cache: list[dict[str, torch.Tensor]] = []
308
+ for layer_idx in range(self.num_layers):
309
+ block = expert.blocks[layer_idx]
310
+ # Build video Q/K/V from current layer input tokens.
311
+ (
312
+ q,
313
+ k,
314
+ v,
315
+ residual_x,
316
+ gate_msa,
317
+ shift_mlp,
318
+ scale_mlp,
319
+ gate_mlp,
320
+ use_gradient_checkpointing,
321
+ ) = self._build_expert_attention_io(
322
+ expert=expert,
323
+ block=block,
324
+ x=x,
325
+ freqs=video_freqs,
326
+ t_mod=video_t_mod,
327
+ )
328
+ # Video prefill uses only video self-attention mask.
329
+ mixed = self._mixed_attention(
330
+ q_cat=q,
331
+ k_cat=k,
332
+ v_cat=v,
333
+ attention_mask=video_attention_mask,
334
+ )
335
+ # Update video tokens for the next layer and persist current layer K/V.
336
+ x = self._apply_post_with_optional_checkpoint(
337
+ block=block,
338
+ residual_x=residual_x,
339
+ gate_msa=gate_msa,
340
+ shift_mlp=shift_mlp,
341
+ scale_mlp=scale_mlp,
342
+ gate_mlp=gate_mlp,
343
+ use_gradient_checkpointing=use_gradient_checkpointing,
344
+ mixed_slice=mixed,
345
+ context_payload=video_context_payload,
346
+ )
347
+ kv_cache.append({"k": k, "v": v})
348
+ return kv_cache
349
+
350
+ def forward_action_with_video_cache(
351
+ self,
352
+ action_tokens: torch.Tensor,
353
+ action_freqs: torch.Tensor,
354
+ action_t_mod: torch.Tensor,
355
+ action_context_payload: Optional[dict],
356
+ video_kv_cache: list[dict[str, torch.Tensor]],
357
+ attention_mask: torch.Tensor,
358
+ video_seq_len: int,
359
+ ) -> torch.Tensor:
360
+ """Run action branch with cached video K/V instead of recomputing video tokens.
361
+
362
+ Args:
363
+ action_tokens: Action tokens before layer 0, shape [B, Sa, D].
364
+ action_freqs: Action RoPE frequencies, shape [Sa, 1, rope_dim].
365
+ action_t_mod: Action time modulation tensor.
366
+ action_context_payload: Optional dict for action cross-attention.
367
+ - `context`: encoder states [B, L, D]
368
+ - `mask`: attention mask [B, Sa, L] or [B, 1, Sa, L]
369
+ video_kv_cache: Layer-wise cached video K/V from `prefill_video_cache`.
370
+ attention_mask: Joint [video+action] mask, shape [Sv+Sa, Sv+Sa].
371
+ video_seq_len: Video token count `Sv` in the joint sequence prefix.
372
+
373
+ Returns:
374
+ Updated action tokens after all layers, shape [B, Sa, D].
375
+ """
376
+ if "action" not in self.mixtures:
377
+ raise ValueError("MoT requires `action` expert for `forward_action_with_video_cache`.")
378
+ if len(video_kv_cache) != self.num_layers:
379
+ raise ValueError(
380
+ f"`video_kv_cache` must contain {self.num_layers} layers, got {len(video_kv_cache)}."
381
+ )
382
+ if attention_mask.ndim != 2:
383
+ raise ValueError(f"`attention_mask` must be 2D [S,S], got shape {tuple(attention_mask.shape)}")
384
+ if attention_mask.shape[0] != attention_mask.shape[1]:
385
+ raise ValueError(f"`attention_mask` must be square, got shape {tuple(attention_mask.shape)}")
386
+
387
+ action_seq_len = int(action_tokens.shape[1])
388
+ total_seq_len = int(video_seq_len) + action_seq_len
389
+ if attention_mask.shape[0] != total_seq_len:
390
+ raise ValueError(
391
+ "`attention_mask` seq length mismatch: "
392
+ f"mask={attention_mask.shape[0]} vs expected_total={total_seq_len}"
393
+ )
394
+ # Use the action query rows from the joint [video+action] mask.
395
+ action_attention_mask = attention_mask[video_seq_len:total_seq_len, :total_seq_len]
396
+
397
+ expert = self.mixtures["action"]
398
+ x = action_tokens
399
+ for layer_idx in range(self.num_layers):
400
+ block = expert.blocks[layer_idx]
401
+ # Action query/key/value are still step-dependent and must be recomputed each step.
402
+ (
403
+ q_action,
404
+ k_action,
405
+ v_action,
406
+ residual_x,
407
+ gate_msa,
408
+ shift_mlp,
409
+ scale_mlp,
410
+ gate_mlp,
411
+ use_gradient_checkpointing,
412
+ ) = self._build_expert_attention_io(
413
+ expert=expert,
414
+ block=block,
415
+ x=x,
416
+ freqs=action_freqs,
417
+ t_mod=action_t_mod,
418
+ )
419
+ layer_cache = video_kv_cache[layer_idx]
420
+ if "k" not in layer_cache or "v" not in layer_cache:
421
+ raise ValueError(
422
+ f"`video_kv_cache[{layer_idx}]` must contain `k` and `v`."
423
+ )
424
+
425
+ k_video = layer_cache["k"]
426
+ v_video = layer_cache["v"]
427
+ if k_video.shape[1] != video_seq_len or v_video.shape[1] != video_seq_len:
428
+ raise ValueError(
429
+ f"`video_kv_cache[{layer_idx}]` seq len mismatch, expected {video_seq_len}."
430
+ )
431
+
432
+ # Mixed attention: action queries attend to cached video K/V plus current action K/V.
433
+ k_cat = torch.cat([k_video, k_action], dim=1)
434
+ v_cat = torch.cat([v_video, v_action], dim=1)
435
+ mixed = self._mixed_attention(
436
+ q_cat=q_action,
437
+ k_cat=k_cat,
438
+ v_cat=v_cat,
439
+ attention_mask=action_attention_mask,
440
+ )
441
+ x = self._apply_post_with_optional_checkpoint(
442
+ block=block,
443
+ residual_x=residual_x,
444
+ gate_msa=gate_msa,
445
+ shift_mlp=shift_mlp,
446
+ scale_mlp=scale_mlp,
447
+ gate_mlp=gate_mlp,
448
+ use_gradient_checkpointing=use_gradient_checkpointing,
449
+ mixed_slice=mixed,
450
+ context_payload=action_context_payload,
451
+ )
452
+ return x
453
+
454
+ def forward(
455
+ self,
456
+ embeds_all: Dict[str, torch.Tensor],
457
+ attention_mask: torch.Tensor,
458
+ freqs_all: Dict[str, torch.Tensor],
459
+ context_all: Dict[str, Optional[dict]],
460
+ t_mod_all: Dict[str, torch.Tensor],
461
+ ):
462
+ missing = [k for k in self.expert_order if k not in embeds_all]
463
+ if missing:
464
+ raise ValueError(f"Missing expert tokens for {missing}")
465
+ missing = [k for k in self.expert_order if k not in freqs_all]
466
+ if missing:
467
+ raise ValueError(f"Missing expert freqs for {missing}")
468
+ missing = [k for k in self.expert_order if k not in t_mod_all]
469
+ if missing:
470
+ raise ValueError(f"Missing expert t_mod for {missing}")
471
+
472
+ if attention_mask.ndim != 2:
473
+ raise ValueError(f"`attention_mask` must be 2D [S, S], got shape {tuple(attention_mask.shape)}")
474
+ if attention_mask.shape[0] != attention_mask.shape[1]:
475
+ raise ValueError(f"`attention_mask` must be square, got shape {tuple(attention_mask.shape)}")
476
+
477
+ tokens_all = {k: v for k, v in embeds_all.items()}
478
+
479
+ for layer_idx in range(self.num_layers):
480
+ q_chunks = []
481
+ k_chunks = []
482
+ v_chunks = []
483
+ cached = {}
484
+ seq_lens = []
485
+
486
+ for name in self.expert_order:
487
+ expert = self.mixtures[name]
488
+ block = expert.blocks[layer_idx]
489
+ x = tokens_all[name]
490
+ freqs = freqs_all[name]
491
+ t_mod = t_mod_all[name]
492
+
493
+ (
494
+ q,
495
+ k,
496
+ v,
497
+ residual_x,
498
+ gate_msa,
499
+ shift_mlp,
500
+ scale_mlp,
501
+ gate_mlp,
502
+ use_gradient_checkpointing,
503
+ ) = self._build_expert_attention_io(
504
+ expert=expert,
505
+ block=block,
506
+ x=x,
507
+ freqs=freqs,
508
+ t_mod=t_mod,
509
+ )
510
+
511
+ q_chunks.append(q)
512
+ k_chunks.append(k)
513
+ v_chunks.append(v)
514
+ seq_lens.append(x.shape[1])
515
+ cached[name] = {
516
+ "block": block,
517
+ "residual_x": residual_x,
518
+ "gate_msa": gate_msa,
519
+ "shift_mlp": shift_mlp,
520
+ "scale_mlp": scale_mlp,
521
+ "gate_mlp": gate_mlp,
522
+ "use_gradient_checkpointing": use_gradient_checkpointing,
523
+ }
524
+
525
+ # 3. concat all tokens for mixed attention
526
+ q_cat = torch.cat(q_chunks, dim=1)
527
+ k_cat = torch.cat(k_chunks, dim=1)
528
+ v_cat = torch.cat(v_chunks, dim=1)
529
+
530
+ total_seq = q_cat.shape[1]
531
+ if attention_mask.shape[0] != total_seq:
532
+ raise ValueError(
533
+ "Attention mask seq length mismatch: "
534
+ f"mask={attention_mask.shape[0]} vs tokens={total_seq}"
535
+ )
536
+
537
+ mixed = self._mixed_attention(q_cat=q_cat, k_cat=k_cat, v_cat=v_cat, attention_mask=attention_mask)
538
+
539
+ start = 0
540
+ for name, seq_len in zip(self.expert_order, seq_lens):
541
+ # 4. split mixed attention output and apply post-attention blocks for each expert
542
+ end = start + seq_len
543
+ mixed_slice = mixed[:, start:end, :]
544
+ cached_expert = cached[name]
545
+ block = cached_expert["block"]
546
+ context_payload = context_all.get(name)
547
+
548
+ updated_tokens = self._apply_post_with_optional_checkpoint(
549
+ block=block,
550
+ residual_x=cached_expert["residual_x"],
551
+ gate_msa=cached_expert["gate_msa"],
552
+ shift_mlp=cached_expert["shift_mlp"],
553
+ scale_mlp=cached_expert["scale_mlp"],
554
+ gate_mlp=cached_expert["gate_mlp"],
555
+ use_gradient_checkpointing=cached_expert["use_gradient_checkpointing"],
556
+ mixed_slice=mixed_slice,
557
+ context_payload=context_payload,
558
+ )
559
+
560
+ tokens_all[name] = updated_tokens
561
+ start = end
562
+
563
+ return tokens_all
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/schedulers/scheduler_continuous.py ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+
4
+ class WanContinuousFlowMatchScheduler:
5
+ """Continuous-time Flow-Matching scheduler with shift-based sampling."""
6
+
7
+ def __init__(self, num_train_timesteps: int = 1000, shift: float = 5.0, eps: float = 1e-10):
8
+ if num_train_timesteps <= 0:
9
+ raise ValueError(f"`num_train_timesteps` must be positive, got {num_train_timesteps}")
10
+ if shift <= 0:
11
+ raise ValueError(f"`shift` must be positive, got {shift}")
12
+ self.num_train_timesteps = int(num_train_timesteps)
13
+ self.shift = float(shift)
14
+ self.eps = float(eps)
15
+ self._y_min, self._weight_norm_const = self._precompute_training_weight_stats()
16
+
17
+ @staticmethod
18
+ def _phi(u: torch.Tensor, shift: float) -> torch.Tensor:
19
+ return shift * u / (1.0 + (shift - 1.0) * u)
20
+
21
+ def _precompute_training_weight_stats(self) -> tuple[float, float]:
22
+ steps = self.num_train_timesteps
23
+ u_grid = torch.linspace(1.0, 0.0, steps + 1, dtype=torch.float64)[:-1]
24
+ t_grid = self._phi(u_grid, self.shift) * float(steps)
25
+ y_grid = torch.exp(-2.0 * ((t_grid - (steps / 2.0)) / steps) ** 2)
26
+ y_min = float(y_grid.min().item())
27
+ y_shifted_grid = y_grid - y_min
28
+ norm_const = float(y_shifted_grid.mean().item())
29
+ return y_min, norm_const
30
+
31
+ def sample_training_t(self, batch_size: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor:
32
+ if batch_size <= 0:
33
+ raise ValueError(f"`batch_size` must be positive, got {batch_size}")
34
+ u = torch.rand((batch_size,), device=device, dtype=torch.float32)
35
+ sigma = self._phi(u, self.shift)
36
+ timestep = sigma * float(self.num_train_timesteps)
37
+ return timestep.to(dtype=dtype)
38
+
39
+ def training_weight(self, timestep: torch.Tensor) -> torch.Tensor:
40
+ t = timestep.to(dtype=torch.float32)
41
+ steps = float(self.num_train_timesteps)
42
+ y = torch.exp(-2.0 * ((t - (steps / 2.0)) / steps) ** 2)
43
+ y_shifted = y - self._y_min
44
+ weight = y_shifted / (self._weight_norm_const + self.eps)
45
+ if weight.numel() == 1:
46
+ return weight.reshape(())
47
+ return weight
48
+
49
+ def add_noise(self, original_samples: torch.Tensor, noise: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor:
50
+ sigma = (timestep / float(self.num_train_timesteps)).to(
51
+ original_samples.device, dtype=original_samples.dtype
52
+ )
53
+ if sigma.ndim == 0:
54
+ return (1 - sigma) * original_samples + sigma * noise
55
+ sigma = sigma.view(-1, *([1] * (original_samples.ndim - 1)))
56
+ return (1 - sigma) * original_samples + sigma * noise
57
+
58
+ @staticmethod
59
+ def training_target(sample: torch.Tensor, noise: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor:
60
+ del timestep
61
+ return noise - sample
62
+
63
+ def build_inference_schedule(
64
+ self,
65
+ num_inference_steps: int,
66
+ device: torch.device,
67
+ dtype: torch.dtype,
68
+ shift_override: float | None = None,
69
+ ) -> tuple[torch.Tensor, torch.Tensor]:
70
+ if num_inference_steps <= 0:
71
+ raise ValueError(f"`num_inference_steps` must be positive, got {num_inference_steps}")
72
+ shift = self.shift if shift_override is None else float(shift_override)
73
+ if shift <= 0:
74
+ raise ValueError(f"`shift` must be positive, got {shift}")
75
+
76
+ u_steps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32)
77
+ sigma_steps = self._phi(u_steps, shift)
78
+ timesteps = sigma_steps[:-1] * float(self.num_train_timesteps)
79
+ deltas = sigma_steps[1:] - sigma_steps[:-1]
80
+ return timesteps.to(dtype=dtype), deltas.to(dtype=dtype)
81
+
82
+ @staticmethod
83
+ def step(model_output: torch.Tensor, delta: torch.Tensor, sample: torch.Tensor) -> torch.Tensor:
84
+ delta = delta.to(sample.device, dtype=sample.dtype)
85
+ if delta.ndim == 0:
86
+ return sample + model_output * delta
87
+ delta = delta.view(-1, *([1] * (sample.ndim - 1)))
88
+ return sample + model_output * delta
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan22.py ADDED
@@ -0,0 +1,405 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import os
3
+ import torch
4
+ import torch.nn.functional as F
5
+ from PIL import Image
6
+ from typing import Any, Optional, Sequence, Union
7
+
8
+ from .helpers.loader import load_wan22_ti2v_5b_components
9
+ from .schedulers.scheduler_continuous import WanContinuousFlowMatchScheduler
10
+ from .wan_video_dit import WanVideoDiT
11
+
12
+
13
+ class Wan22Core(torch.nn.Module):
14
+ """Standalone Wan2.2-TI2V-5B core without pipeline unit graph."""
15
+
16
+ def __init__(
17
+ self,
18
+ dit: WanVideoDiT,
19
+ vae,
20
+ text_encoder,
21
+ tokenizer,
22
+ device="cpu",
23
+ torch_dtype=torch.float32,
24
+ train_shift: float = 5.0,
25
+ infer_shift: float = 5.0,
26
+ num_train_timesteps: int = 1000,
27
+ ):
28
+ super().__init__()
29
+ self.dit = dit
30
+ self.vae = vae
31
+ self.text_encoder = text_encoder
32
+ self.tokenizer = tokenizer
33
+ self.train_scheduler = WanContinuousFlowMatchScheduler(
34
+ num_train_timesteps=num_train_timesteps,
35
+ shift=train_shift,
36
+ )
37
+ self.infer_scheduler = WanContinuousFlowMatchScheduler(
38
+ num_train_timesteps=num_train_timesteps,
39
+ shift=infer_shift,
40
+ )
41
+ self.device = torch.device(device)
42
+ self.torch_dtype = torch_dtype
43
+ self.to(self.device)
44
+
45
+ @classmethod
46
+ def from_wan22_pretrained(
47
+ cls,
48
+ device="cuda",
49
+ torch_dtype=torch.bfloat16,
50
+ model_id="Wan-AI/Wan2.2-TI2V-5B",
51
+ tokenizer_model_id="Wan-AI/Wan2.1-T2V-1.3B",
52
+ tokenizer_max_len: int = 512,
53
+ redirect_common_files=True,
54
+ dit_config: dict[str, Any] | None = None,
55
+ train_shift: float = 5.0,
56
+ infer_shift: float = 5.0,
57
+ num_train_timesteps: int = 1000,
58
+ ):
59
+ if dit_config is None:
60
+ raise ValueError("`dit_config` is required for Wan22Core.from_wan22_pretrained().")
61
+ components = load_wan22_ti2v_5b_components(
62
+ device=device,
63
+ torch_dtype=torch_dtype,
64
+ model_id=model_id,
65
+ tokenizer_model_id=tokenizer_model_id,
66
+ tokenizer_max_len=tokenizer_max_len,
67
+ redirect_common_files=redirect_common_files,
68
+ dit_config=dit_config,
69
+ )
70
+ model = cls(
71
+ dit=components.dit,
72
+ vae=components.vae,
73
+ text_encoder=components.text_encoder,
74
+ tokenizer=components.tokenizer,
75
+ device=device,
76
+ torch_dtype=torch_dtype,
77
+ train_shift=train_shift,
78
+ infer_shift=infer_shift,
79
+ num_train_timesteps=num_train_timesteps,
80
+ )
81
+ model.model_paths = {
82
+ "dit": components.dit_path,
83
+ "vae": components.vae_path,
84
+ "text_encoder": components.text_encoder_path,
85
+ "tokenizer": components.tokenizer_path,
86
+ }
87
+ return model
88
+
89
+ def to(self, *args, **kwargs):
90
+ super().to(*args, **kwargs)
91
+ self.dit.to(*args, **kwargs)
92
+ self.text_encoder.to(*args, **kwargs)
93
+ self.vae.to(*args, **kwargs)
94
+ return self
95
+
96
+ @staticmethod
97
+ def _check_resize_height_width(height, width, num_frames):
98
+ if height % 16 != 0:
99
+ height = (height + 15) // 16 * 16
100
+ if width % 16 != 0:
101
+ width = (width + 15) // 16 * 16
102
+ if num_frames % 4 != 1:
103
+ num_frames = (num_frames + 3) // 4 * 4 + 1
104
+ return height, width, num_frames
105
+
106
+ def encode_prompt(self, prompt: Union[str, Sequence[str]]):
107
+ ids, mask = self.tokenizer(prompt, return_mask=True, add_special_tokens=True)
108
+ ids = ids.to(self.device)
109
+ mask = mask.to(self.device, dtype=torch.bool)
110
+ prompt_emb = self.text_encoder(ids, mask)
111
+ return prompt_emb.to(device=self.device), mask
112
+
113
+ def _encode_video_latents(self, video_tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)):
114
+ z = self.vae.encode(
115
+ video_tensor,
116
+ device=self.device,
117
+ tiled=tiled,
118
+ tile_size=tile_size,
119
+ tile_stride=tile_stride,
120
+ )
121
+ return z
122
+
123
+ def _encode_input_image_latents_tensor(self, input_image: torch.Tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)):
124
+ if input_image.ndim == 3:
125
+ input_image = input_image.unsqueeze(0)
126
+ if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
127
+ raise ValueError(
128
+ f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
129
+ )
130
+ image = input_image.to(device=self.device)[0].unsqueeze(1)
131
+ z = self.vae.encode([image], device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride)
132
+ if isinstance(z, list):
133
+ z = z[0].unsqueeze(0)
134
+ return z
135
+
136
+ def _decode_latents(self, latents, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)):
137
+ video_tensor = self.vae.decode(latents, device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride)
138
+ video_tensor = video_tensor.squeeze(0).detach().float().clamp(-1, 1)
139
+ video_tensor = ((video_tensor + 1.0) * 127.5).to(torch.uint8).cpu()
140
+ frames = []
141
+ for t in range(video_tensor.shape[1]):
142
+ frame = video_tensor[:, t].permute(1, 2, 0).numpy()
143
+ frames.append(Image.fromarray(frame))
144
+ return frames
145
+
146
+ def _model_fn(self,
147
+ latents: torch.Tensor, # [B, C, T, H, W]
148
+ timestep: torch.Tensor, # [B] or [1] (inference mode)
149
+ context: torch.Tensor, # [B, L, D]
150
+ context_mask: Optional[torch.Tensor] = None, # [B, L]
151
+ action: Optional[torch.Tensor] = None, # [B, T-1, a_dim]
152
+ fuse_vae_embedding_in_latents=False):
153
+ return self.dit(
154
+ x=latents,
155
+ timestep=timestep,
156
+ context=context,
157
+ context_mask=context_mask,
158
+ action=action,
159
+ fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents,
160
+ )
161
+
162
+ def build_inputs(self, sample, tiled=False):
163
+ video = sample["video"]
164
+ prompt = sample["prompt"]
165
+ if not isinstance(video, torch.Tensor):
166
+ raise TypeError(
167
+ f"`sample['video']` must be a torch.Tensor with shape [B, 3, T, H, W], got {type(video)}"
168
+ )
169
+ if video.ndim != 5:
170
+ raise ValueError(f"`sample['video']` must be 5D [B, 3, T, H, W], got shape {tuple(video.shape)}")
171
+ if video.shape[1] != 3:
172
+ raise ValueError(f"`sample['video']` channel dimension must be 3, got shape {tuple(video.shape)}")
173
+
174
+ if isinstance(prompt, str):
175
+ prompt_list = [prompt]
176
+ elif isinstance(prompt, Sequence):
177
+ prompt_list = list(prompt)
178
+ else:
179
+ raise TypeError(f"`sample['prompt']` must be str or list[str], got {type(prompt)}")
180
+
181
+ batch_size, _, num_frames, height, width = video.shape
182
+ if len(prompt_list) != batch_size:
183
+ raise ValueError(
184
+ f"Prompt batch mismatch: got len(prompt)={len(prompt_list)} and video batch={batch_size}"
185
+ )
186
+ if height % 16 != 0 or width % 16 != 0:
187
+ raise ValueError(
188
+ f"Video spatial dims must be multiples of 16, got H={height}, W={width}"
189
+ )
190
+ if num_frames % 4 != 1:
191
+ raise ValueError(f"Video T must satisfy T % 4 == 1, got T={num_frames}")
192
+
193
+ input_video = video.to(device=self.device, dtype=self.torch_dtype)
194
+ input_latents = self._encode_video_latents(input_video, tiled=tiled) # [B, C, Latent_T, H', W']
195
+
196
+ first_frame_latents = None
197
+ fuse_flag = False
198
+ if getattr(self.dit, "fuse_vae_embedding_in_latents", False):
199
+ first_frame_latents = input_latents[:, :, 0:1]
200
+ fuse_flag = True
201
+ context, context_mask = self.encode_prompt(prompt_list)
202
+
203
+ action = None
204
+ if "action" in sample:
205
+ action = sample["action"]
206
+ if not isinstance(action, torch.Tensor):
207
+ raise TypeError(
208
+ f"`sample['action']` must be a torch.Tensor with shape [B, T, a_dim], got {type(action)}"
209
+ )
210
+ if action.ndim != 3:
211
+ raise ValueError(f"`sample['action']` must be 3D [B, T, a_dim], got shape {tuple(action.shape)}")
212
+ if action.shape[1] <= 0:
213
+ raise ValueError(f"`sample['action']` temporal dimension must be positive, got {action.shape[1]}")
214
+ if action.shape[1] % (num_frames - 1) != 0:
215
+ raise ValueError(
216
+ "`sample['action']` temporal dimension must be divisible by video transitions "
217
+ f"({num_frames - 1}), got {action.shape[1]}"
218
+ )
219
+ action = action.to(device=self.device, dtype=self.torch_dtype)
220
+
221
+ return {
222
+ "context": context,
223
+ "context_mask": context_mask,
224
+ "input_latents": input_latents,
225
+ "first_frame_latents": first_frame_latents,
226
+ "fuse_vae_embedding_in_latents": fuse_flag,
227
+ "action": action,
228
+ }
229
+
230
+ def training_loss(self, sample, tiled=False):
231
+ inputs = self.build_inputs(sample, tiled=tiled)
232
+ input_latents = inputs["input_latents"]
233
+ batch_size = input_latents.shape[0]
234
+ action = inputs["action"]
235
+ context = inputs["context"]
236
+ context_mask = inputs["context_mask"]
237
+
238
+ # 1. Continuous timestep sampling and noise injection.
239
+ noise = torch.randn_like(input_latents)
240
+ timestep = self.train_scheduler.sample_training_t(
241
+ batch_size=batch_size,
242
+ device=self.device,
243
+ dtype=input_latents.dtype,
244
+ )
245
+ latents = self.train_scheduler.add_noise(input_latents, noise, timestep)
246
+ target = self.train_scheduler.training_target(input_latents, noise, timestep)
247
+
248
+ # 2. fix first latent
249
+ if inputs["first_frame_latents"] is not None:
250
+ latents[:, :, 0: 1] = inputs["first_frame_latents"]
251
+
252
+ pred = self._model_fn(
253
+ latents=latents, # [B, C, Latent_T, H', W']
254
+ timestep=timestep,
255
+ context=context,
256
+ context_mask=context_mask,
257
+ action=action,
258
+ fuse_vae_embedding_in_latents=inputs["fuse_vae_embedding_in_latents"],
259
+ )
260
+ if inputs["first_frame_latents"] is not None:
261
+ pred = pred[:, :, 1:]
262
+ target = target[:, :, 1:]
263
+ loss_per_sample = F.mse_loss(pred.float(), target.float(), reduction="none").mean(dim=(1, 2, 3, 4))
264
+ sample_weight = self.train_scheduler.training_weight(timestep).to(
265
+ loss_per_sample.device, dtype=loss_per_sample.dtype
266
+ )
267
+ loss_total = (loss_per_sample * sample_weight).mean()
268
+ loss_dict = {
269
+ "loss_video": float(loss_total.detach().item()),
270
+ }
271
+ return loss_total, loss_dict
272
+
273
+ @torch.no_grad()
274
+ def infer(
275
+ self,
276
+ prompt: str,
277
+ input_image: torch.Tensor,
278
+ num_frames: int,
279
+ action: Optional[torch.Tensor] = None,
280
+ negative_prompt: Optional[str] = None,
281
+ text_cfg_scale: float = 5.0,
282
+ action_cfg_scale: float = 1.0,
283
+ num_inference_steps: int = 20,
284
+ sigma_shift: Optional[float] = None,
285
+ seed: Optional[int] = None,
286
+ rand_device: str = "cpu",
287
+ tiled: bool = False,
288
+ **kwargs
289
+ ):
290
+ self.eval()
291
+ if input_image.ndim == 3:
292
+ input_image = input_image.unsqueeze(0)
293
+ if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
294
+ raise ValueError(
295
+ f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
296
+ )
297
+ _, _, height, width = input_image.shape
298
+ checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_frames)
299
+ if (checked_h, checked_w) != (height, width):
300
+ raise ValueError(
301
+ f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})"
302
+ )
303
+ if checked_t != num_frames:
304
+ raise ValueError(
305
+ f"`num_frames` must satisfy T % 4 == 1, got {num_frames}"
306
+ )
307
+
308
+ latent_t = (num_frames - 1) // self.vae.temporal_downsample_factor + 1
309
+ latent_h = height // self.vae.upsampling_factor
310
+ latent_w = width // self.vae.upsampling_factor
311
+
312
+ generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
313
+ latents = torch.randn(
314
+ (1, self.vae.model.z_dim, latent_t, latent_h, latent_w),
315
+ generator=generator,
316
+ device=rand_device,
317
+ dtype=torch.float32,
318
+ ).to(device=self.device, dtype=self.torch_dtype)
319
+ input_image = input_image.to(device=self.device, dtype=self.torch_dtype)
320
+ if action is not None:
321
+ action = action.to(device=self.device, dtype=latents.dtype)
322
+ if action.ndim != 2:
323
+ raise ValueError(f"`action` must be 2D [T, a_dim], got shape {tuple(action.shape)}")
324
+ action = action.unsqueeze(0) # [1, T, a_dim]
325
+ if action.shape[1] % (num_frames - 1) != 0:
326
+ raise ValueError(
327
+ "`action` temporal dimension must be divisible by `num_frames - 1`, "
328
+ f"got {action.shape[1]} vs {num_frames - 1}"
329
+ )
330
+ if action_cfg_scale != 1.0 and action is None:
331
+ raise ValueError("`action_cfg_scale` != 1.0 requires non-null `action` input.")
332
+
333
+ z = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled)
334
+ latents[:, :, 0:1] = z
335
+ first_frame_latents = z
336
+ fuse_flag = True
337
+
338
+ context_posi, context_posi_mask = self.encode_prompt(prompt)
339
+ context_nega = None
340
+ context_nega_mask = None
341
+ if text_cfg_scale != 1.0:
342
+ context_nega, context_nega_mask = self.encode_prompt("" if negative_prompt is None else negative_prompt)
343
+ action_nega = torch.zeros_like(action) if (action is not None and action_cfg_scale != 1.0) else None
344
+
345
+ infer_timesteps, infer_deltas = self.infer_scheduler.build_inference_schedule(
346
+ num_inference_steps=num_inference_steps,
347
+ device=self.device,
348
+ dtype=latents.dtype,
349
+ shift_override=sigma_shift,
350
+ )
351
+ for step_t, step_delta in zip(infer_timesteps, infer_deltas):
352
+ timestep = step_t.unsqueeze(0).to(dtype=latents.dtype, device=self.device)
353
+ noise_pred_posi = self._model_fn(
354
+ latents=latents,
355
+ timestep=timestep,
356
+ context=context_posi,
357
+ context_mask=context_posi_mask,
358
+ action=action,
359
+ fuse_vae_embedding_in_latents=fuse_flag,
360
+ )
361
+ noise_pred = noise_pred_posi
362
+ if context_nega is not None:
363
+ noise_pred_text_nega = self._model_fn(
364
+ latents=latents,
365
+ timestep=timestep,
366
+ context=context_nega,
367
+ context_mask=context_nega_mask,
368
+ action=action,
369
+ fuse_vae_embedding_in_latents=fuse_flag,
370
+ )
371
+ noise_pred = noise_pred + (text_cfg_scale - 1.0) * (noise_pred_posi - noise_pred_text_nega)
372
+ if action_nega is not None:
373
+ noise_pred_action_nega = self._model_fn(
374
+ latents=latents,
375
+ timestep=timestep,
376
+ context=context_posi,
377
+ context_mask=context_posi_mask,
378
+ action=action_nega,
379
+ fuse_vae_embedding_in_latents=fuse_flag,
380
+ )
381
+ noise_pred = noise_pred + (action_cfg_scale - 1.0) * (noise_pred_posi - noise_pred_action_nega)
382
+ latents = self.infer_scheduler.step(noise_pred, step_delta, latents)
383
+ latents[:, :, 0:1] = first_frame_latents
384
+
385
+ return {"video": self._decode_latents(latents, tiled=tiled)}
386
+
387
+ def save_checkpoint(self, path, optimizer=None, step=None):
388
+ payload = {
389
+ "dit": self.dit.state_dict(),
390
+ "step": step,
391
+ "torch_dtype": str(self.torch_dtype),
392
+ }
393
+ if optimizer is not None:
394
+ payload["optimizer"] = optimizer.state_dict()
395
+ torch.save(payload, path)
396
+
397
+ def load_checkpoint(self, path, optimizer=None):
398
+ payload = torch.load(path, map_location="cpu")
399
+ self.dit.load_state_dict(payload["dit"], strict=False)
400
+ if optimizer is not None and "optimizer" in payload:
401
+ optimizer.load_state_dict(payload["optimizer"])
402
+ return payload
403
+
404
+ def forward(self, *args, **kwargs):
405
+ return self.training_loss(*args, **kwargs)
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_dit.py ADDED
@@ -0,0 +1,673 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import torch.nn.functional as F
4
+ import math
5
+ from typing import Any, Dict, Tuple, Optional
6
+ from einops import rearrange
7
+ from .helpers.gradient import gradient_checkpoint_forward
8
+
9
+ from fastwam.utils.logging_config import get_logger
10
+
11
+ logger = get_logger(__name__)
12
+
13
+
14
+ def flash_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, num_heads: int, ctx_mask: Optional[torch.Tensor] = None, compatibility_mode=True):
15
+ if compatibility_mode:
16
+ q = rearrange(q, "b s (n d) -> b n s d", n=num_heads)
17
+ k = rearrange(k, "b s (n d) -> b n s d", n=num_heads)
18
+ v = rearrange(v, "b s (n d) -> b n s d", n=num_heads)
19
+ x = F.scaled_dot_product_attention(q, k, v, attn_mask=ctx_mask)
20
+ x = rearrange(x, "b n s d -> b s (n d)", n=num_heads)
21
+ return x
22
+ else:
23
+ raise NotImplementedError("Only compatibility mode is implemented for flash attention. Please set compatibility_mode=True.")
24
+
25
+
26
+
27
+ def modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor):
28
+ return (x * (1 + scale) + shift)
29
+
30
+
31
+ def sinusoidal_embedding_1d(dim, position):
32
+ sinusoid = torch.outer(position.type(torch.float64), torch.pow(
33
+ 10000, -torch.arange(dim//2, dtype=torch.float64, device=position.device).div(dim//2)))
34
+ x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1)
35
+ return x.to(position.dtype)
36
+
37
+
38
+ def precompute_freqs_cis_3d(dim: int, end: int = 1024, theta: float = 10000.0):
39
+ # 3d rope precompute
40
+ f_freqs_cis = precompute_freqs_cis(dim - 2 * (dim // 3), end, theta)
41
+ h_freqs_cis = precompute_freqs_cis(dim // 3, end, theta)
42
+ w_freqs_cis = precompute_freqs_cis(dim // 3, end, theta)
43
+ return f_freqs_cis, h_freqs_cis, w_freqs_cis
44
+
45
+
46
+ def precompute_freqs_cis(dim: int, end: int = 1024, theta: float = 10000.0):
47
+ # 1d rope precompute
48
+ freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)
49
+ [: (dim // 2)].double() / dim))
50
+ freqs = torch.outer(torch.arange(end, device=freqs.device), freqs)
51
+ freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex dtype follows freqs
52
+ return freqs_cis
53
+
54
+
55
+ def rope_apply(x, freqs, num_heads, compute_dtype=torch.float64):
56
+ x = rearrange(x, "b s (n d) -> b s n d", n=num_heads)
57
+ if compute_dtype not in (torch.float32, torch.float64):
58
+ raise ValueError(f"RoPE compute_dtype must be float32 or float64, got {compute_dtype}")
59
+ complex_dtype = torch.complex64 if compute_dtype == torch.float32 else torch.complex128
60
+ x_out = torch.view_as_complex(x.to(compute_dtype).reshape(
61
+ x.shape[0], x.shape[1], x.shape[2], -1, 2))
62
+ if compute_dtype == torch.float64 and freqs.device.type == "npu":
63
+ # Preserve the historical NPU compatibility path.
64
+ freqs = freqs.to(torch.complex64)
65
+ else:
66
+ freqs = freqs.to(complex_dtype)
67
+ x_out = torch.view_as_real(x_out * freqs).flatten(2)
68
+ return x_out.to(x.dtype)
69
+
70
+
71
+ def create_group_causal_attn_mask(
72
+ num_temporal_groups: int, num_query_per_group: int, num_key_per_group: int, mode: str = "causal"
73
+ ) -> torch.Tensor:
74
+ """
75
+ Creates a group-based attention mask for scaled dot-product attention with two modes:
76
+ 'causal' and 'group_diagonal'.
77
+
78
+ Parameters:
79
+ - num_temporal_groups (int): The number of temporal groups (e.g., frames in a video sequence).
80
+ - num_query_per_group (int): The number of query tokens per temporal group. (e.g., latent tokens in a frame, H x W).
81
+ - num_key_per_group (int): The number of key tokens per temporal group. (e.g., action tokens per frame).
82
+ - mode (str): The mode of the attention mask. Options are:
83
+ - 'causal': Query tokens can attend to key tokens from the same or previous temporal groups.
84
+ - 'group_diagonal': Query tokens can attend only to key tokens from the same temporal group.
85
+
86
+ Returns:
87
+ - attn_mask (torch.Tensor): A boolean tensor of shape (L, S), where:
88
+ - L = num_temporal_groups * num_query_per_group (total number of query tokens)
89
+ - S = num_temporal_groups * num_key_per_group (total number of key tokens)
90
+ The mask indicates where attention is allowed (True) and disallowed (False).
91
+
92
+ Example:
93
+ Input:
94
+ num_temporal_groups = 3
95
+ num_query_per_group = 4
96
+ num_key_per_group = 2
97
+ Output:
98
+ Causal Mask Shape: torch.Size([12, 6])
99
+ Group Diagonal Mask Shape: torch.Size([12, 6])
100
+ if mode='causal':
101
+ tensor([[ True, True, False, False, False, False],
102
+ [ True, True, False, False, False, False],
103
+ [ True, True, False, False, False, False],
104
+ [ True, True, False, False, False, False],
105
+ [ True, True, True, True, False, False],
106
+ [ True, True, True, True, False, False],
107
+ [ True, True, True, True, False, False],
108
+ [ True, True, True, True, False, False],
109
+ [ True, True, True, True, True, True],
110
+ [ True, True, True, True, True, True],
111
+ [ True, True, True, True, True, True],
112
+ [ True, True, True, True, True, True]])
113
+
114
+ if mode='group_diagonal':
115
+ tensor([[ True, True, False, False, False, False],
116
+ [ True, True, False, False, False, False],
117
+ [ True, True, False, False, False, False],
118
+ [ True, True, False, False, False, False],
119
+ [False, False, True, True, False, False],
120
+ [False, False, True, True, False, False],
121
+ [False, False, True, True, False, False],
122
+ [False, False, True, True, False, False],
123
+ [False, False, False, False, True, True],
124
+ [False, False, False, False, True, True],
125
+ [False, False, False, False, True, True],
126
+ [False, False, False, False, True, True]])
127
+
128
+ """
129
+ assert mode in ["causal", "group_diagonal"], f"Mode {mode} must be 'causal' or 'group_diagonal'"
130
+
131
+ # Total number of query and key tokens
132
+ total_num_query_tokens = num_temporal_groups * num_query_per_group # Total number of query tokens (L)
133
+ total_num_key_tokens = num_temporal_groups * num_key_per_group # Total number of key tokens (S)
134
+
135
+ # Generate time indices for query and key tokens (shape: [L] and [S])
136
+ query_time_indices = torch.arange(num_temporal_groups).repeat_interleave(num_query_per_group) # Shape: [L]
137
+ key_time_indices = torch.arange(num_temporal_groups).repeat_interleave(num_key_per_group) # Shape: [S]
138
+
139
+ # Expand dimensions to compute outer comparison
140
+ query_time_indices = query_time_indices.unsqueeze(1) # Shape: [L, 1]
141
+ key_time_indices = key_time_indices.unsqueeze(0) # Shape: [1, S]
142
+
143
+ if mode == "causal":
144
+ # Causal Mode: Query can attend to keys where key_time <= query_time
145
+ attn_mask = query_time_indices >= key_time_indices # Shape: [L, S]
146
+ elif mode == "group_diagonal":
147
+ # Group Diagonal Mode: Query can attend only to keys where key_time == query_time
148
+ attn_mask = query_time_indices == key_time_indices # Shape: [L, S]
149
+
150
+ assert attn_mask.shape == (total_num_query_tokens, total_num_key_tokens), "Attention mask shape mismatch"
151
+ return attn_mask
152
+
153
+
154
+ class RMSNorm(nn.Module):
155
+ def __init__(self, dim, eps=1e-5):
156
+ super().__init__()
157
+ self.eps = eps
158
+ self.weight = nn.Parameter(torch.ones(dim))
159
+
160
+ def norm(self, x):
161
+ return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
162
+
163
+ def forward(self, x):
164
+ dtype = x.dtype
165
+ return self.norm(x.float()).to(dtype) * self.weight
166
+
167
+
168
+ class AttentionModule(nn.Module):
169
+ def __init__(self, num_heads):
170
+ super().__init__()
171
+ self.num_heads = num_heads
172
+
173
+ def forward(self, q, k, v, ctx_mask=None):
174
+ x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=ctx_mask)
175
+ return x
176
+
177
+
178
+ class SelfAttention(nn.Module):
179
+ def __init__(self, hidden_dim: int, attn_head_dim: int, num_heads: int, eps: float = 1e-6):
180
+ super().__init__()
181
+ self.hidden_dim = hidden_dim
182
+ self.num_heads = num_heads
183
+ self.attn_head_dim = attn_head_dim
184
+ self.attn_hidden_dim = self.num_heads * self.attn_head_dim
185
+
186
+ self.q = nn.Linear(hidden_dim, self.attn_hidden_dim)
187
+ self.k = nn.Linear(hidden_dim, self.attn_hidden_dim)
188
+ self.v = nn.Linear(hidden_dim, self.attn_hidden_dim)
189
+ self.o = nn.Linear(self.attn_hidden_dim, hidden_dim)
190
+ self.norm_q = RMSNorm(self.attn_hidden_dim, eps=eps)
191
+ self.norm_k = RMSNorm(self.attn_hidden_dim, eps=eps)
192
+
193
+ # self.attn = AttentionModule(self.num_heads)
194
+
195
+ def forward(self, x, freqs, self_attn_mask: Optional[torch.Tensor] = None):
196
+ q = self.norm_q(self.q(x))
197
+ k = self.norm_k(self.k(x))
198
+ v = self.v(x)
199
+ q = rope_apply(q, freqs, self.num_heads)
200
+ k = rope_apply(k, freqs, self.num_heads)
201
+ x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=self_attn_mask)
202
+ return self.o(x)
203
+
204
+
205
+ class CrossAttention(nn.Module):
206
+ def __init__(self, hidden_dim: int, attn_head_dim: int, num_heads: int, eps: float = 1e-6,):
207
+ super().__init__()
208
+ self.hidden_dim = hidden_dim
209
+ self.num_heads = num_heads
210
+ self.attn_head_dim = attn_head_dim
211
+ self.attn_hidden_dim = self.num_heads * self.attn_head_dim
212
+
213
+ self.q = nn.Linear(hidden_dim, self.attn_hidden_dim)
214
+ self.k = nn.Linear(hidden_dim, self.attn_hidden_dim)
215
+ self.v = nn.Linear(hidden_dim, self.attn_hidden_dim)
216
+ self.o = nn.Linear(self.attn_hidden_dim, hidden_dim)
217
+ self.norm_q = RMSNorm(self.attn_hidden_dim, eps=eps)
218
+ self.norm_k = RMSNorm(self.attn_hidden_dim, eps=eps)
219
+
220
+ # self.attn = AttentionModule(self.num_heads)
221
+
222
+ def forward(self, x: torch.Tensor, ctx: torch.Tensor, ctx_mask: Optional[torch.Tensor] = None):
223
+ q = self.norm_q(self.q(x))
224
+ k = self.norm_k(self.k(ctx))
225
+ v = self.v(ctx)
226
+ x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=ctx_mask)
227
+ return self.o(x)
228
+
229
+
230
+ class GateModule(nn.Module):
231
+ def __init__(self,):
232
+ super().__init__()
233
+
234
+ def forward(self, x, gate, residual):
235
+ return x + gate * residual
236
+
237
+ class DiTBlock(nn.Module):
238
+ def __init__(self, hidden_dim: int, attn_head_dim: int, num_heads: int, ffn_dim: int, eps: float = 1e-6):
239
+ super().__init__()
240
+ self.hidden_dim = hidden_dim
241
+ self.attn_head_dim = attn_head_dim
242
+ self.num_heads = num_heads
243
+ self.ffn_dim = ffn_dim
244
+
245
+ self.self_attn = SelfAttention(hidden_dim, attn_head_dim, num_heads, eps)
246
+ self.cross_attn = CrossAttention(
247
+ hidden_dim, attn_head_dim, num_heads, eps)
248
+ self.norm1 = nn.LayerNorm(hidden_dim, eps=eps, elementwise_affine=False)
249
+ self.norm2 = nn.LayerNorm(hidden_dim, eps=eps, elementwise_affine=False)
250
+ self.norm3 = nn.LayerNorm(hidden_dim, eps=eps)
251
+ self.ffn = nn.Sequential(nn.Linear(hidden_dim, ffn_dim), nn.GELU(
252
+ approximate='tanh'), nn.Linear(ffn_dim, hidden_dim))
253
+ self.modulation = nn.Parameter(torch.randn(1, 6, hidden_dim) / hidden_dim**0.5)
254
+ self.gate = GateModule()
255
+
256
+ def forward(self, x, context, t_mod, freqs, context_mask=None, self_attn_mask: Optional[torch.Tensor] = None):
257
+ if context_mask is not None and context_mask.dim() == 3:
258
+ context_mask = context_mask.unsqueeze(1) # (B, 1, seq_len, context_len), 1 for heads
259
+ has_seq = len(t_mod.shape) == 4
260
+ chunk_dim = 2 if has_seq else 1
261
+ # msa: multi-head self-attention mlp: multi-layer perceptron
262
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
263
+ self.modulation.to(dtype=t_mod.dtype, device=t_mod.device) + t_mod).chunk(6, dim=chunk_dim)
264
+ if has_seq:
265
+ # means t_mod has separate modulation for each token, otherwise same modulation for all tokens in the block
266
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
267
+ shift_msa.squeeze(2), scale_msa.squeeze(2), gate_msa.squeeze(2),
268
+ shift_mlp.squeeze(2), scale_mlp.squeeze(2), gate_mlp.squeeze(2),
269
+ )
270
+ input_x = modulate(self.norm1(x), shift_msa, scale_msa)
271
+ x = self.gate(x, gate_msa, self.self_attn(input_x, freqs, self_attn_mask=self_attn_mask))
272
+ x = x + self.cross_attn(self.norm3(x), context, ctx_mask=context_mask)
273
+ input_x = modulate(self.norm2(x), shift_mlp, scale_mlp)
274
+ x = self.gate(x, gate_mlp, self.ffn(input_x))
275
+ return x
276
+
277
+
278
+ class MLP(torch.nn.Module):
279
+ def __init__(self, in_dim, out_dim, has_pos_emb=False):
280
+ super().__init__()
281
+ self.proj = torch.nn.Sequential(
282
+ nn.LayerNorm(in_dim),
283
+ nn.Linear(in_dim, in_dim),
284
+ nn.GELU(),
285
+ nn.Linear(in_dim, out_dim),
286
+ nn.LayerNorm(out_dim)
287
+ )
288
+ self.has_pos_emb = has_pos_emb
289
+ if has_pos_emb:
290
+ self.emb_pos = torch.nn.Parameter(torch.zeros((1, 514, 1280)))
291
+
292
+ def forward(self, x):
293
+ if self.has_pos_emb:
294
+ x = x + self.emb_pos.to(dtype=x.dtype, device=x.device)
295
+ return self.proj(x)
296
+
297
+
298
+ class Head(nn.Module):
299
+ def __init__(self, dim: int, out_dim: int, patch_size: Tuple[int, int, int], eps: float):
300
+ super().__init__()
301
+ self.dim = dim
302
+ self.patch_size = patch_size
303
+ self.norm = nn.LayerNorm(dim, eps=eps, elementwise_affine=False)
304
+ self.head = nn.Linear(dim, out_dim * math.prod(patch_size))
305
+ self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5)
306
+
307
+ def forward(self, x, t_mod):
308
+ if len(t_mod.shape) == 3:
309
+ shift, scale = (self.modulation.unsqueeze(0).to(dtype=t_mod.dtype, device=t_mod.device) + t_mod.unsqueeze(2)).chunk(2, dim=2)
310
+ x = (self.head(self.norm(x) * (1 + scale.squeeze(2)) + shift.squeeze(2)))
311
+ else:
312
+ shift, scale = (self.modulation.to(dtype=t_mod.dtype, device=t_mod.device) + t_mod).chunk(2, dim=1)
313
+ x = (self.head(self.norm(x) * (1 + scale) + shift))
314
+ return x
315
+
316
+
317
+ class WanVideoDiT(torch.nn.Module):
318
+ def __init__(
319
+ self,
320
+ hidden_dim: int,
321
+ in_dim: int,
322
+ ffn_dim: int,
323
+ out_dim: int,
324
+ text_dim: int,
325
+ freq_dim: int,
326
+ eps: float,
327
+ patch_size: Tuple[int, int, int],
328
+ num_heads: int,
329
+ attn_head_dim: int,
330
+ num_layers: int,
331
+ has_image_input: bool,
332
+ has_image_pos_emb: bool = False,
333
+ has_ref_conv: bool = False,
334
+ add_control_adapter: bool = False,
335
+ in_dim_control_adapter: int = 24,
336
+ seperated_timestep: bool = False,
337
+ require_vae_embedding: bool = False,
338
+ require_clip_embedding: bool = False,
339
+ fuse_vae_embedding_in_latents: bool = True,
340
+ action_conditioned: bool = False,
341
+ action_dim: int = 7,
342
+ action_group_causal_mask_mode = "causal",
343
+ video_attention_mask_mode: str = "bidirectional",
344
+ use_gradient_checkpointing: bool = False,
345
+ ):
346
+ super().__init__()
347
+ self.hidden_dim = hidden_dim
348
+ self.in_dim = in_dim
349
+ self.freq_dim = freq_dim
350
+ self.patch_size = patch_size
351
+ self.num_heads = num_heads
352
+ self.attn_head_dim = attn_head_dim
353
+ self.seperated_timestep = seperated_timestep
354
+ self.require_vae_embedding = require_vae_embedding
355
+ self.require_clip_embedding = require_clip_embedding
356
+ self.fuse_vae_embedding_in_latents = fuse_vae_embedding_in_latents
357
+ self.video_attention_mask_mode = str(video_attention_mask_mode)
358
+
359
+ if num_heads <= 0:
360
+ raise ValueError(f"`num_heads` must be > 0, got {num_heads}")
361
+ if attn_head_dim <= 0:
362
+ raise ValueError(f"`attn_head_dim` must be > 0, got {attn_head_dim}")
363
+ if attn_head_dim % 2 != 0:
364
+ raise ValueError(
365
+ f"`attn_head_dim` must be even for RoPE, got {attn_head_dim}"
366
+ )
367
+
368
+ self.action_conditioned = action_conditioned
369
+ self.action_dim = action_dim
370
+ assert has_image_input == False
371
+ assert require_clip_embedding == False
372
+ assert require_vae_embedding == False and fuse_vae_embedding_in_latents == True, "Only support fusing vae embedding in latents"
373
+
374
+ self.patch_embedding = nn.Conv3d(
375
+ in_dim, hidden_dim, kernel_size=patch_size, stride=patch_size)
376
+ self.text_embedding = nn.Sequential(
377
+ nn.Linear(text_dim, hidden_dim),
378
+ nn.GELU(approximate='tanh'),
379
+ nn.Linear(hidden_dim, hidden_dim)
380
+ )
381
+ self.time_embedding = nn.Sequential(
382
+ nn.Linear(freq_dim, hidden_dim),
383
+ nn.SiLU(),
384
+ nn.Linear(hidden_dim, hidden_dim)
385
+ )
386
+ self.time_projection = nn.Sequential(
387
+ nn.SiLU(), nn.Linear(hidden_dim, hidden_dim * 6))
388
+ self.blocks = nn.ModuleList([
389
+ DiTBlock(hidden_dim, attn_head_dim, num_heads, ffn_dim, eps)
390
+ for _ in range(num_layers)
391
+ ])
392
+ self.head = Head(hidden_dim, out_dim, patch_size, eps)
393
+ self.freqs = precompute_freqs_cis_3d(attn_head_dim)
394
+ if has_ref_conv:
395
+ self.ref_conv = nn.Conv2d(16, hidden_dim, kernel_size=(2, 2), stride=(2, 2))
396
+ self.has_image_pos_emb = has_image_pos_emb
397
+ self.has_ref_conv = has_ref_conv
398
+ self.control_adapter = None
399
+
400
+ if self.action_conditioned:
401
+ self.action_embedding = nn.Linear(action_dim, hidden_dim)
402
+ self.action_group_causal_mask_mode = action_group_causal_mask_mode
403
+
404
+ self.use_gradient_checkpointing = use_gradient_checkpointing
405
+ if self.use_gradient_checkpointing:
406
+ logger.info("Using gradient checkpointing for DiT blocks. This will save memory but use more computation.")
407
+
408
+
409
+ def patchify(self, x: torch.Tensor, control_camera_latents_input: Optional[torch.Tensor] = None):
410
+ x = self.patch_embedding(x)
411
+ if self.control_adapter is not None and control_camera_latents_input is not None:
412
+ y_camera = self.control_adapter(control_camera_latents_input)
413
+ x = [u + v for u, v in zip(x, y_camera)]
414
+ x = x[0].unsqueeze(0)
415
+ return x
416
+
417
+ def unpatchify(self, x: torch.Tensor, grid_size: torch.Tensor):
418
+ return rearrange(
419
+ x, 'b (f h w) (x y z c) -> b c (f x) (h y) (w z)',
420
+ f=grid_size[0], h=grid_size[1], w=grid_size[2],
421
+ x=self.patch_size[0], y=self.patch_size[1], z=self.patch_size[2]
422
+ )
423
+
424
+ def _validate_forward_inputs(
425
+ self,
426
+ x: torch.Tensor,
427
+ timestep: torch.Tensor,
428
+ context: torch.Tensor,
429
+ context_mask: Optional[torch.Tensor],
430
+ action: Optional[torch.Tensor],
431
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
432
+ if x.ndim != 5:
433
+ raise ValueError(f"`latents` must be 5D [B, C, T, H, W], got shape {tuple(x.shape)}")
434
+ num_latent_frames = x.shape[2]
435
+ if context.ndim != 3:
436
+ raise ValueError(f"`context` must be 3D [B, L, D], got shape {tuple(context.shape)}")
437
+ if timestep.ndim != 1:
438
+ raise ValueError(f"`timestep` must be 1D [B] or [1], got shape {tuple(timestep.shape)}")
439
+ if self.action_conditioned:
440
+ allow_text_only_single_frame = (num_latent_frames == 1 and action is None)
441
+ if not allow_text_only_single_frame:
442
+ assert action is not None, "Action input is required for action-conditioned model."
443
+ if action.ndim != 3:
444
+ raise ValueError(f"`action` must be 3D [B, action_horizon, action_dim], got shape {tuple(action.shape)}")
445
+ if action.shape[2] != self.action_dim:
446
+ raise ValueError(f"`action` last dimension must be {self.action_dim}, got {action.shape[2]}")
447
+ if num_latent_frames <= 1:
448
+ raise ValueError(f"video length must be > 1 for action-conditioned model, got {num_latent_frames}")
449
+ if action.shape[1] % (num_latent_frames - 1) != 0:
450
+ raise ValueError(
451
+ f"action horizon must be divisible by (num_latent_frames - 1), got action_horizon={action.shape[1]}"
452
+ )
453
+ if context_mask is None:
454
+ context_mask = torch.ones((context.shape[0], context.shape[1]), dtype=torch.bool, device=context.device)
455
+ else:
456
+ if context_mask.ndim != 2:
457
+ raise ValueError(f"`context_mask` must be 2D [B, L], got shape {tuple(context_mask.shape)}")
458
+ if context_mask.shape[0] != context.shape[0] or context_mask.shape[1] != context.shape[1]:
459
+ raise ValueError(f"`context_mask` shape must match `context` shape [B, L], got {tuple(context_mask.shape)} vs {tuple(context.shape)}")
460
+
461
+ batch_size = x.shape[0]
462
+ if batch_size != context.shape[0]:
463
+ if not self.training and batch_size == 1:
464
+ x = x.expand(context.shape[0], -1, -1, -1, -1)
465
+ batch_size = context.shape[0]
466
+ else:
467
+ raise ValueError(
468
+ f"Batch mismatch between latents and context: {batch_size} vs {context.shape[0]}."
469
+ )
470
+
471
+ if timestep.shape[0] not in (1, batch_size):
472
+ raise ValueError(
473
+ f"`timestep` length must be 1 or batch_size({batch_size}), got {timestep.shape[0]}"
474
+ )
475
+ if timestep.shape[0] == 1 and batch_size > 1:
476
+ assert not self.training, "During training, timestep length must match batch_size."
477
+ timestep = timestep.expand(batch_size)
478
+ return x, timestep, context_mask
479
+
480
+ def build_video_to_video_mask(
481
+ self,
482
+ video_seq_len: int,
483
+ video_tokens_per_frame: int,
484
+ device: torch.device,
485
+ ) -> torch.Tensor:
486
+ if video_seq_len <= 0:
487
+ raise ValueError(f"`video_seq_len` must be positive, got {video_seq_len}")
488
+ if video_tokens_per_frame <= 0:
489
+ raise ValueError(f"`video_tokens_per_frame` must be positive, got {video_tokens_per_frame}")
490
+
491
+ if self.video_attention_mask_mode == "bidirectional":
492
+ return torch.ones((video_seq_len, video_seq_len), dtype=torch.bool, device=device)
493
+
494
+ if self.video_attention_mask_mode == "per_frame_causal":
495
+ if video_seq_len % video_tokens_per_frame != 0:
496
+ raise ValueError(
497
+ "`video_seq_len` must be divisible by `video_tokens_per_frame` in `per_frame_causal` mode, "
498
+ f"got {video_seq_len} and {video_tokens_per_frame}"
499
+ )
500
+ num_video_frames = video_seq_len // video_tokens_per_frame
501
+ frame_causal = torch.tril(
502
+ torch.ones((num_video_frames, num_video_frames), dtype=torch.bool, device=device)
503
+ )
504
+ return frame_causal.repeat_interleave(video_tokens_per_frame, dim=0).repeat_interleave(
505
+ video_tokens_per_frame, dim=1
506
+ )
507
+
508
+ if self.video_attention_mask_mode == "first_frame_causal":
509
+ video_mask = torch.ones((video_seq_len, video_seq_len), dtype=torch.bool, device=device)
510
+ first_frame_tokens = min(video_tokens_per_frame, video_seq_len)
511
+ video_mask[:first_frame_tokens, first_frame_tokens:] = False
512
+ return video_mask
513
+
514
+ raise ValueError(f"Unsupported video attention mask mode: {self.video_attention_mask_mode}")
515
+
516
+ def pre_dit(
517
+ self,
518
+ x: torch.Tensor,
519
+ timestep: torch.Tensor,
520
+ context: torch.Tensor,
521
+ context_mask: Optional[torch.Tensor] = None,
522
+ action: Optional[torch.Tensor] = None,
523
+ fuse_vae_embedding_in_latents: bool = False,
524
+ control_camera_latents_input: Optional[torch.Tensor] = None,
525
+ ) -> Dict[str, Any]:
526
+ x, timestep, context_mask = self._validate_forward_inputs(
527
+ x=x,
528
+ timestep=timestep,
529
+ context=context,
530
+ context_mask=context_mask,
531
+ action=action,
532
+ )
533
+
534
+ batch_size = x.shape[0]
535
+ patch_h = int(self.patch_size[1])
536
+ patch_w = int(self.patch_size[2])
537
+ if x.shape[3] % patch_h != 0 or x.shape[4] % patch_w != 0:
538
+ raise ValueError(
539
+ "Latent spatial shape must be divisible by DiT patch size, "
540
+ f"got HxW=({x.shape[3]}, {x.shape[4]}), patch=({patch_h}, {patch_w})"
541
+ )
542
+ tokens_per_frame = (x.shape[3] // patch_h) * (x.shape[4] // patch_w)
543
+
544
+ if self.seperated_timestep and fuse_vae_embedding_in_latents:
545
+ if not hasattr(self, "patch_size") or len(self.patch_size) < 3:
546
+ raise ValueError(f"Invalid dit.patch_size: {getattr(self, 'patch_size', None)}")
547
+
548
+ token_timesteps = torch.ones(
549
+ (batch_size, x.shape[2], tokens_per_frame),
550
+ dtype=timestep.dtype,
551
+ device=timestep.device,
552
+ ) * timestep.view(batch_size, 1, 1)
553
+ token_timesteps[:, 0, :] = 0
554
+ token_timesteps = token_timesteps.reshape(batch_size, -1)
555
+ token_t_emb = sinusoidal_embedding_1d(self.freq_dim, token_timesteps.reshape(-1))
556
+ t = self.time_embedding(token_t_emb).reshape(batch_size, -1, self.hidden_dim)
557
+ t_mod = self.time_projection(t).unflatten(2, (6, self.hidden_dim))
558
+ else:
559
+ raise NotImplementedError("Only support seperated_timestep with fuse_vae_embedding_in_latents for now.")
560
+ t = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timestep))
561
+ t_mod = self.time_projection(t).unflatten(1, (6, self.hidden_dim))
562
+ x = self.patchify(x, control_camera_latents_input=control_camera_latents_input)
563
+ f, h, w = x.shape[2:]
564
+
565
+ context = self.text_embedding(context) # (B, L, dim)
566
+ context_len = context.shape[1]
567
+ if self.action_conditioned and action is not None:
568
+ action_len = action.shape[1]
569
+ action_emb = self.action_embedding(action) # (B, action_len, dim)
570
+ action_pos_embed = sinusoidal_embedding_1d(self.hidden_dim,
571
+ torch.arange(action_len, device=action_emb.device)) # (action_len, dim)
572
+ action_emb = action_emb + action_pos_embed.unsqueeze(0) # (B, action_len, dim)
573
+ context = torch.cat([context, action_emb], dim=1) # (B, context_len + action_len, dim)
574
+
575
+ # new mask
576
+ num_temporal_groups = f - 1 # first latent frame do not attend to actions
577
+ if num_temporal_groups <= 0:
578
+ raise ValueError(
579
+ "Action-conditioned context mask requires at least 2 latent frames when `action` is provided."
580
+ )
581
+ assert action_emb.shape[1] % num_temporal_groups == 0, \
582
+ f"Action embedding length {action_emb.shape[1]} must be divisible by number of temporal groups {num_temporal_groups}"
583
+ # Each latent frame (from the 2nd one) attends to the corresponding group of action tokens
584
+ action_group_mask = create_group_causal_attn_mask(
585
+ num_temporal_groups=num_temporal_groups,
586
+ num_query_per_group=tokens_per_frame,
587
+ num_key_per_group=action_len // num_temporal_groups,
588
+ mode=self.action_group_causal_mask_mode,
589
+ ).to(context.device) # ((f-1)*tokens_per_frame, action_len)
590
+
591
+ seq_len = f * h * w # query length
592
+ final_context_mask = torch.zeros((batch_size, seq_len, context.shape[1]), dtype=torch.bool, device=context.device) # (B, seq_len, L + action_len)
593
+ # all latent frames attend to text tokens
594
+ final_context_mask[:, :, :context_len] = context_mask.unsqueeze(1).expand(-1, seq_len, -1) # (B, seq_len, L)
595
+ # latent frames from the 2nd one attend to action tokens
596
+ final_context_mask[:, tokens_per_frame:, context_len:] = action_group_mask.unsqueeze(0).expand(batch_size, -1, -1) # (B, seq_len, action_len)
597
+ context_mask = final_context_mask
598
+ elif self.action_conditioned and action is None:
599
+ if f != 1:
600
+ raise ValueError(
601
+ "Action-conditioned model requires `action` unless running single-frame text-only mode with num_latent_frames=1."
602
+ )
603
+ context_mask = context_mask.unsqueeze(1).expand(-1, f * h * w, -1) # (B, seq_len, L)
604
+ else:
605
+ context_mask = context_mask.unsqueeze(1).expand(-1, f * h * w, -1) # (B, seq_len, L)
606
+
607
+ x_tokens = rearrange(x, "b c f h w -> b (f h w) c").contiguous()
608
+
609
+ freqs = torch.cat([
610
+ self.freqs[0][:f].view(f, 1, 1, -1).expand(f, h, w, -1),
611
+ self.freqs[1][:h].view(1, h, 1, -1).expand(f, h, w, -1),
612
+ self.freqs[2][:w].view(1, 1, w, -1).expand(f, h, w, -1)
613
+ ], dim=-1).reshape(f * h * w, 1, -1).to(x_tokens.device)
614
+
615
+ return {
616
+ "tokens": x_tokens,
617
+ "freqs": freqs,
618
+ "t": t,
619
+ "t_mod": t_mod,
620
+ "context": context,
621
+ "context_mask": context_mask,
622
+ "meta": {
623
+ "grid_size": (f, h, w),
624
+ "tokens_per_frame": tokens_per_frame,
625
+ "batch_size": batch_size,
626
+ },
627
+ }
628
+
629
+ def post_dit(self, x_tokens: torch.Tensor, pre_state: Dict[str, Any]) -> torch.Tensor:
630
+ f, h, w = pre_state["meta"]["grid_size"]
631
+ x = self.head(x_tokens, pre_state["t"])
632
+ x = self.unpatchify(x, (f, h, w))
633
+ return x
634
+
635
+ def forward(
636
+ self,
637
+ x: torch.Tensor,
638
+ timestep: torch.Tensor,
639
+ context: torch.Tensor,
640
+ context_mask: Optional[torch.Tensor] = None,
641
+ action: Optional[torch.Tensor] = None,
642
+ fuse_vae_embedding_in_latents: bool = False,
643
+ ):
644
+ pre_state = self.pre_dit(
645
+ x=x,
646
+ timestep=timestep,
647
+ context=context,
648
+ context_mask=context_mask,
649
+ action=action,
650
+ fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents,
651
+ )
652
+ x_tokens = pre_state["tokens"]
653
+ context_emb = pre_state["context"]
654
+ t_mod = pre_state["t_mod"]
655
+ freqs = pre_state["freqs"]
656
+ context_attn_mask = pre_state["context_mask"]
657
+ self_attn_mask = self.build_video_to_video_mask(
658
+ video_seq_len=x_tokens.shape[1],
659
+ video_tokens_per_frame=int(pre_state["meta"]["tokens_per_frame"]),
660
+ device=x_tokens.device,
661
+ ) if self.video_attention_mask_mode != "bidirectional" else None # special rule for faster speed
662
+
663
+ for block in self.blocks:
664
+ if self.use_gradient_checkpointing:
665
+ x_tokens = gradient_checkpoint_forward(
666
+ block,
667
+ self.use_gradient_checkpointing,
668
+ x_tokens, context_emb, t_mod, freqs, context_mask=context_attn_mask, self_attn_mask=self_attn_mask
669
+ )
670
+ else:
671
+ x_tokens = block(x_tokens, context_emb, t_mod, freqs, context_mask=context_attn_mask, self_attn_mask=self_attn_mask)
672
+
673
+ return self.post_dit(x_tokens, pre_state)
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_text_encoder.py ADDED
@@ -0,0 +1,342 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import torch
3
+ import torch.nn as nn
4
+ import torch.nn.functional as F
5
+ from transformers import AutoTokenizer
6
+ import html
7
+ import string
8
+ try:
9
+ import ftfy
10
+ except ModuleNotFoundError:
11
+ class _FTFY:
12
+ @staticmethod
13
+ def fix_text(text):
14
+ return text
15
+
16
+ ftfy = _FTFY()
17
+ try:
18
+ import regex as re
19
+ except ModuleNotFoundError:
20
+ import re
21
+
22
+ def fp16_clamp(x):
23
+ if x.dtype == torch.float16 and torch.isinf(x).any():
24
+ clamp = torch.finfo(x.dtype).max - 1000
25
+ x = torch.clamp(x, min=-clamp, max=clamp)
26
+ return x
27
+
28
+
29
+ class GELU(nn.Module):
30
+
31
+ def forward(self, x):
32
+ return 0.5 * x * (1.0 + torch.tanh(
33
+ math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0))))
34
+
35
+
36
+ class T5LayerNorm(nn.Module):
37
+
38
+ def __init__(self, dim, eps=1e-6):
39
+ super(T5LayerNorm, self).__init__()
40
+ self.dim = dim
41
+ self.eps = eps
42
+ self.weight = nn.Parameter(torch.ones(dim))
43
+
44
+ def forward(self, x):
45
+ x = x * torch.rsqrt(x.float().pow(2).mean(dim=-1, keepdim=True) +
46
+ self.eps)
47
+ if self.weight.dtype in [torch.float16, torch.bfloat16]:
48
+ x = x.type_as(self.weight)
49
+ return self.weight * x
50
+
51
+
52
+ class T5Attention(nn.Module):
53
+
54
+ def __init__(self, dim, dim_attn, num_heads, dropout=0.1):
55
+ assert dim_attn % num_heads == 0
56
+ super(T5Attention, self).__init__()
57
+ self.dim = dim
58
+ self.dim_attn = dim_attn
59
+ self.num_heads = num_heads
60
+ self.head_dim = dim_attn // num_heads
61
+
62
+ # layers
63
+ self.q = nn.Linear(dim, dim_attn, bias=False)
64
+ self.k = nn.Linear(dim, dim_attn, bias=False)
65
+ self.v = nn.Linear(dim, dim_attn, bias=False)
66
+ self.o = nn.Linear(dim_attn, dim, bias=False)
67
+ self.dropout = nn.Dropout(dropout)
68
+
69
+ def forward(self, x, context=None, mask=None, pos_bias=None):
70
+ """
71
+ x: [B, L1, C].
72
+ context: [B, L2, C] or None.
73
+ mask: [B, L2] or [B, L1, L2] or None.
74
+ """
75
+ # check inputs
76
+ context = x if context is None else context
77
+ b, n, c = x.size(0), self.num_heads, self.head_dim
78
+
79
+ # compute query, key, value
80
+ q = self.q(x).view(b, -1, n, c)
81
+ k = self.k(context).view(b, -1, n, c)
82
+ v = self.v(context).view(b, -1, n, c)
83
+
84
+ # attention bias
85
+ attn_bias = x.new_zeros(b, n, q.size(1), k.size(1))
86
+ if pos_bias is not None:
87
+ attn_bias += pos_bias
88
+ if mask is not None:
89
+ assert mask.ndim in [2, 3]
90
+ mask = mask.view(b, 1, 1,
91
+ -1) if mask.ndim == 2 else mask.unsqueeze(1)
92
+ attn_bias.masked_fill_(mask == 0, torch.finfo(x.dtype).min)
93
+
94
+ # compute attention (T5 does not use scaling)
95
+ attn = torch.einsum('binc,bjnc->bnij', q, k) + attn_bias
96
+ attn = F.softmax(attn.float(), dim=-1).type_as(attn)
97
+ x = torch.einsum('bnij,bjnc->binc', attn, v)
98
+
99
+ # output
100
+ x = x.reshape(b, -1, n * c)
101
+ x = self.o(x)
102
+ x = self.dropout(x)
103
+ return x
104
+
105
+
106
+ class T5FeedForward(nn.Module):
107
+
108
+ def __init__(self, dim, dim_ffn, dropout=0.1):
109
+ super(T5FeedForward, self).__init__()
110
+ self.dim = dim
111
+ self.dim_ffn = dim_ffn
112
+
113
+ # layers
114
+ self.gate = nn.Sequential(nn.Linear(dim, dim_ffn, bias=False), GELU())
115
+ self.fc1 = nn.Linear(dim, dim_ffn, bias=False)
116
+ self.fc2 = nn.Linear(dim_ffn, dim, bias=False)
117
+ self.dropout = nn.Dropout(dropout)
118
+
119
+ def forward(self, x):
120
+ x = self.fc1(x) * self.gate(x)
121
+ x = self.dropout(x)
122
+ x = self.fc2(x)
123
+ x = self.dropout(x)
124
+ return x
125
+
126
+
127
+ class T5SelfAttention(nn.Module):
128
+
129
+ def __init__(self,
130
+ dim,
131
+ dim_attn,
132
+ dim_ffn,
133
+ num_heads,
134
+ num_buckets,
135
+ shared_pos=True,
136
+ dropout=0.1):
137
+ super(T5SelfAttention, self).__init__()
138
+ self.dim = dim
139
+ self.dim_attn = dim_attn
140
+ self.dim_ffn = dim_ffn
141
+ self.num_heads = num_heads
142
+ self.num_buckets = num_buckets
143
+ self.shared_pos = shared_pos
144
+
145
+ # layers
146
+ self.norm1 = T5LayerNorm(dim)
147
+ self.attn = T5Attention(dim, dim_attn, num_heads, dropout)
148
+ self.norm2 = T5LayerNorm(dim)
149
+ self.ffn = T5FeedForward(dim, dim_ffn, dropout)
150
+ self.pos_embedding = None if shared_pos else T5RelativeEmbedding(
151
+ num_buckets, num_heads, bidirectional=True)
152
+
153
+ def forward(self, x, mask=None, pos_bias=None):
154
+ e = pos_bias if self.shared_pos else self.pos_embedding(
155
+ x.size(1), x.size(1))
156
+ x = fp16_clamp(x + self.attn(self.norm1(x), mask=mask, pos_bias=e))
157
+ x = fp16_clamp(x + self.ffn(self.norm2(x)))
158
+ return x
159
+
160
+
161
+ class T5RelativeEmbedding(nn.Module):
162
+
163
+ def __init__(self, num_buckets, num_heads, bidirectional, max_dist=128):
164
+ super(T5RelativeEmbedding, self).__init__()
165
+ self.num_buckets = num_buckets
166
+ self.num_heads = num_heads
167
+ self.bidirectional = bidirectional
168
+ self.max_dist = max_dist
169
+
170
+ # layers
171
+ self.embedding = nn.Embedding(num_buckets, num_heads)
172
+
173
+ def forward(self, lq, lk):
174
+ device = self.embedding.weight.device
175
+ # rel_pos = torch.arange(lk).unsqueeze(0).to(device) - \
176
+ # torch.arange(lq).unsqueeze(1).to(device)
177
+ rel_pos = torch.arange(lk, device=device).unsqueeze(0) - \
178
+ torch.arange(lq, device=device).unsqueeze(1)
179
+ rel_pos = self._relative_position_bucket(rel_pos)
180
+ rel_pos_embeds = self.embedding(rel_pos)
181
+ rel_pos_embeds = rel_pos_embeds.permute(2, 0, 1).unsqueeze(
182
+ 0) # [1, N, Lq, Lk]
183
+ return rel_pos_embeds.contiguous()
184
+
185
+ def _relative_position_bucket(self, rel_pos):
186
+ # preprocess
187
+ if self.bidirectional:
188
+ num_buckets = self.num_buckets // 2
189
+ rel_buckets = (rel_pos > 0).long() * num_buckets
190
+ rel_pos = torch.abs(rel_pos)
191
+ else:
192
+ num_buckets = self.num_buckets
193
+ rel_buckets = 0
194
+ rel_pos = -torch.min(rel_pos, torch.zeros_like(rel_pos))
195
+
196
+ # embeddings for small and large positions
197
+ max_exact = num_buckets // 2
198
+ rel_pos_large = max_exact + (torch.log(rel_pos.float() / max_exact) /
199
+ math.log(self.max_dist / max_exact) *
200
+ (num_buckets - max_exact)).long()
201
+ rel_pos_large = torch.min(
202
+ rel_pos_large, torch.full_like(rel_pos_large, num_buckets - 1))
203
+ rel_buckets += torch.where(rel_pos < max_exact, rel_pos, rel_pos_large)
204
+ return rel_buckets
205
+
206
+ def init_weights(m):
207
+ if isinstance(m, T5LayerNorm):
208
+ nn.init.ones_(m.weight)
209
+ elif isinstance(m, T5FeedForward):
210
+ nn.init.normal_(m.gate[0].weight, std=m.dim**-0.5)
211
+ nn.init.normal_(m.fc1.weight, std=m.dim**-0.5)
212
+ nn.init.normal_(m.fc2.weight, std=m.dim_ffn**-0.5)
213
+ elif isinstance(m, T5Attention):
214
+ nn.init.normal_(m.q.weight, std=(m.dim * m.dim_attn)**-0.5)
215
+ nn.init.normal_(m.k.weight, std=m.dim**-0.5)
216
+ nn.init.normal_(m.v.weight, std=m.dim**-0.5)
217
+ nn.init.normal_(m.o.weight, std=(m.num_heads * m.dim_attn)**-0.5)
218
+ elif isinstance(m, T5RelativeEmbedding):
219
+ nn.init.normal_(
220
+ m.embedding.weight, std=(2 * m.num_buckets * m.num_heads)**-0.5)
221
+
222
+
223
+ class WanTextEncoder(torch.nn.Module):
224
+
225
+ def __init__(self,
226
+ vocab=256384,
227
+ dim=4096,
228
+ dim_attn=4096,
229
+ dim_ffn=10240,
230
+ num_heads=64,
231
+ num_layers=24,
232
+ num_buckets=32,
233
+ shared_pos=False,
234
+ dropout=0.1):
235
+ super(WanTextEncoder, self).__init__()
236
+ self.dim = dim
237
+ self.dim_attn = dim_attn
238
+ self.dim_ffn = dim_ffn
239
+ self.num_heads = num_heads
240
+ self.num_layers = num_layers
241
+ self.num_buckets = num_buckets
242
+ self.shared_pos = shared_pos
243
+
244
+ # layers
245
+ self.token_embedding = vocab if isinstance(vocab, nn.Embedding) \
246
+ else nn.Embedding(vocab, dim)
247
+ self.pos_embedding = T5RelativeEmbedding(
248
+ num_buckets, num_heads, bidirectional=True) if shared_pos else None
249
+ self.dropout = nn.Dropout(dropout)
250
+ self.blocks = nn.ModuleList([
251
+ T5SelfAttention(dim, dim_attn, dim_ffn, num_heads, num_buckets,
252
+ shared_pos, dropout) for _ in range(num_layers)
253
+ ])
254
+ self.norm = T5LayerNorm(dim)
255
+
256
+ # Skip costly random init when loading pretrained checkpoints.
257
+ # Verified: checkpoint fully covers this module (missing/unexpected keys are both 0).
258
+ # self.apply(init_weights)
259
+
260
+ def forward(self, ids, mask=None):
261
+ x = self.token_embedding(ids)
262
+ x = self.dropout(x)
263
+ e = self.pos_embedding(x.size(1),
264
+ x.size(1)) if self.shared_pos else None
265
+ for block in self.blocks:
266
+ x = block(x, mask, pos_bias=e)
267
+ x = self.norm(x)
268
+ x = self.dropout(x)
269
+ return x
270
+
271
+
272
+ def basic_clean(text):
273
+ text = ftfy.fix_text(text)
274
+ text = html.unescape(html.unescape(text))
275
+ return text.strip()
276
+
277
+
278
+ def whitespace_clean(text):
279
+ text = re.sub(r'\s+', ' ', text)
280
+ text = text.strip()
281
+ return text
282
+
283
+
284
+ def canonicalize(text, keep_punctuation_exact_string=None):
285
+ text = text.replace('_', ' ')
286
+ if keep_punctuation_exact_string:
287
+ text = keep_punctuation_exact_string.join(
288
+ part.translate(str.maketrans('', '', string.punctuation))
289
+ for part in text.split(keep_punctuation_exact_string))
290
+ else:
291
+ text = text.translate(str.maketrans('', '', string.punctuation))
292
+ text = text.lower()
293
+ text = re.sub(r'\s+', ' ', text)
294
+ return text.strip()
295
+
296
+
297
+ class HuggingfaceTokenizer:
298
+
299
+ def __init__(self, name, seq_len=None, clean=None, **kwargs):
300
+ assert clean in (None, 'whitespace', 'lower', 'canonicalize')
301
+ self.name = name
302
+ self.seq_len = seq_len
303
+ self.clean = clean
304
+
305
+ # init tokenizer
306
+ self.tokenizer = AutoTokenizer.from_pretrained(name, **kwargs)
307
+ self.vocab_size = self.tokenizer.vocab_size
308
+
309
+ def __call__(self, sequence, **kwargs):
310
+ return_mask = kwargs.pop('return_mask', False)
311
+
312
+ # arguments
313
+ _kwargs = {'return_tensors': 'pt'}
314
+ if self.seq_len is not None:
315
+ _kwargs.update({
316
+ 'padding': 'max_length',
317
+ 'truncation': True,
318
+ 'max_length': self.seq_len
319
+ })
320
+ _kwargs.update(**kwargs)
321
+
322
+ # tokenization
323
+ if isinstance(sequence, str):
324
+ sequence = [sequence]
325
+ if self.clean:
326
+ sequence = [self._clean(u) for u in sequence]
327
+ ids = self.tokenizer(sequence, **_kwargs)
328
+
329
+ # output
330
+ if return_mask:
331
+ return ids.input_ids, ids.attention_mask
332
+ else:
333
+ return ids.input_ids
334
+
335
+ def _clean(self, text):
336
+ if self.clean == 'whitespace':
337
+ text = whitespace_clean(basic_clean(text))
338
+ elif self.clean == 'lower':
339
+ text = whitespace_clean(basic_clean(text)).lower()
340
+ elif self.clean == 'canonicalize':
341
+ text = canonicalize(basic_clean(text))
342
+ return text
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_vae.py ADDED
@@ -0,0 +1,1384 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from einops import rearrange, repeat
2
+
3
+ import torch
4
+ import torch.nn as nn
5
+ import torch.nn.functional as F
6
+ from tqdm import tqdm
7
+
8
+ CACHE_T = 2
9
+
10
+
11
+ def check_is_instance(model, module_class):
12
+ if isinstance(model, module_class):
13
+ return True
14
+ if hasattr(model, "module") and isinstance(model.module, module_class):
15
+ return True
16
+ return False
17
+
18
+
19
+ def block_causal_mask(x, block_size):
20
+ # params
21
+ b, n, s, _, device = *x.size(), x.device
22
+ assert s % block_size == 0
23
+ num_blocks = s // block_size
24
+
25
+ # build mask
26
+ mask = torch.zeros(b, n, s, s, dtype=torch.bool, device=device)
27
+ for i in range(num_blocks):
28
+ mask[:, :,
29
+ i * block_size:(i + 1) * block_size, :(i + 1) * block_size] = 1
30
+ return mask
31
+
32
+
33
+ class CausalConv3d(nn.Conv3d):
34
+ """
35
+ Causal 3d convolusion.
36
+ """
37
+
38
+ def __init__(self, *args, **kwargs):
39
+ super().__init__(*args, **kwargs)
40
+ self._padding = (self.padding[2], self.padding[2], self.padding[1],
41
+ self.padding[1], 2 * self.padding[0], 0)
42
+ self.padding = (0, 0, 0)
43
+
44
+ def forward(self, x, cache_x=None):
45
+ padding = list(self._padding)
46
+ if cache_x is not None and self._padding[4] > 0:
47
+ cache_x = cache_x.to(x.device)
48
+ x = torch.cat([cache_x, x], dim=2)
49
+ padding[4] -= cache_x.shape[2]
50
+ x = F.pad(x, padding)
51
+
52
+ return super().forward(x)
53
+
54
+
55
+ class RMS_norm(nn.Module):
56
+
57
+ def __init__(self, dim, channel_first=True, images=True, bias=False):
58
+ super().__init__()
59
+ broadcastable_dims = (1, 1, 1) if not images else (1, 1)
60
+ shape = (dim, *broadcastable_dims) if channel_first else (dim,)
61
+
62
+ self.channel_first = channel_first
63
+ self.scale = dim**0.5
64
+ self.gamma = nn.Parameter(torch.ones(shape))
65
+ self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0.
66
+
67
+ def forward(self, x):
68
+ return F.normalize(
69
+ x, dim=(1 if self.channel_first else
70
+ -1)) * self.scale * self.gamma + self.bias
71
+
72
+
73
+ class Upsample(nn.Upsample):
74
+
75
+ def forward(self, x):
76
+ """
77
+ Fix bfloat16 support for nearest neighbor interpolation.
78
+ """
79
+ return super().forward(x.float()).type_as(x)
80
+
81
+
82
+ class Resample(nn.Module):
83
+
84
+ def __init__(self, dim, mode):
85
+ assert mode in ('none', 'upsample2d', 'upsample3d', 'downsample2d',
86
+ 'downsample3d')
87
+ super().__init__()
88
+ self.dim = dim
89
+ self.mode = mode
90
+
91
+ # layers
92
+ if mode == 'upsample2d':
93
+ self.resample = nn.Sequential(
94
+ Upsample(scale_factor=(2., 2.), mode='nearest-exact'),
95
+ nn.Conv2d(dim, dim // 2, 3, padding=1))
96
+ elif mode == 'upsample3d':
97
+ self.resample = nn.Sequential(
98
+ Upsample(scale_factor=(2., 2.), mode='nearest-exact'),
99
+ nn.Conv2d(dim, dim // 2, 3, padding=1))
100
+ self.time_conv = CausalConv3d(dim,
101
+ dim * 2, (3, 1, 1),
102
+ padding=(1, 0, 0))
103
+
104
+ elif mode == 'downsample2d':
105
+ self.resample = nn.Sequential(
106
+ nn.ZeroPad2d((0, 1, 0, 1)),
107
+ nn.Conv2d(dim, dim, 3, stride=(2, 2)))
108
+ elif mode == 'downsample3d':
109
+ self.resample = nn.Sequential(
110
+ nn.ZeroPad2d((0, 1, 0, 1)),
111
+ nn.Conv2d(dim, dim, 3, stride=(2, 2)))
112
+ self.time_conv = CausalConv3d(dim,
113
+ dim, (3, 1, 1),
114
+ stride=(2, 1, 1),
115
+ padding=(0, 0, 0))
116
+
117
+ else:
118
+ self.resample = nn.Identity()
119
+
120
+ def forward(self, x, feat_cache=None, feat_idx=[0]):
121
+ b, c, t, h, w = x.size()
122
+ if self.mode == 'upsample3d':
123
+ if feat_cache is not None:
124
+ idx = feat_idx[0]
125
+ if feat_cache[idx] is None:
126
+ feat_cache[idx] = 'Rep'
127
+ feat_idx[0] += 1
128
+ else:
129
+
130
+ cache_x = x[:, :, -CACHE_T:, :, :].clone()
131
+ if cache_x.shape[2] < 2 and feat_cache[
132
+ idx] is not None and feat_cache[idx] != 'Rep':
133
+ # cache last frame of last two chunk
134
+ cache_x = torch.cat([
135
+ feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(
136
+ cache_x.device), cache_x
137
+ ],
138
+ dim=2)
139
+ if cache_x.shape[2] < 2 and feat_cache[
140
+ idx] is not None and feat_cache[idx] == 'Rep':
141
+ cache_x = torch.cat([
142
+ torch.zeros_like(cache_x).to(cache_x.device),
143
+ cache_x
144
+ ],
145
+ dim=2)
146
+ if feat_cache[idx] == 'Rep':
147
+ x = self.time_conv(x)
148
+ else:
149
+ x = self.time_conv(x, feat_cache[idx])
150
+ feat_cache[idx] = cache_x
151
+ feat_idx[0] += 1
152
+
153
+ x = x.reshape(b, 2, c, t, h, w)
154
+ x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]),
155
+ 3)
156
+ x = x.reshape(b, c, t * 2, h, w)
157
+ t = x.shape[2]
158
+ x = rearrange(x, 'b c t h w -> (b t) c h w')
159
+ x = self.resample(x)
160
+ x = rearrange(x, '(b t) c h w -> b c t h w', t=t)
161
+
162
+ if self.mode == 'downsample3d':
163
+ if feat_cache is not None:
164
+ idx = feat_idx[0]
165
+ if feat_cache[idx] is None:
166
+ feat_cache[idx] = x.clone()
167
+ feat_idx[0] += 1
168
+ else:
169
+ cache_x = x[:, :, -1:, :, :].clone()
170
+ x = self.time_conv(
171
+ torch.cat([feat_cache[idx][:, :, -1:, :, :], x], 2))
172
+ feat_cache[idx] = cache_x
173
+ feat_idx[0] += 1
174
+ return x, feat_cache, feat_idx
175
+
176
+ def init_weight(self, conv):
177
+ conv_weight = conv.weight
178
+ nn.init.zeros_(conv_weight)
179
+ c1, c2, t, h, w = conv_weight.size()
180
+ one_matrix = torch.eye(c1, c2)
181
+ init_matrix = one_matrix
182
+ nn.init.zeros_(conv_weight)
183
+ conv_weight.data[:, :, 1, 0, 0] = init_matrix
184
+ conv.weight.data.copy_(conv_weight)
185
+ nn.init.zeros_(conv.bias.data)
186
+
187
+ def init_weight2(self, conv):
188
+ conv_weight = conv.weight.data
189
+ nn.init.zeros_(conv_weight)
190
+ c1, c2, t, h, w = conv_weight.size()
191
+ init_matrix = torch.eye(c1 // 2, c2)
192
+ conv_weight[:c1 // 2, :, -1, 0, 0] = init_matrix
193
+ conv_weight[c1 // 2:, :, -1, 0, 0] = init_matrix
194
+ conv.weight.data.copy_(conv_weight)
195
+ nn.init.zeros_(conv.bias.data)
196
+
197
+
198
+
199
+ def patchify(x, patch_size):
200
+ if patch_size == 1:
201
+ return x
202
+ if x.dim() == 4:
203
+ x = rearrange(x, "b c (h q) (w r) -> b (c r q) h w", q=patch_size, r=patch_size)
204
+ elif x.dim() == 5:
205
+ x = rearrange(x,
206
+ "b c f (h q) (w r) -> b (c r q) f h w",
207
+ q=patch_size,
208
+ r=patch_size)
209
+ else:
210
+ raise ValueError(f"Invalid input shape: {x.shape}")
211
+ return x
212
+
213
+
214
+ def unpatchify(x, patch_size):
215
+ if patch_size == 1:
216
+ return x
217
+ if x.dim() == 4:
218
+ x = rearrange(x, "b (c r q) h w -> b c (h q) (w r)", q=patch_size, r=patch_size)
219
+ elif x.dim() == 5:
220
+ x = rearrange(x,
221
+ "b (c r q) f h w -> b c f (h q) (w r)",
222
+ q=patch_size,
223
+ r=patch_size)
224
+ return x
225
+
226
+
227
+ class Resample38(Resample):
228
+
229
+ def __init__(self, dim, mode):
230
+ assert mode in (
231
+ "none",
232
+ "upsample2d",
233
+ "upsample3d",
234
+ "downsample2d",
235
+ "downsample3d",
236
+ )
237
+ super(Resample, self).__init__()
238
+ self.dim = dim
239
+ self.mode = mode
240
+
241
+ # layers
242
+ if mode == "upsample2d":
243
+ self.resample = nn.Sequential(
244
+ Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact"),
245
+ nn.Conv2d(dim, dim, 3, padding=1),
246
+ )
247
+ elif mode == "upsample3d":
248
+ self.resample = nn.Sequential(
249
+ Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact"),
250
+ nn.Conv2d(dim, dim, 3, padding=1),
251
+ )
252
+ self.time_conv = CausalConv3d(dim, dim * 2, (3, 1, 1), padding=(1, 0, 0))
253
+ elif mode == "downsample2d":
254
+ self.resample = nn.Sequential(
255
+ nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2))
256
+ )
257
+ elif mode == "downsample3d":
258
+ self.resample = nn.Sequential(
259
+ nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2))
260
+ )
261
+ self.time_conv = CausalConv3d(
262
+ dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0)
263
+ )
264
+ else:
265
+ self.resample = nn.Identity()
266
+
267
+ class ResidualBlock(nn.Module):
268
+
269
+ def __init__(self, in_dim, out_dim, dropout=0.0):
270
+ super().__init__()
271
+ self.in_dim = in_dim
272
+ self.out_dim = out_dim
273
+
274
+ # layers
275
+ self.residual = nn.Sequential(
276
+ RMS_norm(in_dim, images=False), nn.SiLU(),
277
+ CausalConv3d(in_dim, out_dim, 3, padding=1),
278
+ RMS_norm(out_dim, images=False), nn.SiLU(), nn.Dropout(dropout),
279
+ CausalConv3d(out_dim, out_dim, 3, padding=1))
280
+ self.shortcut = CausalConv3d(in_dim, out_dim, 1) \
281
+ if in_dim != out_dim else nn.Identity()
282
+
283
+ def forward(self, x, feat_cache=None, feat_idx=[0]):
284
+ h = self.shortcut(x)
285
+ for layer in self.residual:
286
+ if check_is_instance(layer, CausalConv3d) and feat_cache is not None:
287
+ idx = feat_idx[0]
288
+ cache_x = x[:, :, -CACHE_T:, :, :].clone()
289
+ if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
290
+ # cache last frame of last two chunk
291
+ cache_x = torch.cat([
292
+ feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(
293
+ cache_x.device), cache_x
294
+ ],
295
+ dim=2)
296
+ x = layer(x, feat_cache[idx])
297
+ feat_cache[idx] = cache_x
298
+ feat_idx[0] += 1
299
+ else:
300
+ x = layer(x)
301
+ return x + h, feat_cache, feat_idx
302
+
303
+
304
+ class AttentionBlock(nn.Module):
305
+ """
306
+ Causal self-attention with a single head.
307
+ """
308
+
309
+ def __init__(self, dim):
310
+ super().__init__()
311
+ self.dim = dim
312
+
313
+ # layers
314
+ self.norm = RMS_norm(dim)
315
+ self.to_qkv = nn.Conv2d(dim, dim * 3, 1)
316
+ self.proj = nn.Conv2d(dim, dim, 1)
317
+
318
+ # zero out the last layer params
319
+ nn.init.zeros_(self.proj.weight)
320
+
321
+ def forward(self, x):
322
+ identity = x
323
+ b, c, t, h, w = x.size()
324
+ x = rearrange(x, 'b c t h w -> (b t) c h w')
325
+ x = self.norm(x)
326
+ # compute query, key, value
327
+ q, k, v = self.to_qkv(x).reshape(b * t, 1, c * 3, -1).permute(
328
+ 0, 1, 3, 2).contiguous().chunk(3, dim=-1)
329
+
330
+ # apply attention
331
+ x = F.scaled_dot_product_attention(
332
+ q,
333
+ k,
334
+ v,
335
+ #attn_mask=block_causal_mask(q, block_size=h * w)
336
+ )
337
+ x = x.squeeze(1).permute(0, 2, 1).reshape(b * t, c, h, w)
338
+
339
+ # output
340
+ x = self.proj(x)
341
+ x = rearrange(x, '(b t) c h w-> b c t h w', t=t)
342
+ return x + identity
343
+
344
+
345
+ class AvgDown3D(nn.Module):
346
+ def __init__(
347
+ self,
348
+ in_channels,
349
+ out_channels,
350
+ factor_t,
351
+ factor_s=1,
352
+ ):
353
+ super().__init__()
354
+ self.in_channels = in_channels
355
+ self.out_channels = out_channels
356
+ self.factor_t = factor_t
357
+ self.factor_s = factor_s
358
+ self.factor = self.factor_t * self.factor_s * self.factor_s
359
+
360
+ assert in_channels * self.factor % out_channels == 0
361
+ self.group_size = in_channels * self.factor // out_channels
362
+
363
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
364
+ pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t
365
+ pad = (0, 0, 0, 0, pad_t, 0)
366
+ x = F.pad(x, pad)
367
+ B, C, T, H, W = x.shape
368
+ x = x.view(
369
+ B,
370
+ C,
371
+ T // self.factor_t,
372
+ self.factor_t,
373
+ H // self.factor_s,
374
+ self.factor_s,
375
+ W // self.factor_s,
376
+ self.factor_s,
377
+ )
378
+ x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous()
379
+ x = x.view(
380
+ B,
381
+ C * self.factor,
382
+ T // self.factor_t,
383
+ H // self.factor_s,
384
+ W // self.factor_s,
385
+ )
386
+ x = x.view(
387
+ B,
388
+ self.out_channels,
389
+ self.group_size,
390
+ T // self.factor_t,
391
+ H // self.factor_s,
392
+ W // self.factor_s,
393
+ )
394
+ x = x.mean(dim=2)
395
+ return x
396
+
397
+
398
+ class DupUp3D(nn.Module):
399
+ def __init__(
400
+ self,
401
+ in_channels: int,
402
+ out_channels: int,
403
+ factor_t,
404
+ factor_s=1,
405
+ ):
406
+ super().__init__()
407
+ self.in_channels = in_channels
408
+ self.out_channels = out_channels
409
+
410
+ self.factor_t = factor_t
411
+ self.factor_s = factor_s
412
+ self.factor = self.factor_t * self.factor_s * self.factor_s
413
+
414
+ assert out_channels * self.factor % in_channels == 0
415
+ self.repeats = out_channels * self.factor // in_channels
416
+
417
+ def forward(self, x: torch.Tensor, first_chunk=False) -> torch.Tensor:
418
+ x = x.repeat_interleave(self.repeats, dim=1)
419
+ x = x.view(
420
+ x.size(0),
421
+ self.out_channels,
422
+ self.factor_t,
423
+ self.factor_s,
424
+ self.factor_s,
425
+ x.size(2),
426
+ x.size(3),
427
+ x.size(4),
428
+ )
429
+ x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous()
430
+ x = x.view(
431
+ x.size(0),
432
+ self.out_channels,
433
+ x.size(2) * self.factor_t,
434
+ x.size(4) * self.factor_s,
435
+ x.size(6) * self.factor_s,
436
+ )
437
+ if first_chunk:
438
+ x = x[:, :, self.factor_t - 1 :, :, :]
439
+ return x
440
+
441
+
442
+ class Down_ResidualBlock(nn.Module):
443
+ def __init__(
444
+ self, in_dim, out_dim, dropout, mult, temperal_downsample=False, down_flag=False
445
+ ):
446
+ super().__init__()
447
+
448
+ # Shortcut path with downsample
449
+ self.avg_shortcut = AvgDown3D(
450
+ in_dim,
451
+ out_dim,
452
+ factor_t=2 if temperal_downsample else 1,
453
+ factor_s=2 if down_flag else 1,
454
+ )
455
+
456
+ # Main path with residual blocks and downsample
457
+ downsamples = []
458
+ for _ in range(mult):
459
+ downsamples.append(ResidualBlock(in_dim, out_dim, dropout))
460
+ in_dim = out_dim
461
+
462
+ # Add the final downsample block
463
+ if down_flag:
464
+ mode = "downsample3d" if temperal_downsample else "downsample2d"
465
+ downsamples.append(Resample38(out_dim, mode=mode))
466
+
467
+ self.downsamples = nn.Sequential(*downsamples)
468
+
469
+ def forward(self, x, feat_cache=None, feat_idx=[0]):
470
+ x_copy = x.clone()
471
+ for module in self.downsamples:
472
+ x, feat_cache, feat_idx = module(x, feat_cache, feat_idx)
473
+
474
+ return x + self.avg_shortcut(x_copy), feat_cache, feat_idx
475
+
476
+
477
+ class Up_ResidualBlock(nn.Module):
478
+ def __init__(
479
+ self, in_dim, out_dim, dropout, mult, temperal_upsample=False, up_flag=False
480
+ ):
481
+ super().__init__()
482
+ # Shortcut path with upsample
483
+ if up_flag:
484
+ self.avg_shortcut = DupUp3D(
485
+ in_dim,
486
+ out_dim,
487
+ factor_t=2 if temperal_upsample else 1,
488
+ factor_s=2 if up_flag else 1,
489
+ )
490
+ else:
491
+ self.avg_shortcut = None
492
+
493
+ # Main path with residual blocks and upsample
494
+ upsamples = []
495
+ for _ in range(mult):
496
+ upsamples.append(ResidualBlock(in_dim, out_dim, dropout))
497
+ in_dim = out_dim
498
+
499
+ # Add the final upsample block
500
+ if up_flag:
501
+ mode = "upsample3d" if temperal_upsample else "upsample2d"
502
+ upsamples.append(Resample38(out_dim, mode=mode))
503
+
504
+ self.upsamples = nn.Sequential(*upsamples)
505
+
506
+ def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False):
507
+ x_main = x.clone()
508
+ for module in self.upsamples:
509
+ x_main, feat_cache, feat_idx = module(x_main, feat_cache, feat_idx)
510
+ if self.avg_shortcut is not None:
511
+ x_shortcut = self.avg_shortcut(x, first_chunk)
512
+ x_main = x_main + x_shortcut
513
+ return x_main, feat_cache, feat_idx
514
+
515
+
516
+ class Encoder3d(nn.Module):
517
+
518
+ def __init__(self,
519
+ dim=128,
520
+ z_dim=4,
521
+ dim_mult=[1, 2, 4, 4],
522
+ num_res_blocks=2,
523
+ attn_scales=[],
524
+ temperal_downsample=[True, True, False],
525
+ dropout=0.0):
526
+ super().__init__()
527
+ self.dim = dim
528
+ self.z_dim = z_dim
529
+ self.dim_mult = dim_mult
530
+ self.num_res_blocks = num_res_blocks
531
+ self.attn_scales = attn_scales
532
+ self.temperal_downsample = temperal_downsample
533
+
534
+ # dimensions
535
+ dims = [dim * u for u in [1] + dim_mult]
536
+ scale = 1.0
537
+
538
+ # init block
539
+ self.conv1 = CausalConv3d(3, dims[0], 3, padding=1)
540
+
541
+ # downsample blocks
542
+ downsamples = []
543
+ for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])):
544
+ # residual (+attention) blocks
545
+ for _ in range(num_res_blocks):
546
+ downsamples.append(ResidualBlock(in_dim, out_dim, dropout))
547
+ if scale in attn_scales:
548
+ downsamples.append(AttentionBlock(out_dim))
549
+ in_dim = out_dim
550
+
551
+ # downsample block
552
+ if i != len(dim_mult) - 1:
553
+ mode = 'downsample3d' if temperal_downsample[
554
+ i] else 'downsample2d'
555
+ downsamples.append(Resample(out_dim, mode=mode))
556
+ scale /= 2.0
557
+ self.downsamples = nn.Sequential(*downsamples)
558
+
559
+ # middle blocks
560
+ self.middle = nn.Sequential(ResidualBlock(out_dim, out_dim, dropout),
561
+ AttentionBlock(out_dim),
562
+ ResidualBlock(out_dim, out_dim, dropout))
563
+
564
+ # output blocks
565
+ self.head = nn.Sequential(RMS_norm(out_dim, images=False), nn.SiLU(),
566
+ CausalConv3d(out_dim, z_dim, 3, padding=1))
567
+
568
+ def forward(self, x, feat_cache=None, feat_idx=[0]):
569
+ if feat_cache is not None:
570
+ idx = feat_idx[0]
571
+ cache_x = x[:, :, -CACHE_T:, :, :].clone()
572
+ if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
573
+ # cache last frame of last two chunk
574
+ cache_x = torch.cat([
575
+ feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(
576
+ cache_x.device), cache_x
577
+ ],
578
+ dim=2)
579
+ x = self.conv1(x, feat_cache[idx])
580
+ feat_cache[idx] = cache_x
581
+ feat_idx[0] += 1
582
+ else:
583
+ x = self.conv1(x)
584
+
585
+ ## downsamples
586
+ for layer in self.downsamples:
587
+ if feat_cache is not None:
588
+ x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
589
+ else:
590
+ x = layer(x)
591
+
592
+ ## middle
593
+ for layer in self.middle:
594
+ if check_is_instance(layer, ResidualBlock) and feat_cache is not None:
595
+ x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
596
+ else:
597
+ x = layer(x)
598
+
599
+ ## head
600
+ for layer in self.head:
601
+ if check_is_instance(layer, CausalConv3d) and feat_cache is not None:
602
+ idx = feat_idx[0]
603
+ cache_x = x[:, :, -CACHE_T:, :, :].clone()
604
+ if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
605
+ # cache last frame of last two chunk
606
+ cache_x = torch.cat([
607
+ feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(
608
+ cache_x.device), cache_x
609
+ ],
610
+ dim=2)
611
+ x = layer(x, feat_cache[idx])
612
+ feat_cache[idx] = cache_x
613
+ feat_idx[0] += 1
614
+ else:
615
+ x = layer(x)
616
+ return x, feat_cache, feat_idx
617
+
618
+
619
+ class Encoder3d_38(nn.Module):
620
+
621
+ def __init__(self,
622
+ dim=128,
623
+ z_dim=4,
624
+ dim_mult=[1, 2, 4, 4],
625
+ num_res_blocks=2,
626
+ attn_scales=[],
627
+ temperal_downsample=[False, True, True],
628
+ dropout=0.0):
629
+ super().__init__()
630
+ self.dim = dim
631
+ self.z_dim = z_dim
632
+ self.dim_mult = dim_mult
633
+ self.num_res_blocks = num_res_blocks
634
+ self.attn_scales = attn_scales
635
+ self.temperal_downsample = temperal_downsample
636
+
637
+ # dimensions
638
+ dims = [dim * u for u in [1] + dim_mult]
639
+ scale = 1.0
640
+
641
+ # init block
642
+ self.conv1 = CausalConv3d(12, dims[0], 3, padding=1)
643
+
644
+ # downsample blocks
645
+ downsamples = []
646
+ for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])):
647
+ t_down_flag = (
648
+ temperal_downsample[i] if i < len(temperal_downsample) else False
649
+ )
650
+ downsamples.append(
651
+ Down_ResidualBlock(
652
+ in_dim=in_dim,
653
+ out_dim=out_dim,
654
+ dropout=dropout,
655
+ mult=num_res_blocks,
656
+ temperal_downsample=t_down_flag,
657
+ down_flag=i != len(dim_mult) - 1,
658
+ )
659
+ )
660
+ scale /= 2.0
661
+ self.downsamples = nn.Sequential(*downsamples)
662
+
663
+ # middle blocks
664
+ self.middle = nn.Sequential(
665
+ ResidualBlock(out_dim, out_dim, dropout),
666
+ AttentionBlock(out_dim),
667
+ ResidualBlock(out_dim, out_dim, dropout),
668
+ )
669
+
670
+ # # output blocks
671
+ self.head = nn.Sequential(
672
+ RMS_norm(out_dim, images=False),
673
+ nn.SiLU(),
674
+ CausalConv3d(out_dim, z_dim, 3, padding=1),
675
+ )
676
+
677
+
678
+ def forward(self, x, feat_cache=None, feat_idx=[0]):
679
+
680
+ if feat_cache is not None:
681
+ idx = feat_idx[0]
682
+ cache_x = x[:, :, -CACHE_T:, :, :].clone()
683
+ if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
684
+ cache_x = torch.cat(
685
+ [
686
+ feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device),
687
+ cache_x,
688
+ ],
689
+ dim=2,
690
+ )
691
+ x = self.conv1(x, feat_cache[idx])
692
+ feat_cache[idx] = cache_x
693
+ feat_idx[0] += 1
694
+ else:
695
+ x = self.conv1(x)
696
+
697
+ ## downsamples
698
+ for layer in self.downsamples:
699
+ if feat_cache is not None:
700
+ x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
701
+ else:
702
+ x = layer(x)
703
+
704
+ ## middle
705
+ for layer in self.middle:
706
+ if isinstance(layer, ResidualBlock) and feat_cache is not None:
707
+ x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
708
+ else:
709
+ x = layer(x)
710
+
711
+ ## head
712
+ for layer in self.head:
713
+ if isinstance(layer, CausalConv3d) and feat_cache is not None:
714
+ idx = feat_idx[0]
715
+ cache_x = x[:, :, -CACHE_T:, :, :].clone()
716
+ if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
717
+ cache_x = torch.cat(
718
+ [
719
+ feat_cache[idx][:, :, -1, :, :]
720
+ .unsqueeze(2)
721
+ .to(cache_x.device),
722
+ cache_x,
723
+ ],
724
+ dim=2,
725
+ )
726
+ x = layer(x, feat_cache[idx])
727
+ feat_cache[idx] = cache_x
728
+ feat_idx[0] += 1
729
+ else:
730
+ x = layer(x)
731
+
732
+ return x, feat_cache, feat_idx
733
+
734
+
735
+ class Decoder3d(nn.Module):
736
+
737
+ def __init__(self,
738
+ dim=128,
739
+ z_dim=4,
740
+ dim_mult=[1, 2, 4, 4],
741
+ num_res_blocks=2,
742
+ attn_scales=[],
743
+ temperal_upsample=[False, True, True],
744
+ dropout=0.0):
745
+ super().__init__()
746
+ self.dim = dim
747
+ self.z_dim = z_dim
748
+ self.dim_mult = dim_mult
749
+ self.num_res_blocks = num_res_blocks
750
+ self.attn_scales = attn_scales
751
+ self.temperal_upsample = temperal_upsample
752
+
753
+ # dimensions
754
+ dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]]
755
+ scale = 1.0 / 2**(len(dim_mult) - 2)
756
+
757
+ # init block
758
+ self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1)
759
+
760
+ # middle blocks
761
+ self.middle = nn.Sequential(ResidualBlock(dims[0], dims[0], dropout),
762
+ AttentionBlock(dims[0]),
763
+ ResidualBlock(dims[0], dims[0], dropout))
764
+
765
+ # upsample blocks
766
+ upsamples = []
767
+ for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])):
768
+ # residual (+attention) blocks
769
+ if i == 1 or i == 2 or i == 3:
770
+ in_dim = in_dim // 2
771
+ for _ in range(num_res_blocks + 1):
772
+ upsamples.append(ResidualBlock(in_dim, out_dim, dropout))
773
+ if scale in attn_scales:
774
+ upsamples.append(AttentionBlock(out_dim))
775
+ in_dim = out_dim
776
+
777
+ # upsample block
778
+ if i != len(dim_mult) - 1:
779
+ mode = 'upsample3d' if temperal_upsample[i] else 'upsample2d'
780
+ upsamples.append(Resample(out_dim, mode=mode))
781
+ scale *= 2.0
782
+ self.upsamples = nn.Sequential(*upsamples)
783
+
784
+ # output blocks
785
+ self.head = nn.Sequential(RMS_norm(out_dim, images=False), nn.SiLU(),
786
+ CausalConv3d(out_dim, 3, 3, padding=1))
787
+
788
+ def forward(self, x, feat_cache=None, feat_idx=[0]):
789
+ ## conv1
790
+ if feat_cache is not None:
791
+ idx = feat_idx[0]
792
+ cache_x = x[:, :, -CACHE_T:, :, :].clone()
793
+ if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
794
+ # cache last frame of last two chunk
795
+ cache_x = torch.cat([
796
+ feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(
797
+ cache_x.device), cache_x
798
+ ],
799
+ dim=2)
800
+ x = self.conv1(x, feat_cache[idx])
801
+ feat_cache[idx] = cache_x
802
+ feat_idx[0] += 1
803
+ else:
804
+ x = self.conv1(x)
805
+
806
+ ## middle
807
+ for layer in self.middle:
808
+ if check_is_instance(layer, ResidualBlock) and feat_cache is not None:
809
+ x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
810
+ else:
811
+ x = layer(x)
812
+
813
+ ## upsamples
814
+ for layer in self.upsamples:
815
+ if feat_cache is not None:
816
+ x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
817
+ else:
818
+ x = layer(x)
819
+
820
+ ## head
821
+ for layer in self.head:
822
+ if check_is_instance(layer, CausalConv3d) and feat_cache is not None:
823
+ idx = feat_idx[0]
824
+ cache_x = x[:, :, -CACHE_T:, :, :].clone()
825
+ if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
826
+ # cache last frame of last two chunk
827
+ cache_x = torch.cat([
828
+ feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(
829
+ cache_x.device), cache_x
830
+ ],
831
+ dim=2)
832
+ x = layer(x, feat_cache[idx])
833
+ feat_cache[idx] = cache_x
834
+ feat_idx[0] += 1
835
+ else:
836
+ x = layer(x)
837
+ return x, feat_cache, feat_idx
838
+
839
+
840
+
841
+ class Decoder3d_38(nn.Module):
842
+
843
+ def __init__(self,
844
+ dim=128,
845
+ z_dim=4,
846
+ dim_mult=[1, 2, 4, 4],
847
+ num_res_blocks=2,
848
+ attn_scales=[],
849
+ temperal_upsample=[False, True, True],
850
+ dropout=0.0):
851
+ super().__init__()
852
+ self.dim = dim
853
+ self.z_dim = z_dim
854
+ self.dim_mult = dim_mult
855
+ self.num_res_blocks = num_res_blocks
856
+ self.attn_scales = attn_scales
857
+ self.temperal_upsample = temperal_upsample
858
+
859
+ # dimensions
860
+ dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]]
861
+ scale = 1.0 / 2 ** (len(dim_mult) - 2)
862
+ # init block
863
+ self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1)
864
+
865
+ # middle blocks
866
+ self.middle = nn.Sequential(ResidualBlock(dims[0], dims[0], dropout),
867
+ AttentionBlock(dims[0]),
868
+ ResidualBlock(dims[0], dims[0], dropout))
869
+
870
+ # upsample blocks
871
+ upsamples = []
872
+ for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])):
873
+ t_up_flag = temperal_upsample[i] if i < len(temperal_upsample) else False
874
+ upsamples.append(
875
+ Up_ResidualBlock(in_dim=in_dim,
876
+ out_dim=out_dim,
877
+ dropout=dropout,
878
+ mult=num_res_blocks + 1,
879
+ temperal_upsample=t_up_flag,
880
+ up_flag=i != len(dim_mult) - 1))
881
+ self.upsamples = nn.Sequential(*upsamples)
882
+
883
+ # output blocks
884
+ self.head = nn.Sequential(RMS_norm(out_dim, images=False), nn.SiLU(),
885
+ CausalConv3d(out_dim, 12, 3, padding=1))
886
+
887
+
888
+ def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False):
889
+ if feat_cache is not None:
890
+ idx = feat_idx[0]
891
+ cache_x = x[:, :, -CACHE_T:, :, :].clone()
892
+ if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
893
+ cache_x = torch.cat(
894
+ [
895
+ feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device),
896
+ cache_x,
897
+ ],
898
+ dim=2,
899
+ )
900
+ x = self.conv1(x, feat_cache[idx])
901
+ feat_cache[idx] = cache_x
902
+ feat_idx[0] += 1
903
+ else:
904
+ x = self.conv1(x)
905
+
906
+ for layer in self.middle:
907
+ if check_is_instance(layer, ResidualBlock) and feat_cache is not None:
908
+ x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
909
+ else:
910
+ x = layer(x)
911
+
912
+ ## upsamples
913
+ for layer in self.upsamples:
914
+ if feat_cache is not None:
915
+ x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx, first_chunk)
916
+ else:
917
+ x = layer(x)
918
+
919
+ ## head
920
+ for layer in self.head:
921
+ if check_is_instance(layer, CausalConv3d) and feat_cache is not None:
922
+ idx = feat_idx[0]
923
+ cache_x = x[:, :, -CACHE_T:, :, :].clone()
924
+ if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
925
+ cache_x = torch.cat(
926
+ [
927
+ feat_cache[idx][:, :, -1, :, :]
928
+ .unsqueeze(2)
929
+ .to(cache_x.device),
930
+ cache_x,
931
+ ],
932
+ dim=2,
933
+ )
934
+ x = layer(x, feat_cache[idx])
935
+ feat_cache[idx] = cache_x
936
+ feat_idx[0] += 1
937
+ else:
938
+ x = layer(x)
939
+ return x, feat_cache, feat_idx
940
+
941
+
942
+ def count_conv3d(model):
943
+ count = 0
944
+ for m in model.modules():
945
+ if isinstance(m, CausalConv3d):
946
+ count += 1
947
+ return count
948
+
949
+
950
+ class VideoVAE_(nn.Module):
951
+
952
+ def __init__(self,
953
+ dim=96,
954
+ z_dim=16,
955
+ dim_mult=[1, 2, 4, 4],
956
+ num_res_blocks=2,
957
+ attn_scales=[],
958
+ temperal_downsample=[False, True, True],
959
+ dropout=0.0):
960
+ super().__init__()
961
+ self.dim = dim
962
+ self.z_dim = z_dim
963
+ self.dim_mult = dim_mult
964
+ self.num_res_blocks = num_res_blocks
965
+ self.attn_scales = attn_scales
966
+ self.temperal_downsample = temperal_downsample
967
+ self.temperal_upsample = temperal_downsample[::-1]
968
+
969
+ # modules
970
+ self.encoder = Encoder3d(dim, z_dim * 2, dim_mult, num_res_blocks,
971
+ attn_scales, self.temperal_downsample, dropout)
972
+ self.conv1 = CausalConv3d(z_dim * 2, z_dim * 2, 1)
973
+ self.conv2 = CausalConv3d(z_dim, z_dim, 1)
974
+ self.decoder = Decoder3d(dim, z_dim, dim_mult, num_res_blocks,
975
+ attn_scales, self.temperal_upsample, dropout)
976
+
977
+ def forward(self, x):
978
+ mu, log_var = self.encode(x)
979
+ z = self.reparameterize(mu, log_var)
980
+ x_recon = self.decode(z)
981
+ return x_recon, mu, log_var
982
+
983
+ def encode(self, x, scale):
984
+ self.clear_cache()
985
+ ## cache
986
+ t = x.shape[2]
987
+ iter_ = 1 + (t - 1) // 4
988
+
989
+ for i in range(iter_):
990
+ self._enc_conv_idx = [0]
991
+ if i == 0:
992
+ out, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, :1, :, :],
993
+ feat_cache=self._enc_feat_map,
994
+ feat_idx=self._enc_conv_idx)
995
+ else:
996
+ out_, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, 1 + 4 * (i - 1):1 + 4 * i, :, :],
997
+ feat_cache=self._enc_feat_map,
998
+ feat_idx=self._enc_conv_idx)
999
+ out = torch.cat([out, out_], 2)
1000
+ mu, log_var = self.conv1(out).chunk(2, dim=1)
1001
+ if isinstance(scale[0], torch.Tensor):
1002
+ scale = [s.to(dtype=mu.dtype, device=mu.device) for s in scale]
1003
+ mu = (mu - scale[0].view(1, self.z_dim, 1, 1, 1)) * scale[1].view(
1004
+ 1, self.z_dim, 1, 1, 1)
1005
+ else:
1006
+ scale = scale.to(dtype=mu.dtype, device=mu.device)
1007
+ mu = (mu - scale[0]) * scale[1]
1008
+ return mu
1009
+
1010
+ def decode(self, z, scale):
1011
+ self.clear_cache()
1012
+ # z: [b,c,t,h,w]
1013
+ if isinstance(scale[0], torch.Tensor):
1014
+ scale = [s.to(dtype=z.dtype, device=z.device) for s in scale]
1015
+ z = z / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view(
1016
+ 1, self.z_dim, 1, 1, 1)
1017
+ else:
1018
+ scale = scale.to(dtype=z.dtype, device=z.device)
1019
+ z = z / scale[1] + scale[0]
1020
+ iter_ = z.shape[2]
1021
+ x = self.conv2(z)
1022
+ for i in range(iter_):
1023
+ self._conv_idx = [0]
1024
+ if i == 0:
1025
+ out, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :],
1026
+ feat_cache=self._feat_map,
1027
+ feat_idx=self._conv_idx)
1028
+ else:
1029
+ out_, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :],
1030
+ feat_cache=self._feat_map,
1031
+ feat_idx=self._conv_idx)
1032
+ out = torch.cat([out, out_], 2) # may add tensor offload
1033
+ return out
1034
+
1035
+ def reparameterize(self, mu, log_var):
1036
+ std = torch.exp(0.5 * log_var)
1037
+ eps = torch.randn_like(std)
1038
+ return eps * std + mu
1039
+
1040
+ def sample(self, imgs, deterministic=False):
1041
+ mu, log_var = self.encode(imgs)
1042
+ if deterministic:
1043
+ return mu
1044
+ std = torch.exp(0.5 * log_var.clamp(-30.0, 20.0))
1045
+ return mu + std * torch.randn_like(std)
1046
+
1047
+ def clear_cache(self):
1048
+ self._conv_num = count_conv3d(self.decoder)
1049
+ self._conv_idx = [0]
1050
+ self._feat_map = [None] * self._conv_num
1051
+ # cache encode
1052
+ self._enc_conv_num = count_conv3d(self.encoder)
1053
+ self._enc_conv_idx = [0]
1054
+ self._enc_feat_map = [None] * self._enc_conv_num
1055
+
1056
+
1057
+ class WanVideoVAE(nn.Module):
1058
+
1059
+ def __init__(self, z_dim=16):
1060
+ super().__init__()
1061
+
1062
+ mean = [
1063
+ -0.7571, -0.7089, -0.9113, 0.1075, -0.1745, 0.9653, -0.1517, 1.5508,
1064
+ 0.4134, -0.0715, 0.5517, -0.3632, -0.1922, -0.9497, 0.2503, -0.2921
1065
+ ]
1066
+ std = [
1067
+ 2.8184, 1.4541, 2.3275, 2.6558, 1.2196, 1.7708, 2.6052, 2.0743,
1068
+ 3.2687, 2.1526, 2.8652, 1.5579, 1.6382, 1.1253, 2.8251, 1.9160
1069
+ ]
1070
+ self.mean = torch.tensor(mean)
1071
+ self.std = torch.tensor(std)
1072
+ self.scale = [self.mean, 1.0 / self.std]
1073
+
1074
+ # init model
1075
+ self.model = VideoVAE_(z_dim=z_dim).eval().requires_grad_(False)
1076
+ self.upsampling_factor = 8
1077
+ self.temporal_downsample_factor = 4
1078
+ self.z_dim = z_dim
1079
+
1080
+
1081
+ def build_1d_mask(self, length, left_bound, right_bound, border_width):
1082
+ x = torch.ones((length,))
1083
+ if not left_bound:
1084
+ x[:border_width] = (torch.arange(border_width) + 1) / border_width
1085
+ if not right_bound:
1086
+ x[-border_width:] = torch.flip((torch.arange(border_width) + 1) / border_width, dims=(0,))
1087
+ return x
1088
+
1089
+
1090
+ def build_mask(self, data, is_bound, border_width):
1091
+ _, _, _, H, W = data.shape
1092
+ h = self.build_1d_mask(H, is_bound[0], is_bound[1], border_width[0])
1093
+ w = self.build_1d_mask(W, is_bound[2], is_bound[3], border_width[1])
1094
+
1095
+ h = repeat(h, "H -> H W", H=H, W=W)
1096
+ w = repeat(w, "W -> H W", H=H, W=W)
1097
+
1098
+ mask = torch.stack([h, w]).min(dim=0).values
1099
+ mask = rearrange(mask, "H W -> 1 1 1 H W")
1100
+ return mask
1101
+
1102
+
1103
+ def tiled_decode(self, hidden_states, device, tile_size, tile_stride):
1104
+ _, _, T, H, W = hidden_states.shape
1105
+ size_h, size_w = tile_size
1106
+ stride_h, stride_w = tile_stride
1107
+
1108
+ # Split tasks
1109
+ tasks = []
1110
+ for h in range(0, H, stride_h):
1111
+ if (h-stride_h >= 0 and h-stride_h+size_h >= H): continue
1112
+ for w in range(0, W, stride_w):
1113
+ if (w-stride_w >= 0 and w-stride_w+size_w >= W): continue
1114
+ h_, w_ = h + size_h, w + size_w
1115
+ tasks.append((h, h_, w, w_))
1116
+
1117
+ data_device = "cpu"
1118
+ computation_device = device
1119
+
1120
+ out_T = T * 4 - 3
1121
+ weight = torch.zeros((1, 1, out_T, H * self.upsampling_factor, W * self.upsampling_factor), dtype=hidden_states.dtype, device=data_device)
1122
+ values = torch.zeros((1, 3, out_T, H * self.upsampling_factor, W * self.upsampling_factor), dtype=hidden_states.dtype, device=data_device)
1123
+
1124
+ for h, h_, w, w_ in tqdm(tasks, desc="VAE decoding"):
1125
+ hidden_states_batch = hidden_states[:, :, :, h:h_, w:w_].to(computation_device)
1126
+ hidden_states_batch = self.model.decode(hidden_states_batch, self.scale).to(data_device)
1127
+
1128
+ mask = self.build_mask(
1129
+ hidden_states_batch,
1130
+ is_bound=(h==0, h_>=H, w==0, w_>=W),
1131
+ border_width=((size_h - stride_h) * self.upsampling_factor, (size_w - stride_w) * self.upsampling_factor)
1132
+ ).to(dtype=hidden_states.dtype, device=data_device)
1133
+
1134
+ target_h = h * self.upsampling_factor
1135
+ target_w = w * self.upsampling_factor
1136
+ values[
1137
+ :,
1138
+ :,
1139
+ :,
1140
+ target_h:target_h + hidden_states_batch.shape[3],
1141
+ target_w:target_w + hidden_states_batch.shape[4],
1142
+ ] += hidden_states_batch * mask
1143
+ weight[
1144
+ :,
1145
+ :,
1146
+ :,
1147
+ target_h: target_h + hidden_states_batch.shape[3],
1148
+ target_w: target_w + hidden_states_batch.shape[4],
1149
+ ] += mask
1150
+ values = values / weight
1151
+ values = values.clamp_(-1, 1)
1152
+ return values
1153
+
1154
+
1155
+ def tiled_encode(self, video, device, tile_size, tile_stride):
1156
+ _, _, T, H, W = video.shape
1157
+ size_h, size_w = tile_size
1158
+ stride_h, stride_w = tile_stride
1159
+
1160
+ # Split tasks
1161
+ tasks = []
1162
+ for h in range(0, H, stride_h):
1163
+ if (h-stride_h >= 0 and h-stride_h+size_h >= H): continue
1164
+ for w in range(0, W, stride_w):
1165
+ if (w-stride_w >= 0 and w-stride_w+size_w >= W): continue
1166
+ h_, w_ = h + size_h, w + size_w
1167
+ tasks.append((h, h_, w, w_))
1168
+
1169
+ data_device = "cpu"
1170
+ computation_device = device
1171
+
1172
+ out_T = (T + 3) // 4
1173
+ weight = torch.zeros((1, 1, out_T, H // self.upsampling_factor, W // self.upsampling_factor), dtype=video.dtype, device=data_device)
1174
+ values = torch.zeros((1, self.z_dim, out_T, H // self.upsampling_factor, W // self.upsampling_factor), dtype=video.dtype, device=data_device)
1175
+
1176
+ for h, h_, w, w_ in tqdm(tasks, desc="VAE encoding"):
1177
+ hidden_states_batch = video[:, :, :, h:h_, w:w_].to(computation_device)
1178
+ hidden_states_batch = self.model.encode(hidden_states_batch, self.scale).to(data_device)
1179
+
1180
+ mask = self.build_mask(
1181
+ hidden_states_batch,
1182
+ is_bound=(h==0, h_>=H, w==0, w_>=W),
1183
+ border_width=((size_h - stride_h) // self.upsampling_factor, (size_w - stride_w) // self.upsampling_factor)
1184
+ ).to(dtype=video.dtype, device=data_device)
1185
+
1186
+ target_h = h // self.upsampling_factor
1187
+ target_w = w // self.upsampling_factor
1188
+ values[
1189
+ :,
1190
+ :,
1191
+ :,
1192
+ target_h:target_h + hidden_states_batch.shape[3],
1193
+ target_w:target_w + hidden_states_batch.shape[4],
1194
+ ] += hidden_states_batch * mask
1195
+ weight[
1196
+ :,
1197
+ :,
1198
+ :,
1199
+ target_h: target_h + hidden_states_batch.shape[3],
1200
+ target_w: target_w + hidden_states_batch.shape[4],
1201
+ ] += mask
1202
+ values = values / weight
1203
+ return values
1204
+
1205
+
1206
+ def single_encode(self, video, device):
1207
+ video = video.to(device)
1208
+ x = self.model.encode(video, self.scale)
1209
+ return x
1210
+
1211
+
1212
+ def single_decode(self, hidden_state, device):
1213
+ hidden_state = hidden_state.to(device)
1214
+ video = self.model.decode(hidden_state, self.scale)
1215
+ return video.clamp_(-1, 1)
1216
+
1217
+
1218
+ def encode(self, videos, device, tiled=False, tile_size=(34, 34), tile_stride=(18, 16)):
1219
+ # videos = [video.to("cpu") for video in videos]
1220
+ hidden_states = []
1221
+ for video in videos:
1222
+ video = video.unsqueeze(0)
1223
+ if tiled:
1224
+ raise NotImplementedError("Tiled encoding is not allowed yet.")
1225
+ tile_size = (tile_size[0] * self.upsampling_factor, tile_size[1] * self.upsampling_factor)
1226
+ tile_stride = (tile_stride[0] * self.upsampling_factor, tile_stride[1] * self.upsampling_factor)
1227
+ hidden_state = self.tiled_encode(video, device, tile_size, tile_stride)
1228
+ else:
1229
+ hidden_state = self.single_encode(video, device)
1230
+ hidden_state = hidden_state.squeeze(0)
1231
+ hidden_states.append(hidden_state)
1232
+ hidden_states = torch.stack(hidden_states)
1233
+ return hidden_states
1234
+
1235
+
1236
+ def decode(self, hidden_states, device, tiled=False, tile_size=(34, 34), tile_stride=(18, 16)):
1237
+ hidden_states = [hidden_state.to("cpu") for hidden_state in hidden_states]
1238
+ videos = []
1239
+ for hidden_state in hidden_states:
1240
+ hidden_state = hidden_state.unsqueeze(0)
1241
+ if tiled:
1242
+ video = self.tiled_decode(hidden_state, device, tile_size, tile_stride)
1243
+ else:
1244
+ video = self.single_decode(hidden_state, device)
1245
+ video = video.squeeze(0)
1246
+ videos.append(video)
1247
+ videos = torch.stack(videos)
1248
+ return videos
1249
+
1250
+
1251
+ @staticmethod
1252
+ def state_dict_converter():
1253
+ return WanVideoVAEStateDictConverter()
1254
+
1255
+
1256
+ class WanVideoVAEStateDictConverter:
1257
+
1258
+ def __init__(self):
1259
+ pass
1260
+
1261
+ def from_civitai(self, state_dict):
1262
+ state_dict_ = {}
1263
+ if 'model_state' in state_dict:
1264
+ state_dict = state_dict['model_state']
1265
+ for name in state_dict:
1266
+ state_dict_['model.' + name] = state_dict[name]
1267
+ return state_dict_
1268
+
1269
+
1270
+ class VideoVAE38_(VideoVAE_):
1271
+
1272
+ def __init__(self,
1273
+ dim=160,
1274
+ z_dim=48,
1275
+ dec_dim=256,
1276
+ dim_mult=[1, 2, 4, 4],
1277
+ num_res_blocks=2,
1278
+ attn_scales=[],
1279
+ temperal_downsample=[False, True, True],
1280
+ dropout=0.0):
1281
+ super(VideoVAE_, self).__init__()
1282
+ self.dim = dim
1283
+ self.z_dim = z_dim
1284
+ self.dim_mult = dim_mult
1285
+ self.num_res_blocks = num_res_blocks
1286
+ self.attn_scales = attn_scales
1287
+ self.temperal_downsample = temperal_downsample
1288
+ self.temperal_upsample = temperal_downsample[::-1]
1289
+
1290
+ # modules
1291
+ self.encoder = Encoder3d_38(dim, z_dim * 2, dim_mult, num_res_blocks,
1292
+ attn_scales, self.temperal_downsample, dropout)
1293
+ self.conv1 = CausalConv3d(z_dim * 2, z_dim * 2, 1)
1294
+ self.conv2 = CausalConv3d(z_dim, z_dim, 1)
1295
+ self.decoder = Decoder3d_38(dec_dim, z_dim, dim_mult, num_res_blocks,
1296
+ attn_scales, self.temperal_upsample, dropout)
1297
+
1298
+
1299
+ def encode(self, x, scale):
1300
+ self.clear_cache()
1301
+ x = patchify(x, patch_size=2)
1302
+ t = x.shape[2]
1303
+ iter_ = 1 + (t - 1) // 4
1304
+ for i in range(iter_):
1305
+ self._enc_conv_idx = [0]
1306
+ if i == 0:
1307
+ out, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, :1, :, :],
1308
+ feat_cache=self._enc_feat_map,
1309
+ feat_idx=self._enc_conv_idx)
1310
+ else:
1311
+ out_, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, 1 + 4 * (i - 1):1 + 4 * i, :, :],
1312
+ feat_cache=self._enc_feat_map,
1313
+ feat_idx=self._enc_conv_idx)
1314
+ out = torch.cat([out, out_], 2)
1315
+ mu, log_var = self.conv1(out).chunk(2, dim=1)
1316
+ if isinstance(scale[0], torch.Tensor):
1317
+ scale = [s.to(dtype=mu.dtype, device=mu.device) for s in scale]
1318
+ mu = (mu - scale[0].view(1, self.z_dim, 1, 1, 1)) * scale[1].view(
1319
+ 1, self.z_dim, 1, 1, 1)
1320
+ else:
1321
+ scale = scale.to(dtype=mu.dtype, device=mu.device)
1322
+ mu = (mu - scale[0]) * scale[1]
1323
+ self.clear_cache()
1324
+ return mu
1325
+
1326
+
1327
+ def decode(self, z, scale):
1328
+ self.clear_cache()
1329
+ if isinstance(scale[0], torch.Tensor):
1330
+ scale = [s.to(dtype=z.dtype, device=z.device) for s in scale]
1331
+ z = z / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view(
1332
+ 1, self.z_dim, 1, 1, 1)
1333
+ else:
1334
+ scale = scale.to(dtype=z.dtype, device=z.device)
1335
+ z = z / scale[1] + scale[0]
1336
+ iter_ = z.shape[2]
1337
+ x = self.conv2(z)
1338
+ for i in range(iter_):
1339
+ self._conv_idx = [0]
1340
+ if i == 0:
1341
+ out, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :],
1342
+ feat_cache=self._feat_map,
1343
+ feat_idx=self._conv_idx,
1344
+ first_chunk=True)
1345
+ else:
1346
+ out_, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :],
1347
+ feat_cache=self._feat_map,
1348
+ feat_idx=self._conv_idx)
1349
+ out = torch.cat([out, out_], 2)
1350
+ out = unpatchify(out, patch_size=2)
1351
+ self.clear_cache()
1352
+ return out
1353
+
1354
+
1355
+ class WanVideoVAE38(WanVideoVAE):
1356
+
1357
+ def __init__(self, z_dim=48, dim=160):
1358
+ super(WanVideoVAE, self).__init__()
1359
+
1360
+ mean = [
1361
+ -0.2289, -0.0052, -0.1323, -0.2339, -0.2799, 0.0174, 0.1838, 0.1557,
1362
+ -0.1382, 0.0542, 0.2813, 0.0891, 0.1570, -0.0098, 0.0375, -0.1825,
1363
+ -0.2246, -0.1207, -0.0698, 0.5109, 0.2665, -0.2108, -0.2158, 0.2502,
1364
+ -0.2055, -0.0322, 0.1109, 0.1567, -0.0729, 0.0899, -0.2799, -0.1230,
1365
+ -0.0313, -0.1649, 0.0117, 0.0723, -0.2839, -0.2083, -0.0520, 0.3748,
1366
+ 0.0152, 0.1957, 0.1433, -0.2944, 0.3573, -0.0548, -0.1681, -0.0667
1367
+ ]
1368
+ std = [
1369
+ 0.4765, 1.0364, 0.4514, 1.1677, 0.5313, 0.4990, 0.4818, 0.5013,
1370
+ 0.8158, 1.0344, 0.5894, 1.0901, 0.6885, 0.6165, 0.8454, 0.4978,
1371
+ 0.5759, 0.3523, 0.7135, 0.6804, 0.5833, 1.4146, 0.8986, 0.5659,
1372
+ 0.7069, 0.5338, 0.4889, 0.4917, 0.4069, 0.4999, 0.6866, 0.4093,
1373
+ 0.5709, 0.6065, 0.6415, 0.4944, 0.5726, 1.2042, 0.5458, 1.6887,
1374
+ 0.3971, 1.0600, 0.3943, 0.5537, 0.5444, 0.4089, 0.7468, 0.7744
1375
+ ]
1376
+ self.mean = torch.tensor(mean)
1377
+ self.std = torch.tensor(std)
1378
+ self.scale = [self.mean, 1.0 / self.std]
1379
+
1380
+ # init model
1381
+ self.model = VideoVAE38_(z_dim=z_dim, dim=dim).eval().requires_grad_(False)
1382
+ self.upsampling_factor = 16
1383
+ self.temporal_downsample_factor = 4
1384
+ self.z_dim = z_dim
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/runtime.py ADDED
@@ -0,0 +1,436 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ import os
3
+ import inspect
4
+ from pathlib import Path
5
+
6
+ import torch
7
+ from hydra.utils import instantiate
8
+ from omegaconf import DictConfig
9
+ from PIL import Image
10
+ import numpy as np
11
+ from einops import repeat
12
+ from omegaconf import OmegaConf
13
+
14
+ from .trainer import Wan22Trainer
15
+ from .utils.logging_config import get_logger, setup_logging
16
+ from .utils.video_io import save_mp4
17
+ from .utils import misc
18
+
19
+ logger = get_logger(__name__)
20
+
21
+
22
+ def _normalize_mixed_precision(mixed_precision: str) -> str:
23
+ if not isinstance(mixed_precision, str):
24
+ raise ValueError(f"`mixed_precision` must be str, got {type(mixed_precision)}")
25
+ key = mixed_precision.strip().lower()
26
+ if key not in {"no", "fp16", "bf16"}:
27
+ raise ValueError(
28
+ f"Unsupported mixed_precision: {mixed_precision}. "
29
+ "Expected one of: ['no', 'fp16', 'bf16']."
30
+ )
31
+ return key
32
+
33
+
34
+ def _mixed_precision_to_model_dtype(mixed_precision: str) -> torch.dtype:
35
+ precision = _normalize_mixed_precision(mixed_precision)
36
+ if precision == "no":
37
+ return torch.float32
38
+ if precision == "fp16":
39
+ return torch.float16
40
+ return torch.bfloat16
41
+
42
+
43
+ def create_wan22_model(
44
+ model_id: str,
45
+ tokenizer_model_id: str,
46
+ dit_config,
47
+ tokenizer_max_len: int = 512,
48
+ train_shift: float = 5.0,
49
+ infer_shift: float = 5.0,
50
+ num_train_timesteps: int = 1000,
51
+ redirect_common_files: bool = True,
52
+ model_dtype: torch.dtype = torch.bfloat16,
53
+ device: str = "cuda",
54
+ ):
55
+ from .models.wan22.wan22 import Wan22Core
56
+
57
+ if isinstance(dit_config, DictConfig):
58
+ dit_config = OmegaConf.to_container(dit_config, resolve=True)
59
+ if not isinstance(dit_config, dict):
60
+ raise ValueError(f"`dit_config` must resolve to a dict, got {type(dit_config)}")
61
+
62
+ return Wan22Core.from_wan22_pretrained(
63
+ device=device,
64
+ torch_dtype=model_dtype,
65
+ model_id=model_id,
66
+ tokenizer_model_id=tokenizer_model_id,
67
+ tokenizer_max_len=int(tokenizer_max_len),
68
+ redirect_common_files=bool(redirect_common_files),
69
+ dit_config=dit_config,
70
+ train_shift=float(train_shift),
71
+ infer_shift=float(infer_shift),
72
+ num_train_timesteps=int(num_train_timesteps),
73
+ )
74
+
75
+
76
+ def create_fastwam(
77
+ model_id: str,
78
+ tokenizer_model_id: str,
79
+ video_dit_config,
80
+ tokenizer_max_len: int = 512,
81
+ load_text_encoder: bool = True,
82
+ proprio_dim: int | None = None,
83
+ action_dit_config=None,
84
+ action_dit_pretrained_path: str | None = None,
85
+ skip_dit_load_from_pretrain: bool = False,
86
+ video_scheduler=None,
87
+ action_scheduler=None,
88
+ loss=None,
89
+ mot_checkpoint_mixed_attn: bool = True,
90
+ redirect_common_files: bool = True,
91
+ model_dtype: torch.dtype = torch.bfloat16,
92
+ device: str = "cuda",
93
+ ):
94
+ from .models.wan22.fastwam import FastWAM
95
+
96
+ if isinstance(video_dit_config, DictConfig):
97
+ video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True)
98
+ if not isinstance(video_dit_config, dict):
99
+ raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}")
100
+
101
+ if isinstance(action_dit_config, DictConfig):
102
+ action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True)
103
+ if action_dit_config is None:
104
+ action_dit_config = {}
105
+ if not isinstance(action_dit_config, dict):
106
+ raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}")
107
+
108
+ if isinstance(video_scheduler, DictConfig):
109
+ video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True)
110
+ if video_scheduler is None:
111
+ video_scheduler = {}
112
+ if not isinstance(video_scheduler, dict):
113
+ raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}")
114
+
115
+ if isinstance(action_scheduler, DictConfig):
116
+ action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True)
117
+ if action_scheduler is None:
118
+ raise ValueError("`action_scheduler` is required for FastWAM.")
119
+ if not isinstance(action_scheduler, dict):
120
+ raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}")
121
+ required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"}
122
+ missing_keys = required_action_scheduler_keys - set(action_scheduler.keys())
123
+ if missing_keys:
124
+ raise ValueError(
125
+ f"`action_scheduler` missing required keys: {sorted(missing_keys)}. "
126
+ "Expected keys: train_shift, infer_shift, num_train_timesteps."
127
+ )
128
+
129
+ if isinstance(loss, DictConfig):
130
+ loss = OmegaConf.to_container(loss, resolve=True)
131
+ if loss is None:
132
+ loss = {}
133
+ if not isinstance(loss, dict):
134
+ raise ValueError(f"`loss` must be dict-like, got {type(loss)}")
135
+
136
+ return FastWAM.from_wan22_pretrained(
137
+ device=device,
138
+ torch_dtype=model_dtype,
139
+ model_id=model_id,
140
+ tokenizer_model_id=tokenizer_model_id,
141
+ tokenizer_max_len=int(tokenizer_max_len),
142
+ load_text_encoder=bool(load_text_encoder),
143
+ proprio_dim=(None if proprio_dim is None else int(proprio_dim)),
144
+ redirect_common_files=bool(redirect_common_files),
145
+ video_dit_config=video_dit_config,
146
+ action_dit_config=action_dit_config,
147
+ action_dit_pretrained_path=action_dit_pretrained_path,
148
+ skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain),
149
+ mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn),
150
+ video_train_shift=float(video_scheduler.get("train_shift", 5.0)),
151
+ video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)),
152
+ video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)),
153
+ action_train_shift=float(action_scheduler["train_shift"]),
154
+ action_infer_shift=float(action_scheduler["infer_shift"]),
155
+ action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]),
156
+ loss_lambda_video=float(loss.get("lambda_video", 1.0)),
157
+ loss_lambda_action=float(loss.get("lambda_action", 1.0)),
158
+ )
159
+
160
+
161
+ def create_fastwam_joint(
162
+ model_id: str,
163
+ tokenizer_model_id: str,
164
+ video_dit_config,
165
+ tokenizer_max_len: int = 512,
166
+ load_text_encoder: bool = True,
167
+ proprio_dim: int | None = None,
168
+ action_dit_config=None,
169
+ action_dit_pretrained_path: str | None = None,
170
+ skip_dit_load_from_pretrain: bool = False,
171
+ video_scheduler=None,
172
+ action_scheduler=None,
173
+ loss=None,
174
+ mot_checkpoint_mixed_attn: bool = True,
175
+ redirect_common_files: bool = True,
176
+ model_dtype: torch.dtype = torch.bfloat16,
177
+ device: str = "cuda",
178
+ ):
179
+ from .models.wan22.fastwam_joint import FastWAMJoint
180
+
181
+ if isinstance(video_dit_config, DictConfig):
182
+ video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True)
183
+ if not isinstance(video_dit_config, dict):
184
+ raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}")
185
+
186
+ if isinstance(action_dit_config, DictConfig):
187
+ action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True)
188
+ if action_dit_config is None:
189
+ action_dit_config = {}
190
+ if not isinstance(action_dit_config, dict):
191
+ raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}")
192
+
193
+ if isinstance(video_scheduler, DictConfig):
194
+ video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True)
195
+ if video_scheduler is None:
196
+ video_scheduler = {}
197
+ if not isinstance(video_scheduler, dict):
198
+ raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}")
199
+
200
+ if isinstance(action_scheduler, DictConfig):
201
+ action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True)
202
+ if action_scheduler is None:
203
+ raise ValueError("`action_scheduler` is required for FastWAM.")
204
+ if not isinstance(action_scheduler, dict):
205
+ raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}")
206
+ required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"}
207
+ missing_keys = required_action_scheduler_keys - set(action_scheduler.keys())
208
+ if missing_keys:
209
+ raise ValueError(
210
+ f"`action_scheduler` missing required keys: {sorted(missing_keys)}. "
211
+ "Expected keys: train_shift, infer_shift, num_train_timesteps."
212
+ )
213
+
214
+ if isinstance(loss, DictConfig):
215
+ loss = OmegaConf.to_container(loss, resolve=True)
216
+ if loss is None:
217
+ loss = {}
218
+ if not isinstance(loss, dict):
219
+ raise ValueError(f"`loss` must be dict-like, got {type(loss)}")
220
+
221
+ return FastWAMJoint.from_wan22_pretrained(
222
+ device=device,
223
+ torch_dtype=model_dtype,
224
+ model_id=model_id,
225
+ tokenizer_model_id=tokenizer_model_id,
226
+ tokenizer_max_len=int(tokenizer_max_len),
227
+ load_text_encoder=bool(load_text_encoder),
228
+ proprio_dim=(None if proprio_dim is None else int(proprio_dim)),
229
+ redirect_common_files=bool(redirect_common_files),
230
+ video_dit_config=video_dit_config,
231
+ action_dit_config=action_dit_config,
232
+ action_dit_pretrained_path=action_dit_pretrained_path,
233
+ skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain),
234
+ mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn),
235
+ video_train_shift=float(video_scheduler.get("train_shift", 5.0)),
236
+ video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)),
237
+ video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)),
238
+ action_train_shift=float(action_scheduler["train_shift"]),
239
+ action_infer_shift=float(action_scheduler["infer_shift"]),
240
+ action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]),
241
+ loss_lambda_video=float(loss.get("lambda_video", 1.0)),
242
+ loss_lambda_action=float(loss.get("lambda_action", 1.0)),
243
+ )
244
+
245
+
246
+ def create_fastwam_idm(
247
+ model_id: str,
248
+ tokenizer_model_id: str,
249
+ video_dit_config,
250
+ tokenizer_max_len: int = 512,
251
+ load_text_encoder: bool = True,
252
+ proprio_dim: int | None = None,
253
+ action_dit_config=None,
254
+ action_dit_pretrained_path: str | None = None,
255
+ skip_dit_load_from_pretrain: bool = False,
256
+ video_scheduler=None,
257
+ action_scheduler=None,
258
+ loss=None,
259
+ mot_checkpoint_mixed_attn: bool = True,
260
+ redirect_common_files: bool = True,
261
+ model_dtype: torch.dtype = torch.bfloat16,
262
+ device: str = "cuda",
263
+ ):
264
+ from .models.wan22.fastwam_idm import (
265
+ FastWAMIDM,
266
+ )
267
+
268
+ if isinstance(video_dit_config, DictConfig):
269
+ video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True)
270
+ if not isinstance(video_dit_config, dict):
271
+ raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}")
272
+
273
+ if isinstance(action_dit_config, DictConfig):
274
+ action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True)
275
+ if action_dit_config is None:
276
+ action_dit_config = {}
277
+ if not isinstance(action_dit_config, dict):
278
+ raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}")
279
+
280
+ if isinstance(video_scheduler, DictConfig):
281
+ video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True)
282
+ if video_scheduler is None:
283
+ video_scheduler = {}
284
+ if not isinstance(video_scheduler, dict):
285
+ raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}")
286
+
287
+ if isinstance(action_scheduler, DictConfig):
288
+ action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True)
289
+ if action_scheduler is None:
290
+ raise ValueError("`action_scheduler` is required for FastWAM.")
291
+ if not isinstance(action_scheduler, dict):
292
+ raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}")
293
+ required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"}
294
+ missing_keys = required_action_scheduler_keys - set(action_scheduler.keys())
295
+ if missing_keys:
296
+ raise ValueError(
297
+ f"`action_scheduler` missing required keys: {sorted(missing_keys)}. "
298
+ "Expected keys: train_shift, infer_shift, num_train_timesteps."
299
+ )
300
+
301
+ if isinstance(loss, DictConfig):
302
+ loss = OmegaConf.to_container(loss, resolve=True)
303
+ if loss is None:
304
+ loss = {}
305
+ if not isinstance(loss, dict):
306
+ raise ValueError(f"`loss` must be dict-like, got {type(loss)}")
307
+
308
+ return FastWAMIDM.from_wan22_pretrained(
309
+ device=device,
310
+ torch_dtype=model_dtype,
311
+ model_id=model_id,
312
+ tokenizer_model_id=tokenizer_model_id,
313
+ tokenizer_max_len=int(tokenizer_max_len),
314
+ load_text_encoder=bool(load_text_encoder),
315
+ proprio_dim=(None if proprio_dim is None else int(proprio_dim)),
316
+ redirect_common_files=bool(redirect_common_files),
317
+ video_dit_config=video_dit_config,
318
+ action_dit_config=action_dit_config,
319
+ action_dit_pretrained_path=action_dit_pretrained_path,
320
+ skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain),
321
+ mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn),
322
+ video_train_shift=float(video_scheduler.get("train_shift", 5.0)),
323
+ video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)),
324
+ video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)),
325
+ action_train_shift=float(action_scheduler["train_shift"]),
326
+ action_infer_shift=float(action_scheduler["infer_shift"]),
327
+ action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]),
328
+ loss_lambda_video=float(loss.get("lambda_video", 1.0)),
329
+ loss_lambda_action=float(loss.get("lambda_action", 1.0)),
330
+ )
331
+
332
+
333
+ def build_datasets(data_cfg: DictConfig):
334
+ train_ds = instantiate(data_cfg.train)
335
+ if data_cfg.get("val") is None:
336
+ val_ds = train_ds
337
+ else:
338
+ train_stats_path = data_cfg.train.get("pretrained_norm_stats")
339
+ default_stats_path = os.path.join(misc.get_work_dir(), "dataset_stats.json")
340
+ val_stats_path = data_cfg.val.get("pretrained_norm_stats")
341
+ pretrained_norm_stats = val_stats_path or train_stats_path or default_stats_path
342
+ logger.info("Building val dataset with pretrained_norm_stats: %s", pretrained_norm_stats)
343
+ val_ds = instantiate(data_cfg.val, pretrained_norm_stats=pretrained_norm_stats)
344
+ return train_ds, val_ds
345
+
346
+
347
+ def _resolve_train_device() -> str:
348
+ if not torch.cuda.is_available():
349
+ return "cpu"
350
+ device_count = torch.cuda.device_count()
351
+ if device_count <= 1:
352
+ return "cuda:0"
353
+ local_rank = int(os.environ.get("LOCAL_RANK", "0"))
354
+ if local_rank < 0 or local_rank >= device_count:
355
+ return "cuda:0"
356
+ return f"cuda:{local_rank}"
357
+
358
+
359
+ def run_training(cfg: DictConfig):
360
+ setup_logging(
361
+ log_level=logging.INFO,
362
+ is_main_process=torch.distributed.get_rank() == 0 if torch.distributed.is_initialized() else True,
363
+ )
364
+ misc.register_work_dir(cfg.output_dir)
365
+ config_payload = OmegaConf.to_container(cfg, resolve=True)
366
+ with open(Path(cfg.output_dir) / "config.yaml", "w") as f:
367
+ OmegaConf.save(config_payload, f)
368
+
369
+ model_device = _resolve_train_device()
370
+ mixed_precision = _normalize_mixed_precision(cfg.mixed_precision)
371
+ model_dtype = _mixed_precision_to_model_dtype(mixed_precision)
372
+ model = instantiate(cfg.model, model_dtype=model_dtype, device=model_device)
373
+ train_ds, val_ds = build_datasets(cfg.data)
374
+
375
+ trainer = Wan22Trainer(
376
+ cfg=cfg,
377
+ model=model,
378
+ train_dataset=train_ds,
379
+ val_dataset=val_ds,
380
+ )
381
+ trainer.train()
382
+
383
+ def run_inference(cfg: DictConfig):
384
+ setup_logging(log_level=logging.INFO)
385
+ inference_cfg = cfg.inference
386
+ mixed_precision = _normalize_mixed_precision(cfg.mixed_precision)
387
+ model_dtype = _mixed_precision_to_model_dtype(mixed_precision)
388
+
389
+ model = instantiate(cfg.model, model_dtype=model_dtype, device=str(inference_cfg.device))
390
+ checkpoint_path = inference_cfg.get("checkpoint_path")
391
+ if checkpoint_path:
392
+ ckpt = Path(checkpoint_path)
393
+ if ckpt.exists():
394
+ logger.info("Loading finetuned checkpoint: %s", checkpoint_path)
395
+ model.load_checkpoint(checkpoint_path)
396
+ else:
397
+ logger.warning("Checkpoint not found, skipping load: %s", checkpoint_path)
398
+ model.eval()
399
+
400
+ def center_crop_resize(img: Image, width: int, height: int) -> Image.Image:
401
+ src_w, src_h = img.size
402
+ scale = max(width / src_w, height / src_h)
403
+ resized = img.resize((round(src_w * scale), round(src_h * scale)), resample=Image.BILINEAR)
404
+ rw, rh = resized.size
405
+ left = max((rw - width) // 2, 0)
406
+ top = max((rh - height) // 2, 0)
407
+ return resized.crop((left, top, left + width, top + height))
408
+
409
+ input_image = Image.open(str(inference_cfg.input_image_path)).convert("RGB")
410
+ input_image = center_crop_resize(input_image, width=inference_cfg.width, height=inference_cfg.height)
411
+ arr = np.array(input_image, dtype=np.float32)
412
+ x = torch.from_numpy(arr)
413
+ x = x.to(device=model.device, dtype=model.torch_dtype)
414
+ x = x * (2.0 / 255.0) - 1.0
415
+ x = repeat(x, "H W C -> B C H W", B=1)
416
+ output_mp4 = str(inference_cfg.output_mp4)
417
+
418
+ infer_kwargs = {
419
+ "prompt": str(inference_cfg.prompt),
420
+ "negative_prompt": str(inference_cfg.negative_prompt),
421
+ "text_cfg_scale": float(inference_cfg.text_cfg_scale),
422
+ "action_cfg_scale": float(inference_cfg.action_cfg_scale),
423
+ "input_image": x,
424
+ "num_frames": int(inference_cfg.num_frames),
425
+ "num_inference_steps": int(inference_cfg.num_inference_steps),
426
+ "sigma_shift": None if inference_cfg.get("sigma_shift") is None else float(inference_cfg.sigma_shift),
427
+ "seed": int(inference_cfg.seed),
428
+ "rand_device": str(inference_cfg.rand_device),
429
+ "tiled": bool(inference_cfg.tiled),
430
+ }
431
+
432
+ infer_out = model.infer(**infer_kwargs)
433
+ video = infer_out["video"]
434
+ save_mp4(video, output_mp4, fps=15)
435
+ logger.info("Saved inference video to %s", output_mp4)
436
+ return output_mp4
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/trainer.py ADDED
@@ -0,0 +1,791 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ import json
3
+ import inspect
4
+ import os
5
+ import re
6
+ from math import ceil
7
+ from pathlib import Path
8
+ import time
9
+
10
+ import numpy as np
11
+ import torch
12
+ from accelerate import Accelerator
13
+ from omegaconf import DictConfig
14
+ from PIL import Image
15
+ from torch.optim.lr_scheduler import ConstantLR, CosineAnnealingLR, LinearLR, SequentialLR
16
+ from torch.utils.data import DataLoader
17
+
18
+ from .utils.fs import ensure_dir
19
+ from .utils.logging_config import get_logger, setup_logging
20
+ from .utils.pytorch_utils import set_global_seed
21
+ from .utils.samplers import ResumableEpochSampler
22
+ from .utils.video_io import save_mp4
23
+ from .utils.video_metrics import pil_frames_to_video_tensor, video_psnr, video_ssim
24
+
25
+ logger = get_logger(__name__)
26
+
27
+
28
+ class Wan22Trainer:
29
+ def __init__(self, model, train_dataset, val_dataset=None, *, cfg: DictConfig):
30
+ self.model = model
31
+ self.train_dataset = train_dataset
32
+ self.val_dataset = val_dataset
33
+ self.cfg = cfg
34
+ self.output_dir = str(cfg.output_dir)
35
+ self.learning_rate = float(cfg.learning_rate)
36
+ self.weight_decay = float(cfg.weight_decay)
37
+ self.batch_size = int(cfg.batch_size)
38
+ self.num_workers = int(cfg.num_workers)
39
+ self.num_epochs = int(cfg.num_epochs)
40
+ max_steps = cfg.max_steps
41
+ self.max_steps = int(max_steps) if max_steps is not None else None
42
+ self.log_every = int(cfg.log_every)
43
+ self.save_every = int(cfg.save_every)
44
+ self.eval_every = int(cfg.eval_every)
45
+ self.eval_num_inference_steps = int(cfg.eval_num_inference_steps)
46
+ self.gradient_accumulation_steps = int(cfg.gradient_accumulation_steps)
47
+ self.max_grad_norm = float(cfg.max_grad_norm)
48
+ self.seed = int(cfg.seed)
49
+
50
+ self.resume = cfg.resume
51
+ self.mixed_precision = str(cfg.mixed_precision).strip().lower()
52
+ if self.mixed_precision not in {"no", "fp16", "bf16"}:
53
+ raise ValueError(
54
+ f"Unsupported mixed_precision: {cfg.mixed_precision}. "
55
+ "Expected one of: ['no', 'fp16', 'bf16']."
56
+ )
57
+ self.wandb_enabled = bool(cfg.wandb.enabled)
58
+
59
+ self.accelerator = Accelerator(
60
+ gradient_accumulation_steps=self.gradient_accumulation_steps,
61
+ mixed_precision=self.mixed_precision,
62
+ step_scheduler_with_optimizer=False,
63
+ )
64
+
65
+ logger.info(
66
+ "Accelerate training: distributed_type=%s zero_stage=%s world_size=%d process_index=%d cfg_mixed_precision=%s accelerator_mixed_precision=%s grad_accum=%d grad_clip=%.4f",
67
+ self.accelerator.distributed_type,
68
+ self.accelerator.state.deepspeed_plugin.deepspeed_config.get("zero_optimization", {}).get("stage", "unknown"),
69
+ self.accelerator.num_processes,
70
+ self.accelerator.process_index,
71
+ self.mixed_precision,
72
+ self.accelerator.mixed_precision,
73
+ self.gradient_accumulation_steps,
74
+ self.max_grad_norm,
75
+ )
76
+ logger.info("using accelerator.device=%s", self.accelerator.device)
77
+ worker_init_fn = set_global_seed(self.seed, get_worker_init_fn=True)
78
+ self._assert_dataset_length_consistent(self.train_dataset, "train_dataset")
79
+ if self.val_dataset is not None:
80
+ self._assert_dataset_length_consistent(self.val_dataset, "val_dataset")
81
+
82
+ # Freeze non-trainable modules before optimizer/deepspeed initialization.
83
+ # This keeps DiT (+ optional proprio encoder) as trainable when ZeRO builds optimizer state.
84
+ self._apply_dit_only_train_mode(self.model)
85
+ trainable_params = list(self.model.dit.parameters())
86
+ proprio_encoder = getattr(self.model, "proprio_encoder", None)
87
+ if proprio_encoder is not None:
88
+ trainable_params.extend(list(proprio_encoder.parameters()))
89
+ self.optimizer = torch.optim.AdamW(
90
+ trainable_params,
91
+ lr=self.learning_rate,
92
+ weight_decay=self.weight_decay,
93
+ betas=(0.9, 0.95),
94
+ )
95
+
96
+ self.train_loader = self._build_loader(self.train_dataset, worker_init_fn=worker_init_fn)
97
+ total_train_steps = self._estimate_total_train_steps()
98
+ self.max_steps = total_train_steps
99
+ warmup_steps = int(total_train_steps * 0.05)
100
+ self.scheduler = self._build_scheduler(
101
+ scheduler_type=cfg.lr_scheduler_type,
102
+ total_train_steps=total_train_steps,
103
+ warmup_steps=warmup_steps,
104
+ )
105
+ self.global_step = 0
106
+ self.epoch = 0
107
+ self.batch_in_epoch = 0
108
+
109
+ self.checkpoint_root = os.path.join(self.output_dir, "checkpoints")
110
+ self.weights_dir = os.path.join(self.checkpoint_root, "weights")
111
+ self.state_dir = os.path.join(self.checkpoint_root, "state")
112
+ self.eval_dir = os.path.join(self.output_dir, "eval")
113
+
114
+ ensure_dir(self.output_dir)
115
+ ensure_dir(self.checkpoint_root)
116
+ ensure_dir(self.weights_dir)
117
+ ensure_dir(self.state_dir)
118
+ ensure_dir(self.eval_dir)
119
+
120
+ self.model, self.optimizer, self.train_loader, self.scheduler = self.accelerator.prepare(
121
+ self.model, self.optimizer, self.train_loader, self.scheduler
122
+ )
123
+ self.optimizer.zero_grad(set_to_none=True)
124
+ self.wandb_run = None
125
+ self._init_wandb()
126
+ self._resume_or_load_checkpoint()
127
+
128
+ val_size = len(self.val_dataset) if self.val_dataset is not None else len(self.train_dataset)
129
+ logger.info("Train/val dataset size: %d/%d", len(self.train_dataset), val_size)
130
+
131
+ def _init_wandb(self):
132
+ if not self.wandb_enabled or not self.accelerator.is_main_process:
133
+ return
134
+ try:
135
+ import wandb
136
+ except ImportError as e:
137
+ raise ImportError(
138
+ "wandb logging is enabled in config (`wandb.enabled=true`) but wandb is not installed."
139
+ ) from e
140
+
141
+ self.wandb_run = wandb.init(
142
+ entity=self.cfg.wandb.workspace,
143
+ project=self.cfg.wandb.project,
144
+ name=self.cfg.wandb.name,
145
+ group=None if self.cfg.wandb.group in (None, "null", "") else str(self.cfg.wandb.group),
146
+ mode=self.cfg.wandb.mode,
147
+ dir=self.output_dir,
148
+ )
149
+ logger.info(
150
+ "Initialized wandb run: workspace=%s project=%s name=%s",
151
+ self.cfg.wandb.workspace,
152
+ self.cfg.wandb.project,
153
+ self.cfg.wandb.name,
154
+ )
155
+
156
+ def _wandb_log(self, payload: dict):
157
+ if self.wandb_run is None:
158
+ return
159
+ self.wandb_run.log(payload, step=self.global_step)
160
+
161
+ def _finish_wandb(self):
162
+ if self.wandb_run is None:
163
+ return
164
+ self.wandb_run.finish()
165
+ self.wandb_run = None
166
+
167
+ def _build_loader(self, dataset, worker_init_fn=None):
168
+ self.train_sampler = ResumableEpochSampler(
169
+ dataset=dataset,
170
+ seed=self.seed,
171
+ batch_size=self.batch_size,
172
+ num_processes=self.accelerator.num_processes,
173
+ )
174
+ return DataLoader(
175
+ dataset,
176
+ batch_size=self.batch_size,
177
+ shuffle=False,
178
+ sampler=self.train_sampler,
179
+ num_workers=self.num_workers,
180
+ pin_memory=torch.cuda.is_available(),
181
+ worker_init_fn=worker_init_fn,
182
+ )
183
+
184
+ def _assert_dataset_length_consistent(self, dataset, dataset_name: str):
185
+ if not hasattr(dataset, "__len__"):
186
+ raise TypeError(f"`{dataset_name}` must implement __len__ for rank consistency checks.")
187
+
188
+ local_length = len(dataset)
189
+ gathered_lengths = self.accelerator.gather(
190
+ torch.tensor([local_length], device=self.accelerator.device, dtype=torch.int64)
191
+ ).reshape(-1)
192
+ if torch.all(gathered_lengths == gathered_lengths[0]):
193
+ return
194
+
195
+ if self.accelerator.is_main_process:
196
+ print(f"[dataset-check] {dataset_name} length mismatch across ranks after initialization:")
197
+ for rank, rank_length in enumerate(gathered_lengths.cpu().tolist()):
198
+ print(f"rank {rank}: {rank_length}")
199
+ self.accelerator.wait_for_everyone()
200
+ raise RuntimeError(
201
+ f"{dataset_name} length mismatch across ranks: {gathered_lengths.cpu().tolist()}"
202
+ )
203
+
204
+ def _estimate_total_train_steps(self) -> int:
205
+ if self.max_steps is not None:
206
+ return max(int(self.max_steps), 1)
207
+
208
+ if not hasattr(self.train_dataset, "__len__"):
209
+ raise TypeError("`train_dataset` must implement __len__ when `max_steps` is None.")
210
+
211
+ num_processes = max(int(self.accelerator.num_processes), 1)
212
+ global_batch_size = max(self.batch_size * num_processes, 1)
213
+ micro_steps_per_epoch = max(ceil(len(self.train_dataset) / global_batch_size), 1)
214
+ opt_steps_per_epoch = max(
215
+ ceil(micro_steps_per_epoch / self.gradient_accumulation_steps),
216
+ 1,
217
+ )
218
+ return max(opt_steps_per_epoch * self.num_epochs, 1)
219
+
220
+ def _build_scheduler(self, scheduler_type, total_train_steps: int, warmup_steps: int = 0):
221
+ scheduler_type = str(scheduler_type).strip().lower()
222
+ total_train_steps = max(int(total_train_steps), 1)
223
+ warmup_steps = min(max(int(warmup_steps), 0), total_train_steps - 1)
224
+
225
+ remaining_steps = max(total_train_steps - warmup_steps, 1)
226
+ if scheduler_type == "cosine":
227
+ main_scheduler = CosineAnnealingLR(
228
+ self.optimizer,
229
+ T_max=remaining_steps,
230
+ eta_min=self.learning_rate * 0.01,
231
+ )
232
+ elif scheduler_type == "constant":
233
+ main_scheduler = ConstantLR(self.optimizer, factor=1.0, total_iters=remaining_steps)
234
+ else:
235
+ raise ValueError(
236
+ f"Unsupported lr_scheduler_type: {scheduler_type}. "
237
+ "Expected one of: ['cosine', 'constant']."
238
+ )
239
+
240
+ if warmup_steps <= 0:
241
+ return main_scheduler
242
+
243
+ warmup_scheduler = LinearLR(
244
+ self.optimizer,
245
+ start_factor=1.0 / warmup_steps,
246
+ end_factor=1.0,
247
+ total_iters=warmup_steps,
248
+ )
249
+ return SequentialLR(
250
+ self.optimizer,
251
+ schedulers=[warmup_scheduler, main_scheduler],
252
+ milestones=[warmup_steps],
253
+ )
254
+
255
+ def _estimate_eta(self):
256
+ elapsed = max(time.perf_counter() - self.run_start_time, 1e-6)
257
+ done_steps = max(self.global_step - self.run_start_step, 1)
258
+ steps_per_sec = done_steps / elapsed
259
+ remaining_steps = max(self.max_steps - self.global_step, 0)
260
+ eta_seconds = int(remaining_steps / max(steps_per_sec, 1e-9))
261
+ eta_h, eta_rem = divmod(eta_seconds, 3600)
262
+ eta_m, eta_s = divmod(eta_rem, 60)
263
+ return f"{eta_h:02d}:{eta_m:02d}:{eta_s:02d}", steps_per_sec
264
+
265
+ def _resume_or_load_checkpoint(self):
266
+ resume = self.resume
267
+ if not resume:
268
+ return
269
+ resume_path = Path(str(resume))
270
+ if resume_path.is_dir():
271
+ logger.info("Resuming full training state from directory: %s", resume)
272
+ self.load_training_state(str(resume_path))
273
+ return
274
+ if not resume_path.exists():
275
+ raise FileNotFoundError(f"Resume checkpoint not found: {resume}")
276
+ logger.info("Loading weight checkpoint only: %s", resume)
277
+ self.accelerator.unwrap_model(self.model).load_checkpoint(str(resume_path), optimizer=None)
278
+ logger.warning("Loaded .pt weights only; optimizer/scheduler/step were not restored under ZeRO2.")
279
+
280
+ def _set_dit_only_train_mode(self):
281
+ # Match DiffSynth's freeze_except("dit"): only DiT stays trainable/in-train-mode.
282
+ logger.info("Setting DiT to train mode and freezing other model components.")
283
+ model = self.accelerator.unwrap_model(self.model)
284
+ self._apply_dit_only_train_mode(model)
285
+
286
+ @staticmethod
287
+ def _apply_dit_only_train_mode(model):
288
+ model.eval()
289
+ model.requires_grad_(False)
290
+ model.dit.train()
291
+ model.dit.requires_grad_(True)
292
+ proprio_encoder = getattr(model, "proprio_encoder", None)
293
+ if proprio_encoder is not None:
294
+ proprio_encoder.train()
295
+ proprio_encoder.requires_grad_(True)
296
+
297
+ @staticmethod
298
+ def _to_batched_eval_sample(sample):
299
+ video = sample["video"]
300
+ prompt = sample["prompt"]
301
+ action = sample.get("action", None)
302
+ proprio = sample.get("proprio", None)
303
+ context = sample.get("context", None)
304
+ context_mask = sample.get("context_mask", None)
305
+
306
+ if not isinstance(video, torch.Tensor):
307
+ raise TypeError(
308
+ f"Expected tensor video for evaluation, got {type(video)}. "
309
+ "Evaluation now expects `video` with shape [3,T,H,W] or [B,3,T,H,W]."
310
+ )
311
+ if video.ndim == 4:
312
+ video = video.unsqueeze(0)
313
+ if video.ndim != 5:
314
+ raise ValueError(f"Expected video shape [3,T,H,W] or [B,3,T,H,W], got {tuple(video.shape)}")
315
+ num_video_frames = video.shape[2]
316
+ if num_video_frames <= 1:
317
+ raise ValueError(f"`sample['video']` must have at least 2 frames for action evaluation, got {num_video_frames}")
318
+
319
+ if isinstance(prompt, str):
320
+ prompt = [prompt]
321
+ elif isinstance(prompt, tuple):
322
+ prompt = list(prompt)
323
+ elif not isinstance(prompt, list):
324
+ raise TypeError(f"Expected prompt type str/list[str], got {type(prompt)}")
325
+ if len(prompt) != video.shape[0]:
326
+ raise ValueError(f"Prompt batch mismatch: len(prompt)={len(prompt)} vs video batch={video.shape[0]}")
327
+
328
+ action_horizon = None
329
+ action = None
330
+ if "action" in sample:
331
+ action = sample["action"]
332
+ if not isinstance(action, torch.Tensor):
333
+ raise TypeError(
334
+ f"`sample['action']` must be a torch.Tensor, got {type(action)}"
335
+ )
336
+ if action.ndim == 2:
337
+ action = action.unsqueeze(0)
338
+ if action.ndim != 3:
339
+ raise ValueError(f"`sample['action']` must be 3D [B, T, a_dim], got shape {tuple(action.shape)}")
340
+ if action.shape[1] % (num_video_frames - 1) != 0:
341
+ raise ValueError(f"`sample['action']` temporal dimension must be divisible by video frames-1={num_video_frames - 1}, got {action.shape[1]}")
342
+ action_horizon = int(action.shape[1])
343
+
344
+ proprio = None
345
+ if "proprio" in sample:
346
+ proprio = sample["proprio"]
347
+ if not isinstance(proprio, torch.Tensor):
348
+ raise TypeError(f"`sample['proprio']` must be a torch.Tensor, got {type(proprio)}")
349
+ if proprio.ndim == 2:
350
+ proprio = proprio.unsqueeze(0)
351
+ if proprio.ndim != 3:
352
+ raise ValueError(f"`sample['proprio']` must be 3D [B, T, d], got shape {tuple(proprio.shape)}")
353
+
354
+ if context is not None or context_mask is not None:
355
+ if context is None or context_mask is None:
356
+ raise ValueError("`context` and `context_mask` must both exist in eval sample.")
357
+ if context.ndim == 2:
358
+ context = context.unsqueeze(0)
359
+ if context_mask.ndim == 1:
360
+ context_mask = context_mask.unsqueeze(0)
361
+ if context.ndim != 3 or context_mask.ndim != 2:
362
+ raise ValueError(
363
+ f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}"
364
+ )
365
+
366
+ return {
367
+ "video": video,
368
+ "prompt": prompt,
369
+ "action": action,
370
+ "proprio": proprio,
371
+ "context": context,
372
+ "context_mask": context_mask,
373
+ "action_horizon": action_horizon,
374
+ }
375
+
376
+ @torch.no_grad()
377
+ def evaluate(self):
378
+ if self.val_dataset is None:
379
+ return None
380
+
381
+ model = self.accelerator.unwrap_model(self.model)
382
+ was_dit_training = model.dit.training
383
+ model.eval()
384
+
385
+ # eval_index = (self.global_step + self.accelerator.process_index) % len(self.val_dataset)
386
+ rng = torch.Generator(device="cpu").manual_seed(self.global_step + self.accelerator.process_index)
387
+ eval_index = torch.randint(0, len(self.val_dataset), (1,), generator=rng).item()
388
+ sample = self._to_batched_eval_sample(self.val_dataset[eval_index])
389
+
390
+ # 1. training loss
391
+ with self.accelerator.autocast():
392
+ val_loss, _ = model.training_loss(sample)
393
+ val_loss = val_loss.float().item()
394
+
395
+ prompt = sample["prompt"][0]
396
+ video0 = sample["video"][0] # Tensor [3, T, H, W] in (-1, 1)
397
+ action = sample["action"][0] if "action" in sample and sample["action"] is not None else None
398
+ proprio = sample["proprio"][0, 0] if "proprio" in sample and sample["proprio"] is not None else None # from [1, T, d] to [d]
399
+ input_image = video0[:, 0].unsqueeze(0)
400
+ _, num_frames, _, _ = video0.shape
401
+
402
+ # 2. inference and video saving
403
+ infer_kwargs = {
404
+ "input_image": input_image,
405
+ "num_frames": num_frames,
406
+ "action": action,
407
+ "action_horizon": sample['action_horizon'],
408
+ "proprio": proprio,
409
+ "text_cfg_scale": 1.0,
410
+ "action_cfg_scale": 1.0,
411
+ "num_inference_steps": self.eval_num_inference_steps,
412
+ "seed": 42,
413
+ "tiled": False,
414
+ }
415
+ if sample["context"] is not None:
416
+ infer_kwargs["prompt"] = None
417
+ infer_kwargs["context"] = sample["context"][0]
418
+ infer_kwargs["context_mask"] = sample["context_mask"][0]
419
+ else:
420
+ infer_kwargs["prompt"] = prompt
421
+
422
+ pred = model.infer(
423
+ **infer_kwargs,
424
+ )
425
+
426
+ pred_video = pred["video"]
427
+ pred_action = pred.get("action", None)
428
+
429
+ # 3. inference metrics against GT video
430
+ pred_video_tensor = pil_frames_to_video_tensor(pred_video)
431
+ gt_video_tensor = ((video0.detach().float().cpu().clamp(-1.0, 1.0) + 1.0) * 0.5).contiguous()
432
+
433
+ assert pred_video_tensor.shape == gt_video_tensor.shape, (
434
+ "Eval infer prediction/GT shape mismatch: "
435
+ f"pred={tuple(pred_video_tensor.shape)} vs gt={tuple(gt_video_tensor.shape)}"
436
+ )
437
+
438
+ psnr_rollout_vs_gt = video_psnr(pred=pred_video_tensor, target=gt_video_tensor)
439
+ ssim_rollout_vs_gt = video_ssim(pred=pred_video_tensor, target=gt_video_tensor)
440
+
441
+ action_l1 = None
442
+ action_l2 = None
443
+ if action is not None and pred_action is not None:
444
+ if sample["proprio"] is None:
445
+ raise ValueError("Eval sample must contain `proprio` for action denormalization.")
446
+ proprio = sample["proprio"].detach().to(device="cpu", dtype=torch.float32)
447
+
448
+ processor = self.val_dataset.lerobot_dataset.processor
449
+
450
+ denorm_actions = {}
451
+ action_meta = processor.shape_meta["action"]
452
+ state_meta = processor.shape_meta["state"]
453
+ for action_name, raw_action in (("pred", pred_action), ("gt", action)):
454
+ if not isinstance(raw_action, torch.Tensor):
455
+ raise TypeError(f"{action_name} action must be a torch.Tensor, got {type(raw_action)}")
456
+ if raw_action.ndim == 2:
457
+ action_btd = raw_action.unsqueeze(0)
458
+ elif raw_action.ndim == 3 and raw_action.shape[0] == 1:
459
+ action_btd = raw_action
460
+ else:
461
+ raise ValueError(
462
+ f"{action_name} action must have shape [T, D] or [1, T, D], got {tuple(raw_action.shape)}"
463
+ )
464
+ action_btd = action_btd.detach().to(device="cpu", dtype=torch.float32)
465
+
466
+ batch = {
467
+ "action": action_btd,
468
+ "state": proprio,
469
+ }
470
+ batch = processor.action_state_merger.backward(batch)
471
+ batch = processor.normalizer.backward(batch)
472
+ merged_batch = {
473
+ "action": {meta["key"]: batch["action"][meta["key"]].squeeze(0) for meta in action_meta},
474
+ "state": {meta["key"]: batch["state"][meta["key"]].squeeze(0) for meta in state_meta},
475
+ }
476
+ merged_batch = processor.action_state_merger.forward(merged_batch)
477
+ denorm_action = merged_batch["action"].unsqueeze(0)
478
+ if denorm_action.ndim != 3 or denorm_action.shape[0] != 1:
479
+ raise ValueError(
480
+ f"Denormalized {action_name} action must have shape [1, T, D], got {tuple(denorm_action.shape)}"
481
+ )
482
+ denorm_actions[action_name] = denorm_action
483
+
484
+ pred_action_denorm = denorm_actions["pred"]
485
+ gt_action_denorm = denorm_actions["gt"]
486
+
487
+ if pred_action_denorm.shape != gt_action_denorm.shape:
488
+ raise ValueError(
489
+ "Predicted action/GT action shape mismatch after denormalization: "
490
+ f"pred={tuple(pred_action_denorm.shape)} vs gt={tuple(gt_action_denorm.shape)}"
491
+ )
492
+ action_diff = pred_action_denorm - gt_action_denorm
493
+ action_l1 = action_diff.abs().mean().item()
494
+ action_l2 = action_diff.pow(2).mean().item()
495
+
496
+ # 4. VAE reconstruction metrics against GT video
497
+ gt_video_batch = video0.unsqueeze(0).to(device=model.device, dtype=model.torch_dtype)
498
+ vae_latents = model._encode_video_latents(gt_video_batch, tiled=False)
499
+ vae_recon_video = model._decode_latents(vae_latents, tiled=False)
500
+ vae_video_tensor = pil_frames_to_video_tensor(vae_recon_video)
501
+
502
+ assert vae_video_tensor.shape == gt_video_tensor.shape, (
503
+ "Eval VAE reconstruction/GT shape mismatch: "
504
+ f"vae={tuple(vae_video_tensor.shape)} vs gt={tuple(gt_video_tensor.shape)}"
505
+ )
506
+
507
+ psnr_decode_vs_gt = video_psnr(pred=vae_video_tensor, target=gt_video_tensor)
508
+ ssim_decode_vs_gt = video_ssim(pred=vae_video_tensor, target=gt_video_tensor)
509
+
510
+ psnr_rollout_vs_decode = video_psnr(pred=pred_video_tensor, target=vae_video_tensor)
511
+ ssim_rollout_vs_decode = video_ssim(pred=pred_video_tensor, target=vae_video_tensor)
512
+
513
+ stitched_video_tensor = torch.cat(
514
+ [pred_video_tensor, vae_video_tensor, gt_video_tensor],
515
+ dim=2,
516
+ ).contiguous()
517
+ stitched_frames = []
518
+ for t in range(stitched_video_tensor.shape[1]):
519
+ frame = (stitched_video_tensor[:, t].permute(1, 2, 0).clamp(0.0, 1.0).numpy() * 255.0).astype(np.uint8)
520
+ stitched_frames.append(Image.fromarray(frame))
521
+
522
+ video_path = os.path.join(
523
+ self.eval_dir,
524
+ f"step_{self.global_step:06d}_rank_{self.accelerator.process_index:03d}.mp4",
525
+ )
526
+ save_mp4(stitched_frames, video_path, fps=8)
527
+
528
+ local_metrics = torch.tensor(
529
+ [
530
+ float(val_loss),
531
+ float(psnr_rollout_vs_gt),
532
+ float(ssim_rollout_vs_gt),
533
+ float(psnr_rollout_vs_decode),
534
+ float(ssim_rollout_vs_decode),
535
+ float(psnr_decode_vs_gt),
536
+ float(ssim_decode_vs_gt),
537
+ float(action_l2) if action_l2 is not None else -1.0,
538
+ float(action_l1) if action_l1 is not None else -1.0,
539
+ ],
540
+ device=self.accelerator.device,
541
+ dtype=torch.float32,
542
+ ).unsqueeze(0)
543
+ gathered_metrics = self.accelerator.gather_for_metrics(local_metrics)
544
+ mean_metrics = gathered_metrics[:, :7].mean(dim=0)
545
+ action_l2_mean = gathered_metrics[:, 7].mean().item() if action_l2 is not None else None
546
+ action_l1_mean = gathered_metrics[:, 8].mean().item() if action_l1 is not None else None
547
+
548
+ if was_dit_training:
549
+ self._set_dit_only_train_mode()
550
+
551
+ result = {
552
+ "val_loss": float(mean_metrics[0].item()),
553
+ "psnr_rg": float(mean_metrics[1].item()),
554
+ "ssim_rg": float(mean_metrics[2].item()),
555
+ "psnr_rd": float(mean_metrics[3].item()),
556
+ "ssim_rd": float(mean_metrics[4].item()),
557
+ "psnr_dg": float(mean_metrics[5].item()),
558
+ "ssim_dg": float(mean_metrics[6].item()),
559
+ "video_path": video_path,
560
+ }
561
+ if action_l2_mean is not None:
562
+ result["action_l2"] = float(action_l2_mean)
563
+ if action_l1_mean is not None:
564
+ result["action_l1"] = float(action_l1_mean)
565
+ return result
566
+
567
+ def _save_weights_checkpoint(self, step_tag: str):
568
+ model = self.accelerator.unwrap_model(self.model)
569
+ ckpt_path = os.path.join(self.weights_dir, f"{step_tag}.pt")
570
+ model.save_checkpoint(ckpt_path, optimizer=None, step=self.global_step)
571
+ return ckpt_path
572
+
573
+ def _save_trainer_state(self, state_path: str):
574
+ state_file = os.path.join(state_path, "trainer_state.json")
575
+ payload = {
576
+ "global_step": int(self.global_step),
577
+ "epoch": int(self.epoch),
578
+ "batch_in_epoch": int(self.batch_in_epoch),
579
+ }
580
+ with open(state_file, "w", encoding="utf-8") as f:
581
+ json.dump(payload, f, ensure_ascii=True, indent=2)
582
+
583
+ def save_checkpoint(self):
584
+ step_tag = f"step_{self.global_step:06d}"
585
+
586
+ self.accelerator.wait_for_everyone()
587
+ ckpt_path = None
588
+ if self.accelerator.is_main_process:
589
+ ckpt_path = self._save_weights_checkpoint(step_tag=step_tag)
590
+ self.accelerator.wait_for_everyone()
591
+
592
+ state_path = os.path.join(self.state_dir, step_tag)
593
+ ensure_dir(state_path)
594
+ self.accelerator.save_state(output_dir=state_path)
595
+ if self.accelerator.is_main_process:
596
+ self._save_trainer_state(state_path)
597
+ self.accelerator.wait_for_everyone()
598
+
599
+ return {"weights_path": ckpt_path, "state_path": state_path}
600
+
601
+ def load_training_state(self, state_dir: str):
602
+ self.accelerator.load_state(input_dir=state_dir)
603
+ state_file = Path(state_dir) / "trainer_state.json"
604
+ if state_file.exists():
605
+ with open(state_file, "r", encoding="utf-8") as f:
606
+ payload = json.load(f)
607
+ self.global_step = int(payload["global_step"])
608
+
609
+ if "epoch" in payload and "batch_in_epoch" in payload:
610
+ self.epoch = int(payload["epoch"])
611
+ self.batch_in_epoch = int(payload["batch_in_epoch"])
612
+ self.train_sampler.set_epoch_offset(self.epoch)
613
+ self.train_sampler.set_resume_batch_offset(self.batch_in_epoch)
614
+ logger.info(
615
+ "Restored dataloader progress: epoch=%d batch_in_epoch=%d sample_offset=%d",
616
+ self.epoch,
617
+ self.batch_in_epoch,
618
+ self.batch_in_epoch * self.batch_size * self.accelerator.num_processes,
619
+ )
620
+ else:
621
+ self.epoch = 0
622
+ self.batch_in_epoch = 0
623
+ self.train_sampler.clear_resume_batch_offset()
624
+ logger.warning(
625
+ "State file does not contain `epoch`/`batch_in_epoch`; "
626
+ "optimizer/scheduler were restored, but dataloader progress resume is skipped."
627
+ )
628
+ self.accelerator.wait_for_everyone()
629
+ return
630
+
631
+ match = re.search(r"step[_-](\d+)$", str(state_dir).rstrip("/"))
632
+ if match:
633
+ self.global_step = int(match.group(1))
634
+ else:
635
+ self.global_step = 0
636
+ self.epoch = 0
637
+ self.batch_in_epoch = 0
638
+ self.train_sampler.clear_resume_batch_offset()
639
+ self.accelerator.wait_for_everyone()
640
+ logger.info("Loaded accelerate training state from %s at step=%d", state_dir, self.global_step)
641
+ logger.warning(
642
+ "State file `%s` is missing; dataloader progress resume is skipped.",
643
+ state_file,
644
+ )
645
+
646
+ def train(self):
647
+ self._set_dit_only_train_mode()
648
+
649
+ unwrapped_model = self.accelerator.unwrap_model(self.model)
650
+
651
+ if self.max_steps is None:
652
+ raise ValueError("`max_steps` must be set before entering the while-step training loop.")
653
+
654
+ logger.info("Starting training with max_steps=%d.", self.max_steps)
655
+ data_iter = iter(self.train_loader)
656
+ self.run_start_step = self.global_step
657
+ self.run_start_time = time.perf_counter()
658
+
659
+ while self.global_step < self.max_steps:
660
+ try:
661
+ sample = next(data_iter)
662
+ self.batch_in_epoch += 1
663
+ except StopIteration:
664
+ self.epoch += 1
665
+ self.batch_in_epoch = 0
666
+ self.train_sampler.clear_resume_batch_offset()
667
+ data_iter = iter(self.train_loader)
668
+ continue
669
+
670
+ with self.accelerator.accumulate(self.model):
671
+ train_model = self.model if hasattr(self.model, "training_loss") else self.accelerator.unwrap_model(self.model)
672
+
673
+ with self.accelerator.autocast():
674
+ loss, loss_dict = train_model.training_loss(sample)
675
+ self.accelerator.backward(loss)
676
+
677
+ if self.accelerator.sync_gradients:
678
+ grad_norm = self.accelerator.clip_grad_norm_(self.model.parameters(), self.max_grad_norm)
679
+ self.optimizer.step()
680
+ if not self.accelerator.optimizer_step_was_skipped:
681
+ self.scheduler.step()
682
+ self.optimizer.zero_grad(set_to_none=True)
683
+ self.global_step += 1
684
+ global_loss = float(
685
+ self.accelerator.gather(loss.detach().float().reshape(1)).mean().item()
686
+ )
687
+ global_loss_metrics = {}
688
+ for key, value in loss_dict.items():
689
+ metric_tensor = torch.tensor(float(value), device=loss.device, dtype=torch.float32).reshape(1)
690
+ global_loss_metrics[key] = float(
691
+ self.accelerator.gather(metric_tensor).mean().item()
692
+ )
693
+ grad_norm_tensor = torch.tensor(grad_norm, device=loss.device, dtype=torch.float32)
694
+ global_grad_norm = float(self.accelerator.gather(grad_norm_tensor).mean().item())
695
+
696
+ current_lr = float(self.optimizer.param_groups[0]["lr"])
697
+
698
+ if self.log_every > 0 and self.global_step % self.log_every == 0 and self.accelerator.is_main_process:
699
+ eta_str, steps_per_sec = self._estimate_eta()
700
+ description = "[train] epoch=%d step=%d/%d loss=%.4f " % (
701
+ self.epoch,
702
+ self.global_step,
703
+ self.max_steps,
704
+ global_loss,
705
+ )
706
+ if global_loss_metrics:
707
+ detail_str = " ".join([f"{k}={v:.4f}" for k, v in sorted(global_loss_metrics.items())])
708
+ description += detail_str + " "
709
+ description += "lr=%.2e speed=%.2f step/s, %.2f samples/s eta=%s" % (
710
+ current_lr,
711
+ steps_per_sec,
712
+ steps_per_sec * self.batch_size * self.accelerator.num_processes,
713
+ eta_str,
714
+ )
715
+ logger.info(description)
716
+
717
+ wandb_payload = {
718
+ "train/loss": global_loss,
719
+ "train/grad_norm": global_grad_norm,
720
+ "train/lr": current_lr,
721
+ "performance/steps_per_sec": steps_per_sec,
722
+ "performance/samples_per_sec": steps_per_sec * self.batch_size * self.accelerator.num_processes,
723
+ }
724
+ for key, value in global_loss_metrics.items():
725
+ wandb_payload[f"train/{key}"] = value
726
+ self._wandb_log(wandb_payload)
727
+
728
+ if (
729
+ self.eval_every > 0
730
+ and self.val_dataset is not None
731
+ and self.global_step % self.eval_every == 0
732
+ ):
733
+ metrics = self.evaluate()
734
+ self.accelerator.wait_for_everyone()
735
+ if metrics is not None and self.accelerator.is_main_process:
736
+ description = "[eval] step=%d val_loss=%.4f infer_psnr=%.4f infer_ssim=%.4f" % (
737
+ self.global_step,
738
+ metrics["val_loss"],
739
+ metrics["psnr_rd"],
740
+ metrics["ssim_rd"],
741
+ )
742
+ if "action_l2" in metrics:
743
+ description += " action_l2=%.4f" % metrics["action_l2"]
744
+ if "action_l1" in metrics:
745
+ description += " action_l1=%.4f" % metrics["action_l1"]
746
+ logger.info(description)
747
+ eval_payload = {
748
+ "eval/val_loss": float(metrics["val_loss"]),
749
+ "eval/psnr_rg": float(metrics["psnr_rg"]),
750
+ "eval/ssim_rg": float(metrics["ssim_rg"]),
751
+ "eval/psnr_rd": float(metrics["psnr_rd"]),
752
+ "eval/ssim_rd": float(metrics["ssim_rd"]),
753
+ "eval/psnr_dg": float(metrics["psnr_dg"]),
754
+ "eval/ssim_dg": float(metrics["ssim_dg"]),
755
+ }
756
+ if "action_l2" in metrics:
757
+ eval_payload["eval/action_l2"] = float(metrics["action_l2"])
758
+ if "action_l1" in metrics:
759
+ eval_payload["eval/action_l1"] = float(metrics["action_l1"])
760
+ self._wandb_log(eval_payload)
761
+
762
+ if self.save_every > 0 and self.global_step % self.save_every == 0:
763
+ ckpt_info = self.save_checkpoint()
764
+ if self.accelerator.is_main_process:
765
+ logger.info(
766
+ "[ckpt] step=%d weights=%s state=%s",
767
+ self.global_step,
768
+ ckpt_info["weights_path"],
769
+ ckpt_info["state_path"],
770
+ )
771
+
772
+ if self.global_step >= self.max_steps:
773
+ ckpt_info = self.save_checkpoint()
774
+ if self.accelerator.is_main_process:
775
+ logger.info(
776
+ "[done] max_steps reached step=%d weights=%s state=%s",
777
+ self.global_step,
778
+ ckpt_info["weights_path"],
779
+ ckpt_info["state_path"],
780
+ )
781
+ return
782
+
783
+ ckpt_info = self.save_checkpoint()
784
+ if self.accelerator.is_main_process:
785
+ logger.info(
786
+ "[done] training finished step=%d weights=%s state=%s",
787
+ self.global_step,
788
+ ckpt_info["weights_path"],
789
+ ckpt_info["state_path"],
790
+ )
791
+