Add final SpaceDreamer stack_cups clean40 30-epoch checkpoint and inference package
Browse filesFinal step4350 from bifrost-2026111502493001-shouqx. Original unchanged final_weights.pt, exact selected40 FP32 statistics, matching frozen source and inference adapter. The optimizer completed30epochs; platform failure was traced to a postflight BF16/FP32 comparison false positive. Changes confined to spacedreamer_stack_cups_clean40_30ep.
This view is limited to 50 files because it contains too many changes. See raw diff
- spacedreamer_stack_cups_clean40_30ep/CPU_validation.json +23 -0
- spacedreamer_stack_cups_clean40_30ep/README.md +100 -0
- spacedreamer_stack_cups_clean40_30ep/checkpoint_metadata.json +18 -0
- spacedreamer_stack_cups_clean40_30ep/final_weights.pt +3 -0
- spacedreamer_stack_cups_clean40_30ep/initialization.json +35 -0
- spacedreamer_stack_cups_clean40_30ep/manifest.json +928 -0
- spacedreamer_stack_cups_clean40_30ep/postflight_normalizer_audit.json +64 -0
- spacedreamer_stack_cups_clean40_30ep/reference_norm_stats.json +136 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/requirements.txt +22 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/run_inference.py +29 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/runtime_packages.json +161 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/__init__.py +1 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/__init__.py +0 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/dataset_utils.py +109 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/base_lerobot_dataset.py +412 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/constants.py +54 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py +179 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py +903 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py +527 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py +1304 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/base_processor.py +263 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/fastwam_processor.py +304 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/robot_video_dataset.py +279 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/action_state_merger.py +78 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/image.py +25 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/misc.py +20 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/relative_action.py +104 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/rotation.py +133 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/normalizer.py +291 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/rotation.py +621 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/__init__.py +1 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/__init__.py +0 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/action_dit.py +337 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/attention_masks.py +82 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam.py +1122 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_idm.py +452 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_joint.py +229 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/__init__.py +15 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/gradient.py +25 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/io.py +207 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/loader.py +221 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/state_dict_converters.py +90 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/mot.py +563 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/schedulers/scheduler_continuous.py +88 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan22.py +405 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_dit.py +673 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_text_encoder.py +342 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_vae.py +1384 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/runtime.py +436 -0
- spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/trainer.py +791 -0
spacedreamer_stack_cups_clean40_30ep/CPU_validation.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "PASS_CPU_STRICT_LOAD_EXACT_SELECTED40_NORMALIZATION_PROMPT_AND_FROZEN_SOURCE",
|
| 3 |
+
"checkpoint_step": 4350,
|
| 4 |
+
"epochs": 30,
|
| 5 |
+
"state_dict_keys": 1661,
|
| 6 |
+
"strict_load_missing_keys": [],
|
| 7 |
+
"strict_load_unexpected_keys": [],
|
| 8 |
+
"all_parameters_materialized": true,
|
| 9 |
+
"selected40_FP32_statistics_file_identical": true,
|
| 10 |
+
"training_and_adapter_normalize_decode_bitwise_equal": true,
|
| 11 |
+
"all4_quantized_buffers_correct": true,
|
| 12 |
+
"context_shape": [
|
| 13 |
+
128,
|
| 14 |
+
4096
|
| 15 |
+
],
|
| 16 |
+
"context_prompt_matches": true,
|
| 17 |
+
"shared_dependencies_at_revision": "64a7a5fad88d69ed041b00d233fc1d4374e4bf8e",
|
| 18 |
+
"shared_dependency_sha256": {
|
| 19 |
+
"spacedreamer_20epochs/assets/Wan2.2-TI2V-5B/Wan2.2_VAE.pth": "20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36",
|
| 20 |
+
"spacedreamer_20epochs/assets/metric_video_depth_anything_vitb.pth": "f6f58576b9680a112f2428d4f39aff92656d3ae85745b0164675ace8d5b1fade"
|
| 21 |
+
},
|
| 22 |
+
"scope": "CPU checkpoint reconstruction and packaging; no new GPU sampling or robot success claimed."
|
| 23 |
+
}
|
spacedreamer_stack_cups_clean40_30ep/README.md
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SpaceDreamer stack_cups — clean40,30 epochs
|
| 2 |
+
|
| 3 |
+
Final original `final_weights.pt` from **bifrost-2026111502493001-shouqx**: **30 epochs,4350 optimizer steps**,trained on the approved40 demonstrations from the new67 November3/4 stack_cups recordings (41 continuous segments,37273 unique rows). Fresh initialization from SpaceDreamer-v3 pretrained018126;16H200,global batch256. Right-arm6joint loss weight2;left joints and both grippers1. This is the new selected40-refit-normalization run.
|
| 4 |
+
|
| 5 |
+
The model is **RgbGeometryActionMoT**,joint RGB/Depth/dense-point-map/Action prediction. It predicts **32×14 absolute joint/gripper commands**,not the RGB-only FastWAM40-step model. All final model parameters are contained in this weight file; distributed optimizer/RNG training states are retained on the original training storage and are not part of this inference release.
|
| 6 |
+
|
| 7 |
+
## Files
|
| 8 |
+
|
| 9 |
+
- `final_weights.pt`: unchanged final training export (`state_dict`,`config`,`initialization`).
|
| 10 |
+
- `training_config.json`: original training config; historical absolute dataset/output paths are provenance and are not needed by the inference adapter.
|
| 11 |
+
- `reference_norm_stats.json`: **exact FP32 statistics refit on selected40 only**. Use these for state normalization and action decoding.
|
| 12 |
+
- `text_context.pt`: exact cached128-token training instruction,`Stack the cups.`
|
| 13 |
+
- `checkpoint_metadata.json`: training step,weight size and SHA-256.
|
| 14 |
+
- `runtime/spacedreamer_galexea.py`: inference adapter exposing `infer_action()` through the original joint sampler.
|
| 15 |
+
- `runtime/run_inference.py`: saved-observation prediction CLI,with no robot I/O.
|
| 16 |
+
- `runtime/source/`: matching frozen training source;`training_reference/code/`: exact training recipe.
|
| 17 |
+
- `postflight_normalizer_audit.json`: independent explanation of the platform failure after training.
|
| 18 |
+
- `CPU_validation.json`,`manifest.json`: packaging checks and exact file digests.
|
| 19 |
+
|
| 20 |
+
## Download this model and shared dependencies
|
| 21 |
+
|
| 22 |
+
VAE,VDA and optional text encoder already reside in this repository under `spacedreamer_20epochs/assets`. They are reused rather than duplicated in this new folder. The standard cached instruction needs VAE and,if depth is not provided,VDA. UMT5/tokenizer are needed only for a different instruction.
|
| 23 |
+
|
| 24 |
+
```python
|
| 25 |
+
from huggingface_hub import HfApi,snapshot_download
|
| 26 |
+
repo="qshou-coder/SpaceDreamer_Galexea"
|
| 27 |
+
prefix="spacedreamer_stack_cups_clean40_30ep"
|
| 28 |
+
shared="spacedreamer_20epochs/assets"
|
| 29 |
+
revision=HfApi().model_info(repo).sha # save/pin this revision
|
| 30 |
+
snapshot_download(
|
| 31 |
+
repo_id=repo,revision=revision,local_dir="SpaceDreamer_Galexea",
|
| 32 |
+
allow_patterns=[
|
| 33 |
+
f"{prefix}/**",
|
| 34 |
+
f"{shared}/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
|
| 35 |
+
f"{shared}/metric_video_depth_anything_vitb.pth",
|
| 36 |
+
f"{shared}/calibrations.json",
|
| 37 |
+
],
|
| 38 |
+
)
|
| 39 |
+
# For custom instructions,also download the existing shared UMT5 weights/tokenizer.
|
| 40 |
+
```
|
| 41 |
+
|
| 42 |
+
Known shared hashes:
|
| 43 |
+
|
| 44 |
+
- VAE: `20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36`.
|
| 45 |
+
- Metric VDA: `f6f58576b9680a112f2428d4f39aff92656d3ae85745b0164675ace8d5b1fade`.
|
| 46 |
+
|
| 47 |
+
The calibration manifest is historical training-camera metadata. No training datasets are required for prediction. Deployment camera geometry must correspond to the trained image/depth convention.
|
| 48 |
+
|
| 49 |
+
## Runtime and inference
|
| 50 |
+
|
| 51 |
+
`runtime/runtime_packages.json` records observed package versions. The checked runtime uses Python3.12,PyTorch2.9.0 and torchvision0.24.0. Install the CUDA build appropriate to your machine and the dependencies in `runtime/requirements.txt` in an isolated environment. The checkpoint uses BF16. No Wan diffusion backbone checkpoint is needed beyond this trained export and its VAE.
|
| 52 |
+
|
| 53 |
+
From the downloaded repository root:
|
| 54 |
+
|
| 55 |
+
```bash
|
| 56 |
+
export PYTHONPATH="$PWD/spacedreamer_stack_cups_clean40_30ep/runtime${PYTHONPATH:+:$PYTHONPATH}"
|
| 57 |
+
python spacedreamer_stack_cups_clean40_30ep/runtime/run_inference.py \
|
| 58 |
+
--task-dir spacedreamer_stack_cups_clean40_30ep \
|
| 59 |
+
--assets-dir spacedreamer_20epochs/assets \
|
| 60 |
+
--observation observation.npz \
|
| 61 |
+
--output predicted_actions.npy \
|
| 62 |
+
--steps 10 --seed 0
|
| 63 |
+
```
|
| 64 |
+
|
| 65 |
+
`observation.npz` contains your own `head_rgb` RGB,uint8 image and `state` float14;optionally `depth_m`. No training-camera images are published in this folder.
|
| 66 |
+
|
| 67 |
+
```python
|
| 68 |
+
from spacedreamer_galexea import SpaceDreamerGalaxea
|
| 69 |
+
policy=SpaceDreamerGalaxea(
|
| 70 |
+
"spacedreamer_stack_cups_clean40_30ep",
|
| 71 |
+
"spacedreamer_20epochs/assets",
|
| 72 |
+
device="cuda:0",
|
| 73 |
+
)
|
| 74 |
+
policy.reset_episode() # once at the start of a new episode
|
| 75 |
+
commands=policy.infer_action(
|
| 76 |
+
head_rgb,state,depth_m=depth_m,
|
| 77 |
+
num_inference_steps=10,seed=None,
|
| 78 |
+
)
|
| 79 |
+
assert commands.shape==(32,14)
|
| 80 |
+
```
|
| 81 |
+
|
| 82 |
+
Ten sampling steps is the adapter's default,not a claim that it is optimal for this new checkpoint. If `depth_m` is omitted,the adapter advances causal metric VDA using the head image. Feed successive camera frames through `update_depth` to maintain its temporal history when replanning less often;reset the stream once per episode. The adapter performs the original joint multimodal sampling and returns physical actions;it does not implement a new action-only algorithm or execute robot commands.
|
| 83 |
+
|
| 84 |
+
## Exact input/output contract
|
| 85 |
+
|
| 86 |
+
- One **head-left RGB view**,not FastWAM's concatenated three-camera layout.
|
| 87 |
+
- Accepted image: raw RGB,uint8 **512×640**,or the exact training letterboxed **224×224** image. Arbitrary square resizing of a raw camera frame is not equivalent. The adapter applies the matching letterbox and model resize to256×256.
|
| 88 |
+
- `depth_m`,if supplied: finite metric z-depth,float224×224,aligned with that letterboxed RGB. The adapter applies the training float16 cache quantization,nearest resize and0.05–5m validity convention.
|
| 89 |
+
- State order: left_arm6,left_gripper,right_arm6,right_gripper. Joint values in radians;grippers keep original continuous raw units approximately0–100.
|
| 90 |
+
- State normalization and output decoding use the external **reference_norm_stats.json in FP32**,including native clipping;do not substitute the old81/148-demo statistics or rounded checkpoint buffer values.
|
| 91 |
+
- Returned array is float32 **32×14**,already denormalized **absolute original motion_target.position**. Do not add current state or inverse-normalize again. At the20Hz training rate,32 commands span approximately1.6 seconds. Execution prefix/replanning frequency is controlled separately.
|
| 92 |
+
- Current RGB/depth frame0 condition the model;point frame0 and future RGB/depth/point/action latents are sampled jointly.
|
| 93 |
+
|
| 94 |
+
## Training completion and postflight issue
|
| 95 |
+
|
| 96 |
+
All4350 optimizer steps completed;all logged loss/gradient values were finite. The final100-step mean action loss was0.0086685. The platform then marked the job JOB_FAILED because an export checker required bitwise equality between BF16-saved normalizer buffers converted back toFP32 and the original FP32 statistics.
|
| 97 |
+
|
| 98 |
+
The independent audit found all four stored buffers **exactly equal to the correct selected40 statistics cast to BF16**,and the external inference statistics file byte-identical to the training normalizer. The failure is a postflight comparison false positive,not missing training steps or a switched normalizer. The weight file is uploaded unchanged. The inference adapter explicitly uses the original FP32 JSON for normalization/decoding.
|
| 99 |
+
|
| 100 |
+
Packaging/CPU model reconstruction checks are documented in CPU_validation.json. This upload does not claim new GPU rollout acceptance or a real-robot success rate for this checkpoint.
|
spacedreamer_stack_cups_clean40_30ep/checkpoint_metadata.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"task": "stack_cups",
|
| 3 |
+
"training_job": "bifrost-2026111502493001-shouqx",
|
| 4 |
+
"epochs": 30,
|
| 5 |
+
"global_step": 4350,
|
| 6 |
+
"checkpoint": "final_weights.pt",
|
| 7 |
+
"bytes": 11030029192,
|
| 8 |
+
"sha256": "b9c898a58d03a4b863d3ec9cc5b3b9c87e7ac00427a27478e61a17cbf269a6e7",
|
| 9 |
+
"action_dim": 14,
|
| 10 |
+
"action_horizon": 32,
|
| 11 |
+
"video_frames": 33,
|
| 12 |
+
"action_source": "absolute_original_motion_target_position",
|
| 13 |
+
"joint_order": "left_arm6,left_gripper,right_arm6,right_gripper",
|
| 14 |
+
"initialization": "Fresh fine-tune from original pretrained018126 on approved40 of new67 demos; selected40-refit normalization; right6joint loss2x,grippers1x.",
|
| 15 |
+
"optimizer_training_complete": true,
|
| 16 |
+
"platform_state": "JOB_FAILED",
|
| 17 |
+
"platform_failure_reason": "Postflight BF16-buffer versus FP32-statistics bitwise comparison false positive, independently audited; no missing optimizer steps."
|
| 18 |
+
}
|
spacedreamer_stack_cups_clean40_30ep/final_weights.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b9c898a58d03a4b863d3ec9cc5b3b9c87e7ac00427a27478e61a17cbf269a6e7
|
| 3 |
+
size 11030029192
|
spacedreamer_stack_cups_clean40_30ep/initialization.json
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"loaded_backbone_tensors": 1649,
|
| 3 |
+
"fresh_interface_tensors": [
|
| 4 |
+
"action_normalizer.action_q01",
|
| 5 |
+
"action_normalizer.action_q99",
|
| 6 |
+
"action_normalizer.state_q01",
|
| 7 |
+
"action_normalizer.state_q99",
|
| 8 |
+
"action_state_encoder.0.bias",
|
| 9 |
+
"action_state_encoder.0.weight",
|
| 10 |
+
"action_state_encoder.2.bias",
|
| 11 |
+
"action_state_encoder.2.weight",
|
| 12 |
+
"mot.mixtures.action.action_encoder.bias",
|
| 13 |
+
"mot.mixtures.action.action_encoder.weight",
|
| 14 |
+
"mot.mixtures.action.head.bias",
|
| 15 |
+
"mot.mixtures.action.head.weight"
|
| 16 |
+
],
|
| 17 |
+
"omitted_source_tensors": [
|
| 18 |
+
"action_normalizer.count",
|
| 19 |
+
"action_normalizer.fitted",
|
| 20 |
+
"action_normalizer.mean",
|
| 21 |
+
"action_normalizer.std",
|
| 22 |
+
"camera_encoder.bias",
|
| 23 |
+
"camera_encoder.weight",
|
| 24 |
+
"camera_head.bias",
|
| 25 |
+
"camera_head.weight",
|
| 26 |
+
"camera_normalizer.count",
|
| 27 |
+
"camera_normalizer.fitted",
|
| 28 |
+
"camera_normalizer.mean",
|
| 29 |
+
"camera_normalizer.std",
|
| 30 |
+
"structured_type_embedding"
|
| 31 |
+
],
|
| 32 |
+
"pretrained_action_semantics": "absolute_ee_xyz_so3log_grip",
|
| 33 |
+
"target_action_semantics": "absolute_joint_position_grip",
|
| 34 |
+
"optimizer_restored": false
|
| 35 |
+
}
|
spacedreamer_stack_cups_clean40_30ep/manifest.json
ADDED
|
@@ -0,0 +1,928 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "VALIDATED_ORIGINAL_FINAL30EP_RELEASE",
|
| 3 |
+
"training_job": "bifrost-2026111502493001-shouqx",
|
| 4 |
+
"step": 4350,
|
| 5 |
+
"epochs": 30,
|
| 6 |
+
"files": [
|
| 7 |
+
{
|
| 8 |
+
"path": "CPU_validation.json",
|
| 9 |
+
"bytes": 1008,
|
| 10 |
+
"sha256": "40f3bee5664f3afba5b173ea57168944b7f6c7362118cf311365906c3fafb0cb"
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"path": "README.md",
|
| 14 |
+
"bytes": 7458,
|
| 15 |
+
"sha256": "f75d0fe3b967edfbc5956a70466836a93edeb8266d801fa71cbd0e48b348ed3b"
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"path": "checkpoint_metadata.json",
|
| 19 |
+
"bytes": 856,
|
| 20 |
+
"sha256": "3709afdfb089858776b4c16d5430c53b6a31e31c4f6a419ef34591dceff750d3"
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"path": "final_weights.pt",
|
| 24 |
+
"bytes": 11030029192,
|
| 25 |
+
"sha256": "b9c898a58d03a4b863d3ec9cc5b3b9c87e7ac00427a27478e61a17cbf269a6e7"
|
| 26 |
+
},
|
| 27 |
+
{
|
| 28 |
+
"path": "initialization.json",
|
| 29 |
+
"bytes": 1104,
|
| 30 |
+
"sha256": "e7d9654086d1758bc509f6b47877423cabc6314e3c32439443a54b2568a7d7a6"
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"path": "postflight_normalizer_audit.json",
|
| 34 |
+
"bytes": 2771,
|
| 35 |
+
"sha256": "e02b02ca0501d9ab43a11baec6e7c0400e2ab27154722c510325d529f11fc954"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"path": "reference_norm_stats.json",
|
| 39 |
+
"bytes": 3360,
|
| 40 |
+
"sha256": "10f598cd5148e9ac85fa2b7b8c3b2f2c503127fcb307f33ad3e5277f8a9ff6f8"
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"path": "runtime/requirements.txt",
|
| 44 |
+
"bytes": 456,
|
| 45 |
+
"sha256": "93ed58aca470a8fec133f87b19e479f28744f29c848a0e8e26cd74a729afa16b"
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"path": "runtime/run_inference.py",
|
| 49 |
+
"bytes": 1427,
|
| 50 |
+
"sha256": "86c4c73bd13919001de9991b15883a2c7e6c4aff2f59a0c15aae51bb0b208e53"
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"path": "runtime/runtime_packages.json",
|
| 54 |
+
"bytes": 4403,
|
| 55 |
+
"sha256": "54120c1011271310e5babd300edcbff2b4ccab01fb1a3f84ed8f28f54b0f94e3"
|
| 56 |
+
},
|
| 57 |
+
{
|
| 58 |
+
"path": "runtime/source/model_old/src/fastwam/__init__.py",
|
| 59 |
+
"bytes": 23,
|
| 60 |
+
"sha256": "c4149c5ff0f9abd1517360daf761e692feb8570eb7a0caa2dfde52d4412c3e23"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/__init__.py",
|
| 64 |
+
"bytes": 0,
|
| 65 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/dataset_utils.py",
|
| 69 |
+
"bytes": 3735,
|
| 70 |
+
"sha256": "3bc17590049c7cb54061acf8d731b07892e7f530c5d7eeac2613fa77a285a392"
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/base_lerobot_dataset.py",
|
| 74 |
+
"bytes": 18557,
|
| 75 |
+
"sha256": "9a0690381f0ce0058bb0d526ea4fd4982939a4dbe293dabfc83d5abc086f0512"
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/constants.py",
|
| 79 |
+
"bytes": 1915,
|
| 80 |
+
"sha256": "11ac9260341e1560c5eb0dd7acb79ed63c74180b8ab4113c27622ff213db417f"
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py",
|
| 84 |
+
"bytes": 7067,
|
| 85 |
+
"sha256": "7d7daab219332b5abdfd5ec67ccaf3502e8eb583a17a4e482c3f3d4f7a691e89"
|
| 86 |
+
},
|
| 87 |
+
{
|
| 88 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py",
|
| 89 |
+
"bytes": 33439,
|
| 90 |
+
"sha256": "40f1b94e6ceb50765fb8dfa3d498887c2a0fc2eb49f09770c5f36190e7110c98"
|
| 91 |
+
},
|
| 92 |
+
{
|
| 93 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py",
|
| 94 |
+
"bytes": 19421,
|
| 95 |
+
"sha256": "cb647a1e955c3279e39ce8d7af7a37ea7432bbd71534c183f9ceebc8557dd89f"
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py",
|
| 99 |
+
"bytes": 55587,
|
| 100 |
+
"sha256": "6abe9eccf17077eee478805b574c25980cedf7a4f673a3dee1cb7184db9126f8"
|
| 101 |
+
},
|
| 102 |
+
{
|
| 103 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/processors/base_processor.py",
|
| 104 |
+
"bytes": 10811,
|
| 105 |
+
"sha256": "1f47bc85fee11e3d16714b2f1e3a2dc874f4609a1706a788617156d34b48b58e"
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/processors/fastwam_processor.py",
|
| 109 |
+
"bytes": 13316,
|
| 110 |
+
"sha256": "d71bd147c6594ee9d1a50c04d22fa69ccb914d601c4ad816233496f5b583fda5"
|
| 111 |
+
},
|
| 112 |
+
{
|
| 113 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/robot_video_dataset.py",
|
| 114 |
+
"bytes": 12358,
|
| 115 |
+
"sha256": "a30583e16418c2fd848e1588f42a28889fdc7b3b9dbaeccffbedbfe7fca0b081"
|
| 116 |
+
},
|
| 117 |
+
{
|
| 118 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/action_state_merger.py",
|
| 119 |
+
"bytes": 2522,
|
| 120 |
+
"sha256": "669d6991415fc5eca44b32b129d15fee1e06e2f17d59d90350c252598033b427"
|
| 121 |
+
},
|
| 122 |
+
{
|
| 123 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/image.py",
|
| 124 |
+
"bytes": 725,
|
| 125 |
+
"sha256": "4d7ab61d0844918525244db653831f0f632730046b2714a6163ff57c441a16fa"
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/misc.py",
|
| 129 |
+
"bytes": 422,
|
| 130 |
+
"sha256": "8b505bd9e3a54ab2db7efcd64f0929d2e0303f5db1123de1074fd01fa9887e2b"
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/relative_action.py",
|
| 134 |
+
"bytes": 4032,
|
| 135 |
+
"sha256": "e555e0558e948c4eea3d1119e274fa5d923af30862eec4380fb8528f4a319c47"
|
| 136 |
+
},
|
| 137 |
+
{
|
| 138 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/rotation.py",
|
| 139 |
+
"bytes": 5603,
|
| 140 |
+
"sha256": "0de34b252d60b956cda07c0c19baebbd3c8a28916face700c6664c5815e11e91"
|
| 141 |
+
},
|
| 142 |
+
{
|
| 143 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/utils/normalizer.py",
|
| 144 |
+
"bytes": 11274,
|
| 145 |
+
"sha256": "5ac89a646cd6395e98feaa832f8fdf7c7e00857080af3d9c15710e243c43ad43"
|
| 146 |
+
},
|
| 147 |
+
{
|
| 148 |
+
"path": "runtime/source/model_old/src/fastwam/datasets/lerobot/utils/rotation.py",
|
| 149 |
+
"bytes": 21110,
|
| 150 |
+
"sha256": "6169162f98faf02a3fe5201e2b8a1069fe299840d036ee7cad8af5fa8f561542"
|
| 151 |
+
},
|
| 152 |
+
{
|
| 153 |
+
"path": "runtime/source/model_old/src/fastwam/models/__init__.py",
|
| 154 |
+
"bytes": 35,
|
| 155 |
+
"sha256": "ae4c5733f4bbf7caf31df2ae6f789516f6a10cd84a321f7d0375a7034253ac6d"
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/__init__.py",
|
| 159 |
+
"bytes": 0,
|
| 160 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
| 161 |
+
},
|
| 162 |
+
{
|
| 163 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/action_dit.py",
|
| 164 |
+
"bytes": 13347,
|
| 165 |
+
"sha256": "1301d9224149de43bb701f620a5d41858ecc63c6b19a573ec32edd45a3bdb0a2"
|
| 166 |
+
},
|
| 167 |
+
{
|
| 168 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/attention_masks.py",
|
| 169 |
+
"bytes": 2846,
|
| 170 |
+
"sha256": "5c84fd14dfba35086c75fbd14244609d54e52f482a658ca4f02ddd63800bbb31"
|
| 171 |
+
},
|
| 172 |
+
{
|
| 173 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/fastwam.py",
|
| 174 |
+
"bytes": 48807,
|
| 175 |
+
"sha256": "36a61b652944a89dfd59dbba94ce206a664e40ed27868ca6d7bdbf0642e3bd0a"
|
| 176 |
+
},
|
| 177 |
+
{
|
| 178 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/fastwam_idm.py",
|
| 179 |
+
"bytes": 20427,
|
| 180 |
+
"sha256": "aae332d17645a6744aafd85edde8e8d7dacbb8b110de3988015ca330146c3c65"
|
| 181 |
+
},
|
| 182 |
+
{
|
| 183 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/fastwam_joint.py",
|
| 184 |
+
"bytes": 9795,
|
| 185 |
+
"sha256": "10937e5102839c8e6103aab828e3355fab6015299bdf66e303b056703dc79bb8"
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/__init__.py",
|
| 189 |
+
"bytes": 415,
|
| 190 |
+
"sha256": "f26116de229557a37a4812dc4ee75971f5f9e8b80240786ea4d0e5b0ff703cbf"
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/gradient.py",
|
| 194 |
+
"bytes": 565,
|
| 195 |
+
"sha256": "ba5d8f7272eb029dc6cd2849ca99b70f6ad5abb838d21c818beb0590620dc793"
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/io.py",
|
| 199 |
+
"bytes": 7751,
|
| 200 |
+
"sha256": "1c6c1bc8a975b47092912833194fce25e2e3ac886fbab72d73af3ff5a00b3c62"
|
| 201 |
+
},
|
| 202 |
+
{
|
| 203 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/loader.py",
|
| 204 |
+
"bytes": 8279,
|
| 205 |
+
"sha256": "792f66bd0609c1365256aa24d43301340f81260c1651d89b04d2ae8224a08726"
|
| 206 |
+
},
|
| 207 |
+
{
|
| 208 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/helpers/state_dict_converters.py",
|
| 209 |
+
"bytes": 5226,
|
| 210 |
+
"sha256": "81d213373e5056a8d8bf31768b9b04164138ea78d0b306e316334841499f4bf9"
|
| 211 |
+
},
|
| 212 |
+
{
|
| 213 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/mot.py",
|
| 214 |
+
"bytes": 22491,
|
| 215 |
+
"sha256": "fe0dce22413e18ff2e9cb249f7f9714f25d828b214f1a19e73feee029d29944b"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/schedulers/scheduler_continuous.py",
|
| 219 |
+
"bytes": 4068,
|
| 220 |
+
"sha256": "2ececce284f7b9fe876965fbf5e11ca7b69caaf693979266b0f63062a589f181"
|
| 221 |
+
},
|
| 222 |
+
{
|
| 223 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/wan22.py",
|
| 224 |
+
"bytes": 16945,
|
| 225 |
+
"sha256": "b72998999b4b2c5096773092339b8ca81e9e7d96f398b81c57d3d22d14972ac4"
|
| 226 |
+
},
|
| 227 |
+
{
|
| 228 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/wan_video_dit.py",
|
| 229 |
+
"bytes": 31351,
|
| 230 |
+
"sha256": "c254ac7f2a43da59f103864140bb84474f5588290f5fb17a6ed426d0b64e02c9"
|
| 231 |
+
},
|
| 232 |
+
{
|
| 233 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/wan_video_text_encoder.py",
|
| 234 |
+
"bytes": 11459,
|
| 235 |
+
"sha256": "aedf7f131556a39b99a273deec3a85d67f41b5367cef41f310f33a807ae60a87"
|
| 236 |
+
},
|
| 237 |
+
{
|
| 238 |
+
"path": "runtime/source/model_old/src/fastwam/models/wan22/wan_video_vae.py",
|
| 239 |
+
"bytes": 50058,
|
| 240 |
+
"sha256": "31b3fb9f844a76f5db5af4854e14802aae640b85b61d0de03a4b53b52515abb7"
|
| 241 |
+
},
|
| 242 |
+
{
|
| 243 |
+
"path": "runtime/source/model_old/src/fastwam/runtime.py",
|
| 244 |
+
"bytes": 18262,
|
| 245 |
+
"sha256": "e19cd2abf94f3af9f65a231158406b3d61887136a603bd3a954335d3623319b0"
|
| 246 |
+
},
|
| 247 |
+
{
|
| 248 |
+
"path": "runtime/source/model_old/src/fastwam/trainer.py",
|
| 249 |
+
"bytes": 35849,
|
| 250 |
+
"sha256": "551a81bb01adcbbe9c1bc753d397b3e53df1b9d80cc28041cef56c3a1f32607c"
|
| 251 |
+
},
|
| 252 |
+
{
|
| 253 |
+
"path": "runtime/source/model_old/src/fastwam/utils/__init__.py",
|
| 254 |
+
"bytes": 96,
|
| 255 |
+
"sha256": "2893c05b99b17162941c70a41859c2808f8ae390be367a26cac7d5ab93ee197b"
|
| 256 |
+
},
|
| 257 |
+
{
|
| 258 |
+
"path": "runtime/source/model_old/src/fastwam/utils/config_resolvers.py",
|
| 259 |
+
"bytes": 2526,
|
| 260 |
+
"sha256": "01c48cbd42b86e636b70ccc3e193a81147162ba890777102b2c61086fa415340"
|
| 261 |
+
},
|
| 262 |
+
{
|
| 263 |
+
"path": "runtime/source/model_old/src/fastwam/utils/fs.py",
|
| 264 |
+
"bytes": 76,
|
| 265 |
+
"sha256": "69e6df236b43360428a613686c8f37c531c0d16a3a2c5b06a725c46b49d657c5"
|
| 266 |
+
},
|
| 267 |
+
{
|
| 268 |
+
"path": "runtime/source/model_old/src/fastwam/utils/logging_config.py",
|
| 269 |
+
"bytes": 4679,
|
| 270 |
+
"sha256": "cadc4f6ad487c3f37171a6defad36d23d9ab8bf262f73a461cd9ee9b03b757e7"
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"path": "runtime/source/model_old/src/fastwam/utils/misc.py",
|
| 274 |
+
"bytes": 1223,
|
| 275 |
+
"sha256": "05aa51247936a8d3faac27f90b78b65ff83a688259bed55d114b6a617ced532d"
|
| 276 |
+
},
|
| 277 |
+
{
|
| 278 |
+
"path": "runtime/source/model_old/src/fastwam/utils/pytorch_utils.py",
|
| 279 |
+
"bytes": 5049,
|
| 280 |
+
"sha256": "ffc483e7fb435cd654326750eaffdbfc9243837c3140dcc74e45b24abb751a33"
|
| 281 |
+
},
|
| 282 |
+
{
|
| 283 |
+
"path": "runtime/source/model_old/src/fastwam/utils/samplers.py",
|
| 284 |
+
"bytes": 1339,
|
| 285 |
+
"sha256": "4d3bd9b2d498f4ad8f2cf9765d60c4bf5368fa6fa2a7d42d0db6210f66899652"
|
| 286 |
+
},
|
| 287 |
+
{
|
| 288 |
+
"path": "runtime/source/model_old/src/fastwam/utils/video_io.py",
|
| 289 |
+
"bytes": 819,
|
| 290 |
+
"sha256": "9e9898b021a44e87aa1b4e6d13e3755a9869a2e27a7eb53c72a6802808d5bd80"
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"path": "runtime/source/model_old/src/fastwam/utils/video_metrics.py",
|
| 294 |
+
"bytes": 3446,
|
| 295 |
+
"sha256": "aaa37e458cc9e4bb79a17923b5ff5900c97b4309f7abd50db44cf76e9143ffd6"
|
| 296 |
+
},
|
| 297 |
+
{
|
| 298 |
+
"path": "runtime/source/multimodal/audit_robodojo_warmup_reproduction.py",
|
| 299 |
+
"bytes": 14938,
|
| 300 |
+
"sha256": "ab2d694fc3b00d3047296eac4a196ee7a30180fa7abed2547bf1f68b781a2405"
|
| 301 |
+
},
|
| 302 |
+
{
|
| 303 |
+
"path": "runtime/source/multimodal/build_piper_hdf5_index.py",
|
| 304 |
+
"bytes": 1287,
|
| 305 |
+
"sha256": "c6fa001c3363d4306e85a5eab2fe4e847f18d69c899b068259e8a35f090e1b9a"
|
| 306 |
+
},
|
| 307 |
+
{
|
| 308 |
+
"path": "runtime/source/multimodal/canonical_spacedreamer_data.py",
|
| 309 |
+
"bytes": 39499,
|
| 310 |
+
"sha256": "0b5f5c718c882f24012c3ae8a2070ffb53e386aea53a1ee60d8eb0a361906920"
|
| 311 |
+
},
|
| 312 |
+
{
|
| 313 |
+
"path": "runtime/source/multimodal/controlled_directional_mot.py",
|
| 314 |
+
"bytes": 6970,
|
| 315 |
+
"sha256": "8f1c53f5a7570ed251057274136a733faa4ef653b46a7728ad00539670ab4503"
|
| 316 |
+
},
|
| 317 |
+
{
|
| 318 |
+
"path": "runtime/source/multimodal/depth_preprocessing.py",
|
| 319 |
+
"bytes": 3254,
|
| 320 |
+
"sha256": "4fe296aee8e22ee56e488bf6fcbf6279e7f1ac385b286ae3b7f705d92940d482"
|
| 321 |
+
},
|
| 322 |
+
{
|
| 323 |
+
"path": "runtime/source/multimodal/evaluate_spacedreamer_validation.py",
|
| 324 |
+
"bytes": 2147,
|
| 325 |
+
"sha256": "ec76991435cbe39bec9e3ea506810cabbf466a129a9ad4544bed906b4b7fe550"
|
| 326 |
+
},
|
| 327 |
+
{
|
| 328 |
+
"path": "runtime/source/multimodal/finetune_galexea_rgb_action.py",
|
| 329 |
+
"bytes": 28110,
|
| 330 |
+
"sha256": "dc29a32aa0dc5afad4273f30e690016b3a65e0a385d4aa751b17e6db7c8d2d2a"
|
| 331 |
+
},
|
| 332 |
+
{
|
| 333 |
+
"path": "runtime/source/multimodal/galexea_mcap.py",
|
| 334 |
+
"bytes": 18359,
|
| 335 |
+
"sha256": "2f2e60c5a81bd44a9c9d0272af79ebb2b9dd8004b12fbddffb2a7c8d4e84ace1"
|
| 336 |
+
},
|
| 337 |
+
{
|
| 338 |
+
"path": "runtime/source/multimodal/joint_rgb_action_diffusion.py",
|
| 339 |
+
"bytes": 7935,
|
| 340 |
+
"sha256": "e71d7210e7bbf20e3807215e6b9feacc48f7c2f31b496e7d4e5a4af76f7323eb"
|
| 341 |
+
},
|
| 342 |
+
{
|
| 343 |
+
"path": "runtime/source/multimodal/joint_rgb_action_diffusion_shared_timestep.py",
|
| 344 |
+
"bytes": 5510,
|
| 345 |
+
"sha256": "cfcf934e2c77d4d69a6a7280e0dcf264a55a42e885b99612cb89bf721a35a96f"
|
| 346 |
+
},
|
| 347 |
+
{
|
| 348 |
+
"path": "runtime/source/multimodal/joint_rgb_depth_action_fastwam_diffusion.py",
|
| 349 |
+
"bytes": 4807,
|
| 350 |
+
"sha256": "8bc87f96c4ed0d6076f9c5be3367e290bb6863c3e9b9f5146539971c7b243541"
|
| 351 |
+
},
|
| 352 |
+
{
|
| 353 |
+
"path": "runtime/source/multimodal/joint_rgb_depth_point_action_diffusion.py",
|
| 354 |
+
"bytes": 10866,
|
| 355 |
+
"sha256": "0874a83ebfe1f4452fb8eedace7293d750fd1aa5814fd93ee6bd68c9e12e453b"
|
| 356 |
+
},
|
| 357 |
+
{
|
| 358 |
+
"path": "runtime/source/multimodal/joint_rgb_depth_point_action_fastwam_diffusion.py",
|
| 359 |
+
"bytes": 7018,
|
| 360 |
+
"sha256": "78d7d14c323178994391414e6cf7b102e97adddfb79266d4e66385a8e473d165"
|
| 361 |
+
},
|
| 362 |
+
{
|
| 363 |
+
"path": "runtime/source/multimodal/joint_rgb_geometry_action_diffusion.py",
|
| 364 |
+
"bytes": 9126,
|
| 365 |
+
"sha256": "e94cf04caac709aa806bec600e38db472dae896b122ae23be8d9d35ef24fad56"
|
| 366 |
+
},
|
| 367 |
+
{
|
| 368 |
+
"path": "runtime/source/multimodal/online_egoverse_action.py",
|
| 369 |
+
"bytes": 27776,
|
| 370 |
+
"sha256": "c484a0e11953ba78a154bb38c170ba890d8ef427a0ec515148312466ee966c16"
|
| 371 |
+
},
|
| 372 |
+
{
|
| 373 |
+
"path": "runtime/source/multimodal/optimizer_step_scheduler.py",
|
| 374 |
+
"bytes": 5408,
|
| 375 |
+
"sha256": "84b09ff87013725d120bf2cd5357a71752d3e406724b57fdffe054de262698c7"
|
| 376 |
+
},
|
| 377 |
+
{
|
| 378 |
+
"path": "runtime/source/multimodal/piper_hdf5.py",
|
| 379 |
+
"bytes": 23659,
|
| 380 |
+
"sha256": "544d6e39836339de5c69a22756df198f09cce52adcfdafab97e0d63e4e95d2a0"
|
| 381 |
+
},
|
| 382 |
+
{
|
| 383 |
+
"path": "runtime/source/multimodal/post_training_test.py",
|
| 384 |
+
"bytes": 29116,
|
| 385 |
+
"sha256": "5a41fd8f38ec3baf6a0e69619d074fffa8acb30aaeed22c26612d0576120cbb8"
|
| 386 |
+
},
|
| 387 |
+
{
|
| 388 |
+
"path": "runtime/source/multimodal/post_training_test_action_mot.py",
|
| 389 |
+
"bytes": 21288,
|
| 390 |
+
"sha256": "ba1718fddc337b11e9e7f7c4d7e68241cfe87db6cd5123cfcfd9ee7aa7df7cf1"
|
| 391 |
+
},
|
| 392 |
+
{
|
| 393 |
+
"path": "runtime/source/multimodal/post_training_test_rgb_depth_point.py",
|
| 394 |
+
"bytes": 22379,
|
| 395 |
+
"sha256": "418af7e62aa7ae6115538a01364c0ec5ee7647321c7bea6701e820637385445e"
|
| 396 |
+
},
|
| 397 |
+
{
|
| 398 |
+
"path": "runtime/source/multimodal/precompute_robodojo_exhaustive_rgb_latents.py",
|
| 399 |
+
"bytes": 14671,
|
| 400 |
+
"sha256": "943e83702c1c5eb1ee224d691d14c53aa87f8084c8824304aff91bbbf8d8d28a"
|
| 401 |
+
},
|
| 402 |
+
{
|
| 403 |
+
"path": "runtime/source/multimodal/precompute_robodojo_geometry_latents.py",
|
| 404 |
+
"bytes": 10570,
|
| 405 |
+
"sha256": "27b2c7f5c27b865921f4403ac68e861d2cd02e4f928bb1e96400eea675d0163b"
|
| 406 |
+
},
|
| 407 |
+
{
|
| 408 |
+
"path": "runtime/source/multimodal/precompute_robodojo_native_depth_latents_fastwam.py",
|
| 409 |
+
"bytes": 12125,
|
| 410 |
+
"sha256": "1596c08fd81ca38ef8d4dcc11dec36a7b9df17c57282b057571f44176b7e9ca5"
|
| 411 |
+
},
|
| 412 |
+
{
|
| 413 |
+
"path": "runtime/source/multimodal/precompute_robodojo_native_point_latents_fastwam.py",
|
| 414 |
+
"bytes": 14509,
|
| 415 |
+
"sha256": "ff70aa582dc179104058a451fc4796d0b502394f8b30fe9026e0bd31666bb42e"
|
| 416 |
+
},
|
| 417 |
+
{
|
| 418 |
+
"path": "runtime/source/multimodal/preflight_robodojo_ego_first_cam.py",
|
| 419 |
+
"bytes": 6302,
|
| 420 |
+
"sha256": "bae3d9a15a791cee000c85daca94c0c977895ff88be0da835c5f24361d615d53"
|
| 421 |
+
},
|
| 422 |
+
{
|
| 423 |
+
"path": "runtime/source/multimodal/prepare_point_cloud_cache.py",
|
| 424 |
+
"bytes": 7450,
|
| 425 |
+
"sha256": "b6c3bcacc0ae1d912ae1370525d841896b4f6acba51d2ca53e0cab1a561fddde"
|
| 426 |
+
},
|
| 427 |
+
{
|
| 428 |
+
"path": "runtime/source/multimodal/rgb_action_model_factory.py",
|
| 429 |
+
"bytes": 3791,
|
| 430 |
+
"sha256": "285364b62bf5f9ca488f003328150aacd95a7fd4a98e990965cfe2cad9f2965a"
|
| 431 |
+
},
|
| 432 |
+
{
|
| 433 |
+
"path": "runtime/source/multimodal/rgb_depth_point_action_model_factory.py",
|
| 434 |
+
"bytes": 7916,
|
| 435 |
+
"sha256": "1ac6d134c3f94f4b04604d6b9a35cc4a98bafff728361affc5b0c69211e71ba3"
|
| 436 |
+
},
|
| 437 |
+
{
|
| 438 |
+
"path": "runtime/source/multimodal/robodojo_ego_first_cam.py",
|
| 439 |
+
"bytes": 11763,
|
| 440 |
+
"sha256": "9543f5f29ea78db8c0763450c08fee8bd52a9fc1e9433826e161338b95b3a4f1"
|
| 441 |
+
},
|
| 442 |
+
{
|
| 443 |
+
"path": "runtime/source/multimodal/robodojo_geometry_data.py",
|
| 444 |
+
"bytes": 12794,
|
| 445 |
+
"sha256": "37221e841d324cc8713a142515591cd375c09ce01d7cb896be5f18fd75e5332e"
|
| 446 |
+
},
|
| 447 |
+
{
|
| 448 |
+
"path": "runtime/source/multimodal/robodojo_lerobot_v30.py",
|
| 449 |
+
"bytes": 17380,
|
| 450 |
+
"sha256": "4bfc2df11d6786bc78b5fa1452904f6cca66e1203b3a8e8524d81fca20ee9563"
|
| 451 |
+
},
|
| 452 |
+
{
|
| 453 |
+
"path": "runtime/source/multimodal/robodojo_native_depth_action_model.py",
|
| 454 |
+
"bytes": 15497,
|
| 455 |
+
"sha256": "21ac2f0d5aabcde94fc2961db747449fd4d08e27ad7067b47daf460a813be40e"
|
| 456 |
+
},
|
| 457 |
+
{
|
| 458 |
+
"path": "runtime/source/multimodal/robodojo_native_depth_data.py",
|
| 459 |
+
"bytes": 15063,
|
| 460 |
+
"sha256": "4e982175ceb1211f08386a339a973eb0d377c2a4ac1ff368738857a484820584"
|
| 461 |
+
},
|
| 462 |
+
{
|
| 463 |
+
"path": "runtime/source/multimodal/robodojo_native_depth_point_action_model.py",
|
| 464 |
+
"bytes": 18092,
|
| 465 |
+
"sha256": "eec08ebb5ada5a278916bf6f414fe82e5749d5c5159f7d2e1c576b476f76e6fe"
|
| 466 |
+
},
|
| 467 |
+
{
|
| 468 |
+
"path": "runtime/source/multimodal/robodojo_native_depth_point_data.py",
|
| 469 |
+
"bytes": 12559,
|
| 470 |
+
"sha256": "16895df58500abc5b9926b37a6f5face6f9d6e7d1f8fda15331efee165a5dc28"
|
| 471 |
+
},
|
| 472 |
+
{
|
| 473 |
+
"path": "runtime/source/multimodal/robodojo_rgb_geometry_action_model.py",
|
| 474 |
+
"bytes": 7859,
|
| 475 |
+
"sha256": "5a496d410f42f206542e5111d1ac38003b646b0f50e55dfb78a2ee3e50bc5bc0"
|
| 476 |
+
},
|
| 477 |
+
{
|
| 478 |
+
"path": "runtime/source/multimodal/robodojo_v2_checkpoint_contract.py",
|
| 479 |
+
"bytes": 6497,
|
| 480 |
+
"sha256": "f361f24c019c0e04989208457cc0b92f47dcb4567499902a1f62ac6af5afc89c"
|
| 481 |
+
},
|
| 482 |
+
{
|
| 483 |
+
"path": "runtime/source/multimodal/scripts/audit_piper_pretrained018126.py",
|
| 484 |
+
"bytes": 3894,
|
| 485 |
+
"sha256": "5011329e2f3c0c7962c6c31df7fe0a946042dfaded19a08b51209e32dbbd9ab2"
|
| 486 |
+
},
|
| 487 |
+
{
|
| 488 |
+
"path": "runtime/source/multimodal/scripts/audit_piper_pretrained018126_target_norm.py",
|
| 489 |
+
"bytes": 2207,
|
| 490 |
+
"sha256": "dd817495b883b408eb53a0f04ba298cc87f4084cbbdd8a0fa53582a46cbaf17b"
|
| 491 |
+
},
|
| 492 |
+
{
|
| 493 |
+
"path": "runtime/source/multimodal/scripts/audit_robodojo_full35_training.py",
|
| 494 |
+
"bytes": 4532,
|
| 495 |
+
"sha256": "1f8dd936fb44781533ed3db581b82017d27d99a3b2147f79300af9b8bf283864"
|
| 496 |
+
},
|
| 497 |
+
{
|
| 498 |
+
"path": "runtime/source/multimodal/scripts/audit_robodojo_full_normalizer_noaug_configs.py",
|
| 499 |
+
"bytes": 6127,
|
| 500 |
+
"sha256": "58a4bc39a9d59c1add35f44d8b3931117ba23be8f61bfb70163205a17d368917"
|
| 501 |
+
},
|
| 502 |
+
{
|
| 503 |
+
"path": "runtime/source/multimodal/scripts/audit_source_normalizer_noaug_configs.py",
|
| 504 |
+
"bytes": 6930,
|
| 505 |
+
"sha256": "cb72a0c6361b40340382625022ef93f978e3b1bcc9938a5a447861ab0c3a55b5"
|
| 506 |
+
},
|
| 507 |
+
{
|
| 508 |
+
"path": "runtime/source/multimodal/scripts/audit_spacedreamer_v3_robodojo_srs_resume5000.py",
|
| 509 |
+
"bytes": 2683,
|
| 510 |
+
"sha256": "9bf8dd49834503c92664df78d3a33e7b08244454d275595780550e69d719c53c"
|
| 511 |
+
},
|
| 512 |
+
{
|
| 513 |
+
"path": "runtime/source/multimodal/scripts/audit_spacedreamer_v3_robodojo_srs_trail1.py",
|
| 514 |
+
"bytes": 3388,
|
| 515 |
+
"sha256": "c96e68f282fe17da9ae1d84d21b11eecb2dc8e5fb80e50b028c6eaa6169df3c3"
|
| 516 |
+
},
|
| 517 |
+
{
|
| 518 |
+
"path": "runtime/source/multimodal/scripts/audit_target_normalizer_noaug_configs.py",
|
| 519 |
+
"bytes": 2826,
|
| 520 |
+
"sha256": "4b817fffe274c9e139ba79c1d07ec6ec50af4d92027cfe0a85c2fb39b9a47dc7"
|
| 521 |
+
},
|
| 522 |
+
{
|
| 523 |
+
"path": "runtime/source/multimodal/scripts/build_egoverse_canonical_motion_cache.py",
|
| 524 |
+
"bytes": 12355,
|
| 525 |
+
"sha256": "a34371ef4c32998343a19646977450d5cad51f2514a010cc382579fd6c49697d"
|
| 526 |
+
},
|
| 527 |
+
{
|
| 528 |
+
"path": "runtime/source/multimodal/scripts/build_spacedreamer_canonical_mix.py",
|
| 529 |
+
"bytes": 27136,
|
| 530 |
+
"sha256": "c8769ed42f9e36e62c734bfe960efcc57fe8657fc46af5852a9598ed067568b2"
|
| 531 |
+
},
|
| 532 |
+
{
|
| 533 |
+
"path": "runtime/source/multimodal/scripts/build_spacedreamer_indexed_canonical_manifest.py",
|
| 534 |
+
"bytes": 12295,
|
| 535 |
+
"sha256": "a06fbeda5afc37f141cf97bfc4686613ed11e3fd86db112a7d7e6ba6e58b6a76"
|
| 536 |
+
},
|
| 537 |
+
{
|
| 538 |
+
"path": "runtime/source/multimodal/scripts/evaluate_robodojo_train_action_generation.py",
|
| 539 |
+
"bytes": 23899,
|
| 540 |
+
"sha256": "cc950de9e1efae04e87af46600b8beee9742a2689267416cf929e718580b4ed4"
|
| 541 |
+
},
|
| 542 |
+
{
|
| 543 |
+
"path": "runtime/source/multimodal/scripts/merge_spacedreamer_canonical_parts.py",
|
| 544 |
+
"bytes": 6149,
|
| 545 |
+
"sha256": "d43aa61829542c19efe07950dc52802e6c3086706fbaa1e81dcb19f821c63bd1"
|
| 546 |
+
},
|
| 547 |
+
{
|
| 548 |
+
"path": "runtime/source/multimodal/scripts/prepare_robodojo_full35_base_cache.py",
|
| 549 |
+
"bytes": 1389,
|
| 550 |
+
"sha256": "a446bedc5b09d769761a87e572247c3df4fe7ade950d6ec8c3c3893e4496ae35"
|
| 551 |
+
},
|
| 552 |
+
{
|
| 553 |
+
"path": "runtime/source/multimodal/scripts/run_spacedreamer_canonical_full_build.py",
|
| 554 |
+
"bytes": 4433,
|
| 555 |
+
"sha256": "5fe85a4edeaec5572c7bac49c2541c58b86393e34db5b8d2b66aeba4841c6d28"
|
| 556 |
+
},
|
| 557 |
+
{
|
| 558 |
+
"path": "runtime/source/multimodal/scripts/smoke_robodojo_geometry_cached_dataset.py",
|
| 559 |
+
"bytes": 3709,
|
| 560 |
+
"sha256": "fba36fe8f2bf0d8802f5f4013867bd4a651771222daf07bfb42361aeeb32e909"
|
| 561 |
+
},
|
| 562 |
+
{
|
| 563 |
+
"path": "runtime/source/multimodal/scripts/validate_spacedreamer_canonical_mix.py",
|
| 564 |
+
"bytes": 12597,
|
| 565 |
+
"sha256": "1397bd66a8ff17f59eb2668aa115e34ace8c44b05782d7805994923f01e84e88"
|
| 566 |
+
},
|
| 567 |
+
{
|
| 568 |
+
"path": "runtime/source/multimodal/scripts/validate_spacedreamer_indexed_canonical.py",
|
| 569 |
+
"bytes": 17200,
|
| 570 |
+
"sha256": "1efcc093f42978254a1fbc6c26487261ff9bec25c38adcd8fff218af59d03ba4"
|
| 571 |
+
},
|
| 572 |
+
{
|
| 573 |
+
"path": "runtime/source/multimodal/scripts/verify_robodojo_geometry_experiment.py",
|
| 574 |
+
"bytes": 7763,
|
| 575 |
+
"sha256": "ac10edb1850a3229f52c7c472bde0584a9d212095dd37ea98c87454729c4fb11"
|
| 576 |
+
},
|
| 577 |
+
{
|
| 578 |
+
"path": "runtime/source/multimodal/scripts/visualize_depth_preprocessing.py",
|
| 579 |
+
"bytes": 6814,
|
| 580 |
+
"sha256": "9f20bbef948916866cdc2a2ac3a5010e549ba0787ed350b71435adc2aa66c28a"
|
| 581 |
+
},
|
| 582 |
+
{
|
| 583 |
+
"path": "runtime/source/multimodal/spacedreamer_ego_dataset.py",
|
| 584 |
+
"bytes": 38628,
|
| 585 |
+
"sha256": "633e1c7de3799c0d05329bfe8edf97b1dced4541ed6cec3cea1afd02dc80726c"
|
| 586 |
+
},
|
| 587 |
+
{
|
| 588 |
+
"path": "runtime/source/multimodal/train_galexea_rgb_state_joint_action_ablation.py",
|
| 589 |
+
"bytes": 28500,
|
| 590 |
+
"sha256": "ed1e3cef681348472320ac04a9bb7f7e1f26a203e25f2673cd40239187c84d3f"
|
| 591 |
+
},
|
| 592 |
+
{
|
| 593 |
+
"path": "runtime/source/multimodal/train_joint_rgb_depth.py",
|
| 594 |
+
"bytes": 67200,
|
| 595 |
+
"sha256": "53b1f71eda541f36499b7be813b805068aa22245693cb9fea0e6c302fa0bebd8"
|
| 596 |
+
},
|
| 597 |
+
{
|
| 598 |
+
"path": "runtime/source/multimodal/train_joint_rgb_depth_point.py",
|
| 599 |
+
"bytes": 40421,
|
| 600 |
+
"sha256": "012bcd542e7e43bc99b7606d76dc2af9a1816e9630246fcfa779b1d0c6a833fe"
|
| 601 |
+
},
|
| 602 |
+
{
|
| 603 |
+
"path": "runtime/source/multimodal/train_joint_rgb_depth_point_action.py",
|
| 604 |
+
"bytes": 35857,
|
| 605 |
+
"sha256": "ddbcabac68c212ad3aab2afd1e825ce3de3120be32b14ac791670b2d8b66bf0d"
|
| 606 |
+
},
|
| 607 |
+
{
|
| 608 |
+
"path": "runtime/source/multimodal/train_piper_rgb_state_joint_action_quantile_4way.py",
|
| 609 |
+
"bytes": 27604,
|
| 610 |
+
"sha256": "76353a68caeb954a3be02359a75c626072e93e617cb2c2b079363d72786ec0d9"
|
| 611 |
+
},
|
| 612 |
+
{
|
| 613 |
+
"path": "runtime/source/multimodal/train_robodojo_rgb_geometry_action_absolute_ee.py",
|
| 614 |
+
"bytes": 12293,
|
| 615 |
+
"sha256": "ced9745b4ab2bca6b3bea5508f433247106af98dc708cd08bfe7a9400bbc19ce"
|
| 616 |
+
},
|
| 617 |
+
{
|
| 618 |
+
"path": "runtime/source/multimodal/train_robodojo_rgb_native_depth_point_state_joint_action_fastwam.py",
|
| 619 |
+
"bytes": 13798,
|
| 620 |
+
"sha256": "9e2519a5dbe5a960a5cdedfa247630a6dfd28bef72fecfed0b4c4bbc3cbc18be"
|
| 621 |
+
},
|
| 622 |
+
{
|
| 623 |
+
"path": "runtime/source/multimodal/train_robodojo_rgb_native_depth_state_joint_action_fastwam.py",
|
| 624 |
+
"bytes": 17614,
|
| 625 |
+
"sha256": "32fddebea5accffc5be727e61d7f7c4865a75522c61d2293bca1b503ebe4dd7f"
|
| 626 |
+
},
|
| 627 |
+
{
|
| 628 |
+
"path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_4way.py",
|
| 629 |
+
"bytes": 80135,
|
| 630 |
+
"sha256": "26b1ed4fc9c8c6b079a8b212aae78287a6b36f8bb35c3ed171d8bfa511eaed39"
|
| 631 |
+
},
|
| 632 |
+
{
|
| 633 |
+
"path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_ablation.py",
|
| 634 |
+
"bytes": 42461,
|
| 635 |
+
"sha256": "485b33691060a00f85beae7e180ac11f30881982406f2698622e0ed91a93969e"
|
| 636 |
+
},
|
| 637 |
+
{
|
| 638 |
+
"path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_4way.py",
|
| 639 |
+
"bytes": 7861,
|
| 640 |
+
"sha256": "e069cd1b6198e7c591ba179d5f26cf0b02bad98133cb5b8cf26ae558acc9caa0"
|
| 641 |
+
},
|
| 642 |
+
{
|
| 643 |
+
"path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_quantile_4way.py",
|
| 644 |
+
"bytes": 10551,
|
| 645 |
+
"sha256": "465b849da066d562e12b43a31557f74eb3faac71669ef52f87ac90a185185b8d"
|
| 646 |
+
},
|
| 647 |
+
{
|
| 648 |
+
"path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_absolute_ee_quantile_shared_timestep_4way.py",
|
| 649 |
+
"bytes": 3857,
|
| 650 |
+
"sha256": "119231049ec988daf4fbaa4b4db7f8013d84fd47e23a8bf0596131b4e2911143"
|
| 651 |
+
},
|
| 652 |
+
{
|
| 653 |
+
"path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_ego_first_cam_meanstd_4way.py",
|
| 654 |
+
"bytes": 26471,
|
| 655 |
+
"sha256": "600eca1cd377a5399f1659260a798e548f368f1788566d2a5467ca9d66ed168e"
|
| 656 |
+
},
|
| 657 |
+
{
|
| 658 |
+
"path": "runtime/source/multimodal/train_robodojo_rgb_state_joint_action_episode_anchor_4way.py",
|
| 659 |
+
"bytes": 7408,
|
| 660 |
+
"sha256": "d90b516158b2781c3b913098c6d488b05db1e30714269330ffce6daf8b55fd6d"
|
| 661 |
+
},
|
| 662 |
+
{
|
| 663 |
+
"path": "runtime/source/multimodal/train_spacedreamer_ego.py",
|
| 664 |
+
"bytes": 68897,
|
| 665 |
+
"sha256": "c8f87e0a82f564a07dc58a09d4cb6220fc850d15654662307abf6cd55f8c0140"
|
| 666 |
+
},
|
| 667 |
+
{
|
| 668 |
+
"path": "runtime/source/multimodal/wan_rgb_action_mot.py",
|
| 669 |
+
"bytes": 10421,
|
| 670 |
+
"sha256": "a33437c1dbeed4dcaa6cf70e5505bd6ddc89a2324e11d81f5e18bff56ae115e7"
|
| 671 |
+
},
|
| 672 |
+
{
|
| 673 |
+
"path": "runtime/source/multimodal/wan_rgb_depth_initialization.py",
|
| 674 |
+
"bytes": 24878,
|
| 675 |
+
"sha256": "025d5c2cef3ddfced4ef0f3f4b25644cf68618e5e75be19f4eb20be40be79df3"
|
| 676 |
+
},
|
| 677 |
+
{
|
| 678 |
+
"path": "runtime/source/multimodal/wan_rgb_depth_point_action_mot.py",
|
| 679 |
+
"bytes": 17358,
|
| 680 |
+
"sha256": "238447d95a5b07d876e4ddf235077ac40cd8a945af8b27f4147811c771868557"
|
| 681 |
+
},
|
| 682 |
+
{
|
| 683 |
+
"path": "runtime/source/multimodal/wan_rgb_depth_point_initialization.py",
|
| 684 |
+
"bytes": 13728,
|
| 685 |
+
"sha256": "724f8ef02bb42754d2cc3fa3e2ed0e507e2a0451f96e37e2bb72a00da847ba7d"
|
| 686 |
+
},
|
| 687 |
+
{
|
| 688 |
+
"path": "runtime/source/multimodal/world_point_cloud.py",
|
| 689 |
+
"bytes": 25523,
|
| 690 |
+
"sha256": "3323ae5e1779126182e5a27ffcd39336dac5feb3d4db4f3ab572e5483272180b"
|
| 691 |
+
},
|
| 692 |
+
{
|
| 693 |
+
"path": "runtime/source/vda/utils/dc_utils.py",
|
| 694 |
+
"bytes": 3675,
|
| 695 |
+
"sha256": "45103ac573780e51ae54bbc3a010e391003c0e8bbd02746cb5f493d60d4e7907"
|
| 696 |
+
},
|
| 697 |
+
{
|
| 698 |
+
"path": "runtime/source/vda/utils/util.py",
|
| 699 |
+
"bytes": 2449,
|
| 700 |
+
"sha256": "f02cf85f5a9543ce0f832a4fde1492076026650c8eb25c6a8cc4315bf67a8ed2"
|
| 701 |
+
},
|
| 702 |
+
{
|
| 703 |
+
"path": "runtime/source/vda/video_depth_anything/dinov2.py",
|
| 704 |
+
"bytes": 15178,
|
| 705 |
+
"sha256": "d22df8d0da44c8525ebed898fd9c33b3acfcda919a84d577cc2a08bfd3d50ac1"
|
| 706 |
+
},
|
| 707 |
+
{
|
| 708 |
+
"path": "runtime/source/vda/video_depth_anything/dinov2_layers/__init__.py",
|
| 709 |
+
"bytes": 382,
|
| 710 |
+
"sha256": "e3c28c7b775165b6844000eb69f44d104d6c3803ac7da5d59f6ac3ecfb46d9d8"
|
| 711 |
+
},
|
| 712 |
+
{
|
| 713 |
+
"path": "runtime/source/vda/video_depth_anything/dinov2_layers/attention.py",
|
| 714 |
+
"bytes": 2343,
|
| 715 |
+
"sha256": "0d2410ce8b8b4499a6b0cdab939ae2bceab8971efd8660d999b0f3088f28cbf7"
|
| 716 |
+
},
|
| 717 |
+
{
|
| 718 |
+
"path": "runtime/source/vda/video_depth_anything/dinov2_layers/block.py",
|
| 719 |
+
"bytes": 9332,
|
| 720 |
+
"sha256": "093266ea08084c7cce7b8ddd779ecfaaa13055735faa8a7d68b5924f385b395b"
|
| 721 |
+
},
|
| 722 |
+
{
|
| 723 |
+
"path": "runtime/source/vda/video_depth_anything/dinov2_layers/drop_path.py",
|
| 724 |
+
"bytes": 1160,
|
| 725 |
+
"sha256": "949100aac4d9c19c9d65319dbd362886c5d46a72d7586ba4fd500e9f1d51570c"
|
| 726 |
+
},
|
| 727 |
+
{
|
| 728 |
+
"path": "runtime/source/vda/video_depth_anything/dinov2_layers/layer_scale.py",
|
| 729 |
+
"bytes": 823,
|
| 730 |
+
"sha256": "356817b5d1390737128cbcc03a5333fd3060b73beae478772282ccb851d0ef85"
|
| 731 |
+
},
|
| 732 |
+
{
|
| 733 |
+
"path": "runtime/source/vda/video_depth_anything/dinov2_layers/mlp.py",
|
| 734 |
+
"bytes": 1272,
|
| 735 |
+
"sha256": "c18c5b3f021408597a473bb710070ec3fb3b39f5a2e42e88ae83ced425a608b5"
|
| 736 |
+
},
|
| 737 |
+
{
|
| 738 |
+
"path": "runtime/source/vda/video_depth_anything/dinov2_layers/patch_embed.py",
|
| 739 |
+
"bytes": 2832,
|
| 740 |
+
"sha256": "b3f82e19926c9cfb63f10136aca5e8a723bde1ad62dae1d04f90d405f9439baf"
|
| 741 |
+
},
|
| 742 |
+
{
|
| 743 |
+
"path": "runtime/source/vda/video_depth_anything/dinov2_layers/swiglu_ffn.py",
|
| 744 |
+
"bytes": 1859,
|
| 745 |
+
"sha256": "df81d9901240214edc1831ce1b77558dcd6227b29a2f000fc5f546df55d2d09a"
|
| 746 |
+
},
|
| 747 |
+
{
|
| 748 |
+
"path": "runtime/source/vda/video_depth_anything/dpt.py",
|
| 749 |
+
"bytes": 5467,
|
| 750 |
+
"sha256": "4751fac16206892889d74fd7f564f275e0efa3a903172c9350a1b55ed850c7d8"
|
| 751 |
+
},
|
| 752 |
+
{
|
| 753 |
+
"path": "runtime/source/vda/video_depth_anything/dpt_temporal.py",
|
| 754 |
+
"bytes": 5909,
|
| 755 |
+
"sha256": "b80c448f4fe66d0ca7092be887c0ea3925a4ce7c03cffa0083480e7e2f55805d"
|
| 756 |
+
},
|
| 757 |
+
{
|
| 758 |
+
"path": "runtime/source/vda/video_depth_anything/motion_module/attention.py",
|
| 759 |
+
"bytes": 16719,
|
| 760 |
+
"sha256": "15e4049641f04f73027ea520ada3ba2c81196cea1339f0d6a6063d944df61c8b"
|
| 761 |
+
},
|
| 762 |
+
{
|
| 763 |
+
"path": "runtime/source/vda/video_depth_anything/motion_module/motion_module.py",
|
| 764 |
+
"bytes": 12744,
|
| 765 |
+
"sha256": "73d6a9cee40681532ae2603dd426405e840dc5c592a313005fa367f6fe4c69e8"
|
| 766 |
+
},
|
| 767 |
+
{
|
| 768 |
+
"path": "runtime/source/vda/video_depth_anything/util/blocks.py",
|
| 769 |
+
"bytes": 4064,
|
| 770 |
+
"sha256": "8a9fde84bddbfa2498b5f5ef06895ddc941c1fc12c5919e9891fad481997b7dd"
|
| 771 |
+
},
|
| 772 |
+
{
|
| 773 |
+
"path": "runtime/source/vda/video_depth_anything/util/transform.py",
|
| 774 |
+
"bytes": 6075,
|
| 775 |
+
"sha256": "a97df1dbb687457a40c387fdc3482edee7ddc5c142c7d2735ca6235a8ae2885f"
|
| 776 |
+
},
|
| 777 |
+
{
|
| 778 |
+
"path": "runtime/source/vda/video_depth_anything/video_depth.py",
|
| 779 |
+
"bytes": 6712,
|
| 780 |
+
"sha256": "208e882d60a41434b3a0d9935a7025b01cce0756c17b3d889c6db0c4c945eb29"
|
| 781 |
+
},
|
| 782 |
+
{
|
| 783 |
+
"path": "runtime/source/vda/video_depth_anything/video_depth_stream.py",
|
| 784 |
+
"bytes": 6833,
|
| 785 |
+
"sha256": "23ebab70525f318cf0bfeba38dfcac3f1c345ba50aaa1bb5c6dbae671b0caa6d"
|
| 786 |
+
},
|
| 787 |
+
{
|
| 788 |
+
"path": "runtime/spacedreamer_galexea.py",
|
| 789 |
+
"bytes": 14685,
|
| 790 |
+
"sha256": "d694e516d3f6baca2a49df8086c95b5438147091c9eb27bc684f16ffd319942f"
|
| 791 |
+
},
|
| 792 |
+
{
|
| 793 |
+
"path": "text_context.pt",
|
| 794 |
+
"bytes": 1050977,
|
| 795 |
+
"sha256": "49cf3918423e55ed12776ac91546bd4efb4142fd4645309b34b8526953a7cc92"
|
| 796 |
+
},
|
| 797 |
+
{
|
| 798 |
+
"path": "training_complete.json",
|
| 799 |
+
"bytes": 358,
|
| 800 |
+
"sha256": "92e46d8dd3370060bacf464880b0fcec4b712629c83e06b0203cfec1baccb07a"
|
| 801 |
+
},
|
| 802 |
+
{
|
| 803 |
+
"path": "training_config.json",
|
| 804 |
+
"bytes": 7131,
|
| 805 |
+
"sha256": "a94d0f7e603f5f905ee694c7bed920693bb0b32db77869f80878855fe8311718"
|
| 806 |
+
},
|
| 807 |
+
{
|
| 808 |
+
"path": "training_reference/code/audit.py",
|
| 809 |
+
"bytes": 1956,
|
| 810 |
+
"sha256": "110519fba757e41fb284c9d8853e416f4b9312fe8d33d7d79519fded2a41cc83"
|
| 811 |
+
},
|
| 812 |
+
{
|
| 813 |
+
"path": "training_reference/code/build_smoke_sample.py",
|
| 814 |
+
"bytes": 2493,
|
| 815 |
+
"sha256": "aa2976e044bab50de47268fff191dc73122308ce4d3188169a2cf06f046e6263"
|
| 816 |
+
},
|
| 817 |
+
{
|
| 818 |
+
"path": "training_reference/code/cache_calibrations.py",
|
| 819 |
+
"bytes": 2398,
|
| 820 |
+
"sha256": "65c51b36d239cbbfd4f9dea9d40280f445bcc276bb5164ea9a71806fb1f36e19"
|
| 821 |
+
},
|
| 822 |
+
{
|
| 823 |
+
"path": "training_reference/code/calibration.py",
|
| 824 |
+
"bytes": 3092,
|
| 825 |
+
"sha256": "39f4cfc0ab2a841a2e9c44f04af149aee9b2b52db4c42e6feab2b54b75d4c0ee"
|
| 826 |
+
},
|
| 827 |
+
{
|
| 828 |
+
"path": "training_reference/code/check_global_batch.py",
|
| 829 |
+
"bytes": 2872,
|
| 830 |
+
"sha256": "de59449caa64e50eef846768454895d4d904daf6339f797be80ea959fbd23e6a"
|
| 831 |
+
},
|
| 832 |
+
{
|
| 833 |
+
"path": "training_reference/code/cluster_entry.py",
|
| 834 |
+
"bytes": 7841,
|
| 835 |
+
"sha256": "d42963927ce7637ebd5ad2bee0102b595dff41b564aa318c81781de833d7cd98"
|
| 836 |
+
},
|
| 837 |
+
{
|
| 838 |
+
"path": "training_reference/code/common.py",
|
| 839 |
+
"bytes": 5682,
|
| 840 |
+
"sha256": "54bdfd54a3dca23bfa7073d8d85f011b5009bd81623f0dac7d5a8c0ca561134d"
|
| 841 |
+
},
|
| 842 |
+
{
|
| 843 |
+
"path": "training_reference/code/cpu_bootstrap.py",
|
| 844 |
+
"bytes": 315,
|
| 845 |
+
"sha256": "59ffd83c32036d4bd3a83b01f480058e51f3f5a04838500c85a197afe7d0ae50"
|
| 846 |
+
},
|
| 847 |
+
{
|
| 848 |
+
"path": "training_reference/code/cpu_preflight.py",
|
| 849 |
+
"bytes": 3879,
|
| 850 |
+
"sha256": "53455315a2aa2141293e111e3cbd67b5bc9b600e5df3c69938c03c791713a5b3"
|
| 851 |
+
},
|
| 852 |
+
{
|
| 853 |
+
"path": "training_reference/code/cuda_probe.py",
|
| 854 |
+
"bytes": 2547,
|
| 855 |
+
"sha256": "01949bf05a92ba781aa65f929d5a6bd7bb7972231524b89e3186ee18f14f12c2"
|
| 856 |
+
},
|
| 857 |
+
{
|
| 858 |
+
"path": "training_reference/code/data.py",
|
| 859 |
+
"bytes": 14912,
|
| 860 |
+
"sha256": "697b7dd046b054e2db0c2e61f8350b02b3a973458ac9670edb295f3903c156d6"
|
| 861 |
+
},
|
| 862 |
+
{
|
| 863 |
+
"path": "training_reference/code/decode_selected.py",
|
| 864 |
+
"bytes": 695,
|
| 865 |
+
"sha256": "6754d0f88170e5e87dd2ebe733c350c405debd3492801d71693ce31038bc15f8"
|
| 866 |
+
},
|
| 867 |
+
{
|
| 868 |
+
"path": "training_reference/code/distributed.py",
|
| 869 |
+
"bytes": 1052,
|
| 870 |
+
"sha256": "cdafe58a0101d1adb508b44d06f1dfbf4c52d8fc73ba66cbcdbf7fd3d0dfa843"
|
| 871 |
+
},
|
| 872 |
+
{
|
| 873 |
+
"path": "training_reference/code/model.py",
|
| 874 |
+
"bytes": 7629,
|
| 875 |
+
"sha256": "99bafc6873be3cca2d016df41c30c7741841701518256617038f36972969ef3e"
|
| 876 |
+
},
|
| 877 |
+
{
|
| 878 |
+
"path": "training_reference/code/output_guard.py",
|
| 879 |
+
"bytes": 1453,
|
| 880 |
+
"sha256": "e9c1c6192dc71c478c7d612d544e238d8995d5b45bbf38145f3dd66ae41d4969"
|
| 881 |
+
},
|
| 882 |
+
{
|
| 883 |
+
"path": "training_reference/code/prepare.py",
|
| 884 |
+
"bytes": 7614,
|
| 885 |
+
"sha256": "7e63fdae5a8b765a206c5c2196d507c96770fbdb73b9356cc1b9faa8438df088"
|
| 886 |
+
},
|
| 887 |
+
{
|
| 888 |
+
"path": "training_reference/code/selection_data.py",
|
| 889 |
+
"bytes": 1297,
|
| 890 |
+
"sha256": "175ed3df31c6e13ed2276078e888e8605485d1aff26282a861a6bbbe4c879be3"
|
| 891 |
+
},
|
| 892 |
+
{
|
| 893 |
+
"path": "training_reference/code/staging.py",
|
| 894 |
+
"bytes": 2327,
|
| 895 |
+
"sha256": "6d8f94c6e24db6373fd1129fdf7ee0387c03b9505b37c77f0c421b56c3cf7521"
|
| 896 |
+
},
|
| 897 |
+
{
|
| 898 |
+
"path": "training_reference/code/storage_budget.py",
|
| 899 |
+
"bytes": 12441,
|
| 900 |
+
"sha256": "e7ed733b8ccce3480d59f46c54293cf469b34269e2bc301203ebb71a00b8bb2c"
|
| 901 |
+
},
|
| 902 |
+
{
|
| 903 |
+
"path": "training_reference/code/submit.py",
|
| 904 |
+
"bytes": 4656,
|
| 905 |
+
"sha256": "70dd48052e9b5b8ecea25605811bba0f9e20d441e28b664fb3af860f787910c2"
|
| 906 |
+
},
|
| 907 |
+
{
|
| 908 |
+
"path": "training_reference/code/submit20.py",
|
| 909 |
+
"bytes": 6964,
|
| 910 |
+
"sha256": "41763dfa7b49b466462fca56c2bf4d27dea2ee06c6287c47bfbebaf12ed4ca4a"
|
| 911 |
+
},
|
| 912 |
+
{
|
| 913 |
+
"path": "training_reference/code/train.py",
|
| 914 |
+
"bytes": 14828,
|
| 915 |
+
"sha256": "28a320e6fa8a737ce71e880d5d1103246b1e054975c0db9723ed9a0170ed0788"
|
| 916 |
+
},
|
| 917 |
+
{
|
| 918 |
+
"path": "training_reference/code/vae_runtime.py",
|
| 919 |
+
"bytes": 1005,
|
| 920 |
+
"sha256": "ab129de4f7d5aff6271176b6305d00ba78931b4510595a41d0f2c28cd86192fe"
|
| 921 |
+
}
|
| 922 |
+
],
|
| 923 |
+
"total_bytes": 11033164188,
|
| 924 |
+
"shared_dependencies": {
|
| 925 |
+
"spacedreamer_20epochs/assets/Wan2.2-TI2V-5B/Wan2.2_VAE.pth": "20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36",
|
| 926 |
+
"spacedreamer_20epochs/assets/metric_video_depth_anything_vitb.pth": "f6f58576b9680a112f2428d4f39aff92656d3ae85745b0164675ace8d5b1fade"
|
| 927 |
+
}
|
| 928 |
+
}
|
spacedreamer_stack_cups_clean40_30ep/postflight_normalizer_audit.json
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "TRAINING_COMPLETE_POSTFLIGHT_BF16_COMPARISON_FALSE_POSITIVE_CONFIRMED",
|
| 3 |
+
"platform_state": "JOB_FAILED",
|
| 4 |
+
"job": "bifrost-2026111502493001-shouqx",
|
| 5 |
+
"epochs": 30,
|
| 6 |
+
"optimizer_steps": 4350,
|
| 7 |
+
"rows_of_training_metrics": 4350,
|
| 8 |
+
"all_loss_and_grad_metrics_finite": true,
|
| 9 |
+
"last_metrics": {
|
| 10 |
+
"step": 4350,
|
| 11 |
+
"epoch": 30.0,
|
| 12 |
+
"lr": 3.4098125841150466e-05,
|
| 13 |
+
"seconds": 38622.264301493764,
|
| 14 |
+
"point_prediction_grad_norm": 0.00035875628236681223,
|
| 15 |
+
"loss_rgb": 0.045066364109516144,
|
| 16 |
+
"loss_depth": 0.03409406542778015,
|
| 17 |
+
"loss_point": 0.043846286833286285,
|
| 18 |
+
"loss_action": 0.008644891902804375,
|
| 19 |
+
"video_timestep_mean": 800.71875,
|
| 20 |
+
"action_timestep_mean": 461.4375,
|
| 21 |
+
"prediction_rgb_norm": 1707.3424072265625,
|
| 22 |
+
"prediction_depth_norm": 2257.48876953125,
|
| 23 |
+
"prediction_point_norm": 1777.875732421875,
|
| 24 |
+
"prediction_action_norm": 107.14266967773438,
|
| 25 |
+
"grad_norm": 0.3622361421585083
|
| 26 |
+
},
|
| 27 |
+
"last100_mean_loss_action": 0.008668523519299925,
|
| 28 |
+
"weights": "/workspace/group_share/rc-wbc/shouqx@xiaopeng.com/real_robot_training/stack_cups_new67_clean40_refitnorm_30ep_16h200_r2_20261114/runs/stack_cups/final_weights.pt",
|
| 29 |
+
"weights_bytes": 11030029192,
|
| 30 |
+
"normalizer_sha256": "10f598cd5148e9ac85fa2b7b8c3b2f2c503127fcb307f33ad3e5277f8a9ff6f8",
|
| 31 |
+
"checks": [
|
| 32 |
+
{
|
| 33 |
+
"buffer": "state_q01",
|
| 34 |
+
"dtype": "torch.bfloat16",
|
| 35 |
+
"matches_exact_expected_after_dtype_conversion": true,
|
| 36 |
+
"max_joint_rounding_rad": 0.0037899017333984375,
|
| 37 |
+
"max_gripper_rounding_raw": 0.023340225219726562
|
| 38 |
+
},
|
| 39 |
+
{
|
| 40 |
+
"buffer": "state_q99",
|
| 41 |
+
"dtype": "torch.bfloat16",
|
| 42 |
+
"matches_exact_expected_after_dtype_conversion": true,
|
| 43 |
+
"max_joint_rounding_rad": 0.005093097686767578,
|
| 44 |
+
"max_gripper_rounding_raw": 0.19965362548828125
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"buffer": "action_q01",
|
| 48 |
+
"dtype": "torch.bfloat16",
|
| 49 |
+
"matches_exact_expected_after_dtype_conversion": true,
|
| 50 |
+
"max_joint_rounding_rad": 0.0017759203910827637,
|
| 51 |
+
"max_gripper_rounding_raw": 0.0
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"buffer": "action_q99",
|
| 55 |
+
"dtype": "torch.bfloat16",
|
| 56 |
+
"matches_exact_expected_after_dtype_conversion": true,
|
| 57 |
+
"max_joint_rounding_rad": 0.00553131103515625,
|
| 58 |
+
"max_gripper_rounding_raw": 0.19965362548828125
|
| 59 |
+
}
|
| 60 |
+
],
|
| 61 |
+
"root_cause": "Export verifier converted quantized BF16 buffers back to FP32 and required bitwise equality to original FP32 statistics. All4 buffers exactly match correctly quantized selected40 statistics. Data pipeline normalizes from the original JSON, not these stored buffers.",
|
| 62 |
+
"inference_note": "Use accompanying reference_norm_stats.json for exact FP32 normalization. No new GPU inference acceptance is claimed by this metadata/buffer audit.",
|
| 63 |
+
"checked_utc": "2026-11-15T06:13:04Z"
|
| 64 |
+
}
|
spacedreamer_stack_cups_clean40_30ep/reference_norm_stats.json
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"norm_stats": {
|
| 3 |
+
"state": {
|
| 4 |
+
"mean": [
|
| 5 |
+
-0.13387035604186223,
|
| 6 |
+
0.9388066769062815,
|
| 7 |
+
-0.299460975359783,
|
| 8 |
+
-0.312899377843042,
|
| 9 |
+
0.022158233386334263,
|
| 10 |
+
-0.013364365686538105,
|
| 11 |
+
87.67583665493922,
|
| 12 |
+
0.12845319557789786,
|
| 13 |
+
1.0475177045852482,
|
| 14 |
+
-0.3682720079020069,
|
| 15 |
+
-0.4035744942019698,
|
| 16 |
+
0.09048382540451419,
|
| 17 |
+
0.05871849529736292,
|
| 18 |
+
88.37490834410208
|
| 19 |
+
],
|
| 20 |
+
"std": [
|
| 21 |
+
0.23867227111243974,
|
| 22 |
+
1.0532982660446137,
|
| 23 |
+
0.4497836674554094,
|
| 24 |
+
0.521777337868213,
|
| 25 |
+
0.0756900378886052,
|
| 26 |
+
0.06476722031329171,
|
| 27 |
+
26.71132088139677,
|
| 28 |
+
0.1772370447709506,
|
| 29 |
+
1.0844598923871354,
|
| 30 |
+
0.49809485962894984,
|
| 31 |
+
0.5703715013499049,
|
| 32 |
+
0.1773866395459158,
|
| 33 |
+
0.07316617681887644,
|
| 34 |
+
23.618158731215697
|
| 35 |
+
],
|
| 36 |
+
"q01": [
|
| 37 |
+
-0.7902127504348755,
|
| 38 |
+
0.21595744788646698,
|
| 39 |
+
-1.4180850982666016,
|
| 40 |
+
-1.5263829231262207,
|
| 41 |
+
-0.26163404583930966,
|
| 42 |
+
-0.22297872602939606,
|
| 43 |
+
12.665559768676758,
|
| 44 |
+
-0.04957446828484535,
|
| 45 |
+
0.19914893805980682,
|
| 46 |
+
-1.4697531938552857,
|
| 47 |
+
-1.5238298177719116,
|
| 48 |
+
-0.19659574329853058,
|
| 49 |
+
-0.07021276652812958,
|
| 50 |
+
18.101660156250002
|
| 51 |
+
],
|
| 52 |
+
"q99": [
|
| 53 |
+
0.0274468082934618,
|
| 54 |
+
2.8855319023132324,
|
| 55 |
+
-0.004680851008743048,
|
| 56 |
+
0.15954042673110888,
|
| 57 |
+
0.2176595777273178,
|
| 58 |
+
0.07531914860010147,
|
| 59 |
+
100.07527160644531,
|
| 60 |
+
0.5876595973968506,
|
| 61 |
+
2.826382875442505,
|
| 62 |
+
-0.003617021255195141,
|
| 63 |
+
0.0663829818367958,
|
| 64 |
+
0.8282978534698486,
|
| 65 |
+
0.2704255282878876,
|
| 66 |
+
101.19965362548828
|
| 67 |
+
]
|
| 68 |
+
},
|
| 69 |
+
"actions": {
|
| 70 |
+
"mean": [
|
| 71 |
+
-0.14292097344549323,
|
| 72 |
+
0.7355806430838492,
|
| 73 |
+
-0.29974345955300896,
|
| 74 |
+
-0.3301010056178284,
|
| 75 |
+
-0.0026388765341016007,
|
| 76 |
+
-0.03810476927142731,
|
| 77 |
+
87.42233316834316,
|
| 78 |
+
0.11110324934059856,
|
| 79 |
+
0.8414215713411128,
|
| 80 |
+
-0.3732627181630525,
|
| 81 |
+
-0.42123973672921317,
|
| 82 |
+
0.06530478815310313,
|
| 83 |
+
0.03338203608640254,
|
| 84 |
+
83.2737688514841
|
| 85 |
+
],
|
| 86 |
+
"std": [
|
| 87 |
+
0.2332689717409583,
|
| 88 |
+
1.0724814487217418,
|
| 89 |
+
0.4633568221251692,
|
| 90 |
+
0.5291241592842331,
|
| 91 |
+
0.07604966254227942,
|
| 92 |
+
0.06502434652450503,
|
| 93 |
+
32.40528197209235,
|
| 94 |
+
0.17417586584814845,
|
| 95 |
+
1.10393937977348,
|
| 96 |
+
0.514695628729752,
|
| 97 |
+
0.5803439554075464,
|
| 98 |
+
0.17785820538290803,
|
| 99 |
+
0.07406519902851658,
|
| 100 |
+
36.22980289874895
|
| 101 |
+
],
|
| 102 |
+
"q01": [
|
| 103 |
+
-0.7869321703910828,
|
| 104 |
+
-0.0,
|
| 105 |
+
-1.4373400211334229,
|
| 106 |
+
-1.5707999467849731,
|
| 107 |
+
-0.2991262674331665,
|
| 108 |
+
-0.24697090685367584,
|
| 109 |
+
0.0,
|
| 110 |
+
-0.07516505569219589,
|
| 111 |
+
-0.0,
|
| 112 |
+
-1.4986991882324219,
|
| 113 |
+
-1.5707999467849731,
|
| 114 |
+
-0.22396118938922882,
|
| 115 |
+
-0.09664078801870346,
|
| 116 |
+
0.0
|
| 117 |
+
],
|
| 118 |
+
"q99": [
|
| 119 |
+
0.012271846644580364,
|
| 120 |
+
2.723245506286619,
|
| 121 |
+
0.0,
|
| 122 |
+
0.15800002217292786,
|
| 123 |
+
0.19174760580062866,
|
| 124 |
+
0.05368932709097862,
|
| 125 |
+
100.0,
|
| 126 |
+
0.5599029660224915,
|
| 127 |
+
2.6507186889648438,
|
| 128 |
+
0.0,
|
| 129 |
+
0.06442718952894211,
|
| 130 |
+
0.8053399324417114,
|
| 131 |
+
0.24697090685367584,
|
| 132 |
+
101.19965362548828
|
| 133 |
+
]
|
| 134 |
+
}
|
| 135 |
+
}
|
| 136 |
+
}
|
spacedreamer_stack_cups_clean40_30ep/runtime/requirements.txt
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CPU packaging validation versions; choose the matching PyTorch CUDA wheel index.
|
| 2 |
+
torch==2.9.0+cu128
|
| 3 |
+
torchvision==0.24.0+cu128
|
| 4 |
+
numpy==1.26.4
|
| 5 |
+
opencv-python-headless==4.11.0.86
|
| 6 |
+
Pillow==12.2.0
|
| 7 |
+
accelerate==1.13.0
|
| 8 |
+
einops==0.8.1
|
| 9 |
+
safetensors==0.7.0
|
| 10 |
+
transformers==4.57.3
|
| 11 |
+
ftfy==6.3.1
|
| 12 |
+
sentencepiece==0.2.1
|
| 13 |
+
PyYAML==6.0.3
|
| 14 |
+
h5py==3.14.0
|
| 15 |
+
zarr==2.18.7
|
| 16 |
+
numcodecs==0.15.1
|
| 17 |
+
fasteners==0.20
|
| 18 |
+
asciitree==0.3.3
|
| 19 |
+
zstandard==0.25.0
|
| 20 |
+
lz4==4.4.5
|
| 21 |
+
easydict==1.13
|
| 22 |
+
huggingface-hub==0.36.2
|
spacedreamer_stack_cups_clean40_30ep/runtime/run_inference.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Predict and save an action chunk from one NPZ observation; no robot I/O."""
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
import argparse
|
| 4 |
+
import numpy as np
|
| 5 |
+
from spacedreamer_galexea import SpaceDreamerGalaxea
|
| 6 |
+
|
| 7 |
+
def main():
|
| 8 |
+
p = argparse.ArgumentParser(description=__doc__)
|
| 9 |
+
p.add_argument('--task-dir', type=Path, required=True)
|
| 10 |
+
p.add_argument('--assets-dir', type=Path)
|
| 11 |
+
p.add_argument('--observation', type=Path, required=True,
|
| 12 |
+
help='NPZ containing head_rgb uint8[512,640,3] or letterboxed[224,224,3], state[14], optional depth_m[224,224]')
|
| 13 |
+
p.add_argument('--output', type=Path, required=True)
|
| 14 |
+
p.add_argument('--steps', type=int, default=10)
|
| 15 |
+
p.add_argument('--seed', type=int, default=0)
|
| 16 |
+
p.add_argument('--device', default='cuda:0')
|
| 17 |
+
a = p.parse_args()
|
| 18 |
+
with np.load(a.observation, allow_pickle=False) as obs:
|
| 19 |
+
rgb, state = obs['head_rgb'], obs['state']
|
| 20 |
+
depth = obs['depth_m'] if 'depth_m' in obs else None
|
| 21 |
+
policy = SpaceDreamerGalaxea(a.task_dir, a.assets_dir, device=a.device)
|
| 22 |
+
actions = policy.infer_action(rgb, state, depth_m=depth,
|
| 23 |
+
num_inference_steps=a.steps, seed=a.seed)
|
| 24 |
+
a.output.parent.mkdir(parents=True, exist_ok=True)
|
| 25 |
+
np.save(a.output, actions, allow_pickle=False)
|
| 26 |
+
print(f'Saved {actions.shape} {actions.dtype} absolute commands to {a.output}; no robot commands sent.')
|
| 27 |
+
|
| 28 |
+
if __name__ == '__main__':
|
| 29 |
+
main()
|
spacedreamer_stack_cups_clean40_30ep/runtime/runtime_packages.json
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"python": "3.12.14",
|
| 3 |
+
"context": "Local CPU packaging validation runtime; remote training pod package dump is not claimed.",
|
| 4 |
+
"packages": {
|
| 5 |
+
"absl-py": "2.4.0",
|
| 6 |
+
"accelerate": "1.13.0",
|
| 7 |
+
"aiohappyeyeballs": "2.6.1",
|
| 8 |
+
"aiohttp": "3.14.0",
|
| 9 |
+
"aiosignal": "1.4.0",
|
| 10 |
+
"albucore": "0.0.17",
|
| 11 |
+
"albumentations": "1.4.18",
|
| 12 |
+
"annotated-types": "0.7.0",
|
| 13 |
+
"antlr4-python3-runtime": "4.9.3",
|
| 14 |
+
"asciitree": "0.3.3",
|
| 15 |
+
"attrs": "25.4.0",
|
| 16 |
+
"av": "15.1.0",
|
| 17 |
+
"boto3": "1.35.99",
|
| 18 |
+
"botocore": "1.35.99",
|
| 19 |
+
"certifi": "2026.7.22",
|
| 20 |
+
"cffi": "2.0.0",
|
| 21 |
+
"charset-normalizer": "3.5.2",
|
| 22 |
+
"click": "8.1.8",
|
| 23 |
+
"cloudpickle": "3.1.2",
|
| 24 |
+
"contourpy": "1.3.2",
|
| 25 |
+
"cryptography": "48.0.0",
|
| 26 |
+
"cycler": "0.12.1",
|
| 27 |
+
"datasets": "3.6.0",
|
| 28 |
+
"deepspeed": "0.17.6",
|
| 29 |
+
"diffusers": "0.35.1",
|
| 30 |
+
"dill": "0.3.8",
|
| 31 |
+
"dm-tree": "0.1.9",
|
| 32 |
+
"docstring-parser": "0.17.0",
|
| 33 |
+
"easydict": "1.13",
|
| 34 |
+
"einops": "0.8.1",
|
| 35 |
+
"eval-type-backport": "0.3.1",
|
| 36 |
+
"farama-notifications": "0.0.4",
|
| 37 |
+
"fasteners": "0.20",
|
| 38 |
+
"fastwam": "0.1.0",
|
| 39 |
+
"filelock": "4.0.9",
|
| 40 |
+
"flash-attn": "2.8.3",
|
| 41 |
+
"fonttools": "4.62.1",
|
| 42 |
+
"frozenlist": "1.8.0",
|
| 43 |
+
"fsspec": "2025.3.0",
|
| 44 |
+
"ftfy": "6.3.1",
|
| 45 |
+
"gitdb": "4.0.12",
|
| 46 |
+
"gitpython": "3.1.50",
|
| 47 |
+
"gr00t": "0.1.0",
|
| 48 |
+
"gymnasium": "1.2.2",
|
| 49 |
+
"h5py": "3.14.0",
|
| 50 |
+
"hf-xet": "1.4.2",
|
| 51 |
+
"hjson": "3.1.0",
|
| 52 |
+
"huggingface-hub": "0.36.2",
|
| 53 |
+
"hydra-core": "1.3.2",
|
| 54 |
+
"idna": "3.20",
|
| 55 |
+
"imageio": "2.37.3",
|
| 56 |
+
"imageio-ffmpeg": "0.6.0",
|
| 57 |
+
"importlib-metadata": "8.7.1",
|
| 58 |
+
"inquirerpy": "0.3.4",
|
| 59 |
+
"jinja2": "3.1.6",
|
| 60 |
+
"jmespath": "1.1.0",
|
| 61 |
+
"jsonlines": "4.0.0",
|
| 62 |
+
"kiwisolver": "1.5.0",
|
| 63 |
+
"lazy-loader": "0.5",
|
| 64 |
+
"lmdb": "1.7.5",
|
| 65 |
+
"lz4": "4.4.5",
|
| 66 |
+
"markdown-it-py": "4.0.0",
|
| 67 |
+
"markupsafe": "3.0.3",
|
| 68 |
+
"matplotlib": "3.10.1",
|
| 69 |
+
"mdurl": "0.1.2",
|
| 70 |
+
"ml-dtypes": "0.5.4",
|
| 71 |
+
"modelscope": "1.34.0",
|
| 72 |
+
"mpmath": "1.3.0",
|
| 73 |
+
"msgpack": "1.1.0",
|
| 74 |
+
"msgpack-numpy": "0.4.8",
|
| 75 |
+
"multidict": "6.7.1",
|
| 76 |
+
"multiprocess": "0.70.16",
|
| 77 |
+
"networkx": "3.4.2",
|
| 78 |
+
"ninja": "1.13.0",
|
| 79 |
+
"numcodecs": "0.15.1",
|
| 80 |
+
"numpy": "1.26.4",
|
| 81 |
+
"nvidia-cublas-cu12": "12.8.4.1",
|
| 82 |
+
"nvidia-cuda-cupti-cu12": "12.8.90",
|
| 83 |
+
"nvidia-cuda-nvrtc-cu12": "12.8.93",
|
| 84 |
+
"nvidia-cuda-runtime-cu12": "12.8.90",
|
| 85 |
+
"nvidia-cudnn-cu12": "9.10.2.21",
|
| 86 |
+
"nvidia-cufft-cu12": "11.3.3.83",
|
| 87 |
+
"nvidia-cufile-cu12": "1.13.1.3",
|
| 88 |
+
"nvidia-curand-cu12": "10.3.9.90",
|
| 89 |
+
"nvidia-cusolver-cu12": "11.7.3.90",
|
| 90 |
+
"nvidia-cusparse-cu12": "12.5.8.93",
|
| 91 |
+
"nvidia-cusparselt-cu12": "0.7.1",
|
| 92 |
+
"nvidia-nccl-cu12": "2.27.5",
|
| 93 |
+
"nvidia-nvjitlink-cu12": "12.8.93",
|
| 94 |
+
"nvidia-nvshmem-cu12": "3.3.20",
|
| 95 |
+
"nvidia-nvtx-cu12": "12.8.90",
|
| 96 |
+
"omegaconf": "2.3.1",
|
| 97 |
+
"onnx": "1.20.1",
|
| 98 |
+
"onnx-ir": "0.2.1",
|
| 99 |
+
"onnxscript": "0.7.0",
|
| 100 |
+
"opencv-python-headless": "4.11.0.86",
|
| 101 |
+
"packaging": "26.3",
|
| 102 |
+
"pandas": "2.2.3",
|
| 103 |
+
"peft": "0.17.1",
|
| 104 |
+
"pfzy": "0.3.4",
|
| 105 |
+
"pillow": "12.2.0",
|
| 106 |
+
"pip": "26.2.1",
|
| 107 |
+
"platformdirs": "4.9.4",
|
| 108 |
+
"prompt-toolkit": "3.0.52",
|
| 109 |
+
"propcache": "0.4.1",
|
| 110 |
+
"protobuf": "6.33.5",
|
| 111 |
+
"psutil": "7.2.2",
|
| 112 |
+
"py-cpuinfo": "9.0.0",
|
| 113 |
+
"pyarrow": "23.0.1",
|
| 114 |
+
"pycparser": "3.0",
|
| 115 |
+
"pydantic": "2.12.5",
|
| 116 |
+
"pydantic-core": "2.41.5",
|
| 117 |
+
"pygments": "2.20.0",
|
| 118 |
+
"pyparsing": "3.3.2",
|
| 119 |
+
"python-dateutil": "2.9.0.post0",
|
| 120 |
+
"pytz": "2026.1.post1",
|
| 121 |
+
"pyyaml": "6.0.3",
|
| 122 |
+
"pyzmq": "27.0.1",
|
| 123 |
+
"regex": "2026.2.28",
|
| 124 |
+
"requests": "2.34.2",
|
| 125 |
+
"rich": "14.3.3",
|
| 126 |
+
"s3transfer": "0.10.4",
|
| 127 |
+
"safetensors": "0.7.0",
|
| 128 |
+
"scikit-image": "0.25.2",
|
| 129 |
+
"scipy": "1.15.3",
|
| 130 |
+
"sentencepiece": "0.2.1",
|
| 131 |
+
"sentry-sdk": "2.55.0",
|
| 132 |
+
"setuptools": "84.0.0",
|
| 133 |
+
"shtab": "1.8.0",
|
| 134 |
+
"six": "1.17.0",
|
| 135 |
+
"smmap": "5.0.3",
|
| 136 |
+
"sympy": "1.14.0",
|
| 137 |
+
"termcolor": "3.2.0",
|
| 138 |
+
"tifffile": "2025.5.10",
|
| 139 |
+
"tokenizers": "0.22.2",
|
| 140 |
+
"torch": "2.9.0+cu128",
|
| 141 |
+
"torchcodec": "0.8.0",
|
| 142 |
+
"torchvision": "0.24.0+cu128",
|
| 143 |
+
"tqdm": "4.70.1",
|
| 144 |
+
"transformers": "4.57.3",
|
| 145 |
+
"triton": "3.5.0",
|
| 146 |
+
"typeguard": "4.5.1",
|
| 147 |
+
"typing-extensions": "4.15.0",
|
| 148 |
+
"typing-inspection": "0.4.2",
|
| 149 |
+
"tyro": "0.9.17",
|
| 150 |
+
"tzdata": "2025.3",
|
| 151 |
+
"urllib3": "2.8.0",
|
| 152 |
+
"wandb": "0.23.0",
|
| 153 |
+
"wcwidth": "0.6.0",
|
| 154 |
+
"wrapt": "2.1.2",
|
| 155 |
+
"xxhash": "3.6.0",
|
| 156 |
+
"yarl": "1.23.0",
|
| 157 |
+
"zarr": "2.18.7",
|
| 158 |
+
"zipp": "3.23.0",
|
| 159 |
+
"zstandard": "0.25.0"
|
| 160 |
+
}
|
| 161 |
+
}
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""FastWAM package."""
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/__init__.py
ADDED
|
File without changes
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/dataset_utils.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
import torchvision.transforms.functional as transforms_F
|
| 6 |
+
from PIL import Image
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def obtain_image_size(data: torch.Tensor | Image.Image) -> tuple[int, int]:
|
| 10 |
+
r"""Return spatial size from a PIL image or image/video tensor.
|
| 11 |
+
|
| 12 |
+
Args:
|
| 13 |
+
data (torch.Tensor | Image.Image): Input image or video tensor.
|
| 14 |
+
Returns:
|
| 15 |
+
width (int): Input width.
|
| 16 |
+
height (int): Input height.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
if isinstance(data, Image.Image):
|
| 20 |
+
width, height = data.size
|
| 21 |
+
elif isinstance(data, torch.Tensor):
|
| 22 |
+
height, width = data.size()[-2:]
|
| 23 |
+
else:
|
| 24 |
+
raise ValueError("data to random crop should be PIL Image or tensor")
|
| 25 |
+
|
| 26 |
+
return width, height
|
| 27 |
+
|
| 28 |
+
class ResizeSmallestSideAspectPreserving:
|
| 29 |
+
def __init__(self, args: Optional[dict] = None) -> None:
|
| 30 |
+
self.args = args
|
| 31 |
+
|
| 32 |
+
def __call__(self, video: torch.Tensor | Image.Image) -> torch.Tensor | Image.Image:
|
| 33 |
+
r"""Resize while preserving aspect ratio.
|
| 34 |
+
|
| 35 |
+
The output is scaled so both spatial dimensions are at least the
|
| 36 |
+
requested target size.
|
| 37 |
+
|
| 38 |
+
Args:
|
| 39 |
+
video (torch.Tensor | Image.Image): Input image or video tensor.
|
| 40 |
+
Returns:
|
| 41 |
+
torch.Tensor | Image.Image: Resized image or video tensor.
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
assert self.args is not None, "Please specify args in augmentations"
|
| 45 |
+
|
| 46 |
+
img_w, img_h = self.args["img_w"], self.args["img_h"]
|
| 47 |
+
|
| 48 |
+
orig_w, orig_h = obtain_image_size(video)
|
| 49 |
+
scaling_ratio = max((img_w / orig_w), (img_h / orig_h))
|
| 50 |
+
target_size = (int(scaling_ratio * orig_h + 0.5), int(scaling_ratio * orig_w + 0.5))
|
| 51 |
+
|
| 52 |
+
assert (
|
| 53 |
+
target_size[0] >= img_h and target_size[1] >= img_w
|
| 54 |
+
), f"Resize error. orig {(orig_w, orig_h)} desire {(img_w, img_h)} compute {target_size}"
|
| 55 |
+
|
| 56 |
+
return transforms_F.resize(
|
| 57 |
+
video,
|
| 58 |
+
size=target_size, # type: ignore
|
| 59 |
+
interpolation=self.args.get("interpolation", transforms_F.InterpolationMode.BICUBIC),
|
| 60 |
+
antialias=True,
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
class CenterCrop:
|
| 65 |
+
def __init__(self, args: Optional[dict] = None) -> None:
|
| 66 |
+
self.args = args
|
| 67 |
+
|
| 68 |
+
def __call__(self, video: torch.Tensor | Image.Image) -> torch.Tensor | Image.Image:
|
| 69 |
+
r"""Center crop to the requested spatial size.
|
| 70 |
+
|
| 71 |
+
Args:
|
| 72 |
+
video (torch.Tensor | Image.Image): Input image or video tensor.
|
| 73 |
+
Returns:
|
| 74 |
+
torch.Tensor | Image.Image: Center cropped image or video tensor.
|
| 75 |
+
"""
|
| 76 |
+
assert (
|
| 77 |
+
(self.args is not None) and ("img_w" in self.args) and ("img_h" in self.args)
|
| 78 |
+
), "Please specify size in args"
|
| 79 |
+
|
| 80 |
+
img_w, img_h = self.args["img_w"], self.args["img_h"]
|
| 81 |
+
return transforms_F.center_crop(video, [img_h, img_w])
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class Normalize:
|
| 85 |
+
def __init__(self, args: Optional[dict] = None) -> None:
|
| 86 |
+
self.args = args
|
| 87 |
+
|
| 88 |
+
def __call__(self, video: torch.Tensor | Image.Image) -> torch.Tensor:
|
| 89 |
+
r"""Convert to tensor if needed and normalize by mean/std.
|
| 90 |
+
|
| 91 |
+
Args:
|
| 92 |
+
video (torch.Tensor | Image.Image): Input image or video tensor.
|
| 93 |
+
Returns:
|
| 94 |
+
torch.Tensor: Normalized image or video tensor.
|
| 95 |
+
"""
|
| 96 |
+
assert self.args is not None, "Please specify args"
|
| 97 |
+
|
| 98 |
+
mean = self.args["mean"]
|
| 99 |
+
std = self.args["std"]
|
| 100 |
+
|
| 101 |
+
if isinstance(video, torch.Tensor):
|
| 102 |
+
data = video.to(dtype=torch.float32)
|
| 103 |
+
if video.dtype == torch.uint8:
|
| 104 |
+
data = data / 255.0
|
| 105 |
+
data = data.to(dtype=torch.get_default_dtype())
|
| 106 |
+
else:
|
| 107 |
+
data = transforms_F.to_tensor(video) # division by 255 is applied in to_tensor()
|
| 108 |
+
|
| 109 |
+
return transforms_F.normalize(tensor=data, mean=mean, std=std)
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/base_lerobot_dataset.py
ADDED
|
@@ -0,0 +1,412 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import numpy as np
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from typing import List, Literal, Dict, Optional, Any, DefaultDict
|
| 5 |
+
from tqdm import tqdm
|
| 6 |
+
from .lerobot.lerobot_dataset import LeRobotDatasetMetadata, MultiLeRobotDataset
|
| 7 |
+
|
| 8 |
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 9 |
+
import traceback
|
| 10 |
+
from fastwam.utils.logging_config import get_logger
|
| 11 |
+
from .processors.base_processor import BaseProcessor
|
| 12 |
+
|
| 13 |
+
logger = get_logger(__name__)
|
| 14 |
+
|
| 15 |
+
MAX_GETITEM_ATTEMPT = 5
|
| 16 |
+
|
| 17 |
+
class BaseLerobotDataset(torch.utils.data.Dataset):
|
| 18 |
+
def __init__(
|
| 19 |
+
self,
|
| 20 |
+
dataset_dirs: List[str],
|
| 21 |
+
|
| 22 |
+
# shapes
|
| 23 |
+
shape_meta: Dict[str, Any],
|
| 24 |
+
action_size: int = 1,
|
| 25 |
+
past_action_size: int = 0, # Excludes the current frame
|
| 26 |
+
obs_size: int = 1, # should be
|
| 27 |
+
past_obs_size: int = 0,
|
| 28 |
+
|
| 29 |
+
# train vs val
|
| 30 |
+
val_set_proportion: float = 0.05,
|
| 31 |
+
is_training_set: bool = False,
|
| 32 |
+
seed: int = 42,
|
| 33 |
+
|
| 34 |
+
# sampling
|
| 35 |
+
global_sample_stride: int = 1,
|
| 36 |
+
):
|
| 37 |
+
assert len(dataset_dirs) > 0, "At least one dataset directory is required"
|
| 38 |
+
assert past_action_size == 0
|
| 39 |
+
assert past_obs_size == 0
|
| 40 |
+
assert action_size == obs_size - 1, "In this dataset, action_size should be obs_size - 1"
|
| 41 |
+
|
| 42 |
+
self.dataset_dirs = dataset_dirs
|
| 43 |
+
self.shape_meta = shape_meta
|
| 44 |
+
self.action_size = action_size
|
| 45 |
+
self.past_action_size = past_action_size
|
| 46 |
+
self.obs_size = obs_size
|
| 47 |
+
self.processor = None # Will be set externally
|
| 48 |
+
metas = []
|
| 49 |
+
for ds_dir in dataset_dirs:
|
| 50 |
+
ds_root = Path(ds_dir)
|
| 51 |
+
repo_id = ds_dir
|
| 52 |
+
meta = LeRobotDatasetMetadata(repo_id=repo_id, root=ds_root)
|
| 53 |
+
metas.append(meta)
|
| 54 |
+
|
| 55 |
+
fps_list = [m.fps for m in metas]
|
| 56 |
+
assert len(set(fps_list)) == 1, f"All dataset_dirs must have the same fps, got {fps_list}"
|
| 57 |
+
fps = fps_list[0]
|
| 58 |
+
|
| 59 |
+
self.global_sample_stride = global_sample_stride
|
| 60 |
+
|
| 61 |
+
self.val_set_proportion = val_set_proportion
|
| 62 |
+
self.is_training_set = is_training_set
|
| 63 |
+
|
| 64 |
+
self.image_meta = shape_meta["images"]
|
| 65 |
+
self.state_meta = shape_meta["state"]
|
| 66 |
+
self.action_meta = shape_meta["action"]
|
| 67 |
+
|
| 68 |
+
delta_timestamps = {}
|
| 69 |
+
for meta in self.image_meta:
|
| 70 |
+
key = meta["key"]
|
| 71 |
+
meta["lerobot_key"] = f"observation.images.{key}" if key != "default" else "observation.images"
|
| 72 |
+
delta_timestamps[meta["lerobot_key"]] = [
|
| 73 |
+
(t * global_sample_stride) / fps for t in range(-past_obs_size, -past_obs_size + obs_size)
|
| 74 |
+
]
|
| 75 |
+
|
| 76 |
+
for meta in self.state_meta:
|
| 77 |
+
key = meta["key"]
|
| 78 |
+
meta["lerobot_key"] = f"observation.state.{key}" if key != "default" else "observation.state"
|
| 79 |
+
delta_timestamps[meta["lerobot_key"]] = [
|
| 80 |
+
(t * global_sample_stride) / fps for t in range(-past_obs_size, -past_obs_size + obs_size)
|
| 81 |
+
]
|
| 82 |
+
|
| 83 |
+
for meta in self.action_meta:
|
| 84 |
+
key = meta["key"]
|
| 85 |
+
meta["lerobot_key"] = f"action.{key}" if key != "default" else "action"
|
| 86 |
+
delta_timestamps[meta["lerobot_key"]] = [(t * global_sample_stride) / fps for t in range(-past_action_size, -past_action_size + action_size)]
|
| 87 |
+
|
| 88 |
+
episodes = {}
|
| 89 |
+
if val_set_proportion < 1e-6:
|
| 90 |
+
for meta in metas:
|
| 91 |
+
episodes.update({meta.repo_id: list(range(meta.total_episodes))})
|
| 92 |
+
else:
|
| 93 |
+
for meta in metas:
|
| 94 |
+
split_idx = int(meta.total_episodes * (1 - val_set_proportion))
|
| 95 |
+
# random shuffle episode indices before splitting
|
| 96 |
+
episode_indices = list(range(meta.total_episodes))
|
| 97 |
+
rng = np.random.default_rng(seed)
|
| 98 |
+
rng.shuffle(episode_indices)
|
| 99 |
+
if self.is_training_set:
|
| 100 |
+
episodes.update({meta.repo_id: [episode_indices[i] for i in range(split_idx)]})
|
| 101 |
+
else:
|
| 102 |
+
episodes.update({meta.repo_id: [episode_indices[i] for i in range(split_idx, meta.total_episodes)]})
|
| 103 |
+
|
| 104 |
+
self.multi_dataset = MultiLeRobotDataset(
|
| 105 |
+
dataset_dirs=self.dataset_dirs,
|
| 106 |
+
episodes=episodes,
|
| 107 |
+
delta_timestamps=delta_timestamps,
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
# HACK: lerobot 3.0 will fix this
|
| 111 |
+
episode_data_index = []
|
| 112 |
+
end_index = 0
|
| 113 |
+
for dataset in self.multi_dataset._datasets:
|
| 114 |
+
multi_episode_data_index = {
|
| 115 |
+
"from": dataset.episode_data_index["from"] + end_index,
|
| 116 |
+
"to": dataset.episode_data_index["to"] + end_index,
|
| 117 |
+
}
|
| 118 |
+
episode_data_index.append(multi_episode_data_index)
|
| 119 |
+
end_index = multi_episode_data_index["to"][-1]
|
| 120 |
+
|
| 121 |
+
self.episode_data_index = {
|
| 122 |
+
"from": torch.cat([dataset["from"] for dataset in episode_data_index]),
|
| 123 |
+
"to": torch.cat([dataset["to"] for dataset in episode_data_index]),
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
def _get_action(self, meta, lerobot_sample) -> torch.Tensor:
|
| 127 |
+
key, lerobot_key, raw_shape = meta["key"], meta["lerobot_key"], meta["raw_shape"]
|
| 128 |
+
action: torch.Tensor = lerobot_sample[lerobot_key] # [T, action_dim]
|
| 129 |
+
if action.ndim == 1: # for shape of 1, like gripper
|
| 130 |
+
action = action.unsqueeze(-1)
|
| 131 |
+
assert action.shape[-1] == raw_shape, f"Action '{key}' shape {action.shape[-1]} mismatch with meta {raw_shape}."
|
| 132 |
+
return action
|
| 133 |
+
|
| 134 |
+
def _get_state(self, meta, lerobot_sample) -> torch.Tensor:
|
| 135 |
+
key, lerobot_key, raw_shape = meta["key"], meta["lerobot_key"], meta["raw_shape"]
|
| 136 |
+
state: torch.Tensor = lerobot_sample[lerobot_key]
|
| 137 |
+
if state.ndim == 1: # for shape of 1, like gripper
|
| 138 |
+
state = state.unsqueeze(-1)
|
| 139 |
+
# state = state[..., :-1, :] # use state_{t} as observation_t
|
| 140 |
+
assert state.shape[-1] == raw_shape, f"State '{key}' shape {state.shape[-1]} mismatch with meta {raw_shape}."
|
| 141 |
+
return state
|
| 142 |
+
|
| 143 |
+
def _get_image(self, meta, lerobot_sample) -> torch.Tensor:
|
| 144 |
+
key, lerobot_key, raw_shape = meta["key"], meta["lerobot_key"], meta["raw_shape"]
|
| 145 |
+
image: torch.Tensor = lerobot_sample[lerobot_key]
|
| 146 |
+
if image.ndim == 3: # time dim will lost when obs_size is 1
|
| 147 |
+
image = image.unsqueeze(0)
|
| 148 |
+
image = (image * 255).to(torch.uint8) # (1, 3, H, W)
|
| 149 |
+
# For config simplication
|
| 150 |
+
# assert image.shape[1:] == raw_shape, f"Image '{key}' shape {image.shape[1:]} mismatch with {raw_shape}."
|
| 151 |
+
return image
|
| 152 |
+
|
| 153 |
+
def _split_lerobot_sample(self, lerobot_sample) -> Dict[str, Any]:
|
| 154 |
+
return lerobot_sample
|
| 155 |
+
|
| 156 |
+
def _get_episode_data(self, episode_idx):
|
| 157 |
+
lerobot_sample = self.multi_dataset.get_episode_data(episode_idx)
|
| 158 |
+
lerobot_sample = self._split_lerobot_sample(lerobot_sample)
|
| 159 |
+
state, action = {}, {}
|
| 160 |
+
for meta in self.state_meta:
|
| 161 |
+
s = self._get_state(meta, lerobot_sample)
|
| 162 |
+
state[meta["key"]] = s.unsqueeze(1).float()
|
| 163 |
+
for meta in self.action_meta:
|
| 164 |
+
a = self._get_action(meta, lerobot_sample)
|
| 165 |
+
a = sliding_window_with_replication(a, self.action_size)
|
| 166 |
+
action[meta["key"]] = a.float()
|
| 167 |
+
return {"action": action, "state": state}
|
| 168 |
+
|
| 169 |
+
def _set_return_images(self, flag: bool):
|
| 170 |
+
self.return_images = flag
|
| 171 |
+
self.multi_dataset.set_during_training(flag)
|
| 172 |
+
|
| 173 |
+
def __len__(self):
|
| 174 |
+
return self.multi_dataset.num_frames
|
| 175 |
+
|
| 176 |
+
def _get_additional_data(self, sample, lerobot_sample):
|
| 177 |
+
return sample
|
| 178 |
+
|
| 179 |
+
def __getitem__(self, idx):
|
| 180 |
+
if idx >= len(self):
|
| 181 |
+
raise IndexError(f"Index {idx} out of bounds {len(self)}.")
|
| 182 |
+
|
| 183 |
+
# Retry with random indices until we successfully load a frame.
|
| 184 |
+
sample_idx = idx
|
| 185 |
+
attempt = 0
|
| 186 |
+
last_exception: Optional[Exception] = None
|
| 187 |
+
while attempt < MAX_GETITEM_ATTEMPT:
|
| 188 |
+
try:
|
| 189 |
+
lerobot_sample = self.multi_dataset[sample_idx]
|
| 190 |
+
lerobot_sample = self._split_lerobot_sample(lerobot_sample)
|
| 191 |
+
break
|
| 192 |
+
except Exception as err:
|
| 193 |
+
attempt += 1
|
| 194 |
+
last_exception = err
|
| 195 |
+
logger.warning(
|
| 196 |
+
f"Error loading sample {sample_idx} (attempt {attempt}). "
|
| 197 |
+
"Retrying with a random index. "
|
| 198 |
+
f"Error: {err}"
|
| 199 |
+
)
|
| 200 |
+
sample_idx = np.random.randint(len(self))
|
| 201 |
+
print(traceback.format_exc())
|
| 202 |
+
else:
|
| 203 |
+
raise RuntimeError(
|
| 204 |
+
f"Failed to load a valid sample after {MAX_GETITEM_ATTEMPT} attempts "
|
| 205 |
+
f"for index {idx}."
|
| 206 |
+
) from last_exception
|
| 207 |
+
|
| 208 |
+
# Get data from lerobot, organized in nested dict
|
| 209 |
+
sample = {
|
| 210 |
+
"idx": sample_idx,
|
| 211 |
+
"task": lerobot_sample["task"],
|
| 212 |
+
"action": {},
|
| 213 |
+
"state": {},
|
| 214 |
+
"images": {},
|
| 215 |
+
}
|
| 216 |
+
for meta in self.state_meta:
|
| 217 |
+
sample["state"][meta["key"]] = self._get_state(meta, lerobot_sample)
|
| 218 |
+
|
| 219 |
+
for meta in self.action_meta:
|
| 220 |
+
sample["action"][meta["key"]] = self._get_action(meta, lerobot_sample)
|
| 221 |
+
|
| 222 |
+
for meta in self.image_meta:
|
| 223 |
+
sample["images"][meta["key"]] = self._get_image(meta, lerobot_sample)
|
| 224 |
+
|
| 225 |
+
sample["action_is_pad"] = lerobot_sample[f"{self.action_meta[0]['lerobot_key']}_is_pad"]
|
| 226 |
+
sample["state_is_pad"] = lerobot_sample[f"{self.state_meta[0]['lerobot_key']}_is_pad"]
|
| 227 |
+
sample["image_is_pad"] = lerobot_sample[f"{self.image_meta[0]['lerobot_key']}_is_pad"]
|
| 228 |
+
|
| 229 |
+
sample = self._get_additional_data(sample, lerobot_sample)
|
| 230 |
+
|
| 231 |
+
for key in lerobot_sample:
|
| 232 |
+
if key not in sample and "observation" not in key and "action" not in key:
|
| 233 |
+
sample[key] = lerobot_sample[key]
|
| 234 |
+
|
| 235 |
+
# Preprocess the sample using the processor
|
| 236 |
+
# for quick data loading
|
| 237 |
+
if self.processor is not None:
|
| 238 |
+
sample = self.processor.preprocess(sample)
|
| 239 |
+
|
| 240 |
+
return sample
|
| 241 |
+
|
| 242 |
+
def set_processor(self, processor: BaseProcessor):
|
| 243 |
+
"""Set processor instance from external initialization."""
|
| 244 |
+
self.processor = processor
|
| 245 |
+
if self.is_training_set:
|
| 246 |
+
self.processor.train()
|
| 247 |
+
else:
|
| 248 |
+
self.processor.eval()
|
| 249 |
+
return self
|
| 250 |
+
|
| 251 |
+
def get_dataset_stats(self, preprocessor: BaseProcessor):
|
| 252 |
+
state_min = DefaultDict(list)
|
| 253 |
+
state_max = DefaultDict(list)
|
| 254 |
+
state_mean = DefaultDict(list)
|
| 255 |
+
state_var = DefaultDict(list)
|
| 256 |
+
state_q01 = DefaultDict(list)
|
| 257 |
+
state_q99 = DefaultDict(list)
|
| 258 |
+
|
| 259 |
+
action_min = DefaultDict(list)
|
| 260 |
+
action_max = DefaultDict(list)
|
| 261 |
+
action_mean = DefaultDict(list)
|
| 262 |
+
action_var = DefaultDict(list)
|
| 263 |
+
action_q01 = DefaultDict(list)
|
| 264 |
+
action_q99 = DefaultDict(list)
|
| 265 |
+
|
| 266 |
+
episodes_num = self.multi_dataset.num_episodes
|
| 267 |
+
|
| 268 |
+
def process_episode(episode_idx):
|
| 269 |
+
batch = self._get_episode_data(episode_idx)
|
| 270 |
+
batch = preprocessor.action_state_transform(batch)
|
| 271 |
+
return batch
|
| 272 |
+
|
| 273 |
+
multi_thread = True
|
| 274 |
+
if not multi_thread:
|
| 275 |
+
for episode_idx in tqdm(range(episodes_num), desc="Iterating dataset to get normalization"):
|
| 276 |
+
batch = process_episode(episode_idx)
|
| 277 |
+
for meta in self.state_meta:
|
| 278 |
+
key = meta["key"]
|
| 279 |
+
cur_state: torch.Tensor = batch["state"][key] # (B, T, dim)
|
| 280 |
+
state_min[key].append(cur_state.amin(0))
|
| 281 |
+
state_max[key].append(cur_state.amax(0))
|
| 282 |
+
state_mean[key].append(cur_state.mean(0))
|
| 283 |
+
state_var[key].append(cur_state.var(0))
|
| 284 |
+
state_q01[key].append(torch.quantile(cur_state, 0.01, dim=0, keepdim=False))
|
| 285 |
+
state_q99[key].append(torch.quantile(cur_state, 0.99, dim=0, keepdim=False))
|
| 286 |
+
for meta in self.action_meta:
|
| 287 |
+
key = meta["key"]
|
| 288 |
+
cur_action: torch.Tensor = batch["action"][key] # (B, T, dim)
|
| 289 |
+
action_min[key].append(cur_action.amin(0))
|
| 290 |
+
action_max[key].append(cur_action.amax(0))
|
| 291 |
+
action_mean[key].append(cur_action.mean(0))
|
| 292 |
+
action_var[key].append(cur_action.var(0))
|
| 293 |
+
action_q01[key].append(torch.quantile(cur_action, 0.01, dim=0, keepdim=False))
|
| 294 |
+
action_q99[key].append(torch.quantile(cur_action, 0.99, dim=0, keepdim=False))
|
| 295 |
+
|
| 296 |
+
else:
|
| 297 |
+
with ThreadPoolExecutor() as executor:
|
| 298 |
+
futures = [executor.submit(process_episode, num) for num in range(episodes_num)]
|
| 299 |
+
|
| 300 |
+
for future in tqdm(as_completed(futures), total=episodes_num, desc="Iterating dataset to get normalization"):
|
| 301 |
+
try:
|
| 302 |
+
batch = future.result()
|
| 303 |
+
for meta in self.state_meta:
|
| 304 |
+
key = meta["key"]
|
| 305 |
+
cur_state: torch.Tensor = batch["state"][key] # (B, T, dim)
|
| 306 |
+
state_min[key].append(cur_state.amin(0))
|
| 307 |
+
state_max[key].append(cur_state.amax(0))
|
| 308 |
+
state_mean[key].append(cur_state.mean(0))
|
| 309 |
+
state_var[key].append(cur_state.var(0))
|
| 310 |
+
state_q01[key].append(torch.quantile(cur_state, 0.01, dim=0, keepdim=False))
|
| 311 |
+
state_q99[key].append(torch.quantile(cur_state, 0.99, dim=0, keepdim=False))
|
| 312 |
+
|
| 313 |
+
for meta in self.action_meta:
|
| 314 |
+
key = meta["key"]
|
| 315 |
+
cur_action: torch.Tensor = batch["action"][key] # (B, T, dim)
|
| 316 |
+
action_min[key].append(cur_action.amin(0))
|
| 317 |
+
action_max[key].append(cur_action.amax(0))
|
| 318 |
+
action_mean[key].append(cur_action.mean(0))
|
| 319 |
+
action_var[key].append(cur_action.var(0))
|
| 320 |
+
action_q01[key].append(torch.quantile(cur_action, 0.01, dim=0, keepdim=False))
|
| 321 |
+
action_q99[key].append(torch.quantile(cur_action, 0.99, dim=0, keepdim=False))
|
| 322 |
+
|
| 323 |
+
except Exception as e:
|
| 324 |
+
logger.error(f"Error processing episode: {e}")
|
| 325 |
+
print(traceback.format_exc())
|
| 326 |
+
raise e
|
| 327 |
+
|
| 328 |
+
# assume that each minibatch has equal number of samples
|
| 329 |
+
def get_mean_std(means, vars):
|
| 330 |
+
means = torch.stack(means)
|
| 331 |
+
vars = torch.stack(vars)
|
| 332 |
+
stepwise_mean = means.mean(0)
|
| 333 |
+
stepwise_std = (vars + (means - stepwise_mean) ** 2).mean(0).sqrt()
|
| 334 |
+
global_mean = means.mean((0, 1))
|
| 335 |
+
global_std = (vars + (means - global_mean) ** 2).mean((0, 1)).sqrt()
|
| 336 |
+
return stepwise_mean, stepwise_std, global_mean, global_std
|
| 337 |
+
|
| 338 |
+
stats = {"state": DefaultDict(dict), "action": DefaultDict(dict), "num_episodes": episodes_num, "num_transition": self.multi_dataset.num_frames}
|
| 339 |
+
for meta in self.state_meta:
|
| 340 |
+
key = meta["key"]
|
| 341 |
+
stats["state"][key]["stepwise_min"] = torch.stack(state_min[key]).amin(0)
|
| 342 |
+
stats["state"][key]["stepwise_max"] = torch.stack(state_max[key]).amax(0)
|
| 343 |
+
stats["state"][key]["global_min"] = stats["state"][key]["stepwise_min"].amin(0)
|
| 344 |
+
stats["state"][key]["global_max"] = stats["state"][key]["stepwise_max"].amax(0)
|
| 345 |
+
stats["state"][key]["stepwise_q01"] = torch.stack(state_q01[key]).amin(0)
|
| 346 |
+
stats["state"][key]["stepwise_q99"] = torch.stack(state_q99[key]).amax(0)
|
| 347 |
+
stats["state"][key]["global_q01"] = stats["state"][key]["stepwise_q01"].amin(0)
|
| 348 |
+
stats["state"][key]["global_q99"] = stats["state"][key]["stepwise_q99"].amax(0)
|
| 349 |
+
(
|
| 350 |
+
stats["state"][key]["stepwise_mean"],
|
| 351 |
+
stats["state"][key]["stepwise_std"],
|
| 352 |
+
stats["state"][key]["global_mean"],
|
| 353 |
+
stats["state"][key]["global_std"],
|
| 354 |
+
) = get_mean_std(state_mean[key], state_var[key])
|
| 355 |
+
|
| 356 |
+
for meta in self.action_meta:
|
| 357 |
+
key = meta["key"]
|
| 358 |
+
stats["action"][key]["stepwise_min"] = torch.stack(action_min[key]).amin(0)
|
| 359 |
+
stats["action"][key]["stepwise_max"] = torch.stack(action_max[key]).amax(0)
|
| 360 |
+
stats["action"][key]["global_min"] = stats["action"][key]["stepwise_min"].amin(0)
|
| 361 |
+
stats["action"][key]["global_max"] = stats["action"][key]["stepwise_max"].amax(0)
|
| 362 |
+
stats["action"][key]["stepwise_q01"] = torch.stack(action_q01[key]).amin(0)
|
| 363 |
+
stats["action"][key]["stepwise_q99"] = torch.stack(action_q99[key]).amax(0)
|
| 364 |
+
stats["action"][key]["global_q01"] = stats["action"][key]["stepwise_q01"].amin(0)
|
| 365 |
+
stats["action"][key]["global_q99"] = stats["action"][key]["stepwise_q99"].amax(0)
|
| 366 |
+
(
|
| 367 |
+
stats["action"][key]["stepwise_mean"],
|
| 368 |
+
stats["action"][key]["stepwise_std"],
|
| 369 |
+
stats["action"][key]["global_mean"],
|
| 370 |
+
stats["action"][key]["global_std"],
|
| 371 |
+
) = get_mean_std(action_mean[key], action_var[key])
|
| 372 |
+
|
| 373 |
+
return stats
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
def sliding_window_with_replication(x: torch.Tensor, window_size: int) -> torch.Tensor:
|
| 377 |
+
"""
|
| 378 |
+
Construct a sliding-window tensor from the input tensor x (shape: [N, D]).
|
| 379 |
+
The output shape is [N, window_size, D].
|
| 380 |
+
|
| 381 |
+
For each starting index i:
|
| 382 |
+
out[i, j, :] =
|
| 383 |
+
x[i + j, :] if i + j < N
|
| 384 |
+
x[-1, :] otherwise (replicate the last row when out of bounds)
|
| 385 |
+
|
| 386 |
+
Args:
|
| 387 |
+
x (torch.Tensor): Input tensor of shape [N, D]
|
| 388 |
+
window_size (int): Size of the sliding window
|
| 389 |
+
|
| 390 |
+
Returns:
|
| 391 |
+
torch.Tensor: Tensor of shape [N, window_size, D]
|
| 392 |
+
"""
|
| 393 |
+
assert x.dim() == 2
|
| 394 |
+
assert window_size > 0
|
| 395 |
+
|
| 396 |
+
N, D = x.shape
|
| 397 |
+
|
| 398 |
+
# shape [N, window_size]
|
| 399 |
+
# indices[i, j] = i + j
|
| 400 |
+
i_indices = torch.arange(N).unsqueeze(1) # [N, 1]
|
| 401 |
+
j_indices = torch.arange(window_size).unsqueeze(0) # [1, window_size]
|
| 402 |
+
indices = i_indices + j_indices # [N, window_size]
|
| 403 |
+
|
| 404 |
+
# N-1
|
| 405 |
+
# torch.clamp [0, N-1]
|
| 406 |
+
clamped_indices = torch.clamp(indices, min=0, max=N - 1)
|
| 407 |
+
|
| 408 |
+
# clamped_indices [N, window_size],x [N, D]
|
| 409 |
+
# out[i, j, :] = x[clamped_indices[i, j], :]
|
| 410 |
+
out = x[clamped_indices] # [N, window_size, D]
|
| 411 |
+
|
| 412 |
+
return out
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/constants.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
# keys
|
| 15 |
+
import os
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
from huggingface_hub.constants import HF_HOME
|
| 19 |
+
|
| 20 |
+
OBS_ENV_STATE = "observation.environment_state"
|
| 21 |
+
OBS_STATE = "observation.state"
|
| 22 |
+
OBS_IMAGE = "observation.image"
|
| 23 |
+
OBS_IMAGES = "observation.images"
|
| 24 |
+
ACTION = "action"
|
| 25 |
+
REWARD = "next.reward"
|
| 26 |
+
|
| 27 |
+
ROBOTS = "robots"
|
| 28 |
+
ROBOT_TYPE = "robot_type"
|
| 29 |
+
TELEOPERATORS = "teleoperators"
|
| 30 |
+
|
| 31 |
+
# files & directories
|
| 32 |
+
CHECKPOINTS_DIR = "checkpoints"
|
| 33 |
+
LAST_CHECKPOINT_LINK = "last"
|
| 34 |
+
PRETRAINED_MODEL_DIR = "pretrained_model"
|
| 35 |
+
TRAINING_STATE_DIR = "training_state"
|
| 36 |
+
RNG_STATE = "rng_state.safetensors"
|
| 37 |
+
TRAINING_STEP = "training_step.json"
|
| 38 |
+
OPTIMIZER_STATE = "optimizer_state.safetensors"
|
| 39 |
+
OPTIMIZER_PARAM_GROUPS = "optimizer_param_groups.json"
|
| 40 |
+
SCHEDULER_STATE = "scheduler_state.json"
|
| 41 |
+
|
| 42 |
+
if "LEROBOT_HOME" in os.environ:
|
| 43 |
+
raise ValueError(
|
| 44 |
+
f"You have a 'LEROBOT_HOME' environment variable set to '{os.getenv('LEROBOT_HOME')}'.\n"
|
| 45 |
+
"'LEROBOT_HOME' is deprecated, please use 'HF_LEROBOT_HOME' instead."
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
# cache dir
|
| 49 |
+
default_cache_path = Path(HF_HOME) / "lerobot"
|
| 50 |
+
HF_LEROBOT_HOME = Path(os.getenv("HF_LEROBOT_HOME", default_cache_path)).expanduser()
|
| 51 |
+
|
| 52 |
+
# calibration dir
|
| 53 |
+
default_calibration_path = HF_LEROBOT_HOME / "calibration"
|
| 54 |
+
HF_LEROBOT_CALIBRATION = Path(os.getenv("HF_LEROBOT_CALIBRATION", default_calibration_path)).expanduser()
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
|
| 3 |
+
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
import numpy as np
|
| 17 |
+
|
| 18 |
+
from .utils import load_image_as_numpy
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def estimate_num_samples(
|
| 22 |
+
dataset_len: int, min_num_samples: int = 100, max_num_samples: int = 10_000, power: float = 0.75
|
| 23 |
+
) -> int:
|
| 24 |
+
"""Heuristic to estimate the number of samples based on dataset size.
|
| 25 |
+
The power controls the sample growth relative to dataset size.
|
| 26 |
+
Lower the power for less number of samples.
|
| 27 |
+
|
| 28 |
+
For default arguments, we have:
|
| 29 |
+
- from 1 to ~500, num_samples=100
|
| 30 |
+
- at 1000, num_samples=177
|
| 31 |
+
- at 2000, num_samples=299
|
| 32 |
+
- at 5000, num_samples=594
|
| 33 |
+
- at 10000, num_samples=1000
|
| 34 |
+
- at 20000, num_samples=1681
|
| 35 |
+
"""
|
| 36 |
+
if dataset_len < min_num_samples:
|
| 37 |
+
min_num_samples = dataset_len
|
| 38 |
+
return max(min_num_samples, min(int(dataset_len**power), max_num_samples))
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def sample_indices(data_len: int) -> list[int]:
|
| 42 |
+
num_samples = estimate_num_samples(data_len)
|
| 43 |
+
return np.round(np.linspace(0, data_len - 1, num_samples)).astype(int).tolist()
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def auto_downsample_height_width(img: np.ndarray, target_size: int = 150, max_size_threshold: int = 300):
|
| 47 |
+
_, height, width = img.shape
|
| 48 |
+
|
| 49 |
+
if max(width, height) < max_size_threshold:
|
| 50 |
+
# no downsampling needed
|
| 51 |
+
return img
|
| 52 |
+
|
| 53 |
+
downsample_factor = int(width / target_size) if width > height else int(height / target_size)
|
| 54 |
+
return img[:, ::downsample_factor, ::downsample_factor]
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def sample_images(image_paths: list[str]) -> np.ndarray:
|
| 58 |
+
sampled_indices = sample_indices(len(image_paths))
|
| 59 |
+
|
| 60 |
+
images = None
|
| 61 |
+
for i, idx in enumerate(sampled_indices):
|
| 62 |
+
path = image_paths[idx]
|
| 63 |
+
# we load as uint8 to reduce memory usage
|
| 64 |
+
img = load_image_as_numpy(path, dtype=np.uint8, channel_first=True)
|
| 65 |
+
img = auto_downsample_height_width(img)
|
| 66 |
+
|
| 67 |
+
if images is None:
|
| 68 |
+
images = np.empty((len(sampled_indices), *img.shape), dtype=np.uint8)
|
| 69 |
+
|
| 70 |
+
images[i] = img
|
| 71 |
+
|
| 72 |
+
return images
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def get_feature_stats(array: np.ndarray, axis: tuple, keepdims: bool) -> dict[str, np.ndarray]:
|
| 76 |
+
return {
|
| 77 |
+
"min": np.min(array, axis=axis, keepdims=keepdims),
|
| 78 |
+
"max": np.max(array, axis=axis, keepdims=keepdims),
|
| 79 |
+
"mean": np.mean(array, axis=axis, keepdims=keepdims),
|
| 80 |
+
"std": np.std(array, axis=axis, keepdims=keepdims),
|
| 81 |
+
"count": np.array([len(array)]),
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def compute_episode_stats(episode_data: dict[str, list[str] | np.ndarray], features: dict, is_compute_episode_stats_image : bool) -> dict:
|
| 86 |
+
ep_stats = {}
|
| 87 |
+
for key, data in episode_data.items():
|
| 88 |
+
if features[key]["dtype"] == "string":
|
| 89 |
+
continue # HACK: we should receive np.arrays of strings
|
| 90 |
+
elif features[key]["dtype"] in ["image", "video"]:
|
| 91 |
+
if is_compute_episode_stats_image:
|
| 92 |
+
ep_ft_array = sample_images(data) # data is a list of image paths
|
| 93 |
+
axes_to_reduce = (0, 2, 3) # keep channel dim
|
| 94 |
+
keepdims = True
|
| 95 |
+
else:
|
| 96 |
+
continue
|
| 97 |
+
else:
|
| 98 |
+
ep_ft_array = data # data is already a np.ndarray
|
| 99 |
+
axes_to_reduce = 0 # compute stats over the first axis
|
| 100 |
+
keepdims = data.ndim == 1 # keep as np.array
|
| 101 |
+
|
| 102 |
+
ep_stats[key] = get_feature_stats(ep_ft_array, axis=axes_to_reduce, keepdims=keepdims)
|
| 103 |
+
|
| 104 |
+
# finally, we normalize and remove batch dim for images
|
| 105 |
+
if features[key]["dtype"] in ["image", "video"]:
|
| 106 |
+
ep_stats[key] = {
|
| 107 |
+
k: v if k == "count" else np.squeeze(v / 255.0, axis=0) for k, v in ep_stats[key].items()
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
return ep_stats
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def _assert_type_and_shape(stats_list: list[dict[str, dict]]):
|
| 114 |
+
for i in range(len(stats_list)):
|
| 115 |
+
for fkey in stats_list[i]:
|
| 116 |
+
for k, v in stats_list[i][fkey].items():
|
| 117 |
+
if not isinstance(v, np.ndarray):
|
| 118 |
+
raise ValueError(
|
| 119 |
+
f"Stats must be composed of numpy array, but key '{k}' of feature '{fkey}' is of type '{type(v)}' instead."
|
| 120 |
+
)
|
| 121 |
+
if v.ndim == 0:
|
| 122 |
+
raise ValueError("Number of dimensions must be at least 1, and is 0 instead.")
|
| 123 |
+
if k == "count" and v.shape != (1,):
|
| 124 |
+
raise ValueError(f"Shape of 'count' must be (1), but is {v.shape} instead.")
|
| 125 |
+
if "image" in fkey and k != "count" and v.shape != (3, 1, 1):
|
| 126 |
+
raise ValueError(f"Shape of '{k}' must be (3,1,1), but is {v.shape} instead.")
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def aggregate_feature_stats(stats_ft_list: list[dict[str, dict]]) -> dict[str, dict[str, np.ndarray]]:
|
| 130 |
+
"""Aggregates stats for a single feature."""
|
| 131 |
+
means = np.stack([s["mean"] for s in stats_ft_list])
|
| 132 |
+
variances = np.stack([s["std"] ** 2 for s in stats_ft_list])
|
| 133 |
+
counts = np.stack([s["count"] for s in stats_ft_list])
|
| 134 |
+
total_count = counts.sum(axis=0)
|
| 135 |
+
|
| 136 |
+
# Prepare weighted mean by matching number of dimensions
|
| 137 |
+
while counts.ndim < means.ndim:
|
| 138 |
+
counts = np.expand_dims(counts, axis=-1)
|
| 139 |
+
|
| 140 |
+
# Compute the weighted mean
|
| 141 |
+
weighted_means = means * counts
|
| 142 |
+
total_mean = weighted_means.sum(axis=0) / total_count
|
| 143 |
+
|
| 144 |
+
# Compute the variance using the parallel algorithm
|
| 145 |
+
delta_means = means - total_mean
|
| 146 |
+
weighted_variances = (variances + delta_means**2) * counts
|
| 147 |
+
total_variance = weighted_variances.sum(axis=0) / total_count
|
| 148 |
+
|
| 149 |
+
return {
|
| 150 |
+
"min": np.min(np.stack([s["min"] for s in stats_ft_list]), axis=0),
|
| 151 |
+
"max": np.max(np.stack([s["max"] for s in stats_ft_list]), axis=0),
|
| 152 |
+
"mean": total_mean,
|
| 153 |
+
"std": np.sqrt(total_variance),
|
| 154 |
+
"count": total_count,
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def aggregate_stats(stats_list: list[dict[str, dict]]) -> dict[str, dict[str, np.ndarray]]:
|
| 159 |
+
"""Aggregate stats from multiple compute_stats outputs into a single set of stats.
|
| 160 |
+
|
| 161 |
+
The final stats will have the union of all data keys from each of the stats dicts.
|
| 162 |
+
|
| 163 |
+
For instance:
|
| 164 |
+
- new_min = min(min_dataset_0, min_dataset_1, ...)
|
| 165 |
+
- new_max = max(max_dataset_0, max_dataset_1, ...)
|
| 166 |
+
- new_mean = (mean of all data, weighted by counts)
|
| 167 |
+
- new_std = (std of all data)
|
| 168 |
+
"""
|
| 169 |
+
|
| 170 |
+
_assert_type_and_shape(stats_list)
|
| 171 |
+
|
| 172 |
+
data_keys = {key for stats in stats_list for key in stats}
|
| 173 |
+
aggregated_stats = {key: {} for key in data_keys}
|
| 174 |
+
|
| 175 |
+
for key in data_keys:
|
| 176 |
+
stats_with_key = [stats[key] for stats in stats_list if key in stats]
|
| 177 |
+
aggregated_stats[key] = aggregate_feature_stats(stats_with_key)
|
| 178 |
+
|
| 179 |
+
return aggregated_stats
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py
ADDED
|
@@ -0,0 +1,903 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
|
| 3 |
+
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
import contextlib
|
| 17 |
+
import importlib.resources
|
| 18 |
+
import io
|
| 19 |
+
import json
|
| 20 |
+
import logging
|
| 21 |
+
from collections.abc import Iterator
|
| 22 |
+
from itertools import accumulate
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
from pprint import pformat
|
| 25 |
+
from types import SimpleNamespace
|
| 26 |
+
from typing import Any
|
| 27 |
+
import array
|
| 28 |
+
|
| 29 |
+
import datasets
|
| 30 |
+
import jsonlines
|
| 31 |
+
import numpy as np
|
| 32 |
+
import packaging.version
|
| 33 |
+
import torch
|
| 34 |
+
from datasets.table import embed_table_storage
|
| 35 |
+
from huggingface_hub import DatasetCard, DatasetCardData, HfApi
|
| 36 |
+
from huggingface_hub.errors import RevisionNotFoundError
|
| 37 |
+
from PIL import Image as PILImage
|
| 38 |
+
from torchvision import transforms
|
| 39 |
+
|
| 40 |
+
from functools import partial
|
| 41 |
+
|
| 42 |
+
# from lerobot.configs.types import DictLike, FeatureType, PolicyFeature
|
| 43 |
+
# from lerobot.datasets.backward_compatibility import (
|
| 44 |
+
# V21_MESSAGE,
|
| 45 |
+
# BackwardCompatibilityError,
|
| 46 |
+
# ForwardCompatibilityError,
|
| 47 |
+
# )
|
| 48 |
+
# from lerobot.robots import Robot
|
| 49 |
+
# from lerobot.utils.utils import is_valid_numpy_dtype_string
|
| 50 |
+
|
| 51 |
+
DEFAULT_CHUNK_SIZE = 1000 # Max number of episodes per chunk
|
| 52 |
+
|
| 53 |
+
INFO_PATH = "meta/info.json"
|
| 54 |
+
EPISODES_PATH = "meta/episodes.jsonl"
|
| 55 |
+
STATS_PATH = "meta/stats.json"
|
| 56 |
+
EPISODES_STATS_PATH = "meta/episodes_stats.jsonl"
|
| 57 |
+
TASKS_PATH = "meta/tasks.jsonl"
|
| 58 |
+
|
| 59 |
+
ANNOTATION_PATHS = {
|
| 60 |
+
"subtask": "annotations/subtask_annotations.jsonl",
|
| 61 |
+
"scene": "annotations/scene_annotations.jsonl",
|
| 62 |
+
"gripper_mode": "annotations/gripper_mode_annotation.jsonl",
|
| 63 |
+
"gripper_activity": "annotations/gripper_activity_annotation.jsonl",
|
| 64 |
+
"eef_velocity": "annotations/eef_velocity_annotation.jsonl",
|
| 65 |
+
"eef_acc_mag": "annotations/eef_acc_mag_annotation.jsonl",
|
| 66 |
+
"eef_direction": "annotations/eef_direction_annotation.jsonl",
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
DEFAULT_VIDEO_PATH = "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4"
|
| 70 |
+
DEFAULT_PARQUET_PATH = "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet"
|
| 71 |
+
DEFAULT_IMAGE_PATH = "images/{image_key}/episode_{episode_index:06d}/frame_{frame_index:06d}.jpeg"
|
| 72 |
+
|
| 73 |
+
DATASET_CARD_TEMPLATE = """
|
| 74 |
+
---
|
| 75 |
+
# Metadata will go there
|
| 76 |
+
---
|
| 77 |
+
This dataset was created using [LeRobot](https://github.com/huggingface/lerobot).
|
| 78 |
+
|
| 79 |
+
## {}
|
| 80 |
+
|
| 81 |
+
"""
|
| 82 |
+
|
| 83 |
+
DEFAULT_FEATURES = {
|
| 84 |
+
"timestamp": {"dtype": "float32", "shape": (1,), "names": None},
|
| 85 |
+
"frame_index": {"dtype": "int64", "shape": (1,), "names": None},
|
| 86 |
+
"episode_index": {"dtype": "int64", "shape": (1,), "names": None},
|
| 87 |
+
"index": {"dtype": "int64", "shape": (1,), "names": None},
|
| 88 |
+
"coarse_task_index": {"dtype": "int64", "shape": (1,), "names": None},
|
| 89 |
+
"task_index": {"dtype": "int64", "shape": (1,), "names": None},
|
| 90 |
+
"coarse_quality_index": {"dtype": "int64", "shape": (1,), "names": None},
|
| 91 |
+
"quality_index": {"dtype": "int64", "shape": (1,), "names": None},
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def flatten_dict(d: dict, parent_key: str = "", sep: str = "/") -> dict:
|
| 96 |
+
"""Flatten a nested dictionary structure by collapsing nested keys into one key with a separator.
|
| 97 |
+
|
| 98 |
+
For example:
|
| 99 |
+
```
|
| 100 |
+
>>> dct = {"a": {"b": 1, "c": {"d": 2}}, "e": 3}`
|
| 101 |
+
>>> print(flatten_dict(dct))
|
| 102 |
+
{"a/b": 1, "a/c/d": 2, "e": 3}
|
| 103 |
+
"""
|
| 104 |
+
items = []
|
| 105 |
+
for k, v in d.items():
|
| 106 |
+
new_key = f"{parent_key}{sep}{k}" if parent_key else k
|
| 107 |
+
if isinstance(v, dict):
|
| 108 |
+
items.extend(flatten_dict(v, new_key, sep=sep).items())
|
| 109 |
+
else:
|
| 110 |
+
items.append((new_key, v))
|
| 111 |
+
return dict(items)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def unflatten_dict(d: dict, sep: str = "/") -> dict:
|
| 115 |
+
outdict = {}
|
| 116 |
+
for key, value in d.items():
|
| 117 |
+
parts = key.split(sep)
|
| 118 |
+
d = outdict
|
| 119 |
+
for part in parts[:-1]:
|
| 120 |
+
if part not in d:
|
| 121 |
+
d[part] = {}
|
| 122 |
+
d = d[part]
|
| 123 |
+
d[parts[-1]] = value
|
| 124 |
+
return outdict
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
# def get_nested_item(obj: DictLike, flattened_key: str, sep: str = "/") -> Any:
|
| 128 |
+
# split_keys = flattened_key.split(sep)
|
| 129 |
+
# getter = obj[split_keys[0]]
|
| 130 |
+
# if len(split_keys) == 1:
|
| 131 |
+
# return getter
|
| 132 |
+
|
| 133 |
+
# for key in split_keys[1:]:
|
| 134 |
+
# getter = getter[key]
|
| 135 |
+
|
| 136 |
+
# return getter
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def serialize_dict(stats: dict[str, torch.Tensor | np.ndarray | dict]) -> dict:
|
| 140 |
+
serialized_dict = {}
|
| 141 |
+
for key, value in flatten_dict(stats).items():
|
| 142 |
+
if isinstance(value, (torch.Tensor, np.ndarray)):
|
| 143 |
+
serialized_dict[key] = value.tolist()
|
| 144 |
+
elif isinstance(value, np.generic):
|
| 145 |
+
serialized_dict[key] = value.item()
|
| 146 |
+
elif isinstance(value, (int, float)):
|
| 147 |
+
serialized_dict[key] = value
|
| 148 |
+
else:
|
| 149 |
+
raise NotImplementedError(f"The value '{value}' of type '{type(value)}' is not supported.")
|
| 150 |
+
return unflatten_dict(serialized_dict)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def embed_images(dataset: datasets.Dataset) -> datasets.Dataset:
|
| 154 |
+
# Embed image bytes into the table before saving to parquet
|
| 155 |
+
format = dataset.format
|
| 156 |
+
dataset = dataset.with_format("arrow")
|
| 157 |
+
dataset = dataset.map(embed_table_storage, batched=False)
|
| 158 |
+
dataset = dataset.with_format(**format)
|
| 159 |
+
return dataset
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def load_json(fpath: Path) -> Any:
|
| 163 |
+
with open(fpath) as f:
|
| 164 |
+
return json.load(f)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def write_json(data: dict, fpath: Path) -> None:
|
| 168 |
+
fpath.parent.mkdir(exist_ok=True, parents=True)
|
| 169 |
+
with open(fpath, "w") as f:
|
| 170 |
+
json.dump(data, f, indent=4, ensure_ascii=False)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def load_jsonlines(fpath: Path) -> list[Any]:
|
| 174 |
+
# allow \t and \n
|
| 175 |
+
loose_loader = partial(json.loads, strict=False)
|
| 176 |
+
with jsonlines.open(fpath, "r", loads=loose_loader) as reader:
|
| 177 |
+
return list(reader)
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def write_jsonlines(data: dict, fpath: Path) -> None:
|
| 181 |
+
fpath.parent.mkdir(exist_ok=True, parents=True)
|
| 182 |
+
with jsonlines.open(fpath, "w") as writer:
|
| 183 |
+
writer.write_all(data)
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def append_jsonlines(data: dict, fpath: Path) -> None:
|
| 187 |
+
fpath.parent.mkdir(exist_ok=True, parents=True)
|
| 188 |
+
with jsonlines.open(fpath, "a") as writer:
|
| 189 |
+
writer.write(data)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def write_info(info: dict, local_dir: Path):
|
| 193 |
+
write_json(info, local_dir / INFO_PATH)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def load_info(local_dir: Path) -> dict:
|
| 197 |
+
info = load_json(local_dir / INFO_PATH)
|
| 198 |
+
for ft in info["features"].values():
|
| 199 |
+
ft["shape"] = tuple(ft["shape"])
|
| 200 |
+
return info
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def write_stats(stats: dict, local_dir: Path):
|
| 204 |
+
serialized_stats = serialize_dict(stats)
|
| 205 |
+
write_json(serialized_stats, local_dir / STATS_PATH)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def cast_stats_to_numpy(stats) -> dict[str, dict[str, np.ndarray]]:
|
| 209 |
+
stats = {key: np.array(value) for key, value in flatten_dict(stats).items()}
|
| 210 |
+
return unflatten_dict(stats)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def load_stats(local_dir: Path) -> dict[str, dict[str, np.ndarray]]:
|
| 214 |
+
if not (local_dir / STATS_PATH).exists():
|
| 215 |
+
return None
|
| 216 |
+
stats = load_json(local_dir / STATS_PATH)
|
| 217 |
+
return cast_stats_to_numpy(stats)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def write_task(task_index: int, task: dict, local_dir: Path):
|
| 221 |
+
task_dict = {
|
| 222 |
+
"task_index": task_index,
|
| 223 |
+
"task": task,
|
| 224 |
+
}
|
| 225 |
+
append_jsonlines(task_dict, local_dir / TASKS_PATH)
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def load_tasks(local_dir: Path) -> tuple[dict, dict]:
|
| 229 |
+
tasks = load_jsonlines(local_dir / TASKS_PATH)
|
| 230 |
+
tasks = {item["task_index"]: item["task"] for item in sorted(tasks, key=lambda x: x["task_index"])}
|
| 231 |
+
task_to_task_index = {task: task_index for task_index, task in tasks.items()}
|
| 232 |
+
return tasks, task_to_task_index
|
| 233 |
+
|
| 234 |
+
def load_annotations(local_dir: Path) -> dict[str, dict[int, str]]:
|
| 235 |
+
annotations = {}
|
| 236 |
+
for key, path in ANNOTATION_PATHS.items():
|
| 237 |
+
anno = load_jsonlines(local_dir / path)
|
| 238 |
+
anno = {item[f"{key}_index"]: item[key] for item in sorted(anno, key=lambda x: x[f"{key}_index"])}
|
| 239 |
+
annotations[key] = anno
|
| 240 |
+
return annotations
|
| 241 |
+
|
| 242 |
+
def write_episode(episode: dict, local_dir: Path):
|
| 243 |
+
append_jsonlines(episode, local_dir / EPISODES_PATH)
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def load_episodes(local_dir: Path) -> dict:
|
| 247 |
+
episodes = load_jsonlines(local_dir / EPISODES_PATH)
|
| 248 |
+
return {item["episode_index"]: item for item in sorted(episodes, key=lambda x: x["episode_index"])}
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def write_episode_stats(episode_index: int, episode_stats: dict, local_dir: Path):
|
| 252 |
+
# We wrap episode_stats in a dictionary since `episode_stats["episode_index"]`
|
| 253 |
+
# is a dictionary of stats and not an integer.
|
| 254 |
+
episode_stats = {"episode_index": episode_index, "stats": serialize_dict(episode_stats)}
|
| 255 |
+
append_jsonlines(episode_stats, local_dir / EPISODES_STATS_PATH)
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def load_episodes_stats(local_dir: Path) -> dict:
|
| 259 |
+
episodes_stats = load_jsonlines(local_dir / EPISODES_STATS_PATH)
|
| 260 |
+
return {
|
| 261 |
+
item["episode_index"]: cast_stats_to_numpy(item["stats"])
|
| 262 |
+
for item in sorted(episodes_stats, key=lambda x: x["episode_index"])
|
| 263 |
+
}
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def backward_compatible_episodes_stats(
|
| 267 |
+
stats: dict[str, dict[str, np.ndarray]], episodes: list[int]
|
| 268 |
+
) -> dict[str, dict[str, np.ndarray]]:
|
| 269 |
+
return dict.fromkeys(episodes, stats)
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def load_image_as_numpy(
|
| 273 |
+
fpath: str | Path, dtype: np.dtype = np.float32, channel_first: bool = True
|
| 274 |
+
) -> np.ndarray:
|
| 275 |
+
img = PILImage.open(fpath).convert("RGB")
|
| 276 |
+
img_array = np.array(img, dtype=dtype)
|
| 277 |
+
if channel_first: # (H, W, C) -> (C, H, W)
|
| 278 |
+
img_array = np.transpose(img_array, (2, 0, 1))
|
| 279 |
+
if np.issubdtype(dtype, np.floating):
|
| 280 |
+
img_array /= 255.0
|
| 281 |
+
return img_array
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
def hf_transform_to_torch(items_dict: dict[torch.Tensor | None]):
|
| 285 |
+
"""Get a transform function that convert items from Hugging Face dataset (pyarrow)
|
| 286 |
+
to torch tensors. Importantly, images are converted from PIL, which corresponds to
|
| 287 |
+
a channel last representation (h w c) of uint8 type, to a torch image representation
|
| 288 |
+
with channel first (c h w) of float32 type in range [0,1].
|
| 289 |
+
"""
|
| 290 |
+
for key in items_dict:
|
| 291 |
+
first_item = items_dict[key][0]
|
| 292 |
+
if type(first_item) == dict and 'bytes' in first_item and first_item['bytes'] is not None:
|
| 293 |
+
to_pil = lambda img: PILImage.open(io.BytesIO(img['bytes']))
|
| 294 |
+
items_dict[key] = [to_pil(img) for img in items_dict[key]]
|
| 295 |
+
first_item = items_dict[key][0]
|
| 296 |
+
if isinstance(first_item, PILImage.Image):
|
| 297 |
+
to_tensor = transforms.ToTensor()
|
| 298 |
+
items_dict[key] = [to_tensor(img) for img in items_dict[key]]
|
| 299 |
+
elif first_item is None:
|
| 300 |
+
pass
|
| 301 |
+
else:
|
| 302 |
+
items_dict[key] = [x if isinstance(x, str) else torch.tensor(x) for x in items_dict[key]]
|
| 303 |
+
return items_dict
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def is_valid_version(version: str) -> bool:
|
| 307 |
+
try:
|
| 308 |
+
packaging.version.parse(version)
|
| 309 |
+
return True
|
| 310 |
+
except packaging.version.InvalidVersion:
|
| 311 |
+
return False
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
# def check_version_compatibility(
|
| 315 |
+
# repo_id: str,
|
| 316 |
+
# version_to_check: str | packaging.version.Version,
|
| 317 |
+
# current_version: str | packaging.version.Version,
|
| 318 |
+
# enforce_breaking_major: bool = True,
|
| 319 |
+
# ) -> None:
|
| 320 |
+
# v_check = (
|
| 321 |
+
# packaging.version.parse(version_to_check)
|
| 322 |
+
# if not isinstance(version_to_check, packaging.version.Version)
|
| 323 |
+
# else version_to_check
|
| 324 |
+
# )
|
| 325 |
+
# v_current = (
|
| 326 |
+
# packaging.version.parse(current_version)
|
| 327 |
+
# if not isinstance(current_version, packaging.version.Version)
|
| 328 |
+
# else current_version
|
| 329 |
+
# )
|
| 330 |
+
# if v_check.major < v_current.major and enforce_breaking_major:
|
| 331 |
+
# raise BackwardCompatibilityError(repo_id, v_check)
|
| 332 |
+
# elif v_check.minor < v_current.minor:
|
| 333 |
+
# logging.warning(V21_MESSAGE.format(repo_id=repo_id, version=v_check))
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def get_repo_versions(repo_id: str) -> list[packaging.version.Version]:
|
| 337 |
+
"""Returns available valid versions (branches and tags) on given repo."""
|
| 338 |
+
api = HfApi()
|
| 339 |
+
repo_refs = api.list_repo_refs(repo_id, repo_type="dataset")
|
| 340 |
+
repo_refs = [b.name for b in repo_refs.branches + repo_refs.tags]
|
| 341 |
+
repo_versions = []
|
| 342 |
+
for ref in repo_refs:
|
| 343 |
+
with contextlib.suppress(packaging.version.InvalidVersion):
|
| 344 |
+
repo_versions.append(packaging.version.parse(ref))
|
| 345 |
+
|
| 346 |
+
return repo_versions
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
# def get_safe_version(repo_id: str, version: str | packaging.version.Version) -> str:
|
| 350 |
+
# """
|
| 351 |
+
# Returns the version if available on repo or the latest compatible one.
|
| 352 |
+
# Otherwise, will throw a `CompatibilityError`.
|
| 353 |
+
# """
|
| 354 |
+
# target_version = (
|
| 355 |
+
# packaging.version.parse(version) if not isinstance(version, packaging.version.Version) else version
|
| 356 |
+
# )
|
| 357 |
+
# hub_versions = get_repo_versions(repo_id)
|
| 358 |
+
|
| 359 |
+
# if not hub_versions:
|
| 360 |
+
# raise RevisionNotFoundError(
|
| 361 |
+
# f"""Your dataset must be tagged with a codebase version.
|
| 362 |
+
# Assuming _version_ is the codebase_version value in the info.json, you can run this:
|
| 363 |
+
# ```python
|
| 364 |
+
# from huggingface_hub import HfApi
|
| 365 |
+
|
| 366 |
+
# hub_api = HfApi()
|
| 367 |
+
# hub_api.create_tag("{repo_id}", tag="_version_", repo_type="dataset")
|
| 368 |
+
# ```
|
| 369 |
+
# """
|
| 370 |
+
# )
|
| 371 |
+
|
| 372 |
+
# if target_version in hub_versions:
|
| 373 |
+
# return f"v{target_version}"
|
| 374 |
+
|
| 375 |
+
# compatibles = [
|
| 376 |
+
# v for v in hub_versions if v.major == target_version.major and v.minor <= target_version.minor
|
| 377 |
+
# ]
|
| 378 |
+
# if compatibles:
|
| 379 |
+
# return_version = max(compatibles)
|
| 380 |
+
# if return_version < target_version:
|
| 381 |
+
# logging.warning(f"Revision {version} for {repo_id} not found, using version v{return_version}")
|
| 382 |
+
# return f"v{return_version}"
|
| 383 |
+
|
| 384 |
+
# lower_major = [v for v in hub_versions if v.major < target_version.major]
|
| 385 |
+
# if lower_major:
|
| 386 |
+
# raise BackwardCompatibilityError(repo_id, max(lower_major))
|
| 387 |
+
|
| 388 |
+
# upper_versions = [v for v in hub_versions if v > target_version]
|
| 389 |
+
# assert len(upper_versions) > 0
|
| 390 |
+
# raise ForwardCompatibilityError(repo_id, min(upper_versions))
|
| 391 |
+
|
| 392 |
+
|
| 393 |
+
def get_hf_features_from_features(features: dict) -> datasets.Features:
|
| 394 |
+
hf_features = {}
|
| 395 |
+
for key, ft in features.items():
|
| 396 |
+
if ft["dtype"] == "video":
|
| 397 |
+
continue
|
| 398 |
+
elif ft["dtype"] == "image":
|
| 399 |
+
hf_features[key] = datasets.Image()
|
| 400 |
+
elif ft["shape"] == (1,):
|
| 401 |
+
hf_features[key] = datasets.Value(dtype=ft["dtype"])
|
| 402 |
+
elif len(ft["shape"]) == 1:
|
| 403 |
+
hf_features[key] = datasets.Sequence(
|
| 404 |
+
length=ft["shape"][0], feature=datasets.Value(dtype=ft["dtype"])
|
| 405 |
+
)
|
| 406 |
+
elif len(ft["shape"]) == 2:
|
| 407 |
+
hf_features[key] = datasets.Array2D(shape=ft["shape"], dtype=ft["dtype"])
|
| 408 |
+
elif len(ft["shape"]) == 3:
|
| 409 |
+
hf_features[key] = datasets.Array3D(shape=ft["shape"], dtype=ft["dtype"])
|
| 410 |
+
elif len(ft["shape"]) == 4:
|
| 411 |
+
hf_features[key] = datasets.Array4D(shape=ft["shape"], dtype=ft["dtype"])
|
| 412 |
+
elif len(ft["shape"]) == 5:
|
| 413 |
+
hf_features[key] = datasets.Array5D(shape=ft["shape"], dtype=ft["dtype"])
|
| 414 |
+
else:
|
| 415 |
+
raise ValueError(f"Corresponding feature is not valid: {ft}")
|
| 416 |
+
|
| 417 |
+
return datasets.Features(hf_features)
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
def _validate_feature_names(features: dict[str, dict]) -> None:
|
| 421 |
+
invalid_features = {name: ft for name, ft in features.items() if "/" in name}
|
| 422 |
+
if invalid_features:
|
| 423 |
+
raise ValueError(f"Feature names should not contain '/'. Found '/' in '{invalid_features}'.")
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
def hw_to_dataset_features(
|
| 427 |
+
hw_features: dict[str, type | tuple], prefix: str, use_video: bool = True
|
| 428 |
+
) -> dict[str, dict]:
|
| 429 |
+
features = {}
|
| 430 |
+
joint_fts = {key: ftype for key, ftype in hw_features.items() if ftype is float}
|
| 431 |
+
cam_fts = {key: shape for key, shape in hw_features.items() if isinstance(shape, tuple)}
|
| 432 |
+
|
| 433 |
+
if joint_fts and prefix == "action":
|
| 434 |
+
features[prefix] = {
|
| 435 |
+
"dtype": "float32",
|
| 436 |
+
"shape": (len(joint_fts),),
|
| 437 |
+
"names": list(joint_fts),
|
| 438 |
+
}
|
| 439 |
+
|
| 440 |
+
if joint_fts and prefix == "observation":
|
| 441 |
+
features[f"{prefix}.state"] = {
|
| 442 |
+
"dtype": "float32",
|
| 443 |
+
"shape": (len(joint_fts),),
|
| 444 |
+
"names": list(joint_fts),
|
| 445 |
+
}
|
| 446 |
+
|
| 447 |
+
for key, shape in cam_fts.items():
|
| 448 |
+
features[f"{prefix}.images.{key}"] = {
|
| 449 |
+
"dtype": "video" if use_video else "image",
|
| 450 |
+
"shape": shape,
|
| 451 |
+
"names": ["height", "width", "channels"],
|
| 452 |
+
}
|
| 453 |
+
|
| 454 |
+
_validate_feature_names(features)
|
| 455 |
+
return features
|
| 456 |
+
|
| 457 |
+
|
| 458 |
+
def build_dataset_frame(
|
| 459 |
+
ds_features: dict[str, dict], values: dict[str, Any], prefix: str
|
| 460 |
+
) -> dict[str, np.ndarray]:
|
| 461 |
+
frame = {}
|
| 462 |
+
for key, ft in ds_features.items():
|
| 463 |
+
if key in DEFAULT_FEATURES or not key.startswith(prefix):
|
| 464 |
+
continue
|
| 465 |
+
elif ft["dtype"] == "float32" and len(ft["shape"]) == 1:
|
| 466 |
+
frame[key] = np.array([values[name] for name in ft["names"]], dtype=np.float32)
|
| 467 |
+
elif ft["dtype"] in ["image", "video"]:
|
| 468 |
+
frame[key] = values[key.removeprefix(f"{prefix}.images.")]
|
| 469 |
+
|
| 470 |
+
return frame
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
# def get_features_from_robot(robot: Robot, use_videos: bool = True) -> dict:
|
| 474 |
+
# camera_ft = {}
|
| 475 |
+
# if robot.cameras:
|
| 476 |
+
# camera_ft = {
|
| 477 |
+
# key: {"dtype": "video" if use_videos else "image", **ft}
|
| 478 |
+
# for key, ft in robot.camera_features.items()
|
| 479 |
+
# }
|
| 480 |
+
# return {**robot.motor_features, **camera_ft, **DEFAULT_FEATURES}
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
# def dataset_to_policy_features(features: dict[str, dict]) -> dict[str, PolicyFeature]:
|
| 484 |
+
# # TODO(aliberts): Implement "type" in dataset features and simplify this
|
| 485 |
+
# policy_features = {}
|
| 486 |
+
# for key, ft in features.items():
|
| 487 |
+
# shape = ft["shape"]
|
| 488 |
+
# if ft["dtype"] in ["image", "video"]:
|
| 489 |
+
# type = FeatureType.VISUAL
|
| 490 |
+
# if len(shape) != 3:
|
| 491 |
+
# raise ValueError(f"Number of dimensions of {key} != 3 (shape={shape})")
|
| 492 |
+
|
| 493 |
+
# names = ft["names"]
|
| 494 |
+
# # Backward compatibility for "channel" which is an error introduced in LeRobotDataset v2.0 for ported datasets.
|
| 495 |
+
# if names[2] in ["channel", "channels"]: # (h, w, c) -> (c, h, w)
|
| 496 |
+
# shape = (shape[2], shape[0], shape[1])
|
| 497 |
+
# elif key == "observation.environment_state":
|
| 498 |
+
# type = FeatureType.ENV
|
| 499 |
+
# elif key.startswith("observation"):
|
| 500 |
+
# type = FeatureType.STATE
|
| 501 |
+
# elif key.startswith("action"):
|
| 502 |
+
# type = FeatureType.ACTION
|
| 503 |
+
# else:
|
| 504 |
+
# continue
|
| 505 |
+
|
| 506 |
+
# policy_features[key] = PolicyFeature(
|
| 507 |
+
# type=type,
|
| 508 |
+
# shape=shape,
|
| 509 |
+
# )
|
| 510 |
+
|
| 511 |
+
# return policy_features
|
| 512 |
+
|
| 513 |
+
|
| 514 |
+
def create_empty_dataset_info(
|
| 515 |
+
codebase_version: str,
|
| 516 |
+
fps: int,
|
| 517 |
+
features: dict,
|
| 518 |
+
use_videos: bool,
|
| 519 |
+
robot_type: str | None = None,
|
| 520 |
+
) -> dict:
|
| 521 |
+
return {
|
| 522 |
+
"codebase_version": codebase_version,
|
| 523 |
+
"robot_type": robot_type,
|
| 524 |
+
"total_episodes": 0,
|
| 525 |
+
"total_frames": 0,
|
| 526 |
+
"total_tasks": 0,
|
| 527 |
+
"total_videos": 0,
|
| 528 |
+
"total_chunks": 0,
|
| 529 |
+
"chunks_size": DEFAULT_CHUNK_SIZE,
|
| 530 |
+
"fps": fps,
|
| 531 |
+
"splits": {},
|
| 532 |
+
"data_path": DEFAULT_PARQUET_PATH,
|
| 533 |
+
"video_path": DEFAULT_VIDEO_PATH if use_videos else None,
|
| 534 |
+
"features": features,
|
| 535 |
+
}
|
| 536 |
+
|
| 537 |
+
|
| 538 |
+
def get_episode_data_index(
|
| 539 |
+
episode_dicts: dict[dict], episodes: list[int] | None = None
|
| 540 |
+
) -> dict[str, torch.Tensor]:
|
| 541 |
+
episode_lengths = {ep_idx: ep_dict["length"] for ep_idx, ep_dict in episode_dicts.items()}
|
| 542 |
+
if episodes is not None:
|
| 543 |
+
episode_lengths = {ep_idx: episode_lengths[ep_idx] for ep_idx in episodes}
|
| 544 |
+
|
| 545 |
+
cumulative_lengths = list(accumulate(episode_lengths.values()))
|
| 546 |
+
return {
|
| 547 |
+
"from": torch.LongTensor([0] + cumulative_lengths[:-1]),
|
| 548 |
+
"to": torch.LongTensor(cumulative_lengths),
|
| 549 |
+
}
|
| 550 |
+
|
| 551 |
+
|
| 552 |
+
def check_timestamps_sync(
|
| 553 |
+
timestamps: np.ndarray,
|
| 554 |
+
episode_indices: np.ndarray,
|
| 555 |
+
episode_data_index: dict[str, np.ndarray],
|
| 556 |
+
fps: int,
|
| 557 |
+
tolerance_s: float,
|
| 558 |
+
raise_value_error: bool = True,
|
| 559 |
+
) -> bool:
|
| 560 |
+
"""
|
| 561 |
+
This check is to make sure that each timestamp is separated from the next by (1/fps) +/- tolerance
|
| 562 |
+
to account for possible numerical error.
|
| 563 |
+
|
| 564 |
+
Args:
|
| 565 |
+
timestamps (np.ndarray): Array of timestamps in seconds.
|
| 566 |
+
episode_indices (np.ndarray): Array indicating the episode index for each timestamp.
|
| 567 |
+
episode_data_index (dict[str, np.ndarray]): A dictionary that includes 'to',
|
| 568 |
+
which identifies indices for the end of each episode.
|
| 569 |
+
fps (int): Frames per second. Used to check the expected difference between consecutive timestamps.
|
| 570 |
+
tolerance_s (float): Allowed deviation from the expected (1/fps) difference.
|
| 571 |
+
raise_value_error (bool): Whether to raise a ValueError if the check fails.
|
| 572 |
+
|
| 573 |
+
Returns:
|
| 574 |
+
bool: True if all checked timestamp differences lie within tolerance, False otherwise.
|
| 575 |
+
|
| 576 |
+
Raises:
|
| 577 |
+
ValueError: If the check fails and `raise_value_error` is True.
|
| 578 |
+
"""
|
| 579 |
+
if timestamps.shape != episode_indices.shape:
|
| 580 |
+
raise ValueError(
|
| 581 |
+
"timestamps and episode_indices should have the same shape. "
|
| 582 |
+
f"Found {timestamps.shape=} and {episode_indices.shape=}."
|
| 583 |
+
)
|
| 584 |
+
|
| 585 |
+
# Consecutive differences
|
| 586 |
+
diffs = np.diff(timestamps)
|
| 587 |
+
within_tolerance = np.abs(diffs - (1.0 / fps)) <= tolerance_s
|
| 588 |
+
|
| 589 |
+
# Mask to ignore differences at the boundaries between episodes
|
| 590 |
+
mask = np.ones(len(diffs), dtype=bool)
|
| 591 |
+
ignored_diffs = episode_data_index["to"][:-1] - 1 # indices at the end of each episode
|
| 592 |
+
mask[ignored_diffs] = False
|
| 593 |
+
filtered_within_tolerance = within_tolerance[mask]
|
| 594 |
+
|
| 595 |
+
# Check if all remaining diffs are within tolerance
|
| 596 |
+
if not np.all(filtered_within_tolerance):
|
| 597 |
+
# Track original indices before masking
|
| 598 |
+
original_indices = np.arange(len(diffs))
|
| 599 |
+
filtered_indices = original_indices[mask]
|
| 600 |
+
outside_tolerance_filtered_indices = np.nonzero(~filtered_within_tolerance)[0]
|
| 601 |
+
outside_tolerance_indices = filtered_indices[outside_tolerance_filtered_indices]
|
| 602 |
+
|
| 603 |
+
outside_tolerances = []
|
| 604 |
+
for idx in outside_tolerance_indices:
|
| 605 |
+
entry = {
|
| 606 |
+
"timestamps": [timestamps[idx], timestamps[idx + 1]],
|
| 607 |
+
"diff": diffs[idx],
|
| 608 |
+
"episode_index": episode_indices[idx].item()
|
| 609 |
+
if hasattr(episode_indices[idx], "item")
|
| 610 |
+
else episode_indices[idx],
|
| 611 |
+
}
|
| 612 |
+
outside_tolerances.append(entry)
|
| 613 |
+
|
| 614 |
+
if raise_value_error:
|
| 615 |
+
raise ValueError(
|
| 616 |
+
f"""One or several timestamps unexpectedly violate the tolerance inside episode range.
|
| 617 |
+
This might be due to synchronization issues during data collection.
|
| 618 |
+
\n{pformat(outside_tolerances)}"""
|
| 619 |
+
)
|
| 620 |
+
return False
|
| 621 |
+
|
| 622 |
+
return True
|
| 623 |
+
|
| 624 |
+
|
| 625 |
+
def check_delta_timestamps(
|
| 626 |
+
delta_timestamps: dict[str, list[float]], fps: int, tolerance_s: float, raise_value_error: bool = True
|
| 627 |
+
) -> bool:
|
| 628 |
+
"""This will check if all the values in delta_timestamps are multiples of 1/fps +/- tolerance.
|
| 629 |
+
This is to ensure that these delta_timestamps added to any timestamp from a dataset will themselves be
|
| 630 |
+
actual timestamps from the dataset.
|
| 631 |
+
"""
|
| 632 |
+
outside_tolerance = {}
|
| 633 |
+
for key, delta_ts in delta_timestamps.items():
|
| 634 |
+
within_tolerance = [abs(ts * fps - round(ts * fps)) / fps <= tolerance_s for ts in delta_ts]
|
| 635 |
+
if not all(within_tolerance):
|
| 636 |
+
outside_tolerance[key] = [
|
| 637 |
+
ts for ts, is_within in zip(delta_ts, within_tolerance, strict=True) if not is_within
|
| 638 |
+
]
|
| 639 |
+
|
| 640 |
+
if len(outside_tolerance) > 0:
|
| 641 |
+
if raise_value_error:
|
| 642 |
+
raise ValueError(
|
| 643 |
+
f"""
|
| 644 |
+
The following delta_timestamps are found outside of tolerance range.
|
| 645 |
+
Please make sure they are multiples of 1/{fps} +/- tolerance and adjust
|
| 646 |
+
their values accordingly.
|
| 647 |
+
\n{pformat(outside_tolerance)}
|
| 648 |
+
"""
|
| 649 |
+
)
|
| 650 |
+
return False
|
| 651 |
+
|
| 652 |
+
return True
|
| 653 |
+
|
| 654 |
+
|
| 655 |
+
def get_delta_indices(delta_timestamps: dict[str, list[float]], fps: int) -> dict[str, list[int]]:
|
| 656 |
+
delta_indices = {}
|
| 657 |
+
for key, delta_ts in delta_timestamps.items():
|
| 658 |
+
delta_indices[key] = [round(d * fps) for d in delta_ts]
|
| 659 |
+
|
| 660 |
+
return delta_indices
|
| 661 |
+
|
| 662 |
+
|
| 663 |
+
def cycle(iterable):
|
| 664 |
+
"""The equivalent of itertools.cycle, but safe for Pytorch dataloaders.
|
| 665 |
+
|
| 666 |
+
See https://github.com/pytorch/pytorch/issues/23900 for information on why itertools.cycle is not safe.
|
| 667 |
+
"""
|
| 668 |
+
iterator = iter(iterable)
|
| 669 |
+
while True:
|
| 670 |
+
try:
|
| 671 |
+
yield next(iterator)
|
| 672 |
+
except StopIteration:
|
| 673 |
+
iterator = iter(iterable)
|
| 674 |
+
|
| 675 |
+
|
| 676 |
+
def create_branch(repo_id, *, branch: str, repo_type: str | None = None) -> None:
|
| 677 |
+
"""Create a branch on a existing Hugging Face repo. Delete the branch if it already
|
| 678 |
+
exists before creating it.
|
| 679 |
+
"""
|
| 680 |
+
api = HfApi()
|
| 681 |
+
|
| 682 |
+
branches = api.list_repo_refs(repo_id, repo_type=repo_type).branches
|
| 683 |
+
refs = [branch.ref for branch in branches]
|
| 684 |
+
ref = f"refs/heads/{branch}"
|
| 685 |
+
if ref in refs:
|
| 686 |
+
api.delete_branch(repo_id, repo_type=repo_type, branch=branch)
|
| 687 |
+
|
| 688 |
+
api.create_branch(repo_id, repo_type=repo_type, branch=branch)
|
| 689 |
+
|
| 690 |
+
|
| 691 |
+
def create_lerobot_dataset_card(
|
| 692 |
+
tags: list | None = None,
|
| 693 |
+
dataset_info: dict | None = None,
|
| 694 |
+
**kwargs,
|
| 695 |
+
) -> DatasetCard:
|
| 696 |
+
"""
|
| 697 |
+
Keyword arguments will be used to replace values in src/lerobot/datasets/card_template.md.
|
| 698 |
+
Note: If specified, license must be one of https://huggingface.co/docs/hub/repositories-licenses.
|
| 699 |
+
"""
|
| 700 |
+
card_tags = ["LeRobot"]
|
| 701 |
+
|
| 702 |
+
if tags:
|
| 703 |
+
card_tags += tags
|
| 704 |
+
if dataset_info:
|
| 705 |
+
dataset_structure = "[meta/info.json](meta/info.json):\n"
|
| 706 |
+
dataset_structure += f"```json\n{json.dumps(dataset_info, indent=4)}\n```\n"
|
| 707 |
+
kwargs = {**kwargs, "dataset_structure": dataset_structure}
|
| 708 |
+
card_data = DatasetCardData(
|
| 709 |
+
license=kwargs.get("license"),
|
| 710 |
+
tags=card_tags,
|
| 711 |
+
task_categories=["robotics"],
|
| 712 |
+
configs=[
|
| 713 |
+
{
|
| 714 |
+
"config_name": "default",
|
| 715 |
+
"data_files": "data/*/*.parquet",
|
| 716 |
+
}
|
| 717 |
+
],
|
| 718 |
+
)
|
| 719 |
+
|
| 720 |
+
card_template = (importlib.resources.files("lerobot.datasets") / "card_template.md").read_text()
|
| 721 |
+
|
| 722 |
+
return DatasetCard.from_template(
|
| 723 |
+
card_data=card_data,
|
| 724 |
+
template_str=card_template,
|
| 725 |
+
**kwargs,
|
| 726 |
+
)
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
class IterableNamespace(SimpleNamespace):
|
| 730 |
+
"""
|
| 731 |
+
A namespace object that supports both dictionary-like iteration and dot notation access.
|
| 732 |
+
Automatically converts nested dictionaries into IterableNamespaces.
|
| 733 |
+
|
| 734 |
+
This class extends SimpleNamespace to provide:
|
| 735 |
+
- Dictionary-style iteration over keys
|
| 736 |
+
- Access to items via both dot notation (obj.key) and brackets (obj["key"])
|
| 737 |
+
- Dictionary-like methods: items(), keys(), values()
|
| 738 |
+
- Recursive conversion of nested dictionaries
|
| 739 |
+
|
| 740 |
+
Args:
|
| 741 |
+
dictionary: Optional dictionary to initialize the namespace
|
| 742 |
+
**kwargs: Additional keyword arguments passed to SimpleNamespace
|
| 743 |
+
|
| 744 |
+
Examples:
|
| 745 |
+
>>> data = {"name": "Alice", "details": {"age": 25}}
|
| 746 |
+
>>> ns = IterableNamespace(data)
|
| 747 |
+
>>> ns.name
|
| 748 |
+
'Alice'
|
| 749 |
+
>>> ns.details.age
|
| 750 |
+
25
|
| 751 |
+
>>> list(ns.keys())
|
| 752 |
+
['name', 'details']
|
| 753 |
+
>>> for key, value in ns.items():
|
| 754 |
+
... print(f"{key}: {value}")
|
| 755 |
+
name: Alice
|
| 756 |
+
details: IterableNamespace(age=25)
|
| 757 |
+
"""
|
| 758 |
+
|
| 759 |
+
def __init__(self, dictionary: dict[str, Any] = None, **kwargs):
|
| 760 |
+
super().__init__(**kwargs)
|
| 761 |
+
if dictionary is not None:
|
| 762 |
+
for key, value in dictionary.items():
|
| 763 |
+
if isinstance(value, dict):
|
| 764 |
+
setattr(self, key, IterableNamespace(value))
|
| 765 |
+
else:
|
| 766 |
+
setattr(self, key, value)
|
| 767 |
+
|
| 768 |
+
def __iter__(self) -> Iterator[str]:
|
| 769 |
+
return iter(vars(self))
|
| 770 |
+
|
| 771 |
+
def __getitem__(self, key: str) -> Any:
|
| 772 |
+
return vars(self)[key]
|
| 773 |
+
|
| 774 |
+
def items(self):
|
| 775 |
+
return vars(self).items()
|
| 776 |
+
|
| 777 |
+
def values(self):
|
| 778 |
+
return vars(self).values()
|
| 779 |
+
|
| 780 |
+
def keys(self):
|
| 781 |
+
return vars(self).keys()
|
| 782 |
+
|
| 783 |
+
|
| 784 |
+
def validate_frame(frame: dict, features: dict):
|
| 785 |
+
expected_features = set(features) - set(DEFAULT_FEATURES)
|
| 786 |
+
actual_features = set(frame)
|
| 787 |
+
|
| 788 |
+
error_message = validate_features_presence(actual_features, expected_features)
|
| 789 |
+
|
| 790 |
+
common_features = actual_features & expected_features
|
| 791 |
+
for name in common_features - {"task"}:
|
| 792 |
+
error_message += validate_feature_dtype_and_shape(name, features[name], frame[name])
|
| 793 |
+
|
| 794 |
+
if error_message:
|
| 795 |
+
raise ValueError(error_message)
|
| 796 |
+
|
| 797 |
+
|
| 798 |
+
def validate_features_presence(actual_features: set[str], expected_features: set[str]):
|
| 799 |
+
error_message = ""
|
| 800 |
+
missing_features = expected_features - actual_features
|
| 801 |
+
extra_features = actual_features - expected_features
|
| 802 |
+
|
| 803 |
+
if missing_features or extra_features:
|
| 804 |
+
error_message += "Feature mismatch in `frame` dictionary:\n"
|
| 805 |
+
if missing_features:
|
| 806 |
+
error_message += f"Missing features: {missing_features}\n"
|
| 807 |
+
if extra_features:
|
| 808 |
+
error_message += f"Extra features: {extra_features}\n"
|
| 809 |
+
|
| 810 |
+
return error_message
|
| 811 |
+
|
| 812 |
+
def is_valid_numpy_dtype_string(dtype_str: str) -> bool:
|
| 813 |
+
"""
|
| 814 |
+
Return True if a given string can be converted to a numpy dtype.
|
| 815 |
+
"""
|
| 816 |
+
try:
|
| 817 |
+
# Attempt to convert the string to a numpy dtype
|
| 818 |
+
np.dtype(dtype_str)
|
| 819 |
+
return True
|
| 820 |
+
except TypeError:
|
| 821 |
+
# If a TypeError is raised, the string is not a valid dtype
|
| 822 |
+
return False
|
| 823 |
+
def validate_feature_dtype_and_shape(name: str, feature: dict, value: np.ndarray | PILImage.Image | str | bytes):
|
| 824 |
+
if isinstance(value, bytes) or isinstance(value, array.array): # ROS 1 and 2
|
| 825 |
+
#TODO fix bytes
|
| 826 |
+
return ""
|
| 827 |
+
expected_dtype = feature["dtype"]
|
| 828 |
+
expected_shape = feature["shape"]
|
| 829 |
+
if is_valid_numpy_dtype_string(expected_dtype):
|
| 830 |
+
return validate_feature_numpy_array(name, expected_dtype, expected_shape, value)
|
| 831 |
+
elif expected_dtype in ["image", "video"]:
|
| 832 |
+
return validate_feature_image_or_video(name, expected_shape, value)
|
| 833 |
+
elif expected_dtype == "string":
|
| 834 |
+
return validate_feature_string(name, value)
|
| 835 |
+
else:
|
| 836 |
+
raise NotImplementedError(f"The feature dtype '{expected_dtype}' is not implemented yet.")
|
| 837 |
+
|
| 838 |
+
|
| 839 |
+
def validate_feature_numpy_array(
|
| 840 |
+
name: str, expected_dtype: str, expected_shape: list[int], value: np.ndarray
|
| 841 |
+
):
|
| 842 |
+
error_message = ""
|
| 843 |
+
if isinstance(value, np.ndarray):
|
| 844 |
+
actual_dtype = value.dtype
|
| 845 |
+
actual_shape = value.shape
|
| 846 |
+
|
| 847 |
+
if actual_dtype != np.dtype(expected_dtype):
|
| 848 |
+
error_message += f"The feature '{name}' of dtype '{actual_dtype}' is not of the expected dtype '{expected_dtype}'.\n"
|
| 849 |
+
|
| 850 |
+
if actual_shape != expected_shape:
|
| 851 |
+
error_message += f"The feature '{name}' of shape '{actual_shape}' does not have the expected shape '{expected_shape}'.\n"
|
| 852 |
+
else:
|
| 853 |
+
error_message += f"The feature '{name}' is not a 'np.ndarray'. Expected type is '{expected_dtype}', but type '{type(value)}' provided instead.\n"
|
| 854 |
+
|
| 855 |
+
return error_message
|
| 856 |
+
|
| 857 |
+
|
| 858 |
+
def validate_feature_image_or_video(name: str, expected_shape: list[str], value: np.ndarray | PILImage.Image):
|
| 859 |
+
# Note: The check of pixels range ([0,1] for float and [0,255] for uint8) is done by the image writer threads.
|
| 860 |
+
error_message = ""
|
| 861 |
+
if isinstance(value, np.ndarray):
|
| 862 |
+
actual_shape = value.shape
|
| 863 |
+
c, h, w = expected_shape
|
| 864 |
+
if len(actual_shape) != 3 or (actual_shape != (c, h, w) and actual_shape != (h, w, c)):
|
| 865 |
+
error_message += f"The feature '{name}' of shape '{actual_shape}' does not have the expected shape '{(c, h, w)}' or '{(h, w, c)}'.\n"
|
| 866 |
+
elif isinstance(value, PILImage.Image):
|
| 867 |
+
pass
|
| 868 |
+
else:
|
| 869 |
+
error_message += f"The feature '{name}' is expected to be of type 'PIL.Image' or 'np.ndarray' channel first or channel last, but type '{type(value)}' provided instead.\n"
|
| 870 |
+
|
| 871 |
+
return error_message
|
| 872 |
+
|
| 873 |
+
|
| 874 |
+
def validate_feature_string(name: str, value: str):
|
| 875 |
+
if not isinstance(value, str):
|
| 876 |
+
return f"The feature '{name}' is expected to be of type 'str', but type '{type(value)}' provided instead.\n"
|
| 877 |
+
return ""
|
| 878 |
+
|
| 879 |
+
|
| 880 |
+
def validate_episode_buffer(episode_buffer: dict, total_episodes: int, features: dict):
|
| 881 |
+
if "size" not in episode_buffer:
|
| 882 |
+
raise ValueError("size key not found in episode_buffer")
|
| 883 |
+
|
| 884 |
+
if "task" not in episode_buffer:
|
| 885 |
+
raise ValueError("task key not found in episode_buffer")
|
| 886 |
+
|
| 887 |
+
if episode_buffer["episode_index"] != total_episodes:
|
| 888 |
+
# TODO(aliberts): Add option to use existing episode_index
|
| 889 |
+
raise NotImplementedError(
|
| 890 |
+
"You might have manually provided the episode_buffer with an episode_index that doesn't "
|
| 891 |
+
"match the total number of episodes already in the dataset. This is not supported for now."
|
| 892 |
+
)
|
| 893 |
+
|
| 894 |
+
if episode_buffer["size"] == 0:
|
| 895 |
+
raise ValueError("You must add one or several frames with `add_frame` before calling `add_episode`.")
|
| 896 |
+
|
| 897 |
+
buffer_keys = set(episode_buffer.keys()) - {"task", "size"}
|
| 898 |
+
if not buffer_keys == set(features):
|
| 899 |
+
raise ValueError(
|
| 900 |
+
f"Features from `episode_buffer` don't match the ones in `features`."
|
| 901 |
+
f"In episode_buffer not in features: {buffer_keys - set(features)}"
|
| 902 |
+
f"In features not in episode_buffer: {set(features) - buffer_keys}"
|
| 903 |
+
)
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py
ADDED
|
@@ -0,0 +1,527 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
|
| 3 |
+
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
import glob
|
| 17 |
+
import importlib
|
| 18 |
+
import logging
|
| 19 |
+
import warnings
|
| 20 |
+
from dataclasses import dataclass, field
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
from typing import Any, ClassVar
|
| 23 |
+
|
| 24 |
+
import av
|
| 25 |
+
import pyarrow as pa
|
| 26 |
+
import torch
|
| 27 |
+
import torchvision
|
| 28 |
+
from datasets.features.features import register_feature
|
| 29 |
+
from PIL import Image
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def get_safe_default_codec():
|
| 33 |
+
if importlib.util.find_spec("torchcodec"):
|
| 34 |
+
return "torchcodec"
|
| 35 |
+
else:
|
| 36 |
+
logging.warning(
|
| 37 |
+
"'torchcodec' is not available in your platform, falling back to 'pyav' as a default decoder"
|
| 38 |
+
)
|
| 39 |
+
return "pyav"
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def decode_video_frames(
|
| 43 |
+
video_path: Path | str,
|
| 44 |
+
timestamps: list[float],
|
| 45 |
+
tolerance_s: float,
|
| 46 |
+
backend: str | None = None,
|
| 47 |
+
) -> torch.Tensor:
|
| 48 |
+
"""
|
| 49 |
+
Decodes video frames using the specified backend.
|
| 50 |
+
|
| 51 |
+
Args:
|
| 52 |
+
video_path (Path): Path to the video file.
|
| 53 |
+
timestamps (list[float]): List of timestamps to extract frames.
|
| 54 |
+
tolerance_s (float): Allowed deviation in seconds for frame retrieval.
|
| 55 |
+
backend (str, optional): Backend to use for decoding. Defaults to "torchcodec" when available in the platform; otherwise, defaults to "pyav"..
|
| 56 |
+
|
| 57 |
+
Returns:
|
| 58 |
+
torch.Tensor: Decoded frames.
|
| 59 |
+
|
| 60 |
+
Currently supports torchcodec on cpu and pyav.
|
| 61 |
+
"""
|
| 62 |
+
if backend is None:
|
| 63 |
+
backend = get_safe_default_codec()
|
| 64 |
+
if backend == "torchcodec":
|
| 65 |
+
try:
|
| 66 |
+
return decode_video_frames_torchcodec(video_path, timestamps, tolerance_s)
|
| 67 |
+
except Exception as err:
|
| 68 |
+
warnings.warn(
|
| 69 |
+
f"torchcodec video decode failed ({type(err).__name__}: {err}); falling back to torchvision/pyav."
|
| 70 |
+
)
|
| 71 |
+
return decode_video_frames_torchvision(video_path, timestamps, tolerance_s, backend="pyav")
|
| 72 |
+
elif backend in ["pyav", "video_reader"]:
|
| 73 |
+
return decode_video_frames_torchvision(video_path, timestamps, tolerance_s, backend)
|
| 74 |
+
else:
|
| 75 |
+
raise ValueError(f"Unsupported video backend: {backend}")
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def decode_video_frames_torchvision(
|
| 79 |
+
video_path: Path | str,
|
| 80 |
+
timestamps: list[float],
|
| 81 |
+
tolerance_s: float,
|
| 82 |
+
backend: str = "pyav",
|
| 83 |
+
log_loaded_timestamps: bool = False,
|
| 84 |
+
) -> torch.Tensor:
|
| 85 |
+
"""Loads frames associated to the requested timestamps of a video
|
| 86 |
+
|
| 87 |
+
The backend can be either "pyav" (default) or "video_reader".
|
| 88 |
+
"video_reader" requires installing torchvision from source, see:
|
| 89 |
+
https://github.com/pytorch/vision/blob/main/torchvision/csrc/io/decoder/gpu/README.rst
|
| 90 |
+
(note that you need to compile against ffmpeg<4.3)
|
| 91 |
+
|
| 92 |
+
While both use cpu, "video_reader" is supposedly faster than "pyav" but requires additional setup.
|
| 93 |
+
For more info on video decoding, see `benchmark/video/README.md`
|
| 94 |
+
|
| 95 |
+
See torchvision doc for more info on these two backends:
|
| 96 |
+
https://pytorch.org/vision/0.18/index.html?highlight=backend#torchvision.set_video_backend
|
| 97 |
+
|
| 98 |
+
Note: Video benefits from inter-frame compression. Instead of storing every frame individually,
|
| 99 |
+
the encoder stores a reference frame (or a key frame) and subsequent frames as differences relative to
|
| 100 |
+
that key frame. As a consequence, to access a requested frame, we need to load the preceding key frame,
|
| 101 |
+
and all subsequent frames until reaching the requested frame. The number of key frames in a video
|
| 102 |
+
can be adjusted during encoding to take into account decoding time and video size in bytes.
|
| 103 |
+
"""
|
| 104 |
+
video_path = str(video_path)
|
| 105 |
+
|
| 106 |
+
# set backend
|
| 107 |
+
keyframes_only = False
|
| 108 |
+
torchvision.set_video_backend(backend)
|
| 109 |
+
if backend == "pyav":
|
| 110 |
+
keyframes_only = True # pyav doesn't support accurate seek
|
| 111 |
+
|
| 112 |
+
# set a video stream reader
|
| 113 |
+
# TODO(rcadene): also load audio stream at the same time
|
| 114 |
+
reader = torchvision.io.VideoReader(video_path, "video")
|
| 115 |
+
|
| 116 |
+
# set the first and last requested timestamps
|
| 117 |
+
# Note: previous timestamps are usually loaded, since we need to access the previous key frame
|
| 118 |
+
first_ts = min(timestamps)
|
| 119 |
+
last_ts = max(timestamps)
|
| 120 |
+
|
| 121 |
+
# access closest key frame of the first requested frame
|
| 122 |
+
# Note: closest key frame timestamp is usually smaller than `first_ts` (e.g. key frame can be the first frame of the video)
|
| 123 |
+
# for details on what `seek` is doing see: https://pyav.basswood-io.com/docs/stable/api/container.html?highlight=inputcontainer#av.container.InputContainer.seek
|
| 124 |
+
reader.seek(first_ts, keyframes_only=keyframes_only)
|
| 125 |
+
|
| 126 |
+
# load all frames until last requested frame
|
| 127 |
+
loaded_frames = []
|
| 128 |
+
loaded_ts = []
|
| 129 |
+
for frame in reader:
|
| 130 |
+
current_ts = frame["pts"]
|
| 131 |
+
if log_loaded_timestamps:
|
| 132 |
+
logging.info(f"frame loaded at timestamp={current_ts:.4f}")
|
| 133 |
+
loaded_frames.append(frame["data"])
|
| 134 |
+
loaded_ts.append(current_ts)
|
| 135 |
+
if current_ts >= last_ts:
|
| 136 |
+
break
|
| 137 |
+
|
| 138 |
+
if backend == "pyav":
|
| 139 |
+
reader.container.close()
|
| 140 |
+
|
| 141 |
+
reader = None
|
| 142 |
+
|
| 143 |
+
# Use float32 for timestamp distance computation (torch.cdist doesn't support bfloat16).
|
| 144 |
+
query_ts = torch.tensor(timestamps, dtype=torch.float32)
|
| 145 |
+
loaded_ts = torch.tensor(loaded_ts, dtype=torch.float32)
|
| 146 |
+
|
| 147 |
+
# compute distances between each query timestamp and timestamps of all loaded frames
|
| 148 |
+
dist = torch.cdist(query_ts[:, None], loaded_ts[:, None], p=1)
|
| 149 |
+
min_, argmin_ = dist.min(1)
|
| 150 |
+
|
| 151 |
+
is_within_tol = min_ < tolerance_s
|
| 152 |
+
assert is_within_tol.all(), (
|
| 153 |
+
f"One or several query timestamps unexpectedly violate the tolerance ({min_[~is_within_tol]} > {tolerance_s=})."
|
| 154 |
+
"It means that the closest frame that can be loaded from the video is too far away in time."
|
| 155 |
+
"This might be due to synchronization issues with timestamps during data collection."
|
| 156 |
+
"To be safe, we advise to ignore this item during training."
|
| 157 |
+
f"\nqueried timestamps: {query_ts}"
|
| 158 |
+
f"\nloaded timestamps: {loaded_ts}"
|
| 159 |
+
f"\nvideo: {video_path}"
|
| 160 |
+
f"\nbackend: {backend}"
|
| 161 |
+
)
|
| 162 |
+
|
| 163 |
+
# get closest frames to the query timestamps
|
| 164 |
+
closest_frames = torch.stack([loaded_frames[idx] for idx in argmin_])
|
| 165 |
+
closest_ts = loaded_ts[argmin_]
|
| 166 |
+
|
| 167 |
+
if log_loaded_timestamps:
|
| 168 |
+
logging.info(f"{closest_ts=}")
|
| 169 |
+
|
| 170 |
+
# convert to the pytorch format which is float32 in [0,1] range (and channel first)
|
| 171 |
+
closest_frames = closest_frames.type(torch.float32) / 255
|
| 172 |
+
|
| 173 |
+
assert len(timestamps) == len(closest_frames)
|
| 174 |
+
return closest_frames
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def decode_video_frames_torchcodec(
|
| 178 |
+
video_path: Path | str,
|
| 179 |
+
timestamps: list[float],
|
| 180 |
+
tolerance_s: float,
|
| 181 |
+
device: str = "cpu",
|
| 182 |
+
log_loaded_timestamps: bool = False,
|
| 183 |
+
) -> torch.Tensor:
|
| 184 |
+
"""Loads frames associated with the requested timestamps of a video using torchcodec.
|
| 185 |
+
|
| 186 |
+
Note: Setting device="cuda" outside the main process, e.g. in data loader workers, will lead to CUDA initialization errors.
|
| 187 |
+
|
| 188 |
+
Note: Video benefits from inter-frame compression. Instead of storing every frame individually,
|
| 189 |
+
the encoder stores a reference frame (or a key frame) and subsequent frames as differences relative to
|
| 190 |
+
that key frame. As a consequence, to access a requested frame, we need to load the preceding key frame,
|
| 191 |
+
and all subsequent frames until reaching the requested frame. The number of key frames in a video
|
| 192 |
+
can be adjusted during encoding to take into account decoding time and video size in bytes.
|
| 193 |
+
"""
|
| 194 |
+
|
| 195 |
+
if importlib.util.find_spec("torchcodec"):
|
| 196 |
+
from torchcodec.decoders import VideoDecoder
|
| 197 |
+
else:
|
| 198 |
+
raise ImportError("torchcodec is required but not available.")
|
| 199 |
+
|
| 200 |
+
# initialize video decoder
|
| 201 |
+
decoder = VideoDecoder(video_path, device=device, seek_mode="approximate")
|
| 202 |
+
loaded_frames = []
|
| 203 |
+
loaded_ts = []
|
| 204 |
+
# get metadata for frame information
|
| 205 |
+
metadata = decoder.metadata
|
| 206 |
+
average_fps = metadata.average_fps
|
| 207 |
+
|
| 208 |
+
# convert timestamps to frame indices
|
| 209 |
+
frame_indices = [round(ts * average_fps) for ts in timestamps]
|
| 210 |
+
|
| 211 |
+
# retrieve frames based on indices
|
| 212 |
+
frames_batch = decoder.get_frames_at(indices=frame_indices)
|
| 213 |
+
|
| 214 |
+
for frame, pts in zip(frames_batch.data, frames_batch.pts_seconds, strict=False):
|
| 215 |
+
loaded_frames.append(frame)
|
| 216 |
+
loaded_ts.append(pts.item())
|
| 217 |
+
if log_loaded_timestamps:
|
| 218 |
+
logging.info(f"Frame loaded at timestamp={pts:.4f}")
|
| 219 |
+
|
| 220 |
+
# Use float32 for timestamp distance computation (torch.cdist doesn't support bfloat16).
|
| 221 |
+
query_ts = torch.tensor(timestamps, dtype=torch.float32)
|
| 222 |
+
loaded_ts = torch.tensor(loaded_ts, dtype=torch.float32)
|
| 223 |
+
|
| 224 |
+
# compute distances between each query timestamp and loaded timestamps
|
| 225 |
+
dist = torch.cdist(query_ts[:, None], loaded_ts[:, None], p=1)
|
| 226 |
+
min_, argmin_ = dist.min(1)
|
| 227 |
+
|
| 228 |
+
is_within_tol = min_ < tolerance_s
|
| 229 |
+
assert is_within_tol.all(), (
|
| 230 |
+
f"One or several query timestamps unexpectedly violate the tolerance ({min_[~is_within_tol]} > {tolerance_s=})."
|
| 231 |
+
"It means that the closest frame that can be loaded from the video is too far away in time."
|
| 232 |
+
"This might be due to synchronization issues with timestamps during data collection."
|
| 233 |
+
"To be safe, we advise to ignore this item during training."
|
| 234 |
+
f"\nqueried timestamps: {query_ts}"
|
| 235 |
+
f"\nloaded timestamps: {loaded_ts}"
|
| 236 |
+
f"\nvideo: {video_path}"
|
| 237 |
+
)
|
| 238 |
+
|
| 239 |
+
# get closest frames to the query timestamps
|
| 240 |
+
closest_frames = torch.stack([loaded_frames[idx] for idx in argmin_])
|
| 241 |
+
closest_ts = loaded_ts[argmin_]
|
| 242 |
+
|
| 243 |
+
if log_loaded_timestamps:
|
| 244 |
+
logging.info(f"{closest_ts=}")
|
| 245 |
+
|
| 246 |
+
# convert to float32 in [0,1] range (channel first)
|
| 247 |
+
closest_frames = closest_frames.type(torch.float32) / 255
|
| 248 |
+
|
| 249 |
+
assert len(timestamps) == len(closest_frames)
|
| 250 |
+
return closest_frames
|
| 251 |
+
|
| 252 |
+
def encode_video_frames(
|
| 253 |
+
imgs_dir: Path | str,
|
| 254 |
+
video_path: Path | str,
|
| 255 |
+
fps: int,
|
| 256 |
+
vcodec: str = "libsvtav1",
|
| 257 |
+
pix_fmt: str = "yuv420p",
|
| 258 |
+
g: int | None = 2,
|
| 259 |
+
crf: int | None = 30,
|
| 260 |
+
fast_decode: int = 0,
|
| 261 |
+
log_level: int | None = av.logging.ERROR,
|
| 262 |
+
overwrite: bool = False,
|
| 263 |
+
) -> None:
|
| 264 |
+
"""More info on ffmpeg arguments tuning on `benchmark/video/README.md`"""
|
| 265 |
+
# Check encoder availability
|
| 266 |
+
if vcodec == "h264_nvenc" :
|
| 267 |
+
return encode_video_frames_ffmpeg(
|
| 268 |
+
imgs_dir, video_path, fps, pix_fmt = pix_fmt, overwrite=overwrite
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
if vcodec not in ["h264", "hevc", "libsvtav1"]:
|
| 272 |
+
raise ValueError(f"Unsupported video codec: {vcodec}. Supported codecs are: h264, hevc, libsvtav1.")
|
| 273 |
+
|
| 274 |
+
video_path = Path(video_path)
|
| 275 |
+
imgs_dir = Path(imgs_dir)
|
| 276 |
+
|
| 277 |
+
video_path.parent.mkdir(parents=True, exist_ok=overwrite)
|
| 278 |
+
|
| 279 |
+
# Encoders/pixel formats incompatibility check
|
| 280 |
+
if (vcodec == "libsvtav1" or vcodec == "hevc") and pix_fmt == "yuv444p":
|
| 281 |
+
logging.warning(
|
| 282 |
+
f"Incompatible pixel format 'yuv444p' for codec {vcodec}, auto-selecting format 'yuv420p'"
|
| 283 |
+
)
|
| 284 |
+
pix_fmt = "yuv420p"
|
| 285 |
+
|
| 286 |
+
# Get input frames
|
| 287 |
+
template = "frame_" + ("[0-9]" * 6) + ".jpeg"
|
| 288 |
+
input_list = sorted(
|
| 289 |
+
glob.glob(str(imgs_dir / template)), key=lambda x: int(x.split("_")[-1].split(".")[0])
|
| 290 |
+
)
|
| 291 |
+
|
| 292 |
+
# Define video output frame size (assuming all input frames are the same size)
|
| 293 |
+
if len(input_list) == 0:
|
| 294 |
+
raise FileNotFoundError(f"No images found in {imgs_dir}.")
|
| 295 |
+
dummy_image = Image.open(input_list[0])
|
| 296 |
+
width, height = dummy_image.size
|
| 297 |
+
|
| 298 |
+
# Define video codec options
|
| 299 |
+
video_options = {}
|
| 300 |
+
|
| 301 |
+
if g is not None:
|
| 302 |
+
video_options["g"] = str(g)
|
| 303 |
+
|
| 304 |
+
if crf is not None:
|
| 305 |
+
video_options["crf"] = str(crf)
|
| 306 |
+
|
| 307 |
+
if fast_decode:
|
| 308 |
+
key = "svtav1-params" if vcodec == "libsvtav1" else "tune"
|
| 309 |
+
value = f"fast-decode={fast_decode}" if vcodec == "libsvtav1" else "fastdecode"
|
| 310 |
+
video_options[key] = value
|
| 311 |
+
|
| 312 |
+
# Set logging level
|
| 313 |
+
if log_level is not None:
|
| 314 |
+
# "While less efficient, it is generally preferable to modify logging with Python’s logging"
|
| 315 |
+
logging.getLogger("libav").setLevel(log_level)
|
| 316 |
+
|
| 317 |
+
# Create and open output file (overwrite by default)
|
| 318 |
+
with av.open(str(video_path), "w") as output:
|
| 319 |
+
output_stream = output.add_stream(vcodec, fps, options=video_options)
|
| 320 |
+
output_stream.pix_fmt = pix_fmt
|
| 321 |
+
output_stream.width = width
|
| 322 |
+
output_stream.height = height
|
| 323 |
+
|
| 324 |
+
# Loop through input frames and encode them
|
| 325 |
+
for input_data in input_list:
|
| 326 |
+
input_image = Image.open(input_data).convert("RGB")
|
| 327 |
+
input_frame = av.VideoFrame.from_image(input_image)
|
| 328 |
+
packet = output_stream.encode(input_frame)
|
| 329 |
+
if packet:
|
| 330 |
+
output.mux(packet)
|
| 331 |
+
|
| 332 |
+
# Flush the encoder
|
| 333 |
+
packet = output_stream.encode()
|
| 334 |
+
if packet:
|
| 335 |
+
output.mux(packet)
|
| 336 |
+
|
| 337 |
+
# Reset logging level
|
| 338 |
+
if log_level is not None:
|
| 339 |
+
av.logging.restore_default_callback()
|
| 340 |
+
|
| 341 |
+
if not video_path.exists():
|
| 342 |
+
raise OSError(f"Video encoding did not work. File not found: {video_path}.")
|
| 343 |
+
|
| 344 |
+
# GPU speed up video encoding
|
| 345 |
+
def encode_video_frames_ffmpeg(
|
| 346 |
+
imgs_dir: Path | str,
|
| 347 |
+
video_path: Path | str,
|
| 348 |
+
fps: int,
|
| 349 |
+
vcodec: str = "h264_nvenc",
|
| 350 |
+
pix_fmt: str = "yuv420p",
|
| 351 |
+
g: int | None = 4,
|
| 352 |
+
crf: int | None = 30,
|
| 353 |
+
overwrite: bool = False,
|
| 354 |
+
) -> None:
|
| 355 |
+
import subprocess
|
| 356 |
+
import shutil
|
| 357 |
+
|
| 358 |
+
video_path = Path(video_path)
|
| 359 |
+
imgs_dir = Path(imgs_dir)
|
| 360 |
+
|
| 361 |
+
video_path.parent.mkdir(parents=True, exist_ok=True)
|
| 362 |
+
|
| 363 |
+
# Check if ffmpeg is available
|
| 364 |
+
ffmpeg_path = shutil.which("ffmpeg")
|
| 365 |
+
if ffmpeg_path is None:
|
| 366 |
+
raise RuntimeError("ffmpeg is not installed or not in system PATH")
|
| 367 |
+
|
| 368 |
+
# Build ffmpeg command
|
| 369 |
+
cmd = [
|
| 370 |
+
ffmpeg_path,
|
| 371 |
+
"-y" if overwrite else "-n",
|
| 372 |
+
"-framerate", str(fps),
|
| 373 |
+
"-pattern_type", "sequence",
|
| 374 |
+
"-i", str(imgs_dir / "frame_%06d.jpeg"),
|
| 375 |
+
"-vcodec", vcodec,
|
| 376 |
+
"-pix_fmt", pix_fmt,
|
| 377 |
+
]
|
| 378 |
+
|
| 379 |
+
if g is not None:
|
| 380 |
+
cmd.extend(["-g", str(g)])
|
| 381 |
+
|
| 382 |
+
if crf is not None:
|
| 383 |
+
cmd.extend(["-crf", str(crf)])
|
| 384 |
+
|
| 385 |
+
cmd.append(str(video_path))
|
| 386 |
+
|
| 387 |
+
try:
|
| 388 |
+
result = subprocess.run(
|
| 389 |
+
cmd,
|
| 390 |
+
capture_output=True,
|
| 391 |
+
text=True,
|
| 392 |
+
check=True
|
| 393 |
+
)
|
| 394 |
+
except subprocess.CalledProcessError as e:
|
| 395 |
+
error_msg = f"FFmpeg failed:\nCommand: {' '.join(cmd)}\nStdout: {e.stdout}\nStderr: {e.stderr}"
|
| 396 |
+
logging.error(error_msg)
|
| 397 |
+
logging.error(video_path)
|
| 398 |
+
logging.error(vcodec)
|
| 399 |
+
print(f"ffmpeg {' '.join(cmd)}")
|
| 400 |
+
raise RuntimeError(error_msg)
|
| 401 |
+
except Exception as e:
|
| 402 |
+
error_msg = f"Failed to run FFmpeg command: {' '.join(cmd)}\nError: {str(e)}"
|
| 403 |
+
logging.error(error_msg)
|
| 404 |
+
raise RuntimeError(error_msg)
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
@dataclass
|
| 408 |
+
class VideoFrame:
|
| 409 |
+
# TODO(rcadene, lhoestq): move to Hugging Face `datasets` repo
|
| 410 |
+
"""
|
| 411 |
+
Provides a type for a dataset containing video frames.
|
| 412 |
+
|
| 413 |
+
Example:
|
| 414 |
+
|
| 415 |
+
```python
|
| 416 |
+
data_dict = [{"image": {"path": "videos/episode_0.mp4", "timestamp": 0.3}}]
|
| 417 |
+
features = {"image": VideoFrame()}
|
| 418 |
+
Dataset.from_dict(data_dict, features=Features(features))
|
| 419 |
+
```
|
| 420 |
+
"""
|
| 421 |
+
|
| 422 |
+
pa_type: ClassVar[Any] = pa.struct({"path": pa.string(), "timestamp": pa.float32()})
|
| 423 |
+
_type: str = field(default="VideoFrame", init=False, repr=False)
|
| 424 |
+
|
| 425 |
+
def __call__(self):
|
| 426 |
+
return self.pa_type
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
with warnings.catch_warnings():
|
| 430 |
+
warnings.filterwarnings(
|
| 431 |
+
"ignore",
|
| 432 |
+
"'register_feature' is experimental and might be subject to breaking changes in the future.",
|
| 433 |
+
category=UserWarning,
|
| 434 |
+
)
|
| 435 |
+
# to make VideoFrame available in HuggingFace `datasets`
|
| 436 |
+
register_feature(VideoFrame, "VideoFrame")
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def get_audio_info(video_path: Path | str) -> dict:
|
| 440 |
+
# Set logging level
|
| 441 |
+
logging.getLogger("libav").setLevel(av.logging.ERROR)
|
| 442 |
+
|
| 443 |
+
# Getting audio stream information
|
| 444 |
+
audio_info = {}
|
| 445 |
+
with av.open(str(video_path), "r") as audio_file:
|
| 446 |
+
try:
|
| 447 |
+
audio_stream = audio_file.streams.audio[0]
|
| 448 |
+
except IndexError:
|
| 449 |
+
# Reset logging level
|
| 450 |
+
av.logging.restore_default_callback()
|
| 451 |
+
return {"has_audio": False}
|
| 452 |
+
|
| 453 |
+
audio_info["audio.channels"] = audio_stream.channels
|
| 454 |
+
audio_info["audio.codec"] = audio_stream.codec.canonical_name
|
| 455 |
+
# In an ideal loseless case : bit depth x sample rate x channels = bit rate.
|
| 456 |
+
# In an actual compressed case, the bit rate is set according to the compression level : the lower the bit rate, the more compression is applied.
|
| 457 |
+
audio_info["audio.bit_rate"] = audio_stream.bit_rate
|
| 458 |
+
audio_info["audio.sample_rate"] = audio_stream.sample_rate # Number of samples per second
|
| 459 |
+
# In an ideal loseless case : fixed number of bits per sample.
|
| 460 |
+
# In an actual compressed case : variable number of bits per sample (often reduced to match a given depth rate).
|
| 461 |
+
audio_info["audio.bit_depth"] = audio_stream.format.bits
|
| 462 |
+
audio_info["audio.channel_layout"] = audio_stream.layout.name
|
| 463 |
+
audio_info["has_audio"] = True
|
| 464 |
+
|
| 465 |
+
# Reset logging level
|
| 466 |
+
av.logging.restore_default_callback()
|
| 467 |
+
|
| 468 |
+
return audio_info
|
| 469 |
+
|
| 470 |
+
|
| 471 |
+
def get_video_info(video_path: Path | str) -> dict:
|
| 472 |
+
# Set logging level
|
| 473 |
+
logging.getLogger("libav").setLevel(av.logging.ERROR)
|
| 474 |
+
|
| 475 |
+
# Getting video stream information
|
| 476 |
+
video_info = {}
|
| 477 |
+
with av.open(str(video_path), "r") as video_file:
|
| 478 |
+
try:
|
| 479 |
+
video_stream = video_file.streams.video[0]
|
| 480 |
+
except IndexError:
|
| 481 |
+
# Reset logging level
|
| 482 |
+
av.logging.restore_default_callback()
|
| 483 |
+
return {}
|
| 484 |
+
|
| 485 |
+
video_info["video.height"] = video_stream.height
|
| 486 |
+
video_info["video.width"] = video_stream.width
|
| 487 |
+
video_info["video.codec"] = video_stream.codec.canonical_name
|
| 488 |
+
video_info["video.pix_fmt"] = video_stream.pix_fmt
|
| 489 |
+
video_info["video.is_depth_map"] = False
|
| 490 |
+
|
| 491 |
+
# Calculate fps from r_frame_rate
|
| 492 |
+
video_info["video.fps"] = int(video_stream.base_rate)
|
| 493 |
+
|
| 494 |
+
pixel_channels = get_video_pixel_channels(video_stream.pix_fmt)
|
| 495 |
+
video_info["video.channels"] = pixel_channels
|
| 496 |
+
|
| 497 |
+
# Reset logging level
|
| 498 |
+
av.logging.restore_default_callback()
|
| 499 |
+
|
| 500 |
+
# Adding audio stream information
|
| 501 |
+
video_info.update(**get_audio_info(video_path))
|
| 502 |
+
|
| 503 |
+
return video_info
|
| 504 |
+
|
| 505 |
+
|
| 506 |
+
def get_video_pixel_channels(pix_fmt: str) -> int:
|
| 507 |
+
if "gray" in pix_fmt or "depth" in pix_fmt or "monochrome" in pix_fmt:
|
| 508 |
+
return 1
|
| 509 |
+
elif "rgba" in pix_fmt or "yuva" in pix_fmt:
|
| 510 |
+
return 4
|
| 511 |
+
elif "rgb" in pix_fmt or "yuv" in pix_fmt:
|
| 512 |
+
return 3
|
| 513 |
+
else:
|
| 514 |
+
raise ValueError("Unknown format")
|
| 515 |
+
|
| 516 |
+
|
| 517 |
+
def get_image_pixel_channels(image: Image):
|
| 518 |
+
if image.mode == "L":
|
| 519 |
+
return 1 # Grayscale
|
| 520 |
+
elif image.mode == "LA":
|
| 521 |
+
return 2 # Grayscale + Alpha
|
| 522 |
+
elif image.mode == "RGB":
|
| 523 |
+
return 3 # RGB
|
| 524 |
+
elif image.mode == "RGBA":
|
| 525 |
+
return 4 # RGBA
|
| 526 |
+
else:
|
| 527 |
+
raise ValueError("Unknown format")
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py
ADDED
|
@@ -0,0 +1,1304 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
|
| 3 |
+
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
import contextlib
|
| 17 |
+
import logging
|
| 18 |
+
import shutil
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
from typing import Callable, List, Literal
|
| 21 |
+
import warnings
|
| 22 |
+
|
| 23 |
+
import datasets
|
| 24 |
+
import numpy as np
|
| 25 |
+
import packaging.version
|
| 26 |
+
import PIL.Image
|
| 27 |
+
import torch
|
| 28 |
+
import torch.utils
|
| 29 |
+
import pyarrow.parquet as pq
|
| 30 |
+
from datasets import concatenate_datasets, load_dataset
|
| 31 |
+
from huggingface_hub import HfApi, snapshot_download
|
| 32 |
+
from huggingface_hub.constants import REPOCARD_NAME
|
| 33 |
+
from huggingface_hub.errors import RevisionNotFoundError
|
| 34 |
+
|
| 35 |
+
# adapte to own path
|
| 36 |
+
from ..constants import HF_LEROBOT_HOME
|
| 37 |
+
from .datasets.compute_stats import aggregate_stats, compute_episode_stats
|
| 38 |
+
from .datasets.utils import (
|
| 39 |
+
DEFAULT_FEATURES,
|
| 40 |
+
DEFAULT_IMAGE_PATH,
|
| 41 |
+
INFO_PATH,
|
| 42 |
+
TASKS_PATH,
|
| 43 |
+
_validate_feature_names,
|
| 44 |
+
append_jsonlines,
|
| 45 |
+
backward_compatible_episodes_stats,
|
| 46 |
+
check_delta_timestamps,
|
| 47 |
+
check_timestamps_sync,
|
| 48 |
+
# check_version_compatibility,
|
| 49 |
+
create_empty_dataset_info,
|
| 50 |
+
create_lerobot_dataset_card,
|
| 51 |
+
embed_images,
|
| 52 |
+
get_delta_indices,
|
| 53 |
+
get_episode_data_index,
|
| 54 |
+
get_hf_features_from_features,
|
| 55 |
+
# get_safe_version,
|
| 56 |
+
hf_transform_to_torch,
|
| 57 |
+
is_valid_version,
|
| 58 |
+
load_episodes,
|
| 59 |
+
load_episodes_stats,
|
| 60 |
+
load_info,
|
| 61 |
+
load_stats,
|
| 62 |
+
load_tasks,
|
| 63 |
+
load_annotations,
|
| 64 |
+
validate_episode_buffer,
|
| 65 |
+
validate_frame,
|
| 66 |
+
write_episode,
|
| 67 |
+
write_episode_stats,
|
| 68 |
+
write_info,
|
| 69 |
+
write_json,
|
| 70 |
+
)
|
| 71 |
+
from .datasets.video_utils import (
|
| 72 |
+
VideoFrame,
|
| 73 |
+
decode_video_frames,
|
| 74 |
+
encode_video_frames,
|
| 75 |
+
get_safe_default_codec,
|
| 76 |
+
get_video_info,
|
| 77 |
+
)
|
| 78 |
+
import traceback
|
| 79 |
+
|
| 80 |
+
CODEBASE_VERSION = "v2.1"
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class LeRobotDatasetMetadata:
|
| 84 |
+
def __init__(
|
| 85 |
+
self,
|
| 86 |
+
repo_id: str,
|
| 87 |
+
root: str | Path | None = None,
|
| 88 |
+
revision: str | None = None,
|
| 89 |
+
force_cache_sync: bool = False,
|
| 90 |
+
):
|
| 91 |
+
self.repo_id = repo_id
|
| 92 |
+
self.revision = revision if revision else CODEBASE_VERSION
|
| 93 |
+
self.root = Path(root) if root is not None else HF_LEROBOT_HOME / repo_id
|
| 94 |
+
|
| 95 |
+
try:
|
| 96 |
+
if force_cache_sync:
|
| 97 |
+
raise FileNotFoundError
|
| 98 |
+
self.load_metadata()
|
| 99 |
+
except (FileNotFoundError, NotADirectoryError):
|
| 100 |
+
# if is_valid_version(self.revision):
|
| 101 |
+
# self.revision = get_safe_version(self.repo_id, self.revision)
|
| 102 |
+
|
| 103 |
+
(self.root / "meta").mkdir(exist_ok=True, parents=True)
|
| 104 |
+
self.pull_from_repo(allow_patterns="meta/")
|
| 105 |
+
self.load_metadata()
|
| 106 |
+
|
| 107 |
+
def load_metadata(self):
|
| 108 |
+
self.info = load_info(self.root)
|
| 109 |
+
# TODO add new check
|
| 110 |
+
# check_version_compatibility(self.repo_id, self._version, CODEBASE_VERSION)
|
| 111 |
+
self.tasks, self.task_to_task_index = load_tasks(self.root)
|
| 112 |
+
if (self.root / "annotations").exists():
|
| 113 |
+
self.annotations = load_annotations(self.root)
|
| 114 |
+
self.episodes = load_episodes(self.root)
|
| 115 |
+
if self._version < packaging.version.parse("v2.1"):
|
| 116 |
+
self.stats = load_stats(self.root)
|
| 117 |
+
self.episodes_stats = backward_compatible_episodes_stats(self.stats, self.episodes)
|
| 118 |
+
else:
|
| 119 |
+
self.episodes_stats = load_episodes_stats(self.root)
|
| 120 |
+
self.stats = aggregate_stats(list(self.episodes_stats.values()))
|
| 121 |
+
|
| 122 |
+
def pull_from_repo(
|
| 123 |
+
self,
|
| 124 |
+
allow_patterns: list[str] | str | None = None,
|
| 125 |
+
ignore_patterns: list[str] | str | None = None,
|
| 126 |
+
) -> None:
|
| 127 |
+
snapshot_download(
|
| 128 |
+
self.repo_id,
|
| 129 |
+
repo_type="dataset",
|
| 130 |
+
revision=self.revision,
|
| 131 |
+
local_dir=self.root,
|
| 132 |
+
allow_patterns=allow_patterns,
|
| 133 |
+
ignore_patterns=ignore_patterns,
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
@property
|
| 137 |
+
def _version(self) -> packaging.version.Version:
|
| 138 |
+
"""Codebase version used to create this dataset."""
|
| 139 |
+
return packaging.version.parse(self.info["codebase_version"])
|
| 140 |
+
|
| 141 |
+
def get_data_file_path(self, ep_index: int) -> Path:
|
| 142 |
+
ep_chunk = self.get_episode_chunk(ep_index)
|
| 143 |
+
fpath = self.data_path.format(episode_chunk=ep_chunk, episode_index=ep_index)
|
| 144 |
+
return Path(fpath)
|
| 145 |
+
|
| 146 |
+
def get_video_file_path(self, ep_index: int, vid_key: str) -> Path:
|
| 147 |
+
ep_chunk = self.get_episode_chunk(ep_index)
|
| 148 |
+
fpath = self.video_path.format(episode_chunk=ep_chunk, video_key=vid_key, episode_index=ep_index)
|
| 149 |
+
return Path(fpath)
|
| 150 |
+
|
| 151 |
+
def get_episode_chunk(self, ep_index: int) -> int:
|
| 152 |
+
return ep_index // self.chunks_size
|
| 153 |
+
|
| 154 |
+
@property
|
| 155 |
+
def data_path(self) -> str:
|
| 156 |
+
"""Formattable string for the parquet files."""
|
| 157 |
+
return self.info["data_path"]
|
| 158 |
+
|
| 159 |
+
@property
|
| 160 |
+
def video_path(self) -> str | None:
|
| 161 |
+
"""Formattable string for the video files."""
|
| 162 |
+
return self.info["video_path"]
|
| 163 |
+
|
| 164 |
+
@property
|
| 165 |
+
def robot_type(self) -> str | None:
|
| 166 |
+
"""Robot type used in recording this dataset."""
|
| 167 |
+
return self.info["robot_type"]
|
| 168 |
+
|
| 169 |
+
@property
|
| 170 |
+
def fps(self) -> int:
|
| 171 |
+
"""Frames per second used during data collection."""
|
| 172 |
+
return self.info["fps"]
|
| 173 |
+
|
| 174 |
+
@property
|
| 175 |
+
def features(self) -> dict[str, dict]:
|
| 176 |
+
"""All features contained in the dataset."""
|
| 177 |
+
return self.info["features"]
|
| 178 |
+
|
| 179 |
+
@property
|
| 180 |
+
def image_keys(self) -> list[str]:
|
| 181 |
+
"""Keys to access visual modalities stored as images."""
|
| 182 |
+
return [key for key, ft in self.features.items() if ft["dtype"] == "image"]
|
| 183 |
+
|
| 184 |
+
@property
|
| 185 |
+
def video_keys(self) -> list[str]:
|
| 186 |
+
"""Keys to access visual modalities stored as videos."""
|
| 187 |
+
return [key for key, ft in self.features.items() if ft["dtype"] == "video"]
|
| 188 |
+
|
| 189 |
+
@property
|
| 190 |
+
def camera_keys(self) -> list[str]:
|
| 191 |
+
"""Keys to access visual modalities (regardless of their storage method)."""
|
| 192 |
+
return [key for key, ft in self.features.items() if ft["dtype"] in ["video", "image"]]
|
| 193 |
+
|
| 194 |
+
@property
|
| 195 |
+
def names(self) -> dict[str, list | dict]:
|
| 196 |
+
"""Names of the various dimensions of vector modalities."""
|
| 197 |
+
return {key: ft["names"] for key, ft in self.features.items()}
|
| 198 |
+
|
| 199 |
+
@property
|
| 200 |
+
def shapes(self) -> dict:
|
| 201 |
+
"""Shapes for the different features."""
|
| 202 |
+
return {key: tuple(ft["shape"]) for key, ft in self.features.items()}
|
| 203 |
+
|
| 204 |
+
@property
|
| 205 |
+
def total_episodes(self) -> int:
|
| 206 |
+
"""Total number of episodes available."""
|
| 207 |
+
return self.info["total_episodes"]
|
| 208 |
+
|
| 209 |
+
@property
|
| 210 |
+
def total_frames(self) -> int:
|
| 211 |
+
"""Total number of frames saved in this dataset."""
|
| 212 |
+
return self.info["total_frames"]
|
| 213 |
+
|
| 214 |
+
@property
|
| 215 |
+
def total_tasks(self) -> int:
|
| 216 |
+
"""Total number of different tasks performed in this dataset."""
|
| 217 |
+
return self.info["total_tasks"]
|
| 218 |
+
|
| 219 |
+
@property
|
| 220 |
+
def total_chunks(self) -> int:
|
| 221 |
+
"""Total number of chunks (groups of episodes)."""
|
| 222 |
+
return self.info["total_chunks"]
|
| 223 |
+
|
| 224 |
+
@property
|
| 225 |
+
def chunks_size(self) -> int:
|
| 226 |
+
"""Max number of episodes per chunk."""
|
| 227 |
+
return self.info["chunks_size"]
|
| 228 |
+
|
| 229 |
+
def get_task_index(self, task: str) -> int | None:
|
| 230 |
+
"""
|
| 231 |
+
Given a task in natural language, returns its task_index if the task already exists in the dataset,
|
| 232 |
+
otherwise return None.
|
| 233 |
+
"""
|
| 234 |
+
return self.task_to_task_index.get(task, None)
|
| 235 |
+
|
| 236 |
+
def add_task(self, task: str):
|
| 237 |
+
"""
|
| 238 |
+
Given a task in natural language, add it to the dictionary of tasks.
|
| 239 |
+
"""
|
| 240 |
+
if task in self.task_to_task_index:
|
| 241 |
+
raise ValueError(f"The task '{task}' already exists and can't be added twice.")
|
| 242 |
+
|
| 243 |
+
task_index = self.info["total_tasks"]
|
| 244 |
+
self.task_to_task_index[task] = task_index
|
| 245 |
+
self.tasks[task_index] = task
|
| 246 |
+
self.info["total_tasks"] += 1
|
| 247 |
+
|
| 248 |
+
task_dict = {
|
| 249 |
+
"task_index": task_index,
|
| 250 |
+
"task": task,
|
| 251 |
+
}
|
| 252 |
+
append_jsonlines(task_dict, self.root / TASKS_PATH)
|
| 253 |
+
|
| 254 |
+
def save_episode(
|
| 255 |
+
self,
|
| 256 |
+
episode_index: int,
|
| 257 |
+
episode_length: int,
|
| 258 |
+
episode_tasks: list[str],
|
| 259 |
+
episode_stats: dict[str, dict],
|
| 260 |
+
raw_file_name: str | None = None,
|
| 261 |
+
) -> None:
|
| 262 |
+
self.info["total_episodes"] += 1
|
| 263 |
+
self.info["total_frames"] += episode_length
|
| 264 |
+
|
| 265 |
+
chunk = self.get_episode_chunk(episode_index)
|
| 266 |
+
if chunk >= self.total_chunks:
|
| 267 |
+
self.info["total_chunks"] += 1
|
| 268 |
+
|
| 269 |
+
self.info["splits"] = {"train": f"0:{self.info['total_episodes']}"}
|
| 270 |
+
self.info["total_videos"] += len(self.video_keys)
|
| 271 |
+
if len(self.video_keys) > 0:
|
| 272 |
+
self.update_video_info()
|
| 273 |
+
|
| 274 |
+
write_info(self.info, self.root)
|
| 275 |
+
|
| 276 |
+
episode_dict = {
|
| 277 |
+
"episode_index": episode_index,
|
| 278 |
+
"tasks": episode_tasks,
|
| 279 |
+
"length": episode_length,
|
| 280 |
+
}
|
| 281 |
+
if raw_file_name is not None:
|
| 282 |
+
episode_dict["raw_file_name"] = raw_file_name
|
| 283 |
+
|
| 284 |
+
self.episodes[episode_index] = episode_dict
|
| 285 |
+
write_episode(episode_dict, self.root)
|
| 286 |
+
|
| 287 |
+
self.episodes_stats[episode_index] = episode_stats
|
| 288 |
+
self.stats = aggregate_stats([self.stats, episode_stats]) if self.stats else episode_stats
|
| 289 |
+
write_episode_stats(episode_index, episode_stats, self.root)
|
| 290 |
+
|
| 291 |
+
def update_video_info(self) -> None:
|
| 292 |
+
"""
|
| 293 |
+
Warning: this function writes info from first episode videos, implicitly assuming that all videos have
|
| 294 |
+
been encoded the same way. Also, this means it assumes the first episode exists.
|
| 295 |
+
"""
|
| 296 |
+
for key in self.video_keys:
|
| 297 |
+
if not self.features[key].get("info", None):
|
| 298 |
+
video_path = self.root / self.get_video_file_path(ep_index=0, vid_key=key)
|
| 299 |
+
self.info["features"][key]["info"] = get_video_info(video_path)
|
| 300 |
+
|
| 301 |
+
def __repr__(self):
|
| 302 |
+
feature_keys = list(self.features)
|
| 303 |
+
return (
|
| 304 |
+
f"{self.__class__.__name__}({{\n"
|
| 305 |
+
f" Repository ID: '{self.repo_id}',\n"
|
| 306 |
+
f" Total episodes: '{self.total_episodes}',\n"
|
| 307 |
+
f" Total frames: '{self.total_frames}',\n"
|
| 308 |
+
f" Features: '{feature_keys}',\n"
|
| 309 |
+
"})',\n"
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
@classmethod
|
| 313 |
+
def create(
|
| 314 |
+
cls,
|
| 315 |
+
repo_id: str,
|
| 316 |
+
fps: int,
|
| 317 |
+
features: dict,
|
| 318 |
+
robot_type: str | None = None,
|
| 319 |
+
root: str | Path | None = None,
|
| 320 |
+
use_videos: bool = True,
|
| 321 |
+
) -> "LeRobotDatasetMetadata":
|
| 322 |
+
"""Creates metadata for a LeRobotDataset."""
|
| 323 |
+
obj = cls.__new__(cls)
|
| 324 |
+
obj.repo_id = repo_id
|
| 325 |
+
obj.root = Path(root) if root is not None else HF_LEROBOT_HOME / repo_id
|
| 326 |
+
|
| 327 |
+
obj.root.mkdir(parents=True, exist_ok=False)
|
| 328 |
+
|
| 329 |
+
# TODO(aliberts, rcadene): implement sanity check for features
|
| 330 |
+
features = {**features, **DEFAULT_FEATURES}
|
| 331 |
+
_validate_feature_names(features)
|
| 332 |
+
|
| 333 |
+
obj.tasks, obj.task_to_task_index = {}, {}
|
| 334 |
+
obj.episodes_stats, obj.stats, obj.episodes = {}, {}, {}
|
| 335 |
+
obj.info = create_empty_dataset_info(CODEBASE_VERSION, fps, features, use_videos, robot_type)
|
| 336 |
+
if len(obj.video_keys) > 0 and not use_videos:
|
| 337 |
+
raise ValueError()
|
| 338 |
+
write_json(obj.info, obj.root / INFO_PATH)
|
| 339 |
+
obj.revision = None
|
| 340 |
+
return obj
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
class LeRobotDataset(torch.utils.data.Dataset):
|
| 344 |
+
def __init__(
|
| 345 |
+
self,
|
| 346 |
+
repo_id: str,
|
| 347 |
+
root: str | Path | None = None,
|
| 348 |
+
episodes: list[int] | None = None,
|
| 349 |
+
image_transforms: Callable | None = None,
|
| 350 |
+
delta_timestamps: dict[list[float]] | None = None,
|
| 351 |
+
tolerance_s: float = 1e-4,
|
| 352 |
+
revision: str | None = None,
|
| 353 |
+
force_cache_sync: bool = False,
|
| 354 |
+
download_videos: bool = True,
|
| 355 |
+
video_backend: str | None = None,
|
| 356 |
+
video_codec: Literal["h264", "hevc", "libsvtav1", "h264_nvenc"] = "libsvtav1",
|
| 357 |
+
is_compute_episode_stats_image: bool = True,
|
| 358 |
+
):
|
| 359 |
+
"""
|
| 360 |
+
2 modes are available for instantiating this class, depending on 2 different use cases:
|
| 361 |
+
|
| 362 |
+
1. Your dataset already exists:
|
| 363 |
+
- On your local disk in the 'root' folder. This is typically the case when you recorded your
|
| 364 |
+
dataset locally and you may or may not have pushed it to the hub yet. Instantiating this class
|
| 365 |
+
with 'root' will load your dataset directly from disk. This can happen while you're offline (no
|
| 366 |
+
internet connection).
|
| 367 |
+
|
| 368 |
+
- On the Hugging Face Hub at the address https://huggingface.co/datasets/{repo_id} and not on
|
| 369 |
+
your local disk in the 'root' folder. Instantiating this class with this 'repo_id' will download
|
| 370 |
+
the dataset from that address and load it, pending your dataset is compliant with
|
| 371 |
+
codebase_version v2.0. If your dataset has been created before this new format, you will be
|
| 372 |
+
prompted to convert it using our conversion script from v1.6 to v2.0, which you can find at
|
| 373 |
+
lerobot/datasets/v2/convert_dataset_v1_to_v2.py.
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
2. Your dataset doesn't already exists (either on local disk or on the Hub): you can create an empty
|
| 377 |
+
LeRobotDataset with the 'create' classmethod. This can be used for recording a dataset or port an
|
| 378 |
+
existing dataset to the LeRobotDataset format.
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
In terms of files, LeRobotDataset encapsulates 3 main things:
|
| 382 |
+
- metadata:
|
| 383 |
+
- info contains various information about the dataset like shapes, keys, fps etc.
|
| 384 |
+
- stats stores the dataset statistics of the different modalities for normalization
|
| 385 |
+
- tasks contains the prompts for each task of the dataset, which can be used for
|
| 386 |
+
task-conditioned training.
|
| 387 |
+
- hf_dataset (from datasets.Dataset), which will read any values from parquet files.
|
| 388 |
+
- videos (optional) from which frames are loaded to be synchronous with data from parquet files.
|
| 389 |
+
|
| 390 |
+
A typical LeRobotDataset looks like this from its root path:
|
| 391 |
+
.
|
| 392 |
+
├── data
|
| 393 |
+
│ ├── chunk-000
|
| 394 |
+
│ │ ├── episode_000000.parquet
|
| 395 |
+
│ │ ├── episode_000001.parquet
|
| 396 |
+
│ │ ├── episode_000002.parquet
|
| 397 |
+
│ │ └── ...
|
| 398 |
+
│ ├── chunk-001
|
| 399 |
+
│ │ ├── episode_001000.parquet
|
| 400 |
+
│ │ ├── episode_001001.parquet
|
| 401 |
+
│ │ ├── episode_001002.parquet
|
| 402 |
+
│ │ └── ...
|
| 403 |
+
│ └── ...
|
| 404 |
+
├── meta
|
| 405 |
+
│ ├── episodes.jsonl
|
| 406 |
+
│ ├── info.json
|
| 407 |
+
│ ├── stats.json
|
| 408 |
+
│ └── tasks.jsonl
|
| 409 |
+
└── videos
|
| 410 |
+
├── chunk-000
|
| 411 |
+
│ ├── observation.images.laptop
|
| 412 |
+
│ │ ├── episode_000000.mp4
|
| 413 |
+
│ │ ├── episode_000001.mp4
|
| 414 |
+
│ │ ├── episode_000002.mp4
|
| 415 |
+
│ │ └── ...
|
| 416 |
+
│ ├── observation.images.phone
|
| 417 |
+
│ │ ├── episode_000000.mp4
|
| 418 |
+
│ │ ├── episode_000001.mp4
|
| 419 |
+
│ │ ├── episode_000002.mp4
|
| 420 |
+
│ │ └── ...
|
| 421 |
+
├── chunk-001
|
| 422 |
+
└── ...
|
| 423 |
+
|
| 424 |
+
Note that this file-based structure is designed to be as versatile as possible. The files are split by
|
| 425 |
+
episodes which allows a more granular control over which episodes one wants to use and download. The
|
| 426 |
+
structure of the dataset is entirely described in the info.json file, which can be easily downloaded
|
| 427 |
+
or viewed directly on the hub before downloading any actual data. The type of files used are very
|
| 428 |
+
simple and do not need complex tools to be read, it only uses .parquet, .json and .mp4 files (and .md
|
| 429 |
+
for the README).
|
| 430 |
+
|
| 431 |
+
Args:
|
| 432 |
+
repo_id (str): This is the repo id that will be used to fetch the dataset. Locally, the dataset
|
| 433 |
+
will be stored under root/repo_id.
|
| 434 |
+
root (Path | None, optional): Local directory to use for downloading/writing files. You can also
|
| 435 |
+
set the LEROBOT_HOME environment variable to point to a different location. Defaults to
|
| 436 |
+
'~/.cache/huggingface/lerobot'.
|
| 437 |
+
episodes (list[int] | None, optional): If specified, this will only load episodes specified by
|
| 438 |
+
their episode_index in this list. Defaults to None.
|
| 439 |
+
image_transforms (Callable | None, optional): You can pass standard v2 image transforms from
|
| 440 |
+
torchvision.transforms.v2 here which will be applied to visual modalities (whether they come
|
| 441 |
+
from videos or images). Defaults to None.
|
| 442 |
+
delta_timestamps (dict[list[float]] | None, optional): _description_. Defaults to None.
|
| 443 |
+
tolerance_s (float, optional): Tolerance in seconds used to ensure data timestamps are actually in
|
| 444 |
+
sync with the fps value. It is used at the init of the dataset to make sure that each
|
| 445 |
+
timestamps is separated to the next by 1/fps +/- tolerance_s. This also applies to frames
|
| 446 |
+
decoded from video files. It is also used to check that `delta_timestamps` (when provided) are
|
| 447 |
+
multiples of 1/fps. Defaults to 1e-4.
|
| 448 |
+
revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a
|
| 449 |
+
commit hash. Defaults to current codebase version tag.
|
| 450 |
+
sync_cache_first (bool, optional): Flag to sync and refresh local files first. If True and files
|
| 451 |
+
are already present in the local cache, this will be faster. However, files loaded might not
|
| 452 |
+
be in sync with the version on the hub, especially if you specified 'revision'. Defaults to
|
| 453 |
+
False.
|
| 454 |
+
download_videos (bool, optional): Flag to download the videos. Note that when set to True but the
|
| 455 |
+
video files are already present on local disk, they won't be downloaded again. Defaults to
|
| 456 |
+
True.
|
| 457 |
+
video_backend (str | None, optional): Video backend to use for decoding videos. Defaults to torchcodec when available int the platform; otherwise, defaults to 'pyav'.
|
| 458 |
+
You can also use the 'pyav' decoder used by Torchvision, which used to be the default option, or 'video_reader' which is another decoder of Torchvision.
|
| 459 |
+
"""
|
| 460 |
+
super().__init__()
|
| 461 |
+
self.repo_id = repo_id
|
| 462 |
+
self.root = Path(root) if root else HF_LEROBOT_HOME / repo_id
|
| 463 |
+
self.image_transforms = image_transforms
|
| 464 |
+
self.delta_timestamps = delta_timestamps
|
| 465 |
+
self.episodes = episodes
|
| 466 |
+
self.tolerance_s = tolerance_s
|
| 467 |
+
self.revision = revision if revision else CODEBASE_VERSION
|
| 468 |
+
self.video_backend = video_backend if video_backend else get_safe_default_codec()
|
| 469 |
+
self.video_codec = video_codec
|
| 470 |
+
self.is_compute_episode_stats_image = is_compute_episode_stats_image
|
| 471 |
+
self.delta_indices = None
|
| 472 |
+
self.during_training = True
|
| 473 |
+
|
| 474 |
+
# Unused attributes
|
| 475 |
+
self.image_writer = None
|
| 476 |
+
self.episode_buffer = None
|
| 477 |
+
|
| 478 |
+
self.root.mkdir(exist_ok=True, parents=True)
|
| 479 |
+
|
| 480 |
+
# Load metadata
|
| 481 |
+
self.meta = LeRobotDatasetMetadata(
|
| 482 |
+
self.repo_id, self.root, self.revision, force_cache_sync=force_cache_sync
|
| 483 |
+
)
|
| 484 |
+
if self.episodes is not None and self.meta._version >= packaging.version.parse("v2.1"):
|
| 485 |
+
episodes_stats = [self.meta.episodes_stats[ep_idx] for ep_idx in self.episodes]
|
| 486 |
+
self.stats = aggregate_stats(episodes_stats)
|
| 487 |
+
|
| 488 |
+
# Load actual data
|
| 489 |
+
try:
|
| 490 |
+
if force_cache_sync:
|
| 491 |
+
raise FileNotFoundError
|
| 492 |
+
assert all((self.root / fpath).is_file() for fpath in self.get_episodes_file_paths())
|
| 493 |
+
self.hf_dataset = self.load_hf_dataset()
|
| 494 |
+
except (AssertionError, FileNotFoundError, NotADirectoryError):
|
| 495 |
+
# self.revision = get_safe_version(self.repo_id, self.revision)
|
| 496 |
+
self.revision = CODEBASE_VERSION
|
| 497 |
+
self.download_episodes(download_videos)
|
| 498 |
+
self.hf_dataset = self.load_hf_dataset()
|
| 499 |
+
|
| 500 |
+
self.episode_data_index = get_episode_data_index(self.meta.episodes, self.episodes)
|
| 501 |
+
|
| 502 |
+
# Check timestamps
|
| 503 |
+
timestamps = torch.stack(self.hf_dataset["timestamp"]).numpy()
|
| 504 |
+
episode_indices = torch.stack(self.hf_dataset["episode_index"]).numpy()
|
| 505 |
+
ep_data_index_np = {k: t.numpy() for k, t in self.episode_data_index.items()}
|
| 506 |
+
# check_timestamps_sync(timestamps, episode_indices, ep_data_index_np, self.fps, self.tolerance_s)
|
| 507 |
+
|
| 508 |
+
# Setup delta_indices
|
| 509 |
+
if self.delta_timestamps is not None:
|
| 510 |
+
# check_delta_timestamps(self.delta_timestamps, self.fps, self.tolerance_s)
|
| 511 |
+
self.delta_indices = get_delta_indices(self.delta_timestamps, self.fps)
|
| 512 |
+
|
| 513 |
+
def push_to_hub(
|
| 514 |
+
self,
|
| 515 |
+
branch: str | None = None,
|
| 516 |
+
tags: list | None = None,
|
| 517 |
+
license: str | None = "apache-2.0",
|
| 518 |
+
tag_version: bool = True,
|
| 519 |
+
push_videos: bool = True,
|
| 520 |
+
private: bool = False,
|
| 521 |
+
allow_patterns: list[str] | str | None = None,
|
| 522 |
+
upload_large_folder: bool = False,
|
| 523 |
+
**card_kwargs,
|
| 524 |
+
) -> None:
|
| 525 |
+
ignore_patterns = ["images/"]
|
| 526 |
+
if not push_videos:
|
| 527 |
+
ignore_patterns.append("videos/")
|
| 528 |
+
|
| 529 |
+
hub_api = HfApi()
|
| 530 |
+
hub_api.create_repo(
|
| 531 |
+
repo_id=self.repo_id,
|
| 532 |
+
private=private,
|
| 533 |
+
repo_type="dataset",
|
| 534 |
+
exist_ok=True,
|
| 535 |
+
)
|
| 536 |
+
if branch:
|
| 537 |
+
hub_api.create_branch(
|
| 538 |
+
repo_id=self.repo_id,
|
| 539 |
+
branch=branch,
|
| 540 |
+
revision=self.revision,
|
| 541 |
+
repo_type="dataset",
|
| 542 |
+
exist_ok=True,
|
| 543 |
+
)
|
| 544 |
+
|
| 545 |
+
upload_kwargs = {
|
| 546 |
+
"repo_id": self.repo_id,
|
| 547 |
+
"folder_path": self.root,
|
| 548 |
+
"repo_type": "dataset",
|
| 549 |
+
"revision": branch,
|
| 550 |
+
"allow_patterns": allow_patterns,
|
| 551 |
+
"ignore_patterns": ignore_patterns,
|
| 552 |
+
}
|
| 553 |
+
if upload_large_folder:
|
| 554 |
+
hub_api.upload_large_folder(**upload_kwargs)
|
| 555 |
+
else:
|
| 556 |
+
hub_api.upload_folder(**upload_kwargs)
|
| 557 |
+
|
| 558 |
+
if not hub_api.file_exists(self.repo_id, REPOCARD_NAME, repo_type="dataset", revision=branch):
|
| 559 |
+
card = create_lerobot_dataset_card(
|
| 560 |
+
tags=tags, dataset_info=self.meta.info, license=license, **card_kwargs
|
| 561 |
+
)
|
| 562 |
+
card.push_to_hub(repo_id=self.repo_id, repo_type="dataset", revision=branch)
|
| 563 |
+
|
| 564 |
+
if tag_version:
|
| 565 |
+
with contextlib.suppress(RevisionNotFoundError):
|
| 566 |
+
hub_api.delete_tag(self.repo_id, tag=CODEBASE_VERSION, repo_type="dataset")
|
| 567 |
+
hub_api.create_tag(self.repo_id, tag=CODEBASE_VERSION, revision=branch, repo_type="dataset")
|
| 568 |
+
|
| 569 |
+
def pull_from_repo(
|
| 570 |
+
self,
|
| 571 |
+
allow_patterns: list[str] | str | None = None,
|
| 572 |
+
ignore_patterns: list[str] | str | None = None,
|
| 573 |
+
) -> None:
|
| 574 |
+
snapshot_download(
|
| 575 |
+
self.repo_id,
|
| 576 |
+
repo_type="dataset",
|
| 577 |
+
revision=self.revision,
|
| 578 |
+
local_dir=self.root,
|
| 579 |
+
allow_patterns=allow_patterns,
|
| 580 |
+
ignore_patterns=ignore_patterns,
|
| 581 |
+
)
|
| 582 |
+
|
| 583 |
+
def download_episodes(self, download_videos: bool = True) -> None:
|
| 584 |
+
"""Downloads the dataset from the given 'repo_id' at the provided version. If 'episodes' is given, this
|
| 585 |
+
will only download those episodes (selected by their episode_index). If 'episodes' is None, the whole
|
| 586 |
+
dataset will be downloaded. Thanks to the behavior of snapshot_download, if the files are already present
|
| 587 |
+
in 'local_dir', they won't be downloaded again.
|
| 588 |
+
"""
|
| 589 |
+
# TODO(rcadene, aliberts): implement faster transfer
|
| 590 |
+
# https://huggingface.co/docs/huggingface_hub/en/guides/download#faster-downloads
|
| 591 |
+
files = None
|
| 592 |
+
ignore_patterns = None if download_videos else "videos/"
|
| 593 |
+
if self.episodes is not None:
|
| 594 |
+
files = self.get_episodes_file_paths()
|
| 595 |
+
|
| 596 |
+
self.pull_from_repo(allow_patterns=files, ignore_patterns=ignore_patterns)
|
| 597 |
+
|
| 598 |
+
def get_episodes_file_paths(self) -> list[Path]:
|
| 599 |
+
episodes = self.episodes if self.episodes is not None else list(range(self.meta.total_episodes))
|
| 600 |
+
fpaths = [str(self.meta.get_data_file_path(ep_idx)) for ep_idx in episodes]
|
| 601 |
+
if len(self.meta.video_keys) > 0:
|
| 602 |
+
video_files = [
|
| 603 |
+
str(self.meta.get_video_file_path(ep_idx, vid_key))
|
| 604 |
+
for vid_key in self.meta.video_keys
|
| 605 |
+
for ep_idx in episodes
|
| 606 |
+
]
|
| 607 |
+
fpaths += video_files
|
| 608 |
+
|
| 609 |
+
return fpaths
|
| 610 |
+
|
| 611 |
+
def load_hf_dataset(self) -> datasets.Dataset:
|
| 612 |
+
"""hf_dataset contains all the observations, states, actions, rewards, etc."""
|
| 613 |
+
if self.episodes is None:
|
| 614 |
+
path = str(self.root / "data")
|
| 615 |
+
hf_dataset = load_dataset("parquet", data_dir=path, split="train")
|
| 616 |
+
else:
|
| 617 |
+
files = [str(self.root / self.meta.get_data_file_path(ep_idx)) for ep_idx in self.episodes]
|
| 618 |
+
hf_dataset = load_dataset("parquet", data_files=files, split="train")
|
| 619 |
+
|
| 620 |
+
# TODO(aliberts): hf_dataset.set_format("torch")
|
| 621 |
+
hf_dataset.set_transform(hf_transform_to_torch)
|
| 622 |
+
return hf_dataset
|
| 623 |
+
|
| 624 |
+
def create_hf_dataset(self) -> datasets.Dataset:
|
| 625 |
+
features = get_hf_features_from_features(self.features)
|
| 626 |
+
ft_dict = {col: [] for col in features}
|
| 627 |
+
hf_dataset = datasets.Dataset.from_dict(ft_dict, features=features, split="train")
|
| 628 |
+
|
| 629 |
+
# TODO(aliberts): hf_dataset.set_format("torch")
|
| 630 |
+
hf_dataset.set_transform(hf_transform_to_torch)
|
| 631 |
+
return hf_dataset
|
| 632 |
+
|
| 633 |
+
@property
|
| 634 |
+
def fps(self) -> int:
|
| 635 |
+
"""Frames per second used during data collection."""
|
| 636 |
+
return self.meta.fps
|
| 637 |
+
|
| 638 |
+
@property
|
| 639 |
+
def num_frames(self) -> int:
|
| 640 |
+
"""Number of frames in selected episodes."""
|
| 641 |
+
return len(self.hf_dataset) if self.hf_dataset is not None else self.meta.total_frames
|
| 642 |
+
|
| 643 |
+
@property
|
| 644 |
+
def num_episodes(self) -> int:
|
| 645 |
+
"""Number of episodes selected."""
|
| 646 |
+
return len(self.episodes) if self.episodes is not None else self.meta.total_episodes
|
| 647 |
+
|
| 648 |
+
@property
|
| 649 |
+
def features(self) -> dict[str, dict]:
|
| 650 |
+
return self.meta.features
|
| 651 |
+
|
| 652 |
+
@property
|
| 653 |
+
def hf_features(self) -> datasets.Features:
|
| 654 |
+
"""Features of the hf_dataset."""
|
| 655 |
+
if self.hf_dataset is not None:
|
| 656 |
+
return self.hf_dataset.features
|
| 657 |
+
else:
|
| 658 |
+
return get_hf_features_from_features(self.features)
|
| 659 |
+
|
| 660 |
+
def _get_query_indices(self, idx: int, ep_idx: int) -> tuple[dict[str, list[int | bool]]]:
|
| 661 |
+
ep_start = self.episode_data_index["from"][ep_idx]
|
| 662 |
+
ep_end = self.episode_data_index["to"][ep_idx]
|
| 663 |
+
query_indices = {
|
| 664 |
+
key: [max(ep_start.item(), min(ep_end.item() - 1, idx + delta)) for delta in delta_idx]
|
| 665 |
+
for key, delta_idx in self.delta_indices.items()
|
| 666 |
+
}
|
| 667 |
+
padding = { # Pad values outside of current episode range
|
| 668 |
+
f"{key}_is_pad": torch.BoolTensor(
|
| 669 |
+
[(idx + delta < ep_start.item()) | (idx + delta >= ep_end.item()) for delta in delta_idx]
|
| 670 |
+
)
|
| 671 |
+
for key, delta_idx in self.delta_indices.items()
|
| 672 |
+
}
|
| 673 |
+
return query_indices, padding
|
| 674 |
+
|
| 675 |
+
def _get_query_timestamps(
|
| 676 |
+
self,
|
| 677 |
+
current_ts: float,
|
| 678 |
+
query_indices: dict[str, list[int]] | None = None,
|
| 679 |
+
) -> dict[str, list[float]]:
|
| 680 |
+
query_timestamps = {}
|
| 681 |
+
for key in self.meta.video_keys:
|
| 682 |
+
if query_indices is not None and key in query_indices:
|
| 683 |
+
timestamps = self.hf_dataset.select(query_indices[key])["timestamp"]
|
| 684 |
+
query_timestamps[key] = torch.stack(timestamps).tolist()
|
| 685 |
+
else:
|
| 686 |
+
query_timestamps[key] = [current_ts]
|
| 687 |
+
|
| 688 |
+
return query_timestamps
|
| 689 |
+
|
| 690 |
+
def _query_hf_dataset(self, query_indices: dict[str, list[int]]) -> dict:
|
| 691 |
+
return {
|
| 692 |
+
key: torch.stack(self.hf_dataset.select(q_idx)[key])
|
| 693 |
+
for key, q_idx in query_indices.items()
|
| 694 |
+
if key not in self.meta.video_keys
|
| 695 |
+
}
|
| 696 |
+
|
| 697 |
+
def _query_hf_dataset_fast(self, query_indices: dict[str, list[int]]) -> dict:
|
| 698 |
+
result = {}
|
| 699 |
+
processed_indices = set()
|
| 700 |
+
index_to_selected = {}
|
| 701 |
+
for key, q_idx in query_indices.items():
|
| 702 |
+
if key not in self.meta.video_keys :
|
| 703 |
+
if 'images' in key and not self.during_training:
|
| 704 |
+
continue
|
| 705 |
+
q_idx_tuple = tuple(q_idx)
|
| 706 |
+
if q_idx_tuple not in processed_indices:
|
| 707 |
+
selected_data = self.hf_dataset.select(q_idx)
|
| 708 |
+
index_to_selected[q_idx_tuple] = selected_data
|
| 709 |
+
processed_indices.add(q_idx_tuple)
|
| 710 |
+
else:
|
| 711 |
+
selected_data = index_to_selected[q_idx_tuple]
|
| 712 |
+
result[key] = torch.stack(selected_data[key])
|
| 713 |
+
return result
|
| 714 |
+
|
| 715 |
+
# no videos
|
| 716 |
+
def get_episode_data(self, episode_id: int) -> dict:
|
| 717 |
+
ep_start = self.episode_data_index["from"][episode_id].item()
|
| 718 |
+
ep_end = self.episode_data_index["to"][episode_id].item()
|
| 719 |
+
q_idx = list(range(ep_start, ep_end))
|
| 720 |
+
# selected_data = self.hf_dataset.select(q_idx)
|
| 721 |
+
selected_data = self.hf_dataset[q_idx]
|
| 722 |
+
res_keys = self.meta.features.keys() - set(self.meta.video_keys)
|
| 723 |
+
res = {key : torch.stack(selected_data[key]) for key in res_keys}
|
| 724 |
+
return res
|
| 725 |
+
|
| 726 |
+
def _query_videos(self, query_timestamps: dict[str, list[float]], ep_idx: int) -> dict[str, torch.Tensor]:
|
| 727 |
+
"""Note: When using data workers (e.g. DataLoader with num_workers>0), do not call this function
|
| 728 |
+
in the main process (e.g. by using a second Dataloader with num_workers=0). It will result in a
|
| 729 |
+
Segmentation Fault. This probably happens because a memory reference to the video loader is created in
|
| 730 |
+
the main process and a subprocess fails to access it.
|
| 731 |
+
"""
|
| 732 |
+
item = {}
|
| 733 |
+
for vid_key, query_ts in query_timestamps.items():
|
| 734 |
+
video_path = self.root / self.meta.get_video_file_path(ep_idx, vid_key)
|
| 735 |
+
frames = decode_video_frames(video_path, query_ts, self.tolerance_s, self.video_backend)
|
| 736 |
+
item[vid_key] = frames.squeeze(0)
|
| 737 |
+
|
| 738 |
+
return item
|
| 739 |
+
|
| 740 |
+
def _add_padding_keys(self, item: dict, padding: dict[str, list[bool]]) -> dict:
|
| 741 |
+
for key, val in padding.items():
|
| 742 |
+
item[key] = torch.BoolTensor(val)
|
| 743 |
+
return item
|
| 744 |
+
|
| 745 |
+
def __len__(self):
|
| 746 |
+
return self.num_frames
|
| 747 |
+
|
| 748 |
+
def __getitem__(self, idx) -> dict:
|
| 749 |
+
item = self.hf_dataset[idx]
|
| 750 |
+
ep_idx = item["episode_index"].item()
|
| 751 |
+
|
| 752 |
+
query_indices = None
|
| 753 |
+
if self.delta_indices is not None:
|
| 754 |
+
current_ep_idx = self.episodes.index(ep_idx) if self.episodes is not None else ep_idx
|
| 755 |
+
query_indices, padding = self._get_query_indices(idx, current_ep_idx)
|
| 756 |
+
query_result = self._query_hf_dataset_fast(query_indices)
|
| 757 |
+
item = {**item, **padding}
|
| 758 |
+
for key, val in query_result.items():
|
| 759 |
+
item[key] = val
|
| 760 |
+
|
| 761 |
+
if len(self.meta.video_keys) > 0 and self.during_training:
|
| 762 |
+
current_ts = item["timestamp"].item()
|
| 763 |
+
query_timestamps = self._get_query_timestamps(current_ts, query_indices)
|
| 764 |
+
video_frames = self._query_videos(query_timestamps, ep_idx)
|
| 765 |
+
item = {**video_frames, **item}
|
| 766 |
+
|
| 767 |
+
if self.image_transforms is not None:
|
| 768 |
+
image_keys = self.meta.camera_keys
|
| 769 |
+
for cam in image_keys:
|
| 770 |
+
item[cam] = self.image_transforms(item[cam])
|
| 771 |
+
|
| 772 |
+
# Add task as a string
|
| 773 |
+
task_idx = item["task_index"].item()
|
| 774 |
+
item["task"] = self.meta.tasks[task_idx]
|
| 775 |
+
if "coarse_task_index" in item:
|
| 776 |
+
coarse_task_index = item["coarse_task_index"].item()
|
| 777 |
+
item["coarse_task"] = self.meta.tasks[coarse_task_index]
|
| 778 |
+
|
| 779 |
+
if "operating_hand_index" in item:
|
| 780 |
+
operating_hand_index = item["operating_hand_index"].item()
|
| 781 |
+
item["operating_hand"] = self.meta.tasks[operating_hand_index]
|
| 782 |
+
|
| 783 |
+
if "subtask_annotation" in item and hasattr(self.meta, "annotations"):
|
| 784 |
+
index = item["subtask_annotation"][0].item()
|
| 785 |
+
item["subtask"] = self.meta.annotations["subtask"][index]
|
| 786 |
+
|
| 787 |
+
if "atomic_task_index" in item and item["atomic_task_index"] is not None:
|
| 788 |
+
atomic_task_index = item["atomic_task_index"].item()
|
| 789 |
+
item["subtask"] = self.meta.tasks[int(atomic_task_index)]
|
| 790 |
+
|
| 791 |
+
return item
|
| 792 |
+
|
| 793 |
+
def __repr__(self):
|
| 794 |
+
feature_keys = list(self.features)
|
| 795 |
+
return (
|
| 796 |
+
f"{self.__class__.__name__}({{\n"
|
| 797 |
+
f" Repository ID: '{self.repo_id}',\n"
|
| 798 |
+
f" Number of selected episodes: '{self.num_episodes}',\n"
|
| 799 |
+
f" Number of selected samples: '{self.num_frames}',\n"
|
| 800 |
+
f" Features: '{feature_keys}',\n"
|
| 801 |
+
"})',\n"
|
| 802 |
+
)
|
| 803 |
+
|
| 804 |
+
def create_episode_buffer(self, episode_index: int | None = None) -> dict:
|
| 805 |
+
current_ep_idx = self.meta.total_episodes if episode_index is None else episode_index
|
| 806 |
+
ep_buffer = {}
|
| 807 |
+
# size and task are special cases that are not in self.features
|
| 808 |
+
ep_buffer["size"] = 0
|
| 809 |
+
ep_buffer["task"] = []
|
| 810 |
+
for key in self.features:
|
| 811 |
+
ep_buffer[key] = current_ep_idx if key == "episode_index" else []
|
| 812 |
+
return ep_buffer
|
| 813 |
+
|
| 814 |
+
def _get_image_file_path(self, episode_index: int, image_key: str, frame_index: int) -> Path:
|
| 815 |
+
fpath = DEFAULT_IMAGE_PATH.format(
|
| 816 |
+
image_key=image_key, episode_index=episode_index, frame_index=frame_index
|
| 817 |
+
)
|
| 818 |
+
return self.root / fpath
|
| 819 |
+
|
| 820 |
+
# def _save_image(self, image: torch.Tensor | np.ndarray | PIL.Image.Image | bytes, fpath: Path) -> None:
|
| 821 |
+
# if self.image_writer is None:
|
| 822 |
+
# if isinstance(image, torch.Tensor):
|
| 823 |
+
# image = image.cpu().numpy()
|
| 824 |
+
# write_image(image, fpath)
|
| 825 |
+
# else:
|
| 826 |
+
# self.image_writer.save_image(image=image, fpath=fpath)
|
| 827 |
+
|
| 828 |
+
def add_frame(self, frame: dict, task: List[str], timestamp: float | None = None) -> None:
|
| 829 |
+
"""
|
| 830 |
+
This function only adds the frame to the episode_buffer. Apart from images — which are written in a
|
| 831 |
+
temporary directory — nothing is written to disk. To save those frames, the 'save_episode()' method
|
| 832 |
+
then needs to be called.
|
| 833 |
+
"""
|
| 834 |
+
# Convert torch to numpy if needed
|
| 835 |
+
assert len(task) == 4, "Task frame must be of two elements"
|
| 836 |
+
for name in frame:
|
| 837 |
+
if isinstance(frame[name], torch.Tensor):
|
| 838 |
+
frame[name] = frame[name].numpy()
|
| 839 |
+
|
| 840 |
+
validate_frame(frame, self.features)
|
| 841 |
+
|
| 842 |
+
if self.episode_buffer is None:
|
| 843 |
+
self.episode_buffer = self.create_episode_buffer()
|
| 844 |
+
|
| 845 |
+
# Automatically add frame_index and timestamp to episode buffer
|
| 846 |
+
frame_index = self.episode_buffer["size"]
|
| 847 |
+
if timestamp is None:
|
| 848 |
+
timestamp = frame_index / self.fps
|
| 849 |
+
self.episode_buffer["frame_index"].append(frame_index)
|
| 850 |
+
self.episode_buffer["timestamp"].append(timestamp)
|
| 851 |
+
self.episode_buffer["task"].append(task)
|
| 852 |
+
|
| 853 |
+
# Add frame features to episode_buffer
|
| 854 |
+
for key in frame:
|
| 855 |
+
if key not in self.features:
|
| 856 |
+
raise ValueError(
|
| 857 |
+
f"An element of the frame is not in the features. '{key}' not in '{self.features.keys()}'."
|
| 858 |
+
)
|
| 859 |
+
|
| 860 |
+
if self.features[key]["dtype"] in ["image", "video"]:
|
| 861 |
+
img_path = self._get_image_file_path(
|
| 862 |
+
episode_index=self.episode_buffer["episode_index"], image_key=key, frame_index=frame_index
|
| 863 |
+
)
|
| 864 |
+
if frame_index == 0:
|
| 865 |
+
img_path.parent.mkdir(parents=True, exist_ok=True)
|
| 866 |
+
# self._save_image(frame[key], img_path)
|
| 867 |
+
self.episode_buffer[key].append(str(img_path))
|
| 868 |
+
else:
|
| 869 |
+
self.episode_buffer[key].append(frame[key])
|
| 870 |
+
|
| 871 |
+
self.episode_buffer["size"] += 1
|
| 872 |
+
|
| 873 |
+
def save_episode(self, episode_data: dict | None = None, raw_file_name: str | None = None) -> None:
|
| 874 |
+
"""
|
| 875 |
+
This will save to disk the current episode in self.episode_buffer.
|
| 876 |
+
|
| 877 |
+
Args:
|
| 878 |
+
episode_data (dict | None, optional): Dict containing the episode data to save. If None, this will
|
| 879 |
+
save the current episode in self.episode_buffer, which is filled with 'add_frame'. Defaults to
|
| 880 |
+
None.
|
| 881 |
+
"""
|
| 882 |
+
if not episode_data:
|
| 883 |
+
episode_buffer = self.episode_buffer
|
| 884 |
+
|
| 885 |
+
validate_episode_buffer(episode_buffer, self.meta.total_episodes, self.features)
|
| 886 |
+
|
| 887 |
+
# size and task are special cases that won't be added to hf_dataset
|
| 888 |
+
episode_length = episode_buffer.pop("size")
|
| 889 |
+
tasks = episode_buffer.pop("task")
|
| 890 |
+
episode_tasks = list(set([item for sublist in tasks for item in sublist]))
|
| 891 |
+
episode_index = episode_buffer["episode_index"]
|
| 892 |
+
|
| 893 |
+
episode_buffer["index"] = np.arange(self.meta.total_frames, self.meta.total_frames + episode_length)
|
| 894 |
+
episode_buffer["episode_index"] = np.full((episode_length,), episode_index)
|
| 895 |
+
|
| 896 |
+
# Add new tasks to the tasks dictionary
|
| 897 |
+
for task in episode_tasks:
|
| 898 |
+
task_index = self.meta.get_task_index(task)
|
| 899 |
+
if task_index is None:
|
| 900 |
+
self.meta.add_task(task)
|
| 901 |
+
|
| 902 |
+
# Given tasks in natural language, find their corresponding task indices
|
| 903 |
+
episode_buffer["coarse_task_index"] = np.array([self.meta.get_task_index(task[0]) for task in tasks])
|
| 904 |
+
episode_buffer["task_index"] = np.array([self.meta.get_task_index(task[1]) for task in tasks])
|
| 905 |
+
episode_buffer["coarse_quality_index"] = np.array([self.meta.get_task_index(task[2]) for task in tasks])
|
| 906 |
+
episode_buffer["quality_index"] = np.array([self.meta.get_task_index(task[3]) for task in tasks])
|
| 907 |
+
|
| 908 |
+
for key, ft in self.features.items():
|
| 909 |
+
# index, episode_index, task_index are already processed above, and image and video
|
| 910 |
+
# are processed separately by storing image path and frame info as meta data
|
| 911 |
+
if key in ["index", "episode_index", "coarse_task_index", "task_index", "coarse_quality_index", "quality_index"] or ft["dtype"] in ["image", "video"]:
|
| 912 |
+
continue
|
| 913 |
+
episode_buffer[key] = np.stack(episode_buffer[key])
|
| 914 |
+
|
| 915 |
+
self._wait_image_writer()
|
| 916 |
+
self._save_episode_table(episode_buffer, episode_index)
|
| 917 |
+
ep_stats = compute_episode_stats(episode_buffer, self.features, self.is_compute_episode_stats_image)
|
| 918 |
+
|
| 919 |
+
if len(self.meta.video_keys) > 0:
|
| 920 |
+
video_paths = self.encode_episode_videos(episode_index)
|
| 921 |
+
for key in self.meta.video_keys:
|
| 922 |
+
episode_buffer[key] = video_paths[key]
|
| 923 |
+
|
| 924 |
+
# `meta.save_episode` be executed after encoding the videos
|
| 925 |
+
self.meta.save_episode(episode_index, episode_length, episode_tasks, ep_stats, raw_file_name)
|
| 926 |
+
|
| 927 |
+
ep_data_index = get_episode_data_index(self.meta.episodes, [episode_index])
|
| 928 |
+
ep_data_index_np = {k: t.numpy() for k, t in ep_data_index.items()}
|
| 929 |
+
check_timestamps_sync(
|
| 930 |
+
episode_buffer["timestamp"],
|
| 931 |
+
episode_buffer["episode_index"],
|
| 932 |
+
ep_data_index_np,
|
| 933 |
+
self.fps,
|
| 934 |
+
self.tolerance_s,
|
| 935 |
+
)
|
| 936 |
+
|
| 937 |
+
video_files = list(self.root.rglob("*.mp4"))
|
| 938 |
+
assert len(video_files) == self.num_episodes * len(self.meta.video_keys)
|
| 939 |
+
|
| 940 |
+
parquet_files = list(self.root.rglob("*.parquet"))
|
| 941 |
+
assert len(parquet_files) == self.num_episodes
|
| 942 |
+
|
| 943 |
+
# delete images
|
| 944 |
+
img_dir = self.root / "images"
|
| 945 |
+
if img_dir.is_dir():
|
| 946 |
+
shutil.rmtree(self.root / "images")
|
| 947 |
+
|
| 948 |
+
if not episode_data: # Reset the buffer
|
| 949 |
+
self.episode_buffer = self.create_episode_buffer()
|
| 950 |
+
|
| 951 |
+
def _save_episode_table(self, episode_buffer: dict, episode_index: int) -> None:
|
| 952 |
+
episode_dict = {key: episode_buffer[key] for key in self.hf_features}
|
| 953 |
+
ep_dataset = datasets.Dataset.from_dict(episode_dict, features=self.hf_features, split="train")
|
| 954 |
+
ep_dataset = embed_images(ep_dataset)
|
| 955 |
+
self.hf_dataset = concatenate_datasets([self.hf_dataset, ep_dataset])
|
| 956 |
+
self.hf_dataset.set_transform(hf_transform_to_torch)
|
| 957 |
+
ep_data_path = self.root / self.meta.get_data_file_path(ep_index=episode_index)
|
| 958 |
+
ep_data_path.parent.mkdir(parents=True, exist_ok=True)
|
| 959 |
+
ep_dataset.to_parquet(ep_data_path)
|
| 960 |
+
|
| 961 |
+
def clear_episode_buffer(self) -> None:
|
| 962 |
+
episode_index = self.episode_buffer["episode_index"]
|
| 963 |
+
if self.image_writer is not None:
|
| 964 |
+
for cam_key in self.meta.camera_keys:
|
| 965 |
+
img_dir = self._get_image_file_path(
|
| 966 |
+
episode_index=episode_index, image_key=cam_key, frame_index=0
|
| 967 |
+
).parent
|
| 968 |
+
if img_dir.is_dir():
|
| 969 |
+
shutil.rmtree(img_dir)
|
| 970 |
+
|
| 971 |
+
# Reset the buffer
|
| 972 |
+
self.episode_buffer = self.create_episode_buffer()
|
| 973 |
+
|
| 974 |
+
# def start_image_writer(self, num_processes: int = 0, num_threads: int = 4) -> None:
|
| 975 |
+
# if isinstance(self.image_writer, AsyncImageWriter):
|
| 976 |
+
# logging.warning(
|
| 977 |
+
# "You are starting a new AsyncImageWriter that is replacing an already existing one in the dataset."
|
| 978 |
+
# )
|
| 979 |
+
|
| 980 |
+
# self.image_writer = AsyncImageWriter(
|
| 981 |
+
# num_processes=num_processes,
|
| 982 |
+
# num_threads=num_threads,
|
| 983 |
+
# )
|
| 984 |
+
|
| 985 |
+
def stop_image_writer(self) -> None:
|
| 986 |
+
"""
|
| 987 |
+
Whenever wrapping this dataset inside a parallelized DataLoader, this needs to be called first to
|
| 988 |
+
remove the image_writer in order for the LeRobotDataset object to be picklable and parallelized.
|
| 989 |
+
"""
|
| 990 |
+
if self.image_writer is not None:
|
| 991 |
+
self.image_writer.stop()
|
| 992 |
+
self.image_writer = None
|
| 993 |
+
|
| 994 |
+
def _wait_image_writer(self) -> None:
|
| 995 |
+
"""Wait for asynchronous image writer to finish."""
|
| 996 |
+
if self.image_writer is not None:
|
| 997 |
+
self.image_writer.wait_until_done()
|
| 998 |
+
|
| 999 |
+
def encode_videos(self) -> None:
|
| 1000 |
+
"""
|
| 1001 |
+
Use ffmpeg to convert frames stored as png into mp4 videos.
|
| 1002 |
+
Note: `encode_video_frames` is a blocking call. Making it asynchronous shouldn't speedup encoding,
|
| 1003 |
+
since video encoding with ffmpeg is already using multithreading.
|
| 1004 |
+
"""
|
| 1005 |
+
for ep_idx in range(self.meta.total_episodes):
|
| 1006 |
+
self.encode_episode_videos(ep_idx)
|
| 1007 |
+
|
| 1008 |
+
def encode_episode_videos(self, episode_index: int) -> dict:
|
| 1009 |
+
"""
|
| 1010 |
+
Use ffmpeg to convert frames stored as png into mp4 videos.
|
| 1011 |
+
Note: `encode_video_frames` is a blocking call. Making it asynchronous shouldn't speedup encoding,
|
| 1012 |
+
since video encoding with ffmpeg is already using multithreading.
|
| 1013 |
+
"""
|
| 1014 |
+
video_paths = {}
|
| 1015 |
+
for key in self.meta.video_keys:
|
| 1016 |
+
video_path = self.root / self.meta.get_video_file_path(episode_index, key)
|
| 1017 |
+
video_paths[key] = str(video_path)
|
| 1018 |
+
if video_path.is_file():
|
| 1019 |
+
# Skip if video is already encoded. Could be the case when resuming data recording.
|
| 1020 |
+
continue
|
| 1021 |
+
img_dir = self._get_image_file_path(
|
| 1022 |
+
episode_index=episode_index, image_key=key, frame_index=0
|
| 1023 |
+
).parent
|
| 1024 |
+
encode_video_frames(img_dir, video_path, self.fps, overwrite=True, vcodec=self.video_codec)
|
| 1025 |
+
|
| 1026 |
+
return video_paths
|
| 1027 |
+
|
| 1028 |
+
@classmethod
|
| 1029 |
+
def create(
|
| 1030 |
+
cls,
|
| 1031 |
+
repo_id: str,
|
| 1032 |
+
fps: int,
|
| 1033 |
+
features: dict,
|
| 1034 |
+
root: str | Path | None = None,
|
| 1035 |
+
robot_type: str | None = None,
|
| 1036 |
+
use_videos: bool = True,
|
| 1037 |
+
tolerance_s: float = 1e-4,
|
| 1038 |
+
image_writer_processes: int = 0,
|
| 1039 |
+
image_writer_threads: int = 0,
|
| 1040 |
+
video_backend: str | None = None,
|
| 1041 |
+
video_codec: Literal["h264", "hevc", "libsvtav1", "h264_nvenc"] = "libsvtav1",
|
| 1042 |
+
is_compute_episode_stats_image = True,
|
| 1043 |
+
) -> "LeRobotDataset":
|
| 1044 |
+
"""Create a LeRobot Dataset from scratch in order to record data."""
|
| 1045 |
+
obj = cls.__new__(cls)
|
| 1046 |
+
obj.meta = LeRobotDatasetMetadata.create(
|
| 1047 |
+
repo_id=repo_id,
|
| 1048 |
+
fps=fps,
|
| 1049 |
+
robot_type=robot_type,
|
| 1050 |
+
features=features,
|
| 1051 |
+
root=root,
|
| 1052 |
+
use_videos=use_videos,
|
| 1053 |
+
)
|
| 1054 |
+
obj.repo_id = obj.meta.repo_id
|
| 1055 |
+
obj.root = obj.meta.root
|
| 1056 |
+
obj.revision = None
|
| 1057 |
+
obj.tolerance_s = tolerance_s
|
| 1058 |
+
obj.image_writer = None
|
| 1059 |
+
|
| 1060 |
+
# if image_writer_processes or image_writer_threads:
|
| 1061 |
+
# obj.start_image_writer(image_writer_processes, image_writer_threads)
|
| 1062 |
+
|
| 1063 |
+
# TODO(aliberts, rcadene, alexander-soare): Merge this with OnlineBuffer/DataBuffer
|
| 1064 |
+
obj.episode_buffer = obj.create_episode_buffer()
|
| 1065 |
+
|
| 1066 |
+
obj.episodes = None
|
| 1067 |
+
obj.hf_dataset = obj.create_hf_dataset()
|
| 1068 |
+
obj.image_transforms = None
|
| 1069 |
+
obj.delta_timestamps = None
|
| 1070 |
+
obj.delta_indices = None
|
| 1071 |
+
obj.episode_data_index = None
|
| 1072 |
+
obj.video_backend = video_backend if video_backend is not None else get_safe_default_codec()
|
| 1073 |
+
obj.video_codec = video_codec
|
| 1074 |
+
obj.is_compute_episode_stats_image = is_compute_episode_stats_image
|
| 1075 |
+
return obj
|
| 1076 |
+
|
| 1077 |
+
|
| 1078 |
+
class MultiLeRobotDataset(torch.utils.data.Dataset):
|
| 1079 |
+
"""A dataset consisting of multiple underlying `LeRobotDataset`s.
|
| 1080 |
+
|
| 1081 |
+
The underlying `LeRobotDataset`s are effectively concatenated, and this class adopts much of the API
|
| 1082 |
+
structure of `LeRobotDataset`.
|
| 1083 |
+
"""
|
| 1084 |
+
|
| 1085 |
+
def __init__(
|
| 1086 |
+
self,
|
| 1087 |
+
dataset_dirs: list[str],
|
| 1088 |
+
episodes: dict | None = None,
|
| 1089 |
+
image_transforms: Callable | None = None,
|
| 1090 |
+
delta_timestamps: dict[list[float]] | None = None,
|
| 1091 |
+
tolerances_s: dict | None = None,
|
| 1092 |
+
download_videos: bool = True,
|
| 1093 |
+
video_backend: str | None = None,
|
| 1094 |
+
):
|
| 1095 |
+
super().__init__()
|
| 1096 |
+
self.dataset_dirs = dataset_dirs
|
| 1097 |
+
ds_roots = [Path(ds_dir) for ds_dir in dataset_dirs]
|
| 1098 |
+
ds_names = [ds_dir for ds_dir in dataset_dirs]
|
| 1099 |
+
self.ds_names = ds_names
|
| 1100 |
+
self.ds_roots = ds_roots
|
| 1101 |
+
self.tolerances_s = tolerances_s if tolerances_s else dict.fromkeys(ds_names, 0.0001)
|
| 1102 |
+
# Construct the underlying datasets passing everything but `transform` and `delta_timestamps` which
|
| 1103 |
+
# are handled by this class.
|
| 1104 |
+
self._datasets = []
|
| 1105 |
+
for ds_root, ds_name in zip(ds_roots, ds_names, strict=True):
|
| 1106 |
+
try:
|
| 1107 |
+
_dataset = LeRobotDataset(
|
| 1108 |
+
ds_name,
|
| 1109 |
+
root=ds_root,
|
| 1110 |
+
episodes=episodes[ds_name] if episodes else None,
|
| 1111 |
+
image_transforms=image_transforms,
|
| 1112 |
+
delta_timestamps=delta_timestamps,
|
| 1113 |
+
tolerance_s=self.tolerances_s[ds_name],
|
| 1114 |
+
download_videos=download_videos,
|
| 1115 |
+
video_backend=video_backend,
|
| 1116 |
+
)
|
| 1117 |
+
self._datasets.append(_dataset)
|
| 1118 |
+
except Exception as e:
|
| 1119 |
+
# logging.error(e)
|
| 1120 |
+
logging.error(f"Exception while process ds_root: {ds_root}, ds_name: {ds_name}")
|
| 1121 |
+
traceback.print_exc()
|
| 1122 |
+
raise e
|
| 1123 |
+
|
| 1124 |
+
# Disable any data keys that are not common across all of the datasets. Note: we may relax this
|
| 1125 |
+
# restriction in future iterations of this class. For now, this is necessary at least for being able
|
| 1126 |
+
# to use PyTorch's default DataLoader collate function.
|
| 1127 |
+
self.disabled_features = set()
|
| 1128 |
+
intersection_features = set(self._datasets[0].features)
|
| 1129 |
+
for ds in self._datasets:
|
| 1130 |
+
intersection_features.intersection_update(ds.features)
|
| 1131 |
+
if len(intersection_features) == 0:
|
| 1132 |
+
raise RuntimeError(
|
| 1133 |
+
"Multiple datasets were provided but they had no keys common to all of them. "
|
| 1134 |
+
"The multi-dataset functionality currently only keeps common keys."
|
| 1135 |
+
)
|
| 1136 |
+
for ds_name, ds in zip(self.ds_names, self._datasets, strict=True):
|
| 1137 |
+
# Disable warning of empty extra keys do
|
| 1138 |
+
extra_keys = set(ds.features).difference(intersection_features)
|
| 1139 |
+
if extra_keys:
|
| 1140 |
+
logging.warning(
|
| 1141 |
+
f"keys {extra_keys} of {ds_name} were disabled as they are not contained in all the "
|
| 1142 |
+
"other datasets."
|
| 1143 |
+
)
|
| 1144 |
+
self.disabled_features.update(extra_keys)
|
| 1145 |
+
|
| 1146 |
+
self.image_transforms = image_transforms
|
| 1147 |
+
self.delta_timestamps = delta_timestamps
|
| 1148 |
+
# TODO(rcadene, aliberts): We should not perform this aggregation for datasets
|
| 1149 |
+
# with multiple robots of different ranges. Instead we should have one normalization
|
| 1150 |
+
# per robot.
|
| 1151 |
+
self.stats = aggregate_stats([dataset.meta.stats for dataset in self._datasets])
|
| 1152 |
+
|
| 1153 |
+
def set_during_training(self, during_training: bool):
|
| 1154 |
+
for dataset in self._datasets:
|
| 1155 |
+
dataset.during_training = during_training
|
| 1156 |
+
|
| 1157 |
+
@property
|
| 1158 |
+
def repo_id_to_index(self):
|
| 1159 |
+
"""Return a mapping from dataset repo_id to a dataset index automatically created by this class.
|
| 1160 |
+
|
| 1161 |
+
This index is incorporated as a data key in the dictionary returned by `__getitem__`.
|
| 1162 |
+
"""
|
| 1163 |
+
return {repo_id: i for i, repo_id in enumerate(self.ds_names)}
|
| 1164 |
+
|
| 1165 |
+
@property
|
| 1166 |
+
def repo_index_to_id(self):
|
| 1167 |
+
"""Return the inverse mapping if repo_id_to_index."""
|
| 1168 |
+
return {v: k for k, v in self.repo_id_to_index}
|
| 1169 |
+
|
| 1170 |
+
@property
|
| 1171 |
+
def fps(self) -> int:
|
| 1172 |
+
"""Frames per second used during data collection.
|
| 1173 |
+
|
| 1174 |
+
NOTE: Fow now, this relies on a check in __init__ to make sure all sub-datasets have the same info.
|
| 1175 |
+
"""
|
| 1176 |
+
return self._datasets[0].meta.info["fps"]
|
| 1177 |
+
|
| 1178 |
+
@property
|
| 1179 |
+
def video(self) -> bool:
|
| 1180 |
+
"""Returns True if this dataset loads video frames from mp4 files.
|
| 1181 |
+
|
| 1182 |
+
Returns False if it only loads images from png files.
|
| 1183 |
+
|
| 1184 |
+
NOTE: Fow now, this relies on a check in __init__ to make sure all sub-datasets have the same info.
|
| 1185 |
+
"""
|
| 1186 |
+
return self._datasets[0].meta.info.get("video", False)
|
| 1187 |
+
|
| 1188 |
+
@property
|
| 1189 |
+
def features(self) -> datasets.Features:
|
| 1190 |
+
features = {}
|
| 1191 |
+
for dataset in self._datasets:
|
| 1192 |
+
features.update({k: v for k, v in dataset.hf_features.items() if k not in self.disabled_features})
|
| 1193 |
+
return features
|
| 1194 |
+
|
| 1195 |
+
@property
|
| 1196 |
+
def camera_keys(self) -> list[str]:
|
| 1197 |
+
"""Keys to access image and video stream from cameras."""
|
| 1198 |
+
keys = []
|
| 1199 |
+
for key, feats in self.features.items():
|
| 1200 |
+
if isinstance(feats, (datasets.Image, VideoFrame)):
|
| 1201 |
+
keys.append(key)
|
| 1202 |
+
return keys
|
| 1203 |
+
|
| 1204 |
+
@property
|
| 1205 |
+
def video_frame_keys(self) -> list[str]:
|
| 1206 |
+
"""Keys to access video frames that requires to be decoded into images.
|
| 1207 |
+
|
| 1208 |
+
Note: It is empty if the dataset contains images only,
|
| 1209 |
+
or equal to `self.cameras` if the dataset contains videos only,
|
| 1210 |
+
or can even be a subset of `self.cameras` in a case of a mixed image/video dataset.
|
| 1211 |
+
"""
|
| 1212 |
+
video_frame_keys = []
|
| 1213 |
+
for key, feats in self.features.items():
|
| 1214 |
+
if isinstance(feats, VideoFrame):
|
| 1215 |
+
video_frame_keys.append(key)
|
| 1216 |
+
return video_frame_keys
|
| 1217 |
+
|
| 1218 |
+
@property
|
| 1219 |
+
def num_frames(self) -> int:
|
| 1220 |
+
"""Number of samples/frames."""
|
| 1221 |
+
return sum(d.num_frames for d in self._datasets)
|
| 1222 |
+
|
| 1223 |
+
@property
|
| 1224 |
+
def num_episodes(self) -> int:
|
| 1225 |
+
"""Number of episodes."""
|
| 1226 |
+
return sum(d.num_episodes for d in self._datasets)
|
| 1227 |
+
|
| 1228 |
+
@property
|
| 1229 |
+
def tolerance_s(self) -> float:
|
| 1230 |
+
"""Tolerance in seconds used to discard loaded frames when their timestamps
|
| 1231 |
+
are not close enough from the requested frames. It is only used when `delta_timestamps`
|
| 1232 |
+
is provided or when loading video frames from mp4 files.
|
| 1233 |
+
"""
|
| 1234 |
+
# 1e-4 to account for possible numerical error
|
| 1235 |
+
return 1 / self.fps - 1e-4
|
| 1236 |
+
|
| 1237 |
+
def get_episode_data(self, episode_idx: int) -> dict:
|
| 1238 |
+
for dataset in self._datasets:
|
| 1239 |
+
if episode_idx < dataset.num_episodes:
|
| 1240 |
+
file = str(dataset.root / dataset.meta.get_data_file_path(dataset.episodes[episode_idx]))
|
| 1241 |
+
table = pq.read_table(str(file))
|
| 1242 |
+
|
| 1243 |
+
result_dict = {}
|
| 1244 |
+
for col_name in table.column_names:
|
| 1245 |
+
col = table[col_name]
|
| 1246 |
+
try:
|
| 1247 |
+
np_arr = col.to_numpy(zero_copy_only=True)
|
| 1248 |
+
except Exception:
|
| 1249 |
+
raw = col.to_numpy()
|
| 1250 |
+
np_arr = np.stack(raw) if raw.dtype == object else raw
|
| 1251 |
+
with warnings.catch_warnings():
|
| 1252 |
+
warnings.filterwarnings(
|
| 1253 |
+
"ignore",
|
| 1254 |
+
message="The given NumPy array is not writable",
|
| 1255 |
+
category=UserWarning,
|
| 1256 |
+
)
|
| 1257 |
+
# deal with string in parquet file
|
| 1258 |
+
if np_arr.dtype == 'O':
|
| 1259 |
+
result_dict[col_name] = np_arr
|
| 1260 |
+
else:
|
| 1261 |
+
result_dict[col_name] = torch.from_numpy(np_arr)
|
| 1262 |
+
return result_dict
|
| 1263 |
+
else:
|
| 1264 |
+
episode_idx -= dataset.num_episodes
|
| 1265 |
+
raise IndexError(f"Episode index {episode_idx} out of bounds.")
|
| 1266 |
+
|
| 1267 |
+
def __len__(self):
|
| 1268 |
+
return self.num_frames
|
| 1269 |
+
|
| 1270 |
+
def __getitem__(self, idx: int) -> dict[str, torch.Tensor]:
|
| 1271 |
+
if idx >= len(self):
|
| 1272 |
+
raise IndexError(f"Index {idx} out of bounds.")
|
| 1273 |
+
# Determine which dataset to get an item from based on the index.
|
| 1274 |
+
start_idx = 0
|
| 1275 |
+
dataset_idx = 0
|
| 1276 |
+
for dataset in self._datasets:
|
| 1277 |
+
if idx >= start_idx + dataset.num_frames:
|
| 1278 |
+
start_idx += dataset.num_frames
|
| 1279 |
+
dataset_idx += 1
|
| 1280 |
+
continue
|
| 1281 |
+
break
|
| 1282 |
+
else:
|
| 1283 |
+
raise AssertionError("We expect the loop to break out as long as the index is within bounds.")
|
| 1284 |
+
item = self._datasets[dataset_idx][idx - start_idx]
|
| 1285 |
+
item["dataset_index"] = torch.tensor(dataset_idx)
|
| 1286 |
+
for data_key in self.disabled_features:
|
| 1287 |
+
if data_key in item:
|
| 1288 |
+
del item[data_key]
|
| 1289 |
+
|
| 1290 |
+
return item
|
| 1291 |
+
|
| 1292 |
+
def __repr__(self):
|
| 1293 |
+
return (
|
| 1294 |
+
f"{self.__class__.__name__}(\n"
|
| 1295 |
+
f" Dataset Names: '{self.ds_names}',\n"
|
| 1296 |
+
f" Number of Samples: {self.num_frames},\n"
|
| 1297 |
+
f" Number of Episodes: {self.num_episodes},\n"
|
| 1298 |
+
f" Type: {'video (.mp4)' if self.video else 'image (.png)'},\n"
|
| 1299 |
+
f" Recorded Frames per Second: {self.fps},\n"
|
| 1300 |
+
f" Camera Keys: {self.camera_keys},\n"
|
| 1301 |
+
f" Video Frame Keys: {self.video_frame_keys if self.video else 'N/A'},\n"
|
| 1302 |
+
f" Transformations: {self.image_transforms},\n"
|
| 1303 |
+
f")"
|
| 1304 |
+
)
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/base_processor.py
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from abc import ABC, abstractmethod
|
| 2 |
+
from typing import Dict, Any, Optional, List, Literal
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
import numpy as np
|
| 6 |
+
from copy import deepcopy
|
| 7 |
+
from ..utils.normalizer import LinearNormalizer, NormMode
|
| 8 |
+
from fastwam.utils.pytorch_utils import dict_apply
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class BaseProcessor(ABC):
|
| 12 |
+
def __init__(
|
| 13 |
+
self,
|
| 14 |
+
# keys
|
| 15 |
+
shape_meta: Dict[str, Any],
|
| 16 |
+
num_obs_steps: int,
|
| 17 |
+
num_output_cameras: int,
|
| 18 |
+
action_output_dim: int,
|
| 19 |
+
proprio_output_dim: int,
|
| 20 |
+
|
| 21 |
+
action_state_transforms: Optional[List[Any]],
|
| 22 |
+
|
| 23 |
+
# action & state normalization
|
| 24 |
+
use_stepwise_action_norm: bool,
|
| 25 |
+
norm_default_mode: NormMode,
|
| 26 |
+
norm_exception_mode: Dict[str, Dict[str, NormMode]],
|
| 27 |
+
|
| 28 |
+
action_state_merger,
|
| 29 |
+
|
| 30 |
+
# image transform
|
| 31 |
+
train_transforms: Dict[str, List[Any]] | None,
|
| 32 |
+
val_transforms: Dict[str, List[Any]] | None,
|
| 33 |
+
|
| 34 |
+
# instruction transform
|
| 35 |
+
drop_high_level_prob: float,
|
| 36 |
+
use_zh_instruction: bool,
|
| 37 |
+
|
| 38 |
+
tokenizer: Any
|
| 39 |
+
):
|
| 40 |
+
self.shape_meta = shape_meta
|
| 41 |
+
self.num_obs_steps = num_obs_steps
|
| 42 |
+
self.num_output_cameras = num_output_cameras
|
| 43 |
+
self.action_output_dim = action_output_dim
|
| 44 |
+
self.proprio_output_dim = proprio_output_dim
|
| 45 |
+
|
| 46 |
+
self.drop_high_level_prob = drop_high_level_prob
|
| 47 |
+
self.use_zh_instruction = use_zh_instruction
|
| 48 |
+
|
| 49 |
+
# image
|
| 50 |
+
self.train_transforms = train_transforms
|
| 51 |
+
self.val_transforms = val_transforms
|
| 52 |
+
|
| 53 |
+
self._is_train = None
|
| 54 |
+
|
| 55 |
+
self.action_state_transforms = action_state_transforms
|
| 56 |
+
self.action_state_merger = action_state_merger
|
| 57 |
+
self.action_state_merger.set_shape_meta(self.shape_meta)
|
| 58 |
+
|
| 59 |
+
self.use_stepwise_action_norm = use_stepwise_action_norm
|
| 60 |
+
self.norm_default_mode = norm_default_mode
|
| 61 |
+
self.norm_exception_mode = norm_exception_mode
|
| 62 |
+
self._normalizer = None
|
| 63 |
+
|
| 64 |
+
self.tokenizer = tokenizer
|
| 65 |
+
|
| 66 |
+
@property
|
| 67 |
+
def is_train(self):
|
| 68 |
+
if self._is_train is None:
|
| 69 |
+
raise ValueError("is_train has not been set. Please call train() and eval() first.")
|
| 70 |
+
return self._is_train
|
| 71 |
+
|
| 72 |
+
@property
|
| 73 |
+
def normalizer(self) -> LinearNormalizer:
|
| 74 |
+
if self._normalizer is None:
|
| 75 |
+
raise ValueError("normalizer has not been set. Please call set_normalizer_from_stats() first.")
|
| 76 |
+
return self._normalizer
|
| 77 |
+
|
| 78 |
+
def train(self):
|
| 79 |
+
self._is_train = True
|
| 80 |
+
return self
|
| 81 |
+
|
| 82 |
+
def eval(self):
|
| 83 |
+
self._is_train = False
|
| 84 |
+
return self
|
| 85 |
+
|
| 86 |
+
def set_normalizer_from_stats(self, dataset_stats: Dict[str, Any] = None):
|
| 87 |
+
self._normalizer = LinearNormalizer(
|
| 88 |
+
use_stepwise_action_norm=self.use_stepwise_action_norm,
|
| 89 |
+
shape_meta=self.shape_meta,
|
| 90 |
+
default_mode=self.norm_default_mode,
|
| 91 |
+
exception_mode=self.norm_exception_mode,
|
| 92 |
+
stats=dataset_stats,
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
def augment_instruction(self, data: Dict[str, str] | List[str]) -> List[str]:
|
| 96 |
+
"""
|
| 97 |
+
Args:
|
| 98 |
+
data: Dict[str, str] | List[str], lerobot sample in raw mcap
|
| 99 |
+
|
| 100 |
+
Returns:
|
| 101 |
+
List[str], processed instructions
|
| 102 |
+
"""
|
| 103 |
+
# if single instruction, convert to list
|
| 104 |
+
if "coarse_task" in data:
|
| 105 |
+
high_level_instruction = data["coarse_task"]
|
| 106 |
+
else:
|
| 107 |
+
high_level_instruction = ""
|
| 108 |
+
if "task" not in data:
|
| 109 |
+
return f"[high] {high_level_instruction}"
|
| 110 |
+
|
| 111 |
+
low_level_instruction = data["task"]
|
| 112 |
+
# Galaxea lerobot use @ to split Chinese and English instruction
|
| 113 |
+
if "@" in low_level_instruction:
|
| 114 |
+
zh, eng = low_level_instruction.split("@")
|
| 115 |
+
low_level_instruction = zh if self.use_zh_instruction else eng
|
| 116 |
+
|
| 117 |
+
if np.random.rand() < self.drop_high_level_prob:
|
| 118 |
+
instruction = f"[Low]: {low_level_instruction}"
|
| 119 |
+
else:
|
| 120 |
+
instruction = f"[High]: {high_level_instruction}, [Low]: {low_level_instruction}"
|
| 121 |
+
|
| 122 |
+
return instruction
|
| 123 |
+
|
| 124 |
+
def action_state_transform(self, batch):
|
| 125 |
+
if "action" in batch:
|
| 126 |
+
for meta in self.shape_meta["action"]:
|
| 127 |
+
k, meta_shape = meta["key"], meta["raw_shape"]
|
| 128 |
+
actual_shape = batch["action"][k].shape[-1]
|
| 129 |
+
assert actual_shape == meta_shape, \
|
| 130 |
+
f"Action key {k} actual raw shape {actual_shape} mismatch with meta raw shape {meta_shape}."
|
| 131 |
+
|
| 132 |
+
for meta in self.shape_meta["state"]:
|
| 133 |
+
k, meta_shape = meta["key"], meta["raw_shape"]
|
| 134 |
+
actual_shape = batch["state"][k].shape[-1]
|
| 135 |
+
assert actual_shape == meta_shape, \
|
| 136 |
+
f"State key {k} actual raw shape {actual_shape} mismatch with meta raw shape{meta_shape}."
|
| 137 |
+
|
| 138 |
+
if self.action_state_transforms is not None:
|
| 139 |
+
for trans in self.action_state_transforms:
|
| 140 |
+
batch = trans.forward(batch)
|
| 141 |
+
|
| 142 |
+
if "action" in batch:
|
| 143 |
+
for meta in self.shape_meta["action"]:
|
| 144 |
+
k, meta_shape = meta["key"], meta["shape"]
|
| 145 |
+
actual_shape = batch["action"][k].shape[-1]
|
| 146 |
+
assert actual_shape == meta_shape, \
|
| 147 |
+
f"Action key {k} actual transformed shape {actual_shape} mismatch with meta shape {meta_shape}."
|
| 148 |
+
|
| 149 |
+
for meta in self.shape_meta["state"]:
|
| 150 |
+
k, meta_shape = meta["key"], meta["shape"]
|
| 151 |
+
actual_shape = batch["state"][k].shape[-1]
|
| 152 |
+
assert actual_shape == meta_shape, \
|
| 153 |
+
f"State key {k} actual transformed shape {actual_shape} mismatch with meta raw shape {meta_shape}."
|
| 154 |
+
|
| 155 |
+
return batch
|
| 156 |
+
|
| 157 |
+
def preprocess(self, data: Dict[str, Any]) -> Dict[str, Any]:
|
| 158 |
+
"""
|
| 159 |
+
Preprocess the data for the policy model.
|
| 160 |
+
|
| 161 |
+
Args:
|
| 162 |
+
Data: Dict[str, Any], lerobot sample in raw mcap obtained from dataset __getitem__:
|
| 163 |
+
- "action": Optional, Dict[str, torch.Tensor] -> [action_horizon, action_dim]
|
| 164 |
+
- "state": Dict[str, torch.Tensor] -> [num_obs_steps, state_dim]
|
| 165 |
+
- "images": Dict[str, torch.Tensor] -> [num_obs_steps, C, H, W]
|
| 166 |
+
- "action_is_pad": Optional, torch.Tensor -> [action_horizon,]
|
| 167 |
+
- "state_is_pad": torch.Tensor -> [num_obs_steps,]
|
| 168 |
+
- "image_is_pad": torch.Tensor -> [num_obs_steps,]
|
| 169 |
+
- "idx": int, sample index
|
| 170 |
+
|
| 171 |
+
Returns:
|
| 172 |
+
Sample: Dict[str, Any], which can collated:
|
| 173 |
+
- "input_ids": torch.Tensor -> [max_image_text_tokens,]
|
| 174 |
+
- "attention_mask": torch.Tensor -> [max_image_text_tokens,]
|
| 175 |
+
- "pixel_values": torch.Tensor -> [num_input_cameras, C, H, W]
|
| 176 |
+
- "image_is_pad": torch.Tensor -> [num_obs_steps,]
|
| 177 |
+
- "proprio": torch.Tensor -> [num_obs_steps, proprio_dim]
|
| 178 |
+
- "state_is_pad": torch.Tensor -> [num_obs_steps,]
|
| 179 |
+
- "action": Optional, torch.Tensor -> [action_horizon, action_dim]
|
| 180 |
+
- "action_is_pad": Optional, torch.Tensor -> [action_horizon,]
|
| 181 |
+
- "gt_action: Optional, deepcopy of input action for open loop eval, which is left untouched
|
| 182 |
+
- "idx": int, sample index
|
| 183 |
+
"""
|
| 184 |
+
sample = {}
|
| 185 |
+
# 1. instruction
|
| 186 |
+
sample["instruction"] = self.augment_instruction(data)
|
| 187 |
+
sample["image_is_pad"] = data["image_is_pad"]
|
| 188 |
+
|
| 189 |
+
# 2. image
|
| 190 |
+
processed_images = []
|
| 191 |
+
for meta in self.shape_meta["images"]:
|
| 192 |
+
key, shape = meta["key"], meta["shape"]
|
| 193 |
+
image = data["images"][key] # [num_obs_steps, C, H, W]
|
| 194 |
+
assert image.ndim == 4, f"Expected 4 dimensions (num_obs_steps, C, H, W), got shape {image.shape}"
|
| 195 |
+
|
| 196 |
+
# Apply transforms efficiently on the merged batch
|
| 197 |
+
transforms = self.train_transforms if self.is_train else self.val_transforms
|
| 198 |
+
for trans in transforms[key]:
|
| 199 |
+
image = trans(image)
|
| 200 |
+
|
| 201 |
+
meta_shape = [self.num_obs_steps] + shape
|
| 202 |
+
assert image.shape == meta_shape, \
|
| 203 |
+
f"Expected shape {meta_shape}, got {image.shape} after transforms for key {key}"
|
| 204 |
+
|
| 205 |
+
processed_images.append(image)
|
| 206 |
+
|
| 207 |
+
pixel_values = torch.cat(processed_images, dim=0) # [num_input_cameras, C, H, W]
|
| 208 |
+
if self.num_output_cameras > pixel_values.shape[0]:
|
| 209 |
+
out = torch.zeros((self.num_output_cameras,) + pixel_values.shape[1:], device=pixel_values.device, dtype=pixel_values.dtype)
|
| 210 |
+
out[0: pixel_values.shape[0]] = pixel_values
|
| 211 |
+
sample["pixel_values"] = out
|
| 212 |
+
else:
|
| 213 |
+
sample["pixel_values"] = pixel_values
|
| 214 |
+
|
| 215 |
+
# Copy action before transform for open-loop evaluation,
|
| 216 |
+
# disabled for training dataset as it may cause collating key problem.
|
| 217 |
+
if not self.is_train and "action" in data:
|
| 218 |
+
sample["gt_action"] = deepcopy(data["action"])
|
| 219 |
+
|
| 220 |
+
# 3. action & state
|
| 221 |
+
data = self.action_state_transform(data)
|
| 222 |
+
data = self.normalizer.forward(data)
|
| 223 |
+
data = self.action_state_merger.forward(data)
|
| 224 |
+
|
| 225 |
+
if "action" in data:
|
| 226 |
+
sample["action"] = data["action"] # [action_horizon, action_dim]
|
| 227 |
+
sample["action_is_pad"] = data["action_is_pad"] # [action_horizon,]
|
| 228 |
+
sample["action_dim_is_pad"] = data["action_dim_is_pad"] # [action_dim,]
|
| 229 |
+
assert sample["action"].shape[-1] == self.action_output_dim
|
| 230 |
+
|
| 231 |
+
# TODO: rename all "state" into "proprio"
|
| 232 |
+
sample["proprio"] = data["state"] # [num_obs_steps, proprio_dim]
|
| 233 |
+
sample["proprio_is_pad"] = data["state_is_pad"] # [num_obs_steps,]
|
| 234 |
+
sample["proprio_dim_is_pad"] = data["state_dim_is_pad"] # [proprio_dim,]
|
| 235 |
+
assert sample["proprio"].shape[-1] == self.proprio_output_dim
|
| 236 |
+
|
| 237 |
+
sample["idx"] = data["idx"]
|
| 238 |
+
|
| 239 |
+
sample = self.tokenizer(sample)
|
| 240 |
+
|
| 241 |
+
return sample
|
| 242 |
+
|
| 243 |
+
def postprocess(self, data: Dict[str, Any]) -> Dict[str, Any]:
|
| 244 |
+
"""
|
| 245 |
+
Postprocess the data for the policy model.
|
| 246 |
+
|
| 247 |
+
Args:
|
| 248 |
+
data: Dict[str, Any], lerobot sample in raw mcap
|
| 249 |
+
|
| 250 |
+
Returns:
|
| 251 |
+
data: Dict[str, Any], processed data including unnormalized action
|
| 252 |
+
"""
|
| 253 |
+
assert "action" in data, "Action is required in postprocess"
|
| 254 |
+
data["state"] = data.pop("proprio")
|
| 255 |
+
data = self.action_state_merger.backward(data)
|
| 256 |
+
data = self.normalizer.backward(data)
|
| 257 |
+
if self.action_state_transforms is not None:
|
| 258 |
+
for trans in reversed(self.action_state_transforms):
|
| 259 |
+
data = trans.backward(data)
|
| 260 |
+
|
| 261 |
+
start_obs_step = self.num_obs_steps - 1
|
| 262 |
+
data["action"] = dict_apply(data["action"], lambda x: x[:, start_obs_step:, :])
|
| 263 |
+
return data
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/processors/fastwam_processor.py
ADDED
|
@@ -0,0 +1,304 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from abc import ABC, abstractmethod
|
| 2 |
+
from typing import Dict, Any, Optional, List, Literal
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
import numpy as np
|
| 6 |
+
from copy import deepcopy
|
| 7 |
+
from ..utils.normalizer import LinearNormalizer, NormMode
|
| 8 |
+
from fastwam.utils.pytorch_utils import dict_apply
|
| 9 |
+
from fastwam.utils.logging_config import get_logger
|
| 10 |
+
from .base_processor import BaseProcessor
|
| 11 |
+
|
| 12 |
+
logger = get_logger(__name__)
|
| 13 |
+
|
| 14 |
+
class FastWAMProcessor(BaseProcessor):
|
| 15 |
+
def __init__(
|
| 16 |
+
self,
|
| 17 |
+
# keys
|
| 18 |
+
shape_meta: Dict[str, Any],
|
| 19 |
+
num_obs_steps: int,
|
| 20 |
+
num_output_cameras: int,
|
| 21 |
+
action_output_dim: int,
|
| 22 |
+
proprio_output_dim: int,
|
| 23 |
+
|
| 24 |
+
action_state_transforms: Optional[List[Any]],
|
| 25 |
+
|
| 26 |
+
# action & state normalization
|
| 27 |
+
use_stepwise_action_norm: bool,
|
| 28 |
+
norm_default_mode: NormMode,
|
| 29 |
+
norm_exception_mode: Dict[str, Dict[str, NormMode]],
|
| 30 |
+
|
| 31 |
+
action_state_merger,
|
| 32 |
+
|
| 33 |
+
# image transform
|
| 34 |
+
train_transforms: Dict[str, List[Any]] | None,
|
| 35 |
+
val_transforms: Dict[str, List[Any]] | None,
|
| 36 |
+
|
| 37 |
+
# instruction transform
|
| 38 |
+
drop_high_level_prob: float = 1.0,
|
| 39 |
+
use_zh_instruction: bool = False,
|
| 40 |
+
|
| 41 |
+
tokenizer: Optional[Any] = None,
|
| 42 |
+
delta_action_dim_mask: Optional[Dict[str, List[bool]]] = None,
|
| 43 |
+
):
|
| 44 |
+
self.shape_meta = shape_meta
|
| 45 |
+
self.num_obs_steps = num_obs_steps
|
| 46 |
+
self.num_output_cameras = num_output_cameras
|
| 47 |
+
self.action_output_dim = action_output_dim
|
| 48 |
+
self.proprio_output_dim = proprio_output_dim
|
| 49 |
+
|
| 50 |
+
self.drop_high_level_prob = drop_high_level_prob
|
| 51 |
+
self.use_zh_instruction = use_zh_instruction
|
| 52 |
+
|
| 53 |
+
# image
|
| 54 |
+
self.train_transforms = train_transforms
|
| 55 |
+
self.val_transforms = val_transforms
|
| 56 |
+
|
| 57 |
+
self._is_train = None
|
| 58 |
+
|
| 59 |
+
self.action_state_transforms = action_state_transforms
|
| 60 |
+
self.action_state_merger = action_state_merger
|
| 61 |
+
self.action_state_merger.set_shape_meta(self.shape_meta)
|
| 62 |
+
|
| 63 |
+
self.use_stepwise_action_norm = use_stepwise_action_norm
|
| 64 |
+
self.norm_default_mode = norm_default_mode
|
| 65 |
+
self.norm_exception_mode = norm_exception_mode
|
| 66 |
+
self._normalizer = None
|
| 67 |
+
|
| 68 |
+
self.tokenizer = tokenizer
|
| 69 |
+
if delta_action_dim_mask is None:
|
| 70 |
+
self.delta_action_dim_mask = None
|
| 71 |
+
else:
|
| 72 |
+
action_meta = self.shape_meta["action"]
|
| 73 |
+
expected_keys = [m["key"] for m in action_meta]
|
| 74 |
+
provided_keys = list(delta_action_dim_mask.keys())
|
| 75 |
+
if set(provided_keys) != set(expected_keys):
|
| 76 |
+
raise ValueError(
|
| 77 |
+
f"`delta_action_dim_mask` keys mismatch. Expected {expected_keys}, got {provided_keys}."
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
self.delta_action_dim_mask = {}
|
| 81 |
+
for meta in action_meta:
|
| 82 |
+
key = meta["key"]
|
| 83 |
+
expected_dim = meta["shape"]
|
| 84 |
+
mask = delta_action_dim_mask[key]
|
| 85 |
+
if len(mask) != expected_dim:
|
| 86 |
+
raise ValueError(
|
| 87 |
+
f"`delta_action_dim_mask[{key}]` length must be {expected_dim}, got {len(mask)}."
|
| 88 |
+
)
|
| 89 |
+
self.delta_action_dim_mask[key] = torch.as_tensor(mask, dtype=torch.bool)
|
| 90 |
+
|
| 91 |
+
@property
|
| 92 |
+
def is_train(self):
|
| 93 |
+
if self._is_train is None:
|
| 94 |
+
raise ValueError("is_train has not been set. Please call train() and eval() first.")
|
| 95 |
+
return self._is_train
|
| 96 |
+
|
| 97 |
+
@property
|
| 98 |
+
def normalizer(self) -> LinearNormalizer:
|
| 99 |
+
if self._normalizer is None:
|
| 100 |
+
raise ValueError("normalizer has not been set. Please call set_normalizer_from_stats() first.")
|
| 101 |
+
return self._normalizer
|
| 102 |
+
|
| 103 |
+
def train(self):
|
| 104 |
+
self._is_train = True
|
| 105 |
+
return self
|
| 106 |
+
|
| 107 |
+
def eval(self):
|
| 108 |
+
self._is_train = False
|
| 109 |
+
return self
|
| 110 |
+
|
| 111 |
+
def set_normalizer_from_stats(self, dataset_stats: Dict[str, Any] = None):
|
| 112 |
+
self._normalizer = LinearNormalizer(
|
| 113 |
+
use_stepwise_action_norm=self.use_stepwise_action_norm,
|
| 114 |
+
shape_meta=self.shape_meta,
|
| 115 |
+
default_mode=self.norm_default_mode,
|
| 116 |
+
exception_mode=self.norm_exception_mode,
|
| 117 |
+
stats=dataset_stats,
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
def augment_instruction(self, data: Dict[str, str] | List[str]) -> List[str]:
|
| 121 |
+
"""
|
| 122 |
+
Args:
|
| 123 |
+
data: Dict[str, str] | List[str], lerobot sample in raw mcap
|
| 124 |
+
|
| 125 |
+
Returns:
|
| 126 |
+
List[str], processed instructions
|
| 127 |
+
"""
|
| 128 |
+
# if single instruction, convert to list
|
| 129 |
+
if "coarse_task" in data:
|
| 130 |
+
high_level_instruction = data["coarse_task"]
|
| 131 |
+
else:
|
| 132 |
+
high_level_instruction = ""
|
| 133 |
+
if "task" not in data:
|
| 134 |
+
return f"[high] {high_level_instruction}"
|
| 135 |
+
|
| 136 |
+
low_level_instruction = data["task"]
|
| 137 |
+
# Galaxea lerobot use @ to split Chinese and English instruction
|
| 138 |
+
if "@" in low_level_instruction:
|
| 139 |
+
zh, eng = low_level_instruction.split("@")
|
| 140 |
+
low_level_instruction = zh if self.use_zh_instruction else eng
|
| 141 |
+
|
| 142 |
+
if np.random.rand() < self.drop_high_level_prob:
|
| 143 |
+
instruction = f"{low_level_instruction}"
|
| 144 |
+
else:
|
| 145 |
+
instruction = f"[High]: {high_level_instruction}, [Low]: {low_level_instruction}"
|
| 146 |
+
|
| 147 |
+
return instruction
|
| 148 |
+
|
| 149 |
+
def action_state_transform(self, batch):
|
| 150 |
+
if "action" in batch:
|
| 151 |
+
for meta in self.shape_meta["action"]:
|
| 152 |
+
k, meta_shape = meta["key"], meta["raw_shape"]
|
| 153 |
+
actual_shape = batch["action"][k].shape[-1]
|
| 154 |
+
assert actual_shape == meta_shape, \
|
| 155 |
+
f"Action key {k} actual raw shape {actual_shape} mismatch with meta raw shape {meta_shape}."
|
| 156 |
+
|
| 157 |
+
for meta in self.shape_meta["state"]:
|
| 158 |
+
k, meta_shape = meta["key"], meta["raw_shape"]
|
| 159 |
+
actual_shape = batch["state"][k].shape[-1]
|
| 160 |
+
assert actual_shape == meta_shape, \
|
| 161 |
+
f"State key {k} actual raw shape {actual_shape} mismatch with meta raw shape {meta_shape}."
|
| 162 |
+
|
| 163 |
+
if self.action_state_transforms is not None:
|
| 164 |
+
for trans in self.action_state_transforms:
|
| 165 |
+
batch = trans.forward(batch)
|
| 166 |
+
|
| 167 |
+
if "action" in batch:
|
| 168 |
+
for meta in self.shape_meta["action"]:
|
| 169 |
+
k, meta_shape = meta["key"], meta["shape"]
|
| 170 |
+
actual_shape = batch["action"][k].shape[-1]
|
| 171 |
+
assert actual_shape == meta_shape, \
|
| 172 |
+
f"Action key {k} actual transformed shape {actual_shape} mismatch with meta shape {meta_shape}."
|
| 173 |
+
|
| 174 |
+
for meta in self.shape_meta["state"]:
|
| 175 |
+
k, meta_shape = meta["key"], meta["shape"]
|
| 176 |
+
actual_shape = batch["state"][k].shape[-1]
|
| 177 |
+
assert actual_shape == meta_shape, \
|
| 178 |
+
f"State key {k} actual transformed shape {actual_shape} mismatch with meta raw shape {meta_shape}."
|
| 179 |
+
|
| 180 |
+
return batch
|
| 181 |
+
|
| 182 |
+
def preprocess(self, data: Dict[str, Any]) -> Dict[str, Any]:
|
| 183 |
+
"""
|
| 184 |
+
Preprocess the data for the policy model.
|
| 185 |
+
|
| 186 |
+
Args:
|
| 187 |
+
Data: Dict[str, Any], lerobot sample in raw mcap obtained from dataset __getitem__:
|
| 188 |
+
- "action": Optional, Dict[str, torch.Tensor] -> [action_horizon, action_dim]
|
| 189 |
+
- "state": Dict[str, torch.Tensor] -> [num_obs_steps, state_dim]
|
| 190 |
+
- "images": Dict[str, torch.Tensor] -> [num_obs_steps, C, H, W]
|
| 191 |
+
- "action_is_pad": Optional, torch.Tensor -> [action_horizon,]
|
| 192 |
+
- "state_is_pad": torch.Tensor -> [num_obs_steps,]
|
| 193 |
+
- "image_is_pad": torch.Tensor -> [num_obs_steps,]
|
| 194 |
+
- "idx": int, sample index
|
| 195 |
+
|
| 196 |
+
Returns:
|
| 197 |
+
Sample: Dict[str, Any], which can collated:
|
| 198 |
+
- "input_ids": torch.Tensor -> [max_image_text_tokens,]
|
| 199 |
+
- "attention_mask": torch.Tensor -> [max_image_text_tokens,]
|
| 200 |
+
- "pixel_values": torch.Tensor -> [num_input_cameras, C, H, W]
|
| 201 |
+
- "image_is_pad": torch.Tensor -> [num_obs_steps,]
|
| 202 |
+
- "proprio": torch.Tensor -> [num_obs_steps, proprio_dim]
|
| 203 |
+
- "state_is_pad": torch.Tensor -> [num_obs_steps,]
|
| 204 |
+
- "action": Optional, torch.Tensor -> [action_horizon, action_dim]
|
| 205 |
+
- "action_is_pad": Optional, torch.Tensor -> [action_horizon,]
|
| 206 |
+
- "gt_action: Optional, deepcopy of input action for open loop eval, which is left untouched
|
| 207 |
+
- "idx": int, sample index
|
| 208 |
+
"""
|
| 209 |
+
sample = {}
|
| 210 |
+
# 1. instruction
|
| 211 |
+
sample["instruction"] = self.augment_instruction(data)
|
| 212 |
+
sample["image_is_pad"] = data["image_is_pad"]
|
| 213 |
+
|
| 214 |
+
# 2. image
|
| 215 |
+
processed_images = []
|
| 216 |
+
for meta in self.shape_meta["images"]:
|
| 217 |
+
key, shape = meta["key"], meta["shape"]
|
| 218 |
+
image = data["images"][key] # [num_obs_steps, C, H, W]
|
| 219 |
+
assert image.ndim == 4, f"Expected 4 dimensions (num_obs_steps, C, H, W), got shape {image.shape}"
|
| 220 |
+
|
| 221 |
+
# Apply transforms efficiently on the merged batch
|
| 222 |
+
transforms = self.train_transforms if self.is_train else self.val_transforms
|
| 223 |
+
current_transforms = transforms[key] if isinstance(transforms, dict) else transforms
|
| 224 |
+
for trans in current_transforms:
|
| 225 |
+
image = trans(image)
|
| 226 |
+
|
| 227 |
+
meta_shape = [self.num_obs_steps] + shape
|
| 228 |
+
assert image.shape == meta_shape, \
|
| 229 |
+
f"Expected shape {meta_shape}, got {image.shape} after transforms for key {key}"
|
| 230 |
+
|
| 231 |
+
processed_images.append(image)
|
| 232 |
+
pixel_values = torch.stack(processed_images, dim=0) # [num_input_cameras, T, C, H, W]
|
| 233 |
+
|
| 234 |
+
if self.num_output_cameras > pixel_values.shape[0]:
|
| 235 |
+
out = torch.zeros((self.num_output_cameras,) + pixel_values.shape[1:], device=pixel_values.device, dtype=pixel_values.dtype)
|
| 236 |
+
out[0: pixel_values.shape[0]] = pixel_values
|
| 237 |
+
sample["pixel_values"] = out
|
| 238 |
+
elif self.num_output_cameras < pixel_values.shape[0]:
|
| 239 |
+
logger.warning(f"num_output_cameras {self.num_output_cameras} is less than the number of cameras in data {pixel_values.shape[0]}, "
|
| 240 |
+
f"truncating the input to the first {self.num_output_cameras} cameras.")
|
| 241 |
+
sample["pixel_values"] = pixel_values[:self.num_output_cameras]
|
| 242 |
+
else:
|
| 243 |
+
sample["pixel_values"] = pixel_values
|
| 244 |
+
|
| 245 |
+
# Copy action before transform for open-loop evaluation,
|
| 246 |
+
# disabled for training dataset as it may cause collating key problem.
|
| 247 |
+
if not self.is_train and "action" in data:
|
| 248 |
+
sample["gt_action"] = deepcopy(data["action"])
|
| 249 |
+
|
| 250 |
+
# 3. action & state
|
| 251 |
+
if "action" in data and self.delta_action_dim_mask is not None:
|
| 252 |
+
action_is_pad = torch.as_tensor(data["action_is_pad"], dtype=torch.bool)
|
| 253 |
+
if bool(action_is_pad.any().item()):
|
| 254 |
+
for key, dim_mask in self.delta_action_dim_mask.items():
|
| 255 |
+
cur_action = data["action"][key]
|
| 256 |
+
cur_action_is_pad = action_is_pad.to(device=cur_action.device)
|
| 257 |
+
cur_dim_mask = dim_mask.to(device=cur_action.device)
|
| 258 |
+
pad_delta_mask = cur_action_is_pad.unsqueeze(1) & cur_dim_mask.unsqueeze(0)
|
| 259 |
+
cur_action[pad_delta_mask] = 0.0
|
| 260 |
+
data = self.action_state_transform(data)
|
| 261 |
+
data = self.normalizer.forward(data)
|
| 262 |
+
data = self.action_state_merger.forward(data)
|
| 263 |
+
|
| 264 |
+
if "action" in data:
|
| 265 |
+
sample["action"] = data["action"] # [action_horizon, action_dim]
|
| 266 |
+
sample["action_is_pad"] = data["action_is_pad"] # [action_horizon,]
|
| 267 |
+
sample["action_dim_is_pad"] = data["action_dim_is_pad"] # [action_dim,]
|
| 268 |
+
assert sample["action"].shape[-1] == self.action_output_dim
|
| 269 |
+
# sample["action"][sample["action_is_pad"], :-1] = 0.0 # NOTE: we assume use delta_eef_pose + gripper, so pad action is 0
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
# TODO: rename all "state" into "proprio"
|
| 273 |
+
sample["proprio"] = data["state"] # [num_obs_steps, proprio_dim]
|
| 274 |
+
sample["proprio_is_pad"] = data["state_is_pad"] # [num_obs_steps,]
|
| 275 |
+
sample["proprio_dim_is_pad"] = data["state_dim_is_pad"] # [proprio_dim,]
|
| 276 |
+
assert sample["proprio"].shape[-1] == self.proprio_output_dim
|
| 277 |
+
|
| 278 |
+
sample["idx"] = data["idx"]
|
| 279 |
+
|
| 280 |
+
# sample = self.tokenizer(sample)
|
| 281 |
+
|
| 282 |
+
return sample
|
| 283 |
+
|
| 284 |
+
def postprocess(self, data: Dict[str, Any]) -> Dict[str, Any]:
|
| 285 |
+
"""
|
| 286 |
+
Postprocess the data for the policy model.
|
| 287 |
+
|
| 288 |
+
Args:
|
| 289 |
+
data: Dict[str, Any], lerobot sample in raw mcap
|
| 290 |
+
|
| 291 |
+
Returns:
|
| 292 |
+
data: Dict[str, Any], processed data including unnormalized action
|
| 293 |
+
"""
|
| 294 |
+
assert "action" in data, "Action is required in postprocess"
|
| 295 |
+
data["state"] = data.pop("proprio")
|
| 296 |
+
data = self.action_state_merger.backward(data)
|
| 297 |
+
data = self.normalizer.backward(data)
|
| 298 |
+
if self.action_state_transforms is not None:
|
| 299 |
+
for trans in reversed(self.action_state_transforms):
|
| 300 |
+
data = trans.backward(data)
|
| 301 |
+
|
| 302 |
+
start_obs_step = self.num_obs_steps - 1
|
| 303 |
+
data["action"] = dict_apply(data["action"], lambda x: x[:, start_obs_step:, :])
|
| 304 |
+
return data
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/robot_video_dataset.py
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import hashlib
|
| 2 |
+
import os
|
| 3 |
+
from typing import Optional
|
| 4 |
+
import time
|
| 5 |
+
import numpy as np
|
| 6 |
+
import traceback
|
| 7 |
+
import torch
|
| 8 |
+
import torchvision.transforms.functional as transforms_F
|
| 9 |
+
from contextlib import contextmanager
|
| 10 |
+
|
| 11 |
+
from omegaconf import DictConfig, OmegaConf
|
| 12 |
+
|
| 13 |
+
from hydra.utils import instantiate
|
| 14 |
+
from .base_lerobot_dataset import BaseLerobotDataset
|
| 15 |
+
from .utils.normalizer import save_dataset_stats_to_json, load_dataset_stats_from_json
|
| 16 |
+
from ..dataset_utils import ResizeSmallestSideAspectPreserving, CenterCrop, Normalize
|
| 17 |
+
from fastwam.utils.logging_config import get_logger
|
| 18 |
+
from fastwam.utils import misc, pytorch_utils
|
| 19 |
+
from accelerate import PartialState
|
| 20 |
+
logger = get_logger(__name__)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
DEFAULT_PROMPT = "A video recorded from a robot's point of view executing the following instruction: {task}"
|
| 24 |
+
|
| 25 |
+
class RobotVideoDataset(torch.utils.data.Dataset):
|
| 26 |
+
def __init__(
|
| 27 |
+
self,
|
| 28 |
+
dataset_dirs,
|
| 29 |
+
shape_meta,
|
| 30 |
+
num_frames=33,
|
| 31 |
+
video_size=[384, 640],
|
| 32 |
+
camera_key=None,
|
| 33 |
+
processor=None,
|
| 34 |
+
text_embedding_cache_dir=None,
|
| 35 |
+
context_len=128,
|
| 36 |
+
pretrained_norm_stats=None,
|
| 37 |
+
val_set_proportion=0.05,
|
| 38 |
+
is_training_set=False,
|
| 39 |
+
global_sample_stride=1,
|
| 40 |
+
action_video_freq_ratio: int = 1,
|
| 41 |
+
skip_padding_as_possible: bool = False,
|
| 42 |
+
max_padding_retry: int = 3,
|
| 43 |
+
concat_multi_camera: str = "horizontal", # "horizontal", "vertical", "robotwin", or None
|
| 44 |
+
override_instruction: Optional[str] = None, # whether to hardcode a specific instruction for all samples, for debugging
|
| 45 |
+
):
|
| 46 |
+
self.lerobot_dataset = BaseLerobotDataset(
|
| 47 |
+
dataset_dirs=dataset_dirs,
|
| 48 |
+
shape_meta=OmegaConf.to_container(shape_meta, resolve=True),
|
| 49 |
+
obs_size=num_frames,
|
| 50 |
+
action_size=num_frames - 1,
|
| 51 |
+
val_set_proportion=val_set_proportion,
|
| 52 |
+
is_training_set=is_training_set,
|
| 53 |
+
global_sample_stride=global_sample_stride,
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
self.num_frames = num_frames
|
| 57 |
+
self.action_video_freq_ratio = action_video_freq_ratio
|
| 58 |
+
|
| 59 |
+
assert (num_frames - 1) % self.action_video_freq_ratio == 0, \
|
| 60 |
+
f"num_frames-1 must be divisible by action_video_freq_ratio, got {num_frames - 1} and {self.action_video_freq_ratio}"
|
| 61 |
+
assert ((num_frames - 1) // self.action_video_freq_ratio) % 4 == 0, \
|
| 62 |
+
f"video frames must be divisible by 4 for tokenization, got {(num_frames - 1) // self.action_video_freq_ratio}"
|
| 63 |
+
self.video_sample_indices = list(range(0, num_frames, self.action_video_freq_ratio))
|
| 64 |
+
|
| 65 |
+
self.camera_key = camera_key
|
| 66 |
+
self.lerobot_dataset._set_return_images(True)
|
| 67 |
+
|
| 68 |
+
self.video_size = video_size
|
| 69 |
+
self.text_embedding_cache_dir = text_embedding_cache_dir
|
| 70 |
+
self.context_len = context_len
|
| 71 |
+
self.skip_padding_as_possible = skip_padding_as_possible
|
| 72 |
+
self.max_padding_retry = max_padding_retry
|
| 73 |
+
self.concat_multi_camera = concat_multi_camera
|
| 74 |
+
self.override_instruction = override_instruction
|
| 75 |
+
|
| 76 |
+
self.resize_transform = ResizeSmallestSideAspectPreserving(
|
| 77 |
+
args={"img_w": self.video_size[1], "img_h": self.video_size[0]},
|
| 78 |
+
)
|
| 79 |
+
self.crop_transform = CenterCrop(
|
| 80 |
+
args={"img_w": self.video_size[1], "img_h": self.video_size[0]},
|
| 81 |
+
)
|
| 82 |
+
self.normalize_transform = Normalize(
|
| 83 |
+
args={"mean": 0.5, "std": 0.5},
|
| 84 |
+
)
|
| 85 |
+
if processor is not None:
|
| 86 |
+
if isinstance(processor, DictConfig):
|
| 87 |
+
processor = instantiate(processor)
|
| 88 |
+
if not pretrained_norm_stats:
|
| 89 |
+
if not is_training_set:
|
| 90 |
+
raise ValueError("pretrained_norm_stats must be provided for validation/test sets since we don't want to calculate stats on them.")
|
| 91 |
+
if PartialState().is_main_process:
|
| 92 |
+
logger.info("Calculating dataset stats for normalization...")
|
| 93 |
+
dataset_stats = self.lerobot_dataset.get_dataset_stats(processor)
|
| 94 |
+
work_dir = misc.get_work_dir()
|
| 95 |
+
save_dataset_stats_to_json(dataset_stats, os.path.join(work_dir, "dataset_stats.json"))
|
| 96 |
+
else:
|
| 97 |
+
dataset_stats = None
|
| 98 |
+
if torch.distributed.is_available() and torch.distributed.is_initialized():
|
| 99 |
+
obj_list = [dataset_stats]
|
| 100 |
+
torch.distributed.broadcast_object_list(obj_list, src=0)
|
| 101 |
+
dataset_stats = obj_list[0]
|
| 102 |
+
else:
|
| 103 |
+
dataset_stats = load_dataset_stats_from_json(pretrained_norm_stats)
|
| 104 |
+
logger.info(f"Using dataset stats: {pretrained_norm_stats}")
|
| 105 |
+
if PartialState().is_main_process:
|
| 106 |
+
work_dir = misc.get_work_dir()
|
| 107 |
+
save_dataset_stats_to_json(dataset_stats, os.path.join(work_dir, "dataset_stats.json"))
|
| 108 |
+
|
| 109 |
+
processor.set_normalizer_from_stats(dataset_stats)
|
| 110 |
+
self.lerobot_dataset.set_processor(processor)
|
| 111 |
+
|
| 112 |
+
def __len__(self):
|
| 113 |
+
return len(self.lerobot_dataset)
|
| 114 |
+
|
| 115 |
+
def _get(self, idx):
|
| 116 |
+
sample_idx = idx
|
| 117 |
+
sample = None
|
| 118 |
+
for attempt in range(self.max_padding_retry + 1):
|
| 119 |
+
sample = self.lerobot_dataset[sample_idx]
|
| 120 |
+
|
| 121 |
+
if not self.skip_padding_as_possible:
|
| 122 |
+
break
|
| 123 |
+
|
| 124 |
+
action_is_pad = sample["action_is_pad"]
|
| 125 |
+
image_is_pad = sample["image_is_pad"]
|
| 126 |
+
proprio_is_pad = sample["proprio_is_pad"]
|
| 127 |
+
has_pad = False
|
| 128 |
+
if bool(action_is_pad.any().item()):
|
| 129 |
+
has_pad = True
|
| 130 |
+
if bool(image_is_pad.any().item()):
|
| 131 |
+
has_pad = True
|
| 132 |
+
if bool(proprio_is_pad.any().item()):
|
| 133 |
+
has_pad = True
|
| 134 |
+
|
| 135 |
+
if not has_pad or attempt >= self.max_padding_retry:
|
| 136 |
+
break
|
| 137 |
+
|
| 138 |
+
sample_idx = np.random.randint(len(self.lerobot_dataset))
|
| 139 |
+
|
| 140 |
+
image_is_pad = sample["image_is_pad"]
|
| 141 |
+
|
| 142 |
+
video = sample["pixel_values"] # [T, C, H, W] or [num_cameras, T, C, H, W]
|
| 143 |
+
num_cameras = 1
|
| 144 |
+
if video.ndim == 5:
|
| 145 |
+
video = video[:, self.video_sample_indices, :, :, :] # [num_cameras, T_video, C, H, W]
|
| 146 |
+
num_cameras, T_video, C, H, W = video.shape
|
| 147 |
+
else:
|
| 148 |
+
assert video.ndim == 4, f"Expected video to have shape [T, C, H, W], but got {video.shape}"
|
| 149 |
+
video = video[self.video_sample_indices, :, :, :] # [T_video, C, H, W]
|
| 150 |
+
T_video, C, H, W = video.shape
|
| 151 |
+
image_is_pad = image_is_pad[self.video_sample_indices]
|
| 152 |
+
|
| 153 |
+
video = video.view(num_cameras, T_video, C, H, W) # [num_cameras, T_video, C, H, W]
|
| 154 |
+
if self.concat_multi_camera == "robotwin":
|
| 155 |
+
if num_cameras != 3:
|
| 156 |
+
raise ValueError(
|
| 157 |
+
f"`concat_multi_camera='robotwin'` requires exactly 3 cameras, got {num_cameras}"
|
| 158 |
+
)
|
| 159 |
+
cam_top = transforms_F.resize(
|
| 160 |
+
video[0],
|
| 161 |
+
size=[256, 320],
|
| 162 |
+
interpolation=transforms_F.InterpolationMode.BILINEAR,
|
| 163 |
+
antialias=True,
|
| 164 |
+
) # [T_video, C, 256, 320]
|
| 165 |
+
cam_left = transforms_F.resize(
|
| 166 |
+
video[1],
|
| 167 |
+
size=[128, 160],
|
| 168 |
+
interpolation=transforms_F.InterpolationMode.BILINEAR,
|
| 169 |
+
antialias=True,
|
| 170 |
+
) # [T_video, C, 128, 160]
|
| 171 |
+
cam_right = transforms_F.resize(
|
| 172 |
+
video[2],
|
| 173 |
+
size=[128, 160],
|
| 174 |
+
interpolation=transforms_F.InterpolationMode.BILINEAR,
|
| 175 |
+
antialias=True,
|
| 176 |
+
) # [T_video, C, 128, 160]
|
| 177 |
+
bottom = torch.cat([cam_left, cam_right], dim=-1) # [T_video, C, 128, 320]
|
| 178 |
+
video = torch.cat([cam_top, bottom], dim=-2) # [T_video, C, 384, 320]
|
| 179 |
+
elif num_cameras > 1:
|
| 180 |
+
if self.concat_multi_camera == "horizontal":
|
| 181 |
+
video = torch.cat([video[i] for i in range(num_cameras)], dim=-1) # [T_video, C, H, num_cameras*W]
|
| 182 |
+
elif self.concat_multi_camera == "vertical":
|
| 183 |
+
video = torch.cat([video[i] for i in range(num_cameras)], dim=-2) # [T_video, C, num_cameras*H, W]
|
| 184 |
+
else:
|
| 185 |
+
raise ValueError(
|
| 186 |
+
f"Invalid concat_multi_camera: {self.concat_multi_camera}. "
|
| 187 |
+
"Expected one of: horizontal, vertical, robotwin."
|
| 188 |
+
)
|
| 189 |
+
else:
|
| 190 |
+
video = video.squeeze(0) # [T_video, C, H, W]
|
| 191 |
+
|
| 192 |
+
# final resize and normalization
|
| 193 |
+
video = self.resize_transform(video)
|
| 194 |
+
video = self.crop_transform(video)
|
| 195 |
+
video = self.normalize_transform(video) # [T_video, C, H, W]
|
| 196 |
+
|
| 197 |
+
video = video.permute(1, 0, 2, 3) # [C, T_video, H, W], range [-1, 1]
|
| 198 |
+
|
| 199 |
+
# Proxy (from lerobot):
|
| 200 |
+
# action: [num_frames-1, action_dim] # start from t0, except the last frame
|
| 201 |
+
# proprio: [num_frames, proprio_dim] # start from t0 to the last frame, aligned with video frames
|
| 202 |
+
action = sample["action"] # [T-1, action_dim]
|
| 203 |
+
proprio = sample["proprio"][:-1, :] # [T-1, state_dim], to align with action
|
| 204 |
+
if video.shape[1] <= 1:
|
| 205 |
+
raise ValueError(f"`video` must have at least 2 frames, got shape {tuple(video.shape)}")
|
| 206 |
+
if action.shape[0] % (video.shape[1] - 1) != 0:
|
| 207 |
+
raise ValueError(
|
| 208 |
+
f"`action` horizon must be divisible by `video` transitions, got {action.shape[0]} and {video.shape[1] - 1}"
|
| 209 |
+
)
|
| 210 |
+
|
| 211 |
+
task = sample["instruction"]
|
| 212 |
+
|
| 213 |
+
# FIXME
|
| 214 |
+
if self.override_instruction is not None:
|
| 215 |
+
task = self.override_instruction
|
| 216 |
+
instruction = DEFAULT_PROMPT.format(task=task)
|
| 217 |
+
|
| 218 |
+
context, context_mask = self._get_cached_text_context(instruction)
|
| 219 |
+
# NOTE: to keep consistent with wan2.2's behavior
|
| 220 |
+
context[~context_mask] = 0.0
|
| 221 |
+
context_mask = torch.ones_like(context_mask)
|
| 222 |
+
|
| 223 |
+
data = {
|
| 224 |
+
"video": video,
|
| 225 |
+
"action": action,
|
| 226 |
+
"proprio": proprio,
|
| 227 |
+
"prompt": instruction,
|
| 228 |
+
"context": context,
|
| 229 |
+
"context_mask": context_mask,
|
| 230 |
+
"image_is_pad": image_is_pad,
|
| 231 |
+
"action_is_pad": sample["action_is_pad"],
|
| 232 |
+
"proprio_is_pad": sample["proprio_is_pad"],
|
| 233 |
+
}
|
| 234 |
+
return data
|
| 235 |
+
|
| 236 |
+
def _get_cached_text_context(self, prompt: str):
|
| 237 |
+
if self.text_embedding_cache_dir is None:
|
| 238 |
+
raise ValueError("text_embedding_cache_dir is not set.")
|
| 239 |
+
cache_dir = self.text_embedding_cache_dir
|
| 240 |
+
os.makedirs(cache_dir, exist_ok=True)
|
| 241 |
+
hashed = hashlib.sha256(prompt.encode("utf-8")).hexdigest()
|
| 242 |
+
cache_path = os.path.join(cache_dir, f"{hashed}.t5_len{self.context_len}.wan22ti2v5b.pt")
|
| 243 |
+
if not os.path.exists(cache_path):
|
| 244 |
+
raise FileNotFoundError(
|
| 245 |
+
f"Missing text embedding cache: {cache_path}. "
|
| 246 |
+
"Run scripts/precompute_text_embeds.py first."
|
| 247 |
+
)
|
| 248 |
+
payload = torch.load(cache_path, map_location="cpu")
|
| 249 |
+
context = payload["context"]
|
| 250 |
+
context_mask = payload["mask"].bool()
|
| 251 |
+
if context.ndim != 2:
|
| 252 |
+
raise ValueError(
|
| 253 |
+
f"Cached `context` must be 2D [L, D], got shape {tuple(context.shape)} in {cache_path}"
|
| 254 |
+
)
|
| 255 |
+
if context_mask.ndim != 1:
|
| 256 |
+
raise ValueError(
|
| 257 |
+
f"Cached `mask` must be 1D [L], got shape {tuple(context_mask.shape)} in {cache_path}"
|
| 258 |
+
)
|
| 259 |
+
if context.shape[0] != self.context_len:
|
| 260 |
+
raise ValueError(
|
| 261 |
+
f"Cached context_len mismatch: expected {self.context_len}, got {context.shape[0]} in {cache_path}"
|
| 262 |
+
)
|
| 263 |
+
if context_mask.shape[0] != self.context_len:
|
| 264 |
+
raise ValueError(
|
| 265 |
+
f"Cached mask_len mismatch: expected {self.context_len}, got {context_mask.shape[0]} in {cache_path}"
|
| 266 |
+
)
|
| 267 |
+
|
| 268 |
+
return context, context_mask
|
| 269 |
+
|
| 270 |
+
def __getitem__(self, idx):
|
| 271 |
+
try:
|
| 272 |
+
data = self._get(idx)
|
| 273 |
+
except Exception as e:
|
| 274 |
+
print(f"Error processing sample idx {idx}: {e}. Returning a random sample instead.")
|
| 275 |
+
# trace back
|
| 276 |
+
print(traceback.format_exc())
|
| 277 |
+
random_idx = np.random.randint(len(self))
|
| 278 |
+
data = self._get(random_idx)
|
| 279 |
+
return data
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/action_state_merger.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from torch.nn.functional import pad
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class ConcatLeftAlign:
|
| 8 |
+
def __init__(
|
| 9 |
+
self,
|
| 10 |
+
action_target_dim: int | None = None,
|
| 11 |
+
state_target_dim: int | None = None
|
| 12 |
+
):
|
| 13 |
+
self.action_target_dim = action_target_dim
|
| 14 |
+
self.state_target_dim = state_target_dim
|
| 15 |
+
|
| 16 |
+
def set_shape_meta(self, shape_meta):
|
| 17 |
+
self.action_meta = shape_meta["action"]
|
| 18 |
+
self.state_meta = shape_meta["state"]
|
| 19 |
+
|
| 20 |
+
def forward(self, batch):
|
| 21 |
+
if "action" in batch:
|
| 22 |
+
batch["action"] = self._concat(batch["action"], self.action_meta)
|
| 23 |
+
batch["action"], batch["action_dim_is_pad"] = self._pad(batch["action"], self.action_target_dim)
|
| 24 |
+
|
| 25 |
+
batch["state"] = self._concat(batch["state"], self.state_meta)
|
| 26 |
+
batch["state"], batch["state_dim_is_pad"] = self._pad(batch["state"], self.state_target_dim)
|
| 27 |
+
|
| 28 |
+
return batch
|
| 29 |
+
|
| 30 |
+
def backward(self, batch):
|
| 31 |
+
if self.state_target_dim is not None:
|
| 32 |
+
assert batch["state"].shape[-1] == self.state_target_dim
|
| 33 |
+
batch["state"] = self._crop(batch["state"], self.state_meta)
|
| 34 |
+
batch["state"] = self._split(batch["state"], self.state_meta)
|
| 35 |
+
|
| 36 |
+
if self.action_target_dim is not None:
|
| 37 |
+
assert batch["action"].shape[-1] == self.action_target_dim
|
| 38 |
+
batch["action"] = self._crop(batch["action"], self.action_meta)
|
| 39 |
+
batch["action"] = self._split(batch["action"], self.action_meta)
|
| 40 |
+
|
| 41 |
+
return batch
|
| 42 |
+
|
| 43 |
+
@staticmethod
|
| 44 |
+
def _pad(x: torch.Tensor, dim: int):
|
| 45 |
+
if dim is None:
|
| 46 |
+
dim = x.shape[-1]
|
| 47 |
+
|
| 48 |
+
assert x.ndim == 2 and x.shape[-1] <= dim
|
| 49 |
+
pad_dim = dim - x.shape[-1]
|
| 50 |
+
x_padded = pad(x, (0, pad_dim))
|
| 51 |
+
mask = torch.zeros_like(x[0]).bool()
|
| 52 |
+
mask = pad(mask, (0, pad_dim), value=True)
|
| 53 |
+
return x_padded, mask
|
| 54 |
+
|
| 55 |
+
@staticmethod
|
| 56 |
+
def _crop(x: torch.Tensor, meta: int):
|
| 57 |
+
assert x.ndim == 3
|
| 58 |
+
dim = sum([m["shape"] for m in meta])
|
| 59 |
+
x = x[:, :, :dim]
|
| 60 |
+
return x
|
| 61 |
+
|
| 62 |
+
@staticmethod
|
| 63 |
+
def _concat(x: Dict[str, torch.Tensor], meta: Dict[str, Dict]):
|
| 64 |
+
x = torch.cat([x[m["key"]] for m in meta], dim=-1)
|
| 65 |
+
assert x.ndim == 2
|
| 66 |
+
return x
|
| 67 |
+
|
| 68 |
+
@staticmethod
|
| 69 |
+
def _split(x: torch.Tensor, meta: Dict[str, Dict]):
|
| 70 |
+
assert x.ndim == 3
|
| 71 |
+
y = {}
|
| 72 |
+
idx = 0
|
| 73 |
+
for m in meta:
|
| 74 |
+
key, dim = m["key"], m["shape"]
|
| 75 |
+
y[key] = x[:, :, idx: idx + dim]
|
| 76 |
+
idx += dim
|
| 77 |
+
|
| 78 |
+
return y
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/image.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torchvision.transforms as TF
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class ToTensor(nn.Module):
|
| 7 |
+
def __init__(self):
|
| 8 |
+
super().__init__()
|
| 9 |
+
|
| 10 |
+
def forward(self, x: torch.Tensor):
|
| 11 |
+
assert x.dtype == torch.uint8
|
| 12 |
+
x = x.to(torch.float32) / 255.0
|
| 13 |
+
return x
|
| 14 |
+
|
| 15 |
+
class Pad(nn.Module):
|
| 16 |
+
def __init__(self, padding, fill=0, padding_mode='constant'):
|
| 17 |
+
super().__init__()
|
| 18 |
+
self.padding = padding
|
| 19 |
+
self.fill = fill
|
| 20 |
+
self.padding_mode = padding_mode
|
| 21 |
+
self.pad = TF.Pad(padding=tuple(padding), fill=fill, padding_mode=padding_mode)
|
| 22 |
+
|
| 23 |
+
def forward(self, x: torch.Tensor):
|
| 24 |
+
assert x.ndim == 4, "Can only pad tensor of 4 dims."
|
| 25 |
+
return self.pad(x)
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/misc.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import List
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class WrapStateAngle:
|
| 7 |
+
def __init__(self, keys: List[str]):
|
| 8 |
+
self.keys = keys
|
| 9 |
+
|
| 10 |
+
@staticmethod
|
| 11 |
+
def _wrap(x):
|
| 12 |
+
return torch.atan2(torch.sin(x), torch.cos(x))
|
| 13 |
+
|
| 14 |
+
def forward(self, batch):
|
| 15 |
+
for k in self.keys:
|
| 16 |
+
batch["state"][k] = self._wrap(batch["state"][k])
|
| 17 |
+
return batch
|
| 18 |
+
|
| 19 |
+
def backward(self, batch):
|
| 20 |
+
return batch
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/relative_action.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import List, Dict
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from .rotation import quaternion_to_matrix, matrix_to_quaternion
|
| 5 |
+
|
| 6 |
+
# pose: position and quaternion in x, y, z, i, j, k, r
|
| 7 |
+
# mat: homogeneous transformation matrix in 4×4
|
| 8 |
+
# pos: position in x, y, z
|
| 9 |
+
# quat: quaternion in i, j, k, r
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class RelativePoseTransform:
|
| 13 |
+
def __init__(self, keys: List[str]):
|
| 14 |
+
self.keys = keys
|
| 15 |
+
|
| 16 |
+
def forward(self, batch: Dict):
|
| 17 |
+
# for close-loop eval, "action" may not be in batch
|
| 18 |
+
if "action" not in batch:
|
| 19 |
+
return batch
|
| 20 |
+
|
| 21 |
+
for k in self.keys:
|
| 22 |
+
action = batch["action"][k]
|
| 23 |
+
state = batch["state"][k]
|
| 24 |
+
batch["action"][k] = self._forward(action, state[..., -1:, :])
|
| 25 |
+
|
| 26 |
+
return batch
|
| 27 |
+
|
| 28 |
+
def backward(self, batch: Dict):
|
| 29 |
+
for k in self.keys:
|
| 30 |
+
action = batch["action"][k]
|
| 31 |
+
state = batch["state"][k]
|
| 32 |
+
batch["action"][k] = self._backward(action, state[..., -1:, :])
|
| 33 |
+
|
| 34 |
+
return batch
|
| 35 |
+
|
| 36 |
+
def _forward(self, pose: torch.Tensor, base_pose: torch.Tensor):
|
| 37 |
+
# pose & base_pose: position and quaternion in x, y, z, i, j, k, r
|
| 38 |
+
assert pose.shape[-1] == 7, f"Pose shape must be (..., 7), but got {pose.shape}"
|
| 39 |
+
assert base_pose.shape[-1] == 7, f"Base pose shape must be (..., 7), but got {base_pose.shape}"
|
| 40 |
+
pose_matrix = self._pose_to_matrix(pose)
|
| 41 |
+
base_pose_matrix = self._pose_to_matrix(base_pose)
|
| 42 |
+
pose_matrix = self._absolute_to_relative(pose_matrix, base_pose_matrix)
|
| 43 |
+
pose = self._matrix_to_pose(pose_matrix)
|
| 44 |
+
return pose
|
| 45 |
+
|
| 46 |
+
def _backward(self, pose: torch.Tensor, base_pose: torch.Tensor):
|
| 47 |
+
# pose & base_pose: position and quaternion in x, y, z, i, j, k, r
|
| 48 |
+
assert pose.shape[-1] == 7, f"Pose shape must be (..., 7), but got {pose.shape}"
|
| 49 |
+
assert base_pose.shape[-1] == 7, f"Base pose shape must be (..., 7), but got {base_pose.shape}"
|
| 50 |
+
pose_matrix = self._pose_to_matrix(pose)
|
| 51 |
+
base_pose_matrix = self._pose_to_matrix(base_pose)
|
| 52 |
+
pose_matrix = self._relative_to_absolute(pose_matrix, base_pose_matrix)
|
| 53 |
+
pose = self._matrix_to_pose(pose_matrix)
|
| 54 |
+
return pose
|
| 55 |
+
|
| 56 |
+
@staticmethod
|
| 57 |
+
def _pose_to_matrix(pose: torch.Tensor):
|
| 58 |
+
position = pose[..., 0: 3]
|
| 59 |
+
quaternion = pose[..., [6, 3, 4, 5]] # (i j k r) to (r i j k)
|
| 60 |
+
rotation = quaternion_to_matrix(quaternion)
|
| 61 |
+
matrix = torch.zeros(pose.shape[:-1] + (4, 4), dtype=pose.dtype, device=pose.device)
|
| 62 |
+
matrix[..., 0: 3, 0: 3] = rotation
|
| 63 |
+
matrix[..., 0: 3, 3] = position
|
| 64 |
+
matrix[..., 3, 3] = 1
|
| 65 |
+
return matrix
|
| 66 |
+
|
| 67 |
+
@staticmethod
|
| 68 |
+
def _matrix_to_pose(matrix: torch.Tensor):
|
| 69 |
+
position = matrix[..., 0: 3, 3] / matrix[..., 3, 3][..., None]
|
| 70 |
+
rotation = matrix[..., 0: 3, 0: 3]
|
| 71 |
+
quaternion = matrix_to_quaternion(rotation)
|
| 72 |
+
quaternion = quaternion[..., [1, 2, 3, 0]] # (r i j k) to (i j k r)
|
| 73 |
+
pose = torch.cat([position, quaternion], dim=-1)
|
| 74 |
+
return pose
|
| 75 |
+
@staticmethod
|
| 76 |
+
def _absolute_to_relative(pose_matrix: torch.Tensor, base_pose_matrix: torch.Tensor):
|
| 77 |
+
return torch.linalg.inv(base_pose_matrix) @ pose_matrix
|
| 78 |
+
|
| 79 |
+
@staticmethod
|
| 80 |
+
def _relative_to_absolute(pose_matrix: torch.Tensor, base_pose_matrix: torch.Tensor):
|
| 81 |
+
return base_pose_matrix @ pose_matrix
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class RelativeJointTransform:
|
| 85 |
+
def __init__(self, keys: List[str]):
|
| 86 |
+
self.keys = keys
|
| 87 |
+
|
| 88 |
+
def forward(self, batch: Dict):
|
| 89 |
+
# for close-loop eval, "action" may not be in batch
|
| 90 |
+
if "action" not in batch:
|
| 91 |
+
return batch
|
| 92 |
+
|
| 93 |
+
for k in self.keys:
|
| 94 |
+
# NOTE: fixed to the first frame
|
| 95 |
+
batch["action"][k] = batch["action"][k] - batch["state"][k][..., :1, :]
|
| 96 |
+
|
| 97 |
+
return batch
|
| 98 |
+
|
| 99 |
+
def backward(self, batch: Dict):
|
| 100 |
+
for k in self.keys:
|
| 101 |
+
# NOTE: fixed to the first frame
|
| 102 |
+
batch["action"][k] = batch["action"][k] + batch["state"][k][..., :1, :]
|
| 103 |
+
|
| 104 |
+
return batch
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/transforms/rotation.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
from typing import Dict, List, Tuple
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
from ..utils.rotation import (
|
| 11 |
+
quaternion_to_matrix,
|
| 12 |
+
matrix_to_quaternion,
|
| 13 |
+
matrix_to_rotation_6d,
|
| 14 |
+
rotation_6d_to_matrix,
|
| 15 |
+
matrix_to_rotation_9d,
|
| 16 |
+
rotation_9d_to_matrix,
|
| 17 |
+
quaternion_to_axis_angle,
|
| 18 |
+
axis_angle_to_quaternion,
|
| 19 |
+
axis_angle_to_matrix,
|
| 20 |
+
matrix_to_axis_angle,
|
| 21 |
+
euler_angles_to_matrix,
|
| 22 |
+
matrix_to_euler_angles,
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
class PoseRotationTransform:
|
| 26 |
+
def __init__(self, rotation_type: str, category_keys: Dict[str, List[str]]):
|
| 27 |
+
self.rotation_type = rotation_type
|
| 28 |
+
self.category_keys = category_keys
|
| 29 |
+
self.src_type, self.dst_type = self._parse_rotation_type(rotation_type)
|
| 30 |
+
|
| 31 |
+
def forward(self, batch):
|
| 32 |
+
for cat, ks in self.category_keys.items():
|
| 33 |
+
if cat == "action" and "action" not in batch:
|
| 34 |
+
continue
|
| 35 |
+
|
| 36 |
+
for k in ks:
|
| 37 |
+
batch[cat][k] = self._forward(batch[cat][k])
|
| 38 |
+
|
| 39 |
+
return batch
|
| 40 |
+
|
| 41 |
+
def backward(self, batch):
|
| 42 |
+
for cat, ks in self.category_keys.items():
|
| 43 |
+
for k in ks:
|
| 44 |
+
batch[cat][k] = self._backward(batch[cat][k])
|
| 45 |
+
|
| 46 |
+
return batch
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _forward(self, pose):
|
| 50 |
+
assert pose.shape[-1] >= 6
|
| 51 |
+
if self.src_type == self.dst_type:
|
| 52 |
+
return pose
|
| 53 |
+
position, rotation, others = self._split_pose(pose, self.src_type)
|
| 54 |
+
matrix = self._rotation_to_matrix(rotation, self.src_type)
|
| 55 |
+
rotation_out = self._matrix_to_rotation(matrix, self.dst_type)
|
| 56 |
+
return torch.cat([position, rotation_out, others], axis=-1)
|
| 57 |
+
|
| 58 |
+
def _backward(self, pose: torch.Tensor):
|
| 59 |
+
if self.src_type == self.dst_type:
|
| 60 |
+
return pose
|
| 61 |
+
position, rotation, others = self._split_pose(pose, self.dst_type)
|
| 62 |
+
matrix = self._rotation_to_matrix(rotation, self.dst_type)
|
| 63 |
+
rotation_out = self._matrix_to_rotation(matrix, self.src_type)
|
| 64 |
+
return torch.cat([position, rotation_out, others], axis=-1)
|
| 65 |
+
|
| 66 |
+
def add_noise(self, pose: torch.Tensor, std_position=0.05, std_angle=0.05):
|
| 67 |
+
position, rotation, others = self._split_pose(pose, self.src_type)
|
| 68 |
+
matrix = self._rotation_to_matrix(rotation, self.src_type)
|
| 69 |
+
axis_angles = matrix_to_axis_angle(matrix)
|
| 70 |
+
position = position + std_position * torch.randn_like(position)
|
| 71 |
+
axis_angles = axis_angles + std_angle * torch.randn_like(axis_angles)
|
| 72 |
+
matrix = axis_angle_to_matrix(axis_angles)
|
| 73 |
+
rotation_out = self._matrix_to_rotation(matrix, self.src_type)
|
| 74 |
+
return torch.cat([position, rotation_out, others], axis=-1)
|
| 75 |
+
|
| 76 |
+
def _parse_rotation_type(self, rotation_type: str) -> Tuple[str, str]:
|
| 77 |
+
if "_to_" in rotation_type:
|
| 78 |
+
src, dst = rotation_type.split("_to_", 1)
|
| 79 |
+
else:
|
| 80 |
+
src, dst = "quaternion", rotation_type
|
| 81 |
+
return src, dst
|
| 82 |
+
|
| 83 |
+
def _split_pose(self, pose: torch.Tensor, rotation_type: str) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 84 |
+
rotation_dim = self._rotation_dim(rotation_type)
|
| 85 |
+
position = pose[..., 0:3]
|
| 86 |
+
rotation = pose[..., 3: 3 + rotation_dim]
|
| 87 |
+
others = pose[..., 3 + rotation_dim:] if pose.shape[-1] > 3 + rotation_dim else torch.empty_like(position[..., :0])
|
| 88 |
+
assert rotation.shape[-1] == rotation_dim, f"Expected {rotation_dim} dims for {rotation_type}"
|
| 89 |
+
return position, rotation, others
|
| 90 |
+
|
| 91 |
+
def _rotation_dim(self, rotation_type: str) -> int:
|
| 92 |
+
if rotation_type == "quaternion":
|
| 93 |
+
return 4
|
| 94 |
+
if rotation_type == "axis_angle":
|
| 95 |
+
return 3
|
| 96 |
+
if rotation_type in ("rotation_6d",):
|
| 97 |
+
return 6
|
| 98 |
+
if rotation_type in ("rotation_9d", "matrix"):
|
| 99 |
+
return 9
|
| 100 |
+
if rotation_type.startswith("euler_"):
|
| 101 |
+
return 3
|
| 102 |
+
raise ValueError(f"Unsupported rotation type '{rotation_type}'")
|
| 103 |
+
|
| 104 |
+
def _rotation_to_matrix(self, rotation: torch.Tensor, rotation_type: str) -> torch.Tensor:
|
| 105 |
+
if rotation_type == "quaternion":
|
| 106 |
+
quaternion = rotation[..., [3, 0, 1, 2]]
|
| 107 |
+
return quaternion_to_matrix(quaternion)
|
| 108 |
+
if rotation_type == "axis_angle":
|
| 109 |
+
return axis_angle_to_matrix(rotation)
|
| 110 |
+
if rotation_type == "rotation_6d":
|
| 111 |
+
return rotation_6d_to_matrix(rotation)
|
| 112 |
+
if rotation_type in ("rotation_9d", "matrix"):
|
| 113 |
+
return rotation_9d_to_matrix(rotation)
|
| 114 |
+
if rotation_type.startswith("euler_"):
|
| 115 |
+
convention = rotation_type.split("_", 1)[1].upper()
|
| 116 |
+
return euler_angles_to_matrix(rotation, convention)
|
| 117 |
+
raise ValueError(f"Unsupported rotation type '{rotation_type}'")
|
| 118 |
+
|
| 119 |
+
def _matrix_to_rotation(self, matrix: torch.Tensor, rotation_type: str) -> torch.Tensor:
|
| 120 |
+
if rotation_type == "quaternion":
|
| 121 |
+
quaternion = matrix_to_quaternion(matrix)
|
| 122 |
+
return quaternion[..., [1, 2, 3, 0]]
|
| 123 |
+
if rotation_type == "axis_angle":
|
| 124 |
+
return matrix_to_axis_angle(matrix)
|
| 125 |
+
if rotation_type == "rotation_6d":
|
| 126 |
+
return matrix_to_rotation_6d(matrix)
|
| 127 |
+
if rotation_type in ("rotation_9d", "matrix"):
|
| 128 |
+
return matrix_to_rotation_9d(matrix)
|
| 129 |
+
if rotation_type.startswith("euler_"):
|
| 130 |
+
convention = rotation_type.split("_", 1)[1].upper()
|
| 131 |
+
return matrix_to_euler_angles(matrix, convention)
|
| 132 |
+
raise ValueError(f"Unsupported rotation type '{rotation_type}'")
|
| 133 |
+
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/normalizer.py
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Literal, Dict, Annotated, Union, Any, List, Tuple, Optional
|
| 2 |
+
import torch
|
| 3 |
+
import json
|
| 4 |
+
from collections import defaultdict
|
| 5 |
+
import numpy as np
|
| 6 |
+
from omegaconf import DictConfig, OmegaConf
|
| 7 |
+
import hashlib
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from git import Repo
|
| 10 |
+
from fastwam.utils.logging_config import get_logger
|
| 11 |
+
|
| 12 |
+
from fastwam.utils.pytorch_utils import dict_apply
|
| 13 |
+
|
| 14 |
+
logger = get_logger(__name__)
|
| 15 |
+
|
| 16 |
+
ConstConstStr = Annotated[str, "format: 'const_min/const_max', where const_min and const_max give the constant range"]
|
| 17 |
+
NormMode = Union[Literal["min/max", "q01/q99", "z-score"], ConstConstStr]
|
| 18 |
+
|
| 19 |
+
class LinearNormalizer:
|
| 20 |
+
def __init__(
|
| 21 |
+
self,
|
| 22 |
+
shape_meta,
|
| 23 |
+
use_stepwise_action_norm,
|
| 24 |
+
default_mode: NormMode,
|
| 25 |
+
exception_mode: Dict[str, Dict[str, NormMode]],
|
| 26 |
+
stats: Dict[str, Dict[str, Dict[str, torch.Tensor]]]
|
| 27 |
+
):
|
| 28 |
+
super().__init__()
|
| 29 |
+
self.normalizers = {"action": {}, "state": {}}
|
| 30 |
+
self.stats = stats
|
| 31 |
+
|
| 32 |
+
for meta in shape_meta["action"]:
|
| 33 |
+
key = meta["key"]
|
| 34 |
+
|
| 35 |
+
if use_stepwise_action_norm:
|
| 36 |
+
cur_stats = {k.removeprefix("stepwise_"): v for k, v in stats["action"][key].items() if k.startswith("stepwise_")}
|
| 37 |
+
else:
|
| 38 |
+
cur_stats = {k.removeprefix("global_"): v for k, v in stats["action"][key].items() if k.startswith("global_")}
|
| 39 |
+
|
| 40 |
+
if exception_mode is not None and "action" in exception_mode and key in exception_mode["action"]:
|
| 41 |
+
cur_mode = exception_mode["action"][key]
|
| 42 |
+
else:
|
| 43 |
+
cur_mode = default_mode
|
| 44 |
+
|
| 45 |
+
self.normalizers["action"][key] = SingleFieldLinearNormalizer(
|
| 46 |
+
stats=cur_stats,
|
| 47 |
+
mode=cur_mode,
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
for meta in shape_meta["state"]:
|
| 51 |
+
key = meta["key"]
|
| 52 |
+
cur_stats = {k.removeprefix("global_"): v for k, v in stats["state"][key].items() if k.startswith("global_")}
|
| 53 |
+
|
| 54 |
+
if exception_mode is not None and "state" in exception_mode and key in exception_mode["state"]:
|
| 55 |
+
cur_mode = exception_mode["state"][key]
|
| 56 |
+
else:
|
| 57 |
+
cur_mode = default_mode
|
| 58 |
+
|
| 59 |
+
self.normalizers["state"][key] = SingleFieldLinearNormalizer(
|
| 60 |
+
stats=cur_stats,
|
| 61 |
+
mode=cur_mode,
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
def get_stats(self):
|
| 65 |
+
stats = {
|
| 66 |
+
"action": {key: norm.get_stats() for key, norm in self.normalizers["action"].items()},
|
| 67 |
+
"state": {key: norm.get_stats() for key, norm in self.normalizers["state"].items()}
|
| 68 |
+
}
|
| 69 |
+
return stats
|
| 70 |
+
|
| 71 |
+
def forward(self, batch: Dict[str, Dict[str, torch.Tensor]]) -> torch.Tensor:
|
| 72 |
+
if "action" in batch:
|
| 73 |
+
for key, norm in self.normalizers["action"].items():
|
| 74 |
+
batch["action"][key] = norm.forward(batch["action"][key])
|
| 75 |
+
|
| 76 |
+
for key, norm in self.normalizers["state"].items():
|
| 77 |
+
batch["state"][key] = norm.forward(batch["state"][key])
|
| 78 |
+
|
| 79 |
+
return batch
|
| 80 |
+
|
| 81 |
+
def backward(self, batch: Dict[str, Dict[str, torch.Tensor]]) -> torch.Tensor:
|
| 82 |
+
for key, norm in self.normalizers["action"].items():
|
| 83 |
+
batch["action"][key] = norm.backward(batch["action"][key])
|
| 84 |
+
|
| 85 |
+
for key, norm in self.normalizers["state"].items():
|
| 86 |
+
batch["state"][key] = norm.backward(batch["state"][key])
|
| 87 |
+
|
| 88 |
+
return batch
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
class SingleFieldLinearNormalizer:
|
| 92 |
+
std_reg = 1e-8
|
| 93 |
+
range_tol = 1e-4
|
| 94 |
+
output_max = 1.0
|
| 95 |
+
output_min = -1.0
|
| 96 |
+
def __init__(self, stats, mode: NormMode="min/max"):
|
| 97 |
+
self.stats = stats
|
| 98 |
+
self.mode = mode
|
| 99 |
+
|
| 100 |
+
if mode == "z-score":
|
| 101 |
+
input_mean, input_std = stats["mean"], stats["std"]
|
| 102 |
+
scale = 1.0 / (input_std + self.std_reg)
|
| 103 |
+
offset = - input_mean / (input_std + self.std_reg)
|
| 104 |
+
else:
|
| 105 |
+
if mode == "min/max":
|
| 106 |
+
input_min, input_max = stats["min"], stats["max"]
|
| 107 |
+
elif mode == "q01/q99":
|
| 108 |
+
input_min, input_max = stats["q01"], stats["q99"]
|
| 109 |
+
else:
|
| 110 |
+
# parse const_min/const_max
|
| 111 |
+
input_min, input_max = map(float, mode.split("/"))
|
| 112 |
+
input_min = torch.full_like(stats["min"], input_min)
|
| 113 |
+
input_max = torch.full_like(stats["max"], input_max)
|
| 114 |
+
|
| 115 |
+
input_range = input_max - input_min
|
| 116 |
+
ignore_dim = input_range < self.range_tol
|
| 117 |
+
input_range[ignore_dim] = self.output_max - self.output_min
|
| 118 |
+
scale = (self.output_max - self.output_min) / input_range
|
| 119 |
+
offset = self.output_min - scale * input_min
|
| 120 |
+
offset[ignore_dim] = (self.output_max + self.output_min) / 2 - input_min[ignore_dim]
|
| 121 |
+
|
| 122 |
+
self.scale = scale
|
| 123 |
+
self.offset = offset
|
| 124 |
+
def get_stats(self):
|
| 125 |
+
return self.stats
|
| 126 |
+
|
| 127 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 128 |
+
x = x * self.scale + self.offset
|
| 129 |
+
x = torch.clamp(x, -5.0, 5.0)
|
| 130 |
+
return x
|
| 131 |
+
def backward(self, x: torch.Tensor) -> torch.Tensor:
|
| 132 |
+
x = (x - self.offset) / self.scale
|
| 133 |
+
return x
|
| 134 |
+
|
| 135 |
+
def save_dataset_stats_to_json(dataset_stats: dict, file_path: str):
|
| 136 |
+
|
| 137 |
+
def convert_tensor(obj):
|
| 138 |
+
if isinstance(obj, torch.Tensor):
|
| 139 |
+
return obj.detach().cpu().numpy().tolist()
|
| 140 |
+
elif isinstance(obj, (defaultdict, dict)):
|
| 141 |
+
return {k: convert_tensor(v) for k, v in dict(obj).items()}
|
| 142 |
+
elif isinstance(obj, (list, tuple)):
|
| 143 |
+
return [convert_tensor(item) for item in obj]
|
| 144 |
+
elif isinstance(obj, (int, float, str, bool, type(None))):
|
| 145 |
+
return obj
|
| 146 |
+
else:
|
| 147 |
+
return str(obj)
|
| 148 |
+
|
| 149 |
+
serializable_stats = convert_tensor(dataset_stats)
|
| 150 |
+
|
| 151 |
+
with open(file_path, 'w', encoding='utf-8') as f:
|
| 152 |
+
json.dump(serializable_stats, f, ensure_ascii=False, indent=2)
|
| 153 |
+
|
| 154 |
+
def load_dataset_stats_from_json(file_path: str,
|
| 155 |
+
try_convert_tensor: bool = True) -> Dict[str, Any]:
|
| 156 |
+
|
| 157 |
+
def is_numeric_list(obj):
|
| 158 |
+
if isinstance(obj, list):
|
| 159 |
+
if not obj:
|
| 160 |
+
return True
|
| 161 |
+
first = obj[0]
|
| 162 |
+
if isinstance(first, (int, float)):
|
| 163 |
+
return all(isinstance(x, (int, float)) for x in obj)
|
| 164 |
+
elif isinstance(first, list):
|
| 165 |
+
return all(is_numeric_list(item) for item in obj)
|
| 166 |
+
else:
|
| 167 |
+
return False
|
| 168 |
+
return False
|
| 169 |
+
|
| 170 |
+
def convert_back_to_tensor(obj):
|
| 171 |
+
if isinstance(obj, dict):
|
| 172 |
+
return {k: convert_back_to_tensor(v) for k, v in obj.items()}
|
| 173 |
+
elif isinstance(obj, list):
|
| 174 |
+
if is_numeric_list(obj):
|
| 175 |
+
try:
|
| 176 |
+
arr = np.array(obj)
|
| 177 |
+
return torch.from_numpy(arr)
|
| 178 |
+
except Exception:
|
| 179 |
+
return [convert_back_to_tensor(item) for item in obj]
|
| 180 |
+
else:
|
| 181 |
+
return [convert_back_to_tensor(item) for item in obj]
|
| 182 |
+
else:
|
| 183 |
+
return obj
|
| 184 |
+
|
| 185 |
+
with open(file_path, 'r', encoding='utf-8') as f:
|
| 186 |
+
data = json.load(f)
|
| 187 |
+
|
| 188 |
+
if try_convert_tensor:
|
| 189 |
+
data = convert_back_to_tensor(data)
|
| 190 |
+
|
| 191 |
+
data = dict_apply(
|
| 192 |
+
data,
|
| 193 |
+
lambda x: x.to(torch.float32) if isinstance(x, torch.Tensor) else x,
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
return data
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def search_dataset_stats_cache_json(cache_dir: str | Path, data_config: DictConfig) -> Tuple[bool, str | None]:
|
| 200 |
+
if isinstance(cache_dir, str):
|
| 201 |
+
cache_dir = Path(cache_dir)
|
| 202 |
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
| 203 |
+
|
| 204 |
+
def get_git_hash() -> Optional[str]:
|
| 205 |
+
repo = Repo(__file__, search_parent_directories=True)
|
| 206 |
+
return repo.head.commit.hexsha
|
| 207 |
+
|
| 208 |
+
def to_plain(value: Any) -> Any:
|
| 209 |
+
if OmegaConf.is_config(value):
|
| 210 |
+
return OmegaConf.to_container(value, resolve=True)
|
| 211 |
+
return value
|
| 212 |
+
|
| 213 |
+
def normalize_str_list(value: Any) -> List[str]:
|
| 214 |
+
value = to_plain(value)
|
| 215 |
+
if value is None:
|
| 216 |
+
return []
|
| 217 |
+
if isinstance(value, str):
|
| 218 |
+
value = [value]
|
| 219 |
+
return [str(item) for item in value if item is not None]
|
| 220 |
+
|
| 221 |
+
def normalize_transforms(value: Any) -> Any:
|
| 222 |
+
value = to_plain(value)
|
| 223 |
+
if isinstance(value, dict):
|
| 224 |
+
return [value]
|
| 225 |
+
return value
|
| 226 |
+
|
| 227 |
+
def normalize_dataset_dirs(cfg: DictConfig) -> Any:
|
| 228 |
+
dataset_cfg = cfg.get("dataset")
|
| 229 |
+
if dataset_cfg is None:
|
| 230 |
+
return None
|
| 231 |
+
embodiment_datasets = dataset_cfg.get("embodiment_datasets")
|
| 232 |
+
if embodiment_datasets is not None:
|
| 233 |
+
emb_dirs: Dict[str, List[str]] = {}
|
| 234 |
+
for emb, emb_cfg in embodiment_datasets.items():
|
| 235 |
+
dataset_groups = emb_cfg.get("dataset_groups")
|
| 236 |
+
if dataset_groups is None:
|
| 237 |
+
emb_dirs[emb] = []
|
| 238 |
+
continue
|
| 239 |
+
dirs: List[str] = []
|
| 240 |
+
for group in dataset_groups:
|
| 241 |
+
group_dirs = group.get("dataset_dirs")
|
| 242 |
+
if group_dirs is None:
|
| 243 |
+
continue
|
| 244 |
+
dirs.extend(normalize_str_list(group_dirs))
|
| 245 |
+
emb_dirs[emb] = sorted(dirs)
|
| 246 |
+
return emb_dirs
|
| 247 |
+
|
| 248 |
+
dataset_dirs = dataset_cfg.get("dataset_dirs")
|
| 249 |
+
return sorted(normalize_str_list(dataset_dirs))
|
| 250 |
+
|
| 251 |
+
def normalize_action_state_transforms(cfg: DictConfig) -> Any:
|
| 252 |
+
processor_cfg = cfg.get("processor")
|
| 253 |
+
if processor_cfg is None:
|
| 254 |
+
return None
|
| 255 |
+
embodiment_processors = processor_cfg.get("embodiment_processors")
|
| 256 |
+
if embodiment_processors is not None:
|
| 257 |
+
emb_transforms: Dict[str, Any] = {}
|
| 258 |
+
for emb, emb_cfg in embodiment_processors.items():
|
| 259 |
+
transforms = emb_cfg.get("action_state_transforms")
|
| 260 |
+
emb_transforms[emb] = normalize_transforms(transforms)
|
| 261 |
+
return emb_transforms
|
| 262 |
+
|
| 263 |
+
transforms = processor_cfg.get("action_state_transforms")
|
| 264 |
+
return normalize_transforms(transforms)
|
| 265 |
+
|
| 266 |
+
signature = {
|
| 267 |
+
"action_size": data_config.dataset.action_size,
|
| 268 |
+
"dataset_dirs": normalize_dataset_dirs(data_config),
|
| 269 |
+
"action_state_transforms": normalize_action_state_transforms(data_config),
|
| 270 |
+
}
|
| 271 |
+
signature_json = json.dumps(signature, sort_keys=True, separators=(",", ":"))
|
| 272 |
+
dataset_hash = hashlib.sha256(signature_json.encode("utf-8"), usedforsecurity=False).hexdigest()
|
| 273 |
+
|
| 274 |
+
git_hash = get_git_hash()
|
| 275 |
+
precise_name = f"dataset_stats_{dataset_hash}_{git_hash}.json"
|
| 276 |
+
precise = cache_dir / precise_name
|
| 277 |
+
if precise.exists():
|
| 278 |
+
logger.info(f"Found dataset stats cache with precisely matching dataset and git hash: {precise_name}.")
|
| 279 |
+
return True, str(precise)
|
| 280 |
+
|
| 281 |
+
candidates = sorted(cache_dir.glob(f"dataset_stats_{dataset_hash}_*.json"))
|
| 282 |
+
if not candidates:
|
| 283 |
+
logger.info(f"No dataset stats cache found for dataset hash {dataset_hash}")
|
| 284 |
+
return False, str(precise) # return precise cache path for saving cache
|
| 285 |
+
|
| 286 |
+
picked = candidates[0]
|
| 287 |
+
prefix = f"dataset_stats_{dataset_hash}_"
|
| 288 |
+
picked_git_hash = picked.name[len(prefix):-5]
|
| 289 |
+
assert picked_git_hash != git_hash
|
| 290 |
+
logger.warning(f"Found substitute dataset stats cache {picked.name} which mismatch current git hash {git_hash}.")
|
| 291 |
+
return True, str(picked)
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/datasets/lerobot/utils/rotation.py
ADDED
|
@@ -0,0 +1,621 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
from typing import Optional, Union, Literal, List
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
|
| 6 |
+
Device = Union[str, torch.device]
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
# [Important] Below is adapted from pytorch3d, which uses (r, i, j, k) quat order
|
| 10 |
+
|
| 11 |
+
"""
|
| 12 |
+
The transformation matrices returned from the functions in this file assume
|
| 13 |
+
the points on which the transformation will be applied are column vectors.
|
| 14 |
+
i.e. the R matrix is structured as
|
| 15 |
+
|
| 16 |
+
R = [
|
| 17 |
+
[Rxx, Rxy, Rxz],
|
| 18 |
+
[Ryx, Ryy, Ryz],
|
| 19 |
+
[Rzx, Rzy, Rzz],
|
| 20 |
+
] # (3, 3)
|
| 21 |
+
|
| 22 |
+
This matrix can be applied to column vectors by post multiplication
|
| 23 |
+
by the points e.g.
|
| 24 |
+
|
| 25 |
+
points = [[0], [1], [2]] # (3 x 1) xyz coordinates of a point
|
| 26 |
+
transformed_points = R * points
|
| 27 |
+
|
| 28 |
+
To apply the same matrix to points which are row vectors, the R matrix
|
| 29 |
+
can be transposed and pre multiplied by the points:
|
| 30 |
+
|
| 31 |
+
e.g.
|
| 32 |
+
points = [[0, 1, 2]] # (1 x 3) xyz coordinates of a point
|
| 33 |
+
transformed_points = points * R.transpose(1, 0)
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def quaternion_to_matrix(quaternions: torch.Tensor) -> torch.Tensor:
|
| 38 |
+
"""
|
| 39 |
+
Convert rotations given as quaternions to rotation matrices.
|
| 40 |
+
|
| 41 |
+
Args:
|
| 42 |
+
quaternions: quaternions with real part first,
|
| 43 |
+
as tensor of shape (..., 4).
|
| 44 |
+
|
| 45 |
+
Returns:
|
| 46 |
+
Rotation matrices as tensor of shape (..., 3, 3).
|
| 47 |
+
"""
|
| 48 |
+
r, i, j, k = torch.unbind(quaternions, -1)
|
| 49 |
+
# pyre-fixme[58]: `/` is not supported for operand types `float` and `Tensor`.
|
| 50 |
+
two_s = 2.0 / (quaternions * quaternions).sum(-1)
|
| 51 |
+
|
| 52 |
+
o = torch.stack(
|
| 53 |
+
(
|
| 54 |
+
1 - two_s * (j * j + k * k),
|
| 55 |
+
two_s * (i * j - k * r),
|
| 56 |
+
two_s * (i * k + j * r),
|
| 57 |
+
two_s * (i * j + k * r),
|
| 58 |
+
1 - two_s * (i * i + k * k),
|
| 59 |
+
two_s * (j * k - i * r),
|
| 60 |
+
two_s * (i * k - j * r),
|
| 61 |
+
two_s * (j * k + i * r),
|
| 62 |
+
1 - two_s * (i * i + j * j),
|
| 63 |
+
),
|
| 64 |
+
-1,
|
| 65 |
+
)
|
| 66 |
+
return o.reshape(quaternions.shape[:-1] + (3, 3))
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _copysign(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
| 70 |
+
"""
|
| 71 |
+
Return a tensor where each element has the absolute value taken from the,
|
| 72 |
+
corresponding element of a, with sign taken from the corresponding
|
| 73 |
+
element of b. This is like the standard copysign floating-point operation,
|
| 74 |
+
but is not careful about negative 0 and NaN.
|
| 75 |
+
|
| 76 |
+
Args:
|
| 77 |
+
a: source tensor.
|
| 78 |
+
b: tensor whose signs will be used, of the same shape as a.
|
| 79 |
+
|
| 80 |
+
Returns:
|
| 81 |
+
Tensor of the same shape as a with the signs of b.
|
| 82 |
+
"""
|
| 83 |
+
signs_differ = (a < 0) != (b < 0)
|
| 84 |
+
return torch.where(signs_differ, -a, a)
|
| 85 |
+
|
| 86 |
+
@torch.jit.script
|
| 87 |
+
def _sqrt_positive_part(x: torch.Tensor) -> torch.Tensor:
|
| 88 |
+
"""
|
| 89 |
+
Returns torch.sqrt(torch.max(0, x))
|
| 90 |
+
but with a zero subgradient where x is 0.
|
| 91 |
+
"""
|
| 92 |
+
ret = torch.zeros_like(x)
|
| 93 |
+
positive_mask = x > 0
|
| 94 |
+
ret[positive_mask] = torch.sqrt(x[positive_mask])
|
| 95 |
+
return ret
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def matrix_to_quaternion(matrix: torch.Tensor) -> torch.Tensor:
|
| 99 |
+
"""
|
| 100 |
+
Convert rotations given as rotation matrices to quaternions.
|
| 101 |
+
|
| 102 |
+
Args:
|
| 103 |
+
matrix: Rotation matrices as tensor of shape (..., 3, 3).
|
| 104 |
+
|
| 105 |
+
Returns:
|
| 106 |
+
quaternions with real part first, as tensor of shape (..., 4).
|
| 107 |
+
"""
|
| 108 |
+
if matrix.size(-1) != 3 or matrix.size(-2) != 3:
|
| 109 |
+
raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.")
|
| 110 |
+
|
| 111 |
+
batch_dim = matrix.shape[:-2]
|
| 112 |
+
m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.unbind(
|
| 113 |
+
matrix.reshape(batch_dim + (9,)), dim=-1
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
q_abs = _sqrt_positive_part(
|
| 117 |
+
torch.stack(
|
| 118 |
+
[
|
| 119 |
+
1.0 + m00 + m11 + m22,
|
| 120 |
+
1.0 + m00 - m11 - m22,
|
| 121 |
+
1.0 - m00 + m11 - m22,
|
| 122 |
+
1.0 - m00 - m11 + m22,
|
| 123 |
+
],
|
| 124 |
+
dim=-1,
|
| 125 |
+
)
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
# we produce the desired quaternion multiplied by each of r, i, j, k
|
| 129 |
+
quat_by_rijk = torch.stack(
|
| 130 |
+
[
|
| 131 |
+
# pyre-fixme[58]: `**` is not supported for operand types `Tensor` and
|
| 132 |
+
# `int`.
|
| 133 |
+
torch.stack([q_abs[..., 0] ** 2, m21 - m12, m02 - m20, m10 - m01], dim=-1),
|
| 134 |
+
# pyre-fixme[58]: `**` is not supported for operand types `Tensor` and
|
| 135 |
+
# `int`.
|
| 136 |
+
torch.stack([m21 - m12, q_abs[..., 1] ** 2, m10 + m01, m02 + m20], dim=-1),
|
| 137 |
+
# pyre-fixme[58]: `**` is not supported for operand types `Tensor` and
|
| 138 |
+
# `int`.
|
| 139 |
+
torch.stack([m02 - m20, m10 + m01, q_abs[..., 2] ** 2, m12 + m21], dim=-1),
|
| 140 |
+
# pyre-fixme[58]: `**` is not supported for operand types `Tensor` and
|
| 141 |
+
# `int`.
|
| 142 |
+
torch.stack([m10 - m01, m20 + m02, m21 + m12, q_abs[..., 3] ** 2], dim=-1),
|
| 143 |
+
],
|
| 144 |
+
dim=-2,
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
# We floor here at 0.1 but the exact level is not important; if q_abs is small,
|
| 148 |
+
# the candidate won't be picked.
|
| 149 |
+
flr = torch.tensor(0.1).to(dtype=q_abs.dtype, device=q_abs.device)
|
| 150 |
+
quat_candidates = quat_by_rijk / (2.0 * q_abs[..., None].max(flr))
|
| 151 |
+
|
| 152 |
+
# if not for numerical problems, quat_candidates[i] should be same (up to a sign),
|
| 153 |
+
# forall i; we pick the best-conditioned one (with the largest denominator)
|
| 154 |
+
out = quat_candidates[
|
| 155 |
+
F.one_hot(q_abs.argmax(dim=-1), num_classes=4) > 0.5, :
|
| 156 |
+
].reshape(batch_dim + (4,))
|
| 157 |
+
return standardize_quaternion(out)
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def _axis_angle_rotation(axis: str, angle: torch.Tensor) -> torch.Tensor:
|
| 161 |
+
"""
|
| 162 |
+
Return the rotation matrices for one of the rotations about an axis
|
| 163 |
+
of which Euler angles describe, for each value of the angle given.
|
| 164 |
+
|
| 165 |
+
Args:
|
| 166 |
+
axis: Axis label "X" or "Y or "Z".
|
| 167 |
+
angle: any shape tensor of Euler angles in radians
|
| 168 |
+
|
| 169 |
+
Returns:
|
| 170 |
+
Rotation matrices as tensor of shape (..., 3, 3).
|
| 171 |
+
"""
|
| 172 |
+
|
| 173 |
+
cos = torch.cos(angle)
|
| 174 |
+
sin = torch.sin(angle)
|
| 175 |
+
one = torch.ones_like(angle)
|
| 176 |
+
zero = torch.zeros_like(angle)
|
| 177 |
+
|
| 178 |
+
if axis == "X":
|
| 179 |
+
R_flat = (one, zero, zero, zero, cos, -sin, zero, sin, cos)
|
| 180 |
+
elif axis == "Y":
|
| 181 |
+
R_flat = (cos, zero, sin, zero, one, zero, -sin, zero, cos)
|
| 182 |
+
elif axis == "Z":
|
| 183 |
+
R_flat = (cos, -sin, zero, sin, cos, zero, zero, zero, one)
|
| 184 |
+
else:
|
| 185 |
+
raise ValueError("letter must be either X, Y or Z.")
|
| 186 |
+
|
| 187 |
+
return torch.stack(R_flat, -1).reshape(angle.shape + (3, 3))
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def euler_angles_to_matrix(euler_angles: torch.Tensor, convention: str) -> torch.Tensor:
|
| 191 |
+
"""
|
| 192 |
+
Convert rotations given as Euler angles in radians to rotation matrices.
|
| 193 |
+
|
| 194 |
+
Args:
|
| 195 |
+
euler_angles: Euler angles in radians as tensor of shape (..., 3).
|
| 196 |
+
convention: Convention string of three uppercase letters from
|
| 197 |
+
{"X", "Y", and "Z"}.
|
| 198 |
+
|
| 199 |
+
Returns:
|
| 200 |
+
Rotation matrices as tensor of shape (..., 3, 3).
|
| 201 |
+
"""
|
| 202 |
+
if euler_angles.dim() == 0 or euler_angles.shape[-1] != 3:
|
| 203 |
+
raise ValueError("Invalid input euler angles.")
|
| 204 |
+
if len(convention) != 3:
|
| 205 |
+
raise ValueError("Convention must have 3 letters.")
|
| 206 |
+
if convention[1] in (convention[0], convention[2]):
|
| 207 |
+
raise ValueError(f"Invalid convention {convention}.")
|
| 208 |
+
for letter in convention:
|
| 209 |
+
if letter not in ("X", "Y", "Z"):
|
| 210 |
+
raise ValueError(f"Invalid letter {letter} in convention string.")
|
| 211 |
+
matrices = [
|
| 212 |
+
_axis_angle_rotation(c, e)
|
| 213 |
+
for c, e in zip(convention, torch.unbind(euler_angles, -1))
|
| 214 |
+
]
|
| 215 |
+
# return functools.reduce(torch.matmul, matrices)
|
| 216 |
+
return torch.matmul(torch.matmul(matrices[0], matrices[1]), matrices[2])
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def _angle_from_tan(
|
| 220 |
+
axis: str, other_axis: str, data, horizontal: bool, tait_bryan: bool
|
| 221 |
+
) -> torch.Tensor:
|
| 222 |
+
"""
|
| 223 |
+
Extract the first or third Euler angle from the two members of
|
| 224 |
+
the matrix which are positive constant times its sine and cosine.
|
| 225 |
+
|
| 226 |
+
Args:
|
| 227 |
+
axis: Axis label "X" or "Y or "Z" for the angle we are finding.
|
| 228 |
+
other_axis: Axis label "X" or "Y or "Z" for the middle axis in the
|
| 229 |
+
convention.
|
| 230 |
+
data: Rotation matrices as tensor of shape (..., 3, 3).
|
| 231 |
+
horizontal: Whether we are looking for the angle for the third axis,
|
| 232 |
+
which means the relevant entries are in the same row of the
|
| 233 |
+
rotation matrix. If not, they are in the same column.
|
| 234 |
+
tait_bryan: Whether the first and third axes in the convention differ.
|
| 235 |
+
|
| 236 |
+
Returns:
|
| 237 |
+
Euler Angles in radians for each matrix in data as a tensor
|
| 238 |
+
of shape (...).
|
| 239 |
+
"""
|
| 240 |
+
|
| 241 |
+
i1, i2 = {"X": (2, 1), "Y": (0, 2), "Z": (1, 0)}[axis]
|
| 242 |
+
if horizontal:
|
| 243 |
+
i2, i1 = i1, i2
|
| 244 |
+
even = (axis + other_axis) in ["XY", "YZ", "ZX"]
|
| 245 |
+
if horizontal == even:
|
| 246 |
+
return torch.atan2(data[..., i1], data[..., i2])
|
| 247 |
+
if tait_bryan:
|
| 248 |
+
return torch.atan2(-data[..., i2], data[..., i1])
|
| 249 |
+
return torch.atan2(data[..., i2], -data[..., i1])
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def _index_from_letter(letter: str) -> int:
|
| 253 |
+
if letter == "X":
|
| 254 |
+
return 0
|
| 255 |
+
if letter == "Y":
|
| 256 |
+
return 1
|
| 257 |
+
if letter == "Z":
|
| 258 |
+
return 2
|
| 259 |
+
raise ValueError("letter must be either X, Y or Z.")
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def matrix_to_euler_angles(matrix: torch.Tensor, convention: str) -> torch.Tensor:
|
| 263 |
+
"""
|
| 264 |
+
Convert rotations given as rotation matrices to Euler angles in radians.
|
| 265 |
+
|
| 266 |
+
Args:
|
| 267 |
+
matrix: Rotation matrices as tensor of shape (..., 3, 3).
|
| 268 |
+
convention: Convention string of three uppercase letters.
|
| 269 |
+
|
| 270 |
+
Returns:
|
| 271 |
+
Euler angles in radians as tensor of shape (..., 3).
|
| 272 |
+
"""
|
| 273 |
+
if len(convention) != 3:
|
| 274 |
+
raise ValueError("Convention must have 3 letters.")
|
| 275 |
+
if convention[1] in (convention[0], convention[2]):
|
| 276 |
+
raise ValueError(f"Invalid convention {convention}.")
|
| 277 |
+
for letter in convention:
|
| 278 |
+
if letter not in ("X", "Y", "Z"):
|
| 279 |
+
raise ValueError(f"Invalid letter {letter} in convention string.")
|
| 280 |
+
if matrix.size(-1) != 3 or matrix.size(-2) != 3:
|
| 281 |
+
raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.")
|
| 282 |
+
i0 = _index_from_letter(convention[0])
|
| 283 |
+
i2 = _index_from_letter(convention[2])
|
| 284 |
+
tait_bryan = i0 != i2
|
| 285 |
+
if tait_bryan:
|
| 286 |
+
central_angle = torch.asin(
|
| 287 |
+
matrix[..., i0, i2] * (-1.0 if i0 - i2 in [-1, 2] else 1.0)
|
| 288 |
+
)
|
| 289 |
+
else:
|
| 290 |
+
central_angle = torch.acos(matrix[..., i0, i0])
|
| 291 |
+
|
| 292 |
+
o = (
|
| 293 |
+
_angle_from_tan(
|
| 294 |
+
convention[0], convention[1], matrix[..., i2], False, tait_bryan
|
| 295 |
+
),
|
| 296 |
+
central_angle,
|
| 297 |
+
_angle_from_tan(
|
| 298 |
+
convention[2], convention[1], matrix[..., i0, :], True, tait_bryan
|
| 299 |
+
),
|
| 300 |
+
)
|
| 301 |
+
return torch.stack(o, -1)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def random_quaternions(
|
| 305 |
+
n: int, dtype: Optional[torch.dtype] = None, device: Optional[Device] = None
|
| 306 |
+
) -> torch.Tensor:
|
| 307 |
+
"""
|
| 308 |
+
Generate random quaternions representing rotations,
|
| 309 |
+
i.e. versors with nonnegative real part.
|
| 310 |
+
|
| 311 |
+
Args:
|
| 312 |
+
n: Number of quaternions in a batch to return.
|
| 313 |
+
dtype: Type to return.
|
| 314 |
+
device: Desired device of returned tensor. Default:
|
| 315 |
+
uses the current device for the default tensor type.
|
| 316 |
+
|
| 317 |
+
Returns:
|
| 318 |
+
Quaternions as tensor of shape (N, 4).
|
| 319 |
+
"""
|
| 320 |
+
if isinstance(device, str):
|
| 321 |
+
device = torch.device(device)
|
| 322 |
+
o = torch.randn((n, 4), dtype=dtype, device=device)
|
| 323 |
+
s = (o * o).sum(1)
|
| 324 |
+
o = o / _copysign(torch.sqrt(s), o[:, 0])[:, None]
|
| 325 |
+
return o
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def random_rotations(
|
| 329 |
+
n: int, dtype: Optional[torch.dtype] = None, device: Optional[Device] = None
|
| 330 |
+
) -> torch.Tensor:
|
| 331 |
+
"""
|
| 332 |
+
Generate random rotations as 3x3 rotation matrices.
|
| 333 |
+
|
| 334 |
+
Args:
|
| 335 |
+
n: Number of rotation matrices in a batch to return.
|
| 336 |
+
dtype: Type to return.
|
| 337 |
+
device: Device of returned tensor. Default: if None,
|
| 338 |
+
uses the current device for the default tensor type.
|
| 339 |
+
|
| 340 |
+
Returns:
|
| 341 |
+
Rotation matrices as tensor of shape (n, 3, 3).
|
| 342 |
+
"""
|
| 343 |
+
quaternions = random_quaternions(n, dtype=dtype, device=device)
|
| 344 |
+
return quaternion_to_matrix(quaternions)
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
def random_rotation(
|
| 348 |
+
dtype: Optional[torch.dtype] = None, device: Optional[Device] = None
|
| 349 |
+
) -> torch.Tensor:
|
| 350 |
+
"""
|
| 351 |
+
Generate a single random 3x3 rotation matrix.
|
| 352 |
+
|
| 353 |
+
Args:
|
| 354 |
+
dtype: Type to return
|
| 355 |
+
device: Device of returned tensor. Default: if None,
|
| 356 |
+
uses the current device for the default tensor type
|
| 357 |
+
|
| 358 |
+
Returns:
|
| 359 |
+
Rotation matrix as tensor of shape (3, 3).
|
| 360 |
+
"""
|
| 361 |
+
return random_rotations(1, dtype, device)[0]
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor:
|
| 365 |
+
"""
|
| 366 |
+
Convert a unit quaternion to a standard form: one in which the real
|
| 367 |
+
part is non negative.
|
| 368 |
+
|
| 369 |
+
Args:
|
| 370 |
+
quaternions: Quaternions with real part first,
|
| 371 |
+
as tensor of shape (..., 4).
|
| 372 |
+
|
| 373 |
+
Returns:
|
| 374 |
+
Standardized quaternions as tensor of shape (..., 4).
|
| 375 |
+
"""
|
| 376 |
+
return torch.where(quaternions[..., 0:1] < 0, -quaternions, quaternions)
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def quaternion_raw_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
| 380 |
+
"""
|
| 381 |
+
Multiply two quaternions.
|
| 382 |
+
Usual torch rules for broadcasting apply.
|
| 383 |
+
|
| 384 |
+
Args:
|
| 385 |
+
a: Quaternions as tensor of shape (..., 4), real part first.
|
| 386 |
+
b: Quaternions as tensor of shape (..., 4), real part first.
|
| 387 |
+
|
| 388 |
+
Returns:
|
| 389 |
+
The product of a and b, a tensor of quaternions shape (..., 4).
|
| 390 |
+
"""
|
| 391 |
+
aw, ax, ay, az = torch.unbind(a, -1)
|
| 392 |
+
bw, bx, by, bz = torch.unbind(b, -1)
|
| 393 |
+
ow = aw * bw - ax * bx - ay * by - az * bz
|
| 394 |
+
ox = aw * bx + ax * bw + ay * bz - az * by
|
| 395 |
+
oy = aw * by - ax * bz + ay * bw + az * bx
|
| 396 |
+
oz = aw * bz + ax * by - ay * bx + az * bw
|
| 397 |
+
return torch.stack((ow, ox, oy, oz), -1)
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def quaternion_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
| 401 |
+
"""
|
| 402 |
+
Multiply two quaternions representing rotations, returning the quaternion
|
| 403 |
+
representing their composition, i.e. the versor with nonnegative real part.
|
| 404 |
+
Usual torch rules for broadcasting apply.
|
| 405 |
+
|
| 406 |
+
Args:
|
| 407 |
+
a: Quaternions as tensor of shape (..., 4), real part first.
|
| 408 |
+
b: Quaternions as tensor of shape (..., 4), real part first.
|
| 409 |
+
|
| 410 |
+
Returns:
|
| 411 |
+
The product of a and b, a tensor of quaternions of shape (..., 4).
|
| 412 |
+
"""
|
| 413 |
+
ab = quaternion_raw_multiply(a, b)
|
| 414 |
+
return standardize_quaternion(ab)
|
| 415 |
+
|
| 416 |
+
|
| 417 |
+
def quaternion_invert(quaternion: torch.Tensor) -> torch.Tensor:
|
| 418 |
+
"""
|
| 419 |
+
Given a quaternion representing rotation, get the quaternion representing
|
| 420 |
+
its inverse.
|
| 421 |
+
|
| 422 |
+
Args:
|
| 423 |
+
quaternion: Quaternions as tensor of shape (..., 4), with real part
|
| 424 |
+
first, which must be versors (unit quaternions).
|
| 425 |
+
|
| 426 |
+
Returns:
|
| 427 |
+
The inverse, a tensor of quaternions of shape (..., 4).
|
| 428 |
+
"""
|
| 429 |
+
|
| 430 |
+
scaling = torch.tensor([1, -1, -1, -1], device=quaternion.device)
|
| 431 |
+
return quaternion * scaling
|
| 432 |
+
|
| 433 |
+
|
| 434 |
+
def quaternion_apply(quaternion: torch.Tensor, point: torch.Tensor) -> torch.Tensor:
|
| 435 |
+
"""
|
| 436 |
+
Apply the rotation given by a quaternion to a 3D point.
|
| 437 |
+
Usual torch rules for broadcasting apply.
|
| 438 |
+
|
| 439 |
+
Args:
|
| 440 |
+
quaternion: Tensor of quaternions, real part first, of shape (..., 4).
|
| 441 |
+
point: Tensor of 3D points of shape (..., 3).
|
| 442 |
+
|
| 443 |
+
Returns:
|
| 444 |
+
Tensor of rotated points of shape (..., 3).
|
| 445 |
+
"""
|
| 446 |
+
if point.size(-1) != 3:
|
| 447 |
+
raise ValueError(f"Points are not in 3D, {point.shape}.")
|
| 448 |
+
real_parts = point.new_zeros(point.shape[:-1] + (1,))
|
| 449 |
+
point_as_quaternion = torch.cat((real_parts, point), -1)
|
| 450 |
+
out = quaternion_raw_multiply(
|
| 451 |
+
quaternion_raw_multiply(quaternion, point_as_quaternion),
|
| 452 |
+
quaternion_invert(quaternion),
|
| 453 |
+
)
|
| 454 |
+
return out[..., 1:]
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def axis_angle_to_matrix(axis_angle: torch.Tensor) -> torch.Tensor:
|
| 458 |
+
"""
|
| 459 |
+
Convert rotations given as axis/angle to rotation matrices.
|
| 460 |
+
|
| 461 |
+
Args:
|
| 462 |
+
axis_angle: Rotations given as a vector in axis angle form,
|
| 463 |
+
as a tensor of shape (..., 3), where the magnitude is
|
| 464 |
+
the angle turned anticlockwise in radians around the
|
| 465 |
+
vector's direction.
|
| 466 |
+
|
| 467 |
+
Returns:
|
| 468 |
+
Rotation matrices as tensor of shape (..., 3, 3).
|
| 469 |
+
"""
|
| 470 |
+
return quaternion_to_matrix(axis_angle_to_quaternion(axis_angle))
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
def matrix_to_axis_angle(matrix: torch.Tensor) -> torch.Tensor:
|
| 474 |
+
"""
|
| 475 |
+
Convert rotations given as rotation matrices to axis/angle.
|
| 476 |
+
|
| 477 |
+
Args:
|
| 478 |
+
matrix: Rotation matrices as tensor of shape (..., 3, 3).
|
| 479 |
+
|
| 480 |
+
Returns:
|
| 481 |
+
Rotations given as a vector in axis angle form, as a tensor
|
| 482 |
+
of shape (..., 3), where the magnitude is the angle
|
| 483 |
+
turned anticlockwise in radians around the vector's
|
| 484 |
+
direction.
|
| 485 |
+
"""
|
| 486 |
+
return quaternion_to_axis_angle(matrix_to_quaternion(matrix))
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
def axis_angle_to_quaternion(axis_angle: torch.Tensor) -> torch.Tensor:
|
| 490 |
+
"""
|
| 491 |
+
Convert rotations given as axis/angle to quaternions.
|
| 492 |
+
|
| 493 |
+
Args:
|
| 494 |
+
axis_angle: Rotations given as a vector in axis angle form,
|
| 495 |
+
as a tensor of shape (..., 3), where the magnitude is
|
| 496 |
+
the angle turned anticlockwise in radians around the
|
| 497 |
+
vector's direction.
|
| 498 |
+
|
| 499 |
+
Returns:
|
| 500 |
+
quaternions with real part first, as tensor of shape (..., 4).
|
| 501 |
+
"""
|
| 502 |
+
angles = torch.norm(axis_angle, p=2, dim=-1, keepdim=True)
|
| 503 |
+
half_angles = angles * 0.5
|
| 504 |
+
eps = 1e-6
|
| 505 |
+
small_angles = angles.abs() < eps
|
| 506 |
+
sin_half_angles_over_angles = torch.empty_like(angles)
|
| 507 |
+
sin_half_angles_over_angles[~small_angles] = (
|
| 508 |
+
torch.sin(half_angles[~small_angles]) / angles[~small_angles]
|
| 509 |
+
)
|
| 510 |
+
# for x small, sin(x/2) is about x/2 - (x/2)^3/6
|
| 511 |
+
# so sin(x/2)/x is about 1/2 - (x*x)/48
|
| 512 |
+
sin_half_angles_over_angles[small_angles] = (
|
| 513 |
+
0.5 - (angles[small_angles] * angles[small_angles]) / 48
|
| 514 |
+
)
|
| 515 |
+
quaternions = torch.cat(
|
| 516 |
+
[torch.cos(half_angles), axis_angle * sin_half_angles_over_angles], dim=-1
|
| 517 |
+
)
|
| 518 |
+
return quaternions
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
def quaternion_to_axis_angle(quaternions: torch.Tensor) -> torch.Tensor:
|
| 522 |
+
"""
|
| 523 |
+
Convert rotations given as quaternions to axis/angle.
|
| 524 |
+
|
| 525 |
+
Args:
|
| 526 |
+
quaternions: quaternions with real part first,
|
| 527 |
+
as tensor of shape (..., 4).
|
| 528 |
+
|
| 529 |
+
Returns:
|
| 530 |
+
Rotations given as a vector in axis angle form, as a tensor
|
| 531 |
+
of shape (..., 3), where the magnitude is the angle
|
| 532 |
+
turned anticlockwise in radians around the vector's
|
| 533 |
+
direction.
|
| 534 |
+
"""
|
| 535 |
+
norms = torch.norm(quaternions[..., 1:], p=2, dim=-1, keepdim=True)
|
| 536 |
+
half_angles = torch.atan2(norms, quaternions[..., :1])
|
| 537 |
+
angles = 2 * half_angles
|
| 538 |
+
eps = 1e-6
|
| 539 |
+
small_angles = angles.abs() < eps
|
| 540 |
+
sin_half_angles_over_angles = torch.empty_like(angles)
|
| 541 |
+
sin_half_angles_over_angles[~small_angles] = (
|
| 542 |
+
torch.sin(half_angles[~small_angles]) / angles[~small_angles]
|
| 543 |
+
)
|
| 544 |
+
# for x small, sin(x/2) is about x/2 - (x/2)^3/6
|
| 545 |
+
# so sin(x/2)/x is about 1/2 - (x*x)/48
|
| 546 |
+
sin_half_angles_over_angles[small_angles] = (
|
| 547 |
+
0.5 - (angles[small_angles] * angles[small_angles]) / 48
|
| 548 |
+
)
|
| 549 |
+
return quaternions[..., 1:] / sin_half_angles_over_angles
|
| 550 |
+
|
| 551 |
+
|
| 552 |
+
def rotation_6d_to_matrix(d6: torch.Tensor) -> torch.Tensor:
|
| 553 |
+
"""
|
| 554 |
+
Converts 6D rotation representation by Zhou et al. [1] to rotation matrix
|
| 555 |
+
using Gram--Schmidt orthogonalization per Section B of [1].
|
| 556 |
+
Args:
|
| 557 |
+
d6: 6D rotation representation, of size (*, 6)
|
| 558 |
+
|
| 559 |
+
Returns:
|
| 560 |
+
batch of rotation matrices of size (*, 3, 3)
|
| 561 |
+
|
| 562 |
+
[1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H.
|
| 563 |
+
On the Continuity of Rotation Representations in Neural Networks.
|
| 564 |
+
IEEE Conference on Computer Vision and Pattern Recognition, 2019.
|
| 565 |
+
Retrieved from http://arxiv.org/abs/1812.07035
|
| 566 |
+
"""
|
| 567 |
+
|
| 568 |
+
a1, a2 = d6[..., :3], d6[..., 3:]
|
| 569 |
+
b1 = F.normalize(a1, dim=-1)
|
| 570 |
+
b2 = a2 - (b1 * a2).sum(-1, keepdim=True) * b1
|
| 571 |
+
b2 = F.normalize(b2, dim=-1)
|
| 572 |
+
b3 = torch.cross(b1, b2, dim=-1)
|
| 573 |
+
return torch.stack((b1, b2, b3), dim=-2)
|
| 574 |
+
|
| 575 |
+
|
| 576 |
+
def matrix_to_rotation_6d(matrix: torch.Tensor) -> torch.Tensor:
|
| 577 |
+
"""
|
| 578 |
+
Converts rotation matrices to 6D rotation representation by Zhou et al. [1]
|
| 579 |
+
by dropping the last row. Note that 6D representation is not unique.
|
| 580 |
+
Args:
|
| 581 |
+
matrix: batch of rotation matrices of size (*, 3, 3)
|
| 582 |
+
|
| 583 |
+
Returns:
|
| 584 |
+
6D rotation representation, of size (*, 6)
|
| 585 |
+
|
| 586 |
+
[1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H.
|
| 587 |
+
On the Continuity of Rotation Representations in Neural Networks.
|
| 588 |
+
IEEE Conference on Computer Vision and Pattern Recognition, 2019.
|
| 589 |
+
Retrieved from http://arxiv.org/abs/1812.07035
|
| 590 |
+
"""
|
| 591 |
+
batch_dim = matrix.size()[:-2]
|
| 592 |
+
return matrix[..., :2, :].clone().reshape(batch_dim + (6,))
|
| 593 |
+
|
| 594 |
+
|
| 595 |
+
def rotation_9d_to_matrix(d9: torch.Tensor) -> torch.Tensor:
|
| 596 |
+
""""
|
| 597 |
+
Args:
|
| 598 |
+
d9: 9D rotaion representation of size (..., 9)
|
| 599 |
+
|
| 600 |
+
Returns:
|
| 601 |
+
batch of rotation matrices of size (..., 3, 3)
|
| 602 |
+
"""
|
| 603 |
+
assert d9.shape[-1] == 9
|
| 604 |
+
matrices = d9.reshape(-1, 3, 3).permute(0, 2, 1)
|
| 605 |
+
u, s, vt = torch.linalg.svd(matrices) # torch.svd returns v instead of vt
|
| 606 |
+
det = torch.det(torch.matmul(u, vt))
|
| 607 |
+
vt[:, :, -1] *= det[:, None]
|
| 608 |
+
rot_matrices = torch.matmul(u, vt)
|
| 609 |
+
rot_matrices = rot_matrices.unflatten(0, d9.shape[:-1])
|
| 610 |
+
return rot_matrices
|
| 611 |
+
|
| 612 |
+
|
| 613 |
+
def matrix_to_rotation_9d(matrix: torch.Tensor) -> torch.Tensor:
|
| 614 |
+
""""
|
| 615 |
+
Args:
|
| 616 |
+
matrix: batch of rotation matrices of size (..., 3, 3)
|
| 617 |
+
Returns:
|
| 618 |
+
9D rotaion representation of size (..., 9)
|
| 619 |
+
"""
|
| 620 |
+
assert matrix.shape[-2:] == (3, 3)
|
| 621 |
+
return matrix.transpose(-2, -1).flatten(-2, -1)
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Model namespace for fastwam."""
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/__init__.py
ADDED
|
File without changes
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/action_dit.py
ADDED
|
@@ -0,0 +1,337 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
from typing import Any, Dict, Optional
|
| 5 |
+
|
| 6 |
+
from fastwam.utils.logging_config import get_logger
|
| 7 |
+
|
| 8 |
+
from .helpers.gradient import gradient_checkpoint_forward
|
| 9 |
+
from .wan_video_dit import (
|
| 10 |
+
DiTBlock,
|
| 11 |
+
sinusoidal_embedding_1d,
|
| 12 |
+
precompute_freqs_cis,
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
logger = get_logger(__name__)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class ActionHead(nn.Module):
|
| 19 |
+
def __init__(self, hidden_dim: int, out_dim: int, eps: float):
|
| 20 |
+
super().__init__()
|
| 21 |
+
self.norm = nn.LayerNorm(hidden_dim, eps=eps, elementwise_affine=False)
|
| 22 |
+
self.proj = nn.Linear(hidden_dim, out_dim)
|
| 23 |
+
self.modulation = nn.Parameter(torch.randn(1, 2, hidden_dim) / hidden_dim**0.5)
|
| 24 |
+
|
| 25 |
+
def forward(self, x: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
|
| 26 |
+
shift, scale = (self.modulation.to(dtype=t.dtype, device=t.device) + t.unsqueeze(1)).chunk(2, dim=1)
|
| 27 |
+
shift = shift.squeeze(1)
|
| 28 |
+
scale = scale.squeeze(1)
|
| 29 |
+
return self.proj(self.norm(x) * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1))
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class ActionDiT(nn.Module):
|
| 33 |
+
ACTION_BACKBONE_SKIP_PREFIXES = ("action_encoder.", "head.")
|
| 34 |
+
ACTION_BACKBONE_META_KEYS = (
|
| 35 |
+
"hidden_dim",
|
| 36 |
+
"ffn_dim",
|
| 37 |
+
"num_layers",
|
| 38 |
+
"num_heads",
|
| 39 |
+
"attn_head_dim",
|
| 40 |
+
"text_dim",
|
| 41 |
+
"freq_dim",
|
| 42 |
+
"eps",
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
def __init__(
|
| 46 |
+
self,
|
| 47 |
+
hidden_dim: int,
|
| 48 |
+
action_dim: int,
|
| 49 |
+
ffn_dim: int,
|
| 50 |
+
text_dim: int,
|
| 51 |
+
freq_dim: int,
|
| 52 |
+
eps: float,
|
| 53 |
+
num_heads: int,
|
| 54 |
+
attn_head_dim: int,
|
| 55 |
+
num_layers: int,
|
| 56 |
+
use_gradient_checkpointing: bool = False,
|
| 57 |
+
):
|
| 58 |
+
super().__init__()
|
| 59 |
+
self.hidden_dim = hidden_dim
|
| 60 |
+
self.action_dim = action_dim
|
| 61 |
+
self.ffn_dim = ffn_dim
|
| 62 |
+
self.text_dim = text_dim
|
| 63 |
+
self.freq_dim = freq_dim
|
| 64 |
+
self.num_heads = num_heads
|
| 65 |
+
self.attn_head_dim = attn_head_dim
|
| 66 |
+
|
| 67 |
+
if num_heads <= 0:
|
| 68 |
+
raise ValueError(f"`num_heads` must be > 0, got {num_heads}")
|
| 69 |
+
if attn_head_dim <= 0:
|
| 70 |
+
raise ValueError(f"`attn_head_dim` must be > 0, got {attn_head_dim}")
|
| 71 |
+
if attn_head_dim % 2 != 0:
|
| 72 |
+
raise ValueError(f"`attn_head_dim` must be even for RoPE, got {attn_head_dim}")
|
| 73 |
+
|
| 74 |
+
self.action_encoder = nn.Linear(action_dim, hidden_dim)
|
| 75 |
+
self.text_embedding = nn.Sequential(
|
| 76 |
+
nn.Linear(text_dim, hidden_dim),
|
| 77 |
+
nn.GELU(approximate="tanh"),
|
| 78 |
+
nn.Linear(hidden_dim, hidden_dim),
|
| 79 |
+
)
|
| 80 |
+
self.time_embedding = nn.Sequential(
|
| 81 |
+
nn.Linear(freq_dim, hidden_dim),
|
| 82 |
+
nn.SiLU(),
|
| 83 |
+
nn.Linear(hidden_dim, hidden_dim),
|
| 84 |
+
)
|
| 85 |
+
self.time_projection = nn.Sequential(nn.SiLU(), nn.Linear(hidden_dim, hidden_dim * 6))
|
| 86 |
+
self.blocks = nn.ModuleList(
|
| 87 |
+
[
|
| 88 |
+
DiTBlock(
|
| 89 |
+
hidden_dim=hidden_dim,
|
| 90 |
+
attn_head_dim=attn_head_dim,
|
| 91 |
+
num_heads=num_heads,
|
| 92 |
+
ffn_dim=ffn_dim,
|
| 93 |
+
eps=eps,
|
| 94 |
+
)
|
| 95 |
+
for _ in range(num_layers)
|
| 96 |
+
]
|
| 97 |
+
)
|
| 98 |
+
self.head = nn.Linear(hidden_dim, action_dim)
|
| 99 |
+
self.freqs = precompute_freqs_cis(attn_head_dim, end=1024)
|
| 100 |
+
|
| 101 |
+
self.use_gradient_checkpointing = use_gradient_checkpointing
|
| 102 |
+
|
| 103 |
+
@classmethod
|
| 104 |
+
def backbone_key_set(cls, keys) -> set[str]:
|
| 105 |
+
return {
|
| 106 |
+
key
|
| 107 |
+
for key in keys
|
| 108 |
+
if not any(key.startswith(prefix) for prefix in cls.ACTION_BACKBONE_SKIP_PREFIXES)
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
@classmethod
|
| 112 |
+
def from_pretrained(
|
| 113 |
+
cls,
|
| 114 |
+
action_dit_config: dict[str, Any],
|
| 115 |
+
action_dit_pretrained_path: str | None = None,
|
| 116 |
+
skip_dit_load_from_pretrain: bool = False,
|
| 117 |
+
device: str = "cuda",
|
| 118 |
+
torch_dtype: torch.dtype = torch.bfloat16,
|
| 119 |
+
) -> "ActionDiT":
|
| 120 |
+
if action_dit_config is None:
|
| 121 |
+
raise ValueError("`action_dit_config` is required for ActionDiT.from_pretrained().")
|
| 122 |
+
if skip_dit_load_from_pretrain:
|
| 123 |
+
logger.info(
|
| 124 |
+
"Skipping ActionDiT pretrained load (`skip_dit_load_from_pretrain=True`); "
|
| 125 |
+
"initializing action expert randomly and expecting checkpoint override."
|
| 126 |
+
)
|
| 127 |
+
return cls(**action_dit_config).to(device=device, dtype=torch_dtype)
|
| 128 |
+
if not action_dit_pretrained_path:
|
| 129 |
+
logger.info("No `action_dit_pretrained_path` provided, initializing ActionDiT with random weights.")
|
| 130 |
+
return cls(**action_dit_config).to(device=device, dtype=torch_dtype)
|
| 131 |
+
from pathlib import Path
|
| 132 |
+
p = Path(action_dit_pretrained_path)
|
| 133 |
+
if not p.is_absolute():
|
| 134 |
+
p = Path(__file__).resolve().parents[4] / p
|
| 135 |
+
action_dit_pretrained_path = str(p)
|
| 136 |
+
if not os.path.isfile(action_dit_pretrained_path):
|
| 137 |
+
raise FileNotFoundError(
|
| 138 |
+
f"`action_dit_pretrained_path` does not exist: {action_dit_pretrained_path}"
|
| 139 |
+
)
|
| 140 |
+
|
| 141 |
+
action_cfg = dict(action_dit_config)
|
| 142 |
+
action_expert = cls(**action_cfg).to(device=device, dtype=torch_dtype)
|
| 143 |
+
action_state = action_expert.state_dict()
|
| 144 |
+
expected_backbone_keys = cls.backbone_key_set(action_state.keys())
|
| 145 |
+
|
| 146 |
+
payload = torch.load(action_dit_pretrained_path, map_location="cpu")
|
| 147 |
+
if not isinstance(payload, dict):
|
| 148 |
+
raise ValueError(
|
| 149 |
+
f"Invalid action backbone payload type from {action_dit_pretrained_path}: {type(payload)}"
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
policy = payload.get("policy", {})
|
| 153 |
+
if policy:
|
| 154 |
+
logger.info(f"ActionDiT backbone payload policy: {policy}")
|
| 155 |
+
|
| 156 |
+
meta = payload.get("meta")
|
| 157 |
+
expected_meta = {
|
| 158 |
+
"hidden_dim": int(action_cfg["hidden_dim"]),
|
| 159 |
+
"ffn_dim": int(action_cfg["ffn_dim"]),
|
| 160 |
+
"num_layers": int(action_cfg["num_layers"]),
|
| 161 |
+
"num_heads": int(action_cfg["num_heads"]),
|
| 162 |
+
"attn_head_dim": int(action_cfg["attn_head_dim"]),
|
| 163 |
+
"text_dim": int(action_cfg["text_dim"]),
|
| 164 |
+
"freq_dim": int(action_cfg["freq_dim"]),
|
| 165 |
+
"eps": float(action_cfg["eps"]),
|
| 166 |
+
}
|
| 167 |
+
for key in cls.ACTION_BACKBONE_META_KEYS:
|
| 168 |
+
if key not in meta:
|
| 169 |
+
raise ValueError(f"`meta.{key}` missing in {action_dit_pretrained_path}")
|
| 170 |
+
expected_value = expected_meta[key]
|
| 171 |
+
got_value = meta[key]
|
| 172 |
+
if key == "eps":
|
| 173 |
+
if abs(float(got_value) - float(expected_value)) > 1e-12:
|
| 174 |
+
raise ValueError(
|
| 175 |
+
f"`meta.{key}` mismatch in {action_dit_pretrained_path}: "
|
| 176 |
+
f"expected {expected_value}, got {got_value}"
|
| 177 |
+
)
|
| 178 |
+
elif int(got_value) != int(expected_value):
|
| 179 |
+
raise ValueError(
|
| 180 |
+
f"`meta.{key}` mismatch in {action_dit_pretrained_path}: "
|
| 181 |
+
f"expected {expected_value}, got {got_value}"
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
backbone_state_dict = payload.get("backbone_state_dict")
|
| 185 |
+
if not isinstance(backbone_state_dict, dict):
|
| 186 |
+
raise ValueError(
|
| 187 |
+
f"`backbone_state_dict` must be a dict in {action_dit_pretrained_path}, "
|
| 188 |
+
f"got {type(backbone_state_dict)}"
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
provided_keys = set(backbone_state_dict.keys())
|
| 192 |
+
missing_keys = sorted(expected_backbone_keys - provided_keys)
|
| 193 |
+
unexpected_keys = sorted(provided_keys - expected_backbone_keys)
|
| 194 |
+
if missing_keys or unexpected_keys:
|
| 195 |
+
raise ValueError(
|
| 196 |
+
"Action backbone key mismatch in preprocessed payload. "
|
| 197 |
+
f"missing={missing_keys[:10]}{'...' if len(missing_keys) > 10 else ''}, "
|
| 198 |
+
f"unexpected={unexpected_keys[:10]}{'...' if len(unexpected_keys) > 10 else ''}"
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
merged_state = dict(action_state)
|
| 202 |
+
for key in expected_backbone_keys:
|
| 203 |
+
value = backbone_state_dict[key]
|
| 204 |
+
if not isinstance(value, torch.Tensor):
|
| 205 |
+
raise ValueError(
|
| 206 |
+
f"`backbone_state_dict[{key}]` must be torch.Tensor in {action_dit_pretrained_path}, "
|
| 207 |
+
f"got {type(value)}"
|
| 208 |
+
)
|
| 209 |
+
target = merged_state[key]
|
| 210 |
+
if tuple(value.shape) != tuple(target.shape):
|
| 211 |
+
raise ValueError(
|
| 212 |
+
f"Shape mismatch for `{key}` in {action_dit_pretrained_path}: "
|
| 213 |
+
f"expected {tuple(target.shape)}, got {tuple(value.shape)}"
|
| 214 |
+
)
|
| 215 |
+
merged_state[key] = value.to(device=target.device, dtype=target.dtype)
|
| 216 |
+
|
| 217 |
+
action_expert.load_state_dict(merged_state, strict=True)
|
| 218 |
+
logger.info(
|
| 219 |
+
"Loaded ActionDiT backbone from %s (keys=%d; random_kept_prefixes=%s).",
|
| 220 |
+
action_dit_pretrained_path,
|
| 221 |
+
len(expected_backbone_keys),
|
| 222 |
+
list(cls.ACTION_BACKBONE_SKIP_PREFIXES),
|
| 223 |
+
)
|
| 224 |
+
return action_expert.to(device=device, dtype=torch_dtype)
|
| 225 |
+
|
| 226 |
+
def pre_dit(
|
| 227 |
+
self,
|
| 228 |
+
action_tokens: torch.Tensor,
|
| 229 |
+
timestep: torch.Tensor,
|
| 230 |
+
context: torch.Tensor,
|
| 231 |
+
context_mask: Optional[torch.Tensor] = None,
|
| 232 |
+
) -> Dict[str, Any]:
|
| 233 |
+
if action_tokens.ndim != 3:
|
| 234 |
+
raise ValueError(
|
| 235 |
+
f"`action_tokens` must be 3D [B, T, action_dim], got shape {tuple(action_tokens.shape)}"
|
| 236 |
+
)
|
| 237 |
+
if action_tokens.shape[2] != self.action_dim:
|
| 238 |
+
raise ValueError(
|
| 239 |
+
f"`action_tokens` last dim must be {self.action_dim}, got {action_tokens.shape[2]}"
|
| 240 |
+
)
|
| 241 |
+
if timestep.ndim != 1:
|
| 242 |
+
raise ValueError(f"`timestep` must be 1D [B] or [1], got shape {tuple(timestep.shape)}")
|
| 243 |
+
if context.ndim != 3:
|
| 244 |
+
raise ValueError(
|
| 245 |
+
f"`context` must be 3D [B, L, D], got shape {tuple(context.shape)}"
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
batch_size = action_tokens.shape[0]
|
| 249 |
+
if context.shape[0] != batch_size:
|
| 250 |
+
raise ValueError(
|
| 251 |
+
f"Batch mismatch between action tokens and text context: {batch_size} vs {context.shape[0]}"
|
| 252 |
+
)
|
| 253 |
+
if timestep.shape[0] not in (1, batch_size):
|
| 254 |
+
raise ValueError(
|
| 255 |
+
f"`timestep` length must be 1 or batch_size({batch_size}), got {timestep.shape[0]}"
|
| 256 |
+
)
|
| 257 |
+
if timestep.shape[0] == 1 and batch_size > 1:
|
| 258 |
+
if self.training:
|
| 259 |
+
raise ValueError("During training, action timestep length must match batch_size.")
|
| 260 |
+
timestep = timestep.expand(batch_size)
|
| 261 |
+
|
| 262 |
+
if context_mask is None:
|
| 263 |
+
context_mask = torch.ones(
|
| 264 |
+
(batch_size, context.shape[1]), dtype=torch.bool, device=context.device
|
| 265 |
+
)
|
| 266 |
+
else:
|
| 267 |
+
if context_mask.ndim != 2:
|
| 268 |
+
raise ValueError(f"`context_mask` must be 2D [B, L], got shape {tuple(context_mask.shape)}")
|
| 269 |
+
if context_mask.shape[0] != batch_size or context_mask.shape[1] != context.shape[1]:
|
| 270 |
+
raise ValueError(
|
| 271 |
+
f"`context_mask` shape must match `context` shape [B, L], got {tuple(context_mask.shape)} vs {tuple(context.shape)}"
|
| 272 |
+
)
|
| 273 |
+
|
| 274 |
+
seq_len = action_tokens.shape[1]
|
| 275 |
+
if seq_len > self.freqs.shape[0]:
|
| 276 |
+
raise ValueError(
|
| 277 |
+
f"Action token length {seq_len} exceeds RoPE cache {self.freqs.shape[0]}."
|
| 278 |
+
)
|
| 279 |
+
|
| 280 |
+
t = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timestep))
|
| 281 |
+
t_mod = self.time_projection(t).unflatten(1, (6, self.hidden_dim))
|
| 282 |
+
|
| 283 |
+
tokens = self.action_encoder(action_tokens)
|
| 284 |
+
context_emb = self.text_embedding(context)
|
| 285 |
+
context_attn_mask = context_mask.unsqueeze(1).expand(-1, seq_len, -1)
|
| 286 |
+
freqs = self.freqs[:seq_len].view(seq_len, 1, -1).to(tokens.device)
|
| 287 |
+
|
| 288 |
+
return {
|
| 289 |
+
"tokens": tokens,
|
| 290 |
+
"freqs": freqs,
|
| 291 |
+
"t": t,
|
| 292 |
+
"t_mod": t_mod,
|
| 293 |
+
"context": context_emb,
|
| 294 |
+
"context_mask": context_attn_mask,
|
| 295 |
+
"meta": {
|
| 296 |
+
"batch_size": batch_size,
|
| 297 |
+
"seq_len": seq_len,
|
| 298 |
+
},
|
| 299 |
+
}
|
| 300 |
+
|
| 301 |
+
def post_dit(self, tokens: torch.Tensor, pre_state: Dict[str, Any]) -> torch.Tensor:
|
| 302 |
+
return self.head(tokens)
|
| 303 |
+
|
| 304 |
+
def forward(
|
| 305 |
+
self,
|
| 306 |
+
action_tokens: torch.Tensor,
|
| 307 |
+
timestep: torch.Tensor,
|
| 308 |
+
context: torch.Tensor,
|
| 309 |
+
context_mask: Optional[torch.Tensor] = None,
|
| 310 |
+
) -> torch.Tensor:
|
| 311 |
+
pre_state = self.pre_dit(
|
| 312 |
+
action_tokens=action_tokens,
|
| 313 |
+
timestep=timestep,
|
| 314 |
+
context=context,
|
| 315 |
+
context_mask=context_mask,
|
| 316 |
+
)
|
| 317 |
+
x = pre_state["tokens"]
|
| 318 |
+
context = pre_state["context"]
|
| 319 |
+
t_mod = pre_state["t_mod"]
|
| 320 |
+
freqs = pre_state["freqs"]
|
| 321 |
+
context_mask = pre_state["context_mask"]
|
| 322 |
+
|
| 323 |
+
for block in self.blocks:
|
| 324 |
+
if self.use_gradient_checkpointing:
|
| 325 |
+
x = gradient_checkpoint_forward(
|
| 326 |
+
block,
|
| 327 |
+
self.use_gradient_checkpointing,
|
| 328 |
+
x,
|
| 329 |
+
context,
|
| 330 |
+
t_mod,
|
| 331 |
+
freqs,
|
| 332 |
+
context_mask=context_mask,
|
| 333 |
+
)
|
| 334 |
+
else:
|
| 335 |
+
x = block(x, context, t_mod, freqs, context_mask=context_mask)
|
| 336 |
+
|
| 337 |
+
return self.post_dit(x, pre_state)
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/attention_masks.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Structured attention masks for video/action joint denoising.
|
| 2 |
+
|
| 3 |
+
Boolean masks follow PyTorch SDPA semantics: ``True`` means that the query is
|
| 4 |
+
allowed to attend to the key. Rows are queries and columns are keys.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def build_frame_causal_video_mask(
|
| 13 |
+
video_seq_len: int,
|
| 14 |
+
video_tokens_per_frame: int,
|
| 15 |
+
*,
|
| 16 |
+
device: torch.device | str | None = None,
|
| 17 |
+
) -> torch.Tensor:
|
| 18 |
+
"""Build block-causal video attention with bidirectional intra-frame attention.
|
| 19 |
+
|
| 20 |
+
Every token in frame ``i`` can see every token in frames ``0..i``. Thus
|
| 21 |
+
spatial tokens inside a frame are bidirectional, while information cannot
|
| 22 |
+
travel directly from a future frame to a past frame through this sub-mask.
|
| 23 |
+
"""
|
| 24 |
+
if video_seq_len <= 0:
|
| 25 |
+
raise ValueError(f"`video_seq_len` must be > 0, got {video_seq_len}")
|
| 26 |
+
if video_tokens_per_frame <= 0:
|
| 27 |
+
raise ValueError(
|
| 28 |
+
"`video_tokens_per_frame` must be > 0, "
|
| 29 |
+
f"got {video_tokens_per_frame}"
|
| 30 |
+
)
|
| 31 |
+
if video_seq_len % video_tokens_per_frame != 0:
|
| 32 |
+
raise ValueError(
|
| 33 |
+
"`video_seq_len` must be divisible by `video_tokens_per_frame`, "
|
| 34 |
+
f"got {video_seq_len} and {video_tokens_per_frame}"
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
num_frames = video_seq_len // video_tokens_per_frame
|
| 38 |
+
frame_ids = torch.arange(num_frames, device=device).repeat_interleave(
|
| 39 |
+
video_tokens_per_frame
|
| 40 |
+
)
|
| 41 |
+
query_frame = frame_ids[:, None]
|
| 42 |
+
key_frame = frame_ids[None, :]
|
| 43 |
+
return key_frame <= query_frame
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def build_joint_denoise_attention_mask(
|
| 47 |
+
video_seq_len: int,
|
| 48 |
+
action_seq_len: int,
|
| 49 |
+
video_tokens_per_frame: int,
|
| 50 |
+
*,
|
| 51 |
+
device: torch.device | str | None = None,
|
| 52 |
+
) -> torch.Tensor:
|
| 53 |
+
"""Build shared-self-attention visibility for joint video/action denoising.
|
| 54 |
+
|
| 55 |
+
The four query/key quadrants are:
|
| 56 |
+
|
| 57 |
+
* video -> video: frame causal, intra-frame bidirectional;
|
| 58 |
+
* video -> action: fully visible;
|
| 59 |
+
* action -> video: fully visible;
|
| 60 |
+
* action -> action: bidirectional.
|
| 61 |
+
|
| 62 |
+
This differs from upstream ``FastWAMJoint``, whose released mask permits
|
| 63 |
+
action -> video but blocks video -> action.
|
| 64 |
+
|
| 65 |
+
Full cross-modal visibility can indirectly carry future-video information
|
| 66 |
+
through action tokens in deeper layers. Consequently only the direct
|
| 67 |
+
video/video sub-mask is causal; the whole multimodal graph is not strictly
|
| 68 |
+
causal.
|
| 69 |
+
"""
|
| 70 |
+
if action_seq_len <= 0:
|
| 71 |
+
raise ValueError(f"`action_seq_len` must be > 0, got {action_seq_len}")
|
| 72 |
+
|
| 73 |
+
total_seq_len = video_seq_len + action_seq_len
|
| 74 |
+
mask = torch.ones(
|
| 75 |
+
(total_seq_len, total_seq_len), dtype=torch.bool, device=device
|
| 76 |
+
)
|
| 77 |
+
mask[:video_seq_len, :video_seq_len] = build_frame_causal_video_mask(
|
| 78 |
+
video_seq_len,
|
| 79 |
+
video_tokens_per_frame,
|
| 80 |
+
device=device,
|
| 81 |
+
)
|
| 82 |
+
return mask
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam.py
ADDED
|
@@ -0,0 +1,1122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, Optional, Sequence, Union
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
from PIL import Image
|
| 7 |
+
|
| 8 |
+
from fastwam.utils.logging_config import get_logger
|
| 9 |
+
|
| 10 |
+
from .action_dit import ActionDiT
|
| 11 |
+
from .helpers.loader import load_wan22_ti2v_5b_components
|
| 12 |
+
from .mot import MoT
|
| 13 |
+
from .schedulers.scheduler_continuous import WanContinuousFlowMatchScheduler
|
| 14 |
+
|
| 15 |
+
logger = get_logger(__name__)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class FastWAM(torch.nn.Module):
|
| 19 |
+
"""MoT world model with video/action experts."""
|
| 20 |
+
|
| 21 |
+
def __init__(
|
| 22 |
+
self,
|
| 23 |
+
video_expert,
|
| 24 |
+
action_expert: ActionDiT,
|
| 25 |
+
mot: MoT,
|
| 26 |
+
vae,
|
| 27 |
+
text_encoder=None,
|
| 28 |
+
tokenizer=None,
|
| 29 |
+
text_dim: Optional[int] = None,
|
| 30 |
+
proprio_dim: Optional[int] = None,
|
| 31 |
+
device: str = "cpu",
|
| 32 |
+
torch_dtype: torch.dtype = torch.float32,
|
| 33 |
+
video_train_shift: float = 5.0,
|
| 34 |
+
video_infer_shift: float = 5.0,
|
| 35 |
+
video_num_train_timesteps: int = 1000,
|
| 36 |
+
action_train_shift: float = 5.0,
|
| 37 |
+
action_infer_shift: float = 5.0,
|
| 38 |
+
action_num_train_timesteps: int = 1000,
|
| 39 |
+
loss_lambda_video: float = 1.0,
|
| 40 |
+
loss_lambda_action: float = 1.0,
|
| 41 |
+
):
|
| 42 |
+
super().__init__()
|
| 43 |
+
self.video_expert = video_expert
|
| 44 |
+
self.action_expert = action_expert
|
| 45 |
+
self.mot = mot
|
| 46 |
+
# Keep trainer compatibility: optimizer and freeze logic use `model.dit`.
|
| 47 |
+
self.dit = self.mot
|
| 48 |
+
|
| 49 |
+
self.vae = vae
|
| 50 |
+
self.text_encoder = text_encoder
|
| 51 |
+
self.tokenizer = tokenizer
|
| 52 |
+
if text_dim is None:
|
| 53 |
+
if self.text_encoder is None:
|
| 54 |
+
raise ValueError("`text_dim` is required when `text_encoder` is not loaded.")
|
| 55 |
+
text_dim = int(self.text_encoder.dim)
|
| 56 |
+
self.text_dim = int(text_dim)
|
| 57 |
+
self.proprio_dim = None if proprio_dim is None else int(proprio_dim)
|
| 58 |
+
if self.proprio_dim is not None:
|
| 59 |
+
self.proprio_encoder = nn.Linear(self.proprio_dim, self.text_dim).to(torch_dtype)
|
| 60 |
+
else:
|
| 61 |
+
self.proprio_encoder = None
|
| 62 |
+
|
| 63 |
+
self.train_video_scheduler = WanContinuousFlowMatchScheduler(
|
| 64 |
+
num_train_timesteps=video_num_train_timesteps,
|
| 65 |
+
shift=video_train_shift,
|
| 66 |
+
)
|
| 67 |
+
self.infer_video_scheduler = WanContinuousFlowMatchScheduler(
|
| 68 |
+
num_train_timesteps=video_num_train_timesteps,
|
| 69 |
+
shift=video_infer_shift,
|
| 70 |
+
)
|
| 71 |
+
self.train_action_scheduler = WanContinuousFlowMatchScheduler(
|
| 72 |
+
num_train_timesteps=action_num_train_timesteps,
|
| 73 |
+
shift=action_train_shift,
|
| 74 |
+
)
|
| 75 |
+
self.infer_action_scheduler = WanContinuousFlowMatchScheduler(
|
| 76 |
+
num_train_timesteps=action_num_train_timesteps,
|
| 77 |
+
shift=action_infer_shift,
|
| 78 |
+
)
|
| 79 |
+
# Optional aliases for consistency with Wan22Core naming.
|
| 80 |
+
self.train_scheduler = self.train_video_scheduler
|
| 81 |
+
self.infer_scheduler = self.infer_video_scheduler
|
| 82 |
+
|
| 83 |
+
self.device = torch.device(device)
|
| 84 |
+
self.torch_dtype = torch_dtype
|
| 85 |
+
self.loss_lambda_video = float(loss_lambda_video)
|
| 86 |
+
self.loss_lambda_action = float(loss_lambda_action)
|
| 87 |
+
|
| 88 |
+
self.to(self.device)
|
| 89 |
+
|
| 90 |
+
@classmethod
|
| 91 |
+
def from_wan22_pretrained(
|
| 92 |
+
cls,
|
| 93 |
+
device: str = "cuda",
|
| 94 |
+
torch_dtype: torch.dtype = torch.bfloat16,
|
| 95 |
+
model_id: str = "Wan-AI/Wan2.2-TI2V-5B",
|
| 96 |
+
tokenizer_model_id: str = "Wan-AI/Wan2.1-T2V-1.3B",
|
| 97 |
+
tokenizer_max_len: int = 512,
|
| 98 |
+
load_text_encoder: bool = True,
|
| 99 |
+
proprio_dim: Optional[int] = None,
|
| 100 |
+
redirect_common_files: bool = True,
|
| 101 |
+
video_dit_config: dict[str, Any] | None = None,
|
| 102 |
+
action_dit_config: dict[str, Any] | None = None,
|
| 103 |
+
action_dit_pretrained_path: str | None = None,
|
| 104 |
+
skip_dit_load_from_pretrain: bool = False,
|
| 105 |
+
mot_checkpoint_mixed_attn: bool = True,
|
| 106 |
+
video_train_shift: float = 5.0,
|
| 107 |
+
video_infer_shift: float = 5.0,
|
| 108 |
+
video_num_train_timesteps: int = 1000,
|
| 109 |
+
action_train_shift: float = 5.0,
|
| 110 |
+
action_infer_shift: float = 5.0,
|
| 111 |
+
action_num_train_timesteps: int = 1000,
|
| 112 |
+
loss_lambda_video: float = 1.0,
|
| 113 |
+
loss_lambda_action: float = 1.0,
|
| 114 |
+
):
|
| 115 |
+
if video_dit_config is None:
|
| 116 |
+
raise ValueError("`video_dit_config` is required for FastWAM.from_wan22_pretrained().")
|
| 117 |
+
if "text_dim" not in video_dit_config:
|
| 118 |
+
raise ValueError("`video_dit_config['text_dim']` is required for FastWAM.")
|
| 119 |
+
|
| 120 |
+
components = load_wan22_ti2v_5b_components(
|
| 121 |
+
device=device,
|
| 122 |
+
torch_dtype=torch_dtype,
|
| 123 |
+
model_id=model_id,
|
| 124 |
+
tokenizer_model_id=tokenizer_model_id,
|
| 125 |
+
tokenizer_max_len=tokenizer_max_len,
|
| 126 |
+
redirect_common_files=redirect_common_files,
|
| 127 |
+
dit_config=video_dit_config,
|
| 128 |
+
skip_dit_load_from_pretrain=skip_dit_load_from_pretrain,
|
| 129 |
+
load_text_encoder=load_text_encoder,
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
video_expert = components.dit
|
| 133 |
+
action_expert = ActionDiT.from_pretrained(
|
| 134 |
+
action_dit_config=action_dit_config,
|
| 135 |
+
action_dit_pretrained_path=action_dit_pretrained_path,
|
| 136 |
+
skip_dit_load_from_pretrain=skip_dit_load_from_pretrain,
|
| 137 |
+
device=device,
|
| 138 |
+
torch_dtype=torch_dtype,
|
| 139 |
+
)
|
| 140 |
+
if int(action_expert.num_heads) != int(video_expert.num_heads):
|
| 141 |
+
raise ValueError("ActionDiT `num_heads` must match video expert for MoT mixed attention.")
|
| 142 |
+
if int(action_expert.attn_head_dim) != int(video_expert.attn_head_dim):
|
| 143 |
+
raise ValueError("ActionDiT `attn_head_dim` must match video expert for MoT mixed attention.")
|
| 144 |
+
if int(len(action_expert.blocks)) != int(len(video_expert.blocks)):
|
| 145 |
+
raise ValueError("ActionDiT `num_layers` must match video expert.")
|
| 146 |
+
|
| 147 |
+
mot = MoT(
|
| 148 |
+
mixtures={"video": video_expert, "action": action_expert},
|
| 149 |
+
mot_checkpoint_mixed_attn=mot_checkpoint_mixed_attn,
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
model = cls(
|
| 153 |
+
video_expert=video_expert,
|
| 154 |
+
action_expert=action_expert,
|
| 155 |
+
mot=mot,
|
| 156 |
+
vae=components.vae,
|
| 157 |
+
text_encoder=components.text_encoder,
|
| 158 |
+
tokenizer=components.tokenizer,
|
| 159 |
+
text_dim=int(video_dit_config["text_dim"]),
|
| 160 |
+
proprio_dim=proprio_dim,
|
| 161 |
+
device=device,
|
| 162 |
+
torch_dtype=torch_dtype,
|
| 163 |
+
video_train_shift=video_train_shift,
|
| 164 |
+
video_infer_shift=video_infer_shift,
|
| 165 |
+
video_num_train_timesteps=video_num_train_timesteps,
|
| 166 |
+
action_train_shift=action_train_shift,
|
| 167 |
+
action_infer_shift=action_infer_shift,
|
| 168 |
+
action_num_train_timesteps=action_num_train_timesteps,
|
| 169 |
+
loss_lambda_video=loss_lambda_video,
|
| 170 |
+
loss_lambda_action=loss_lambda_action,
|
| 171 |
+
)
|
| 172 |
+
model.model_paths = {
|
| 173 |
+
"video_dit": components.dit_path,
|
| 174 |
+
"vae": components.vae_path,
|
| 175 |
+
"text_encoder": components.text_encoder_path,
|
| 176 |
+
"tokenizer": components.tokenizer_path,
|
| 177 |
+
"action_dit_backbone": (
|
| 178 |
+
"SKIPPED_PRETRAIN" if skip_dit_load_from_pretrain else action_dit_pretrained_path
|
| 179 |
+
),
|
| 180 |
+
}
|
| 181 |
+
return model
|
| 182 |
+
|
| 183 |
+
def to(self, *args, **kwargs):
|
| 184 |
+
super().to(*args, **kwargs)
|
| 185 |
+
self.mot.to(*args, **kwargs)
|
| 186 |
+
if self.text_encoder is not None:
|
| 187 |
+
self.text_encoder.to(*args, **kwargs)
|
| 188 |
+
self.vae.to(*args, **kwargs)
|
| 189 |
+
return self
|
| 190 |
+
|
| 191 |
+
@staticmethod
|
| 192 |
+
def _check_resize_height_width(height, width, num_frames):
|
| 193 |
+
if height % 16 != 0:
|
| 194 |
+
height = (height + 15) // 16 * 16
|
| 195 |
+
if width % 16 != 0:
|
| 196 |
+
width = (width + 15) // 16 * 16
|
| 197 |
+
if num_frames % 4 != 1:
|
| 198 |
+
num_frames = (num_frames + 3) // 4 * 4 + 1
|
| 199 |
+
return height, width, num_frames
|
| 200 |
+
|
| 201 |
+
@torch.no_grad()
|
| 202 |
+
def encode_prompt(self, prompt: Union[str, Sequence[str]]):
|
| 203 |
+
if self.text_encoder is None or self.tokenizer is None:
|
| 204 |
+
raise ValueError(
|
| 205 |
+
"Prompt encoding requires loaded text encoder/tokenizer. "
|
| 206 |
+
"Set `load_text_encoder=true` or provide precomputed `context/context_mask`."
|
| 207 |
+
)
|
| 208 |
+
ids, mask = self.tokenizer(prompt, return_mask=True, add_special_tokens=True)
|
| 209 |
+
ids = ids.to(self.device)
|
| 210 |
+
mask = mask.to(self.device, dtype=torch.bool)
|
| 211 |
+
prompt_emb = self.text_encoder(ids, mask)
|
| 212 |
+
# FIXME: original implementation's zero padding is visible in cross-attn.
|
| 213 |
+
seq_lens = mask.gt(0).sum(dim=1).long()
|
| 214 |
+
for i, v in enumerate(seq_lens):
|
| 215 |
+
prompt_emb[i, v:] = 0
|
| 216 |
+
mask = torch.ones_like(mask)
|
| 217 |
+
return prompt_emb.to(device=self.device), mask
|
| 218 |
+
|
| 219 |
+
def _append_proprio_to_context(
|
| 220 |
+
self,
|
| 221 |
+
context: torch.Tensor,
|
| 222 |
+
context_mask: torch.Tensor,
|
| 223 |
+
proprio: Optional[torch.Tensor],
|
| 224 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 225 |
+
if self.proprio_encoder is None or proprio is None:
|
| 226 |
+
return context, context_mask
|
| 227 |
+
if proprio.ndim != 2:
|
| 228 |
+
raise ValueError(f"`proprio` must be 2D [B, D], got shape {tuple(proprio.shape)}")
|
| 229 |
+
if self.proprio_dim is None or proprio.shape[1] != self.proprio_dim:
|
| 230 |
+
raise ValueError(
|
| 231 |
+
f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}"
|
| 232 |
+
)
|
| 233 |
+
proprio_token = self.proprio_encoder(
|
| 234 |
+
proprio.to(device=self.device, dtype=context.dtype).unsqueeze(1)
|
| 235 |
+
).to(dtype=context.dtype) # [B, 1, D]
|
| 236 |
+
proprio_mask = torch.ones((context_mask.shape[0], 1), dtype=torch.bool, device=context_mask.device)
|
| 237 |
+
return (
|
| 238 |
+
torch.cat([context, proprio_token], dim=1),
|
| 239 |
+
torch.cat([context_mask, proprio_mask], dim=1),
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
@torch.no_grad()
|
| 243 |
+
def _encode_video_latents(self, video_tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)):
|
| 244 |
+
z = self.vae.encode(
|
| 245 |
+
video_tensor,
|
| 246 |
+
device=self.device,
|
| 247 |
+
tiled=tiled,
|
| 248 |
+
tile_size=tile_size,
|
| 249 |
+
tile_stride=tile_stride,
|
| 250 |
+
)
|
| 251 |
+
return z
|
| 252 |
+
|
| 253 |
+
@torch.no_grad()
|
| 254 |
+
def _encode_input_image_latents_tensor(self, input_image: torch.Tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)):
|
| 255 |
+
if input_image.ndim == 3:
|
| 256 |
+
input_image = input_image.unsqueeze(0)
|
| 257 |
+
if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
|
| 258 |
+
raise ValueError(
|
| 259 |
+
f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
|
| 260 |
+
)
|
| 261 |
+
image = input_image.to(device=self.device)[0].unsqueeze(1)
|
| 262 |
+
z = self.vae.encode([image], device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride)
|
| 263 |
+
if isinstance(z, list):
|
| 264 |
+
z = z[0].unsqueeze(0)
|
| 265 |
+
return z
|
| 266 |
+
|
| 267 |
+
def _decode_latents(self, latents, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)):
|
| 268 |
+
video_tensor = self.vae.decode(latents, device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride)
|
| 269 |
+
video_tensor = video_tensor.squeeze(0).detach().float().clamp(-1, 1)
|
| 270 |
+
video_tensor = ((video_tensor + 1.0) * 127.5).to(torch.uint8).cpu()
|
| 271 |
+
frames = []
|
| 272 |
+
for t in range(video_tensor.shape[1]):
|
| 273 |
+
frame = video_tensor[:, t].permute(1, 2, 0).numpy()
|
| 274 |
+
frames.append(Image.fromarray(frame))
|
| 275 |
+
return frames
|
| 276 |
+
|
| 277 |
+
def build_inputs(self, sample, tiled: bool = False):
|
| 278 |
+
video = sample["video"]
|
| 279 |
+
if "context" not in sample or "context_mask" not in sample:
|
| 280 |
+
raise ValueError(
|
| 281 |
+
"FastWAM training requires `sample['context']` and `sample['context_mask']`."
|
| 282 |
+
)
|
| 283 |
+
context = sample["context"]
|
| 284 |
+
context_mask = sample["context_mask"]
|
| 285 |
+
proprio = sample.get("proprio", None)
|
| 286 |
+
if video.ndim != 5:
|
| 287 |
+
raise ValueError(f"`sample['video']` must be 5D [B, 3, T, H, W], got shape {tuple(video.shape)}")
|
| 288 |
+
if video.shape[1] != 3:
|
| 289 |
+
raise ValueError(f"`sample['video']` channel dimension must be 3, got shape {tuple(video.shape)}")
|
| 290 |
+
|
| 291 |
+
batch_size, _, num_frames, height, width = video.shape
|
| 292 |
+
if height % 16 != 0 or width % 16 != 0:
|
| 293 |
+
raise ValueError(
|
| 294 |
+
f"Video spatial dims must be multiples of 16, got H={height}, W={width}"
|
| 295 |
+
)
|
| 296 |
+
if num_frames % 4 != 1:
|
| 297 |
+
raise ValueError(f"Video T must satisfy T % 4 == 1, got T={num_frames}")
|
| 298 |
+
if num_frames <= 1:
|
| 299 |
+
raise ValueError(f"Video T must be > 1 for action-conditioned training, got T={num_frames}")
|
| 300 |
+
|
| 301 |
+
if "action" not in sample:
|
| 302 |
+
raise ValueError("`sample['action']` is required for FastWAM training.")
|
| 303 |
+
|
| 304 |
+
action = sample["action"]
|
| 305 |
+
if action.ndim != 3:
|
| 306 |
+
raise ValueError(f"`sample['action']` must be 3D [B, T, a_dim], got shape {tuple(action.shape)}")
|
| 307 |
+
action_horizon = int(action.shape[1])
|
| 308 |
+
if action_horizon % (num_frames - 1) != 0:
|
| 309 |
+
raise ValueError(
|
| 310 |
+
f"`sample['action']` temporal dimension must be divisible by video transitions ({num_frames - 1}), got {action_horizon}"
|
| 311 |
+
)
|
| 312 |
+
|
| 313 |
+
action_is_pad = sample.get("action_is_pad", None)
|
| 314 |
+
if action_is_pad is not None:
|
| 315 |
+
if action_is_pad.ndim != 2:
|
| 316 |
+
raise ValueError(
|
| 317 |
+
f"`sample['action_is_pad']` must be 2D [B, T], got shape {tuple(action_is_pad.shape)}"
|
| 318 |
+
)
|
| 319 |
+
if action_is_pad.shape[0] != batch_size or action_is_pad.shape[1] != action_horizon:
|
| 320 |
+
raise ValueError(
|
| 321 |
+
"`sample['action_is_pad']` shape mismatch: "
|
| 322 |
+
f"got {tuple(action_is_pad.shape)} vs expected ({batch_size}, {action_horizon})"
|
| 323 |
+
)
|
| 324 |
+
|
| 325 |
+
image_is_pad = sample.get("image_is_pad", None)
|
| 326 |
+
if image_is_pad is not None:
|
| 327 |
+
if image_is_pad.ndim != 2:
|
| 328 |
+
raise ValueError(
|
| 329 |
+
f"`sample['image_is_pad']` must be 2D [B, T], got shape {tuple(image_is_pad.shape)}"
|
| 330 |
+
)
|
| 331 |
+
if image_is_pad.shape[0] != batch_size or image_is_pad.shape[1] != num_frames:
|
| 332 |
+
raise ValueError(
|
| 333 |
+
"`sample['image_is_pad']` shape mismatch: "
|
| 334 |
+
f"got {tuple(image_is_pad.shape)} vs expected ({batch_size}, {num_frames})"
|
| 335 |
+
)
|
| 336 |
+
|
| 337 |
+
input_video = video.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
|
| 338 |
+
input_latents = self._encode_video_latents(input_video, tiled=tiled)
|
| 339 |
+
|
| 340 |
+
first_frame_latents = None
|
| 341 |
+
fuse_flag = False
|
| 342 |
+
if getattr(self.video_expert, "fuse_vae_embedding_in_latents", False):
|
| 343 |
+
first_frame_latents = input_latents[:, :, 0:1]
|
| 344 |
+
fuse_flag = True
|
| 345 |
+
|
| 346 |
+
if context.ndim != 3 or context_mask.ndim != 2:
|
| 347 |
+
raise ValueError(
|
| 348 |
+
f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}"
|
| 349 |
+
)
|
| 350 |
+
context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
|
| 351 |
+
context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True)
|
| 352 |
+
if self.proprio_encoder is not None:
|
| 353 |
+
if proprio is None:
|
| 354 |
+
raise ValueError("`sample['proprio']` is required when `proprio_dim` is enabled.")
|
| 355 |
+
if proprio.ndim != 3:
|
| 356 |
+
raise ValueError(f"`sample['proprio']` must be 3D [B, T, d], got shape {tuple(proprio.shape)}")
|
| 357 |
+
if proprio.shape[2] != self.proprio_dim:
|
| 358 |
+
raise ValueError(
|
| 359 |
+
f"`sample['proprio']` last dim must be {self.proprio_dim}, got {proprio.shape[2]}"
|
| 360 |
+
)
|
| 361 |
+
proprio = proprio[:, 0, :] # [B, D]
|
| 362 |
+
context, context_mask = self._append_proprio_to_context(
|
| 363 |
+
context=context,
|
| 364 |
+
context_mask=context_mask,
|
| 365 |
+
proprio=proprio.to(device=self.device, dtype=self.torch_dtype),
|
| 366 |
+
)
|
| 367 |
+
action = action.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
|
| 368 |
+
|
| 369 |
+
if action_is_pad is not None:
|
| 370 |
+
action_is_pad = action_is_pad.to(device=self.device, dtype=torch.bool, non_blocking=True)
|
| 371 |
+
if image_is_pad is not None:
|
| 372 |
+
image_is_pad = image_is_pad.to(device=self.device, dtype=torch.bool, non_blocking=True)
|
| 373 |
+
|
| 374 |
+
return {
|
| 375 |
+
"context": context,
|
| 376 |
+
"context_mask": context_mask,
|
| 377 |
+
"input_latents": input_latents,
|
| 378 |
+
"first_frame_latents": first_frame_latents,
|
| 379 |
+
"fuse_vae_embedding_in_latents": fuse_flag,
|
| 380 |
+
"action": action,
|
| 381 |
+
"action_is_pad": action_is_pad,
|
| 382 |
+
"image_is_pad": image_is_pad,
|
| 383 |
+
}
|
| 384 |
+
|
| 385 |
+
@torch.no_grad()
|
| 386 |
+
def _build_mot_attention_mask(
|
| 387 |
+
self,
|
| 388 |
+
video_seq_len: int,
|
| 389 |
+
action_seq_len: int,
|
| 390 |
+
video_tokens_per_frame: int,
|
| 391 |
+
device: torch.device,
|
| 392 |
+
) -> torch.Tensor:
|
| 393 |
+
total_seq_len = video_seq_len + action_seq_len
|
| 394 |
+
mask = torch.zeros((total_seq_len, total_seq_len), dtype=torch.bool, device=device)
|
| 395 |
+
|
| 396 |
+
# video -> video
|
| 397 |
+
mask[:video_seq_len, :video_seq_len] = self.video_expert.build_video_to_video_mask(
|
| 398 |
+
video_seq_len=video_seq_len,
|
| 399 |
+
video_tokens_per_frame=video_tokens_per_frame,
|
| 400 |
+
device=device,
|
| 401 |
+
)
|
| 402 |
+
# action -> action
|
| 403 |
+
mask[video_seq_len:, video_seq_len:] = True
|
| 404 |
+
# action -> first-frame video only
|
| 405 |
+
first_frame_tokens = min(video_tokens_per_frame, video_seq_len)
|
| 406 |
+
mask[video_seq_len:, :first_frame_tokens] = True
|
| 407 |
+
return mask
|
| 408 |
+
|
| 409 |
+
def _compute_video_loss_per_sample(
|
| 410 |
+
self,
|
| 411 |
+
pred_video: torch.Tensor,
|
| 412 |
+
target_video: torch.Tensor,
|
| 413 |
+
image_is_pad: Optional[torch.Tensor],
|
| 414 |
+
include_initial_video_step: bool,
|
| 415 |
+
) -> torch.Tensor:
|
| 416 |
+
video_loss_token = F.mse_loss(pred_video.float(), target_video.float(), reduction="none").mean(dim=(1, 3, 4))
|
| 417 |
+
if image_is_pad is None:
|
| 418 |
+
return video_loss_token.mean(dim=1)
|
| 419 |
+
|
| 420 |
+
temporal_factor = int(self.vae.temporal_downsample_factor)
|
| 421 |
+
if temporal_factor <= 0:
|
| 422 |
+
raise ValueError(f"`vae.temporal_downsample_factor` must be positive, got {temporal_factor}.")
|
| 423 |
+
if image_is_pad.shape[1] < 1:
|
| 424 |
+
raise ValueError("`image_is_pad` must contain at least one frame.")
|
| 425 |
+
if (image_is_pad.shape[1] - 1) % temporal_factor != 0:
|
| 426 |
+
raise ValueError(
|
| 427 |
+
"Cannot align `image_is_pad` with video latent steps: "
|
| 428 |
+
f"num_frames={image_is_pad.shape[1]}, temporal_downsample_factor={temporal_factor}."
|
| 429 |
+
)
|
| 430 |
+
|
| 431 |
+
tail_is_pad = image_is_pad[:, 1:]
|
| 432 |
+
latent_tail_is_pad = tail_is_pad.view(image_is_pad.shape[0], -1, temporal_factor).all(dim=2)
|
| 433 |
+
if include_initial_video_step:
|
| 434 |
+
video_is_pad = torch.cat([image_is_pad[:, :1], latent_tail_is_pad], dim=1)
|
| 435 |
+
else:
|
| 436 |
+
video_is_pad = latent_tail_is_pad
|
| 437 |
+
|
| 438 |
+
if video_is_pad.shape[1] != video_loss_token.shape[1]:
|
| 439 |
+
raise ValueError(
|
| 440 |
+
"Video-loss mask shape mismatch: "
|
| 441 |
+
f"mask steps={video_is_pad.shape[1]}, loss steps={video_loss_token.shape[1]}."
|
| 442 |
+
)
|
| 443 |
+
|
| 444 |
+
valid = (~video_is_pad).to(device=video_loss_token.device, dtype=video_loss_token.dtype)
|
| 445 |
+
valid_sum = valid.sum(dim=1).clamp(min=1.0)
|
| 446 |
+
return (video_loss_token * valid).sum(dim=1) / valid_sum
|
| 447 |
+
|
| 448 |
+
def training_loss(self, sample, tiled: bool = False):
|
| 449 |
+
inputs = self.build_inputs(sample, tiled=tiled)
|
| 450 |
+
input_latents = inputs["input_latents"]
|
| 451 |
+
batch_size = input_latents.shape[0]
|
| 452 |
+
context = inputs["context"]
|
| 453 |
+
context_mask = inputs["context_mask"]
|
| 454 |
+
action = inputs["action"]
|
| 455 |
+
action_is_pad = inputs["action_is_pad"]
|
| 456 |
+
image_is_pad = inputs["image_is_pad"]
|
| 457 |
+
|
| 458 |
+
noise_video = torch.randn_like(input_latents)
|
| 459 |
+
timestep_video = self.train_video_scheduler.sample_training_t(
|
| 460 |
+
batch_size=batch_size,
|
| 461 |
+
device=self.device,
|
| 462 |
+
dtype=input_latents.dtype,
|
| 463 |
+
)
|
| 464 |
+
latents = self.train_video_scheduler.add_noise(input_latents, noise_video, timestep_video)
|
| 465 |
+
target_video = self.train_video_scheduler.training_target(input_latents, noise_video, timestep_video)
|
| 466 |
+
|
| 467 |
+
if inputs["first_frame_latents"] is not None:
|
| 468 |
+
latents[:, :, 0:1] = inputs["first_frame_latents"]
|
| 469 |
+
|
| 470 |
+
noise_action = torch.randn_like(action)
|
| 471 |
+
timestep_action = self.train_action_scheduler.sample_training_t(
|
| 472 |
+
batch_size=batch_size,
|
| 473 |
+
device=self.device,
|
| 474 |
+
dtype=action.dtype,
|
| 475 |
+
)
|
| 476 |
+
noisy_action = self.train_action_scheduler.add_noise(action, noise_action, timestep_action)
|
| 477 |
+
target_action = self.train_action_scheduler.training_target(action, noise_action, timestep_action)
|
| 478 |
+
|
| 479 |
+
video_pre = self.video_expert.pre_dit(
|
| 480 |
+
x=latents,
|
| 481 |
+
timestep=timestep_video,
|
| 482 |
+
context=context,
|
| 483 |
+
context_mask=context_mask,
|
| 484 |
+
action=action,
|
| 485 |
+
fuse_vae_embedding_in_latents=inputs["fuse_vae_embedding_in_latents"],
|
| 486 |
+
)
|
| 487 |
+
|
| 488 |
+
action_pre = self.action_expert.pre_dit(
|
| 489 |
+
action_tokens=noisy_action,
|
| 490 |
+
timestep=timestep_action,
|
| 491 |
+
context=context,
|
| 492 |
+
context_mask=context_mask,
|
| 493 |
+
)
|
| 494 |
+
|
| 495 |
+
video_tokens = video_pre["tokens"]
|
| 496 |
+
action_tokens = action_pre["tokens"]
|
| 497 |
+
|
| 498 |
+
attention_mask = self._build_mot_attention_mask(
|
| 499 |
+
video_seq_len=video_tokens.shape[1],
|
| 500 |
+
action_seq_len=action_tokens.shape[1],
|
| 501 |
+
video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]),
|
| 502 |
+
device=video_tokens.device,
|
| 503 |
+
)
|
| 504 |
+
tokens_out = self.mot(
|
| 505 |
+
embeds_all={
|
| 506 |
+
"video": video_tokens,
|
| 507 |
+
"action": action_tokens,
|
| 508 |
+
},
|
| 509 |
+
attention_mask=attention_mask,
|
| 510 |
+
freqs_all={
|
| 511 |
+
"video": video_pre["freqs"],
|
| 512 |
+
"action": action_pre["freqs"],
|
| 513 |
+
},
|
| 514 |
+
context_all={
|
| 515 |
+
"video": {
|
| 516 |
+
"context": video_pre["context"],
|
| 517 |
+
"mask": video_pre["context_mask"],
|
| 518 |
+
},
|
| 519 |
+
"action": {
|
| 520 |
+
"context": action_pre["context"],
|
| 521 |
+
"mask": action_pre["context_mask"],
|
| 522 |
+
},
|
| 523 |
+
},
|
| 524 |
+
t_mod_all={
|
| 525 |
+
"video": video_pre["t_mod"],
|
| 526 |
+
"action": action_pre["t_mod"],
|
| 527 |
+
},
|
| 528 |
+
)
|
| 529 |
+
|
| 530 |
+
pred_video = self.video_expert.post_dit(tokens_out["video"], video_pre)
|
| 531 |
+
|
| 532 |
+
pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre)
|
| 533 |
+
|
| 534 |
+
include_initial_video_step = inputs["first_frame_latents"] is None
|
| 535 |
+
if inputs["first_frame_latents"] is not None:
|
| 536 |
+
pred_video = pred_video[:, :, 1:]
|
| 537 |
+
target_video = target_video[:, :, 1:]
|
| 538 |
+
|
| 539 |
+
loss_video_per_sample = self._compute_video_loss_per_sample(
|
| 540 |
+
pred_video=pred_video,
|
| 541 |
+
target_video=target_video,
|
| 542 |
+
image_is_pad=image_is_pad,
|
| 543 |
+
include_initial_video_step=include_initial_video_step,
|
| 544 |
+
)
|
| 545 |
+
video_weight = self.train_video_scheduler.training_weight(timestep_video).to(
|
| 546 |
+
loss_video_per_sample.device, dtype=loss_video_per_sample.dtype
|
| 547 |
+
)
|
| 548 |
+
loss_video = (loss_video_per_sample * video_weight).mean()
|
| 549 |
+
|
| 550 |
+
action_loss_token = F.mse_loss(pred_action.float(), target_action.float(), reduction="none").mean(dim=2) # [B, T]
|
| 551 |
+
if action_is_pad is not None:
|
| 552 |
+
valid = (~action_is_pad).to(device=action_loss_token.device, dtype=action_loss_token.dtype)
|
| 553 |
+
valid_sum = valid.sum(dim=1).clamp(min=1.0)
|
| 554 |
+
action_loss_per_sample = (action_loss_token * valid).sum(dim=1) / valid_sum
|
| 555 |
+
else:
|
| 556 |
+
action_loss_per_sample = action_loss_token.mean(dim=1)
|
| 557 |
+
|
| 558 |
+
action_weight = self.train_action_scheduler.training_weight(timestep_action).to(
|
| 559 |
+
action_loss_per_sample.device, dtype=action_loss_per_sample.dtype
|
| 560 |
+
)
|
| 561 |
+
loss_action = (action_loss_per_sample * action_weight).mean()
|
| 562 |
+
|
| 563 |
+
loss_total = self.loss_lambda_video * loss_video + self.loss_lambda_action * loss_action
|
| 564 |
+
loss_dict = {
|
| 565 |
+
"loss_video": self.loss_lambda_video * float(loss_video.detach().item()),
|
| 566 |
+
"loss_action": self.loss_lambda_action * float(loss_action.detach().item()),
|
| 567 |
+
}
|
| 568 |
+
return loss_total, loss_dict
|
| 569 |
+
|
| 570 |
+
@torch.no_grad()
|
| 571 |
+
def _predict_joint_noise(
|
| 572 |
+
self,
|
| 573 |
+
latents_video: torch.Tensor,
|
| 574 |
+
latents_action: torch.Tensor,
|
| 575 |
+
timestep_video: torch.Tensor,
|
| 576 |
+
timestep_action: torch.Tensor,
|
| 577 |
+
context: torch.Tensor,
|
| 578 |
+
context_mask: torch.Tensor,
|
| 579 |
+
fuse_vae_embedding_in_latents: bool,
|
| 580 |
+
gt_action: Optional[torch.Tensor] = None,
|
| 581 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 582 |
+
video_pre = self.video_expert.pre_dit(
|
| 583 |
+
x=latents_video,
|
| 584 |
+
timestep=timestep_video,
|
| 585 |
+
context=context,
|
| 586 |
+
context_mask=context_mask,
|
| 587 |
+
action=gt_action,
|
| 588 |
+
fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents,
|
| 589 |
+
)
|
| 590 |
+
action_pre = self.action_expert.pre_dit(
|
| 591 |
+
action_tokens=latents_action,
|
| 592 |
+
timestep=timestep_action,
|
| 593 |
+
context=context,
|
| 594 |
+
context_mask=context_mask,
|
| 595 |
+
)
|
| 596 |
+
|
| 597 |
+
attention_mask = self._build_mot_attention_mask(
|
| 598 |
+
video_seq_len=video_pre["tokens"].shape[1],
|
| 599 |
+
action_seq_len=action_pre["tokens"].shape[1],
|
| 600 |
+
video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]),
|
| 601 |
+
device=video_pre["tokens"].device,
|
| 602 |
+
)
|
| 603 |
+
|
| 604 |
+
tokens_out = self.mot(
|
| 605 |
+
embeds_all={
|
| 606 |
+
"video": video_pre["tokens"],
|
| 607 |
+
"action": action_pre["tokens"],
|
| 608 |
+
},
|
| 609 |
+
attention_mask=attention_mask,
|
| 610 |
+
freqs_all={
|
| 611 |
+
"video": video_pre["freqs"],
|
| 612 |
+
"action": action_pre["freqs"],
|
| 613 |
+
},
|
| 614 |
+
context_all={
|
| 615 |
+
"video": {
|
| 616 |
+
"context": video_pre["context"],
|
| 617 |
+
"mask": video_pre["context_mask"],
|
| 618 |
+
},
|
| 619 |
+
"action": {
|
| 620 |
+
"context": action_pre["context"],
|
| 621 |
+
"mask": action_pre["context_mask"],
|
| 622 |
+
},
|
| 623 |
+
},
|
| 624 |
+
t_mod_all={
|
| 625 |
+
"video": video_pre["t_mod"],
|
| 626 |
+
"action": action_pre["t_mod"],
|
| 627 |
+
},
|
| 628 |
+
)
|
| 629 |
+
|
| 630 |
+
pred_video = self.video_expert.post_dit(tokens_out["video"], video_pre)
|
| 631 |
+
pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre)
|
| 632 |
+
return pred_video, pred_action
|
| 633 |
+
|
| 634 |
+
@torch.no_grad()
|
| 635 |
+
def _predict_action_noise(
|
| 636 |
+
self,
|
| 637 |
+
first_frame_latents: torch.Tensor,
|
| 638 |
+
latents_action: torch.Tensor,
|
| 639 |
+
timestep_action: torch.Tensor,
|
| 640 |
+
context: torch.Tensor,
|
| 641 |
+
context_mask: torch.Tensor,
|
| 642 |
+
fuse_vae_embedding_in_latents: bool,
|
| 643 |
+
) -> torch.Tensor:
|
| 644 |
+
timestep_video = torch.zeros_like(timestep_action, dtype=first_frame_latents.dtype, device=self.device)
|
| 645 |
+
video_pre = self.video_expert.pre_dit(
|
| 646 |
+
x=first_frame_latents,
|
| 647 |
+
timestep=timestep_video,
|
| 648 |
+
context=context,
|
| 649 |
+
context_mask=context_mask,
|
| 650 |
+
action=None,
|
| 651 |
+
fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents,
|
| 652 |
+
)
|
| 653 |
+
action_pre = self.action_expert.pre_dit(
|
| 654 |
+
action_tokens=latents_action,
|
| 655 |
+
timestep=timestep_action,
|
| 656 |
+
context=context,
|
| 657 |
+
context_mask=context_mask,
|
| 658 |
+
)
|
| 659 |
+
|
| 660 |
+
attention_mask = self._build_mot_attention_mask(
|
| 661 |
+
video_seq_len=video_pre["tokens"].shape[1],
|
| 662 |
+
action_seq_len=action_pre["tokens"].shape[1],
|
| 663 |
+
video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]),
|
| 664 |
+
device=video_pre["tokens"].device,
|
| 665 |
+
)
|
| 666 |
+
tokens_out = self.mot(
|
| 667 |
+
embeds_all={
|
| 668 |
+
"video": video_pre["tokens"],
|
| 669 |
+
"action": action_pre["tokens"],
|
| 670 |
+
},
|
| 671 |
+
attention_mask=attention_mask,
|
| 672 |
+
freqs_all={
|
| 673 |
+
"video": video_pre["freqs"],
|
| 674 |
+
"action": action_pre["freqs"],
|
| 675 |
+
},
|
| 676 |
+
context_all={
|
| 677 |
+
"video": {
|
| 678 |
+
"context": video_pre["context"],
|
| 679 |
+
"mask": video_pre["context_mask"],
|
| 680 |
+
},
|
| 681 |
+
"action": {
|
| 682 |
+
"context": action_pre["context"],
|
| 683 |
+
"mask": action_pre["context_mask"],
|
| 684 |
+
},
|
| 685 |
+
},
|
| 686 |
+
t_mod_all={
|
| 687 |
+
"video": video_pre["t_mod"],
|
| 688 |
+
"action": action_pre["t_mod"],
|
| 689 |
+
},
|
| 690 |
+
)
|
| 691 |
+
pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre)
|
| 692 |
+
return pred_action
|
| 693 |
+
|
| 694 |
+
@torch.no_grad()
|
| 695 |
+
def _predict_action_noise_with_cache(
|
| 696 |
+
self,
|
| 697 |
+
latents_action: torch.Tensor,
|
| 698 |
+
timestep_action: torch.Tensor,
|
| 699 |
+
context: torch.Tensor,
|
| 700 |
+
context_mask: torch.Tensor,
|
| 701 |
+
video_kv_cache: list[dict[str, torch.Tensor]],
|
| 702 |
+
attention_mask: torch.Tensor,
|
| 703 |
+
video_seq_len: int,
|
| 704 |
+
) -> torch.Tensor:
|
| 705 |
+
action_pre = self.action_expert.pre_dit(
|
| 706 |
+
action_tokens=latents_action,
|
| 707 |
+
timestep=timestep_action,
|
| 708 |
+
context=context,
|
| 709 |
+
context_mask=context_mask,
|
| 710 |
+
)
|
| 711 |
+
action_tokens = self.mot.forward_action_with_video_cache(
|
| 712 |
+
action_tokens=action_pre["tokens"],
|
| 713 |
+
action_freqs=action_pre["freqs"],
|
| 714 |
+
action_t_mod=action_pre["t_mod"],
|
| 715 |
+
action_context_payload={
|
| 716 |
+
"context": action_pre["context"],
|
| 717 |
+
"mask": action_pre["context_mask"],
|
| 718 |
+
},
|
| 719 |
+
video_kv_cache=video_kv_cache,
|
| 720 |
+
attention_mask=attention_mask,
|
| 721 |
+
video_seq_len=video_seq_len,
|
| 722 |
+
)
|
| 723 |
+
return self.action_expert.post_dit(action_tokens, action_pre)
|
| 724 |
+
|
| 725 |
+
@torch.no_grad()
|
| 726 |
+
def infer_joint(
|
| 727 |
+
self,
|
| 728 |
+
prompt: Optional[str],
|
| 729 |
+
input_image: torch.Tensor,
|
| 730 |
+
num_video_frames: int,
|
| 731 |
+
action_horizon: int,
|
| 732 |
+
action: Optional[torch.Tensor] = None, # NOTE: this is gt action for conditioning videos, not for action expert
|
| 733 |
+
proprio: Optional[torch.Tensor] = None,
|
| 734 |
+
context: Optional[torch.Tensor] = None,
|
| 735 |
+
context_mask: Optional[torch.Tensor] = None,
|
| 736 |
+
negative_prompt: Optional[str] = None,
|
| 737 |
+
text_cfg_scale: float = 1.0,
|
| 738 |
+
num_inference_steps: int = 20,
|
| 739 |
+
sigma_shift: Optional[float] = None,
|
| 740 |
+
seed: Optional[int] = None,
|
| 741 |
+
rand_device: str = "cpu",
|
| 742 |
+
tiled: bool = False,
|
| 743 |
+
test_action_with_infer_action: bool = True,
|
| 744 |
+
) -> dict[str, Any]:
|
| 745 |
+
self.eval()
|
| 746 |
+
if test_action_with_infer_action:
|
| 747 |
+
if seed is None:
|
| 748 |
+
raise ValueError("`test_action_with_infer_action=True` requires non-null `seed`.")
|
| 749 |
+
action_only_out = self.infer_action(
|
| 750 |
+
prompt=prompt,
|
| 751 |
+
input_image=input_image.clone(),
|
| 752 |
+
action_horizon=action_horizon,
|
| 753 |
+
context=context.clone() if context is not None else None,
|
| 754 |
+
context_mask=context_mask.clone() if context_mask is not None else None,
|
| 755 |
+
num_inference_steps=num_inference_steps,
|
| 756 |
+
sigma_shift=sigma_shift,
|
| 757 |
+
seed=seed,
|
| 758 |
+
rand_device=rand_device,
|
| 759 |
+
tiled=tiled,
|
| 760 |
+
proprio=proprio.clone() if proprio is not None else None,
|
| 761 |
+
)["action"]
|
| 762 |
+
|
| 763 |
+
if input_image.ndim == 3:
|
| 764 |
+
input_image = input_image.unsqueeze(0)
|
| 765 |
+
if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
|
| 766 |
+
raise ValueError(
|
| 767 |
+
f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
|
| 768 |
+
)
|
| 769 |
+
_, _, height, width = input_image.shape
|
| 770 |
+
checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_video_frames)
|
| 771 |
+
if (checked_h, checked_w) != (height, width):
|
| 772 |
+
raise ValueError(
|
| 773 |
+
f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})"
|
| 774 |
+
)
|
| 775 |
+
if checked_t != num_video_frames:
|
| 776 |
+
raise ValueError(
|
| 777 |
+
f"`num_video_frames` must satisfy T % 4 == 1, got {num_video_frames}"
|
| 778 |
+
)
|
| 779 |
+
if action is not None:
|
| 780 |
+
if action.ndim == 2:
|
| 781 |
+
action = action.unsqueeze(0)
|
| 782 |
+
if action.ndim != 3 or action.shape[0] != 1 or action.shape[1] != action_horizon:
|
| 783 |
+
# NOTE: This enforces action condition to have the same shape as action horizon to predict, which may be unnecessary
|
| 784 |
+
raise ValueError(
|
| 785 |
+
f"`action` must have shape [1, T, a_dim] or [T, a_dim], got {tuple(action.shape)} with action_horizon={action_horizon}"
|
| 786 |
+
)
|
| 787 |
+
action = action.to(device=self.device, dtype=self.torch_dtype)
|
| 788 |
+
if proprio is not None:
|
| 789 |
+
if self.proprio_dim is None:
|
| 790 |
+
raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.")
|
| 791 |
+
if proprio.ndim == 1:
|
| 792 |
+
proprio = proprio.unsqueeze(0)
|
| 793 |
+
elif proprio.ndim == 2 and proprio.shape[0] == 1:
|
| 794 |
+
pass
|
| 795 |
+
else:
|
| 796 |
+
raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}")
|
| 797 |
+
if proprio.shape[1] != self.proprio_dim:
|
| 798 |
+
raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}")
|
| 799 |
+
proprio = proprio.to(device=self.device, dtype=self.torch_dtype)
|
| 800 |
+
|
| 801 |
+
latent_t = (num_video_frames - 1) // self.vae.temporal_downsample_factor + 1
|
| 802 |
+
latent_h = height // self.vae.upsampling_factor
|
| 803 |
+
latent_w = width // self.vae.upsampling_factor
|
| 804 |
+
|
| 805 |
+
video_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
|
| 806 |
+
action_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
|
| 807 |
+
latents_video = torch.randn(
|
| 808 |
+
(1, self.vae.model.z_dim, latent_t, latent_h, latent_w),
|
| 809 |
+
generator=video_generator,
|
| 810 |
+
device=rand_device,
|
| 811 |
+
dtype=torch.float32,
|
| 812 |
+
).to(device=self.device, dtype=self.torch_dtype)
|
| 813 |
+
latents_action = torch.randn(
|
| 814 |
+
(1, action_horizon, self.action_expert.action_dim),
|
| 815 |
+
generator=action_generator,
|
| 816 |
+
device=rand_device,
|
| 817 |
+
dtype=torch.float32,
|
| 818 |
+
).to(device=self.device, dtype=self.torch_dtype)
|
| 819 |
+
|
| 820 |
+
input_image = input_image.to(device=self.device, dtype=self.torch_dtype)
|
| 821 |
+
first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled)
|
| 822 |
+
latents_video[:, :, 0:1] = first_frame_latents.clone()
|
| 823 |
+
fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False))
|
| 824 |
+
|
| 825 |
+
use_prompt = prompt is not None
|
| 826 |
+
use_context = context is not None or context_mask is not None
|
| 827 |
+
if use_prompt and use_context:
|
| 828 |
+
raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.")
|
| 829 |
+
if not use_prompt and not use_context:
|
| 830 |
+
raise ValueError("Either `prompt` or both `context/context_mask` must be provided.")
|
| 831 |
+
|
| 832 |
+
if use_prompt:
|
| 833 |
+
context, context_mask = self.encode_prompt(prompt)
|
| 834 |
+
else:
|
| 835 |
+
if context is None or context_mask is None:
|
| 836 |
+
raise ValueError("`context` and `context_mask` must be both provided together.")
|
| 837 |
+
if context.ndim == 2:
|
| 838 |
+
context = context.unsqueeze(0)
|
| 839 |
+
if context_mask.ndim == 1:
|
| 840 |
+
context_mask = context_mask.unsqueeze(0)
|
| 841 |
+
if context.ndim != 3 or context_mask.ndim != 2:
|
| 842 |
+
raise ValueError(
|
| 843 |
+
f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}"
|
| 844 |
+
)
|
| 845 |
+
context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
|
| 846 |
+
context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True)
|
| 847 |
+
if proprio is not None:
|
| 848 |
+
context, context_mask = self._append_proprio_to_context(
|
| 849 |
+
context=context,
|
| 850 |
+
context_mask=context_mask,
|
| 851 |
+
proprio=proprio,
|
| 852 |
+
)
|
| 853 |
+
|
| 854 |
+
infer_timesteps_video, infer_deltas_video = self.infer_video_scheduler.build_inference_schedule(
|
| 855 |
+
num_inference_steps=num_inference_steps,
|
| 856 |
+
device=self.device,
|
| 857 |
+
dtype=latents_video.dtype,
|
| 858 |
+
shift_override=sigma_shift,
|
| 859 |
+
)
|
| 860 |
+
infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule(
|
| 861 |
+
num_inference_steps=num_inference_steps,
|
| 862 |
+
device=self.device,
|
| 863 |
+
dtype=latents_action.dtype,
|
| 864 |
+
shift_override=sigma_shift,
|
| 865 |
+
)
|
| 866 |
+
for step_t_video, step_delta_video, step_t_action, step_delta_action in zip(
|
| 867 |
+
infer_timesteps_video,
|
| 868 |
+
infer_deltas_video,
|
| 869 |
+
infer_timesteps_action,
|
| 870 |
+
infer_deltas_action,
|
| 871 |
+
):
|
| 872 |
+
timestep_video = step_t_video.unsqueeze(0).to(dtype=latents_video.dtype, device=self.device)
|
| 873 |
+
timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device)
|
| 874 |
+
|
| 875 |
+
pred_video_posi, pred_action_posi = self._predict_joint_noise(
|
| 876 |
+
latents_video=latents_video,
|
| 877 |
+
latents_action=latents_action,
|
| 878 |
+
timestep_video=timestep_video,
|
| 879 |
+
timestep_action=timestep_action,
|
| 880 |
+
context=context,
|
| 881 |
+
context_mask=context_mask,
|
| 882 |
+
fuse_vae_embedding_in_latents=fuse_flag,
|
| 883 |
+
gt_action=action,
|
| 884 |
+
)
|
| 885 |
+
pred_video = pred_video_posi
|
| 886 |
+
pred_action = pred_action_posi
|
| 887 |
+
|
| 888 |
+
latents_video = self.infer_video_scheduler.step(pred_video, step_delta_video, latents_video)
|
| 889 |
+
latents_action = self.infer_action_scheduler.step(pred_action, step_delta_action, latents_action)
|
| 890 |
+
latents_video[:, :, 0:1] = first_frame_latents.clone()
|
| 891 |
+
|
| 892 |
+
action_out = latents_action[0].detach().to(device="cpu", dtype=torch.float32)
|
| 893 |
+
if test_action_with_infer_action:
|
| 894 |
+
if not torch.allclose(action_out, action_only_out, atol=1e-2, rtol=1e-2):
|
| 895 |
+
max_abs_diff = (action_out - action_only_out).abs().max().item()
|
| 896 |
+
logger.warning(
|
| 897 |
+
f"Action from infer_joint and infer_action differ with max abs diff {max_abs_diff:.6f}. "
|
| 898 |
+
)
|
| 899 |
+
|
| 900 |
+
return {
|
| 901 |
+
"video": self._decode_latents(latents_video, tiled=tiled),
|
| 902 |
+
"action": action_out,
|
| 903 |
+
}
|
| 904 |
+
|
| 905 |
+
@torch.no_grad()
|
| 906 |
+
def infer_action(
|
| 907 |
+
self,
|
| 908 |
+
prompt: Optional[str],
|
| 909 |
+
input_image: torch.Tensor,
|
| 910 |
+
action_horizon: int,
|
| 911 |
+
proprio: Optional[torch.Tensor] = None,
|
| 912 |
+
context: Optional[torch.Tensor] = None,
|
| 913 |
+
context_mask: Optional[torch.Tensor] = None,
|
| 914 |
+
negative_prompt: Optional[str] = None,
|
| 915 |
+
text_cfg_scale: float = 1.0,
|
| 916 |
+
num_inference_steps: int = 20,
|
| 917 |
+
sigma_shift: Optional[float] = None,
|
| 918 |
+
seed: Optional[int] = None,
|
| 919 |
+
rand_device: str = "cpu",
|
| 920 |
+
tiled: bool = False,
|
| 921 |
+
) -> dict[str, Any]:
|
| 922 |
+
self.eval()
|
| 923 |
+
if str(getattr(self.video_expert, "video_attention_mask_mode", "")) != "first_frame_causal":
|
| 924 |
+
raise ValueError(
|
| 925 |
+
"`infer_action` requires `video_attention_mask_mode='first_frame_causal'`."
|
| 926 |
+
)
|
| 927 |
+
|
| 928 |
+
if input_image.ndim == 3:
|
| 929 |
+
input_image = input_image.unsqueeze(0)
|
| 930 |
+
if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
|
| 931 |
+
raise ValueError(
|
| 932 |
+
f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
|
| 933 |
+
)
|
| 934 |
+
_, _, height, width = input_image.shape
|
| 935 |
+
if height % 16 != 0 or width % 16 != 0:
|
| 936 |
+
raise ValueError(
|
| 937 |
+
f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})"
|
| 938 |
+
)
|
| 939 |
+
if proprio is not None:
|
| 940 |
+
if self.proprio_dim is None:
|
| 941 |
+
raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.")
|
| 942 |
+
if proprio.ndim == 1:
|
| 943 |
+
proprio = proprio.unsqueeze(0)
|
| 944 |
+
elif proprio.ndim == 2 and proprio.shape[0] == 1:
|
| 945 |
+
pass
|
| 946 |
+
else:
|
| 947 |
+
raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}")
|
| 948 |
+
if proprio.shape[1] != self.proprio_dim:
|
| 949 |
+
raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}")
|
| 950 |
+
proprio = proprio.to(device=self.device, dtype=self.torch_dtype)
|
| 951 |
+
|
| 952 |
+
generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
|
| 953 |
+
latents_action = torch.randn(
|
| 954 |
+
(1, action_horizon, self.action_expert.action_dim),
|
| 955 |
+
generator=generator,
|
| 956 |
+
device=rand_device,
|
| 957 |
+
dtype=torch.float32,
|
| 958 |
+
).to(device=self.device, dtype=self.torch_dtype)
|
| 959 |
+
|
| 960 |
+
input_image = input_image.to(device=self.device, dtype=self.torch_dtype)
|
| 961 |
+
first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled)
|
| 962 |
+
fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False))
|
| 963 |
+
|
| 964 |
+
use_prompt = prompt is not None
|
| 965 |
+
use_context = context is not None or context_mask is not None
|
| 966 |
+
if use_prompt and use_context:
|
| 967 |
+
raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.")
|
| 968 |
+
if not use_prompt and not use_context:
|
| 969 |
+
raise ValueError("Either `prompt` or both `context/context_mask` must be provided.")
|
| 970 |
+
|
| 971 |
+
if use_prompt:
|
| 972 |
+
context, context_mask = self.encode_prompt(prompt)
|
| 973 |
+
else:
|
| 974 |
+
if context is None or context_mask is None:
|
| 975 |
+
raise ValueError("`context` and `context_mask` must be both provided together.")
|
| 976 |
+
if context.ndim == 2:
|
| 977 |
+
context = context.unsqueeze(0)
|
| 978 |
+
if context_mask.ndim == 1:
|
| 979 |
+
context_mask = context_mask.unsqueeze(0)
|
| 980 |
+
if context.ndim != 3 or context_mask.ndim != 2:
|
| 981 |
+
raise ValueError(
|
| 982 |
+
f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}"
|
| 983 |
+
)
|
| 984 |
+
context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
|
| 985 |
+
context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True)
|
| 986 |
+
if proprio is not None:
|
| 987 |
+
context, context_mask = self._append_proprio_to_context(
|
| 988 |
+
context=context,
|
| 989 |
+
context_mask=context_mask,
|
| 990 |
+
proprio=proprio,
|
| 991 |
+
)
|
| 992 |
+
|
| 993 |
+
timestep_video = torch.zeros(
|
| 994 |
+
(first_frame_latents.shape[0],),
|
| 995 |
+
dtype=first_frame_latents.dtype,
|
| 996 |
+
device=self.device,
|
| 997 |
+
)
|
| 998 |
+
video_pre = self.video_expert.pre_dit(
|
| 999 |
+
x=first_frame_latents,
|
| 1000 |
+
timestep=timestep_video,
|
| 1001 |
+
context=context,
|
| 1002 |
+
context_mask=context_mask,
|
| 1003 |
+
action=None,
|
| 1004 |
+
fuse_vae_embedding_in_latents=fuse_flag,
|
| 1005 |
+
)
|
| 1006 |
+
video_seq_len = int(video_pre["tokens"].shape[1])
|
| 1007 |
+
attention_mask = self._build_mot_attention_mask(
|
| 1008 |
+
video_seq_len=video_seq_len,
|
| 1009 |
+
action_seq_len=latents_action.shape[1],
|
| 1010 |
+
video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]),
|
| 1011 |
+
device=video_pre["tokens"].device,
|
| 1012 |
+
)
|
| 1013 |
+
video_kv_cache = self.mot.prefill_video_cache(
|
| 1014 |
+
video_tokens=video_pre["tokens"],
|
| 1015 |
+
video_freqs=video_pre["freqs"],
|
| 1016 |
+
video_t_mod=video_pre["t_mod"],
|
| 1017 |
+
video_context_payload={
|
| 1018 |
+
"context": video_pre["context"],
|
| 1019 |
+
"mask": video_pre["context_mask"],
|
| 1020 |
+
},
|
| 1021 |
+
video_attention_mask=attention_mask[:video_seq_len, :video_seq_len],
|
| 1022 |
+
)
|
| 1023 |
+
|
| 1024 |
+
infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule(
|
| 1025 |
+
num_inference_steps=num_inference_steps,
|
| 1026 |
+
device=self.device,
|
| 1027 |
+
dtype=latents_action.dtype,
|
| 1028 |
+
shift_override=sigma_shift,
|
| 1029 |
+
)
|
| 1030 |
+
for step_t_action, step_delta_action in zip(infer_timesteps_action, infer_deltas_action):
|
| 1031 |
+
timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device)
|
| 1032 |
+
|
| 1033 |
+
pred_action_posi = self._predict_action_noise_with_cache(
|
| 1034 |
+
latents_action=latents_action,
|
| 1035 |
+
timestep_action=timestep_action,
|
| 1036 |
+
context=context,
|
| 1037 |
+
context_mask=context_mask,
|
| 1038 |
+
video_kv_cache=video_kv_cache,
|
| 1039 |
+
attention_mask=attention_mask,
|
| 1040 |
+
video_seq_len=video_seq_len,
|
| 1041 |
+
)
|
| 1042 |
+
pred_action = pred_action_posi
|
| 1043 |
+
|
| 1044 |
+
latents_action = self.infer_action_scheduler.step(pred_action, step_delta_action, latents_action)
|
| 1045 |
+
|
| 1046 |
+
return {
|
| 1047 |
+
"action": latents_action[0].detach().to(device="cpu", dtype=torch.float32),
|
| 1048 |
+
}
|
| 1049 |
+
|
| 1050 |
+
@torch.no_grad()
|
| 1051 |
+
def infer(
|
| 1052 |
+
self,
|
| 1053 |
+
prompt: Optional[str],
|
| 1054 |
+
input_image: torch.Tensor,
|
| 1055 |
+
num_frames: int,
|
| 1056 |
+
action: Optional[torch.Tensor] = None,
|
| 1057 |
+
action_horizon: Optional[int] = None,
|
| 1058 |
+
proprio: Optional[torch.Tensor] = None,
|
| 1059 |
+
context: Optional[torch.Tensor] = None,
|
| 1060 |
+
context_mask: Optional[torch.Tensor] = None,
|
| 1061 |
+
negative_prompt: Optional[str] = None,
|
| 1062 |
+
text_cfg_scale: float = 5.0,
|
| 1063 |
+
action_cfg_scale: float = 1.0,
|
| 1064 |
+
num_inference_steps: int = 20,
|
| 1065 |
+
sigma_shift: Optional[float] = None,
|
| 1066 |
+
seed: Optional[int] = None,
|
| 1067 |
+
rand_device: str = "cpu",
|
| 1068 |
+
tiled: bool = False,
|
| 1069 |
+
):
|
| 1070 |
+
return self.infer_joint(
|
| 1071 |
+
prompt=prompt,
|
| 1072 |
+
input_image=input_image,
|
| 1073 |
+
num_video_frames=num_frames,
|
| 1074 |
+
action_horizon=action_horizon,
|
| 1075 |
+
action=action,
|
| 1076 |
+
proprio=proprio,
|
| 1077 |
+
context=context,
|
| 1078 |
+
context_mask=context_mask,
|
| 1079 |
+
negative_prompt=negative_prompt,
|
| 1080 |
+
text_cfg_scale=text_cfg_scale,
|
| 1081 |
+
num_inference_steps=num_inference_steps,
|
| 1082 |
+
sigma_shift=sigma_shift,
|
| 1083 |
+
seed=seed,
|
| 1084 |
+
rand_device=rand_device,
|
| 1085 |
+
tiled=tiled,
|
| 1086 |
+
)
|
| 1087 |
+
|
| 1088 |
+
def save_checkpoint(self, path, optimizer=None, step=None):
|
| 1089 |
+
payload = {
|
| 1090 |
+
"mot": self.mot.state_dict(),
|
| 1091 |
+
"step": step,
|
| 1092 |
+
"torch_dtype": str(self.torch_dtype),
|
| 1093 |
+
}
|
| 1094 |
+
if self.proprio_encoder is not None:
|
| 1095 |
+
payload["proprio_encoder"] = self.proprio_encoder.state_dict()
|
| 1096 |
+
if optimizer is not None:
|
| 1097 |
+
payload["optimizer"] = optimizer.state_dict()
|
| 1098 |
+
torch.save(payload, path)
|
| 1099 |
+
|
| 1100 |
+
def load_checkpoint(self, path, optimizer=None):
|
| 1101 |
+
payload = torch.load(path, map_location="cpu")
|
| 1102 |
+
if "mot" in payload:
|
| 1103 |
+
self.mot.load_state_dict(payload["mot"], strict=False)
|
| 1104 |
+
elif "dit" in payload:
|
| 1105 |
+
logger.warning("Loading legacy `dit` checkpoint into video expert only.")
|
| 1106 |
+
self.video_expert.load_state_dict(payload["dit"], strict=False)
|
| 1107 |
+
else:
|
| 1108 |
+
raise ValueError(f"Checkpoint missing both `mot` and `dit` keys: {path}")
|
| 1109 |
+
if self.proprio_encoder is not None:
|
| 1110 |
+
if "proprio_encoder" in payload:
|
| 1111 |
+
self.proprio_encoder.load_state_dict(payload["proprio_encoder"], strict=True)
|
| 1112 |
+
else:
|
| 1113 |
+
logger.warning("Checkpoint has no `proprio_encoder` weights; keeping current `proprio_encoder` params.")
|
| 1114 |
+
elif "proprio_encoder" in payload:
|
| 1115 |
+
logger.warning("Checkpoint contains `proprio_encoder` weights but current model has `proprio_dim=None`; ignoring.")
|
| 1116 |
+
|
| 1117 |
+
if optimizer is not None and "optimizer" in payload:
|
| 1118 |
+
optimizer.load_state_dict(payload["optimizer"])
|
| 1119 |
+
return payload
|
| 1120 |
+
|
| 1121 |
+
def forward(self, *args, **kwargs):
|
| 1122 |
+
return self.training_loss(*args, **kwargs)
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_idm.py
ADDED
|
@@ -0,0 +1,452 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, Optional
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
|
| 6 |
+
from fastwam.utils.logging_config import get_logger
|
| 7 |
+
|
| 8 |
+
from .fastwam_joint import FastWAMJoint
|
| 9 |
+
|
| 10 |
+
logger = get_logger(__name__)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class FastWAMIDM(FastWAMJoint):
|
| 14 |
+
"""IDM variant with teacher-forcing video conditioning for action denoising."""
|
| 15 |
+
|
| 16 |
+
# Hardcoded probability: during training, cond-video is noised with this chance.
|
| 17 |
+
video_cond_noise_prob = 0.5
|
| 18 |
+
|
| 19 |
+
@torch.no_grad()
|
| 20 |
+
def _build_teacher_forcing_attention_mask(
|
| 21 |
+
self,
|
| 22 |
+
noisy_video_seq_len: int,
|
| 23 |
+
cond_video_seq_len: int,
|
| 24 |
+
action_seq_len: int,
|
| 25 |
+
noisy_video_tokens_per_frame: int,
|
| 26 |
+
cond_video_tokens_per_frame: int,
|
| 27 |
+
device: torch.device,
|
| 28 |
+
) -> torch.Tensor:
|
| 29 |
+
if noisy_video_tokens_per_frame != cond_video_tokens_per_frame:
|
| 30 |
+
raise ValueError(
|
| 31 |
+
"Teacher-forcing requires identical `tokens_per_frame` for noisy and cond video branches, "
|
| 32 |
+
f"got {noisy_video_tokens_per_frame} and {cond_video_tokens_per_frame}."
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
noisy_end = noisy_video_seq_len
|
| 36 |
+
cond_end = noisy_video_seq_len + cond_video_seq_len
|
| 37 |
+
total_seq_len = cond_end + action_seq_len
|
| 38 |
+
mask = torch.zeros((total_seq_len, total_seq_len), dtype=torch.bool, device=device)
|
| 39 |
+
|
| 40 |
+
# noisy_video -> noisy_video
|
| 41 |
+
mask[:noisy_end, :noisy_end] = self.video_expert.build_video_to_video_mask(
|
| 42 |
+
video_seq_len=noisy_video_seq_len,
|
| 43 |
+
video_tokens_per_frame=noisy_video_tokens_per_frame,
|
| 44 |
+
device=device,
|
| 45 |
+
)
|
| 46 |
+
# cond_video -> cond_video
|
| 47 |
+
mask[noisy_end:cond_end, noisy_end:cond_end] = self.video_expert.build_video_to_video_mask(
|
| 48 |
+
video_seq_len=cond_video_seq_len,
|
| 49 |
+
video_tokens_per_frame=cond_video_tokens_per_frame,
|
| 50 |
+
device=device,
|
| 51 |
+
)
|
| 52 |
+
# action -> action
|
| 53 |
+
mask[cond_end:, cond_end:] = True
|
| 54 |
+
# action -> cond_video only
|
| 55 |
+
mask[cond_end:, noisy_end:cond_end] = True
|
| 56 |
+
return mask
|
| 57 |
+
|
| 58 |
+
def training_loss(self, sample, tiled: bool = False):
|
| 59 |
+
inputs = self.build_inputs(sample, tiled=tiled)
|
| 60 |
+
input_latents = inputs["input_latents"]
|
| 61 |
+
batch_size = input_latents.shape[0]
|
| 62 |
+
context = inputs["context"]
|
| 63 |
+
context_mask = inputs["context_mask"]
|
| 64 |
+
action = inputs["action"]
|
| 65 |
+
action_is_pad = inputs["action_is_pad"]
|
| 66 |
+
image_is_pad = inputs["image_is_pad"]
|
| 67 |
+
fuse_flag = inputs["fuse_vae_embedding_in_latents"]
|
| 68 |
+
|
| 69 |
+
# Branch A: noisy video (for video denoising target).
|
| 70 |
+
noise_video = torch.randn_like(input_latents)
|
| 71 |
+
timestep_video = self.train_video_scheduler.sample_training_t(
|
| 72 |
+
batch_size=batch_size,
|
| 73 |
+
device=self.device,
|
| 74 |
+
dtype=input_latents.dtype,
|
| 75 |
+
)
|
| 76 |
+
latents_noisy = self.train_video_scheduler.add_noise(input_latents, noise_video, timestep_video)
|
| 77 |
+
target_video = self.train_video_scheduler.training_target(input_latents, noise_video, timestep_video)
|
| 78 |
+
if inputs["first_frame_latents"] is not None:
|
| 79 |
+
latents_noisy[:, :, 0:1] = inputs["first_frame_latents"]
|
| 80 |
+
|
| 81 |
+
# Branch B: noisy action.
|
| 82 |
+
noise_action = torch.randn_like(action)
|
| 83 |
+
timestep_action = self.train_action_scheduler.sample_training_t(
|
| 84 |
+
batch_size=batch_size,
|
| 85 |
+
device=self.device,
|
| 86 |
+
dtype=action.dtype,
|
| 87 |
+
)
|
| 88 |
+
noisy_action = self.train_action_scheduler.add_noise(action, noise_action, timestep_action)
|
| 89 |
+
target_action = self.train_action_scheduler.training_target(action, noise_action, timestep_action)
|
| 90 |
+
|
| 91 |
+
# Branch C: teacher-forcing cond-video.
|
| 92 |
+
# Each sample is independently noised with probability `video_cond_noise_prob`.
|
| 93 |
+
cond_noise_mask = torch.rand((batch_size,), device=self.device) < float(self.video_cond_noise_prob)
|
| 94 |
+
timestep_video_cond = torch.zeros_like(timestep_video, dtype=input_latents.dtype, device=self.device)
|
| 95 |
+
latents_cond = input_latents
|
| 96 |
+
if bool(cond_noise_mask.any()):
|
| 97 |
+
timestep_video_cond_sampled = self.train_video_scheduler.sample_training_t(
|
| 98 |
+
batch_size=batch_size,
|
| 99 |
+
device=self.device,
|
| 100 |
+
dtype=input_latents.dtype,
|
| 101 |
+
)
|
| 102 |
+
timestep_video_cond = torch.where(cond_noise_mask, timestep_video_cond_sampled, timestep_video_cond)
|
| 103 |
+
noise_video_cond = torch.randn_like(input_latents)
|
| 104 |
+
latents_cond_noisy = self.train_video_scheduler.add_noise(
|
| 105 |
+
input_latents, noise_video_cond, timestep_video_cond_sampled
|
| 106 |
+
)
|
| 107 |
+
cond_noise_selector = cond_noise_mask.view(batch_size, 1, 1, 1, 1)
|
| 108 |
+
latents_cond = torch.where(cond_noise_selector, latents_cond_noisy, input_latents)
|
| 109 |
+
if inputs["first_frame_latents"] is not None:
|
| 110 |
+
latents_cond = latents_cond.clone()
|
| 111 |
+
latents_cond[:, :, 0:1] = inputs["first_frame_latents"]
|
| 112 |
+
|
| 113 |
+
video_pre_noisy = self.video_expert.pre_dit(
|
| 114 |
+
x=latents_noisy,
|
| 115 |
+
timestep=timestep_video,
|
| 116 |
+
context=context,
|
| 117 |
+
context_mask=context_mask,
|
| 118 |
+
action=None,
|
| 119 |
+
fuse_vae_embedding_in_latents=fuse_flag,
|
| 120 |
+
)
|
| 121 |
+
video_pre_cond = self.video_expert.pre_dit(
|
| 122 |
+
x=latents_cond,
|
| 123 |
+
timestep=timestep_video_cond,
|
| 124 |
+
context=context,
|
| 125 |
+
context_mask=context_mask,
|
| 126 |
+
action=None,
|
| 127 |
+
fuse_vae_embedding_in_latents=fuse_flag,
|
| 128 |
+
)
|
| 129 |
+
if video_pre_noisy["t_mod"].ndim != 4 or video_pre_cond["t_mod"].ndim != 4:
|
| 130 |
+
raise ValueError(
|
| 131 |
+
"Teacher-forcing requires token-wise `t_mod`; "
|
| 132 |
+
"ensure `seperated_timestep=true` and `fuse_vae_embedding_in_latents=true`."
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
action_pre = self.action_expert.pre_dit(
|
| 136 |
+
action_tokens=noisy_action,
|
| 137 |
+
timestep=timestep_action,
|
| 138 |
+
context=context,
|
| 139 |
+
context_mask=context_mask,
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
noisy_video_seq_len = int(video_pre_noisy["tokens"].shape[1])
|
| 143 |
+
cond_video_seq_len = int(video_pre_cond["tokens"].shape[1])
|
| 144 |
+
noisy_video_tokens_per_frame = int(video_pre_noisy["meta"]["tokens_per_frame"])
|
| 145 |
+
cond_video_tokens_per_frame = int(video_pre_cond["meta"]["tokens_per_frame"])
|
| 146 |
+
|
| 147 |
+
# Concatenate [noisy_video, cond_video] as the video expert sequence.
|
| 148 |
+
merged_video_tokens = torch.cat([video_pre_noisy["tokens"], video_pre_cond["tokens"]], dim=1)
|
| 149 |
+
merged_video_freqs = torch.cat([video_pre_noisy["freqs"], video_pre_cond["freqs"]], dim=0)
|
| 150 |
+
merged_video_t_mod = torch.cat([video_pre_noisy["t_mod"], video_pre_cond["t_mod"]], dim=1)
|
| 151 |
+
merged_video_context_mask = torch.cat([video_pre_noisy["context_mask"], video_pre_cond["context_mask"]], dim=1)
|
| 152 |
+
|
| 153 |
+
attention_mask = self._build_teacher_forcing_attention_mask(
|
| 154 |
+
noisy_video_seq_len=noisy_video_seq_len,
|
| 155 |
+
cond_video_seq_len=cond_video_seq_len,
|
| 156 |
+
action_seq_len=action_pre["tokens"].shape[1],
|
| 157 |
+
noisy_video_tokens_per_frame=noisy_video_tokens_per_frame,
|
| 158 |
+
cond_video_tokens_per_frame=cond_video_tokens_per_frame,
|
| 159 |
+
device=merged_video_tokens.device,
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
tokens_out = self.mot(
|
| 163 |
+
embeds_all={
|
| 164 |
+
"video": merged_video_tokens,
|
| 165 |
+
"action": action_pre["tokens"],
|
| 166 |
+
},
|
| 167 |
+
attention_mask=attention_mask,
|
| 168 |
+
freqs_all={
|
| 169 |
+
"video": merged_video_freqs,
|
| 170 |
+
"action": action_pre["freqs"],
|
| 171 |
+
},
|
| 172 |
+
context_all={
|
| 173 |
+
"video": {
|
| 174 |
+
"context": video_pre_noisy["context"],
|
| 175 |
+
"mask": merged_video_context_mask,
|
| 176 |
+
},
|
| 177 |
+
"action": {
|
| 178 |
+
"context": action_pre["context"],
|
| 179 |
+
"mask": action_pre["context_mask"],
|
| 180 |
+
},
|
| 181 |
+
},
|
| 182 |
+
t_mod_all={
|
| 183 |
+
"video": merged_video_t_mod,
|
| 184 |
+
"action": action_pre["t_mod"],
|
| 185 |
+
},
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
# Only the noisy-video half contributes to video denoising loss.
|
| 189 |
+
pred_video_tokens = tokens_out["video"][:, :noisy_video_seq_len]
|
| 190 |
+
pred_video = self.video_expert.post_dit(pred_video_tokens, video_pre_noisy)
|
| 191 |
+
pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre)
|
| 192 |
+
|
| 193 |
+
include_initial_video_step = inputs["first_frame_latents"] is None
|
| 194 |
+
if inputs["first_frame_latents"] is not None:
|
| 195 |
+
pred_video = pred_video[:, :, 1:]
|
| 196 |
+
target_video = target_video[:, :, 1:]
|
| 197 |
+
|
| 198 |
+
loss_video_per_sample = self._compute_video_loss_per_sample(
|
| 199 |
+
pred_video=pred_video,
|
| 200 |
+
target_video=target_video,
|
| 201 |
+
image_is_pad=image_is_pad,
|
| 202 |
+
include_initial_video_step=include_initial_video_step,
|
| 203 |
+
)
|
| 204 |
+
video_weight = self.train_video_scheduler.training_weight(timestep_video).to(
|
| 205 |
+
loss_video_per_sample.device, dtype=loss_video_per_sample.dtype
|
| 206 |
+
)
|
| 207 |
+
loss_video = (loss_video_per_sample * video_weight).mean()
|
| 208 |
+
|
| 209 |
+
action_loss_token = F.mse_loss(pred_action.float(), target_action.float(), reduction="none").mean(dim=2)
|
| 210 |
+
if action_is_pad is not None:
|
| 211 |
+
valid = (~action_is_pad).to(device=action_loss_token.device, dtype=action_loss_token.dtype)
|
| 212 |
+
valid_sum = valid.sum(dim=1).clamp(min=1.0)
|
| 213 |
+
action_loss_per_sample = (action_loss_token * valid).sum(dim=1) / valid_sum
|
| 214 |
+
else:
|
| 215 |
+
action_loss_per_sample = action_loss_token.mean(dim=1)
|
| 216 |
+
|
| 217 |
+
action_weight = self.train_action_scheduler.training_weight(timestep_action).to(
|
| 218 |
+
action_loss_per_sample.device, dtype=action_loss_per_sample.dtype
|
| 219 |
+
)
|
| 220 |
+
loss_action = (action_loss_per_sample * action_weight).mean()
|
| 221 |
+
|
| 222 |
+
loss_total = self.loss_lambda_video * loss_video + self.loss_lambda_action * loss_action
|
| 223 |
+
loss_dict = {
|
| 224 |
+
"loss_video": self.loss_lambda_video * float(loss_video.detach().item()),
|
| 225 |
+
"loss_action": self.loss_lambda_action * float(loss_action.detach().item()),
|
| 226 |
+
}
|
| 227 |
+
return loss_total, loss_dict
|
| 228 |
+
|
| 229 |
+
@torch.no_grad()
|
| 230 |
+
def infer_action(
|
| 231 |
+
self,
|
| 232 |
+
prompt: Optional[str],
|
| 233 |
+
input_image: torch.Tensor,
|
| 234 |
+
action_horizon: int,
|
| 235 |
+
num_video_frames: int,
|
| 236 |
+
proprio: Optional[torch.Tensor] = None,
|
| 237 |
+
context: Optional[torch.Tensor] = None,
|
| 238 |
+
context_mask: Optional[torch.Tensor] = None,
|
| 239 |
+
negative_prompt: Optional[str] = None,
|
| 240 |
+
text_cfg_scale: float = 1.0,
|
| 241 |
+
num_inference_steps: int = 20,
|
| 242 |
+
sigma_shift: Optional[float] = None,
|
| 243 |
+
seed: Optional[int] = None,
|
| 244 |
+
rand_device: str = "cpu",
|
| 245 |
+
tiled: bool = False,
|
| 246 |
+
) -> dict[str, Any]:
|
| 247 |
+
# Reuse infer_joint pipeline and keep infer_action output contract.
|
| 248 |
+
out = self.infer_joint(
|
| 249 |
+
prompt=prompt,
|
| 250 |
+
input_image=input_image,
|
| 251 |
+
num_video_frames=num_video_frames,
|
| 252 |
+
action_horizon=action_horizon,
|
| 253 |
+
action=None,
|
| 254 |
+
proprio=proprio,
|
| 255 |
+
context=context,
|
| 256 |
+
context_mask=context_mask,
|
| 257 |
+
negative_prompt=negative_prompt,
|
| 258 |
+
text_cfg_scale=text_cfg_scale,
|
| 259 |
+
num_inference_steps=num_inference_steps,
|
| 260 |
+
sigma_shift=sigma_shift,
|
| 261 |
+
seed=seed,
|
| 262 |
+
rand_device=rand_device,
|
| 263 |
+
tiled=tiled,
|
| 264 |
+
test_action_with_infer_action=False,
|
| 265 |
+
)
|
| 266 |
+
return {"action": out["action"]}
|
| 267 |
+
|
| 268 |
+
@torch.no_grad()
|
| 269 |
+
def infer_joint(
|
| 270 |
+
self,
|
| 271 |
+
prompt: Optional[str],
|
| 272 |
+
input_image: torch.Tensor,
|
| 273 |
+
num_video_frames: int,
|
| 274 |
+
action_horizon: int,
|
| 275 |
+
action: Optional[torch.Tensor] = None,
|
| 276 |
+
proprio: Optional[torch.Tensor] = None,
|
| 277 |
+
context: Optional[torch.Tensor] = None,
|
| 278 |
+
context_mask: Optional[torch.Tensor] = None,
|
| 279 |
+
negative_prompt: Optional[str] = None,
|
| 280 |
+
text_cfg_scale: float = 1.0,
|
| 281 |
+
num_inference_steps: int = 20,
|
| 282 |
+
sigma_shift: Optional[float] = None,
|
| 283 |
+
seed: Optional[int] = None,
|
| 284 |
+
rand_device: str = "cpu",
|
| 285 |
+
tiled: bool = False,
|
| 286 |
+
test_action_with_infer_action: bool = True,
|
| 287 |
+
) -> dict[str, Any]:
|
| 288 |
+
del negative_prompt, text_cfg_scale, test_action_with_infer_action
|
| 289 |
+
self.eval()
|
| 290 |
+
|
| 291 |
+
if action is not None:
|
| 292 |
+
logger.warning(
|
| 293 |
+
"`FastWAMIDM.infer_joint` ignores `action` input; "
|
| 294 |
+
"video is denoised in a standalone first stage."
|
| 295 |
+
)
|
| 296 |
+
|
| 297 |
+
if input_image.ndim == 3:
|
| 298 |
+
input_image = input_image.unsqueeze(0)
|
| 299 |
+
if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
|
| 300 |
+
raise ValueError(
|
| 301 |
+
f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
|
| 302 |
+
)
|
| 303 |
+
_, _, height, width = input_image.shape
|
| 304 |
+
checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_video_frames)
|
| 305 |
+
if (checked_h, checked_w) != (height, width):
|
| 306 |
+
raise ValueError(
|
| 307 |
+
f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})"
|
| 308 |
+
)
|
| 309 |
+
if checked_t != num_video_frames:
|
| 310 |
+
raise ValueError(
|
| 311 |
+
f"`num_video_frames` must satisfy T % 4 == 1, got {num_video_frames}"
|
| 312 |
+
)
|
| 313 |
+
|
| 314 |
+
if proprio is not None:
|
| 315 |
+
if self.proprio_dim is None:
|
| 316 |
+
raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.")
|
| 317 |
+
if proprio.ndim == 1:
|
| 318 |
+
proprio = proprio.unsqueeze(0)
|
| 319 |
+
elif proprio.ndim == 2 and proprio.shape[0] == 1:
|
| 320 |
+
pass
|
| 321 |
+
else:
|
| 322 |
+
raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}")
|
| 323 |
+
if proprio.shape[1] != self.proprio_dim:
|
| 324 |
+
raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}")
|
| 325 |
+
proprio = proprio.to(device=self.device, dtype=self.torch_dtype)
|
| 326 |
+
|
| 327 |
+
latent_t = (num_video_frames - 1) // self.vae.temporal_downsample_factor + 1
|
| 328 |
+
latent_h = height // self.vae.upsampling_factor
|
| 329 |
+
latent_w = width // self.vae.upsampling_factor
|
| 330 |
+
|
| 331 |
+
video_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
|
| 332 |
+
action_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
|
| 333 |
+
latents_video = torch.randn(
|
| 334 |
+
(1, self.vae.model.z_dim, latent_t, latent_h, latent_w),
|
| 335 |
+
generator=video_generator,
|
| 336 |
+
device=rand_device,
|
| 337 |
+
dtype=torch.float32,
|
| 338 |
+
).to(device=self.device, dtype=self.torch_dtype)
|
| 339 |
+
latents_action = torch.randn(
|
| 340 |
+
(1, action_horizon, self.action_expert.action_dim),
|
| 341 |
+
generator=action_generator,
|
| 342 |
+
device=rand_device,
|
| 343 |
+
dtype=torch.float32,
|
| 344 |
+
).to(device=self.device, dtype=self.torch_dtype)
|
| 345 |
+
|
| 346 |
+
input_image = input_image.to(device=self.device, dtype=self.torch_dtype)
|
| 347 |
+
first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled)
|
| 348 |
+
latents_video[:, :, 0:1] = first_frame_latents.clone()
|
| 349 |
+
fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False))
|
| 350 |
+
|
| 351 |
+
use_prompt = prompt is not None
|
| 352 |
+
use_context = context is not None or context_mask is not None
|
| 353 |
+
if use_prompt and use_context:
|
| 354 |
+
raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.")
|
| 355 |
+
if not use_prompt and not use_context:
|
| 356 |
+
raise ValueError("Either `prompt` or both `context/context_mask` must be provided.")
|
| 357 |
+
|
| 358 |
+
if use_prompt:
|
| 359 |
+
context, context_mask = self.encode_prompt(prompt)
|
| 360 |
+
else:
|
| 361 |
+
if context is None or context_mask is None:
|
| 362 |
+
raise ValueError("`context` and `context_mask` must be both provided together.")
|
| 363 |
+
if context.ndim == 2:
|
| 364 |
+
context = context.unsqueeze(0)
|
| 365 |
+
if context_mask.ndim == 1:
|
| 366 |
+
context_mask = context_mask.unsqueeze(0)
|
| 367 |
+
if context.ndim != 3 or context_mask.ndim != 2:
|
| 368 |
+
raise ValueError(
|
| 369 |
+
f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}"
|
| 370 |
+
)
|
| 371 |
+
context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
|
| 372 |
+
context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True)
|
| 373 |
+
if proprio is not None:
|
| 374 |
+
context, context_mask = self._append_proprio_to_context(
|
| 375 |
+
context=context,
|
| 376 |
+
context_mask=context_mask,
|
| 377 |
+
proprio=proprio,
|
| 378 |
+
)
|
| 379 |
+
|
| 380 |
+
# Stage 1: denoise video only.
|
| 381 |
+
infer_timesteps_video, infer_deltas_video = self.infer_video_scheduler.build_inference_schedule(
|
| 382 |
+
num_inference_steps=num_inference_steps,
|
| 383 |
+
device=self.device,
|
| 384 |
+
dtype=latents_video.dtype,
|
| 385 |
+
shift_override=sigma_shift,
|
| 386 |
+
)
|
| 387 |
+
for step_t_video, step_delta_video in zip(infer_timesteps_video, infer_deltas_video):
|
| 388 |
+
timestep_video = step_t_video.unsqueeze(0).to(dtype=latents_video.dtype, device=self.device)
|
| 389 |
+
pred_video = self.video_expert(
|
| 390 |
+
x=latents_video,
|
| 391 |
+
timestep=timestep_video,
|
| 392 |
+
context=context,
|
| 393 |
+
context_mask=context_mask,
|
| 394 |
+
action=None,
|
| 395 |
+
fuse_vae_embedding_in_latents=fuse_flag,
|
| 396 |
+
)
|
| 397 |
+
latents_video = self.infer_video_scheduler.step(pred_video, step_delta_video, latents_video)
|
| 398 |
+
latents_video[:, :, 0:1] = first_frame_latents.clone()
|
| 399 |
+
|
| 400 |
+
# Stage 2: freeze denoised video as cond and denoise action via video K/V cache.
|
| 401 |
+
timestep_video_cond = torch.zeros(
|
| 402 |
+
(latents_video.shape[0],), dtype=latents_video.dtype, device=self.device
|
| 403 |
+
)
|
| 404 |
+
video_pre_cond = self.video_expert.pre_dit(
|
| 405 |
+
x=latents_video,
|
| 406 |
+
timestep=timestep_video_cond,
|
| 407 |
+
context=context,
|
| 408 |
+
context_mask=context_mask,
|
| 409 |
+
action=None,
|
| 410 |
+
fuse_vae_embedding_in_latents=fuse_flag,
|
| 411 |
+
)
|
| 412 |
+
video_seq_len = int(video_pre_cond["tokens"].shape[1])
|
| 413 |
+
attention_mask = self._build_mot_attention_mask(
|
| 414 |
+
video_seq_len=video_seq_len,
|
| 415 |
+
action_seq_len=latents_action.shape[1],
|
| 416 |
+
video_tokens_per_frame=int(video_pre_cond["meta"]["tokens_per_frame"]),
|
| 417 |
+
device=video_pre_cond["tokens"].device,
|
| 418 |
+
)
|
| 419 |
+
video_kv_cache = self.mot.prefill_video_cache(
|
| 420 |
+
video_tokens=video_pre_cond["tokens"],
|
| 421 |
+
video_freqs=video_pre_cond["freqs"],
|
| 422 |
+
video_t_mod=video_pre_cond["t_mod"],
|
| 423 |
+
video_context_payload={
|
| 424 |
+
"context": video_pre_cond["context"],
|
| 425 |
+
"mask": video_pre_cond["context_mask"],
|
| 426 |
+
},
|
| 427 |
+
video_attention_mask=attention_mask[:video_seq_len, :video_seq_len],
|
| 428 |
+
)
|
| 429 |
+
|
| 430 |
+
infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule(
|
| 431 |
+
num_inference_steps=num_inference_steps,
|
| 432 |
+
device=self.device,
|
| 433 |
+
dtype=latents_action.dtype,
|
| 434 |
+
shift_override=sigma_shift,
|
| 435 |
+
)
|
| 436 |
+
for step_t_action, step_delta_action in zip(infer_timesteps_action, infer_deltas_action):
|
| 437 |
+
timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device)
|
| 438 |
+
pred_action = self._predict_action_noise_with_cache(
|
| 439 |
+
latents_action=latents_action,
|
| 440 |
+
timestep_action=timestep_action,
|
| 441 |
+
context=context,
|
| 442 |
+
context_mask=context_mask,
|
| 443 |
+
video_kv_cache=video_kv_cache,
|
| 444 |
+
attention_mask=attention_mask,
|
| 445 |
+
video_seq_len=video_seq_len,
|
| 446 |
+
)
|
| 447 |
+
latents_action = self.infer_action_scheduler.step(pred_action, step_delta_action, latents_action)
|
| 448 |
+
|
| 449 |
+
return {
|
| 450 |
+
"video": self._decode_latents(latents_video, tiled=tiled),
|
| 451 |
+
"action": latents_action[0].detach().to(device="cpu", dtype=torch.float32),
|
| 452 |
+
}
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/fastwam_joint.py
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, Optional
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
from fastwam.utils.logging_config import get_logger
|
| 6 |
+
|
| 7 |
+
from .attention_masks import build_joint_denoise_attention_mask
|
| 8 |
+
from .fastwam import FastWAM
|
| 9 |
+
|
| 10 |
+
logger = get_logger(__name__)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class FastWAMJoint(FastWAM):
|
| 14 |
+
"""MoT variant with bidirectional video/action joint denoising."""
|
| 15 |
+
|
| 16 |
+
@classmethod
|
| 17 |
+
def from_wan22_pretrained(cls, **kwargs):
|
| 18 |
+
video_dit_config = kwargs.get("video_dit_config", None)
|
| 19 |
+
if not isinstance(video_dit_config, dict):
|
| 20 |
+
raise ValueError(
|
| 21 |
+
"`video_dit_config` must be provided as dict for FastWAMJoint."
|
| 22 |
+
)
|
| 23 |
+
if bool(video_dit_config.get("action_conditioned", False)):
|
| 24 |
+
raise ValueError(
|
| 25 |
+
"FastWAMJoint requires `video_dit_config['action_conditioned']=false`."
|
| 26 |
+
)
|
| 27 |
+
return super().from_wan22_pretrained(**kwargs)
|
| 28 |
+
|
| 29 |
+
@torch.no_grad()
|
| 30 |
+
def _build_mot_attention_mask(
|
| 31 |
+
self,
|
| 32 |
+
video_seq_len: int,
|
| 33 |
+
action_seq_len: int,
|
| 34 |
+
video_tokens_per_frame: int,
|
| 35 |
+
device: torch.device,
|
| 36 |
+
) -> torch.Tensor:
|
| 37 |
+
return build_joint_denoise_attention_mask(
|
| 38 |
+
video_seq_len=video_seq_len,
|
| 39 |
+
action_seq_len=action_seq_len,
|
| 40 |
+
video_tokens_per_frame=video_tokens_per_frame,
|
| 41 |
+
device=device,
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
@torch.no_grad()
|
| 45 |
+
def infer_joint(
|
| 46 |
+
self,
|
| 47 |
+
prompt: Optional[str],
|
| 48 |
+
input_image: torch.Tensor,
|
| 49 |
+
num_video_frames: int,
|
| 50 |
+
action_horizon: int,
|
| 51 |
+
action: Optional[torch.Tensor] = None,
|
| 52 |
+
proprio: Optional[torch.Tensor] = None,
|
| 53 |
+
context: Optional[torch.Tensor] = None,
|
| 54 |
+
context_mask: Optional[torch.Tensor] = None,
|
| 55 |
+
negative_prompt: Optional[str] = None,
|
| 56 |
+
text_cfg_scale: float = 1.0,
|
| 57 |
+
num_inference_steps: int = 20,
|
| 58 |
+
sigma_shift: Optional[float] = None,
|
| 59 |
+
seed: Optional[int] = None,
|
| 60 |
+
rand_device: str = "cpu",
|
| 61 |
+
tiled: bool = False,
|
| 62 |
+
test_action_with_infer_action: bool = True,
|
| 63 |
+
) -> dict[str, Any]:
|
| 64 |
+
if test_action_with_infer_action:
|
| 65 |
+
logger.warning(
|
| 66 |
+
"`FastWAMJoint.infer_joint` ignores `test_action_with_infer_action=True` "
|
| 67 |
+
"and always runs with `test_action_with_infer_action=False`."
|
| 68 |
+
)
|
| 69 |
+
return super().infer_joint(
|
| 70 |
+
prompt=prompt,
|
| 71 |
+
input_image=input_image,
|
| 72 |
+
num_video_frames=num_video_frames,
|
| 73 |
+
action_horizon=action_horizon,
|
| 74 |
+
action=action,
|
| 75 |
+
proprio=proprio,
|
| 76 |
+
context=context,
|
| 77 |
+
context_mask=context_mask,
|
| 78 |
+
negative_prompt=negative_prompt,
|
| 79 |
+
text_cfg_scale=text_cfg_scale,
|
| 80 |
+
num_inference_steps=num_inference_steps,
|
| 81 |
+
sigma_shift=sigma_shift,
|
| 82 |
+
seed=seed,
|
| 83 |
+
rand_device=rand_device,
|
| 84 |
+
tiled=tiled,
|
| 85 |
+
test_action_with_infer_action=False,
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
@torch.no_grad()
|
| 89 |
+
def infer_action(
|
| 90 |
+
self,
|
| 91 |
+
prompt: Optional[str],
|
| 92 |
+
input_image: torch.Tensor,
|
| 93 |
+
action_horizon: int,
|
| 94 |
+
num_video_frames: int,
|
| 95 |
+
proprio: Optional[torch.Tensor] = None,
|
| 96 |
+
context: Optional[torch.Tensor] = None,
|
| 97 |
+
context_mask: Optional[torch.Tensor] = None,
|
| 98 |
+
negative_prompt: Optional[str] = None,
|
| 99 |
+
text_cfg_scale: float = 1.0,
|
| 100 |
+
num_inference_steps: int = 20,
|
| 101 |
+
sigma_shift: Optional[float] = None,
|
| 102 |
+
seed: Optional[int] = None,
|
| 103 |
+
rand_device: str = "cpu",
|
| 104 |
+
tiled: bool = False,
|
| 105 |
+
) -> dict[str, Any]:
|
| 106 |
+
self.eval()
|
| 107 |
+
|
| 108 |
+
if input_image.ndim == 3:
|
| 109 |
+
input_image = input_image.unsqueeze(0)
|
| 110 |
+
if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
|
| 111 |
+
raise ValueError(
|
| 112 |
+
f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
|
| 113 |
+
)
|
| 114 |
+
_, _, height, width = input_image.shape
|
| 115 |
+
checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_video_frames)
|
| 116 |
+
if (checked_h, checked_w) != (height, width):
|
| 117 |
+
raise ValueError(
|
| 118 |
+
f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})"
|
| 119 |
+
)
|
| 120 |
+
if checked_t != num_video_frames:
|
| 121 |
+
raise ValueError(
|
| 122 |
+
f"`num_video_frames` must satisfy T % 4 == 1, got {num_video_frames}"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
if proprio is not None:
|
| 126 |
+
if self.proprio_dim is None:
|
| 127 |
+
raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.")
|
| 128 |
+
if proprio.ndim == 1:
|
| 129 |
+
proprio = proprio.unsqueeze(0)
|
| 130 |
+
elif proprio.ndim == 2 and proprio.shape[0] == 1:
|
| 131 |
+
pass
|
| 132 |
+
else:
|
| 133 |
+
raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}")
|
| 134 |
+
if proprio.shape[1] != self.proprio_dim:
|
| 135 |
+
raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}")
|
| 136 |
+
proprio = proprio.to(device=self.device, dtype=self.torch_dtype)
|
| 137 |
+
|
| 138 |
+
latent_t = (num_video_frames - 1) // self.vae.temporal_downsample_factor + 1
|
| 139 |
+
latent_h = height // self.vae.upsampling_factor
|
| 140 |
+
latent_w = width // self.vae.upsampling_factor
|
| 141 |
+
|
| 142 |
+
video_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
|
| 143 |
+
action_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
|
| 144 |
+
latents_video = torch.randn(
|
| 145 |
+
(1, self.vae.model.z_dim, latent_t, latent_h, latent_w),
|
| 146 |
+
generator=video_generator,
|
| 147 |
+
device=rand_device,
|
| 148 |
+
dtype=torch.float32,
|
| 149 |
+
).to(device=self.device, dtype=self.torch_dtype)
|
| 150 |
+
latents_action = torch.randn(
|
| 151 |
+
(1, action_horizon, self.action_expert.action_dim),
|
| 152 |
+
generator=action_generator,
|
| 153 |
+
device=rand_device,
|
| 154 |
+
dtype=torch.float32,
|
| 155 |
+
).to(device=self.device, dtype=self.torch_dtype)
|
| 156 |
+
|
| 157 |
+
input_image = input_image.to(device=self.device, dtype=self.torch_dtype)
|
| 158 |
+
first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled)
|
| 159 |
+
latents_video[:, :, 0:1] = first_frame_latents.clone()
|
| 160 |
+
fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False))
|
| 161 |
+
|
| 162 |
+
use_prompt = prompt is not None
|
| 163 |
+
use_context = context is not None or context_mask is not None
|
| 164 |
+
if use_prompt and use_context:
|
| 165 |
+
raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.")
|
| 166 |
+
if not use_prompt and not use_context:
|
| 167 |
+
raise ValueError("Either `prompt` or both `context/context_mask` must be provided.")
|
| 168 |
+
|
| 169 |
+
if use_prompt:
|
| 170 |
+
context, context_mask = self.encode_prompt(prompt)
|
| 171 |
+
else:
|
| 172 |
+
if context is None or context_mask is None:
|
| 173 |
+
raise ValueError("`context` and `context_mask` must be both provided together.")
|
| 174 |
+
if context.ndim == 2:
|
| 175 |
+
context = context.unsqueeze(0)
|
| 176 |
+
if context_mask.ndim == 1:
|
| 177 |
+
context_mask = context_mask.unsqueeze(0)
|
| 178 |
+
if context.ndim != 3 or context_mask.ndim != 2:
|
| 179 |
+
raise ValueError(
|
| 180 |
+
f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}"
|
| 181 |
+
)
|
| 182 |
+
context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True)
|
| 183 |
+
context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True)
|
| 184 |
+
if proprio is not None:
|
| 185 |
+
context, context_mask = self._append_proprio_to_context(
|
| 186 |
+
context=context,
|
| 187 |
+
context_mask=context_mask,
|
| 188 |
+
proprio=proprio,
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
infer_timesteps_video, infer_deltas_video = self.infer_video_scheduler.build_inference_schedule(
|
| 192 |
+
num_inference_steps=num_inference_steps,
|
| 193 |
+
device=self.device,
|
| 194 |
+
dtype=latents_video.dtype,
|
| 195 |
+
shift_override=sigma_shift,
|
| 196 |
+
)
|
| 197 |
+
infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule(
|
| 198 |
+
num_inference_steps=num_inference_steps,
|
| 199 |
+
device=self.device,
|
| 200 |
+
dtype=latents_action.dtype,
|
| 201 |
+
shift_override=sigma_shift,
|
| 202 |
+
)
|
| 203 |
+
for step_t_video, step_delta_video, step_t_action, step_delta_action in zip(
|
| 204 |
+
infer_timesteps_video,
|
| 205 |
+
infer_deltas_video,
|
| 206 |
+
infer_timesteps_action,
|
| 207 |
+
infer_deltas_action,
|
| 208 |
+
):
|
| 209 |
+
timestep_video = step_t_video.unsqueeze(0).to(dtype=latents_video.dtype, device=self.device)
|
| 210 |
+
timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device)
|
| 211 |
+
|
| 212 |
+
pred_video_posi, pred_action_posi = self._predict_joint_noise(
|
| 213 |
+
latents_video=latents_video,
|
| 214 |
+
latents_action=latents_action,
|
| 215 |
+
timestep_video=timestep_video,
|
| 216 |
+
timestep_action=timestep_action,
|
| 217 |
+
context=context,
|
| 218 |
+
context_mask=context_mask,
|
| 219 |
+
fuse_vae_embedding_in_latents=fuse_flag,
|
| 220 |
+
gt_action=None,
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
latents_video = self.infer_video_scheduler.step(pred_video_posi, step_delta_video, latents_video)
|
| 224 |
+
latents_action = self.infer_action_scheduler.step(pred_action_posi, step_delta_action, latents_action)
|
| 225 |
+
latents_video[:, :, 0:1] = first_frame_latents.clone()
|
| 226 |
+
|
| 227 |
+
return {
|
| 228 |
+
"action": latents_action[0].detach().to(device="cpu", dtype=torch.float32),
|
| 229 |
+
}
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .io import ModelConfig, hash_model_file, load_state_dict
|
| 2 |
+
from .state_dict_converters import (
|
| 3 |
+
wan_video_dit_from_diffusers,
|
| 4 |
+
wan_video_dit_state_dict_converter,
|
| 5 |
+
wan_video_vae_state_dict_converter,
|
| 6 |
+
)
|
| 7 |
+
|
| 8 |
+
__all__ = [
|
| 9 |
+
"ModelConfig",
|
| 10 |
+
"hash_model_file",
|
| 11 |
+
"load_state_dict",
|
| 12 |
+
"wan_video_dit_from_diffusers",
|
| 13 |
+
"wan_video_dit_state_dict_converter",
|
| 14 |
+
"wan_video_vae_state_dict_converter",
|
| 15 |
+
]
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/gradient.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def create_custom_forward(module):
|
| 5 |
+
def custom_forward(*inputs, **kwargs):
|
| 6 |
+
return module(*inputs, **kwargs)
|
| 7 |
+
return custom_forward
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def gradient_checkpoint_forward(
|
| 11 |
+
model,
|
| 12 |
+
use_gradient_checkpointing,
|
| 13 |
+
*args,
|
| 14 |
+
**kwargs,
|
| 15 |
+
):
|
| 16 |
+
if use_gradient_checkpointing:
|
| 17 |
+
model_output = torch.utils.checkpoint.checkpoint(
|
| 18 |
+
create_custom_forward(model),
|
| 19 |
+
*args,
|
| 20 |
+
**kwargs,
|
| 21 |
+
use_reentrant=False,
|
| 22 |
+
)
|
| 23 |
+
else:
|
| 24 |
+
model_output = model(*args, **kwargs)
|
| 25 |
+
return model_output
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/io.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import glob
|
| 2 |
+
import hashlib
|
| 3 |
+
import os
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from typing import Dict, Optional, Union
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from safetensors import safe_open
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@dataclass
|
| 12 |
+
class ModelConfig:
|
| 13 |
+
path: Union[str, list[str], None] = None
|
| 14 |
+
model_id: Optional[str] = None
|
| 15 |
+
origin_file_pattern: Union[str, list[str], None] = None
|
| 16 |
+
download_source: Optional[str] = None
|
| 17 |
+
local_model_path: Optional[str] = None
|
| 18 |
+
skip_download: Optional[bool] = None
|
| 19 |
+
state_dict: Optional[Dict[str, torch.Tensor]] = None
|
| 20 |
+
|
| 21 |
+
def check_input(self):
|
| 22 |
+
if self.path is None and self.model_id is None:
|
| 23 |
+
raise ValueError("ModelConfig requires either `path` or (`model_id`, `origin_file_pattern`).")
|
| 24 |
+
|
| 25 |
+
def parse_original_file_pattern(self):
|
| 26 |
+
if self.origin_file_pattern in [None, "", "./"]:
|
| 27 |
+
return "*"
|
| 28 |
+
if isinstance(self.origin_file_pattern, list):
|
| 29 |
+
return self.origin_file_pattern
|
| 30 |
+
if self.origin_file_pattern.endswith("/"):
|
| 31 |
+
return self.origin_file_pattern + "*"
|
| 32 |
+
return self.origin_file_pattern
|
| 33 |
+
|
| 34 |
+
def parse_download_source(self):
|
| 35 |
+
if self.download_source is not None:
|
| 36 |
+
return self.download_source
|
| 37 |
+
env = os.environ.get("DIFFSYNTH_DOWNLOAD_SOURCE")
|
| 38 |
+
return env if env is not None else "modelscope"
|
| 39 |
+
|
| 40 |
+
def parse_skip_download(self):
|
| 41 |
+
if self.skip_download is not None:
|
| 42 |
+
return self.skip_download
|
| 43 |
+
env = os.environ.get("DIFFSYNTH_SKIP_DOWNLOAD")
|
| 44 |
+
if env is None:
|
| 45 |
+
return False
|
| 46 |
+
return env.lower() == "true"
|
| 47 |
+
|
| 48 |
+
def reset_local_model_path(self):
|
| 49 |
+
if os.environ.get("DIFFSYNTH_MODEL_BASE_PATH") is not None:
|
| 50 |
+
self.local_model_path = os.environ.get("DIFFSYNTH_MODEL_BASE_PATH")
|
| 51 |
+
elif self.local_model_path is None:
|
| 52 |
+
self.local_model_path = "./checkpoints/"
|
| 53 |
+
|
| 54 |
+
def require_downloading(self):
|
| 55 |
+
if self.path is not None:
|
| 56 |
+
return False
|
| 57 |
+
origin_file_pattern = self.parse_original_file_pattern()
|
| 58 |
+
local_root = os.path.join(self.local_model_path, self.model_id)
|
| 59 |
+
if isinstance(origin_file_pattern, list):
|
| 60 |
+
all_exist = True
|
| 61 |
+
for pattern in origin_file_pattern:
|
| 62 |
+
matches = glob.glob(os.path.join(local_root, pattern))
|
| 63 |
+
if len(matches) == 0:
|
| 64 |
+
all_exist = False
|
| 65 |
+
break
|
| 66 |
+
if all_exist:
|
| 67 |
+
return False
|
| 68 |
+
else:
|
| 69 |
+
if len(glob.glob(os.path.join(local_root, origin_file_pattern))) > 0:
|
| 70 |
+
return False
|
| 71 |
+
return not self.parse_skip_download()
|
| 72 |
+
|
| 73 |
+
def download(self):
|
| 74 |
+
origin_file_pattern = self.parse_original_file_pattern()
|
| 75 |
+
root = os.path.join(self.local_model_path, self.model_id)
|
| 76 |
+
downloaded_files = glob.glob(origin_file_pattern, root_dir=root)
|
| 77 |
+
download_source = self.parse_download_source().lower()
|
| 78 |
+
if download_source == "modelscope":
|
| 79 |
+
from modelscope import snapshot_download
|
| 80 |
+
|
| 81 |
+
snapshot_download(
|
| 82 |
+
self.model_id,
|
| 83 |
+
local_dir=root,
|
| 84 |
+
allow_file_pattern=origin_file_pattern,
|
| 85 |
+
ignore_file_pattern=downloaded_files,
|
| 86 |
+
local_files_only=False,
|
| 87 |
+
)
|
| 88 |
+
elif download_source == "huggingface":
|
| 89 |
+
from huggingface_hub import snapshot_download as hf_snapshot_download
|
| 90 |
+
|
| 91 |
+
hf_snapshot_download(
|
| 92 |
+
self.model_id,
|
| 93 |
+
local_dir=root,
|
| 94 |
+
allow_patterns=origin_file_pattern,
|
| 95 |
+
ignore_patterns=downloaded_files,
|
| 96 |
+
local_files_only=False,
|
| 97 |
+
)
|
| 98 |
+
else:
|
| 99 |
+
raise ValueError("`download_source` should be `modelscope` or `huggingface`.")
|
| 100 |
+
|
| 101 |
+
def download_if_necessary(self):
|
| 102 |
+
self.check_input()
|
| 103 |
+
self.reset_local_model_path()
|
| 104 |
+
if self.require_downloading():
|
| 105 |
+
self.download()
|
| 106 |
+
if self.path is None:
|
| 107 |
+
if self.origin_file_pattern in [None, "", "./"]:
|
| 108 |
+
self.path = os.path.join(self.local_model_path, self.model_id)
|
| 109 |
+
else:
|
| 110 |
+
matches = glob.glob(os.path.join(self.local_model_path, self.model_id, self.origin_file_pattern))
|
| 111 |
+
matches.sort()
|
| 112 |
+
self.path = matches
|
| 113 |
+
if isinstance(self.path, list) and len(self.path) == 1:
|
| 114 |
+
self.path = self.path[0]
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def load_state_dict(file_path, torch_dtype=None, device="cpu"):
|
| 118 |
+
if isinstance(file_path, list):
|
| 119 |
+
state_dict = {}
|
| 120 |
+
for file_path_ in file_path:
|
| 121 |
+
state_dict.update(load_state_dict(file_path_, torch_dtype=torch_dtype, device=device))
|
| 122 |
+
return state_dict
|
| 123 |
+
if file_path.endswith(".safetensors"):
|
| 124 |
+
return load_state_dict_from_safetensors(file_path, torch_dtype=torch_dtype, device=device)
|
| 125 |
+
return load_state_dict_from_bin(file_path, torch_dtype=torch_dtype, device=device)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def load_state_dict_from_safetensors(file_path, torch_dtype=None, device="cpu"):
|
| 129 |
+
state_dict = {}
|
| 130 |
+
with safe_open(file_path, framework="pt", device=str(device)) as f:
|
| 131 |
+
for key in f.keys():
|
| 132 |
+
value = f.get_tensor(key)
|
| 133 |
+
if torch_dtype is not None:
|
| 134 |
+
value = value.to(torch_dtype)
|
| 135 |
+
state_dict[key] = value
|
| 136 |
+
return state_dict
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def load_state_dict_from_bin(file_path, torch_dtype=None, device="cpu"):
|
| 140 |
+
state_dict = torch.load(file_path, map_location=device, weights_only=True)
|
| 141 |
+
if len(state_dict) == 1:
|
| 142 |
+
if "state_dict" in state_dict:
|
| 143 |
+
state_dict = state_dict["state_dict"]
|
| 144 |
+
elif "module" in state_dict:
|
| 145 |
+
state_dict = state_dict["module"]
|
| 146 |
+
elif "model_state" in state_dict:
|
| 147 |
+
state_dict = state_dict["model_state"]
|
| 148 |
+
if torch_dtype is not None:
|
| 149 |
+
for key in state_dict:
|
| 150 |
+
if isinstance(state_dict[key], torch.Tensor):
|
| 151 |
+
state_dict[key] = state_dict[key].to(torch_dtype)
|
| 152 |
+
return state_dict
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def _load_keys_dict_from_safetensors(file_path):
|
| 156 |
+
keys_dict = {}
|
| 157 |
+
with safe_open(file_path, framework="pt", device="cpu") as f:
|
| 158 |
+
for key in f.keys():
|
| 159 |
+
keys_dict[key] = f.get_slice(key).get_shape()
|
| 160 |
+
return keys_dict
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def _convert_state_dict_to_keys_dict(state_dict):
|
| 164 |
+
keys_dict = {}
|
| 165 |
+
for key, value in state_dict.items():
|
| 166 |
+
if isinstance(value, torch.Tensor):
|
| 167 |
+
keys_dict[key] = list(value.shape)
|
| 168 |
+
else:
|
| 169 |
+
keys_dict[key] = _convert_state_dict_to_keys_dict(value)
|
| 170 |
+
return keys_dict
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _load_keys_dict_from_bin(file_path):
|
| 174 |
+
state_dict = load_state_dict_from_bin(file_path)
|
| 175 |
+
return _convert_state_dict_to_keys_dict(state_dict)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _load_keys_dict(file_path):
|
| 179 |
+
if isinstance(file_path, list):
|
| 180 |
+
merged = {}
|
| 181 |
+
for path in file_path:
|
| 182 |
+
merged.update(_load_keys_dict(path))
|
| 183 |
+
return merged
|
| 184 |
+
if file_path.endswith(".safetensors"):
|
| 185 |
+
return _load_keys_dict_from_safetensors(file_path)
|
| 186 |
+
return _load_keys_dict_from_bin(file_path)
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def _convert_keys_dict_to_single_str(keys_dict, with_shape=True):
|
| 190 |
+
keys = []
|
| 191 |
+
for key, value in keys_dict.items():
|
| 192 |
+
if isinstance(key, str):
|
| 193 |
+
if isinstance(value, dict):
|
| 194 |
+
keys.append(key + "|" + _convert_keys_dict_to_single_str(value, with_shape=with_shape))
|
| 195 |
+
else:
|
| 196 |
+
if with_shape:
|
| 197 |
+
shape = "_".join(map(str, list(value)))
|
| 198 |
+
keys.append(key + ":" + shape)
|
| 199 |
+
keys.append(key)
|
| 200 |
+
keys.sort()
|
| 201 |
+
return ",".join(keys)
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def hash_model_file(path, with_shape=True):
|
| 205 |
+
keys_dict = _load_keys_dict(path)
|
| 206 |
+
keys_str = _convert_keys_dict_to_single_str(keys_dict, with_shape=with_shape).encode("UTF-8")
|
| 207 |
+
return hashlib.md5(keys_str).hexdigest()
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/loader.py
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
import inspect
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import time
|
| 7 |
+
|
| 8 |
+
from .io import ModelConfig, hash_model_file, load_state_dict
|
| 9 |
+
from .state_dict_converters import (
|
| 10 |
+
wan_video_vae_state_dict_converter,
|
| 11 |
+
)
|
| 12 |
+
from ..wan_video_dit import WanVideoDiT
|
| 13 |
+
from ..wan_video_text_encoder import HuggingfaceTokenizer, WanTextEncoder
|
| 14 |
+
from ..wan_video_vae import WanVideoVAE38
|
| 15 |
+
from fastwam.utils.logging_config import get_logger
|
| 16 |
+
|
| 17 |
+
logger = get_logger(__name__)
|
| 18 |
+
SKIPPED_PRETRAIN_SENTINEL = "SKIPPED_PRETRAIN"
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass
|
| 22 |
+
class Wan22LoadedComponents:
|
| 23 |
+
dit: WanVideoDiT
|
| 24 |
+
vae: WanVideoVAE38
|
| 25 |
+
text_encoder: WanTextEncoder | None
|
| 26 |
+
tokenizer: HuggingfaceTokenizer | None
|
| 27 |
+
dit_path: str
|
| 28 |
+
vae_path: str
|
| 29 |
+
text_encoder_path: str | None
|
| 30 |
+
tokenizer_path: str | None
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
WAN22_MODEL_REGISTRY = [
|
| 34 |
+
{
|
| 35 |
+
# Example: ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth")
|
| 36 |
+
"model_hash": "9c8818c2cbea55eca56c7b447df170da",
|
| 37 |
+
"model_name": "wan_video_text_encoder",
|
| 38 |
+
"model_class": WanTextEncoder,
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
# Example: ModelConfig(model_id="Wan-AI/Wan2.2-TI2V-5B", origin_file_pattern="diffusion_pytorch_model*.safetensors")
|
| 42 |
+
"model_hash": "1f5ab7703c6fc803fdded85ff040c316",
|
| 43 |
+
"model_name": "wan_video_dit",
|
| 44 |
+
"model_class": WanVideoDiT,
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
# Example: ModelConfig(model_id="Wan-AI/Wan2.2-TI2V-5B", origin_file_pattern="Wan2.2_VAE.pth")
|
| 48 |
+
"model_hash": "e1de6c02cdac79f8b739f4d3698cd216",
|
| 49 |
+
"model_name": "wan_video_vae",
|
| 50 |
+
"model_class": WanVideoVAE38,
|
| 51 |
+
"state_dict_converter": wan_video_vae_state_dict_converter,
|
| 52 |
+
},
|
| 53 |
+
]
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _validate_dit_config(dit_config: dict[str, Any]) -> dict[str, Any]:
|
| 57 |
+
if not isinstance(dit_config, dict):
|
| 58 |
+
raise ValueError(f"`dit_config` must be a dict, got {type(dit_config)}")
|
| 59 |
+
|
| 60 |
+
validated = dict(dit_config)
|
| 61 |
+
|
| 62 |
+
signature = inspect.signature(WanVideoDiT.__init__)
|
| 63 |
+
allowed_keys = set()
|
| 64 |
+
required_keys = set()
|
| 65 |
+
for name, param in signature.parameters.items():
|
| 66 |
+
if name == "self":
|
| 67 |
+
continue
|
| 68 |
+
allowed_keys.add(name)
|
| 69 |
+
if param.default is inspect.Signature.empty:
|
| 70 |
+
required_keys.add(name)
|
| 71 |
+
|
| 72 |
+
unknown_keys = sorted(set(validated) - allowed_keys)
|
| 73 |
+
if unknown_keys:
|
| 74 |
+
raise ValueError(
|
| 75 |
+
f"Unknown keys in `dit_config`: {unknown_keys}. "
|
| 76 |
+
f"Allowed keys: {sorted(allowed_keys)}"
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
missing_keys = sorted(required_keys - set(validated))
|
| 80 |
+
if missing_keys:
|
| 81 |
+
raise ValueError(
|
| 82 |
+
f"Missing required keys in `dit_config`: {missing_keys}. "
|
| 83 |
+
"Please specify all required WanVideoDiT constructor args."
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
return validated
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _load_registered_model(
|
| 90 |
+
path,
|
| 91 |
+
model_name: str,
|
| 92 |
+
torch_dtype: torch.dtype,
|
| 93 |
+
device: str,
|
| 94 |
+
model_kwargs_override: dict[str, Any] | None = None,
|
| 95 |
+
):
|
| 96 |
+
model_hash = hash_model_file(path)
|
| 97 |
+
|
| 98 |
+
matched_config = None
|
| 99 |
+
for config in WAN22_MODEL_REGISTRY:
|
| 100 |
+
if config["model_hash"] == model_hash and config["model_name"] == model_name:
|
| 101 |
+
matched_config = config
|
| 102 |
+
break
|
| 103 |
+
if matched_config is None:
|
| 104 |
+
raise ValueError(
|
| 105 |
+
f"Cannot detect model type for {model_name}. File: {path}. "
|
| 106 |
+
f"Model hash: {model_hash}. This standalone package follows DiffSynth hash-based loading."
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
+
model_class = matched_config["model_class"]
|
| 110 |
+
model_kwargs = dict(matched_config.get("extra_kwargs", {}))
|
| 111 |
+
if model_kwargs_override is not None:
|
| 112 |
+
model_kwargs.update(model_kwargs_override)
|
| 113 |
+
state_dict_converter = matched_config.get("state_dict_converter")
|
| 114 |
+
|
| 115 |
+
model = model_class(**model_kwargs)
|
| 116 |
+
state_dict = load_state_dict(path, torch_dtype=torch_dtype, device="cpu")
|
| 117 |
+
if state_dict_converter is not None:
|
| 118 |
+
state_dict = state_dict_converter(state_dict)
|
| 119 |
+
|
| 120 |
+
model.load_state_dict(state_dict, strict=False)
|
| 121 |
+
model = model.to(device=device, dtype=torch_dtype)
|
| 122 |
+
return model
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def _resolve_configs(model_id: str, tokenizer_model_id: str, redirect_common_files: bool = True):
|
| 126 |
+
dit_config = ModelConfig(model_id=model_id, origin_file_pattern="diffusion_pytorch_model*.safetensors")
|
| 127 |
+
text_config = ModelConfig(model_id=model_id, origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth")
|
| 128 |
+
vae_config = ModelConfig(model_id=model_id, origin_file_pattern="Wan2.2_VAE.pth")
|
| 129 |
+
tokenizer_config = ModelConfig(model_id=tokenizer_model_id, origin_file_pattern="google/umt5-xxl/")
|
| 130 |
+
|
| 131 |
+
if redirect_common_files:
|
| 132 |
+
redirect_dict = {
|
| 133 |
+
"models_t5_umt5-xxl-enc-bf16.pth": ("DiffSynth-Studio/Wan-Series-Converted-Safetensors", "models_t5_umt5-xxl-enc-bf16.safetensors"),
|
| 134 |
+
"Wan2.2_VAE.pth": ("DiffSynth-Studio/Wan-Series-Converted-Safetensors", "Wan2.2_VAE.safetensors"),
|
| 135 |
+
}
|
| 136 |
+
text_config.model_id, text_config.origin_file_pattern = redirect_dict[text_config.origin_file_pattern]
|
| 137 |
+
vae_config.model_id, vae_config.origin_file_pattern = redirect_dict[vae_config.origin_file_pattern]
|
| 138 |
+
return dit_config, text_config, vae_config, tokenizer_config
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def load_wan22_ti2v_5b_components(
|
| 142 |
+
device: str = "cuda",
|
| 143 |
+
torch_dtype: torch.dtype = torch.bfloat16,
|
| 144 |
+
model_id: str = "Wan-AI/Wan2.2-TI2V-5B",
|
| 145 |
+
tokenizer_model_id: str = "Wan-AI/Wan2.1-T2V-1.3B",
|
| 146 |
+
tokenizer_max_len: int = 512,
|
| 147 |
+
redirect_common_files: bool = True,
|
| 148 |
+
dit_config: dict[str, Any] | None = None,
|
| 149 |
+
skip_dit_load_from_pretrain: bool = False,
|
| 150 |
+
load_text_encoder: bool = True,
|
| 151 |
+
):
|
| 152 |
+
logger.info("Loading Wan2.2-TI2V-5B components...")
|
| 153 |
+
start = time.time()
|
| 154 |
+
|
| 155 |
+
if dit_config is None:
|
| 156 |
+
raise ValueError("`dit_config` is required for Wan2.2-TI2V-5B loading.")
|
| 157 |
+
validated_dit_config = _validate_dit_config(dit_config)
|
| 158 |
+
|
| 159 |
+
dit_model_config, text_config, vae_config, tokenizer_config = _resolve_configs(
|
| 160 |
+
model_id=model_id,
|
| 161 |
+
tokenizer_model_id=tokenizer_model_id,
|
| 162 |
+
redirect_common_files=redirect_common_files,
|
| 163 |
+
)
|
| 164 |
+
|
| 165 |
+
vae_config.download_if_necessary()
|
| 166 |
+
if load_text_encoder:
|
| 167 |
+
text_config.download_if_necessary()
|
| 168 |
+
tokenizer_config.download_if_necessary()
|
| 169 |
+
|
| 170 |
+
if skip_dit_load_from_pretrain:
|
| 171 |
+
logger.info(
|
| 172 |
+
"Skipping pretrained video DiT load (`skip_dit_load_from_pretrain=True`); "
|
| 173 |
+
"initializing video expert randomly and expecting checkpoint override."
|
| 174 |
+
)
|
| 175 |
+
dit: WanVideoDiT = WanVideoDiT(**validated_dit_config).to(device=device, dtype=torch_dtype)
|
| 176 |
+
dit_path = SKIPPED_PRETRAIN_SENTINEL
|
| 177 |
+
else:
|
| 178 |
+
dit_model_config.download_if_necessary()
|
| 179 |
+
dit = _load_registered_model(
|
| 180 |
+
dit_model_config.path,
|
| 181 |
+
"wan_video_dit",
|
| 182 |
+
torch_dtype=torch_dtype,
|
| 183 |
+
device=device,
|
| 184 |
+
model_kwargs_override=validated_dit_config,
|
| 185 |
+
)
|
| 186 |
+
dit_path = str(dit_model_config.path)
|
| 187 |
+
text_encoder: WanTextEncoder | None = None
|
| 188 |
+
tokenizer: HuggingfaceTokenizer | None = None
|
| 189 |
+
text_encoder_path: str | None = None
|
| 190 |
+
tokenizer_path: str | None = None
|
| 191 |
+
if load_text_encoder:
|
| 192 |
+
text_encoder = _load_registered_model(
|
| 193 |
+
text_config.path,
|
| 194 |
+
"wan_video_text_encoder",
|
| 195 |
+
torch_dtype=torch_dtype,
|
| 196 |
+
device=device,
|
| 197 |
+
)
|
| 198 |
+
tokenizer = HuggingfaceTokenizer(
|
| 199 |
+
name=tokenizer_config.path,
|
| 200 |
+
seq_len=int(tokenizer_max_len),
|
| 201 |
+
clean="whitespace",
|
| 202 |
+
)
|
| 203 |
+
text_encoder_path = str(text_config.path)
|
| 204 |
+
tokenizer_path = str(tokenizer_config.path)
|
| 205 |
+
else:
|
| 206 |
+
logger.info(
|
| 207 |
+
"Skipping pretrained text encoder/tokenizer load (`load_text_encoder=False`); "
|
| 208 |
+
"training must provide cached `context/context_mask`."
|
| 209 |
+
)
|
| 210 |
+
vae: WanVideoVAE38 = _load_registered_model(vae_config.path, "wan_video_vae", torch_dtype=torch_dtype, device=device)
|
| 211 |
+
logger.info("Finished loading Wan2.2-TI2V-5B components in %.2f seconds.", time.time() - start)
|
| 212 |
+
return Wan22LoadedComponents(
|
| 213 |
+
dit=dit,
|
| 214 |
+
vae=vae,
|
| 215 |
+
text_encoder=text_encoder,
|
| 216 |
+
tokenizer=tokenizer,
|
| 217 |
+
dit_path=dit_path,
|
| 218 |
+
vae_path=str(vae_config.path),
|
| 219 |
+
text_encoder_path=text_encoder_path,
|
| 220 |
+
tokenizer_path=tokenizer_path,
|
| 221 |
+
)
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/helpers/state_dict_converters.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def wan_video_vae_state_dict_converter(state_dict):
|
| 2 |
+
converted = {}
|
| 3 |
+
if "model_state" in state_dict:
|
| 4 |
+
state_dict = state_dict["model_state"]
|
| 5 |
+
for name, value in state_dict.items():
|
| 6 |
+
converted[f"model.{name}"] = value
|
| 7 |
+
return converted
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def wan_video_dit_from_diffusers(state_dict):
|
| 11 |
+
rename_dict = {
|
| 12 |
+
"blocks.0.attn1.norm_k.weight": "blocks.0.self_attn.norm_k.weight",
|
| 13 |
+
"blocks.0.attn1.norm_q.weight": "blocks.0.self_attn.norm_q.weight",
|
| 14 |
+
"blocks.0.attn1.to_k.bias": "blocks.0.self_attn.k.bias",
|
| 15 |
+
"blocks.0.attn1.to_k.weight": "blocks.0.self_attn.k.weight",
|
| 16 |
+
"blocks.0.attn1.to_out.0.bias": "blocks.0.self_attn.o.bias",
|
| 17 |
+
"blocks.0.attn1.to_out.0.weight": "blocks.0.self_attn.o.weight",
|
| 18 |
+
"blocks.0.attn1.to_q.bias": "blocks.0.self_attn.q.bias",
|
| 19 |
+
"blocks.0.attn1.to_q.weight": "blocks.0.self_attn.q.weight",
|
| 20 |
+
"blocks.0.attn1.to_v.bias": "blocks.0.self_attn.v.bias",
|
| 21 |
+
"blocks.0.attn1.to_v.weight": "blocks.0.self_attn.v.weight",
|
| 22 |
+
"blocks.0.attn2.norm_k.weight": "blocks.0.cross_attn.norm_k.weight",
|
| 23 |
+
"blocks.0.attn2.norm_q.weight": "blocks.0.cross_attn.norm_q.weight",
|
| 24 |
+
"blocks.0.attn2.to_k.bias": "blocks.0.cross_attn.k.bias",
|
| 25 |
+
"blocks.0.attn2.to_k.weight": "blocks.0.cross_attn.k.weight",
|
| 26 |
+
"blocks.0.attn2.to_out.0.bias": "blocks.0.cross_attn.o.bias",
|
| 27 |
+
"blocks.0.attn2.to_out.0.weight": "blocks.0.cross_attn.o.weight",
|
| 28 |
+
"blocks.0.attn2.to_q.bias": "blocks.0.cross_attn.q.bias",
|
| 29 |
+
"blocks.0.attn2.to_q.weight": "blocks.0.cross_attn.q.weight",
|
| 30 |
+
"blocks.0.attn2.to_v.bias": "blocks.0.cross_attn.v.bias",
|
| 31 |
+
"blocks.0.attn2.to_v.weight": "blocks.0.cross_attn.v.weight",
|
| 32 |
+
"blocks.0.attn2.add_k_proj.bias": "blocks.0.cross_attn.k_img.bias",
|
| 33 |
+
"blocks.0.attn2.add_k_proj.weight": "blocks.0.cross_attn.k_img.weight",
|
| 34 |
+
"blocks.0.attn2.add_v_proj.bias": "blocks.0.cross_attn.v_img.bias",
|
| 35 |
+
"blocks.0.attn2.add_v_proj.weight": "blocks.0.cross_attn.v_img.weight",
|
| 36 |
+
"blocks.0.attn2.norm_added_k.weight": "blocks.0.cross_attn.norm_k_img.weight",
|
| 37 |
+
"blocks.0.ffn.net.0.proj.bias": "blocks.0.ffn.0.bias",
|
| 38 |
+
"blocks.0.ffn.net.0.proj.weight": "blocks.0.ffn.0.weight",
|
| 39 |
+
"blocks.0.ffn.net.2.bias": "blocks.0.ffn.2.bias",
|
| 40 |
+
"blocks.0.ffn.net.2.weight": "blocks.0.ffn.2.weight",
|
| 41 |
+
"blocks.0.norm2.bias": "blocks.0.norm3.bias",
|
| 42 |
+
"blocks.0.norm2.weight": "blocks.0.norm3.weight",
|
| 43 |
+
"blocks.0.scale_shift_table": "blocks.0.modulation",
|
| 44 |
+
"condition_embedder.text_embedder.linear_1.bias": "text_embedding.0.bias",
|
| 45 |
+
"condition_embedder.text_embedder.linear_1.weight": "text_embedding.0.weight",
|
| 46 |
+
"condition_embedder.text_embedder.linear_2.bias": "text_embedding.2.bias",
|
| 47 |
+
"condition_embedder.text_embedder.linear_2.weight": "text_embedding.2.weight",
|
| 48 |
+
"condition_embedder.time_embedder.linear_1.bias": "time_embedding.0.bias",
|
| 49 |
+
"condition_embedder.time_embedder.linear_1.weight": "time_embedding.0.weight",
|
| 50 |
+
"condition_embedder.time_embedder.linear_2.bias": "time_embedding.2.bias",
|
| 51 |
+
"condition_embedder.time_embedder.linear_2.weight": "time_embedding.2.weight",
|
| 52 |
+
"condition_embedder.time_proj.bias": "time_projection.1.bias",
|
| 53 |
+
"condition_embedder.time_proj.weight": "time_projection.1.weight",
|
| 54 |
+
"condition_embedder.image_embedder.ff.net.0.proj.bias": "img_emb.proj.1.bias",
|
| 55 |
+
"condition_embedder.image_embedder.ff.net.0.proj.weight": "img_emb.proj.1.weight",
|
| 56 |
+
"condition_embedder.image_embedder.ff.net.2.bias": "img_emb.proj.3.bias",
|
| 57 |
+
"condition_embedder.image_embedder.ff.net.2.weight": "img_emb.proj.3.weight",
|
| 58 |
+
"condition_embedder.image_embedder.norm1.bias": "img_emb.proj.0.bias",
|
| 59 |
+
"condition_embedder.image_embedder.norm1.weight": "img_emb.proj.0.weight",
|
| 60 |
+
"condition_embedder.image_embedder.norm2.bias": "img_emb.proj.4.bias",
|
| 61 |
+
"condition_embedder.image_embedder.norm2.weight": "img_emb.proj.4.weight",
|
| 62 |
+
"patch_embedding.bias": "patch_embedding.bias",
|
| 63 |
+
"patch_embedding.weight": "patch_embedding.weight",
|
| 64 |
+
"scale_shift_table": "head.modulation",
|
| 65 |
+
"proj_out.bias": "head.head.bias",
|
| 66 |
+
"proj_out.weight": "head.head.weight",
|
| 67 |
+
}
|
| 68 |
+
converted = {}
|
| 69 |
+
for name, value in state_dict.items():
|
| 70 |
+
if name in rename_dict:
|
| 71 |
+
converted[rename_dict[name]] = value
|
| 72 |
+
else:
|
| 73 |
+
probe_name = ".".join(name.split(".")[:1] + ["0"] + name.split(".")[2:])
|
| 74 |
+
if probe_name in rename_dict:
|
| 75 |
+
mapped = rename_dict[probe_name]
|
| 76 |
+
mapped = ".".join(mapped.split(".")[:1] + [name.split(".")[1]] + mapped.split(".")[2:])
|
| 77 |
+
converted[mapped] = value
|
| 78 |
+
return converted
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def wan_video_dit_state_dict_converter(state_dict):
|
| 82 |
+
converted = {}
|
| 83 |
+
for name, value in state_dict.items():
|
| 84 |
+
if name.startswith("vace"):
|
| 85 |
+
continue
|
| 86 |
+
if name.split(".")[0] in ["pose_patch_embedding", "face_adapter", "face_encoder", "motion_encoder"]:
|
| 87 |
+
continue
|
| 88 |
+
key = name[6:] if name.startswith("model.") else name
|
| 89 |
+
converted[key] = value
|
| 90 |
+
return converted
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/mot.py
ADDED
|
@@ -0,0 +1,563 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Dict, Optional
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
|
| 8 |
+
from .wan_video_dit import flash_attention, modulate, rope_apply
|
| 9 |
+
from fastwam.utils.logging_config import get_logger
|
| 10 |
+
|
| 11 |
+
logger = get_logger(__name__)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class MoT(nn.Module):
|
| 15 |
+
def __init__(
|
| 16 |
+
self,
|
| 17 |
+
mixtures: Dict[str, nn.Module],
|
| 18 |
+
mot_checkpoint_mixed_attn: bool = True,
|
| 19 |
+
):
|
| 20 |
+
super().__init__()
|
| 21 |
+
if not mixtures:
|
| 22 |
+
raise ValueError("`mixtures` cannot be empty.")
|
| 23 |
+
if "video" not in mixtures or "action" not in mixtures:
|
| 24 |
+
raise ValueError("`mixtures` must include both 'video' and 'action' experts.")
|
| 25 |
+
|
| 26 |
+
self.mixtures = nn.ModuleDict(mixtures)
|
| 27 |
+
self.expert_order = list(self.mixtures.keys())
|
| 28 |
+
self.mot_checkpoint_mixed_attn = mot_checkpoint_mixed_attn
|
| 29 |
+
# Opt-in performance switches. Defaults preserve the historical path.
|
| 30 |
+
self.omit_all_true_attention_mask = False
|
| 31 |
+
self.rope_compute_dtype = torch.float64
|
| 32 |
+
if mot_checkpoint_mixed_attn:
|
| 33 |
+
logger.info("Using gradient checkpointing for mixture attention. This will save memory but use more computation.")
|
| 34 |
+
|
| 35 |
+
first_expert = self.mixtures[self.expert_order[0]]
|
| 36 |
+
self.num_layers = len(first_expert.blocks)
|
| 37 |
+
self.num_heads = first_expert.num_heads
|
| 38 |
+
self.attn_head_dim = first_expert.attn_head_dim
|
| 39 |
+
|
| 40 |
+
for name in self.expert_order[1:]:
|
| 41 |
+
expert = self.mixtures[name]
|
| 42 |
+
if len(expert.blocks) != self.num_layers:
|
| 43 |
+
raise ValueError(
|
| 44 |
+
f"All experts must have same number of layers; got {self.num_layers} and {len(expert.blocks)}"
|
| 45 |
+
)
|
| 46 |
+
if expert.num_heads != self.num_heads:
|
| 47 |
+
raise ValueError(
|
| 48 |
+
f"All experts must have same num_heads; got {self.num_heads} and {expert.num_heads}"
|
| 49 |
+
)
|
| 50 |
+
if expert.attn_head_dim != self.attn_head_dim:
|
| 51 |
+
raise ValueError(
|
| 52 |
+
"All experts must have same attn_head_dim; "
|
| 53 |
+
f"got {self.attn_head_dim} and {expert.attn_head_dim}"
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
logger.info(f"Initialized MoT with experts: {self.expert_order}, num_layers={self.num_layers}")
|
| 57 |
+
for name in self.expert_order:
|
| 58 |
+
expert = self.mixtures[name]
|
| 59 |
+
logger.info(f" Expert '{name}': num_params={sum(p.numel() for p in expert.parameters()) / 1e9:.2f} B")
|
| 60 |
+
|
| 61 |
+
@staticmethod
|
| 62 |
+
def _split_modulation(block, t_mod: torch.Tensor):
|
| 63 |
+
has_seq = len(t_mod.shape) == 4
|
| 64 |
+
chunk_dim = 2 if has_seq else 1
|
| 65 |
+
|
| 66 |
+
base_mod = block.modulation.to(dtype=t_mod.dtype, device=t_mod.device)
|
| 67 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (base_mod + t_mod).chunk(6, dim=chunk_dim)
|
| 68 |
+
if has_seq:
|
| 69 |
+
# means t_mod has separate modulation for each token, otherwise same modulation for all tokens in the block
|
| 70 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
|
| 71 |
+
shift_msa.squeeze(2),
|
| 72 |
+
scale_msa.squeeze(2),
|
| 73 |
+
gate_msa.squeeze(2),
|
| 74 |
+
shift_mlp.squeeze(2),
|
| 75 |
+
scale_mlp.squeeze(2),
|
| 76 |
+
gate_mlp.squeeze(2),
|
| 77 |
+
)
|
| 78 |
+
return shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp
|
| 79 |
+
|
| 80 |
+
def _mixed_attention(
|
| 81 |
+
self,
|
| 82 |
+
q_cat: torch.Tensor,
|
| 83 |
+
k_cat: torch.Tensor,
|
| 84 |
+
v_cat: torch.Tensor,
|
| 85 |
+
attention_mask: Optional[torch.Tensor],
|
| 86 |
+
) -> torch.Tensor:
|
| 87 |
+
attn_mask = (
|
| 88 |
+
None
|
| 89 |
+
if attention_mask is None or self.omit_all_true_attention_mask
|
| 90 |
+
else attention_mask.to(device=q_cat.device)
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
def _forward(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
|
| 94 |
+
return flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=attn_mask)
|
| 95 |
+
|
| 96 |
+
if self.mot_checkpoint_mixed_attn and self.training:
|
| 97 |
+
return torch.utils.checkpoint.checkpoint(
|
| 98 |
+
_forward,
|
| 99 |
+
q_cat,
|
| 100 |
+
k_cat,
|
| 101 |
+
v_cat,
|
| 102 |
+
use_reentrant=False,
|
| 103 |
+
)
|
| 104 |
+
return _forward(q_cat, k_cat, v_cat)
|
| 105 |
+
|
| 106 |
+
@staticmethod
|
| 107 |
+
def _apply_expert_post_block(
|
| 108 |
+
block,
|
| 109 |
+
residual_x: torch.Tensor,
|
| 110 |
+
mixed_attn_out: torch.Tensor,
|
| 111 |
+
gate_msa: torch.Tensor,
|
| 112 |
+
shift_mlp: torch.Tensor,
|
| 113 |
+
scale_mlp: torch.Tensor,
|
| 114 |
+
gate_mlp: torch.Tensor,
|
| 115 |
+
context_payload: Optional[dict],
|
| 116 |
+
) -> torch.Tensor:
|
| 117 |
+
x = block.gate(residual_x, gate_msa, block.self_attn.o(mixed_attn_out))
|
| 118 |
+
|
| 119 |
+
if context_payload is not None:
|
| 120 |
+
context = context_payload.get("context")
|
| 121 |
+
if context is not None:
|
| 122 |
+
context_mask = context_payload.get("mask")
|
| 123 |
+
if context_mask is not None and context_mask.dim() == 3:
|
| 124 |
+
context_mask = context_mask.unsqueeze(1)
|
| 125 |
+
x = x + block.cross_attn(block.norm3(x), context, ctx_mask=context_mask)
|
| 126 |
+
|
| 127 |
+
mlp_input = modulate(block.norm2(x), shift_mlp, scale_mlp)
|
| 128 |
+
x = block.gate(x, gate_mlp, block.ffn(mlp_input))
|
| 129 |
+
return x
|
| 130 |
+
|
| 131 |
+
def _build_expert_attention_io(
|
| 132 |
+
self,
|
| 133 |
+
expert,
|
| 134 |
+
block,
|
| 135 |
+
x: torch.Tensor,
|
| 136 |
+
freqs: torch.Tensor,
|
| 137 |
+
t_mod: torch.Tensor,
|
| 138 |
+
) -> tuple[
|
| 139 |
+
torch.Tensor,
|
| 140 |
+
torch.Tensor,
|
| 141 |
+
torch.Tensor,
|
| 142 |
+
torch.Tensor,
|
| 143 |
+
torch.Tensor,
|
| 144 |
+
torch.Tensor,
|
| 145 |
+
torch.Tensor,
|
| 146 |
+
torch.Tensor,
|
| 147 |
+
bool,
|
| 148 |
+
]:
|
| 149 |
+
"""Build per-expert attention tensors and post-block states.
|
| 150 |
+
|
| 151 |
+
Args:
|
| 152 |
+
expert: Expert module that owns this `block`; only used to read
|
| 153 |
+
`use_gradient_checkpointing`.
|
| 154 |
+
block: Transformer block for current layer (`expert.blocks[layer_idx]`).
|
| 155 |
+
x: Current expert tokens, shape [B, S, D].
|
| 156 |
+
freqs: RoPE frequencies aligned with token sequence, shape [S, 1, rope_dim].
|
| 157 |
+
t_mod: Time modulation tensor for this expert/layer.
|
| 158 |
+
|
| 159 |
+
Returns:
|
| 160 |
+
q: Query after q-proj, RMSNorm, and RoPE, shape [B, S, H*Dh].
|
| 161 |
+
k: Key after k-proj, RMSNorm, and RoPE, shape [B, S, H*Dh].
|
| 162 |
+
v: Value after v-proj, shape [B, S, H*Dh].
|
| 163 |
+
residual_x: Original input `x` for residual path in post block.
|
| 164 |
+
gate_msa: Gating tensor for self-attention residual branch.
|
| 165 |
+
shift_mlp: Shift tensor for MLP modulation.
|
| 166 |
+
scale_mlp: Scale tensor for MLP modulation.
|
| 167 |
+
gate_mlp: Gating tensor for MLP residual branch.
|
| 168 |
+
use_gradient_checkpointing: Whether this expert enables checkpointing.
|
| 169 |
+
"""
|
| 170 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self._split_modulation(block, t_mod)
|
| 171 |
+
attn_input = modulate(block.norm1(x), shift_msa, scale_msa)
|
| 172 |
+
|
| 173 |
+
q = block.self_attn.norm_q(block.self_attn.q(attn_input))
|
| 174 |
+
k = block.self_attn.norm_k(block.self_attn.k(attn_input))
|
| 175 |
+
v = block.self_attn.v(attn_input)
|
| 176 |
+
|
| 177 |
+
q = rope_apply(q, freqs, block.num_heads, self.rope_compute_dtype)
|
| 178 |
+
k = rope_apply(k, freqs, block.num_heads, self.rope_compute_dtype)
|
| 179 |
+
|
| 180 |
+
use_gradient_checkpointing = bool(getattr(expert, "use_gradient_checkpointing", False))
|
| 181 |
+
return (
|
| 182 |
+
q,
|
| 183 |
+
k,
|
| 184 |
+
v,
|
| 185 |
+
x,
|
| 186 |
+
gate_msa,
|
| 187 |
+
shift_mlp,
|
| 188 |
+
scale_mlp,
|
| 189 |
+
gate_mlp,
|
| 190 |
+
use_gradient_checkpointing,
|
| 191 |
+
)
|
| 192 |
+
|
| 193 |
+
def _apply_post_with_optional_checkpoint(
|
| 194 |
+
self,
|
| 195 |
+
block,
|
| 196 |
+
residual_x: torch.Tensor,
|
| 197 |
+
gate_msa: torch.Tensor,
|
| 198 |
+
shift_mlp: torch.Tensor,
|
| 199 |
+
scale_mlp: torch.Tensor,
|
| 200 |
+
gate_mlp: torch.Tensor,
|
| 201 |
+
use_gradient_checkpointing: bool,
|
| 202 |
+
mixed_slice: torch.Tensor,
|
| 203 |
+
context_payload: Optional[dict],
|
| 204 |
+
) -> torch.Tensor:
|
| 205 |
+
"""Apply post-attention computations, with optional checkpointing.
|
| 206 |
+
|
| 207 |
+
Args:
|
| 208 |
+
block: Transformer block for current layer.
|
| 209 |
+
residual_x: Residual input tokens before attention update, shape [B, S, D].
|
| 210 |
+
gate_msa: Gating tensor used after mixed self-attention.
|
| 211 |
+
shift_mlp: Shift tensor for MLP input modulation.
|
| 212 |
+
scale_mlp: Scale tensor for MLP input modulation.
|
| 213 |
+
gate_mlp: Gating tensor used after MLP.
|
| 214 |
+
use_gradient_checkpointing: If True and training, checkpoint this post block.
|
| 215 |
+
mixed_slice: Mixed-attention output for this expert, shape [B, S, H*Dh].
|
| 216 |
+
context_payload: Optional dict for cross-attention.
|
| 217 |
+
- `context`: encoder states [B, L, D]
|
| 218 |
+
- `mask`: attention mask [B, S, L] or [B, 1, S, L]
|
| 219 |
+
|
| 220 |
+
Returns:
|
| 221 |
+
Updated expert tokens after self-attn residual, optional cross-attn, and MLP.
|
| 222 |
+
"""
|
| 223 |
+
def _post_fn(
|
| 224 |
+
_mixed_slice: torch.Tensor,
|
| 225 |
+
_x: torch.Tensor,
|
| 226 |
+
_gate_msa: torch.Tensor,
|
| 227 |
+
_shift_mlp: torch.Tensor,
|
| 228 |
+
_scale_mlp: torch.Tensor,
|
| 229 |
+
_gate_mlp: torch.Tensor,
|
| 230 |
+
_block=block,
|
| 231 |
+
_context_payload=context_payload,
|
| 232 |
+
) -> torch.Tensor:
|
| 233 |
+
return self._apply_expert_post_block(
|
| 234 |
+
block=_block,
|
| 235 |
+
residual_x=_x,
|
| 236 |
+
mixed_attn_out=_mixed_slice,
|
| 237 |
+
gate_msa=_gate_msa,
|
| 238 |
+
shift_mlp=_shift_mlp,
|
| 239 |
+
scale_mlp=_scale_mlp,
|
| 240 |
+
gate_mlp=_gate_mlp,
|
| 241 |
+
context_payload=_context_payload,
|
| 242 |
+
)
|
| 243 |
+
|
| 244 |
+
if use_gradient_checkpointing and self.training:
|
| 245 |
+
return torch.utils.checkpoint.checkpoint(
|
| 246 |
+
_post_fn,
|
| 247 |
+
mixed_slice,
|
| 248 |
+
residual_x,
|
| 249 |
+
gate_msa,
|
| 250 |
+
shift_mlp,
|
| 251 |
+
scale_mlp,
|
| 252 |
+
gate_mlp,
|
| 253 |
+
use_reentrant=False,
|
| 254 |
+
)
|
| 255 |
+
return _post_fn(
|
| 256 |
+
mixed_slice,
|
| 257 |
+
residual_x,
|
| 258 |
+
gate_msa,
|
| 259 |
+
shift_mlp,
|
| 260 |
+
scale_mlp,
|
| 261 |
+
gate_mlp,
|
| 262 |
+
)
|
| 263 |
+
|
| 264 |
+
def prefill_video_cache(
|
| 265 |
+
self,
|
| 266 |
+
video_tokens: torch.Tensor,
|
| 267 |
+
video_freqs: torch.Tensor,
|
| 268 |
+
video_t_mod: torch.Tensor,
|
| 269 |
+
video_context_payload: Optional[dict],
|
| 270 |
+
video_attention_mask: torch.Tensor,
|
| 271 |
+
) -> list[dict[str, torch.Tensor]]:
|
| 272 |
+
"""Prefill video branch once and cache per-layer K/V for action denoising.
|
| 273 |
+
|
| 274 |
+
Args:
|
| 275 |
+
video_tokens: Video tokens before layer 0, shape [B, Sv, D].
|
| 276 |
+
video_freqs: Video RoPE frequencies, shape [Sv, 1, rope_dim].
|
| 277 |
+
video_t_mod: Video time modulation tensor.
|
| 278 |
+
video_context_payload: Optional dict for video cross-attention.
|
| 279 |
+
- `context`: encoder states [B, L, D]
|
| 280 |
+
- `mask`: attention mask [B, Sv, L] or [B, 1, Sv, L]
|
| 281 |
+
video_attention_mask: Video self-attention mask, shape [Sv, Sv].
|
| 282 |
+
|
| 283 |
+
Returns:
|
| 284 |
+
Layer-wise cache list with length `num_layers`.
|
| 285 |
+
Each entry contains:
|
| 286 |
+
- `k`: video key tensor [B, Sv, H*Dh]
|
| 287 |
+
- `v`: video value tensor [B, Sv, H*Dh]
|
| 288 |
+
"""
|
| 289 |
+
if "video" not in self.mixtures:
|
| 290 |
+
raise ValueError("MoT requires `video` expert for `prefill_video_cache`.")
|
| 291 |
+
if video_attention_mask.ndim != 2:
|
| 292 |
+
raise ValueError(
|
| 293 |
+
f"`video_attention_mask` must be 2D [S,S], got shape {tuple(video_attention_mask.shape)}"
|
| 294 |
+
)
|
| 295 |
+
if video_attention_mask.shape[0] != video_attention_mask.shape[1]:
|
| 296 |
+
raise ValueError(
|
| 297 |
+
f"`video_attention_mask` must be square, got shape {tuple(video_attention_mask.shape)}"
|
| 298 |
+
)
|
| 299 |
+
if video_attention_mask.shape[0] != video_tokens.shape[1]:
|
| 300 |
+
raise ValueError(
|
| 301 |
+
"`video_attention_mask` seq length mismatch: "
|
| 302 |
+
f"mask={video_attention_mask.shape[0]} vs tokens={video_tokens.shape[1]}"
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
expert = self.mixtures["video"]
|
| 306 |
+
x = video_tokens
|
| 307 |
+
kv_cache: list[dict[str, torch.Tensor]] = []
|
| 308 |
+
for layer_idx in range(self.num_layers):
|
| 309 |
+
block = expert.blocks[layer_idx]
|
| 310 |
+
# Build video Q/K/V from current layer input tokens.
|
| 311 |
+
(
|
| 312 |
+
q,
|
| 313 |
+
k,
|
| 314 |
+
v,
|
| 315 |
+
residual_x,
|
| 316 |
+
gate_msa,
|
| 317 |
+
shift_mlp,
|
| 318 |
+
scale_mlp,
|
| 319 |
+
gate_mlp,
|
| 320 |
+
use_gradient_checkpointing,
|
| 321 |
+
) = self._build_expert_attention_io(
|
| 322 |
+
expert=expert,
|
| 323 |
+
block=block,
|
| 324 |
+
x=x,
|
| 325 |
+
freqs=video_freqs,
|
| 326 |
+
t_mod=video_t_mod,
|
| 327 |
+
)
|
| 328 |
+
# Video prefill uses only video self-attention mask.
|
| 329 |
+
mixed = self._mixed_attention(
|
| 330 |
+
q_cat=q,
|
| 331 |
+
k_cat=k,
|
| 332 |
+
v_cat=v,
|
| 333 |
+
attention_mask=video_attention_mask,
|
| 334 |
+
)
|
| 335 |
+
# Update video tokens for the next layer and persist current layer K/V.
|
| 336 |
+
x = self._apply_post_with_optional_checkpoint(
|
| 337 |
+
block=block,
|
| 338 |
+
residual_x=residual_x,
|
| 339 |
+
gate_msa=gate_msa,
|
| 340 |
+
shift_mlp=shift_mlp,
|
| 341 |
+
scale_mlp=scale_mlp,
|
| 342 |
+
gate_mlp=gate_mlp,
|
| 343 |
+
use_gradient_checkpointing=use_gradient_checkpointing,
|
| 344 |
+
mixed_slice=mixed,
|
| 345 |
+
context_payload=video_context_payload,
|
| 346 |
+
)
|
| 347 |
+
kv_cache.append({"k": k, "v": v})
|
| 348 |
+
return kv_cache
|
| 349 |
+
|
| 350 |
+
def forward_action_with_video_cache(
|
| 351 |
+
self,
|
| 352 |
+
action_tokens: torch.Tensor,
|
| 353 |
+
action_freqs: torch.Tensor,
|
| 354 |
+
action_t_mod: torch.Tensor,
|
| 355 |
+
action_context_payload: Optional[dict],
|
| 356 |
+
video_kv_cache: list[dict[str, torch.Tensor]],
|
| 357 |
+
attention_mask: torch.Tensor,
|
| 358 |
+
video_seq_len: int,
|
| 359 |
+
) -> torch.Tensor:
|
| 360 |
+
"""Run action branch with cached video K/V instead of recomputing video tokens.
|
| 361 |
+
|
| 362 |
+
Args:
|
| 363 |
+
action_tokens: Action tokens before layer 0, shape [B, Sa, D].
|
| 364 |
+
action_freqs: Action RoPE frequencies, shape [Sa, 1, rope_dim].
|
| 365 |
+
action_t_mod: Action time modulation tensor.
|
| 366 |
+
action_context_payload: Optional dict for action cross-attention.
|
| 367 |
+
- `context`: encoder states [B, L, D]
|
| 368 |
+
- `mask`: attention mask [B, Sa, L] or [B, 1, Sa, L]
|
| 369 |
+
video_kv_cache: Layer-wise cached video K/V from `prefill_video_cache`.
|
| 370 |
+
attention_mask: Joint [video+action] mask, shape [Sv+Sa, Sv+Sa].
|
| 371 |
+
video_seq_len: Video token count `Sv` in the joint sequence prefix.
|
| 372 |
+
|
| 373 |
+
Returns:
|
| 374 |
+
Updated action tokens after all layers, shape [B, Sa, D].
|
| 375 |
+
"""
|
| 376 |
+
if "action" not in self.mixtures:
|
| 377 |
+
raise ValueError("MoT requires `action` expert for `forward_action_with_video_cache`.")
|
| 378 |
+
if len(video_kv_cache) != self.num_layers:
|
| 379 |
+
raise ValueError(
|
| 380 |
+
f"`video_kv_cache` must contain {self.num_layers} layers, got {len(video_kv_cache)}."
|
| 381 |
+
)
|
| 382 |
+
if attention_mask.ndim != 2:
|
| 383 |
+
raise ValueError(f"`attention_mask` must be 2D [S,S], got shape {tuple(attention_mask.shape)}")
|
| 384 |
+
if attention_mask.shape[0] != attention_mask.shape[1]:
|
| 385 |
+
raise ValueError(f"`attention_mask` must be square, got shape {tuple(attention_mask.shape)}")
|
| 386 |
+
|
| 387 |
+
action_seq_len = int(action_tokens.shape[1])
|
| 388 |
+
total_seq_len = int(video_seq_len) + action_seq_len
|
| 389 |
+
if attention_mask.shape[0] != total_seq_len:
|
| 390 |
+
raise ValueError(
|
| 391 |
+
"`attention_mask` seq length mismatch: "
|
| 392 |
+
f"mask={attention_mask.shape[0]} vs expected_total={total_seq_len}"
|
| 393 |
+
)
|
| 394 |
+
# Use the action query rows from the joint [video+action] mask.
|
| 395 |
+
action_attention_mask = attention_mask[video_seq_len:total_seq_len, :total_seq_len]
|
| 396 |
+
|
| 397 |
+
expert = self.mixtures["action"]
|
| 398 |
+
x = action_tokens
|
| 399 |
+
for layer_idx in range(self.num_layers):
|
| 400 |
+
block = expert.blocks[layer_idx]
|
| 401 |
+
# Action query/key/value are still step-dependent and must be recomputed each step.
|
| 402 |
+
(
|
| 403 |
+
q_action,
|
| 404 |
+
k_action,
|
| 405 |
+
v_action,
|
| 406 |
+
residual_x,
|
| 407 |
+
gate_msa,
|
| 408 |
+
shift_mlp,
|
| 409 |
+
scale_mlp,
|
| 410 |
+
gate_mlp,
|
| 411 |
+
use_gradient_checkpointing,
|
| 412 |
+
) = self._build_expert_attention_io(
|
| 413 |
+
expert=expert,
|
| 414 |
+
block=block,
|
| 415 |
+
x=x,
|
| 416 |
+
freqs=action_freqs,
|
| 417 |
+
t_mod=action_t_mod,
|
| 418 |
+
)
|
| 419 |
+
layer_cache = video_kv_cache[layer_idx]
|
| 420 |
+
if "k" not in layer_cache or "v" not in layer_cache:
|
| 421 |
+
raise ValueError(
|
| 422 |
+
f"`video_kv_cache[{layer_idx}]` must contain `k` and `v`."
|
| 423 |
+
)
|
| 424 |
+
|
| 425 |
+
k_video = layer_cache["k"]
|
| 426 |
+
v_video = layer_cache["v"]
|
| 427 |
+
if k_video.shape[1] != video_seq_len or v_video.shape[1] != video_seq_len:
|
| 428 |
+
raise ValueError(
|
| 429 |
+
f"`video_kv_cache[{layer_idx}]` seq len mismatch, expected {video_seq_len}."
|
| 430 |
+
)
|
| 431 |
+
|
| 432 |
+
# Mixed attention: action queries attend to cached video K/V plus current action K/V.
|
| 433 |
+
k_cat = torch.cat([k_video, k_action], dim=1)
|
| 434 |
+
v_cat = torch.cat([v_video, v_action], dim=1)
|
| 435 |
+
mixed = self._mixed_attention(
|
| 436 |
+
q_cat=q_action,
|
| 437 |
+
k_cat=k_cat,
|
| 438 |
+
v_cat=v_cat,
|
| 439 |
+
attention_mask=action_attention_mask,
|
| 440 |
+
)
|
| 441 |
+
x = self._apply_post_with_optional_checkpoint(
|
| 442 |
+
block=block,
|
| 443 |
+
residual_x=residual_x,
|
| 444 |
+
gate_msa=gate_msa,
|
| 445 |
+
shift_mlp=shift_mlp,
|
| 446 |
+
scale_mlp=scale_mlp,
|
| 447 |
+
gate_mlp=gate_mlp,
|
| 448 |
+
use_gradient_checkpointing=use_gradient_checkpointing,
|
| 449 |
+
mixed_slice=mixed,
|
| 450 |
+
context_payload=action_context_payload,
|
| 451 |
+
)
|
| 452 |
+
return x
|
| 453 |
+
|
| 454 |
+
def forward(
|
| 455 |
+
self,
|
| 456 |
+
embeds_all: Dict[str, torch.Tensor],
|
| 457 |
+
attention_mask: torch.Tensor,
|
| 458 |
+
freqs_all: Dict[str, torch.Tensor],
|
| 459 |
+
context_all: Dict[str, Optional[dict]],
|
| 460 |
+
t_mod_all: Dict[str, torch.Tensor],
|
| 461 |
+
):
|
| 462 |
+
missing = [k for k in self.expert_order if k not in embeds_all]
|
| 463 |
+
if missing:
|
| 464 |
+
raise ValueError(f"Missing expert tokens for {missing}")
|
| 465 |
+
missing = [k for k in self.expert_order if k not in freqs_all]
|
| 466 |
+
if missing:
|
| 467 |
+
raise ValueError(f"Missing expert freqs for {missing}")
|
| 468 |
+
missing = [k for k in self.expert_order if k not in t_mod_all]
|
| 469 |
+
if missing:
|
| 470 |
+
raise ValueError(f"Missing expert t_mod for {missing}")
|
| 471 |
+
|
| 472 |
+
if attention_mask.ndim != 2:
|
| 473 |
+
raise ValueError(f"`attention_mask` must be 2D [S, S], got shape {tuple(attention_mask.shape)}")
|
| 474 |
+
if attention_mask.shape[0] != attention_mask.shape[1]:
|
| 475 |
+
raise ValueError(f"`attention_mask` must be square, got shape {tuple(attention_mask.shape)}")
|
| 476 |
+
|
| 477 |
+
tokens_all = {k: v for k, v in embeds_all.items()}
|
| 478 |
+
|
| 479 |
+
for layer_idx in range(self.num_layers):
|
| 480 |
+
q_chunks = []
|
| 481 |
+
k_chunks = []
|
| 482 |
+
v_chunks = []
|
| 483 |
+
cached = {}
|
| 484 |
+
seq_lens = []
|
| 485 |
+
|
| 486 |
+
for name in self.expert_order:
|
| 487 |
+
expert = self.mixtures[name]
|
| 488 |
+
block = expert.blocks[layer_idx]
|
| 489 |
+
x = tokens_all[name]
|
| 490 |
+
freqs = freqs_all[name]
|
| 491 |
+
t_mod = t_mod_all[name]
|
| 492 |
+
|
| 493 |
+
(
|
| 494 |
+
q,
|
| 495 |
+
k,
|
| 496 |
+
v,
|
| 497 |
+
residual_x,
|
| 498 |
+
gate_msa,
|
| 499 |
+
shift_mlp,
|
| 500 |
+
scale_mlp,
|
| 501 |
+
gate_mlp,
|
| 502 |
+
use_gradient_checkpointing,
|
| 503 |
+
) = self._build_expert_attention_io(
|
| 504 |
+
expert=expert,
|
| 505 |
+
block=block,
|
| 506 |
+
x=x,
|
| 507 |
+
freqs=freqs,
|
| 508 |
+
t_mod=t_mod,
|
| 509 |
+
)
|
| 510 |
+
|
| 511 |
+
q_chunks.append(q)
|
| 512 |
+
k_chunks.append(k)
|
| 513 |
+
v_chunks.append(v)
|
| 514 |
+
seq_lens.append(x.shape[1])
|
| 515 |
+
cached[name] = {
|
| 516 |
+
"block": block,
|
| 517 |
+
"residual_x": residual_x,
|
| 518 |
+
"gate_msa": gate_msa,
|
| 519 |
+
"shift_mlp": shift_mlp,
|
| 520 |
+
"scale_mlp": scale_mlp,
|
| 521 |
+
"gate_mlp": gate_mlp,
|
| 522 |
+
"use_gradient_checkpointing": use_gradient_checkpointing,
|
| 523 |
+
}
|
| 524 |
+
|
| 525 |
+
# 3. concat all tokens for mixed attention
|
| 526 |
+
q_cat = torch.cat(q_chunks, dim=1)
|
| 527 |
+
k_cat = torch.cat(k_chunks, dim=1)
|
| 528 |
+
v_cat = torch.cat(v_chunks, dim=1)
|
| 529 |
+
|
| 530 |
+
total_seq = q_cat.shape[1]
|
| 531 |
+
if attention_mask.shape[0] != total_seq:
|
| 532 |
+
raise ValueError(
|
| 533 |
+
"Attention mask seq length mismatch: "
|
| 534 |
+
f"mask={attention_mask.shape[0]} vs tokens={total_seq}"
|
| 535 |
+
)
|
| 536 |
+
|
| 537 |
+
mixed = self._mixed_attention(q_cat=q_cat, k_cat=k_cat, v_cat=v_cat, attention_mask=attention_mask)
|
| 538 |
+
|
| 539 |
+
start = 0
|
| 540 |
+
for name, seq_len in zip(self.expert_order, seq_lens):
|
| 541 |
+
# 4. split mixed attention output and apply post-attention blocks for each expert
|
| 542 |
+
end = start + seq_len
|
| 543 |
+
mixed_slice = mixed[:, start:end, :]
|
| 544 |
+
cached_expert = cached[name]
|
| 545 |
+
block = cached_expert["block"]
|
| 546 |
+
context_payload = context_all.get(name)
|
| 547 |
+
|
| 548 |
+
updated_tokens = self._apply_post_with_optional_checkpoint(
|
| 549 |
+
block=block,
|
| 550 |
+
residual_x=cached_expert["residual_x"],
|
| 551 |
+
gate_msa=cached_expert["gate_msa"],
|
| 552 |
+
shift_mlp=cached_expert["shift_mlp"],
|
| 553 |
+
scale_mlp=cached_expert["scale_mlp"],
|
| 554 |
+
gate_mlp=cached_expert["gate_mlp"],
|
| 555 |
+
use_gradient_checkpointing=cached_expert["use_gradient_checkpointing"],
|
| 556 |
+
mixed_slice=mixed_slice,
|
| 557 |
+
context_payload=context_payload,
|
| 558 |
+
)
|
| 559 |
+
|
| 560 |
+
tokens_all[name] = updated_tokens
|
| 561 |
+
start = end
|
| 562 |
+
|
| 563 |
+
return tokens_all
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/schedulers/scheduler_continuous.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class WanContinuousFlowMatchScheduler:
|
| 5 |
+
"""Continuous-time Flow-Matching scheduler with shift-based sampling."""
|
| 6 |
+
|
| 7 |
+
def __init__(self, num_train_timesteps: int = 1000, shift: float = 5.0, eps: float = 1e-10):
|
| 8 |
+
if num_train_timesteps <= 0:
|
| 9 |
+
raise ValueError(f"`num_train_timesteps` must be positive, got {num_train_timesteps}")
|
| 10 |
+
if shift <= 0:
|
| 11 |
+
raise ValueError(f"`shift` must be positive, got {shift}")
|
| 12 |
+
self.num_train_timesteps = int(num_train_timesteps)
|
| 13 |
+
self.shift = float(shift)
|
| 14 |
+
self.eps = float(eps)
|
| 15 |
+
self._y_min, self._weight_norm_const = self._precompute_training_weight_stats()
|
| 16 |
+
|
| 17 |
+
@staticmethod
|
| 18 |
+
def _phi(u: torch.Tensor, shift: float) -> torch.Tensor:
|
| 19 |
+
return shift * u / (1.0 + (shift - 1.0) * u)
|
| 20 |
+
|
| 21 |
+
def _precompute_training_weight_stats(self) -> tuple[float, float]:
|
| 22 |
+
steps = self.num_train_timesteps
|
| 23 |
+
u_grid = torch.linspace(1.0, 0.0, steps + 1, dtype=torch.float64)[:-1]
|
| 24 |
+
t_grid = self._phi(u_grid, self.shift) * float(steps)
|
| 25 |
+
y_grid = torch.exp(-2.0 * ((t_grid - (steps / 2.0)) / steps) ** 2)
|
| 26 |
+
y_min = float(y_grid.min().item())
|
| 27 |
+
y_shifted_grid = y_grid - y_min
|
| 28 |
+
norm_const = float(y_shifted_grid.mean().item())
|
| 29 |
+
return y_min, norm_const
|
| 30 |
+
|
| 31 |
+
def sample_training_t(self, batch_size: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor:
|
| 32 |
+
if batch_size <= 0:
|
| 33 |
+
raise ValueError(f"`batch_size` must be positive, got {batch_size}")
|
| 34 |
+
u = torch.rand((batch_size,), device=device, dtype=torch.float32)
|
| 35 |
+
sigma = self._phi(u, self.shift)
|
| 36 |
+
timestep = sigma * float(self.num_train_timesteps)
|
| 37 |
+
return timestep.to(dtype=dtype)
|
| 38 |
+
|
| 39 |
+
def training_weight(self, timestep: torch.Tensor) -> torch.Tensor:
|
| 40 |
+
t = timestep.to(dtype=torch.float32)
|
| 41 |
+
steps = float(self.num_train_timesteps)
|
| 42 |
+
y = torch.exp(-2.0 * ((t - (steps / 2.0)) / steps) ** 2)
|
| 43 |
+
y_shifted = y - self._y_min
|
| 44 |
+
weight = y_shifted / (self._weight_norm_const + self.eps)
|
| 45 |
+
if weight.numel() == 1:
|
| 46 |
+
return weight.reshape(())
|
| 47 |
+
return weight
|
| 48 |
+
|
| 49 |
+
def add_noise(self, original_samples: torch.Tensor, noise: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor:
|
| 50 |
+
sigma = (timestep / float(self.num_train_timesteps)).to(
|
| 51 |
+
original_samples.device, dtype=original_samples.dtype
|
| 52 |
+
)
|
| 53 |
+
if sigma.ndim == 0:
|
| 54 |
+
return (1 - sigma) * original_samples + sigma * noise
|
| 55 |
+
sigma = sigma.view(-1, *([1] * (original_samples.ndim - 1)))
|
| 56 |
+
return (1 - sigma) * original_samples + sigma * noise
|
| 57 |
+
|
| 58 |
+
@staticmethod
|
| 59 |
+
def training_target(sample: torch.Tensor, noise: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor:
|
| 60 |
+
del timestep
|
| 61 |
+
return noise - sample
|
| 62 |
+
|
| 63 |
+
def build_inference_schedule(
|
| 64 |
+
self,
|
| 65 |
+
num_inference_steps: int,
|
| 66 |
+
device: torch.device,
|
| 67 |
+
dtype: torch.dtype,
|
| 68 |
+
shift_override: float | None = None,
|
| 69 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 70 |
+
if num_inference_steps <= 0:
|
| 71 |
+
raise ValueError(f"`num_inference_steps` must be positive, got {num_inference_steps}")
|
| 72 |
+
shift = self.shift if shift_override is None else float(shift_override)
|
| 73 |
+
if shift <= 0:
|
| 74 |
+
raise ValueError(f"`shift` must be positive, got {shift}")
|
| 75 |
+
|
| 76 |
+
u_steps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32)
|
| 77 |
+
sigma_steps = self._phi(u_steps, shift)
|
| 78 |
+
timesteps = sigma_steps[:-1] * float(self.num_train_timesteps)
|
| 79 |
+
deltas = sigma_steps[1:] - sigma_steps[:-1]
|
| 80 |
+
return timesteps.to(dtype=dtype), deltas.to(dtype=dtype)
|
| 81 |
+
|
| 82 |
+
@staticmethod
|
| 83 |
+
def step(model_output: torch.Tensor, delta: torch.Tensor, sample: torch.Tensor) -> torch.Tensor:
|
| 84 |
+
delta = delta.to(sample.device, dtype=sample.dtype)
|
| 85 |
+
if delta.ndim == 0:
|
| 86 |
+
return sample + model_output * delta
|
| 87 |
+
delta = delta.view(-1, *([1] * (sample.ndim - 1)))
|
| 88 |
+
return sample + model_output * delta
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan22.py
ADDED
|
@@ -0,0 +1,405 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import os
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
from PIL import Image
|
| 6 |
+
from typing import Any, Optional, Sequence, Union
|
| 7 |
+
|
| 8 |
+
from .helpers.loader import load_wan22_ti2v_5b_components
|
| 9 |
+
from .schedulers.scheduler_continuous import WanContinuousFlowMatchScheduler
|
| 10 |
+
from .wan_video_dit import WanVideoDiT
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class Wan22Core(torch.nn.Module):
|
| 14 |
+
"""Standalone Wan2.2-TI2V-5B core without pipeline unit graph."""
|
| 15 |
+
|
| 16 |
+
def __init__(
|
| 17 |
+
self,
|
| 18 |
+
dit: WanVideoDiT,
|
| 19 |
+
vae,
|
| 20 |
+
text_encoder,
|
| 21 |
+
tokenizer,
|
| 22 |
+
device="cpu",
|
| 23 |
+
torch_dtype=torch.float32,
|
| 24 |
+
train_shift: float = 5.0,
|
| 25 |
+
infer_shift: float = 5.0,
|
| 26 |
+
num_train_timesteps: int = 1000,
|
| 27 |
+
):
|
| 28 |
+
super().__init__()
|
| 29 |
+
self.dit = dit
|
| 30 |
+
self.vae = vae
|
| 31 |
+
self.text_encoder = text_encoder
|
| 32 |
+
self.tokenizer = tokenizer
|
| 33 |
+
self.train_scheduler = WanContinuousFlowMatchScheduler(
|
| 34 |
+
num_train_timesteps=num_train_timesteps,
|
| 35 |
+
shift=train_shift,
|
| 36 |
+
)
|
| 37 |
+
self.infer_scheduler = WanContinuousFlowMatchScheduler(
|
| 38 |
+
num_train_timesteps=num_train_timesteps,
|
| 39 |
+
shift=infer_shift,
|
| 40 |
+
)
|
| 41 |
+
self.device = torch.device(device)
|
| 42 |
+
self.torch_dtype = torch_dtype
|
| 43 |
+
self.to(self.device)
|
| 44 |
+
|
| 45 |
+
@classmethod
|
| 46 |
+
def from_wan22_pretrained(
|
| 47 |
+
cls,
|
| 48 |
+
device="cuda",
|
| 49 |
+
torch_dtype=torch.bfloat16,
|
| 50 |
+
model_id="Wan-AI/Wan2.2-TI2V-5B",
|
| 51 |
+
tokenizer_model_id="Wan-AI/Wan2.1-T2V-1.3B",
|
| 52 |
+
tokenizer_max_len: int = 512,
|
| 53 |
+
redirect_common_files=True,
|
| 54 |
+
dit_config: dict[str, Any] | None = None,
|
| 55 |
+
train_shift: float = 5.0,
|
| 56 |
+
infer_shift: float = 5.0,
|
| 57 |
+
num_train_timesteps: int = 1000,
|
| 58 |
+
):
|
| 59 |
+
if dit_config is None:
|
| 60 |
+
raise ValueError("`dit_config` is required for Wan22Core.from_wan22_pretrained().")
|
| 61 |
+
components = load_wan22_ti2v_5b_components(
|
| 62 |
+
device=device,
|
| 63 |
+
torch_dtype=torch_dtype,
|
| 64 |
+
model_id=model_id,
|
| 65 |
+
tokenizer_model_id=tokenizer_model_id,
|
| 66 |
+
tokenizer_max_len=tokenizer_max_len,
|
| 67 |
+
redirect_common_files=redirect_common_files,
|
| 68 |
+
dit_config=dit_config,
|
| 69 |
+
)
|
| 70 |
+
model = cls(
|
| 71 |
+
dit=components.dit,
|
| 72 |
+
vae=components.vae,
|
| 73 |
+
text_encoder=components.text_encoder,
|
| 74 |
+
tokenizer=components.tokenizer,
|
| 75 |
+
device=device,
|
| 76 |
+
torch_dtype=torch_dtype,
|
| 77 |
+
train_shift=train_shift,
|
| 78 |
+
infer_shift=infer_shift,
|
| 79 |
+
num_train_timesteps=num_train_timesteps,
|
| 80 |
+
)
|
| 81 |
+
model.model_paths = {
|
| 82 |
+
"dit": components.dit_path,
|
| 83 |
+
"vae": components.vae_path,
|
| 84 |
+
"text_encoder": components.text_encoder_path,
|
| 85 |
+
"tokenizer": components.tokenizer_path,
|
| 86 |
+
}
|
| 87 |
+
return model
|
| 88 |
+
|
| 89 |
+
def to(self, *args, **kwargs):
|
| 90 |
+
super().to(*args, **kwargs)
|
| 91 |
+
self.dit.to(*args, **kwargs)
|
| 92 |
+
self.text_encoder.to(*args, **kwargs)
|
| 93 |
+
self.vae.to(*args, **kwargs)
|
| 94 |
+
return self
|
| 95 |
+
|
| 96 |
+
@staticmethod
|
| 97 |
+
def _check_resize_height_width(height, width, num_frames):
|
| 98 |
+
if height % 16 != 0:
|
| 99 |
+
height = (height + 15) // 16 * 16
|
| 100 |
+
if width % 16 != 0:
|
| 101 |
+
width = (width + 15) // 16 * 16
|
| 102 |
+
if num_frames % 4 != 1:
|
| 103 |
+
num_frames = (num_frames + 3) // 4 * 4 + 1
|
| 104 |
+
return height, width, num_frames
|
| 105 |
+
|
| 106 |
+
def encode_prompt(self, prompt: Union[str, Sequence[str]]):
|
| 107 |
+
ids, mask = self.tokenizer(prompt, return_mask=True, add_special_tokens=True)
|
| 108 |
+
ids = ids.to(self.device)
|
| 109 |
+
mask = mask.to(self.device, dtype=torch.bool)
|
| 110 |
+
prompt_emb = self.text_encoder(ids, mask)
|
| 111 |
+
return prompt_emb.to(device=self.device), mask
|
| 112 |
+
|
| 113 |
+
def _encode_video_latents(self, video_tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)):
|
| 114 |
+
z = self.vae.encode(
|
| 115 |
+
video_tensor,
|
| 116 |
+
device=self.device,
|
| 117 |
+
tiled=tiled,
|
| 118 |
+
tile_size=tile_size,
|
| 119 |
+
tile_stride=tile_stride,
|
| 120 |
+
)
|
| 121 |
+
return z
|
| 122 |
+
|
| 123 |
+
def _encode_input_image_latents_tensor(self, input_image: torch.Tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)):
|
| 124 |
+
if input_image.ndim == 3:
|
| 125 |
+
input_image = input_image.unsqueeze(0)
|
| 126 |
+
if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
|
| 127 |
+
raise ValueError(
|
| 128 |
+
f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
|
| 129 |
+
)
|
| 130 |
+
image = input_image.to(device=self.device)[0].unsqueeze(1)
|
| 131 |
+
z = self.vae.encode([image], device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride)
|
| 132 |
+
if isinstance(z, list):
|
| 133 |
+
z = z[0].unsqueeze(0)
|
| 134 |
+
return z
|
| 135 |
+
|
| 136 |
+
def _decode_latents(self, latents, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)):
|
| 137 |
+
video_tensor = self.vae.decode(latents, device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride)
|
| 138 |
+
video_tensor = video_tensor.squeeze(0).detach().float().clamp(-1, 1)
|
| 139 |
+
video_tensor = ((video_tensor + 1.0) * 127.5).to(torch.uint8).cpu()
|
| 140 |
+
frames = []
|
| 141 |
+
for t in range(video_tensor.shape[1]):
|
| 142 |
+
frame = video_tensor[:, t].permute(1, 2, 0).numpy()
|
| 143 |
+
frames.append(Image.fromarray(frame))
|
| 144 |
+
return frames
|
| 145 |
+
|
| 146 |
+
def _model_fn(self,
|
| 147 |
+
latents: torch.Tensor, # [B, C, T, H, W]
|
| 148 |
+
timestep: torch.Tensor, # [B] or [1] (inference mode)
|
| 149 |
+
context: torch.Tensor, # [B, L, D]
|
| 150 |
+
context_mask: Optional[torch.Tensor] = None, # [B, L]
|
| 151 |
+
action: Optional[torch.Tensor] = None, # [B, T-1, a_dim]
|
| 152 |
+
fuse_vae_embedding_in_latents=False):
|
| 153 |
+
return self.dit(
|
| 154 |
+
x=latents,
|
| 155 |
+
timestep=timestep,
|
| 156 |
+
context=context,
|
| 157 |
+
context_mask=context_mask,
|
| 158 |
+
action=action,
|
| 159 |
+
fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents,
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
def build_inputs(self, sample, tiled=False):
|
| 163 |
+
video = sample["video"]
|
| 164 |
+
prompt = sample["prompt"]
|
| 165 |
+
if not isinstance(video, torch.Tensor):
|
| 166 |
+
raise TypeError(
|
| 167 |
+
f"`sample['video']` must be a torch.Tensor with shape [B, 3, T, H, W], got {type(video)}"
|
| 168 |
+
)
|
| 169 |
+
if video.ndim != 5:
|
| 170 |
+
raise ValueError(f"`sample['video']` must be 5D [B, 3, T, H, W], got shape {tuple(video.shape)}")
|
| 171 |
+
if video.shape[1] != 3:
|
| 172 |
+
raise ValueError(f"`sample['video']` channel dimension must be 3, got shape {tuple(video.shape)}")
|
| 173 |
+
|
| 174 |
+
if isinstance(prompt, str):
|
| 175 |
+
prompt_list = [prompt]
|
| 176 |
+
elif isinstance(prompt, Sequence):
|
| 177 |
+
prompt_list = list(prompt)
|
| 178 |
+
else:
|
| 179 |
+
raise TypeError(f"`sample['prompt']` must be str or list[str], got {type(prompt)}")
|
| 180 |
+
|
| 181 |
+
batch_size, _, num_frames, height, width = video.shape
|
| 182 |
+
if len(prompt_list) != batch_size:
|
| 183 |
+
raise ValueError(
|
| 184 |
+
f"Prompt batch mismatch: got len(prompt)={len(prompt_list)} and video batch={batch_size}"
|
| 185 |
+
)
|
| 186 |
+
if height % 16 != 0 or width % 16 != 0:
|
| 187 |
+
raise ValueError(
|
| 188 |
+
f"Video spatial dims must be multiples of 16, got H={height}, W={width}"
|
| 189 |
+
)
|
| 190 |
+
if num_frames % 4 != 1:
|
| 191 |
+
raise ValueError(f"Video T must satisfy T % 4 == 1, got T={num_frames}")
|
| 192 |
+
|
| 193 |
+
input_video = video.to(device=self.device, dtype=self.torch_dtype)
|
| 194 |
+
input_latents = self._encode_video_latents(input_video, tiled=tiled) # [B, C, Latent_T, H', W']
|
| 195 |
+
|
| 196 |
+
first_frame_latents = None
|
| 197 |
+
fuse_flag = False
|
| 198 |
+
if getattr(self.dit, "fuse_vae_embedding_in_latents", False):
|
| 199 |
+
first_frame_latents = input_latents[:, :, 0:1]
|
| 200 |
+
fuse_flag = True
|
| 201 |
+
context, context_mask = self.encode_prompt(prompt_list)
|
| 202 |
+
|
| 203 |
+
action = None
|
| 204 |
+
if "action" in sample:
|
| 205 |
+
action = sample["action"]
|
| 206 |
+
if not isinstance(action, torch.Tensor):
|
| 207 |
+
raise TypeError(
|
| 208 |
+
f"`sample['action']` must be a torch.Tensor with shape [B, T, a_dim], got {type(action)}"
|
| 209 |
+
)
|
| 210 |
+
if action.ndim != 3:
|
| 211 |
+
raise ValueError(f"`sample['action']` must be 3D [B, T, a_dim], got shape {tuple(action.shape)}")
|
| 212 |
+
if action.shape[1] <= 0:
|
| 213 |
+
raise ValueError(f"`sample['action']` temporal dimension must be positive, got {action.shape[1]}")
|
| 214 |
+
if action.shape[1] % (num_frames - 1) != 0:
|
| 215 |
+
raise ValueError(
|
| 216 |
+
"`sample['action']` temporal dimension must be divisible by video transitions "
|
| 217 |
+
f"({num_frames - 1}), got {action.shape[1]}"
|
| 218 |
+
)
|
| 219 |
+
action = action.to(device=self.device, dtype=self.torch_dtype)
|
| 220 |
+
|
| 221 |
+
return {
|
| 222 |
+
"context": context,
|
| 223 |
+
"context_mask": context_mask,
|
| 224 |
+
"input_latents": input_latents,
|
| 225 |
+
"first_frame_latents": first_frame_latents,
|
| 226 |
+
"fuse_vae_embedding_in_latents": fuse_flag,
|
| 227 |
+
"action": action,
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
def training_loss(self, sample, tiled=False):
|
| 231 |
+
inputs = self.build_inputs(sample, tiled=tiled)
|
| 232 |
+
input_latents = inputs["input_latents"]
|
| 233 |
+
batch_size = input_latents.shape[0]
|
| 234 |
+
action = inputs["action"]
|
| 235 |
+
context = inputs["context"]
|
| 236 |
+
context_mask = inputs["context_mask"]
|
| 237 |
+
|
| 238 |
+
# 1. Continuous timestep sampling and noise injection.
|
| 239 |
+
noise = torch.randn_like(input_latents)
|
| 240 |
+
timestep = self.train_scheduler.sample_training_t(
|
| 241 |
+
batch_size=batch_size,
|
| 242 |
+
device=self.device,
|
| 243 |
+
dtype=input_latents.dtype,
|
| 244 |
+
)
|
| 245 |
+
latents = self.train_scheduler.add_noise(input_latents, noise, timestep)
|
| 246 |
+
target = self.train_scheduler.training_target(input_latents, noise, timestep)
|
| 247 |
+
|
| 248 |
+
# 2. fix first latent
|
| 249 |
+
if inputs["first_frame_latents"] is not None:
|
| 250 |
+
latents[:, :, 0: 1] = inputs["first_frame_latents"]
|
| 251 |
+
|
| 252 |
+
pred = self._model_fn(
|
| 253 |
+
latents=latents, # [B, C, Latent_T, H', W']
|
| 254 |
+
timestep=timestep,
|
| 255 |
+
context=context,
|
| 256 |
+
context_mask=context_mask,
|
| 257 |
+
action=action,
|
| 258 |
+
fuse_vae_embedding_in_latents=inputs["fuse_vae_embedding_in_latents"],
|
| 259 |
+
)
|
| 260 |
+
if inputs["first_frame_latents"] is not None:
|
| 261 |
+
pred = pred[:, :, 1:]
|
| 262 |
+
target = target[:, :, 1:]
|
| 263 |
+
loss_per_sample = F.mse_loss(pred.float(), target.float(), reduction="none").mean(dim=(1, 2, 3, 4))
|
| 264 |
+
sample_weight = self.train_scheduler.training_weight(timestep).to(
|
| 265 |
+
loss_per_sample.device, dtype=loss_per_sample.dtype
|
| 266 |
+
)
|
| 267 |
+
loss_total = (loss_per_sample * sample_weight).mean()
|
| 268 |
+
loss_dict = {
|
| 269 |
+
"loss_video": float(loss_total.detach().item()),
|
| 270 |
+
}
|
| 271 |
+
return loss_total, loss_dict
|
| 272 |
+
|
| 273 |
+
@torch.no_grad()
|
| 274 |
+
def infer(
|
| 275 |
+
self,
|
| 276 |
+
prompt: str,
|
| 277 |
+
input_image: torch.Tensor,
|
| 278 |
+
num_frames: int,
|
| 279 |
+
action: Optional[torch.Tensor] = None,
|
| 280 |
+
negative_prompt: Optional[str] = None,
|
| 281 |
+
text_cfg_scale: float = 5.0,
|
| 282 |
+
action_cfg_scale: float = 1.0,
|
| 283 |
+
num_inference_steps: int = 20,
|
| 284 |
+
sigma_shift: Optional[float] = None,
|
| 285 |
+
seed: Optional[int] = None,
|
| 286 |
+
rand_device: str = "cpu",
|
| 287 |
+
tiled: bool = False,
|
| 288 |
+
**kwargs
|
| 289 |
+
):
|
| 290 |
+
self.eval()
|
| 291 |
+
if input_image.ndim == 3:
|
| 292 |
+
input_image = input_image.unsqueeze(0)
|
| 293 |
+
if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3:
|
| 294 |
+
raise ValueError(
|
| 295 |
+
f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}"
|
| 296 |
+
)
|
| 297 |
+
_, _, height, width = input_image.shape
|
| 298 |
+
checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_frames)
|
| 299 |
+
if (checked_h, checked_w) != (height, width):
|
| 300 |
+
raise ValueError(
|
| 301 |
+
f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})"
|
| 302 |
+
)
|
| 303 |
+
if checked_t != num_frames:
|
| 304 |
+
raise ValueError(
|
| 305 |
+
f"`num_frames` must satisfy T % 4 == 1, got {num_frames}"
|
| 306 |
+
)
|
| 307 |
+
|
| 308 |
+
latent_t = (num_frames - 1) // self.vae.temporal_downsample_factor + 1
|
| 309 |
+
latent_h = height // self.vae.upsampling_factor
|
| 310 |
+
latent_w = width // self.vae.upsampling_factor
|
| 311 |
+
|
| 312 |
+
generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed)
|
| 313 |
+
latents = torch.randn(
|
| 314 |
+
(1, self.vae.model.z_dim, latent_t, latent_h, latent_w),
|
| 315 |
+
generator=generator,
|
| 316 |
+
device=rand_device,
|
| 317 |
+
dtype=torch.float32,
|
| 318 |
+
).to(device=self.device, dtype=self.torch_dtype)
|
| 319 |
+
input_image = input_image.to(device=self.device, dtype=self.torch_dtype)
|
| 320 |
+
if action is not None:
|
| 321 |
+
action = action.to(device=self.device, dtype=latents.dtype)
|
| 322 |
+
if action.ndim != 2:
|
| 323 |
+
raise ValueError(f"`action` must be 2D [T, a_dim], got shape {tuple(action.shape)}")
|
| 324 |
+
action = action.unsqueeze(0) # [1, T, a_dim]
|
| 325 |
+
if action.shape[1] % (num_frames - 1) != 0:
|
| 326 |
+
raise ValueError(
|
| 327 |
+
"`action` temporal dimension must be divisible by `num_frames - 1`, "
|
| 328 |
+
f"got {action.shape[1]} vs {num_frames - 1}"
|
| 329 |
+
)
|
| 330 |
+
if action_cfg_scale != 1.0 and action is None:
|
| 331 |
+
raise ValueError("`action_cfg_scale` != 1.0 requires non-null `action` input.")
|
| 332 |
+
|
| 333 |
+
z = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled)
|
| 334 |
+
latents[:, :, 0:1] = z
|
| 335 |
+
first_frame_latents = z
|
| 336 |
+
fuse_flag = True
|
| 337 |
+
|
| 338 |
+
context_posi, context_posi_mask = self.encode_prompt(prompt)
|
| 339 |
+
context_nega = None
|
| 340 |
+
context_nega_mask = None
|
| 341 |
+
if text_cfg_scale != 1.0:
|
| 342 |
+
context_nega, context_nega_mask = self.encode_prompt("" if negative_prompt is None else negative_prompt)
|
| 343 |
+
action_nega = torch.zeros_like(action) if (action is not None and action_cfg_scale != 1.0) else None
|
| 344 |
+
|
| 345 |
+
infer_timesteps, infer_deltas = self.infer_scheduler.build_inference_schedule(
|
| 346 |
+
num_inference_steps=num_inference_steps,
|
| 347 |
+
device=self.device,
|
| 348 |
+
dtype=latents.dtype,
|
| 349 |
+
shift_override=sigma_shift,
|
| 350 |
+
)
|
| 351 |
+
for step_t, step_delta in zip(infer_timesteps, infer_deltas):
|
| 352 |
+
timestep = step_t.unsqueeze(0).to(dtype=latents.dtype, device=self.device)
|
| 353 |
+
noise_pred_posi = self._model_fn(
|
| 354 |
+
latents=latents,
|
| 355 |
+
timestep=timestep,
|
| 356 |
+
context=context_posi,
|
| 357 |
+
context_mask=context_posi_mask,
|
| 358 |
+
action=action,
|
| 359 |
+
fuse_vae_embedding_in_latents=fuse_flag,
|
| 360 |
+
)
|
| 361 |
+
noise_pred = noise_pred_posi
|
| 362 |
+
if context_nega is not None:
|
| 363 |
+
noise_pred_text_nega = self._model_fn(
|
| 364 |
+
latents=latents,
|
| 365 |
+
timestep=timestep,
|
| 366 |
+
context=context_nega,
|
| 367 |
+
context_mask=context_nega_mask,
|
| 368 |
+
action=action,
|
| 369 |
+
fuse_vae_embedding_in_latents=fuse_flag,
|
| 370 |
+
)
|
| 371 |
+
noise_pred = noise_pred + (text_cfg_scale - 1.0) * (noise_pred_posi - noise_pred_text_nega)
|
| 372 |
+
if action_nega is not None:
|
| 373 |
+
noise_pred_action_nega = self._model_fn(
|
| 374 |
+
latents=latents,
|
| 375 |
+
timestep=timestep,
|
| 376 |
+
context=context_posi,
|
| 377 |
+
context_mask=context_posi_mask,
|
| 378 |
+
action=action_nega,
|
| 379 |
+
fuse_vae_embedding_in_latents=fuse_flag,
|
| 380 |
+
)
|
| 381 |
+
noise_pred = noise_pred + (action_cfg_scale - 1.0) * (noise_pred_posi - noise_pred_action_nega)
|
| 382 |
+
latents = self.infer_scheduler.step(noise_pred, step_delta, latents)
|
| 383 |
+
latents[:, :, 0:1] = first_frame_latents
|
| 384 |
+
|
| 385 |
+
return {"video": self._decode_latents(latents, tiled=tiled)}
|
| 386 |
+
|
| 387 |
+
def save_checkpoint(self, path, optimizer=None, step=None):
|
| 388 |
+
payload = {
|
| 389 |
+
"dit": self.dit.state_dict(),
|
| 390 |
+
"step": step,
|
| 391 |
+
"torch_dtype": str(self.torch_dtype),
|
| 392 |
+
}
|
| 393 |
+
if optimizer is not None:
|
| 394 |
+
payload["optimizer"] = optimizer.state_dict()
|
| 395 |
+
torch.save(payload, path)
|
| 396 |
+
|
| 397 |
+
def load_checkpoint(self, path, optimizer=None):
|
| 398 |
+
payload = torch.load(path, map_location="cpu")
|
| 399 |
+
self.dit.load_state_dict(payload["dit"], strict=False)
|
| 400 |
+
if optimizer is not None and "optimizer" in payload:
|
| 401 |
+
optimizer.load_state_dict(payload["optimizer"])
|
| 402 |
+
return payload
|
| 403 |
+
|
| 404 |
+
def forward(self, *args, **kwargs):
|
| 405 |
+
return self.training_loss(*args, **kwargs)
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_dit.py
ADDED
|
@@ -0,0 +1,673 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
import math
|
| 5 |
+
from typing import Any, Dict, Tuple, Optional
|
| 6 |
+
from einops import rearrange
|
| 7 |
+
from .helpers.gradient import gradient_checkpoint_forward
|
| 8 |
+
|
| 9 |
+
from fastwam.utils.logging_config import get_logger
|
| 10 |
+
|
| 11 |
+
logger = get_logger(__name__)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def flash_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, num_heads: int, ctx_mask: Optional[torch.Tensor] = None, compatibility_mode=True):
|
| 15 |
+
if compatibility_mode:
|
| 16 |
+
q = rearrange(q, "b s (n d) -> b n s d", n=num_heads)
|
| 17 |
+
k = rearrange(k, "b s (n d) -> b n s d", n=num_heads)
|
| 18 |
+
v = rearrange(v, "b s (n d) -> b n s d", n=num_heads)
|
| 19 |
+
x = F.scaled_dot_product_attention(q, k, v, attn_mask=ctx_mask)
|
| 20 |
+
x = rearrange(x, "b n s d -> b s (n d)", n=num_heads)
|
| 21 |
+
return x
|
| 22 |
+
else:
|
| 23 |
+
raise NotImplementedError("Only compatibility mode is implemented for flash attention. Please set compatibility_mode=True.")
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor):
|
| 28 |
+
return (x * (1 + scale) + shift)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def sinusoidal_embedding_1d(dim, position):
|
| 32 |
+
sinusoid = torch.outer(position.type(torch.float64), torch.pow(
|
| 33 |
+
10000, -torch.arange(dim//2, dtype=torch.float64, device=position.device).div(dim//2)))
|
| 34 |
+
x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1)
|
| 35 |
+
return x.to(position.dtype)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def precompute_freqs_cis_3d(dim: int, end: int = 1024, theta: float = 10000.0):
|
| 39 |
+
# 3d rope precompute
|
| 40 |
+
f_freqs_cis = precompute_freqs_cis(dim - 2 * (dim // 3), end, theta)
|
| 41 |
+
h_freqs_cis = precompute_freqs_cis(dim // 3, end, theta)
|
| 42 |
+
w_freqs_cis = precompute_freqs_cis(dim // 3, end, theta)
|
| 43 |
+
return f_freqs_cis, h_freqs_cis, w_freqs_cis
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def precompute_freqs_cis(dim: int, end: int = 1024, theta: float = 10000.0):
|
| 47 |
+
# 1d rope precompute
|
| 48 |
+
freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)
|
| 49 |
+
[: (dim // 2)].double() / dim))
|
| 50 |
+
freqs = torch.outer(torch.arange(end, device=freqs.device), freqs)
|
| 51 |
+
freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex dtype follows freqs
|
| 52 |
+
return freqs_cis
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def rope_apply(x, freqs, num_heads, compute_dtype=torch.float64):
|
| 56 |
+
x = rearrange(x, "b s (n d) -> b s n d", n=num_heads)
|
| 57 |
+
if compute_dtype not in (torch.float32, torch.float64):
|
| 58 |
+
raise ValueError(f"RoPE compute_dtype must be float32 or float64, got {compute_dtype}")
|
| 59 |
+
complex_dtype = torch.complex64 if compute_dtype == torch.float32 else torch.complex128
|
| 60 |
+
x_out = torch.view_as_complex(x.to(compute_dtype).reshape(
|
| 61 |
+
x.shape[0], x.shape[1], x.shape[2], -1, 2))
|
| 62 |
+
if compute_dtype == torch.float64 and freqs.device.type == "npu":
|
| 63 |
+
# Preserve the historical NPU compatibility path.
|
| 64 |
+
freqs = freqs.to(torch.complex64)
|
| 65 |
+
else:
|
| 66 |
+
freqs = freqs.to(complex_dtype)
|
| 67 |
+
x_out = torch.view_as_real(x_out * freqs).flatten(2)
|
| 68 |
+
return x_out.to(x.dtype)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def create_group_causal_attn_mask(
|
| 72 |
+
num_temporal_groups: int, num_query_per_group: int, num_key_per_group: int, mode: str = "causal"
|
| 73 |
+
) -> torch.Tensor:
|
| 74 |
+
"""
|
| 75 |
+
Creates a group-based attention mask for scaled dot-product attention with two modes:
|
| 76 |
+
'causal' and 'group_diagonal'.
|
| 77 |
+
|
| 78 |
+
Parameters:
|
| 79 |
+
- num_temporal_groups (int): The number of temporal groups (e.g., frames in a video sequence).
|
| 80 |
+
- num_query_per_group (int): The number of query tokens per temporal group. (e.g., latent tokens in a frame, H x W).
|
| 81 |
+
- num_key_per_group (int): The number of key tokens per temporal group. (e.g., action tokens per frame).
|
| 82 |
+
- mode (str): The mode of the attention mask. Options are:
|
| 83 |
+
- 'causal': Query tokens can attend to key tokens from the same or previous temporal groups.
|
| 84 |
+
- 'group_diagonal': Query tokens can attend only to key tokens from the same temporal group.
|
| 85 |
+
|
| 86 |
+
Returns:
|
| 87 |
+
- attn_mask (torch.Tensor): A boolean tensor of shape (L, S), where:
|
| 88 |
+
- L = num_temporal_groups * num_query_per_group (total number of query tokens)
|
| 89 |
+
- S = num_temporal_groups * num_key_per_group (total number of key tokens)
|
| 90 |
+
The mask indicates where attention is allowed (True) and disallowed (False).
|
| 91 |
+
|
| 92 |
+
Example:
|
| 93 |
+
Input:
|
| 94 |
+
num_temporal_groups = 3
|
| 95 |
+
num_query_per_group = 4
|
| 96 |
+
num_key_per_group = 2
|
| 97 |
+
Output:
|
| 98 |
+
Causal Mask Shape: torch.Size([12, 6])
|
| 99 |
+
Group Diagonal Mask Shape: torch.Size([12, 6])
|
| 100 |
+
if mode='causal':
|
| 101 |
+
tensor([[ True, True, False, False, False, False],
|
| 102 |
+
[ True, True, False, False, False, False],
|
| 103 |
+
[ True, True, False, False, False, False],
|
| 104 |
+
[ True, True, False, False, False, False],
|
| 105 |
+
[ True, True, True, True, False, False],
|
| 106 |
+
[ True, True, True, True, False, False],
|
| 107 |
+
[ True, True, True, True, False, False],
|
| 108 |
+
[ True, True, True, True, False, False],
|
| 109 |
+
[ True, True, True, True, True, True],
|
| 110 |
+
[ True, True, True, True, True, True],
|
| 111 |
+
[ True, True, True, True, True, True],
|
| 112 |
+
[ True, True, True, True, True, True]])
|
| 113 |
+
|
| 114 |
+
if mode='group_diagonal':
|
| 115 |
+
tensor([[ True, True, False, False, False, False],
|
| 116 |
+
[ True, True, False, False, False, False],
|
| 117 |
+
[ True, True, False, False, False, False],
|
| 118 |
+
[ True, True, False, False, False, False],
|
| 119 |
+
[False, False, True, True, False, False],
|
| 120 |
+
[False, False, True, True, False, False],
|
| 121 |
+
[False, False, True, True, False, False],
|
| 122 |
+
[False, False, True, True, False, False],
|
| 123 |
+
[False, False, False, False, True, True],
|
| 124 |
+
[False, False, False, False, True, True],
|
| 125 |
+
[False, False, False, False, True, True],
|
| 126 |
+
[False, False, False, False, True, True]])
|
| 127 |
+
|
| 128 |
+
"""
|
| 129 |
+
assert mode in ["causal", "group_diagonal"], f"Mode {mode} must be 'causal' or 'group_diagonal'"
|
| 130 |
+
|
| 131 |
+
# Total number of query and key tokens
|
| 132 |
+
total_num_query_tokens = num_temporal_groups * num_query_per_group # Total number of query tokens (L)
|
| 133 |
+
total_num_key_tokens = num_temporal_groups * num_key_per_group # Total number of key tokens (S)
|
| 134 |
+
|
| 135 |
+
# Generate time indices for query and key tokens (shape: [L] and [S])
|
| 136 |
+
query_time_indices = torch.arange(num_temporal_groups).repeat_interleave(num_query_per_group) # Shape: [L]
|
| 137 |
+
key_time_indices = torch.arange(num_temporal_groups).repeat_interleave(num_key_per_group) # Shape: [S]
|
| 138 |
+
|
| 139 |
+
# Expand dimensions to compute outer comparison
|
| 140 |
+
query_time_indices = query_time_indices.unsqueeze(1) # Shape: [L, 1]
|
| 141 |
+
key_time_indices = key_time_indices.unsqueeze(0) # Shape: [1, S]
|
| 142 |
+
|
| 143 |
+
if mode == "causal":
|
| 144 |
+
# Causal Mode: Query can attend to keys where key_time <= query_time
|
| 145 |
+
attn_mask = query_time_indices >= key_time_indices # Shape: [L, S]
|
| 146 |
+
elif mode == "group_diagonal":
|
| 147 |
+
# Group Diagonal Mode: Query can attend only to keys where key_time == query_time
|
| 148 |
+
attn_mask = query_time_indices == key_time_indices # Shape: [L, S]
|
| 149 |
+
|
| 150 |
+
assert attn_mask.shape == (total_num_query_tokens, total_num_key_tokens), "Attention mask shape mismatch"
|
| 151 |
+
return attn_mask
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
class RMSNorm(nn.Module):
|
| 155 |
+
def __init__(self, dim, eps=1e-5):
|
| 156 |
+
super().__init__()
|
| 157 |
+
self.eps = eps
|
| 158 |
+
self.weight = nn.Parameter(torch.ones(dim))
|
| 159 |
+
|
| 160 |
+
def norm(self, x):
|
| 161 |
+
return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
|
| 162 |
+
|
| 163 |
+
def forward(self, x):
|
| 164 |
+
dtype = x.dtype
|
| 165 |
+
return self.norm(x.float()).to(dtype) * self.weight
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
class AttentionModule(nn.Module):
|
| 169 |
+
def __init__(self, num_heads):
|
| 170 |
+
super().__init__()
|
| 171 |
+
self.num_heads = num_heads
|
| 172 |
+
|
| 173 |
+
def forward(self, q, k, v, ctx_mask=None):
|
| 174 |
+
x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=ctx_mask)
|
| 175 |
+
return x
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
class SelfAttention(nn.Module):
|
| 179 |
+
def __init__(self, hidden_dim: int, attn_head_dim: int, num_heads: int, eps: float = 1e-6):
|
| 180 |
+
super().__init__()
|
| 181 |
+
self.hidden_dim = hidden_dim
|
| 182 |
+
self.num_heads = num_heads
|
| 183 |
+
self.attn_head_dim = attn_head_dim
|
| 184 |
+
self.attn_hidden_dim = self.num_heads * self.attn_head_dim
|
| 185 |
+
|
| 186 |
+
self.q = nn.Linear(hidden_dim, self.attn_hidden_dim)
|
| 187 |
+
self.k = nn.Linear(hidden_dim, self.attn_hidden_dim)
|
| 188 |
+
self.v = nn.Linear(hidden_dim, self.attn_hidden_dim)
|
| 189 |
+
self.o = nn.Linear(self.attn_hidden_dim, hidden_dim)
|
| 190 |
+
self.norm_q = RMSNorm(self.attn_hidden_dim, eps=eps)
|
| 191 |
+
self.norm_k = RMSNorm(self.attn_hidden_dim, eps=eps)
|
| 192 |
+
|
| 193 |
+
# self.attn = AttentionModule(self.num_heads)
|
| 194 |
+
|
| 195 |
+
def forward(self, x, freqs, self_attn_mask: Optional[torch.Tensor] = None):
|
| 196 |
+
q = self.norm_q(self.q(x))
|
| 197 |
+
k = self.norm_k(self.k(x))
|
| 198 |
+
v = self.v(x)
|
| 199 |
+
q = rope_apply(q, freqs, self.num_heads)
|
| 200 |
+
k = rope_apply(k, freqs, self.num_heads)
|
| 201 |
+
x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=self_attn_mask)
|
| 202 |
+
return self.o(x)
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
class CrossAttention(nn.Module):
|
| 206 |
+
def __init__(self, hidden_dim: int, attn_head_dim: int, num_heads: int, eps: float = 1e-6,):
|
| 207 |
+
super().__init__()
|
| 208 |
+
self.hidden_dim = hidden_dim
|
| 209 |
+
self.num_heads = num_heads
|
| 210 |
+
self.attn_head_dim = attn_head_dim
|
| 211 |
+
self.attn_hidden_dim = self.num_heads * self.attn_head_dim
|
| 212 |
+
|
| 213 |
+
self.q = nn.Linear(hidden_dim, self.attn_hidden_dim)
|
| 214 |
+
self.k = nn.Linear(hidden_dim, self.attn_hidden_dim)
|
| 215 |
+
self.v = nn.Linear(hidden_dim, self.attn_hidden_dim)
|
| 216 |
+
self.o = nn.Linear(self.attn_hidden_dim, hidden_dim)
|
| 217 |
+
self.norm_q = RMSNorm(self.attn_hidden_dim, eps=eps)
|
| 218 |
+
self.norm_k = RMSNorm(self.attn_hidden_dim, eps=eps)
|
| 219 |
+
|
| 220 |
+
# self.attn = AttentionModule(self.num_heads)
|
| 221 |
+
|
| 222 |
+
def forward(self, x: torch.Tensor, ctx: torch.Tensor, ctx_mask: Optional[torch.Tensor] = None):
|
| 223 |
+
q = self.norm_q(self.q(x))
|
| 224 |
+
k = self.norm_k(self.k(ctx))
|
| 225 |
+
v = self.v(ctx)
|
| 226 |
+
x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=ctx_mask)
|
| 227 |
+
return self.o(x)
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
class GateModule(nn.Module):
|
| 231 |
+
def __init__(self,):
|
| 232 |
+
super().__init__()
|
| 233 |
+
|
| 234 |
+
def forward(self, x, gate, residual):
|
| 235 |
+
return x + gate * residual
|
| 236 |
+
|
| 237 |
+
class DiTBlock(nn.Module):
|
| 238 |
+
def __init__(self, hidden_dim: int, attn_head_dim: int, num_heads: int, ffn_dim: int, eps: float = 1e-6):
|
| 239 |
+
super().__init__()
|
| 240 |
+
self.hidden_dim = hidden_dim
|
| 241 |
+
self.attn_head_dim = attn_head_dim
|
| 242 |
+
self.num_heads = num_heads
|
| 243 |
+
self.ffn_dim = ffn_dim
|
| 244 |
+
|
| 245 |
+
self.self_attn = SelfAttention(hidden_dim, attn_head_dim, num_heads, eps)
|
| 246 |
+
self.cross_attn = CrossAttention(
|
| 247 |
+
hidden_dim, attn_head_dim, num_heads, eps)
|
| 248 |
+
self.norm1 = nn.LayerNorm(hidden_dim, eps=eps, elementwise_affine=False)
|
| 249 |
+
self.norm2 = nn.LayerNorm(hidden_dim, eps=eps, elementwise_affine=False)
|
| 250 |
+
self.norm3 = nn.LayerNorm(hidden_dim, eps=eps)
|
| 251 |
+
self.ffn = nn.Sequential(nn.Linear(hidden_dim, ffn_dim), nn.GELU(
|
| 252 |
+
approximate='tanh'), nn.Linear(ffn_dim, hidden_dim))
|
| 253 |
+
self.modulation = nn.Parameter(torch.randn(1, 6, hidden_dim) / hidden_dim**0.5)
|
| 254 |
+
self.gate = GateModule()
|
| 255 |
+
|
| 256 |
+
def forward(self, x, context, t_mod, freqs, context_mask=None, self_attn_mask: Optional[torch.Tensor] = None):
|
| 257 |
+
if context_mask is not None and context_mask.dim() == 3:
|
| 258 |
+
context_mask = context_mask.unsqueeze(1) # (B, 1, seq_len, context_len), 1 for heads
|
| 259 |
+
has_seq = len(t_mod.shape) == 4
|
| 260 |
+
chunk_dim = 2 if has_seq else 1
|
| 261 |
+
# msa: multi-head self-attention mlp: multi-layer perceptron
|
| 262 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
|
| 263 |
+
self.modulation.to(dtype=t_mod.dtype, device=t_mod.device) + t_mod).chunk(6, dim=chunk_dim)
|
| 264 |
+
if has_seq:
|
| 265 |
+
# means t_mod has separate modulation for each token, otherwise same modulation for all tokens in the block
|
| 266 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
|
| 267 |
+
shift_msa.squeeze(2), scale_msa.squeeze(2), gate_msa.squeeze(2),
|
| 268 |
+
shift_mlp.squeeze(2), scale_mlp.squeeze(2), gate_mlp.squeeze(2),
|
| 269 |
+
)
|
| 270 |
+
input_x = modulate(self.norm1(x), shift_msa, scale_msa)
|
| 271 |
+
x = self.gate(x, gate_msa, self.self_attn(input_x, freqs, self_attn_mask=self_attn_mask))
|
| 272 |
+
x = x + self.cross_attn(self.norm3(x), context, ctx_mask=context_mask)
|
| 273 |
+
input_x = modulate(self.norm2(x), shift_mlp, scale_mlp)
|
| 274 |
+
x = self.gate(x, gate_mlp, self.ffn(input_x))
|
| 275 |
+
return x
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
class MLP(torch.nn.Module):
|
| 279 |
+
def __init__(self, in_dim, out_dim, has_pos_emb=False):
|
| 280 |
+
super().__init__()
|
| 281 |
+
self.proj = torch.nn.Sequential(
|
| 282 |
+
nn.LayerNorm(in_dim),
|
| 283 |
+
nn.Linear(in_dim, in_dim),
|
| 284 |
+
nn.GELU(),
|
| 285 |
+
nn.Linear(in_dim, out_dim),
|
| 286 |
+
nn.LayerNorm(out_dim)
|
| 287 |
+
)
|
| 288 |
+
self.has_pos_emb = has_pos_emb
|
| 289 |
+
if has_pos_emb:
|
| 290 |
+
self.emb_pos = torch.nn.Parameter(torch.zeros((1, 514, 1280)))
|
| 291 |
+
|
| 292 |
+
def forward(self, x):
|
| 293 |
+
if self.has_pos_emb:
|
| 294 |
+
x = x + self.emb_pos.to(dtype=x.dtype, device=x.device)
|
| 295 |
+
return self.proj(x)
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
class Head(nn.Module):
|
| 299 |
+
def __init__(self, dim: int, out_dim: int, patch_size: Tuple[int, int, int], eps: float):
|
| 300 |
+
super().__init__()
|
| 301 |
+
self.dim = dim
|
| 302 |
+
self.patch_size = patch_size
|
| 303 |
+
self.norm = nn.LayerNorm(dim, eps=eps, elementwise_affine=False)
|
| 304 |
+
self.head = nn.Linear(dim, out_dim * math.prod(patch_size))
|
| 305 |
+
self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5)
|
| 306 |
+
|
| 307 |
+
def forward(self, x, t_mod):
|
| 308 |
+
if len(t_mod.shape) == 3:
|
| 309 |
+
shift, scale = (self.modulation.unsqueeze(0).to(dtype=t_mod.dtype, device=t_mod.device) + t_mod.unsqueeze(2)).chunk(2, dim=2)
|
| 310 |
+
x = (self.head(self.norm(x) * (1 + scale.squeeze(2)) + shift.squeeze(2)))
|
| 311 |
+
else:
|
| 312 |
+
shift, scale = (self.modulation.to(dtype=t_mod.dtype, device=t_mod.device) + t_mod).chunk(2, dim=1)
|
| 313 |
+
x = (self.head(self.norm(x) * (1 + scale) + shift))
|
| 314 |
+
return x
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
class WanVideoDiT(torch.nn.Module):
|
| 318 |
+
def __init__(
|
| 319 |
+
self,
|
| 320 |
+
hidden_dim: int,
|
| 321 |
+
in_dim: int,
|
| 322 |
+
ffn_dim: int,
|
| 323 |
+
out_dim: int,
|
| 324 |
+
text_dim: int,
|
| 325 |
+
freq_dim: int,
|
| 326 |
+
eps: float,
|
| 327 |
+
patch_size: Tuple[int, int, int],
|
| 328 |
+
num_heads: int,
|
| 329 |
+
attn_head_dim: int,
|
| 330 |
+
num_layers: int,
|
| 331 |
+
has_image_input: bool,
|
| 332 |
+
has_image_pos_emb: bool = False,
|
| 333 |
+
has_ref_conv: bool = False,
|
| 334 |
+
add_control_adapter: bool = False,
|
| 335 |
+
in_dim_control_adapter: int = 24,
|
| 336 |
+
seperated_timestep: bool = False,
|
| 337 |
+
require_vae_embedding: bool = False,
|
| 338 |
+
require_clip_embedding: bool = False,
|
| 339 |
+
fuse_vae_embedding_in_latents: bool = True,
|
| 340 |
+
action_conditioned: bool = False,
|
| 341 |
+
action_dim: int = 7,
|
| 342 |
+
action_group_causal_mask_mode = "causal",
|
| 343 |
+
video_attention_mask_mode: str = "bidirectional",
|
| 344 |
+
use_gradient_checkpointing: bool = False,
|
| 345 |
+
):
|
| 346 |
+
super().__init__()
|
| 347 |
+
self.hidden_dim = hidden_dim
|
| 348 |
+
self.in_dim = in_dim
|
| 349 |
+
self.freq_dim = freq_dim
|
| 350 |
+
self.patch_size = patch_size
|
| 351 |
+
self.num_heads = num_heads
|
| 352 |
+
self.attn_head_dim = attn_head_dim
|
| 353 |
+
self.seperated_timestep = seperated_timestep
|
| 354 |
+
self.require_vae_embedding = require_vae_embedding
|
| 355 |
+
self.require_clip_embedding = require_clip_embedding
|
| 356 |
+
self.fuse_vae_embedding_in_latents = fuse_vae_embedding_in_latents
|
| 357 |
+
self.video_attention_mask_mode = str(video_attention_mask_mode)
|
| 358 |
+
|
| 359 |
+
if num_heads <= 0:
|
| 360 |
+
raise ValueError(f"`num_heads` must be > 0, got {num_heads}")
|
| 361 |
+
if attn_head_dim <= 0:
|
| 362 |
+
raise ValueError(f"`attn_head_dim` must be > 0, got {attn_head_dim}")
|
| 363 |
+
if attn_head_dim % 2 != 0:
|
| 364 |
+
raise ValueError(
|
| 365 |
+
f"`attn_head_dim` must be even for RoPE, got {attn_head_dim}"
|
| 366 |
+
)
|
| 367 |
+
|
| 368 |
+
self.action_conditioned = action_conditioned
|
| 369 |
+
self.action_dim = action_dim
|
| 370 |
+
assert has_image_input == False
|
| 371 |
+
assert require_clip_embedding == False
|
| 372 |
+
assert require_vae_embedding == False and fuse_vae_embedding_in_latents == True, "Only support fusing vae embedding in latents"
|
| 373 |
+
|
| 374 |
+
self.patch_embedding = nn.Conv3d(
|
| 375 |
+
in_dim, hidden_dim, kernel_size=patch_size, stride=patch_size)
|
| 376 |
+
self.text_embedding = nn.Sequential(
|
| 377 |
+
nn.Linear(text_dim, hidden_dim),
|
| 378 |
+
nn.GELU(approximate='tanh'),
|
| 379 |
+
nn.Linear(hidden_dim, hidden_dim)
|
| 380 |
+
)
|
| 381 |
+
self.time_embedding = nn.Sequential(
|
| 382 |
+
nn.Linear(freq_dim, hidden_dim),
|
| 383 |
+
nn.SiLU(),
|
| 384 |
+
nn.Linear(hidden_dim, hidden_dim)
|
| 385 |
+
)
|
| 386 |
+
self.time_projection = nn.Sequential(
|
| 387 |
+
nn.SiLU(), nn.Linear(hidden_dim, hidden_dim * 6))
|
| 388 |
+
self.blocks = nn.ModuleList([
|
| 389 |
+
DiTBlock(hidden_dim, attn_head_dim, num_heads, ffn_dim, eps)
|
| 390 |
+
for _ in range(num_layers)
|
| 391 |
+
])
|
| 392 |
+
self.head = Head(hidden_dim, out_dim, patch_size, eps)
|
| 393 |
+
self.freqs = precompute_freqs_cis_3d(attn_head_dim)
|
| 394 |
+
if has_ref_conv:
|
| 395 |
+
self.ref_conv = nn.Conv2d(16, hidden_dim, kernel_size=(2, 2), stride=(2, 2))
|
| 396 |
+
self.has_image_pos_emb = has_image_pos_emb
|
| 397 |
+
self.has_ref_conv = has_ref_conv
|
| 398 |
+
self.control_adapter = None
|
| 399 |
+
|
| 400 |
+
if self.action_conditioned:
|
| 401 |
+
self.action_embedding = nn.Linear(action_dim, hidden_dim)
|
| 402 |
+
self.action_group_causal_mask_mode = action_group_causal_mask_mode
|
| 403 |
+
|
| 404 |
+
self.use_gradient_checkpointing = use_gradient_checkpointing
|
| 405 |
+
if self.use_gradient_checkpointing:
|
| 406 |
+
logger.info("Using gradient checkpointing for DiT blocks. This will save memory but use more computation.")
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
def patchify(self, x: torch.Tensor, control_camera_latents_input: Optional[torch.Tensor] = None):
|
| 410 |
+
x = self.patch_embedding(x)
|
| 411 |
+
if self.control_adapter is not None and control_camera_latents_input is not None:
|
| 412 |
+
y_camera = self.control_adapter(control_camera_latents_input)
|
| 413 |
+
x = [u + v for u, v in zip(x, y_camera)]
|
| 414 |
+
x = x[0].unsqueeze(0)
|
| 415 |
+
return x
|
| 416 |
+
|
| 417 |
+
def unpatchify(self, x: torch.Tensor, grid_size: torch.Tensor):
|
| 418 |
+
return rearrange(
|
| 419 |
+
x, 'b (f h w) (x y z c) -> b c (f x) (h y) (w z)',
|
| 420 |
+
f=grid_size[0], h=grid_size[1], w=grid_size[2],
|
| 421 |
+
x=self.patch_size[0], y=self.patch_size[1], z=self.patch_size[2]
|
| 422 |
+
)
|
| 423 |
+
|
| 424 |
+
def _validate_forward_inputs(
|
| 425 |
+
self,
|
| 426 |
+
x: torch.Tensor,
|
| 427 |
+
timestep: torch.Tensor,
|
| 428 |
+
context: torch.Tensor,
|
| 429 |
+
context_mask: Optional[torch.Tensor],
|
| 430 |
+
action: Optional[torch.Tensor],
|
| 431 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 432 |
+
if x.ndim != 5:
|
| 433 |
+
raise ValueError(f"`latents` must be 5D [B, C, T, H, W], got shape {tuple(x.shape)}")
|
| 434 |
+
num_latent_frames = x.shape[2]
|
| 435 |
+
if context.ndim != 3:
|
| 436 |
+
raise ValueError(f"`context` must be 3D [B, L, D], got shape {tuple(context.shape)}")
|
| 437 |
+
if timestep.ndim != 1:
|
| 438 |
+
raise ValueError(f"`timestep` must be 1D [B] or [1], got shape {tuple(timestep.shape)}")
|
| 439 |
+
if self.action_conditioned:
|
| 440 |
+
allow_text_only_single_frame = (num_latent_frames == 1 and action is None)
|
| 441 |
+
if not allow_text_only_single_frame:
|
| 442 |
+
assert action is not None, "Action input is required for action-conditioned model."
|
| 443 |
+
if action.ndim != 3:
|
| 444 |
+
raise ValueError(f"`action` must be 3D [B, action_horizon, action_dim], got shape {tuple(action.shape)}")
|
| 445 |
+
if action.shape[2] != self.action_dim:
|
| 446 |
+
raise ValueError(f"`action` last dimension must be {self.action_dim}, got {action.shape[2]}")
|
| 447 |
+
if num_latent_frames <= 1:
|
| 448 |
+
raise ValueError(f"video length must be > 1 for action-conditioned model, got {num_latent_frames}")
|
| 449 |
+
if action.shape[1] % (num_latent_frames - 1) != 0:
|
| 450 |
+
raise ValueError(
|
| 451 |
+
f"action horizon must be divisible by (num_latent_frames - 1), got action_horizon={action.shape[1]}"
|
| 452 |
+
)
|
| 453 |
+
if context_mask is None:
|
| 454 |
+
context_mask = torch.ones((context.shape[0], context.shape[1]), dtype=torch.bool, device=context.device)
|
| 455 |
+
else:
|
| 456 |
+
if context_mask.ndim != 2:
|
| 457 |
+
raise ValueError(f"`context_mask` must be 2D [B, L], got shape {tuple(context_mask.shape)}")
|
| 458 |
+
if context_mask.shape[0] != context.shape[0] or context_mask.shape[1] != context.shape[1]:
|
| 459 |
+
raise ValueError(f"`context_mask` shape must match `context` shape [B, L], got {tuple(context_mask.shape)} vs {tuple(context.shape)}")
|
| 460 |
+
|
| 461 |
+
batch_size = x.shape[0]
|
| 462 |
+
if batch_size != context.shape[0]:
|
| 463 |
+
if not self.training and batch_size == 1:
|
| 464 |
+
x = x.expand(context.shape[0], -1, -1, -1, -1)
|
| 465 |
+
batch_size = context.shape[0]
|
| 466 |
+
else:
|
| 467 |
+
raise ValueError(
|
| 468 |
+
f"Batch mismatch between latents and context: {batch_size} vs {context.shape[0]}."
|
| 469 |
+
)
|
| 470 |
+
|
| 471 |
+
if timestep.shape[0] not in (1, batch_size):
|
| 472 |
+
raise ValueError(
|
| 473 |
+
f"`timestep` length must be 1 or batch_size({batch_size}), got {timestep.shape[0]}"
|
| 474 |
+
)
|
| 475 |
+
if timestep.shape[0] == 1 and batch_size > 1:
|
| 476 |
+
assert not self.training, "During training, timestep length must match batch_size."
|
| 477 |
+
timestep = timestep.expand(batch_size)
|
| 478 |
+
return x, timestep, context_mask
|
| 479 |
+
|
| 480 |
+
def build_video_to_video_mask(
|
| 481 |
+
self,
|
| 482 |
+
video_seq_len: int,
|
| 483 |
+
video_tokens_per_frame: int,
|
| 484 |
+
device: torch.device,
|
| 485 |
+
) -> torch.Tensor:
|
| 486 |
+
if video_seq_len <= 0:
|
| 487 |
+
raise ValueError(f"`video_seq_len` must be positive, got {video_seq_len}")
|
| 488 |
+
if video_tokens_per_frame <= 0:
|
| 489 |
+
raise ValueError(f"`video_tokens_per_frame` must be positive, got {video_tokens_per_frame}")
|
| 490 |
+
|
| 491 |
+
if self.video_attention_mask_mode == "bidirectional":
|
| 492 |
+
return torch.ones((video_seq_len, video_seq_len), dtype=torch.bool, device=device)
|
| 493 |
+
|
| 494 |
+
if self.video_attention_mask_mode == "per_frame_causal":
|
| 495 |
+
if video_seq_len % video_tokens_per_frame != 0:
|
| 496 |
+
raise ValueError(
|
| 497 |
+
"`video_seq_len` must be divisible by `video_tokens_per_frame` in `per_frame_causal` mode, "
|
| 498 |
+
f"got {video_seq_len} and {video_tokens_per_frame}"
|
| 499 |
+
)
|
| 500 |
+
num_video_frames = video_seq_len // video_tokens_per_frame
|
| 501 |
+
frame_causal = torch.tril(
|
| 502 |
+
torch.ones((num_video_frames, num_video_frames), dtype=torch.bool, device=device)
|
| 503 |
+
)
|
| 504 |
+
return frame_causal.repeat_interleave(video_tokens_per_frame, dim=0).repeat_interleave(
|
| 505 |
+
video_tokens_per_frame, dim=1
|
| 506 |
+
)
|
| 507 |
+
|
| 508 |
+
if self.video_attention_mask_mode == "first_frame_causal":
|
| 509 |
+
video_mask = torch.ones((video_seq_len, video_seq_len), dtype=torch.bool, device=device)
|
| 510 |
+
first_frame_tokens = min(video_tokens_per_frame, video_seq_len)
|
| 511 |
+
video_mask[:first_frame_tokens, first_frame_tokens:] = False
|
| 512 |
+
return video_mask
|
| 513 |
+
|
| 514 |
+
raise ValueError(f"Unsupported video attention mask mode: {self.video_attention_mask_mode}")
|
| 515 |
+
|
| 516 |
+
def pre_dit(
|
| 517 |
+
self,
|
| 518 |
+
x: torch.Tensor,
|
| 519 |
+
timestep: torch.Tensor,
|
| 520 |
+
context: torch.Tensor,
|
| 521 |
+
context_mask: Optional[torch.Tensor] = None,
|
| 522 |
+
action: Optional[torch.Tensor] = None,
|
| 523 |
+
fuse_vae_embedding_in_latents: bool = False,
|
| 524 |
+
control_camera_latents_input: Optional[torch.Tensor] = None,
|
| 525 |
+
) -> Dict[str, Any]:
|
| 526 |
+
x, timestep, context_mask = self._validate_forward_inputs(
|
| 527 |
+
x=x,
|
| 528 |
+
timestep=timestep,
|
| 529 |
+
context=context,
|
| 530 |
+
context_mask=context_mask,
|
| 531 |
+
action=action,
|
| 532 |
+
)
|
| 533 |
+
|
| 534 |
+
batch_size = x.shape[0]
|
| 535 |
+
patch_h = int(self.patch_size[1])
|
| 536 |
+
patch_w = int(self.patch_size[2])
|
| 537 |
+
if x.shape[3] % patch_h != 0 or x.shape[4] % patch_w != 0:
|
| 538 |
+
raise ValueError(
|
| 539 |
+
"Latent spatial shape must be divisible by DiT patch size, "
|
| 540 |
+
f"got HxW=({x.shape[3]}, {x.shape[4]}), patch=({patch_h}, {patch_w})"
|
| 541 |
+
)
|
| 542 |
+
tokens_per_frame = (x.shape[3] // patch_h) * (x.shape[4] // patch_w)
|
| 543 |
+
|
| 544 |
+
if self.seperated_timestep and fuse_vae_embedding_in_latents:
|
| 545 |
+
if not hasattr(self, "patch_size") or len(self.patch_size) < 3:
|
| 546 |
+
raise ValueError(f"Invalid dit.patch_size: {getattr(self, 'patch_size', None)}")
|
| 547 |
+
|
| 548 |
+
token_timesteps = torch.ones(
|
| 549 |
+
(batch_size, x.shape[2], tokens_per_frame),
|
| 550 |
+
dtype=timestep.dtype,
|
| 551 |
+
device=timestep.device,
|
| 552 |
+
) * timestep.view(batch_size, 1, 1)
|
| 553 |
+
token_timesteps[:, 0, :] = 0
|
| 554 |
+
token_timesteps = token_timesteps.reshape(batch_size, -1)
|
| 555 |
+
token_t_emb = sinusoidal_embedding_1d(self.freq_dim, token_timesteps.reshape(-1))
|
| 556 |
+
t = self.time_embedding(token_t_emb).reshape(batch_size, -1, self.hidden_dim)
|
| 557 |
+
t_mod = self.time_projection(t).unflatten(2, (6, self.hidden_dim))
|
| 558 |
+
else:
|
| 559 |
+
raise NotImplementedError("Only support seperated_timestep with fuse_vae_embedding_in_latents for now.")
|
| 560 |
+
t = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timestep))
|
| 561 |
+
t_mod = self.time_projection(t).unflatten(1, (6, self.hidden_dim))
|
| 562 |
+
x = self.patchify(x, control_camera_latents_input=control_camera_latents_input)
|
| 563 |
+
f, h, w = x.shape[2:]
|
| 564 |
+
|
| 565 |
+
context = self.text_embedding(context) # (B, L, dim)
|
| 566 |
+
context_len = context.shape[1]
|
| 567 |
+
if self.action_conditioned and action is not None:
|
| 568 |
+
action_len = action.shape[1]
|
| 569 |
+
action_emb = self.action_embedding(action) # (B, action_len, dim)
|
| 570 |
+
action_pos_embed = sinusoidal_embedding_1d(self.hidden_dim,
|
| 571 |
+
torch.arange(action_len, device=action_emb.device)) # (action_len, dim)
|
| 572 |
+
action_emb = action_emb + action_pos_embed.unsqueeze(0) # (B, action_len, dim)
|
| 573 |
+
context = torch.cat([context, action_emb], dim=1) # (B, context_len + action_len, dim)
|
| 574 |
+
|
| 575 |
+
# new mask
|
| 576 |
+
num_temporal_groups = f - 1 # first latent frame do not attend to actions
|
| 577 |
+
if num_temporal_groups <= 0:
|
| 578 |
+
raise ValueError(
|
| 579 |
+
"Action-conditioned context mask requires at least 2 latent frames when `action` is provided."
|
| 580 |
+
)
|
| 581 |
+
assert action_emb.shape[1] % num_temporal_groups == 0, \
|
| 582 |
+
f"Action embedding length {action_emb.shape[1]} must be divisible by number of temporal groups {num_temporal_groups}"
|
| 583 |
+
# Each latent frame (from the 2nd one) attends to the corresponding group of action tokens
|
| 584 |
+
action_group_mask = create_group_causal_attn_mask(
|
| 585 |
+
num_temporal_groups=num_temporal_groups,
|
| 586 |
+
num_query_per_group=tokens_per_frame,
|
| 587 |
+
num_key_per_group=action_len // num_temporal_groups,
|
| 588 |
+
mode=self.action_group_causal_mask_mode,
|
| 589 |
+
).to(context.device) # ((f-1)*tokens_per_frame, action_len)
|
| 590 |
+
|
| 591 |
+
seq_len = f * h * w # query length
|
| 592 |
+
final_context_mask = torch.zeros((batch_size, seq_len, context.shape[1]), dtype=torch.bool, device=context.device) # (B, seq_len, L + action_len)
|
| 593 |
+
# all latent frames attend to text tokens
|
| 594 |
+
final_context_mask[:, :, :context_len] = context_mask.unsqueeze(1).expand(-1, seq_len, -1) # (B, seq_len, L)
|
| 595 |
+
# latent frames from the 2nd one attend to action tokens
|
| 596 |
+
final_context_mask[:, tokens_per_frame:, context_len:] = action_group_mask.unsqueeze(0).expand(batch_size, -1, -1) # (B, seq_len, action_len)
|
| 597 |
+
context_mask = final_context_mask
|
| 598 |
+
elif self.action_conditioned and action is None:
|
| 599 |
+
if f != 1:
|
| 600 |
+
raise ValueError(
|
| 601 |
+
"Action-conditioned model requires `action` unless running single-frame text-only mode with num_latent_frames=1."
|
| 602 |
+
)
|
| 603 |
+
context_mask = context_mask.unsqueeze(1).expand(-1, f * h * w, -1) # (B, seq_len, L)
|
| 604 |
+
else:
|
| 605 |
+
context_mask = context_mask.unsqueeze(1).expand(-1, f * h * w, -1) # (B, seq_len, L)
|
| 606 |
+
|
| 607 |
+
x_tokens = rearrange(x, "b c f h w -> b (f h w) c").contiguous()
|
| 608 |
+
|
| 609 |
+
freqs = torch.cat([
|
| 610 |
+
self.freqs[0][:f].view(f, 1, 1, -1).expand(f, h, w, -1),
|
| 611 |
+
self.freqs[1][:h].view(1, h, 1, -1).expand(f, h, w, -1),
|
| 612 |
+
self.freqs[2][:w].view(1, 1, w, -1).expand(f, h, w, -1)
|
| 613 |
+
], dim=-1).reshape(f * h * w, 1, -1).to(x_tokens.device)
|
| 614 |
+
|
| 615 |
+
return {
|
| 616 |
+
"tokens": x_tokens,
|
| 617 |
+
"freqs": freqs,
|
| 618 |
+
"t": t,
|
| 619 |
+
"t_mod": t_mod,
|
| 620 |
+
"context": context,
|
| 621 |
+
"context_mask": context_mask,
|
| 622 |
+
"meta": {
|
| 623 |
+
"grid_size": (f, h, w),
|
| 624 |
+
"tokens_per_frame": tokens_per_frame,
|
| 625 |
+
"batch_size": batch_size,
|
| 626 |
+
},
|
| 627 |
+
}
|
| 628 |
+
|
| 629 |
+
def post_dit(self, x_tokens: torch.Tensor, pre_state: Dict[str, Any]) -> torch.Tensor:
|
| 630 |
+
f, h, w = pre_state["meta"]["grid_size"]
|
| 631 |
+
x = self.head(x_tokens, pre_state["t"])
|
| 632 |
+
x = self.unpatchify(x, (f, h, w))
|
| 633 |
+
return x
|
| 634 |
+
|
| 635 |
+
def forward(
|
| 636 |
+
self,
|
| 637 |
+
x: torch.Tensor,
|
| 638 |
+
timestep: torch.Tensor,
|
| 639 |
+
context: torch.Tensor,
|
| 640 |
+
context_mask: Optional[torch.Tensor] = None,
|
| 641 |
+
action: Optional[torch.Tensor] = None,
|
| 642 |
+
fuse_vae_embedding_in_latents: bool = False,
|
| 643 |
+
):
|
| 644 |
+
pre_state = self.pre_dit(
|
| 645 |
+
x=x,
|
| 646 |
+
timestep=timestep,
|
| 647 |
+
context=context,
|
| 648 |
+
context_mask=context_mask,
|
| 649 |
+
action=action,
|
| 650 |
+
fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents,
|
| 651 |
+
)
|
| 652 |
+
x_tokens = pre_state["tokens"]
|
| 653 |
+
context_emb = pre_state["context"]
|
| 654 |
+
t_mod = pre_state["t_mod"]
|
| 655 |
+
freqs = pre_state["freqs"]
|
| 656 |
+
context_attn_mask = pre_state["context_mask"]
|
| 657 |
+
self_attn_mask = self.build_video_to_video_mask(
|
| 658 |
+
video_seq_len=x_tokens.shape[1],
|
| 659 |
+
video_tokens_per_frame=int(pre_state["meta"]["tokens_per_frame"]),
|
| 660 |
+
device=x_tokens.device,
|
| 661 |
+
) if self.video_attention_mask_mode != "bidirectional" else None # special rule for faster speed
|
| 662 |
+
|
| 663 |
+
for block in self.blocks:
|
| 664 |
+
if self.use_gradient_checkpointing:
|
| 665 |
+
x_tokens = gradient_checkpoint_forward(
|
| 666 |
+
block,
|
| 667 |
+
self.use_gradient_checkpointing,
|
| 668 |
+
x_tokens, context_emb, t_mod, freqs, context_mask=context_attn_mask, self_attn_mask=self_attn_mask
|
| 669 |
+
)
|
| 670 |
+
else:
|
| 671 |
+
x_tokens = block(x_tokens, context_emb, t_mod, freqs, context_mask=context_attn_mask, self_attn_mask=self_attn_mask)
|
| 672 |
+
|
| 673 |
+
return self.post_dit(x_tokens, pre_state)
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_text_encoder.py
ADDED
|
@@ -0,0 +1,342 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
from transformers import AutoTokenizer
|
| 6 |
+
import html
|
| 7 |
+
import string
|
| 8 |
+
try:
|
| 9 |
+
import ftfy
|
| 10 |
+
except ModuleNotFoundError:
|
| 11 |
+
class _FTFY:
|
| 12 |
+
@staticmethod
|
| 13 |
+
def fix_text(text):
|
| 14 |
+
return text
|
| 15 |
+
|
| 16 |
+
ftfy = _FTFY()
|
| 17 |
+
try:
|
| 18 |
+
import regex as re
|
| 19 |
+
except ModuleNotFoundError:
|
| 20 |
+
import re
|
| 21 |
+
|
| 22 |
+
def fp16_clamp(x):
|
| 23 |
+
if x.dtype == torch.float16 and torch.isinf(x).any():
|
| 24 |
+
clamp = torch.finfo(x.dtype).max - 1000
|
| 25 |
+
x = torch.clamp(x, min=-clamp, max=clamp)
|
| 26 |
+
return x
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class GELU(nn.Module):
|
| 30 |
+
|
| 31 |
+
def forward(self, x):
|
| 32 |
+
return 0.5 * x * (1.0 + torch.tanh(
|
| 33 |
+
math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0))))
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class T5LayerNorm(nn.Module):
|
| 37 |
+
|
| 38 |
+
def __init__(self, dim, eps=1e-6):
|
| 39 |
+
super(T5LayerNorm, self).__init__()
|
| 40 |
+
self.dim = dim
|
| 41 |
+
self.eps = eps
|
| 42 |
+
self.weight = nn.Parameter(torch.ones(dim))
|
| 43 |
+
|
| 44 |
+
def forward(self, x):
|
| 45 |
+
x = x * torch.rsqrt(x.float().pow(2).mean(dim=-1, keepdim=True) +
|
| 46 |
+
self.eps)
|
| 47 |
+
if self.weight.dtype in [torch.float16, torch.bfloat16]:
|
| 48 |
+
x = x.type_as(self.weight)
|
| 49 |
+
return self.weight * x
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class T5Attention(nn.Module):
|
| 53 |
+
|
| 54 |
+
def __init__(self, dim, dim_attn, num_heads, dropout=0.1):
|
| 55 |
+
assert dim_attn % num_heads == 0
|
| 56 |
+
super(T5Attention, self).__init__()
|
| 57 |
+
self.dim = dim
|
| 58 |
+
self.dim_attn = dim_attn
|
| 59 |
+
self.num_heads = num_heads
|
| 60 |
+
self.head_dim = dim_attn // num_heads
|
| 61 |
+
|
| 62 |
+
# layers
|
| 63 |
+
self.q = nn.Linear(dim, dim_attn, bias=False)
|
| 64 |
+
self.k = nn.Linear(dim, dim_attn, bias=False)
|
| 65 |
+
self.v = nn.Linear(dim, dim_attn, bias=False)
|
| 66 |
+
self.o = nn.Linear(dim_attn, dim, bias=False)
|
| 67 |
+
self.dropout = nn.Dropout(dropout)
|
| 68 |
+
|
| 69 |
+
def forward(self, x, context=None, mask=None, pos_bias=None):
|
| 70 |
+
"""
|
| 71 |
+
x: [B, L1, C].
|
| 72 |
+
context: [B, L2, C] or None.
|
| 73 |
+
mask: [B, L2] or [B, L1, L2] or None.
|
| 74 |
+
"""
|
| 75 |
+
# check inputs
|
| 76 |
+
context = x if context is None else context
|
| 77 |
+
b, n, c = x.size(0), self.num_heads, self.head_dim
|
| 78 |
+
|
| 79 |
+
# compute query, key, value
|
| 80 |
+
q = self.q(x).view(b, -1, n, c)
|
| 81 |
+
k = self.k(context).view(b, -1, n, c)
|
| 82 |
+
v = self.v(context).view(b, -1, n, c)
|
| 83 |
+
|
| 84 |
+
# attention bias
|
| 85 |
+
attn_bias = x.new_zeros(b, n, q.size(1), k.size(1))
|
| 86 |
+
if pos_bias is not None:
|
| 87 |
+
attn_bias += pos_bias
|
| 88 |
+
if mask is not None:
|
| 89 |
+
assert mask.ndim in [2, 3]
|
| 90 |
+
mask = mask.view(b, 1, 1,
|
| 91 |
+
-1) if mask.ndim == 2 else mask.unsqueeze(1)
|
| 92 |
+
attn_bias.masked_fill_(mask == 0, torch.finfo(x.dtype).min)
|
| 93 |
+
|
| 94 |
+
# compute attention (T5 does not use scaling)
|
| 95 |
+
attn = torch.einsum('binc,bjnc->bnij', q, k) + attn_bias
|
| 96 |
+
attn = F.softmax(attn.float(), dim=-1).type_as(attn)
|
| 97 |
+
x = torch.einsum('bnij,bjnc->binc', attn, v)
|
| 98 |
+
|
| 99 |
+
# output
|
| 100 |
+
x = x.reshape(b, -1, n * c)
|
| 101 |
+
x = self.o(x)
|
| 102 |
+
x = self.dropout(x)
|
| 103 |
+
return x
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
class T5FeedForward(nn.Module):
|
| 107 |
+
|
| 108 |
+
def __init__(self, dim, dim_ffn, dropout=0.1):
|
| 109 |
+
super(T5FeedForward, self).__init__()
|
| 110 |
+
self.dim = dim
|
| 111 |
+
self.dim_ffn = dim_ffn
|
| 112 |
+
|
| 113 |
+
# layers
|
| 114 |
+
self.gate = nn.Sequential(nn.Linear(dim, dim_ffn, bias=False), GELU())
|
| 115 |
+
self.fc1 = nn.Linear(dim, dim_ffn, bias=False)
|
| 116 |
+
self.fc2 = nn.Linear(dim_ffn, dim, bias=False)
|
| 117 |
+
self.dropout = nn.Dropout(dropout)
|
| 118 |
+
|
| 119 |
+
def forward(self, x):
|
| 120 |
+
x = self.fc1(x) * self.gate(x)
|
| 121 |
+
x = self.dropout(x)
|
| 122 |
+
x = self.fc2(x)
|
| 123 |
+
x = self.dropout(x)
|
| 124 |
+
return x
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
class T5SelfAttention(nn.Module):
|
| 128 |
+
|
| 129 |
+
def __init__(self,
|
| 130 |
+
dim,
|
| 131 |
+
dim_attn,
|
| 132 |
+
dim_ffn,
|
| 133 |
+
num_heads,
|
| 134 |
+
num_buckets,
|
| 135 |
+
shared_pos=True,
|
| 136 |
+
dropout=0.1):
|
| 137 |
+
super(T5SelfAttention, self).__init__()
|
| 138 |
+
self.dim = dim
|
| 139 |
+
self.dim_attn = dim_attn
|
| 140 |
+
self.dim_ffn = dim_ffn
|
| 141 |
+
self.num_heads = num_heads
|
| 142 |
+
self.num_buckets = num_buckets
|
| 143 |
+
self.shared_pos = shared_pos
|
| 144 |
+
|
| 145 |
+
# layers
|
| 146 |
+
self.norm1 = T5LayerNorm(dim)
|
| 147 |
+
self.attn = T5Attention(dim, dim_attn, num_heads, dropout)
|
| 148 |
+
self.norm2 = T5LayerNorm(dim)
|
| 149 |
+
self.ffn = T5FeedForward(dim, dim_ffn, dropout)
|
| 150 |
+
self.pos_embedding = None if shared_pos else T5RelativeEmbedding(
|
| 151 |
+
num_buckets, num_heads, bidirectional=True)
|
| 152 |
+
|
| 153 |
+
def forward(self, x, mask=None, pos_bias=None):
|
| 154 |
+
e = pos_bias if self.shared_pos else self.pos_embedding(
|
| 155 |
+
x.size(1), x.size(1))
|
| 156 |
+
x = fp16_clamp(x + self.attn(self.norm1(x), mask=mask, pos_bias=e))
|
| 157 |
+
x = fp16_clamp(x + self.ffn(self.norm2(x)))
|
| 158 |
+
return x
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
class T5RelativeEmbedding(nn.Module):
|
| 162 |
+
|
| 163 |
+
def __init__(self, num_buckets, num_heads, bidirectional, max_dist=128):
|
| 164 |
+
super(T5RelativeEmbedding, self).__init__()
|
| 165 |
+
self.num_buckets = num_buckets
|
| 166 |
+
self.num_heads = num_heads
|
| 167 |
+
self.bidirectional = bidirectional
|
| 168 |
+
self.max_dist = max_dist
|
| 169 |
+
|
| 170 |
+
# layers
|
| 171 |
+
self.embedding = nn.Embedding(num_buckets, num_heads)
|
| 172 |
+
|
| 173 |
+
def forward(self, lq, lk):
|
| 174 |
+
device = self.embedding.weight.device
|
| 175 |
+
# rel_pos = torch.arange(lk).unsqueeze(0).to(device) - \
|
| 176 |
+
# torch.arange(lq).unsqueeze(1).to(device)
|
| 177 |
+
rel_pos = torch.arange(lk, device=device).unsqueeze(0) - \
|
| 178 |
+
torch.arange(lq, device=device).unsqueeze(1)
|
| 179 |
+
rel_pos = self._relative_position_bucket(rel_pos)
|
| 180 |
+
rel_pos_embeds = self.embedding(rel_pos)
|
| 181 |
+
rel_pos_embeds = rel_pos_embeds.permute(2, 0, 1).unsqueeze(
|
| 182 |
+
0) # [1, N, Lq, Lk]
|
| 183 |
+
return rel_pos_embeds.contiguous()
|
| 184 |
+
|
| 185 |
+
def _relative_position_bucket(self, rel_pos):
|
| 186 |
+
# preprocess
|
| 187 |
+
if self.bidirectional:
|
| 188 |
+
num_buckets = self.num_buckets // 2
|
| 189 |
+
rel_buckets = (rel_pos > 0).long() * num_buckets
|
| 190 |
+
rel_pos = torch.abs(rel_pos)
|
| 191 |
+
else:
|
| 192 |
+
num_buckets = self.num_buckets
|
| 193 |
+
rel_buckets = 0
|
| 194 |
+
rel_pos = -torch.min(rel_pos, torch.zeros_like(rel_pos))
|
| 195 |
+
|
| 196 |
+
# embeddings for small and large positions
|
| 197 |
+
max_exact = num_buckets // 2
|
| 198 |
+
rel_pos_large = max_exact + (torch.log(rel_pos.float() / max_exact) /
|
| 199 |
+
math.log(self.max_dist / max_exact) *
|
| 200 |
+
(num_buckets - max_exact)).long()
|
| 201 |
+
rel_pos_large = torch.min(
|
| 202 |
+
rel_pos_large, torch.full_like(rel_pos_large, num_buckets - 1))
|
| 203 |
+
rel_buckets += torch.where(rel_pos < max_exact, rel_pos, rel_pos_large)
|
| 204 |
+
return rel_buckets
|
| 205 |
+
|
| 206 |
+
def init_weights(m):
|
| 207 |
+
if isinstance(m, T5LayerNorm):
|
| 208 |
+
nn.init.ones_(m.weight)
|
| 209 |
+
elif isinstance(m, T5FeedForward):
|
| 210 |
+
nn.init.normal_(m.gate[0].weight, std=m.dim**-0.5)
|
| 211 |
+
nn.init.normal_(m.fc1.weight, std=m.dim**-0.5)
|
| 212 |
+
nn.init.normal_(m.fc2.weight, std=m.dim_ffn**-0.5)
|
| 213 |
+
elif isinstance(m, T5Attention):
|
| 214 |
+
nn.init.normal_(m.q.weight, std=(m.dim * m.dim_attn)**-0.5)
|
| 215 |
+
nn.init.normal_(m.k.weight, std=m.dim**-0.5)
|
| 216 |
+
nn.init.normal_(m.v.weight, std=m.dim**-0.5)
|
| 217 |
+
nn.init.normal_(m.o.weight, std=(m.num_heads * m.dim_attn)**-0.5)
|
| 218 |
+
elif isinstance(m, T5RelativeEmbedding):
|
| 219 |
+
nn.init.normal_(
|
| 220 |
+
m.embedding.weight, std=(2 * m.num_buckets * m.num_heads)**-0.5)
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
class WanTextEncoder(torch.nn.Module):
|
| 224 |
+
|
| 225 |
+
def __init__(self,
|
| 226 |
+
vocab=256384,
|
| 227 |
+
dim=4096,
|
| 228 |
+
dim_attn=4096,
|
| 229 |
+
dim_ffn=10240,
|
| 230 |
+
num_heads=64,
|
| 231 |
+
num_layers=24,
|
| 232 |
+
num_buckets=32,
|
| 233 |
+
shared_pos=False,
|
| 234 |
+
dropout=0.1):
|
| 235 |
+
super(WanTextEncoder, self).__init__()
|
| 236 |
+
self.dim = dim
|
| 237 |
+
self.dim_attn = dim_attn
|
| 238 |
+
self.dim_ffn = dim_ffn
|
| 239 |
+
self.num_heads = num_heads
|
| 240 |
+
self.num_layers = num_layers
|
| 241 |
+
self.num_buckets = num_buckets
|
| 242 |
+
self.shared_pos = shared_pos
|
| 243 |
+
|
| 244 |
+
# layers
|
| 245 |
+
self.token_embedding = vocab if isinstance(vocab, nn.Embedding) \
|
| 246 |
+
else nn.Embedding(vocab, dim)
|
| 247 |
+
self.pos_embedding = T5RelativeEmbedding(
|
| 248 |
+
num_buckets, num_heads, bidirectional=True) if shared_pos else None
|
| 249 |
+
self.dropout = nn.Dropout(dropout)
|
| 250 |
+
self.blocks = nn.ModuleList([
|
| 251 |
+
T5SelfAttention(dim, dim_attn, dim_ffn, num_heads, num_buckets,
|
| 252 |
+
shared_pos, dropout) for _ in range(num_layers)
|
| 253 |
+
])
|
| 254 |
+
self.norm = T5LayerNorm(dim)
|
| 255 |
+
|
| 256 |
+
# Skip costly random init when loading pretrained checkpoints.
|
| 257 |
+
# Verified: checkpoint fully covers this module (missing/unexpected keys are both 0).
|
| 258 |
+
# self.apply(init_weights)
|
| 259 |
+
|
| 260 |
+
def forward(self, ids, mask=None):
|
| 261 |
+
x = self.token_embedding(ids)
|
| 262 |
+
x = self.dropout(x)
|
| 263 |
+
e = self.pos_embedding(x.size(1),
|
| 264 |
+
x.size(1)) if self.shared_pos else None
|
| 265 |
+
for block in self.blocks:
|
| 266 |
+
x = block(x, mask, pos_bias=e)
|
| 267 |
+
x = self.norm(x)
|
| 268 |
+
x = self.dropout(x)
|
| 269 |
+
return x
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def basic_clean(text):
|
| 273 |
+
text = ftfy.fix_text(text)
|
| 274 |
+
text = html.unescape(html.unescape(text))
|
| 275 |
+
return text.strip()
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def whitespace_clean(text):
|
| 279 |
+
text = re.sub(r'\s+', ' ', text)
|
| 280 |
+
text = text.strip()
|
| 281 |
+
return text
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
def canonicalize(text, keep_punctuation_exact_string=None):
|
| 285 |
+
text = text.replace('_', ' ')
|
| 286 |
+
if keep_punctuation_exact_string:
|
| 287 |
+
text = keep_punctuation_exact_string.join(
|
| 288 |
+
part.translate(str.maketrans('', '', string.punctuation))
|
| 289 |
+
for part in text.split(keep_punctuation_exact_string))
|
| 290 |
+
else:
|
| 291 |
+
text = text.translate(str.maketrans('', '', string.punctuation))
|
| 292 |
+
text = text.lower()
|
| 293 |
+
text = re.sub(r'\s+', ' ', text)
|
| 294 |
+
return text.strip()
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
class HuggingfaceTokenizer:
|
| 298 |
+
|
| 299 |
+
def __init__(self, name, seq_len=None, clean=None, **kwargs):
|
| 300 |
+
assert clean in (None, 'whitespace', 'lower', 'canonicalize')
|
| 301 |
+
self.name = name
|
| 302 |
+
self.seq_len = seq_len
|
| 303 |
+
self.clean = clean
|
| 304 |
+
|
| 305 |
+
# init tokenizer
|
| 306 |
+
self.tokenizer = AutoTokenizer.from_pretrained(name, **kwargs)
|
| 307 |
+
self.vocab_size = self.tokenizer.vocab_size
|
| 308 |
+
|
| 309 |
+
def __call__(self, sequence, **kwargs):
|
| 310 |
+
return_mask = kwargs.pop('return_mask', False)
|
| 311 |
+
|
| 312 |
+
# arguments
|
| 313 |
+
_kwargs = {'return_tensors': 'pt'}
|
| 314 |
+
if self.seq_len is not None:
|
| 315 |
+
_kwargs.update({
|
| 316 |
+
'padding': 'max_length',
|
| 317 |
+
'truncation': True,
|
| 318 |
+
'max_length': self.seq_len
|
| 319 |
+
})
|
| 320 |
+
_kwargs.update(**kwargs)
|
| 321 |
+
|
| 322 |
+
# tokenization
|
| 323 |
+
if isinstance(sequence, str):
|
| 324 |
+
sequence = [sequence]
|
| 325 |
+
if self.clean:
|
| 326 |
+
sequence = [self._clean(u) for u in sequence]
|
| 327 |
+
ids = self.tokenizer(sequence, **_kwargs)
|
| 328 |
+
|
| 329 |
+
# output
|
| 330 |
+
if return_mask:
|
| 331 |
+
return ids.input_ids, ids.attention_mask
|
| 332 |
+
else:
|
| 333 |
+
return ids.input_ids
|
| 334 |
+
|
| 335 |
+
def _clean(self, text):
|
| 336 |
+
if self.clean == 'whitespace':
|
| 337 |
+
text = whitespace_clean(basic_clean(text))
|
| 338 |
+
elif self.clean == 'lower':
|
| 339 |
+
text = whitespace_clean(basic_clean(text)).lower()
|
| 340 |
+
elif self.clean == 'canonicalize':
|
| 341 |
+
text = canonicalize(basic_clean(text))
|
| 342 |
+
return text
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/models/wan22/wan_video_vae.py
ADDED
|
@@ -0,0 +1,1384 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from einops import rearrange, repeat
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
from tqdm import tqdm
|
| 7 |
+
|
| 8 |
+
CACHE_T = 2
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def check_is_instance(model, module_class):
|
| 12 |
+
if isinstance(model, module_class):
|
| 13 |
+
return True
|
| 14 |
+
if hasattr(model, "module") and isinstance(model.module, module_class):
|
| 15 |
+
return True
|
| 16 |
+
return False
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def block_causal_mask(x, block_size):
|
| 20 |
+
# params
|
| 21 |
+
b, n, s, _, device = *x.size(), x.device
|
| 22 |
+
assert s % block_size == 0
|
| 23 |
+
num_blocks = s // block_size
|
| 24 |
+
|
| 25 |
+
# build mask
|
| 26 |
+
mask = torch.zeros(b, n, s, s, dtype=torch.bool, device=device)
|
| 27 |
+
for i in range(num_blocks):
|
| 28 |
+
mask[:, :,
|
| 29 |
+
i * block_size:(i + 1) * block_size, :(i + 1) * block_size] = 1
|
| 30 |
+
return mask
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class CausalConv3d(nn.Conv3d):
|
| 34 |
+
"""
|
| 35 |
+
Causal 3d convolusion.
|
| 36 |
+
"""
|
| 37 |
+
|
| 38 |
+
def __init__(self, *args, **kwargs):
|
| 39 |
+
super().__init__(*args, **kwargs)
|
| 40 |
+
self._padding = (self.padding[2], self.padding[2], self.padding[1],
|
| 41 |
+
self.padding[1], 2 * self.padding[0], 0)
|
| 42 |
+
self.padding = (0, 0, 0)
|
| 43 |
+
|
| 44 |
+
def forward(self, x, cache_x=None):
|
| 45 |
+
padding = list(self._padding)
|
| 46 |
+
if cache_x is not None and self._padding[4] > 0:
|
| 47 |
+
cache_x = cache_x.to(x.device)
|
| 48 |
+
x = torch.cat([cache_x, x], dim=2)
|
| 49 |
+
padding[4] -= cache_x.shape[2]
|
| 50 |
+
x = F.pad(x, padding)
|
| 51 |
+
|
| 52 |
+
return super().forward(x)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class RMS_norm(nn.Module):
|
| 56 |
+
|
| 57 |
+
def __init__(self, dim, channel_first=True, images=True, bias=False):
|
| 58 |
+
super().__init__()
|
| 59 |
+
broadcastable_dims = (1, 1, 1) if not images else (1, 1)
|
| 60 |
+
shape = (dim, *broadcastable_dims) if channel_first else (dim,)
|
| 61 |
+
|
| 62 |
+
self.channel_first = channel_first
|
| 63 |
+
self.scale = dim**0.5
|
| 64 |
+
self.gamma = nn.Parameter(torch.ones(shape))
|
| 65 |
+
self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0.
|
| 66 |
+
|
| 67 |
+
def forward(self, x):
|
| 68 |
+
return F.normalize(
|
| 69 |
+
x, dim=(1 if self.channel_first else
|
| 70 |
+
-1)) * self.scale * self.gamma + self.bias
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class Upsample(nn.Upsample):
|
| 74 |
+
|
| 75 |
+
def forward(self, x):
|
| 76 |
+
"""
|
| 77 |
+
Fix bfloat16 support for nearest neighbor interpolation.
|
| 78 |
+
"""
|
| 79 |
+
return super().forward(x.float()).type_as(x)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class Resample(nn.Module):
|
| 83 |
+
|
| 84 |
+
def __init__(self, dim, mode):
|
| 85 |
+
assert mode in ('none', 'upsample2d', 'upsample3d', 'downsample2d',
|
| 86 |
+
'downsample3d')
|
| 87 |
+
super().__init__()
|
| 88 |
+
self.dim = dim
|
| 89 |
+
self.mode = mode
|
| 90 |
+
|
| 91 |
+
# layers
|
| 92 |
+
if mode == 'upsample2d':
|
| 93 |
+
self.resample = nn.Sequential(
|
| 94 |
+
Upsample(scale_factor=(2., 2.), mode='nearest-exact'),
|
| 95 |
+
nn.Conv2d(dim, dim // 2, 3, padding=1))
|
| 96 |
+
elif mode == 'upsample3d':
|
| 97 |
+
self.resample = nn.Sequential(
|
| 98 |
+
Upsample(scale_factor=(2., 2.), mode='nearest-exact'),
|
| 99 |
+
nn.Conv2d(dim, dim // 2, 3, padding=1))
|
| 100 |
+
self.time_conv = CausalConv3d(dim,
|
| 101 |
+
dim * 2, (3, 1, 1),
|
| 102 |
+
padding=(1, 0, 0))
|
| 103 |
+
|
| 104 |
+
elif mode == 'downsample2d':
|
| 105 |
+
self.resample = nn.Sequential(
|
| 106 |
+
nn.ZeroPad2d((0, 1, 0, 1)),
|
| 107 |
+
nn.Conv2d(dim, dim, 3, stride=(2, 2)))
|
| 108 |
+
elif mode == 'downsample3d':
|
| 109 |
+
self.resample = nn.Sequential(
|
| 110 |
+
nn.ZeroPad2d((0, 1, 0, 1)),
|
| 111 |
+
nn.Conv2d(dim, dim, 3, stride=(2, 2)))
|
| 112 |
+
self.time_conv = CausalConv3d(dim,
|
| 113 |
+
dim, (3, 1, 1),
|
| 114 |
+
stride=(2, 1, 1),
|
| 115 |
+
padding=(0, 0, 0))
|
| 116 |
+
|
| 117 |
+
else:
|
| 118 |
+
self.resample = nn.Identity()
|
| 119 |
+
|
| 120 |
+
def forward(self, x, feat_cache=None, feat_idx=[0]):
|
| 121 |
+
b, c, t, h, w = x.size()
|
| 122 |
+
if self.mode == 'upsample3d':
|
| 123 |
+
if feat_cache is not None:
|
| 124 |
+
idx = feat_idx[0]
|
| 125 |
+
if feat_cache[idx] is None:
|
| 126 |
+
feat_cache[idx] = 'Rep'
|
| 127 |
+
feat_idx[0] += 1
|
| 128 |
+
else:
|
| 129 |
+
|
| 130 |
+
cache_x = x[:, :, -CACHE_T:, :, :].clone()
|
| 131 |
+
if cache_x.shape[2] < 2 and feat_cache[
|
| 132 |
+
idx] is not None and feat_cache[idx] != 'Rep':
|
| 133 |
+
# cache last frame of last two chunk
|
| 134 |
+
cache_x = torch.cat([
|
| 135 |
+
feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(
|
| 136 |
+
cache_x.device), cache_x
|
| 137 |
+
],
|
| 138 |
+
dim=2)
|
| 139 |
+
if cache_x.shape[2] < 2 and feat_cache[
|
| 140 |
+
idx] is not None and feat_cache[idx] == 'Rep':
|
| 141 |
+
cache_x = torch.cat([
|
| 142 |
+
torch.zeros_like(cache_x).to(cache_x.device),
|
| 143 |
+
cache_x
|
| 144 |
+
],
|
| 145 |
+
dim=2)
|
| 146 |
+
if feat_cache[idx] == 'Rep':
|
| 147 |
+
x = self.time_conv(x)
|
| 148 |
+
else:
|
| 149 |
+
x = self.time_conv(x, feat_cache[idx])
|
| 150 |
+
feat_cache[idx] = cache_x
|
| 151 |
+
feat_idx[0] += 1
|
| 152 |
+
|
| 153 |
+
x = x.reshape(b, 2, c, t, h, w)
|
| 154 |
+
x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]),
|
| 155 |
+
3)
|
| 156 |
+
x = x.reshape(b, c, t * 2, h, w)
|
| 157 |
+
t = x.shape[2]
|
| 158 |
+
x = rearrange(x, 'b c t h w -> (b t) c h w')
|
| 159 |
+
x = self.resample(x)
|
| 160 |
+
x = rearrange(x, '(b t) c h w -> b c t h w', t=t)
|
| 161 |
+
|
| 162 |
+
if self.mode == 'downsample3d':
|
| 163 |
+
if feat_cache is not None:
|
| 164 |
+
idx = feat_idx[0]
|
| 165 |
+
if feat_cache[idx] is None:
|
| 166 |
+
feat_cache[idx] = x.clone()
|
| 167 |
+
feat_idx[0] += 1
|
| 168 |
+
else:
|
| 169 |
+
cache_x = x[:, :, -1:, :, :].clone()
|
| 170 |
+
x = self.time_conv(
|
| 171 |
+
torch.cat([feat_cache[idx][:, :, -1:, :, :], x], 2))
|
| 172 |
+
feat_cache[idx] = cache_x
|
| 173 |
+
feat_idx[0] += 1
|
| 174 |
+
return x, feat_cache, feat_idx
|
| 175 |
+
|
| 176 |
+
def init_weight(self, conv):
|
| 177 |
+
conv_weight = conv.weight
|
| 178 |
+
nn.init.zeros_(conv_weight)
|
| 179 |
+
c1, c2, t, h, w = conv_weight.size()
|
| 180 |
+
one_matrix = torch.eye(c1, c2)
|
| 181 |
+
init_matrix = one_matrix
|
| 182 |
+
nn.init.zeros_(conv_weight)
|
| 183 |
+
conv_weight.data[:, :, 1, 0, 0] = init_matrix
|
| 184 |
+
conv.weight.data.copy_(conv_weight)
|
| 185 |
+
nn.init.zeros_(conv.bias.data)
|
| 186 |
+
|
| 187 |
+
def init_weight2(self, conv):
|
| 188 |
+
conv_weight = conv.weight.data
|
| 189 |
+
nn.init.zeros_(conv_weight)
|
| 190 |
+
c1, c2, t, h, w = conv_weight.size()
|
| 191 |
+
init_matrix = torch.eye(c1 // 2, c2)
|
| 192 |
+
conv_weight[:c1 // 2, :, -1, 0, 0] = init_matrix
|
| 193 |
+
conv_weight[c1 // 2:, :, -1, 0, 0] = init_matrix
|
| 194 |
+
conv.weight.data.copy_(conv_weight)
|
| 195 |
+
nn.init.zeros_(conv.bias.data)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def patchify(x, patch_size):
|
| 200 |
+
if patch_size == 1:
|
| 201 |
+
return x
|
| 202 |
+
if x.dim() == 4:
|
| 203 |
+
x = rearrange(x, "b c (h q) (w r) -> b (c r q) h w", q=patch_size, r=patch_size)
|
| 204 |
+
elif x.dim() == 5:
|
| 205 |
+
x = rearrange(x,
|
| 206 |
+
"b c f (h q) (w r) -> b (c r q) f h w",
|
| 207 |
+
q=patch_size,
|
| 208 |
+
r=patch_size)
|
| 209 |
+
else:
|
| 210 |
+
raise ValueError(f"Invalid input shape: {x.shape}")
|
| 211 |
+
return x
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def unpatchify(x, patch_size):
|
| 215 |
+
if patch_size == 1:
|
| 216 |
+
return x
|
| 217 |
+
if x.dim() == 4:
|
| 218 |
+
x = rearrange(x, "b (c r q) h w -> b c (h q) (w r)", q=patch_size, r=patch_size)
|
| 219 |
+
elif x.dim() == 5:
|
| 220 |
+
x = rearrange(x,
|
| 221 |
+
"b (c r q) f h w -> b c f (h q) (w r)",
|
| 222 |
+
q=patch_size,
|
| 223 |
+
r=patch_size)
|
| 224 |
+
return x
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
class Resample38(Resample):
|
| 228 |
+
|
| 229 |
+
def __init__(self, dim, mode):
|
| 230 |
+
assert mode in (
|
| 231 |
+
"none",
|
| 232 |
+
"upsample2d",
|
| 233 |
+
"upsample3d",
|
| 234 |
+
"downsample2d",
|
| 235 |
+
"downsample3d",
|
| 236 |
+
)
|
| 237 |
+
super(Resample, self).__init__()
|
| 238 |
+
self.dim = dim
|
| 239 |
+
self.mode = mode
|
| 240 |
+
|
| 241 |
+
# layers
|
| 242 |
+
if mode == "upsample2d":
|
| 243 |
+
self.resample = nn.Sequential(
|
| 244 |
+
Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact"),
|
| 245 |
+
nn.Conv2d(dim, dim, 3, padding=1),
|
| 246 |
+
)
|
| 247 |
+
elif mode == "upsample3d":
|
| 248 |
+
self.resample = nn.Sequential(
|
| 249 |
+
Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact"),
|
| 250 |
+
nn.Conv2d(dim, dim, 3, padding=1),
|
| 251 |
+
)
|
| 252 |
+
self.time_conv = CausalConv3d(dim, dim * 2, (3, 1, 1), padding=(1, 0, 0))
|
| 253 |
+
elif mode == "downsample2d":
|
| 254 |
+
self.resample = nn.Sequential(
|
| 255 |
+
nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2))
|
| 256 |
+
)
|
| 257 |
+
elif mode == "downsample3d":
|
| 258 |
+
self.resample = nn.Sequential(
|
| 259 |
+
nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2))
|
| 260 |
+
)
|
| 261 |
+
self.time_conv = CausalConv3d(
|
| 262 |
+
dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0)
|
| 263 |
+
)
|
| 264 |
+
else:
|
| 265 |
+
self.resample = nn.Identity()
|
| 266 |
+
|
| 267 |
+
class ResidualBlock(nn.Module):
|
| 268 |
+
|
| 269 |
+
def __init__(self, in_dim, out_dim, dropout=0.0):
|
| 270 |
+
super().__init__()
|
| 271 |
+
self.in_dim = in_dim
|
| 272 |
+
self.out_dim = out_dim
|
| 273 |
+
|
| 274 |
+
# layers
|
| 275 |
+
self.residual = nn.Sequential(
|
| 276 |
+
RMS_norm(in_dim, images=False), nn.SiLU(),
|
| 277 |
+
CausalConv3d(in_dim, out_dim, 3, padding=1),
|
| 278 |
+
RMS_norm(out_dim, images=False), nn.SiLU(), nn.Dropout(dropout),
|
| 279 |
+
CausalConv3d(out_dim, out_dim, 3, padding=1))
|
| 280 |
+
self.shortcut = CausalConv3d(in_dim, out_dim, 1) \
|
| 281 |
+
if in_dim != out_dim else nn.Identity()
|
| 282 |
+
|
| 283 |
+
def forward(self, x, feat_cache=None, feat_idx=[0]):
|
| 284 |
+
h = self.shortcut(x)
|
| 285 |
+
for layer in self.residual:
|
| 286 |
+
if check_is_instance(layer, CausalConv3d) and feat_cache is not None:
|
| 287 |
+
idx = feat_idx[0]
|
| 288 |
+
cache_x = x[:, :, -CACHE_T:, :, :].clone()
|
| 289 |
+
if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
|
| 290 |
+
# cache last frame of last two chunk
|
| 291 |
+
cache_x = torch.cat([
|
| 292 |
+
feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(
|
| 293 |
+
cache_x.device), cache_x
|
| 294 |
+
],
|
| 295 |
+
dim=2)
|
| 296 |
+
x = layer(x, feat_cache[idx])
|
| 297 |
+
feat_cache[idx] = cache_x
|
| 298 |
+
feat_idx[0] += 1
|
| 299 |
+
else:
|
| 300 |
+
x = layer(x)
|
| 301 |
+
return x + h, feat_cache, feat_idx
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
class AttentionBlock(nn.Module):
|
| 305 |
+
"""
|
| 306 |
+
Causal self-attention with a single head.
|
| 307 |
+
"""
|
| 308 |
+
|
| 309 |
+
def __init__(self, dim):
|
| 310 |
+
super().__init__()
|
| 311 |
+
self.dim = dim
|
| 312 |
+
|
| 313 |
+
# layers
|
| 314 |
+
self.norm = RMS_norm(dim)
|
| 315 |
+
self.to_qkv = nn.Conv2d(dim, dim * 3, 1)
|
| 316 |
+
self.proj = nn.Conv2d(dim, dim, 1)
|
| 317 |
+
|
| 318 |
+
# zero out the last layer params
|
| 319 |
+
nn.init.zeros_(self.proj.weight)
|
| 320 |
+
|
| 321 |
+
def forward(self, x):
|
| 322 |
+
identity = x
|
| 323 |
+
b, c, t, h, w = x.size()
|
| 324 |
+
x = rearrange(x, 'b c t h w -> (b t) c h w')
|
| 325 |
+
x = self.norm(x)
|
| 326 |
+
# compute query, key, value
|
| 327 |
+
q, k, v = self.to_qkv(x).reshape(b * t, 1, c * 3, -1).permute(
|
| 328 |
+
0, 1, 3, 2).contiguous().chunk(3, dim=-1)
|
| 329 |
+
|
| 330 |
+
# apply attention
|
| 331 |
+
x = F.scaled_dot_product_attention(
|
| 332 |
+
q,
|
| 333 |
+
k,
|
| 334 |
+
v,
|
| 335 |
+
#attn_mask=block_causal_mask(q, block_size=h * w)
|
| 336 |
+
)
|
| 337 |
+
x = x.squeeze(1).permute(0, 2, 1).reshape(b * t, c, h, w)
|
| 338 |
+
|
| 339 |
+
# output
|
| 340 |
+
x = self.proj(x)
|
| 341 |
+
x = rearrange(x, '(b t) c h w-> b c t h w', t=t)
|
| 342 |
+
return x + identity
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
class AvgDown3D(nn.Module):
|
| 346 |
+
def __init__(
|
| 347 |
+
self,
|
| 348 |
+
in_channels,
|
| 349 |
+
out_channels,
|
| 350 |
+
factor_t,
|
| 351 |
+
factor_s=1,
|
| 352 |
+
):
|
| 353 |
+
super().__init__()
|
| 354 |
+
self.in_channels = in_channels
|
| 355 |
+
self.out_channels = out_channels
|
| 356 |
+
self.factor_t = factor_t
|
| 357 |
+
self.factor_s = factor_s
|
| 358 |
+
self.factor = self.factor_t * self.factor_s * self.factor_s
|
| 359 |
+
|
| 360 |
+
assert in_channels * self.factor % out_channels == 0
|
| 361 |
+
self.group_size = in_channels * self.factor // out_channels
|
| 362 |
+
|
| 363 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 364 |
+
pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t
|
| 365 |
+
pad = (0, 0, 0, 0, pad_t, 0)
|
| 366 |
+
x = F.pad(x, pad)
|
| 367 |
+
B, C, T, H, W = x.shape
|
| 368 |
+
x = x.view(
|
| 369 |
+
B,
|
| 370 |
+
C,
|
| 371 |
+
T // self.factor_t,
|
| 372 |
+
self.factor_t,
|
| 373 |
+
H // self.factor_s,
|
| 374 |
+
self.factor_s,
|
| 375 |
+
W // self.factor_s,
|
| 376 |
+
self.factor_s,
|
| 377 |
+
)
|
| 378 |
+
x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous()
|
| 379 |
+
x = x.view(
|
| 380 |
+
B,
|
| 381 |
+
C * self.factor,
|
| 382 |
+
T // self.factor_t,
|
| 383 |
+
H // self.factor_s,
|
| 384 |
+
W // self.factor_s,
|
| 385 |
+
)
|
| 386 |
+
x = x.view(
|
| 387 |
+
B,
|
| 388 |
+
self.out_channels,
|
| 389 |
+
self.group_size,
|
| 390 |
+
T // self.factor_t,
|
| 391 |
+
H // self.factor_s,
|
| 392 |
+
W // self.factor_s,
|
| 393 |
+
)
|
| 394 |
+
x = x.mean(dim=2)
|
| 395 |
+
return x
|
| 396 |
+
|
| 397 |
+
|
| 398 |
+
class DupUp3D(nn.Module):
|
| 399 |
+
def __init__(
|
| 400 |
+
self,
|
| 401 |
+
in_channels: int,
|
| 402 |
+
out_channels: int,
|
| 403 |
+
factor_t,
|
| 404 |
+
factor_s=1,
|
| 405 |
+
):
|
| 406 |
+
super().__init__()
|
| 407 |
+
self.in_channels = in_channels
|
| 408 |
+
self.out_channels = out_channels
|
| 409 |
+
|
| 410 |
+
self.factor_t = factor_t
|
| 411 |
+
self.factor_s = factor_s
|
| 412 |
+
self.factor = self.factor_t * self.factor_s * self.factor_s
|
| 413 |
+
|
| 414 |
+
assert out_channels * self.factor % in_channels == 0
|
| 415 |
+
self.repeats = out_channels * self.factor // in_channels
|
| 416 |
+
|
| 417 |
+
def forward(self, x: torch.Tensor, first_chunk=False) -> torch.Tensor:
|
| 418 |
+
x = x.repeat_interleave(self.repeats, dim=1)
|
| 419 |
+
x = x.view(
|
| 420 |
+
x.size(0),
|
| 421 |
+
self.out_channels,
|
| 422 |
+
self.factor_t,
|
| 423 |
+
self.factor_s,
|
| 424 |
+
self.factor_s,
|
| 425 |
+
x.size(2),
|
| 426 |
+
x.size(3),
|
| 427 |
+
x.size(4),
|
| 428 |
+
)
|
| 429 |
+
x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous()
|
| 430 |
+
x = x.view(
|
| 431 |
+
x.size(0),
|
| 432 |
+
self.out_channels,
|
| 433 |
+
x.size(2) * self.factor_t,
|
| 434 |
+
x.size(4) * self.factor_s,
|
| 435 |
+
x.size(6) * self.factor_s,
|
| 436 |
+
)
|
| 437 |
+
if first_chunk:
|
| 438 |
+
x = x[:, :, self.factor_t - 1 :, :, :]
|
| 439 |
+
return x
|
| 440 |
+
|
| 441 |
+
|
| 442 |
+
class Down_ResidualBlock(nn.Module):
|
| 443 |
+
def __init__(
|
| 444 |
+
self, in_dim, out_dim, dropout, mult, temperal_downsample=False, down_flag=False
|
| 445 |
+
):
|
| 446 |
+
super().__init__()
|
| 447 |
+
|
| 448 |
+
# Shortcut path with downsample
|
| 449 |
+
self.avg_shortcut = AvgDown3D(
|
| 450 |
+
in_dim,
|
| 451 |
+
out_dim,
|
| 452 |
+
factor_t=2 if temperal_downsample else 1,
|
| 453 |
+
factor_s=2 if down_flag else 1,
|
| 454 |
+
)
|
| 455 |
+
|
| 456 |
+
# Main path with residual blocks and downsample
|
| 457 |
+
downsamples = []
|
| 458 |
+
for _ in range(mult):
|
| 459 |
+
downsamples.append(ResidualBlock(in_dim, out_dim, dropout))
|
| 460 |
+
in_dim = out_dim
|
| 461 |
+
|
| 462 |
+
# Add the final downsample block
|
| 463 |
+
if down_flag:
|
| 464 |
+
mode = "downsample3d" if temperal_downsample else "downsample2d"
|
| 465 |
+
downsamples.append(Resample38(out_dim, mode=mode))
|
| 466 |
+
|
| 467 |
+
self.downsamples = nn.Sequential(*downsamples)
|
| 468 |
+
|
| 469 |
+
def forward(self, x, feat_cache=None, feat_idx=[0]):
|
| 470 |
+
x_copy = x.clone()
|
| 471 |
+
for module in self.downsamples:
|
| 472 |
+
x, feat_cache, feat_idx = module(x, feat_cache, feat_idx)
|
| 473 |
+
|
| 474 |
+
return x + self.avg_shortcut(x_copy), feat_cache, feat_idx
|
| 475 |
+
|
| 476 |
+
|
| 477 |
+
class Up_ResidualBlock(nn.Module):
|
| 478 |
+
def __init__(
|
| 479 |
+
self, in_dim, out_dim, dropout, mult, temperal_upsample=False, up_flag=False
|
| 480 |
+
):
|
| 481 |
+
super().__init__()
|
| 482 |
+
# Shortcut path with upsample
|
| 483 |
+
if up_flag:
|
| 484 |
+
self.avg_shortcut = DupUp3D(
|
| 485 |
+
in_dim,
|
| 486 |
+
out_dim,
|
| 487 |
+
factor_t=2 if temperal_upsample else 1,
|
| 488 |
+
factor_s=2 if up_flag else 1,
|
| 489 |
+
)
|
| 490 |
+
else:
|
| 491 |
+
self.avg_shortcut = None
|
| 492 |
+
|
| 493 |
+
# Main path with residual blocks and upsample
|
| 494 |
+
upsamples = []
|
| 495 |
+
for _ in range(mult):
|
| 496 |
+
upsamples.append(ResidualBlock(in_dim, out_dim, dropout))
|
| 497 |
+
in_dim = out_dim
|
| 498 |
+
|
| 499 |
+
# Add the final upsample block
|
| 500 |
+
if up_flag:
|
| 501 |
+
mode = "upsample3d" if temperal_upsample else "upsample2d"
|
| 502 |
+
upsamples.append(Resample38(out_dim, mode=mode))
|
| 503 |
+
|
| 504 |
+
self.upsamples = nn.Sequential(*upsamples)
|
| 505 |
+
|
| 506 |
+
def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False):
|
| 507 |
+
x_main = x.clone()
|
| 508 |
+
for module in self.upsamples:
|
| 509 |
+
x_main, feat_cache, feat_idx = module(x_main, feat_cache, feat_idx)
|
| 510 |
+
if self.avg_shortcut is not None:
|
| 511 |
+
x_shortcut = self.avg_shortcut(x, first_chunk)
|
| 512 |
+
x_main = x_main + x_shortcut
|
| 513 |
+
return x_main, feat_cache, feat_idx
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
class Encoder3d(nn.Module):
|
| 517 |
+
|
| 518 |
+
def __init__(self,
|
| 519 |
+
dim=128,
|
| 520 |
+
z_dim=4,
|
| 521 |
+
dim_mult=[1, 2, 4, 4],
|
| 522 |
+
num_res_blocks=2,
|
| 523 |
+
attn_scales=[],
|
| 524 |
+
temperal_downsample=[True, True, False],
|
| 525 |
+
dropout=0.0):
|
| 526 |
+
super().__init__()
|
| 527 |
+
self.dim = dim
|
| 528 |
+
self.z_dim = z_dim
|
| 529 |
+
self.dim_mult = dim_mult
|
| 530 |
+
self.num_res_blocks = num_res_blocks
|
| 531 |
+
self.attn_scales = attn_scales
|
| 532 |
+
self.temperal_downsample = temperal_downsample
|
| 533 |
+
|
| 534 |
+
# dimensions
|
| 535 |
+
dims = [dim * u for u in [1] + dim_mult]
|
| 536 |
+
scale = 1.0
|
| 537 |
+
|
| 538 |
+
# init block
|
| 539 |
+
self.conv1 = CausalConv3d(3, dims[0], 3, padding=1)
|
| 540 |
+
|
| 541 |
+
# downsample blocks
|
| 542 |
+
downsamples = []
|
| 543 |
+
for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])):
|
| 544 |
+
# residual (+attention) blocks
|
| 545 |
+
for _ in range(num_res_blocks):
|
| 546 |
+
downsamples.append(ResidualBlock(in_dim, out_dim, dropout))
|
| 547 |
+
if scale in attn_scales:
|
| 548 |
+
downsamples.append(AttentionBlock(out_dim))
|
| 549 |
+
in_dim = out_dim
|
| 550 |
+
|
| 551 |
+
# downsample block
|
| 552 |
+
if i != len(dim_mult) - 1:
|
| 553 |
+
mode = 'downsample3d' if temperal_downsample[
|
| 554 |
+
i] else 'downsample2d'
|
| 555 |
+
downsamples.append(Resample(out_dim, mode=mode))
|
| 556 |
+
scale /= 2.0
|
| 557 |
+
self.downsamples = nn.Sequential(*downsamples)
|
| 558 |
+
|
| 559 |
+
# middle blocks
|
| 560 |
+
self.middle = nn.Sequential(ResidualBlock(out_dim, out_dim, dropout),
|
| 561 |
+
AttentionBlock(out_dim),
|
| 562 |
+
ResidualBlock(out_dim, out_dim, dropout))
|
| 563 |
+
|
| 564 |
+
# output blocks
|
| 565 |
+
self.head = nn.Sequential(RMS_norm(out_dim, images=False), nn.SiLU(),
|
| 566 |
+
CausalConv3d(out_dim, z_dim, 3, padding=1))
|
| 567 |
+
|
| 568 |
+
def forward(self, x, feat_cache=None, feat_idx=[0]):
|
| 569 |
+
if feat_cache is not None:
|
| 570 |
+
idx = feat_idx[0]
|
| 571 |
+
cache_x = x[:, :, -CACHE_T:, :, :].clone()
|
| 572 |
+
if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
|
| 573 |
+
# cache last frame of last two chunk
|
| 574 |
+
cache_x = torch.cat([
|
| 575 |
+
feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(
|
| 576 |
+
cache_x.device), cache_x
|
| 577 |
+
],
|
| 578 |
+
dim=2)
|
| 579 |
+
x = self.conv1(x, feat_cache[idx])
|
| 580 |
+
feat_cache[idx] = cache_x
|
| 581 |
+
feat_idx[0] += 1
|
| 582 |
+
else:
|
| 583 |
+
x = self.conv1(x)
|
| 584 |
+
|
| 585 |
+
## downsamples
|
| 586 |
+
for layer in self.downsamples:
|
| 587 |
+
if feat_cache is not None:
|
| 588 |
+
x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
|
| 589 |
+
else:
|
| 590 |
+
x = layer(x)
|
| 591 |
+
|
| 592 |
+
## middle
|
| 593 |
+
for layer in self.middle:
|
| 594 |
+
if check_is_instance(layer, ResidualBlock) and feat_cache is not None:
|
| 595 |
+
x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
|
| 596 |
+
else:
|
| 597 |
+
x = layer(x)
|
| 598 |
+
|
| 599 |
+
## head
|
| 600 |
+
for layer in self.head:
|
| 601 |
+
if check_is_instance(layer, CausalConv3d) and feat_cache is not None:
|
| 602 |
+
idx = feat_idx[0]
|
| 603 |
+
cache_x = x[:, :, -CACHE_T:, :, :].clone()
|
| 604 |
+
if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
|
| 605 |
+
# cache last frame of last two chunk
|
| 606 |
+
cache_x = torch.cat([
|
| 607 |
+
feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(
|
| 608 |
+
cache_x.device), cache_x
|
| 609 |
+
],
|
| 610 |
+
dim=2)
|
| 611 |
+
x = layer(x, feat_cache[idx])
|
| 612 |
+
feat_cache[idx] = cache_x
|
| 613 |
+
feat_idx[0] += 1
|
| 614 |
+
else:
|
| 615 |
+
x = layer(x)
|
| 616 |
+
return x, feat_cache, feat_idx
|
| 617 |
+
|
| 618 |
+
|
| 619 |
+
class Encoder3d_38(nn.Module):
|
| 620 |
+
|
| 621 |
+
def __init__(self,
|
| 622 |
+
dim=128,
|
| 623 |
+
z_dim=4,
|
| 624 |
+
dim_mult=[1, 2, 4, 4],
|
| 625 |
+
num_res_blocks=2,
|
| 626 |
+
attn_scales=[],
|
| 627 |
+
temperal_downsample=[False, True, True],
|
| 628 |
+
dropout=0.0):
|
| 629 |
+
super().__init__()
|
| 630 |
+
self.dim = dim
|
| 631 |
+
self.z_dim = z_dim
|
| 632 |
+
self.dim_mult = dim_mult
|
| 633 |
+
self.num_res_blocks = num_res_blocks
|
| 634 |
+
self.attn_scales = attn_scales
|
| 635 |
+
self.temperal_downsample = temperal_downsample
|
| 636 |
+
|
| 637 |
+
# dimensions
|
| 638 |
+
dims = [dim * u for u in [1] + dim_mult]
|
| 639 |
+
scale = 1.0
|
| 640 |
+
|
| 641 |
+
# init block
|
| 642 |
+
self.conv1 = CausalConv3d(12, dims[0], 3, padding=1)
|
| 643 |
+
|
| 644 |
+
# downsample blocks
|
| 645 |
+
downsamples = []
|
| 646 |
+
for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])):
|
| 647 |
+
t_down_flag = (
|
| 648 |
+
temperal_downsample[i] if i < len(temperal_downsample) else False
|
| 649 |
+
)
|
| 650 |
+
downsamples.append(
|
| 651 |
+
Down_ResidualBlock(
|
| 652 |
+
in_dim=in_dim,
|
| 653 |
+
out_dim=out_dim,
|
| 654 |
+
dropout=dropout,
|
| 655 |
+
mult=num_res_blocks,
|
| 656 |
+
temperal_downsample=t_down_flag,
|
| 657 |
+
down_flag=i != len(dim_mult) - 1,
|
| 658 |
+
)
|
| 659 |
+
)
|
| 660 |
+
scale /= 2.0
|
| 661 |
+
self.downsamples = nn.Sequential(*downsamples)
|
| 662 |
+
|
| 663 |
+
# middle blocks
|
| 664 |
+
self.middle = nn.Sequential(
|
| 665 |
+
ResidualBlock(out_dim, out_dim, dropout),
|
| 666 |
+
AttentionBlock(out_dim),
|
| 667 |
+
ResidualBlock(out_dim, out_dim, dropout),
|
| 668 |
+
)
|
| 669 |
+
|
| 670 |
+
# # output blocks
|
| 671 |
+
self.head = nn.Sequential(
|
| 672 |
+
RMS_norm(out_dim, images=False),
|
| 673 |
+
nn.SiLU(),
|
| 674 |
+
CausalConv3d(out_dim, z_dim, 3, padding=1),
|
| 675 |
+
)
|
| 676 |
+
|
| 677 |
+
|
| 678 |
+
def forward(self, x, feat_cache=None, feat_idx=[0]):
|
| 679 |
+
|
| 680 |
+
if feat_cache is not None:
|
| 681 |
+
idx = feat_idx[0]
|
| 682 |
+
cache_x = x[:, :, -CACHE_T:, :, :].clone()
|
| 683 |
+
if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
|
| 684 |
+
cache_x = torch.cat(
|
| 685 |
+
[
|
| 686 |
+
feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device),
|
| 687 |
+
cache_x,
|
| 688 |
+
],
|
| 689 |
+
dim=2,
|
| 690 |
+
)
|
| 691 |
+
x = self.conv1(x, feat_cache[idx])
|
| 692 |
+
feat_cache[idx] = cache_x
|
| 693 |
+
feat_idx[0] += 1
|
| 694 |
+
else:
|
| 695 |
+
x = self.conv1(x)
|
| 696 |
+
|
| 697 |
+
## downsamples
|
| 698 |
+
for layer in self.downsamples:
|
| 699 |
+
if feat_cache is not None:
|
| 700 |
+
x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
|
| 701 |
+
else:
|
| 702 |
+
x = layer(x)
|
| 703 |
+
|
| 704 |
+
## middle
|
| 705 |
+
for layer in self.middle:
|
| 706 |
+
if isinstance(layer, ResidualBlock) and feat_cache is not None:
|
| 707 |
+
x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
|
| 708 |
+
else:
|
| 709 |
+
x = layer(x)
|
| 710 |
+
|
| 711 |
+
## head
|
| 712 |
+
for layer in self.head:
|
| 713 |
+
if isinstance(layer, CausalConv3d) and feat_cache is not None:
|
| 714 |
+
idx = feat_idx[0]
|
| 715 |
+
cache_x = x[:, :, -CACHE_T:, :, :].clone()
|
| 716 |
+
if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
|
| 717 |
+
cache_x = torch.cat(
|
| 718 |
+
[
|
| 719 |
+
feat_cache[idx][:, :, -1, :, :]
|
| 720 |
+
.unsqueeze(2)
|
| 721 |
+
.to(cache_x.device),
|
| 722 |
+
cache_x,
|
| 723 |
+
],
|
| 724 |
+
dim=2,
|
| 725 |
+
)
|
| 726 |
+
x = layer(x, feat_cache[idx])
|
| 727 |
+
feat_cache[idx] = cache_x
|
| 728 |
+
feat_idx[0] += 1
|
| 729 |
+
else:
|
| 730 |
+
x = layer(x)
|
| 731 |
+
|
| 732 |
+
return x, feat_cache, feat_idx
|
| 733 |
+
|
| 734 |
+
|
| 735 |
+
class Decoder3d(nn.Module):
|
| 736 |
+
|
| 737 |
+
def __init__(self,
|
| 738 |
+
dim=128,
|
| 739 |
+
z_dim=4,
|
| 740 |
+
dim_mult=[1, 2, 4, 4],
|
| 741 |
+
num_res_blocks=2,
|
| 742 |
+
attn_scales=[],
|
| 743 |
+
temperal_upsample=[False, True, True],
|
| 744 |
+
dropout=0.0):
|
| 745 |
+
super().__init__()
|
| 746 |
+
self.dim = dim
|
| 747 |
+
self.z_dim = z_dim
|
| 748 |
+
self.dim_mult = dim_mult
|
| 749 |
+
self.num_res_blocks = num_res_blocks
|
| 750 |
+
self.attn_scales = attn_scales
|
| 751 |
+
self.temperal_upsample = temperal_upsample
|
| 752 |
+
|
| 753 |
+
# dimensions
|
| 754 |
+
dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]]
|
| 755 |
+
scale = 1.0 / 2**(len(dim_mult) - 2)
|
| 756 |
+
|
| 757 |
+
# init block
|
| 758 |
+
self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1)
|
| 759 |
+
|
| 760 |
+
# middle blocks
|
| 761 |
+
self.middle = nn.Sequential(ResidualBlock(dims[0], dims[0], dropout),
|
| 762 |
+
AttentionBlock(dims[0]),
|
| 763 |
+
ResidualBlock(dims[0], dims[0], dropout))
|
| 764 |
+
|
| 765 |
+
# upsample blocks
|
| 766 |
+
upsamples = []
|
| 767 |
+
for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])):
|
| 768 |
+
# residual (+attention) blocks
|
| 769 |
+
if i == 1 or i == 2 or i == 3:
|
| 770 |
+
in_dim = in_dim // 2
|
| 771 |
+
for _ in range(num_res_blocks + 1):
|
| 772 |
+
upsamples.append(ResidualBlock(in_dim, out_dim, dropout))
|
| 773 |
+
if scale in attn_scales:
|
| 774 |
+
upsamples.append(AttentionBlock(out_dim))
|
| 775 |
+
in_dim = out_dim
|
| 776 |
+
|
| 777 |
+
# upsample block
|
| 778 |
+
if i != len(dim_mult) - 1:
|
| 779 |
+
mode = 'upsample3d' if temperal_upsample[i] else 'upsample2d'
|
| 780 |
+
upsamples.append(Resample(out_dim, mode=mode))
|
| 781 |
+
scale *= 2.0
|
| 782 |
+
self.upsamples = nn.Sequential(*upsamples)
|
| 783 |
+
|
| 784 |
+
# output blocks
|
| 785 |
+
self.head = nn.Sequential(RMS_norm(out_dim, images=False), nn.SiLU(),
|
| 786 |
+
CausalConv3d(out_dim, 3, 3, padding=1))
|
| 787 |
+
|
| 788 |
+
def forward(self, x, feat_cache=None, feat_idx=[0]):
|
| 789 |
+
## conv1
|
| 790 |
+
if feat_cache is not None:
|
| 791 |
+
idx = feat_idx[0]
|
| 792 |
+
cache_x = x[:, :, -CACHE_T:, :, :].clone()
|
| 793 |
+
if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
|
| 794 |
+
# cache last frame of last two chunk
|
| 795 |
+
cache_x = torch.cat([
|
| 796 |
+
feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(
|
| 797 |
+
cache_x.device), cache_x
|
| 798 |
+
],
|
| 799 |
+
dim=2)
|
| 800 |
+
x = self.conv1(x, feat_cache[idx])
|
| 801 |
+
feat_cache[idx] = cache_x
|
| 802 |
+
feat_idx[0] += 1
|
| 803 |
+
else:
|
| 804 |
+
x = self.conv1(x)
|
| 805 |
+
|
| 806 |
+
## middle
|
| 807 |
+
for layer in self.middle:
|
| 808 |
+
if check_is_instance(layer, ResidualBlock) and feat_cache is not None:
|
| 809 |
+
x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
|
| 810 |
+
else:
|
| 811 |
+
x = layer(x)
|
| 812 |
+
|
| 813 |
+
## upsamples
|
| 814 |
+
for layer in self.upsamples:
|
| 815 |
+
if feat_cache is not None:
|
| 816 |
+
x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
|
| 817 |
+
else:
|
| 818 |
+
x = layer(x)
|
| 819 |
+
|
| 820 |
+
## head
|
| 821 |
+
for layer in self.head:
|
| 822 |
+
if check_is_instance(layer, CausalConv3d) and feat_cache is not None:
|
| 823 |
+
idx = feat_idx[0]
|
| 824 |
+
cache_x = x[:, :, -CACHE_T:, :, :].clone()
|
| 825 |
+
if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
|
| 826 |
+
# cache last frame of last two chunk
|
| 827 |
+
cache_x = torch.cat([
|
| 828 |
+
feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(
|
| 829 |
+
cache_x.device), cache_x
|
| 830 |
+
],
|
| 831 |
+
dim=2)
|
| 832 |
+
x = layer(x, feat_cache[idx])
|
| 833 |
+
feat_cache[idx] = cache_x
|
| 834 |
+
feat_idx[0] += 1
|
| 835 |
+
else:
|
| 836 |
+
x = layer(x)
|
| 837 |
+
return x, feat_cache, feat_idx
|
| 838 |
+
|
| 839 |
+
|
| 840 |
+
|
| 841 |
+
class Decoder3d_38(nn.Module):
|
| 842 |
+
|
| 843 |
+
def __init__(self,
|
| 844 |
+
dim=128,
|
| 845 |
+
z_dim=4,
|
| 846 |
+
dim_mult=[1, 2, 4, 4],
|
| 847 |
+
num_res_blocks=2,
|
| 848 |
+
attn_scales=[],
|
| 849 |
+
temperal_upsample=[False, True, True],
|
| 850 |
+
dropout=0.0):
|
| 851 |
+
super().__init__()
|
| 852 |
+
self.dim = dim
|
| 853 |
+
self.z_dim = z_dim
|
| 854 |
+
self.dim_mult = dim_mult
|
| 855 |
+
self.num_res_blocks = num_res_blocks
|
| 856 |
+
self.attn_scales = attn_scales
|
| 857 |
+
self.temperal_upsample = temperal_upsample
|
| 858 |
+
|
| 859 |
+
# dimensions
|
| 860 |
+
dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]]
|
| 861 |
+
scale = 1.0 / 2 ** (len(dim_mult) - 2)
|
| 862 |
+
# init block
|
| 863 |
+
self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1)
|
| 864 |
+
|
| 865 |
+
# middle blocks
|
| 866 |
+
self.middle = nn.Sequential(ResidualBlock(dims[0], dims[0], dropout),
|
| 867 |
+
AttentionBlock(dims[0]),
|
| 868 |
+
ResidualBlock(dims[0], dims[0], dropout))
|
| 869 |
+
|
| 870 |
+
# upsample blocks
|
| 871 |
+
upsamples = []
|
| 872 |
+
for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])):
|
| 873 |
+
t_up_flag = temperal_upsample[i] if i < len(temperal_upsample) else False
|
| 874 |
+
upsamples.append(
|
| 875 |
+
Up_ResidualBlock(in_dim=in_dim,
|
| 876 |
+
out_dim=out_dim,
|
| 877 |
+
dropout=dropout,
|
| 878 |
+
mult=num_res_blocks + 1,
|
| 879 |
+
temperal_upsample=t_up_flag,
|
| 880 |
+
up_flag=i != len(dim_mult) - 1))
|
| 881 |
+
self.upsamples = nn.Sequential(*upsamples)
|
| 882 |
+
|
| 883 |
+
# output blocks
|
| 884 |
+
self.head = nn.Sequential(RMS_norm(out_dim, images=False), nn.SiLU(),
|
| 885 |
+
CausalConv3d(out_dim, 12, 3, padding=1))
|
| 886 |
+
|
| 887 |
+
|
| 888 |
+
def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False):
|
| 889 |
+
if feat_cache is not None:
|
| 890 |
+
idx = feat_idx[0]
|
| 891 |
+
cache_x = x[:, :, -CACHE_T:, :, :].clone()
|
| 892 |
+
if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
|
| 893 |
+
cache_x = torch.cat(
|
| 894 |
+
[
|
| 895 |
+
feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device),
|
| 896 |
+
cache_x,
|
| 897 |
+
],
|
| 898 |
+
dim=2,
|
| 899 |
+
)
|
| 900 |
+
x = self.conv1(x, feat_cache[idx])
|
| 901 |
+
feat_cache[idx] = cache_x
|
| 902 |
+
feat_idx[0] += 1
|
| 903 |
+
else:
|
| 904 |
+
x = self.conv1(x)
|
| 905 |
+
|
| 906 |
+
for layer in self.middle:
|
| 907 |
+
if check_is_instance(layer, ResidualBlock) and feat_cache is not None:
|
| 908 |
+
x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx)
|
| 909 |
+
else:
|
| 910 |
+
x = layer(x)
|
| 911 |
+
|
| 912 |
+
## upsamples
|
| 913 |
+
for layer in self.upsamples:
|
| 914 |
+
if feat_cache is not None:
|
| 915 |
+
x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx, first_chunk)
|
| 916 |
+
else:
|
| 917 |
+
x = layer(x)
|
| 918 |
+
|
| 919 |
+
## head
|
| 920 |
+
for layer in self.head:
|
| 921 |
+
if check_is_instance(layer, CausalConv3d) and feat_cache is not None:
|
| 922 |
+
idx = feat_idx[0]
|
| 923 |
+
cache_x = x[:, :, -CACHE_T:, :, :].clone()
|
| 924 |
+
if cache_x.shape[2] < 2 and feat_cache[idx] is not None:
|
| 925 |
+
cache_x = torch.cat(
|
| 926 |
+
[
|
| 927 |
+
feat_cache[idx][:, :, -1, :, :]
|
| 928 |
+
.unsqueeze(2)
|
| 929 |
+
.to(cache_x.device),
|
| 930 |
+
cache_x,
|
| 931 |
+
],
|
| 932 |
+
dim=2,
|
| 933 |
+
)
|
| 934 |
+
x = layer(x, feat_cache[idx])
|
| 935 |
+
feat_cache[idx] = cache_x
|
| 936 |
+
feat_idx[0] += 1
|
| 937 |
+
else:
|
| 938 |
+
x = layer(x)
|
| 939 |
+
return x, feat_cache, feat_idx
|
| 940 |
+
|
| 941 |
+
|
| 942 |
+
def count_conv3d(model):
|
| 943 |
+
count = 0
|
| 944 |
+
for m in model.modules():
|
| 945 |
+
if isinstance(m, CausalConv3d):
|
| 946 |
+
count += 1
|
| 947 |
+
return count
|
| 948 |
+
|
| 949 |
+
|
| 950 |
+
class VideoVAE_(nn.Module):
|
| 951 |
+
|
| 952 |
+
def __init__(self,
|
| 953 |
+
dim=96,
|
| 954 |
+
z_dim=16,
|
| 955 |
+
dim_mult=[1, 2, 4, 4],
|
| 956 |
+
num_res_blocks=2,
|
| 957 |
+
attn_scales=[],
|
| 958 |
+
temperal_downsample=[False, True, True],
|
| 959 |
+
dropout=0.0):
|
| 960 |
+
super().__init__()
|
| 961 |
+
self.dim = dim
|
| 962 |
+
self.z_dim = z_dim
|
| 963 |
+
self.dim_mult = dim_mult
|
| 964 |
+
self.num_res_blocks = num_res_blocks
|
| 965 |
+
self.attn_scales = attn_scales
|
| 966 |
+
self.temperal_downsample = temperal_downsample
|
| 967 |
+
self.temperal_upsample = temperal_downsample[::-1]
|
| 968 |
+
|
| 969 |
+
# modules
|
| 970 |
+
self.encoder = Encoder3d(dim, z_dim * 2, dim_mult, num_res_blocks,
|
| 971 |
+
attn_scales, self.temperal_downsample, dropout)
|
| 972 |
+
self.conv1 = CausalConv3d(z_dim * 2, z_dim * 2, 1)
|
| 973 |
+
self.conv2 = CausalConv3d(z_dim, z_dim, 1)
|
| 974 |
+
self.decoder = Decoder3d(dim, z_dim, dim_mult, num_res_blocks,
|
| 975 |
+
attn_scales, self.temperal_upsample, dropout)
|
| 976 |
+
|
| 977 |
+
def forward(self, x):
|
| 978 |
+
mu, log_var = self.encode(x)
|
| 979 |
+
z = self.reparameterize(mu, log_var)
|
| 980 |
+
x_recon = self.decode(z)
|
| 981 |
+
return x_recon, mu, log_var
|
| 982 |
+
|
| 983 |
+
def encode(self, x, scale):
|
| 984 |
+
self.clear_cache()
|
| 985 |
+
## cache
|
| 986 |
+
t = x.shape[2]
|
| 987 |
+
iter_ = 1 + (t - 1) // 4
|
| 988 |
+
|
| 989 |
+
for i in range(iter_):
|
| 990 |
+
self._enc_conv_idx = [0]
|
| 991 |
+
if i == 0:
|
| 992 |
+
out, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, :1, :, :],
|
| 993 |
+
feat_cache=self._enc_feat_map,
|
| 994 |
+
feat_idx=self._enc_conv_idx)
|
| 995 |
+
else:
|
| 996 |
+
out_, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, 1 + 4 * (i - 1):1 + 4 * i, :, :],
|
| 997 |
+
feat_cache=self._enc_feat_map,
|
| 998 |
+
feat_idx=self._enc_conv_idx)
|
| 999 |
+
out = torch.cat([out, out_], 2)
|
| 1000 |
+
mu, log_var = self.conv1(out).chunk(2, dim=1)
|
| 1001 |
+
if isinstance(scale[0], torch.Tensor):
|
| 1002 |
+
scale = [s.to(dtype=mu.dtype, device=mu.device) for s in scale]
|
| 1003 |
+
mu = (mu - scale[0].view(1, self.z_dim, 1, 1, 1)) * scale[1].view(
|
| 1004 |
+
1, self.z_dim, 1, 1, 1)
|
| 1005 |
+
else:
|
| 1006 |
+
scale = scale.to(dtype=mu.dtype, device=mu.device)
|
| 1007 |
+
mu = (mu - scale[0]) * scale[1]
|
| 1008 |
+
return mu
|
| 1009 |
+
|
| 1010 |
+
def decode(self, z, scale):
|
| 1011 |
+
self.clear_cache()
|
| 1012 |
+
# z: [b,c,t,h,w]
|
| 1013 |
+
if isinstance(scale[0], torch.Tensor):
|
| 1014 |
+
scale = [s.to(dtype=z.dtype, device=z.device) for s in scale]
|
| 1015 |
+
z = z / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view(
|
| 1016 |
+
1, self.z_dim, 1, 1, 1)
|
| 1017 |
+
else:
|
| 1018 |
+
scale = scale.to(dtype=z.dtype, device=z.device)
|
| 1019 |
+
z = z / scale[1] + scale[0]
|
| 1020 |
+
iter_ = z.shape[2]
|
| 1021 |
+
x = self.conv2(z)
|
| 1022 |
+
for i in range(iter_):
|
| 1023 |
+
self._conv_idx = [0]
|
| 1024 |
+
if i == 0:
|
| 1025 |
+
out, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :],
|
| 1026 |
+
feat_cache=self._feat_map,
|
| 1027 |
+
feat_idx=self._conv_idx)
|
| 1028 |
+
else:
|
| 1029 |
+
out_, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :],
|
| 1030 |
+
feat_cache=self._feat_map,
|
| 1031 |
+
feat_idx=self._conv_idx)
|
| 1032 |
+
out = torch.cat([out, out_], 2) # may add tensor offload
|
| 1033 |
+
return out
|
| 1034 |
+
|
| 1035 |
+
def reparameterize(self, mu, log_var):
|
| 1036 |
+
std = torch.exp(0.5 * log_var)
|
| 1037 |
+
eps = torch.randn_like(std)
|
| 1038 |
+
return eps * std + mu
|
| 1039 |
+
|
| 1040 |
+
def sample(self, imgs, deterministic=False):
|
| 1041 |
+
mu, log_var = self.encode(imgs)
|
| 1042 |
+
if deterministic:
|
| 1043 |
+
return mu
|
| 1044 |
+
std = torch.exp(0.5 * log_var.clamp(-30.0, 20.0))
|
| 1045 |
+
return mu + std * torch.randn_like(std)
|
| 1046 |
+
|
| 1047 |
+
def clear_cache(self):
|
| 1048 |
+
self._conv_num = count_conv3d(self.decoder)
|
| 1049 |
+
self._conv_idx = [0]
|
| 1050 |
+
self._feat_map = [None] * self._conv_num
|
| 1051 |
+
# cache encode
|
| 1052 |
+
self._enc_conv_num = count_conv3d(self.encoder)
|
| 1053 |
+
self._enc_conv_idx = [0]
|
| 1054 |
+
self._enc_feat_map = [None] * self._enc_conv_num
|
| 1055 |
+
|
| 1056 |
+
|
| 1057 |
+
class WanVideoVAE(nn.Module):
|
| 1058 |
+
|
| 1059 |
+
def __init__(self, z_dim=16):
|
| 1060 |
+
super().__init__()
|
| 1061 |
+
|
| 1062 |
+
mean = [
|
| 1063 |
+
-0.7571, -0.7089, -0.9113, 0.1075, -0.1745, 0.9653, -0.1517, 1.5508,
|
| 1064 |
+
0.4134, -0.0715, 0.5517, -0.3632, -0.1922, -0.9497, 0.2503, -0.2921
|
| 1065 |
+
]
|
| 1066 |
+
std = [
|
| 1067 |
+
2.8184, 1.4541, 2.3275, 2.6558, 1.2196, 1.7708, 2.6052, 2.0743,
|
| 1068 |
+
3.2687, 2.1526, 2.8652, 1.5579, 1.6382, 1.1253, 2.8251, 1.9160
|
| 1069 |
+
]
|
| 1070 |
+
self.mean = torch.tensor(mean)
|
| 1071 |
+
self.std = torch.tensor(std)
|
| 1072 |
+
self.scale = [self.mean, 1.0 / self.std]
|
| 1073 |
+
|
| 1074 |
+
# init model
|
| 1075 |
+
self.model = VideoVAE_(z_dim=z_dim).eval().requires_grad_(False)
|
| 1076 |
+
self.upsampling_factor = 8
|
| 1077 |
+
self.temporal_downsample_factor = 4
|
| 1078 |
+
self.z_dim = z_dim
|
| 1079 |
+
|
| 1080 |
+
|
| 1081 |
+
def build_1d_mask(self, length, left_bound, right_bound, border_width):
|
| 1082 |
+
x = torch.ones((length,))
|
| 1083 |
+
if not left_bound:
|
| 1084 |
+
x[:border_width] = (torch.arange(border_width) + 1) / border_width
|
| 1085 |
+
if not right_bound:
|
| 1086 |
+
x[-border_width:] = torch.flip((torch.arange(border_width) + 1) / border_width, dims=(0,))
|
| 1087 |
+
return x
|
| 1088 |
+
|
| 1089 |
+
|
| 1090 |
+
def build_mask(self, data, is_bound, border_width):
|
| 1091 |
+
_, _, _, H, W = data.shape
|
| 1092 |
+
h = self.build_1d_mask(H, is_bound[0], is_bound[1], border_width[0])
|
| 1093 |
+
w = self.build_1d_mask(W, is_bound[2], is_bound[3], border_width[1])
|
| 1094 |
+
|
| 1095 |
+
h = repeat(h, "H -> H W", H=H, W=W)
|
| 1096 |
+
w = repeat(w, "W -> H W", H=H, W=W)
|
| 1097 |
+
|
| 1098 |
+
mask = torch.stack([h, w]).min(dim=0).values
|
| 1099 |
+
mask = rearrange(mask, "H W -> 1 1 1 H W")
|
| 1100 |
+
return mask
|
| 1101 |
+
|
| 1102 |
+
|
| 1103 |
+
def tiled_decode(self, hidden_states, device, tile_size, tile_stride):
|
| 1104 |
+
_, _, T, H, W = hidden_states.shape
|
| 1105 |
+
size_h, size_w = tile_size
|
| 1106 |
+
stride_h, stride_w = tile_stride
|
| 1107 |
+
|
| 1108 |
+
# Split tasks
|
| 1109 |
+
tasks = []
|
| 1110 |
+
for h in range(0, H, stride_h):
|
| 1111 |
+
if (h-stride_h >= 0 and h-stride_h+size_h >= H): continue
|
| 1112 |
+
for w in range(0, W, stride_w):
|
| 1113 |
+
if (w-stride_w >= 0 and w-stride_w+size_w >= W): continue
|
| 1114 |
+
h_, w_ = h + size_h, w + size_w
|
| 1115 |
+
tasks.append((h, h_, w, w_))
|
| 1116 |
+
|
| 1117 |
+
data_device = "cpu"
|
| 1118 |
+
computation_device = device
|
| 1119 |
+
|
| 1120 |
+
out_T = T * 4 - 3
|
| 1121 |
+
weight = torch.zeros((1, 1, out_T, H * self.upsampling_factor, W * self.upsampling_factor), dtype=hidden_states.dtype, device=data_device)
|
| 1122 |
+
values = torch.zeros((1, 3, out_T, H * self.upsampling_factor, W * self.upsampling_factor), dtype=hidden_states.dtype, device=data_device)
|
| 1123 |
+
|
| 1124 |
+
for h, h_, w, w_ in tqdm(tasks, desc="VAE decoding"):
|
| 1125 |
+
hidden_states_batch = hidden_states[:, :, :, h:h_, w:w_].to(computation_device)
|
| 1126 |
+
hidden_states_batch = self.model.decode(hidden_states_batch, self.scale).to(data_device)
|
| 1127 |
+
|
| 1128 |
+
mask = self.build_mask(
|
| 1129 |
+
hidden_states_batch,
|
| 1130 |
+
is_bound=(h==0, h_>=H, w==0, w_>=W),
|
| 1131 |
+
border_width=((size_h - stride_h) * self.upsampling_factor, (size_w - stride_w) * self.upsampling_factor)
|
| 1132 |
+
).to(dtype=hidden_states.dtype, device=data_device)
|
| 1133 |
+
|
| 1134 |
+
target_h = h * self.upsampling_factor
|
| 1135 |
+
target_w = w * self.upsampling_factor
|
| 1136 |
+
values[
|
| 1137 |
+
:,
|
| 1138 |
+
:,
|
| 1139 |
+
:,
|
| 1140 |
+
target_h:target_h + hidden_states_batch.shape[3],
|
| 1141 |
+
target_w:target_w + hidden_states_batch.shape[4],
|
| 1142 |
+
] += hidden_states_batch * mask
|
| 1143 |
+
weight[
|
| 1144 |
+
:,
|
| 1145 |
+
:,
|
| 1146 |
+
:,
|
| 1147 |
+
target_h: target_h + hidden_states_batch.shape[3],
|
| 1148 |
+
target_w: target_w + hidden_states_batch.shape[4],
|
| 1149 |
+
] += mask
|
| 1150 |
+
values = values / weight
|
| 1151 |
+
values = values.clamp_(-1, 1)
|
| 1152 |
+
return values
|
| 1153 |
+
|
| 1154 |
+
|
| 1155 |
+
def tiled_encode(self, video, device, tile_size, tile_stride):
|
| 1156 |
+
_, _, T, H, W = video.shape
|
| 1157 |
+
size_h, size_w = tile_size
|
| 1158 |
+
stride_h, stride_w = tile_stride
|
| 1159 |
+
|
| 1160 |
+
# Split tasks
|
| 1161 |
+
tasks = []
|
| 1162 |
+
for h in range(0, H, stride_h):
|
| 1163 |
+
if (h-stride_h >= 0 and h-stride_h+size_h >= H): continue
|
| 1164 |
+
for w in range(0, W, stride_w):
|
| 1165 |
+
if (w-stride_w >= 0 and w-stride_w+size_w >= W): continue
|
| 1166 |
+
h_, w_ = h + size_h, w + size_w
|
| 1167 |
+
tasks.append((h, h_, w, w_))
|
| 1168 |
+
|
| 1169 |
+
data_device = "cpu"
|
| 1170 |
+
computation_device = device
|
| 1171 |
+
|
| 1172 |
+
out_T = (T + 3) // 4
|
| 1173 |
+
weight = torch.zeros((1, 1, out_T, H // self.upsampling_factor, W // self.upsampling_factor), dtype=video.dtype, device=data_device)
|
| 1174 |
+
values = torch.zeros((1, self.z_dim, out_T, H // self.upsampling_factor, W // self.upsampling_factor), dtype=video.dtype, device=data_device)
|
| 1175 |
+
|
| 1176 |
+
for h, h_, w, w_ in tqdm(tasks, desc="VAE encoding"):
|
| 1177 |
+
hidden_states_batch = video[:, :, :, h:h_, w:w_].to(computation_device)
|
| 1178 |
+
hidden_states_batch = self.model.encode(hidden_states_batch, self.scale).to(data_device)
|
| 1179 |
+
|
| 1180 |
+
mask = self.build_mask(
|
| 1181 |
+
hidden_states_batch,
|
| 1182 |
+
is_bound=(h==0, h_>=H, w==0, w_>=W),
|
| 1183 |
+
border_width=((size_h - stride_h) // self.upsampling_factor, (size_w - stride_w) // self.upsampling_factor)
|
| 1184 |
+
).to(dtype=video.dtype, device=data_device)
|
| 1185 |
+
|
| 1186 |
+
target_h = h // self.upsampling_factor
|
| 1187 |
+
target_w = w // self.upsampling_factor
|
| 1188 |
+
values[
|
| 1189 |
+
:,
|
| 1190 |
+
:,
|
| 1191 |
+
:,
|
| 1192 |
+
target_h:target_h + hidden_states_batch.shape[3],
|
| 1193 |
+
target_w:target_w + hidden_states_batch.shape[4],
|
| 1194 |
+
] += hidden_states_batch * mask
|
| 1195 |
+
weight[
|
| 1196 |
+
:,
|
| 1197 |
+
:,
|
| 1198 |
+
:,
|
| 1199 |
+
target_h: target_h + hidden_states_batch.shape[3],
|
| 1200 |
+
target_w: target_w + hidden_states_batch.shape[4],
|
| 1201 |
+
] += mask
|
| 1202 |
+
values = values / weight
|
| 1203 |
+
return values
|
| 1204 |
+
|
| 1205 |
+
|
| 1206 |
+
def single_encode(self, video, device):
|
| 1207 |
+
video = video.to(device)
|
| 1208 |
+
x = self.model.encode(video, self.scale)
|
| 1209 |
+
return x
|
| 1210 |
+
|
| 1211 |
+
|
| 1212 |
+
def single_decode(self, hidden_state, device):
|
| 1213 |
+
hidden_state = hidden_state.to(device)
|
| 1214 |
+
video = self.model.decode(hidden_state, self.scale)
|
| 1215 |
+
return video.clamp_(-1, 1)
|
| 1216 |
+
|
| 1217 |
+
|
| 1218 |
+
def encode(self, videos, device, tiled=False, tile_size=(34, 34), tile_stride=(18, 16)):
|
| 1219 |
+
# videos = [video.to("cpu") for video in videos]
|
| 1220 |
+
hidden_states = []
|
| 1221 |
+
for video in videos:
|
| 1222 |
+
video = video.unsqueeze(0)
|
| 1223 |
+
if tiled:
|
| 1224 |
+
raise NotImplementedError("Tiled encoding is not allowed yet.")
|
| 1225 |
+
tile_size = (tile_size[0] * self.upsampling_factor, tile_size[1] * self.upsampling_factor)
|
| 1226 |
+
tile_stride = (tile_stride[0] * self.upsampling_factor, tile_stride[1] * self.upsampling_factor)
|
| 1227 |
+
hidden_state = self.tiled_encode(video, device, tile_size, tile_stride)
|
| 1228 |
+
else:
|
| 1229 |
+
hidden_state = self.single_encode(video, device)
|
| 1230 |
+
hidden_state = hidden_state.squeeze(0)
|
| 1231 |
+
hidden_states.append(hidden_state)
|
| 1232 |
+
hidden_states = torch.stack(hidden_states)
|
| 1233 |
+
return hidden_states
|
| 1234 |
+
|
| 1235 |
+
|
| 1236 |
+
def decode(self, hidden_states, device, tiled=False, tile_size=(34, 34), tile_stride=(18, 16)):
|
| 1237 |
+
hidden_states = [hidden_state.to("cpu") for hidden_state in hidden_states]
|
| 1238 |
+
videos = []
|
| 1239 |
+
for hidden_state in hidden_states:
|
| 1240 |
+
hidden_state = hidden_state.unsqueeze(0)
|
| 1241 |
+
if tiled:
|
| 1242 |
+
video = self.tiled_decode(hidden_state, device, tile_size, tile_stride)
|
| 1243 |
+
else:
|
| 1244 |
+
video = self.single_decode(hidden_state, device)
|
| 1245 |
+
video = video.squeeze(0)
|
| 1246 |
+
videos.append(video)
|
| 1247 |
+
videos = torch.stack(videos)
|
| 1248 |
+
return videos
|
| 1249 |
+
|
| 1250 |
+
|
| 1251 |
+
@staticmethod
|
| 1252 |
+
def state_dict_converter():
|
| 1253 |
+
return WanVideoVAEStateDictConverter()
|
| 1254 |
+
|
| 1255 |
+
|
| 1256 |
+
class WanVideoVAEStateDictConverter:
|
| 1257 |
+
|
| 1258 |
+
def __init__(self):
|
| 1259 |
+
pass
|
| 1260 |
+
|
| 1261 |
+
def from_civitai(self, state_dict):
|
| 1262 |
+
state_dict_ = {}
|
| 1263 |
+
if 'model_state' in state_dict:
|
| 1264 |
+
state_dict = state_dict['model_state']
|
| 1265 |
+
for name in state_dict:
|
| 1266 |
+
state_dict_['model.' + name] = state_dict[name]
|
| 1267 |
+
return state_dict_
|
| 1268 |
+
|
| 1269 |
+
|
| 1270 |
+
class VideoVAE38_(VideoVAE_):
|
| 1271 |
+
|
| 1272 |
+
def __init__(self,
|
| 1273 |
+
dim=160,
|
| 1274 |
+
z_dim=48,
|
| 1275 |
+
dec_dim=256,
|
| 1276 |
+
dim_mult=[1, 2, 4, 4],
|
| 1277 |
+
num_res_blocks=2,
|
| 1278 |
+
attn_scales=[],
|
| 1279 |
+
temperal_downsample=[False, True, True],
|
| 1280 |
+
dropout=0.0):
|
| 1281 |
+
super(VideoVAE_, self).__init__()
|
| 1282 |
+
self.dim = dim
|
| 1283 |
+
self.z_dim = z_dim
|
| 1284 |
+
self.dim_mult = dim_mult
|
| 1285 |
+
self.num_res_blocks = num_res_blocks
|
| 1286 |
+
self.attn_scales = attn_scales
|
| 1287 |
+
self.temperal_downsample = temperal_downsample
|
| 1288 |
+
self.temperal_upsample = temperal_downsample[::-1]
|
| 1289 |
+
|
| 1290 |
+
# modules
|
| 1291 |
+
self.encoder = Encoder3d_38(dim, z_dim * 2, dim_mult, num_res_blocks,
|
| 1292 |
+
attn_scales, self.temperal_downsample, dropout)
|
| 1293 |
+
self.conv1 = CausalConv3d(z_dim * 2, z_dim * 2, 1)
|
| 1294 |
+
self.conv2 = CausalConv3d(z_dim, z_dim, 1)
|
| 1295 |
+
self.decoder = Decoder3d_38(dec_dim, z_dim, dim_mult, num_res_blocks,
|
| 1296 |
+
attn_scales, self.temperal_upsample, dropout)
|
| 1297 |
+
|
| 1298 |
+
|
| 1299 |
+
def encode(self, x, scale):
|
| 1300 |
+
self.clear_cache()
|
| 1301 |
+
x = patchify(x, patch_size=2)
|
| 1302 |
+
t = x.shape[2]
|
| 1303 |
+
iter_ = 1 + (t - 1) // 4
|
| 1304 |
+
for i in range(iter_):
|
| 1305 |
+
self._enc_conv_idx = [0]
|
| 1306 |
+
if i == 0:
|
| 1307 |
+
out, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, :1, :, :],
|
| 1308 |
+
feat_cache=self._enc_feat_map,
|
| 1309 |
+
feat_idx=self._enc_conv_idx)
|
| 1310 |
+
else:
|
| 1311 |
+
out_, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, 1 + 4 * (i - 1):1 + 4 * i, :, :],
|
| 1312 |
+
feat_cache=self._enc_feat_map,
|
| 1313 |
+
feat_idx=self._enc_conv_idx)
|
| 1314 |
+
out = torch.cat([out, out_], 2)
|
| 1315 |
+
mu, log_var = self.conv1(out).chunk(2, dim=1)
|
| 1316 |
+
if isinstance(scale[0], torch.Tensor):
|
| 1317 |
+
scale = [s.to(dtype=mu.dtype, device=mu.device) for s in scale]
|
| 1318 |
+
mu = (mu - scale[0].view(1, self.z_dim, 1, 1, 1)) * scale[1].view(
|
| 1319 |
+
1, self.z_dim, 1, 1, 1)
|
| 1320 |
+
else:
|
| 1321 |
+
scale = scale.to(dtype=mu.dtype, device=mu.device)
|
| 1322 |
+
mu = (mu - scale[0]) * scale[1]
|
| 1323 |
+
self.clear_cache()
|
| 1324 |
+
return mu
|
| 1325 |
+
|
| 1326 |
+
|
| 1327 |
+
def decode(self, z, scale):
|
| 1328 |
+
self.clear_cache()
|
| 1329 |
+
if isinstance(scale[0], torch.Tensor):
|
| 1330 |
+
scale = [s.to(dtype=z.dtype, device=z.device) for s in scale]
|
| 1331 |
+
z = z / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view(
|
| 1332 |
+
1, self.z_dim, 1, 1, 1)
|
| 1333 |
+
else:
|
| 1334 |
+
scale = scale.to(dtype=z.dtype, device=z.device)
|
| 1335 |
+
z = z / scale[1] + scale[0]
|
| 1336 |
+
iter_ = z.shape[2]
|
| 1337 |
+
x = self.conv2(z)
|
| 1338 |
+
for i in range(iter_):
|
| 1339 |
+
self._conv_idx = [0]
|
| 1340 |
+
if i == 0:
|
| 1341 |
+
out, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :],
|
| 1342 |
+
feat_cache=self._feat_map,
|
| 1343 |
+
feat_idx=self._conv_idx,
|
| 1344 |
+
first_chunk=True)
|
| 1345 |
+
else:
|
| 1346 |
+
out_, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :],
|
| 1347 |
+
feat_cache=self._feat_map,
|
| 1348 |
+
feat_idx=self._conv_idx)
|
| 1349 |
+
out = torch.cat([out, out_], 2)
|
| 1350 |
+
out = unpatchify(out, patch_size=2)
|
| 1351 |
+
self.clear_cache()
|
| 1352 |
+
return out
|
| 1353 |
+
|
| 1354 |
+
|
| 1355 |
+
class WanVideoVAE38(WanVideoVAE):
|
| 1356 |
+
|
| 1357 |
+
def __init__(self, z_dim=48, dim=160):
|
| 1358 |
+
super(WanVideoVAE, self).__init__()
|
| 1359 |
+
|
| 1360 |
+
mean = [
|
| 1361 |
+
-0.2289, -0.0052, -0.1323, -0.2339, -0.2799, 0.0174, 0.1838, 0.1557,
|
| 1362 |
+
-0.1382, 0.0542, 0.2813, 0.0891, 0.1570, -0.0098, 0.0375, -0.1825,
|
| 1363 |
+
-0.2246, -0.1207, -0.0698, 0.5109, 0.2665, -0.2108, -0.2158, 0.2502,
|
| 1364 |
+
-0.2055, -0.0322, 0.1109, 0.1567, -0.0729, 0.0899, -0.2799, -0.1230,
|
| 1365 |
+
-0.0313, -0.1649, 0.0117, 0.0723, -0.2839, -0.2083, -0.0520, 0.3748,
|
| 1366 |
+
0.0152, 0.1957, 0.1433, -0.2944, 0.3573, -0.0548, -0.1681, -0.0667
|
| 1367 |
+
]
|
| 1368 |
+
std = [
|
| 1369 |
+
0.4765, 1.0364, 0.4514, 1.1677, 0.5313, 0.4990, 0.4818, 0.5013,
|
| 1370 |
+
0.8158, 1.0344, 0.5894, 1.0901, 0.6885, 0.6165, 0.8454, 0.4978,
|
| 1371 |
+
0.5759, 0.3523, 0.7135, 0.6804, 0.5833, 1.4146, 0.8986, 0.5659,
|
| 1372 |
+
0.7069, 0.5338, 0.4889, 0.4917, 0.4069, 0.4999, 0.6866, 0.4093,
|
| 1373 |
+
0.5709, 0.6065, 0.6415, 0.4944, 0.5726, 1.2042, 0.5458, 1.6887,
|
| 1374 |
+
0.3971, 1.0600, 0.3943, 0.5537, 0.5444, 0.4089, 0.7468, 0.7744
|
| 1375 |
+
]
|
| 1376 |
+
self.mean = torch.tensor(mean)
|
| 1377 |
+
self.std = torch.tensor(std)
|
| 1378 |
+
self.scale = [self.mean, 1.0 / self.std]
|
| 1379 |
+
|
| 1380 |
+
# init model
|
| 1381 |
+
self.model = VideoVAE38_(z_dim=z_dim, dim=dim).eval().requires_grad_(False)
|
| 1382 |
+
self.upsampling_factor = 16
|
| 1383 |
+
self.temporal_downsample_factor = 4
|
| 1384 |
+
self.z_dim = z_dim
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/runtime.py
ADDED
|
@@ -0,0 +1,436 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import os
|
| 3 |
+
import inspect
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from hydra.utils import instantiate
|
| 8 |
+
from omegaconf import DictConfig
|
| 9 |
+
from PIL import Image
|
| 10 |
+
import numpy as np
|
| 11 |
+
from einops import repeat
|
| 12 |
+
from omegaconf import OmegaConf
|
| 13 |
+
|
| 14 |
+
from .trainer import Wan22Trainer
|
| 15 |
+
from .utils.logging_config import get_logger, setup_logging
|
| 16 |
+
from .utils.video_io import save_mp4
|
| 17 |
+
from .utils import misc
|
| 18 |
+
|
| 19 |
+
logger = get_logger(__name__)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _normalize_mixed_precision(mixed_precision: str) -> str:
|
| 23 |
+
if not isinstance(mixed_precision, str):
|
| 24 |
+
raise ValueError(f"`mixed_precision` must be str, got {type(mixed_precision)}")
|
| 25 |
+
key = mixed_precision.strip().lower()
|
| 26 |
+
if key not in {"no", "fp16", "bf16"}:
|
| 27 |
+
raise ValueError(
|
| 28 |
+
f"Unsupported mixed_precision: {mixed_precision}. "
|
| 29 |
+
"Expected one of: ['no', 'fp16', 'bf16']."
|
| 30 |
+
)
|
| 31 |
+
return key
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _mixed_precision_to_model_dtype(mixed_precision: str) -> torch.dtype:
|
| 35 |
+
precision = _normalize_mixed_precision(mixed_precision)
|
| 36 |
+
if precision == "no":
|
| 37 |
+
return torch.float32
|
| 38 |
+
if precision == "fp16":
|
| 39 |
+
return torch.float16
|
| 40 |
+
return torch.bfloat16
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def create_wan22_model(
|
| 44 |
+
model_id: str,
|
| 45 |
+
tokenizer_model_id: str,
|
| 46 |
+
dit_config,
|
| 47 |
+
tokenizer_max_len: int = 512,
|
| 48 |
+
train_shift: float = 5.0,
|
| 49 |
+
infer_shift: float = 5.0,
|
| 50 |
+
num_train_timesteps: int = 1000,
|
| 51 |
+
redirect_common_files: bool = True,
|
| 52 |
+
model_dtype: torch.dtype = torch.bfloat16,
|
| 53 |
+
device: str = "cuda",
|
| 54 |
+
):
|
| 55 |
+
from .models.wan22.wan22 import Wan22Core
|
| 56 |
+
|
| 57 |
+
if isinstance(dit_config, DictConfig):
|
| 58 |
+
dit_config = OmegaConf.to_container(dit_config, resolve=True)
|
| 59 |
+
if not isinstance(dit_config, dict):
|
| 60 |
+
raise ValueError(f"`dit_config` must resolve to a dict, got {type(dit_config)}")
|
| 61 |
+
|
| 62 |
+
return Wan22Core.from_wan22_pretrained(
|
| 63 |
+
device=device,
|
| 64 |
+
torch_dtype=model_dtype,
|
| 65 |
+
model_id=model_id,
|
| 66 |
+
tokenizer_model_id=tokenizer_model_id,
|
| 67 |
+
tokenizer_max_len=int(tokenizer_max_len),
|
| 68 |
+
redirect_common_files=bool(redirect_common_files),
|
| 69 |
+
dit_config=dit_config,
|
| 70 |
+
train_shift=float(train_shift),
|
| 71 |
+
infer_shift=float(infer_shift),
|
| 72 |
+
num_train_timesteps=int(num_train_timesteps),
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def create_fastwam(
|
| 77 |
+
model_id: str,
|
| 78 |
+
tokenizer_model_id: str,
|
| 79 |
+
video_dit_config,
|
| 80 |
+
tokenizer_max_len: int = 512,
|
| 81 |
+
load_text_encoder: bool = True,
|
| 82 |
+
proprio_dim: int | None = None,
|
| 83 |
+
action_dit_config=None,
|
| 84 |
+
action_dit_pretrained_path: str | None = None,
|
| 85 |
+
skip_dit_load_from_pretrain: bool = False,
|
| 86 |
+
video_scheduler=None,
|
| 87 |
+
action_scheduler=None,
|
| 88 |
+
loss=None,
|
| 89 |
+
mot_checkpoint_mixed_attn: bool = True,
|
| 90 |
+
redirect_common_files: bool = True,
|
| 91 |
+
model_dtype: torch.dtype = torch.bfloat16,
|
| 92 |
+
device: str = "cuda",
|
| 93 |
+
):
|
| 94 |
+
from .models.wan22.fastwam import FastWAM
|
| 95 |
+
|
| 96 |
+
if isinstance(video_dit_config, DictConfig):
|
| 97 |
+
video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True)
|
| 98 |
+
if not isinstance(video_dit_config, dict):
|
| 99 |
+
raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}")
|
| 100 |
+
|
| 101 |
+
if isinstance(action_dit_config, DictConfig):
|
| 102 |
+
action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True)
|
| 103 |
+
if action_dit_config is None:
|
| 104 |
+
action_dit_config = {}
|
| 105 |
+
if not isinstance(action_dit_config, dict):
|
| 106 |
+
raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}")
|
| 107 |
+
|
| 108 |
+
if isinstance(video_scheduler, DictConfig):
|
| 109 |
+
video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True)
|
| 110 |
+
if video_scheduler is None:
|
| 111 |
+
video_scheduler = {}
|
| 112 |
+
if not isinstance(video_scheduler, dict):
|
| 113 |
+
raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}")
|
| 114 |
+
|
| 115 |
+
if isinstance(action_scheduler, DictConfig):
|
| 116 |
+
action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True)
|
| 117 |
+
if action_scheduler is None:
|
| 118 |
+
raise ValueError("`action_scheduler` is required for FastWAM.")
|
| 119 |
+
if not isinstance(action_scheduler, dict):
|
| 120 |
+
raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}")
|
| 121 |
+
required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"}
|
| 122 |
+
missing_keys = required_action_scheduler_keys - set(action_scheduler.keys())
|
| 123 |
+
if missing_keys:
|
| 124 |
+
raise ValueError(
|
| 125 |
+
f"`action_scheduler` missing required keys: {sorted(missing_keys)}. "
|
| 126 |
+
"Expected keys: train_shift, infer_shift, num_train_timesteps."
|
| 127 |
+
)
|
| 128 |
+
|
| 129 |
+
if isinstance(loss, DictConfig):
|
| 130 |
+
loss = OmegaConf.to_container(loss, resolve=True)
|
| 131 |
+
if loss is None:
|
| 132 |
+
loss = {}
|
| 133 |
+
if not isinstance(loss, dict):
|
| 134 |
+
raise ValueError(f"`loss` must be dict-like, got {type(loss)}")
|
| 135 |
+
|
| 136 |
+
return FastWAM.from_wan22_pretrained(
|
| 137 |
+
device=device,
|
| 138 |
+
torch_dtype=model_dtype,
|
| 139 |
+
model_id=model_id,
|
| 140 |
+
tokenizer_model_id=tokenizer_model_id,
|
| 141 |
+
tokenizer_max_len=int(tokenizer_max_len),
|
| 142 |
+
load_text_encoder=bool(load_text_encoder),
|
| 143 |
+
proprio_dim=(None if proprio_dim is None else int(proprio_dim)),
|
| 144 |
+
redirect_common_files=bool(redirect_common_files),
|
| 145 |
+
video_dit_config=video_dit_config,
|
| 146 |
+
action_dit_config=action_dit_config,
|
| 147 |
+
action_dit_pretrained_path=action_dit_pretrained_path,
|
| 148 |
+
skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain),
|
| 149 |
+
mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn),
|
| 150 |
+
video_train_shift=float(video_scheduler.get("train_shift", 5.0)),
|
| 151 |
+
video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)),
|
| 152 |
+
video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)),
|
| 153 |
+
action_train_shift=float(action_scheduler["train_shift"]),
|
| 154 |
+
action_infer_shift=float(action_scheduler["infer_shift"]),
|
| 155 |
+
action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]),
|
| 156 |
+
loss_lambda_video=float(loss.get("lambda_video", 1.0)),
|
| 157 |
+
loss_lambda_action=float(loss.get("lambda_action", 1.0)),
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def create_fastwam_joint(
|
| 162 |
+
model_id: str,
|
| 163 |
+
tokenizer_model_id: str,
|
| 164 |
+
video_dit_config,
|
| 165 |
+
tokenizer_max_len: int = 512,
|
| 166 |
+
load_text_encoder: bool = True,
|
| 167 |
+
proprio_dim: int | None = None,
|
| 168 |
+
action_dit_config=None,
|
| 169 |
+
action_dit_pretrained_path: str | None = None,
|
| 170 |
+
skip_dit_load_from_pretrain: bool = False,
|
| 171 |
+
video_scheduler=None,
|
| 172 |
+
action_scheduler=None,
|
| 173 |
+
loss=None,
|
| 174 |
+
mot_checkpoint_mixed_attn: bool = True,
|
| 175 |
+
redirect_common_files: bool = True,
|
| 176 |
+
model_dtype: torch.dtype = torch.bfloat16,
|
| 177 |
+
device: str = "cuda",
|
| 178 |
+
):
|
| 179 |
+
from .models.wan22.fastwam_joint import FastWAMJoint
|
| 180 |
+
|
| 181 |
+
if isinstance(video_dit_config, DictConfig):
|
| 182 |
+
video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True)
|
| 183 |
+
if not isinstance(video_dit_config, dict):
|
| 184 |
+
raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}")
|
| 185 |
+
|
| 186 |
+
if isinstance(action_dit_config, DictConfig):
|
| 187 |
+
action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True)
|
| 188 |
+
if action_dit_config is None:
|
| 189 |
+
action_dit_config = {}
|
| 190 |
+
if not isinstance(action_dit_config, dict):
|
| 191 |
+
raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}")
|
| 192 |
+
|
| 193 |
+
if isinstance(video_scheduler, DictConfig):
|
| 194 |
+
video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True)
|
| 195 |
+
if video_scheduler is None:
|
| 196 |
+
video_scheduler = {}
|
| 197 |
+
if not isinstance(video_scheduler, dict):
|
| 198 |
+
raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}")
|
| 199 |
+
|
| 200 |
+
if isinstance(action_scheduler, DictConfig):
|
| 201 |
+
action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True)
|
| 202 |
+
if action_scheduler is None:
|
| 203 |
+
raise ValueError("`action_scheduler` is required for FastWAM.")
|
| 204 |
+
if not isinstance(action_scheduler, dict):
|
| 205 |
+
raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}")
|
| 206 |
+
required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"}
|
| 207 |
+
missing_keys = required_action_scheduler_keys - set(action_scheduler.keys())
|
| 208 |
+
if missing_keys:
|
| 209 |
+
raise ValueError(
|
| 210 |
+
f"`action_scheduler` missing required keys: {sorted(missing_keys)}. "
|
| 211 |
+
"Expected keys: train_shift, infer_shift, num_train_timesteps."
|
| 212 |
+
)
|
| 213 |
+
|
| 214 |
+
if isinstance(loss, DictConfig):
|
| 215 |
+
loss = OmegaConf.to_container(loss, resolve=True)
|
| 216 |
+
if loss is None:
|
| 217 |
+
loss = {}
|
| 218 |
+
if not isinstance(loss, dict):
|
| 219 |
+
raise ValueError(f"`loss` must be dict-like, got {type(loss)}")
|
| 220 |
+
|
| 221 |
+
return FastWAMJoint.from_wan22_pretrained(
|
| 222 |
+
device=device,
|
| 223 |
+
torch_dtype=model_dtype,
|
| 224 |
+
model_id=model_id,
|
| 225 |
+
tokenizer_model_id=tokenizer_model_id,
|
| 226 |
+
tokenizer_max_len=int(tokenizer_max_len),
|
| 227 |
+
load_text_encoder=bool(load_text_encoder),
|
| 228 |
+
proprio_dim=(None if proprio_dim is None else int(proprio_dim)),
|
| 229 |
+
redirect_common_files=bool(redirect_common_files),
|
| 230 |
+
video_dit_config=video_dit_config,
|
| 231 |
+
action_dit_config=action_dit_config,
|
| 232 |
+
action_dit_pretrained_path=action_dit_pretrained_path,
|
| 233 |
+
skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain),
|
| 234 |
+
mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn),
|
| 235 |
+
video_train_shift=float(video_scheduler.get("train_shift", 5.0)),
|
| 236 |
+
video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)),
|
| 237 |
+
video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)),
|
| 238 |
+
action_train_shift=float(action_scheduler["train_shift"]),
|
| 239 |
+
action_infer_shift=float(action_scheduler["infer_shift"]),
|
| 240 |
+
action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]),
|
| 241 |
+
loss_lambda_video=float(loss.get("lambda_video", 1.0)),
|
| 242 |
+
loss_lambda_action=float(loss.get("lambda_action", 1.0)),
|
| 243 |
+
)
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def create_fastwam_idm(
|
| 247 |
+
model_id: str,
|
| 248 |
+
tokenizer_model_id: str,
|
| 249 |
+
video_dit_config,
|
| 250 |
+
tokenizer_max_len: int = 512,
|
| 251 |
+
load_text_encoder: bool = True,
|
| 252 |
+
proprio_dim: int | None = None,
|
| 253 |
+
action_dit_config=None,
|
| 254 |
+
action_dit_pretrained_path: str | None = None,
|
| 255 |
+
skip_dit_load_from_pretrain: bool = False,
|
| 256 |
+
video_scheduler=None,
|
| 257 |
+
action_scheduler=None,
|
| 258 |
+
loss=None,
|
| 259 |
+
mot_checkpoint_mixed_attn: bool = True,
|
| 260 |
+
redirect_common_files: bool = True,
|
| 261 |
+
model_dtype: torch.dtype = torch.bfloat16,
|
| 262 |
+
device: str = "cuda",
|
| 263 |
+
):
|
| 264 |
+
from .models.wan22.fastwam_idm import (
|
| 265 |
+
FastWAMIDM,
|
| 266 |
+
)
|
| 267 |
+
|
| 268 |
+
if isinstance(video_dit_config, DictConfig):
|
| 269 |
+
video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True)
|
| 270 |
+
if not isinstance(video_dit_config, dict):
|
| 271 |
+
raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}")
|
| 272 |
+
|
| 273 |
+
if isinstance(action_dit_config, DictConfig):
|
| 274 |
+
action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True)
|
| 275 |
+
if action_dit_config is None:
|
| 276 |
+
action_dit_config = {}
|
| 277 |
+
if not isinstance(action_dit_config, dict):
|
| 278 |
+
raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}")
|
| 279 |
+
|
| 280 |
+
if isinstance(video_scheduler, DictConfig):
|
| 281 |
+
video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True)
|
| 282 |
+
if video_scheduler is None:
|
| 283 |
+
video_scheduler = {}
|
| 284 |
+
if not isinstance(video_scheduler, dict):
|
| 285 |
+
raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}")
|
| 286 |
+
|
| 287 |
+
if isinstance(action_scheduler, DictConfig):
|
| 288 |
+
action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True)
|
| 289 |
+
if action_scheduler is None:
|
| 290 |
+
raise ValueError("`action_scheduler` is required for FastWAM.")
|
| 291 |
+
if not isinstance(action_scheduler, dict):
|
| 292 |
+
raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}")
|
| 293 |
+
required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"}
|
| 294 |
+
missing_keys = required_action_scheduler_keys - set(action_scheduler.keys())
|
| 295 |
+
if missing_keys:
|
| 296 |
+
raise ValueError(
|
| 297 |
+
f"`action_scheduler` missing required keys: {sorted(missing_keys)}. "
|
| 298 |
+
"Expected keys: train_shift, infer_shift, num_train_timesteps."
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
if isinstance(loss, DictConfig):
|
| 302 |
+
loss = OmegaConf.to_container(loss, resolve=True)
|
| 303 |
+
if loss is None:
|
| 304 |
+
loss = {}
|
| 305 |
+
if not isinstance(loss, dict):
|
| 306 |
+
raise ValueError(f"`loss` must be dict-like, got {type(loss)}")
|
| 307 |
+
|
| 308 |
+
return FastWAMIDM.from_wan22_pretrained(
|
| 309 |
+
device=device,
|
| 310 |
+
torch_dtype=model_dtype,
|
| 311 |
+
model_id=model_id,
|
| 312 |
+
tokenizer_model_id=tokenizer_model_id,
|
| 313 |
+
tokenizer_max_len=int(tokenizer_max_len),
|
| 314 |
+
load_text_encoder=bool(load_text_encoder),
|
| 315 |
+
proprio_dim=(None if proprio_dim is None else int(proprio_dim)),
|
| 316 |
+
redirect_common_files=bool(redirect_common_files),
|
| 317 |
+
video_dit_config=video_dit_config,
|
| 318 |
+
action_dit_config=action_dit_config,
|
| 319 |
+
action_dit_pretrained_path=action_dit_pretrained_path,
|
| 320 |
+
skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain),
|
| 321 |
+
mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn),
|
| 322 |
+
video_train_shift=float(video_scheduler.get("train_shift", 5.0)),
|
| 323 |
+
video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)),
|
| 324 |
+
video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)),
|
| 325 |
+
action_train_shift=float(action_scheduler["train_shift"]),
|
| 326 |
+
action_infer_shift=float(action_scheduler["infer_shift"]),
|
| 327 |
+
action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]),
|
| 328 |
+
loss_lambda_video=float(loss.get("lambda_video", 1.0)),
|
| 329 |
+
loss_lambda_action=float(loss.get("lambda_action", 1.0)),
|
| 330 |
+
)
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def build_datasets(data_cfg: DictConfig):
|
| 334 |
+
train_ds = instantiate(data_cfg.train)
|
| 335 |
+
if data_cfg.get("val") is None:
|
| 336 |
+
val_ds = train_ds
|
| 337 |
+
else:
|
| 338 |
+
train_stats_path = data_cfg.train.get("pretrained_norm_stats")
|
| 339 |
+
default_stats_path = os.path.join(misc.get_work_dir(), "dataset_stats.json")
|
| 340 |
+
val_stats_path = data_cfg.val.get("pretrained_norm_stats")
|
| 341 |
+
pretrained_norm_stats = val_stats_path or train_stats_path or default_stats_path
|
| 342 |
+
logger.info("Building val dataset with pretrained_norm_stats: %s", pretrained_norm_stats)
|
| 343 |
+
val_ds = instantiate(data_cfg.val, pretrained_norm_stats=pretrained_norm_stats)
|
| 344 |
+
return train_ds, val_ds
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
def _resolve_train_device() -> str:
|
| 348 |
+
if not torch.cuda.is_available():
|
| 349 |
+
return "cpu"
|
| 350 |
+
device_count = torch.cuda.device_count()
|
| 351 |
+
if device_count <= 1:
|
| 352 |
+
return "cuda:0"
|
| 353 |
+
local_rank = int(os.environ.get("LOCAL_RANK", "0"))
|
| 354 |
+
if local_rank < 0 or local_rank >= device_count:
|
| 355 |
+
return "cuda:0"
|
| 356 |
+
return f"cuda:{local_rank}"
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
def run_training(cfg: DictConfig):
|
| 360 |
+
setup_logging(
|
| 361 |
+
log_level=logging.INFO,
|
| 362 |
+
is_main_process=torch.distributed.get_rank() == 0 if torch.distributed.is_initialized() else True,
|
| 363 |
+
)
|
| 364 |
+
misc.register_work_dir(cfg.output_dir)
|
| 365 |
+
config_payload = OmegaConf.to_container(cfg, resolve=True)
|
| 366 |
+
with open(Path(cfg.output_dir) / "config.yaml", "w") as f:
|
| 367 |
+
OmegaConf.save(config_payload, f)
|
| 368 |
+
|
| 369 |
+
model_device = _resolve_train_device()
|
| 370 |
+
mixed_precision = _normalize_mixed_precision(cfg.mixed_precision)
|
| 371 |
+
model_dtype = _mixed_precision_to_model_dtype(mixed_precision)
|
| 372 |
+
model = instantiate(cfg.model, model_dtype=model_dtype, device=model_device)
|
| 373 |
+
train_ds, val_ds = build_datasets(cfg.data)
|
| 374 |
+
|
| 375 |
+
trainer = Wan22Trainer(
|
| 376 |
+
cfg=cfg,
|
| 377 |
+
model=model,
|
| 378 |
+
train_dataset=train_ds,
|
| 379 |
+
val_dataset=val_ds,
|
| 380 |
+
)
|
| 381 |
+
trainer.train()
|
| 382 |
+
|
| 383 |
+
def run_inference(cfg: DictConfig):
|
| 384 |
+
setup_logging(log_level=logging.INFO)
|
| 385 |
+
inference_cfg = cfg.inference
|
| 386 |
+
mixed_precision = _normalize_mixed_precision(cfg.mixed_precision)
|
| 387 |
+
model_dtype = _mixed_precision_to_model_dtype(mixed_precision)
|
| 388 |
+
|
| 389 |
+
model = instantiate(cfg.model, model_dtype=model_dtype, device=str(inference_cfg.device))
|
| 390 |
+
checkpoint_path = inference_cfg.get("checkpoint_path")
|
| 391 |
+
if checkpoint_path:
|
| 392 |
+
ckpt = Path(checkpoint_path)
|
| 393 |
+
if ckpt.exists():
|
| 394 |
+
logger.info("Loading finetuned checkpoint: %s", checkpoint_path)
|
| 395 |
+
model.load_checkpoint(checkpoint_path)
|
| 396 |
+
else:
|
| 397 |
+
logger.warning("Checkpoint not found, skipping load: %s", checkpoint_path)
|
| 398 |
+
model.eval()
|
| 399 |
+
|
| 400 |
+
def center_crop_resize(img: Image, width: int, height: int) -> Image.Image:
|
| 401 |
+
src_w, src_h = img.size
|
| 402 |
+
scale = max(width / src_w, height / src_h)
|
| 403 |
+
resized = img.resize((round(src_w * scale), round(src_h * scale)), resample=Image.BILINEAR)
|
| 404 |
+
rw, rh = resized.size
|
| 405 |
+
left = max((rw - width) // 2, 0)
|
| 406 |
+
top = max((rh - height) // 2, 0)
|
| 407 |
+
return resized.crop((left, top, left + width, top + height))
|
| 408 |
+
|
| 409 |
+
input_image = Image.open(str(inference_cfg.input_image_path)).convert("RGB")
|
| 410 |
+
input_image = center_crop_resize(input_image, width=inference_cfg.width, height=inference_cfg.height)
|
| 411 |
+
arr = np.array(input_image, dtype=np.float32)
|
| 412 |
+
x = torch.from_numpy(arr)
|
| 413 |
+
x = x.to(device=model.device, dtype=model.torch_dtype)
|
| 414 |
+
x = x * (2.0 / 255.0) - 1.0
|
| 415 |
+
x = repeat(x, "H W C -> B C H W", B=1)
|
| 416 |
+
output_mp4 = str(inference_cfg.output_mp4)
|
| 417 |
+
|
| 418 |
+
infer_kwargs = {
|
| 419 |
+
"prompt": str(inference_cfg.prompt),
|
| 420 |
+
"negative_prompt": str(inference_cfg.negative_prompt),
|
| 421 |
+
"text_cfg_scale": float(inference_cfg.text_cfg_scale),
|
| 422 |
+
"action_cfg_scale": float(inference_cfg.action_cfg_scale),
|
| 423 |
+
"input_image": x,
|
| 424 |
+
"num_frames": int(inference_cfg.num_frames),
|
| 425 |
+
"num_inference_steps": int(inference_cfg.num_inference_steps),
|
| 426 |
+
"sigma_shift": None if inference_cfg.get("sigma_shift") is None else float(inference_cfg.sigma_shift),
|
| 427 |
+
"seed": int(inference_cfg.seed),
|
| 428 |
+
"rand_device": str(inference_cfg.rand_device),
|
| 429 |
+
"tiled": bool(inference_cfg.tiled),
|
| 430 |
+
}
|
| 431 |
+
|
| 432 |
+
infer_out = model.infer(**infer_kwargs)
|
| 433 |
+
video = infer_out["video"]
|
| 434 |
+
save_mp4(video, output_mp4, fps=15)
|
| 435 |
+
logger.info("Saved inference video to %s", output_mp4)
|
| 436 |
+
return output_mp4
|
spacedreamer_stack_cups_clean40_30ep/runtime/source/model_old/src/fastwam/trainer.py
ADDED
|
@@ -0,0 +1,791 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import json
|
| 3 |
+
import inspect
|
| 4 |
+
import os
|
| 5 |
+
import re
|
| 6 |
+
from math import ceil
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
import time
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import torch
|
| 12 |
+
from accelerate import Accelerator
|
| 13 |
+
from omegaconf import DictConfig
|
| 14 |
+
from PIL import Image
|
| 15 |
+
from torch.optim.lr_scheduler import ConstantLR, CosineAnnealingLR, LinearLR, SequentialLR
|
| 16 |
+
from torch.utils.data import DataLoader
|
| 17 |
+
|
| 18 |
+
from .utils.fs import ensure_dir
|
| 19 |
+
from .utils.logging_config import get_logger, setup_logging
|
| 20 |
+
from .utils.pytorch_utils import set_global_seed
|
| 21 |
+
from .utils.samplers import ResumableEpochSampler
|
| 22 |
+
from .utils.video_io import save_mp4
|
| 23 |
+
from .utils.video_metrics import pil_frames_to_video_tensor, video_psnr, video_ssim
|
| 24 |
+
|
| 25 |
+
logger = get_logger(__name__)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class Wan22Trainer:
|
| 29 |
+
def __init__(self, model, train_dataset, val_dataset=None, *, cfg: DictConfig):
|
| 30 |
+
self.model = model
|
| 31 |
+
self.train_dataset = train_dataset
|
| 32 |
+
self.val_dataset = val_dataset
|
| 33 |
+
self.cfg = cfg
|
| 34 |
+
self.output_dir = str(cfg.output_dir)
|
| 35 |
+
self.learning_rate = float(cfg.learning_rate)
|
| 36 |
+
self.weight_decay = float(cfg.weight_decay)
|
| 37 |
+
self.batch_size = int(cfg.batch_size)
|
| 38 |
+
self.num_workers = int(cfg.num_workers)
|
| 39 |
+
self.num_epochs = int(cfg.num_epochs)
|
| 40 |
+
max_steps = cfg.max_steps
|
| 41 |
+
self.max_steps = int(max_steps) if max_steps is not None else None
|
| 42 |
+
self.log_every = int(cfg.log_every)
|
| 43 |
+
self.save_every = int(cfg.save_every)
|
| 44 |
+
self.eval_every = int(cfg.eval_every)
|
| 45 |
+
self.eval_num_inference_steps = int(cfg.eval_num_inference_steps)
|
| 46 |
+
self.gradient_accumulation_steps = int(cfg.gradient_accumulation_steps)
|
| 47 |
+
self.max_grad_norm = float(cfg.max_grad_norm)
|
| 48 |
+
self.seed = int(cfg.seed)
|
| 49 |
+
|
| 50 |
+
self.resume = cfg.resume
|
| 51 |
+
self.mixed_precision = str(cfg.mixed_precision).strip().lower()
|
| 52 |
+
if self.mixed_precision not in {"no", "fp16", "bf16"}:
|
| 53 |
+
raise ValueError(
|
| 54 |
+
f"Unsupported mixed_precision: {cfg.mixed_precision}. "
|
| 55 |
+
"Expected one of: ['no', 'fp16', 'bf16']."
|
| 56 |
+
)
|
| 57 |
+
self.wandb_enabled = bool(cfg.wandb.enabled)
|
| 58 |
+
|
| 59 |
+
self.accelerator = Accelerator(
|
| 60 |
+
gradient_accumulation_steps=self.gradient_accumulation_steps,
|
| 61 |
+
mixed_precision=self.mixed_precision,
|
| 62 |
+
step_scheduler_with_optimizer=False,
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
logger.info(
|
| 66 |
+
"Accelerate training: distributed_type=%s zero_stage=%s world_size=%d process_index=%d cfg_mixed_precision=%s accelerator_mixed_precision=%s grad_accum=%d grad_clip=%.4f",
|
| 67 |
+
self.accelerator.distributed_type,
|
| 68 |
+
self.accelerator.state.deepspeed_plugin.deepspeed_config.get("zero_optimization", {}).get("stage", "unknown"),
|
| 69 |
+
self.accelerator.num_processes,
|
| 70 |
+
self.accelerator.process_index,
|
| 71 |
+
self.mixed_precision,
|
| 72 |
+
self.accelerator.mixed_precision,
|
| 73 |
+
self.gradient_accumulation_steps,
|
| 74 |
+
self.max_grad_norm,
|
| 75 |
+
)
|
| 76 |
+
logger.info("using accelerator.device=%s", self.accelerator.device)
|
| 77 |
+
worker_init_fn = set_global_seed(self.seed, get_worker_init_fn=True)
|
| 78 |
+
self._assert_dataset_length_consistent(self.train_dataset, "train_dataset")
|
| 79 |
+
if self.val_dataset is not None:
|
| 80 |
+
self._assert_dataset_length_consistent(self.val_dataset, "val_dataset")
|
| 81 |
+
|
| 82 |
+
# Freeze non-trainable modules before optimizer/deepspeed initialization.
|
| 83 |
+
# This keeps DiT (+ optional proprio encoder) as trainable when ZeRO builds optimizer state.
|
| 84 |
+
self._apply_dit_only_train_mode(self.model)
|
| 85 |
+
trainable_params = list(self.model.dit.parameters())
|
| 86 |
+
proprio_encoder = getattr(self.model, "proprio_encoder", None)
|
| 87 |
+
if proprio_encoder is not None:
|
| 88 |
+
trainable_params.extend(list(proprio_encoder.parameters()))
|
| 89 |
+
self.optimizer = torch.optim.AdamW(
|
| 90 |
+
trainable_params,
|
| 91 |
+
lr=self.learning_rate,
|
| 92 |
+
weight_decay=self.weight_decay,
|
| 93 |
+
betas=(0.9, 0.95),
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
self.train_loader = self._build_loader(self.train_dataset, worker_init_fn=worker_init_fn)
|
| 97 |
+
total_train_steps = self._estimate_total_train_steps()
|
| 98 |
+
self.max_steps = total_train_steps
|
| 99 |
+
warmup_steps = int(total_train_steps * 0.05)
|
| 100 |
+
self.scheduler = self._build_scheduler(
|
| 101 |
+
scheduler_type=cfg.lr_scheduler_type,
|
| 102 |
+
total_train_steps=total_train_steps,
|
| 103 |
+
warmup_steps=warmup_steps,
|
| 104 |
+
)
|
| 105 |
+
self.global_step = 0
|
| 106 |
+
self.epoch = 0
|
| 107 |
+
self.batch_in_epoch = 0
|
| 108 |
+
|
| 109 |
+
self.checkpoint_root = os.path.join(self.output_dir, "checkpoints")
|
| 110 |
+
self.weights_dir = os.path.join(self.checkpoint_root, "weights")
|
| 111 |
+
self.state_dir = os.path.join(self.checkpoint_root, "state")
|
| 112 |
+
self.eval_dir = os.path.join(self.output_dir, "eval")
|
| 113 |
+
|
| 114 |
+
ensure_dir(self.output_dir)
|
| 115 |
+
ensure_dir(self.checkpoint_root)
|
| 116 |
+
ensure_dir(self.weights_dir)
|
| 117 |
+
ensure_dir(self.state_dir)
|
| 118 |
+
ensure_dir(self.eval_dir)
|
| 119 |
+
|
| 120 |
+
self.model, self.optimizer, self.train_loader, self.scheduler = self.accelerator.prepare(
|
| 121 |
+
self.model, self.optimizer, self.train_loader, self.scheduler
|
| 122 |
+
)
|
| 123 |
+
self.optimizer.zero_grad(set_to_none=True)
|
| 124 |
+
self.wandb_run = None
|
| 125 |
+
self._init_wandb()
|
| 126 |
+
self._resume_or_load_checkpoint()
|
| 127 |
+
|
| 128 |
+
val_size = len(self.val_dataset) if self.val_dataset is not None else len(self.train_dataset)
|
| 129 |
+
logger.info("Train/val dataset size: %d/%d", len(self.train_dataset), val_size)
|
| 130 |
+
|
| 131 |
+
def _init_wandb(self):
|
| 132 |
+
if not self.wandb_enabled or not self.accelerator.is_main_process:
|
| 133 |
+
return
|
| 134 |
+
try:
|
| 135 |
+
import wandb
|
| 136 |
+
except ImportError as e:
|
| 137 |
+
raise ImportError(
|
| 138 |
+
"wandb logging is enabled in config (`wandb.enabled=true`) but wandb is not installed."
|
| 139 |
+
) from e
|
| 140 |
+
|
| 141 |
+
self.wandb_run = wandb.init(
|
| 142 |
+
entity=self.cfg.wandb.workspace,
|
| 143 |
+
project=self.cfg.wandb.project,
|
| 144 |
+
name=self.cfg.wandb.name,
|
| 145 |
+
group=None if self.cfg.wandb.group in (None, "null", "") else str(self.cfg.wandb.group),
|
| 146 |
+
mode=self.cfg.wandb.mode,
|
| 147 |
+
dir=self.output_dir,
|
| 148 |
+
)
|
| 149 |
+
logger.info(
|
| 150 |
+
"Initialized wandb run: workspace=%s project=%s name=%s",
|
| 151 |
+
self.cfg.wandb.workspace,
|
| 152 |
+
self.cfg.wandb.project,
|
| 153 |
+
self.cfg.wandb.name,
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
def _wandb_log(self, payload: dict):
|
| 157 |
+
if self.wandb_run is None:
|
| 158 |
+
return
|
| 159 |
+
self.wandb_run.log(payload, step=self.global_step)
|
| 160 |
+
|
| 161 |
+
def _finish_wandb(self):
|
| 162 |
+
if self.wandb_run is None:
|
| 163 |
+
return
|
| 164 |
+
self.wandb_run.finish()
|
| 165 |
+
self.wandb_run = None
|
| 166 |
+
|
| 167 |
+
def _build_loader(self, dataset, worker_init_fn=None):
|
| 168 |
+
self.train_sampler = ResumableEpochSampler(
|
| 169 |
+
dataset=dataset,
|
| 170 |
+
seed=self.seed,
|
| 171 |
+
batch_size=self.batch_size,
|
| 172 |
+
num_processes=self.accelerator.num_processes,
|
| 173 |
+
)
|
| 174 |
+
return DataLoader(
|
| 175 |
+
dataset,
|
| 176 |
+
batch_size=self.batch_size,
|
| 177 |
+
shuffle=False,
|
| 178 |
+
sampler=self.train_sampler,
|
| 179 |
+
num_workers=self.num_workers,
|
| 180 |
+
pin_memory=torch.cuda.is_available(),
|
| 181 |
+
worker_init_fn=worker_init_fn,
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
def _assert_dataset_length_consistent(self, dataset, dataset_name: str):
|
| 185 |
+
if not hasattr(dataset, "__len__"):
|
| 186 |
+
raise TypeError(f"`{dataset_name}` must implement __len__ for rank consistency checks.")
|
| 187 |
+
|
| 188 |
+
local_length = len(dataset)
|
| 189 |
+
gathered_lengths = self.accelerator.gather(
|
| 190 |
+
torch.tensor([local_length], device=self.accelerator.device, dtype=torch.int64)
|
| 191 |
+
).reshape(-1)
|
| 192 |
+
if torch.all(gathered_lengths == gathered_lengths[0]):
|
| 193 |
+
return
|
| 194 |
+
|
| 195 |
+
if self.accelerator.is_main_process:
|
| 196 |
+
print(f"[dataset-check] {dataset_name} length mismatch across ranks after initialization:")
|
| 197 |
+
for rank, rank_length in enumerate(gathered_lengths.cpu().tolist()):
|
| 198 |
+
print(f"rank {rank}: {rank_length}")
|
| 199 |
+
self.accelerator.wait_for_everyone()
|
| 200 |
+
raise RuntimeError(
|
| 201 |
+
f"{dataset_name} length mismatch across ranks: {gathered_lengths.cpu().tolist()}"
|
| 202 |
+
)
|
| 203 |
+
|
| 204 |
+
def _estimate_total_train_steps(self) -> int:
|
| 205 |
+
if self.max_steps is not None:
|
| 206 |
+
return max(int(self.max_steps), 1)
|
| 207 |
+
|
| 208 |
+
if not hasattr(self.train_dataset, "__len__"):
|
| 209 |
+
raise TypeError("`train_dataset` must implement __len__ when `max_steps` is None.")
|
| 210 |
+
|
| 211 |
+
num_processes = max(int(self.accelerator.num_processes), 1)
|
| 212 |
+
global_batch_size = max(self.batch_size * num_processes, 1)
|
| 213 |
+
micro_steps_per_epoch = max(ceil(len(self.train_dataset) / global_batch_size), 1)
|
| 214 |
+
opt_steps_per_epoch = max(
|
| 215 |
+
ceil(micro_steps_per_epoch / self.gradient_accumulation_steps),
|
| 216 |
+
1,
|
| 217 |
+
)
|
| 218 |
+
return max(opt_steps_per_epoch * self.num_epochs, 1)
|
| 219 |
+
|
| 220 |
+
def _build_scheduler(self, scheduler_type, total_train_steps: int, warmup_steps: int = 0):
|
| 221 |
+
scheduler_type = str(scheduler_type).strip().lower()
|
| 222 |
+
total_train_steps = max(int(total_train_steps), 1)
|
| 223 |
+
warmup_steps = min(max(int(warmup_steps), 0), total_train_steps - 1)
|
| 224 |
+
|
| 225 |
+
remaining_steps = max(total_train_steps - warmup_steps, 1)
|
| 226 |
+
if scheduler_type == "cosine":
|
| 227 |
+
main_scheduler = CosineAnnealingLR(
|
| 228 |
+
self.optimizer,
|
| 229 |
+
T_max=remaining_steps,
|
| 230 |
+
eta_min=self.learning_rate * 0.01,
|
| 231 |
+
)
|
| 232 |
+
elif scheduler_type == "constant":
|
| 233 |
+
main_scheduler = ConstantLR(self.optimizer, factor=1.0, total_iters=remaining_steps)
|
| 234 |
+
else:
|
| 235 |
+
raise ValueError(
|
| 236 |
+
f"Unsupported lr_scheduler_type: {scheduler_type}. "
|
| 237 |
+
"Expected one of: ['cosine', 'constant']."
|
| 238 |
+
)
|
| 239 |
+
|
| 240 |
+
if warmup_steps <= 0:
|
| 241 |
+
return main_scheduler
|
| 242 |
+
|
| 243 |
+
warmup_scheduler = LinearLR(
|
| 244 |
+
self.optimizer,
|
| 245 |
+
start_factor=1.0 / warmup_steps,
|
| 246 |
+
end_factor=1.0,
|
| 247 |
+
total_iters=warmup_steps,
|
| 248 |
+
)
|
| 249 |
+
return SequentialLR(
|
| 250 |
+
self.optimizer,
|
| 251 |
+
schedulers=[warmup_scheduler, main_scheduler],
|
| 252 |
+
milestones=[warmup_steps],
|
| 253 |
+
)
|
| 254 |
+
|
| 255 |
+
def _estimate_eta(self):
|
| 256 |
+
elapsed = max(time.perf_counter() - self.run_start_time, 1e-6)
|
| 257 |
+
done_steps = max(self.global_step - self.run_start_step, 1)
|
| 258 |
+
steps_per_sec = done_steps / elapsed
|
| 259 |
+
remaining_steps = max(self.max_steps - self.global_step, 0)
|
| 260 |
+
eta_seconds = int(remaining_steps / max(steps_per_sec, 1e-9))
|
| 261 |
+
eta_h, eta_rem = divmod(eta_seconds, 3600)
|
| 262 |
+
eta_m, eta_s = divmod(eta_rem, 60)
|
| 263 |
+
return f"{eta_h:02d}:{eta_m:02d}:{eta_s:02d}", steps_per_sec
|
| 264 |
+
|
| 265 |
+
def _resume_or_load_checkpoint(self):
|
| 266 |
+
resume = self.resume
|
| 267 |
+
if not resume:
|
| 268 |
+
return
|
| 269 |
+
resume_path = Path(str(resume))
|
| 270 |
+
if resume_path.is_dir():
|
| 271 |
+
logger.info("Resuming full training state from directory: %s", resume)
|
| 272 |
+
self.load_training_state(str(resume_path))
|
| 273 |
+
return
|
| 274 |
+
if not resume_path.exists():
|
| 275 |
+
raise FileNotFoundError(f"Resume checkpoint not found: {resume}")
|
| 276 |
+
logger.info("Loading weight checkpoint only: %s", resume)
|
| 277 |
+
self.accelerator.unwrap_model(self.model).load_checkpoint(str(resume_path), optimizer=None)
|
| 278 |
+
logger.warning("Loaded .pt weights only; optimizer/scheduler/step were not restored under ZeRO2.")
|
| 279 |
+
|
| 280 |
+
def _set_dit_only_train_mode(self):
|
| 281 |
+
# Match DiffSynth's freeze_except("dit"): only DiT stays trainable/in-train-mode.
|
| 282 |
+
logger.info("Setting DiT to train mode and freezing other model components.")
|
| 283 |
+
model = self.accelerator.unwrap_model(self.model)
|
| 284 |
+
self._apply_dit_only_train_mode(model)
|
| 285 |
+
|
| 286 |
+
@staticmethod
|
| 287 |
+
def _apply_dit_only_train_mode(model):
|
| 288 |
+
model.eval()
|
| 289 |
+
model.requires_grad_(False)
|
| 290 |
+
model.dit.train()
|
| 291 |
+
model.dit.requires_grad_(True)
|
| 292 |
+
proprio_encoder = getattr(model, "proprio_encoder", None)
|
| 293 |
+
if proprio_encoder is not None:
|
| 294 |
+
proprio_encoder.train()
|
| 295 |
+
proprio_encoder.requires_grad_(True)
|
| 296 |
+
|
| 297 |
+
@staticmethod
|
| 298 |
+
def _to_batched_eval_sample(sample):
|
| 299 |
+
video = sample["video"]
|
| 300 |
+
prompt = sample["prompt"]
|
| 301 |
+
action = sample.get("action", None)
|
| 302 |
+
proprio = sample.get("proprio", None)
|
| 303 |
+
context = sample.get("context", None)
|
| 304 |
+
context_mask = sample.get("context_mask", None)
|
| 305 |
+
|
| 306 |
+
if not isinstance(video, torch.Tensor):
|
| 307 |
+
raise TypeError(
|
| 308 |
+
f"Expected tensor video for evaluation, got {type(video)}. "
|
| 309 |
+
"Evaluation now expects `video` with shape [3,T,H,W] or [B,3,T,H,W]."
|
| 310 |
+
)
|
| 311 |
+
if video.ndim == 4:
|
| 312 |
+
video = video.unsqueeze(0)
|
| 313 |
+
if video.ndim != 5:
|
| 314 |
+
raise ValueError(f"Expected video shape [3,T,H,W] or [B,3,T,H,W], got {tuple(video.shape)}")
|
| 315 |
+
num_video_frames = video.shape[2]
|
| 316 |
+
if num_video_frames <= 1:
|
| 317 |
+
raise ValueError(f"`sample['video']` must have at least 2 frames for action evaluation, got {num_video_frames}")
|
| 318 |
+
|
| 319 |
+
if isinstance(prompt, str):
|
| 320 |
+
prompt = [prompt]
|
| 321 |
+
elif isinstance(prompt, tuple):
|
| 322 |
+
prompt = list(prompt)
|
| 323 |
+
elif not isinstance(prompt, list):
|
| 324 |
+
raise TypeError(f"Expected prompt type str/list[str], got {type(prompt)}")
|
| 325 |
+
if len(prompt) != video.shape[0]:
|
| 326 |
+
raise ValueError(f"Prompt batch mismatch: len(prompt)={len(prompt)} vs video batch={video.shape[0]}")
|
| 327 |
+
|
| 328 |
+
action_horizon = None
|
| 329 |
+
action = None
|
| 330 |
+
if "action" in sample:
|
| 331 |
+
action = sample["action"]
|
| 332 |
+
if not isinstance(action, torch.Tensor):
|
| 333 |
+
raise TypeError(
|
| 334 |
+
f"`sample['action']` must be a torch.Tensor, got {type(action)}"
|
| 335 |
+
)
|
| 336 |
+
if action.ndim == 2:
|
| 337 |
+
action = action.unsqueeze(0)
|
| 338 |
+
if action.ndim != 3:
|
| 339 |
+
raise ValueError(f"`sample['action']` must be 3D [B, T, a_dim], got shape {tuple(action.shape)}")
|
| 340 |
+
if action.shape[1] % (num_video_frames - 1) != 0:
|
| 341 |
+
raise ValueError(f"`sample['action']` temporal dimension must be divisible by video frames-1={num_video_frames - 1}, got {action.shape[1]}")
|
| 342 |
+
action_horizon = int(action.shape[1])
|
| 343 |
+
|
| 344 |
+
proprio = None
|
| 345 |
+
if "proprio" in sample:
|
| 346 |
+
proprio = sample["proprio"]
|
| 347 |
+
if not isinstance(proprio, torch.Tensor):
|
| 348 |
+
raise TypeError(f"`sample['proprio']` must be a torch.Tensor, got {type(proprio)}")
|
| 349 |
+
if proprio.ndim == 2:
|
| 350 |
+
proprio = proprio.unsqueeze(0)
|
| 351 |
+
if proprio.ndim != 3:
|
| 352 |
+
raise ValueError(f"`sample['proprio']` must be 3D [B, T, d], got shape {tuple(proprio.shape)}")
|
| 353 |
+
|
| 354 |
+
if context is not None or context_mask is not None:
|
| 355 |
+
if context is None or context_mask is None:
|
| 356 |
+
raise ValueError("`context` and `context_mask` must both exist in eval sample.")
|
| 357 |
+
if context.ndim == 2:
|
| 358 |
+
context = context.unsqueeze(0)
|
| 359 |
+
if context_mask.ndim == 1:
|
| 360 |
+
context_mask = context_mask.unsqueeze(0)
|
| 361 |
+
if context.ndim != 3 or context_mask.ndim != 2:
|
| 362 |
+
raise ValueError(
|
| 363 |
+
f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}"
|
| 364 |
+
)
|
| 365 |
+
|
| 366 |
+
return {
|
| 367 |
+
"video": video,
|
| 368 |
+
"prompt": prompt,
|
| 369 |
+
"action": action,
|
| 370 |
+
"proprio": proprio,
|
| 371 |
+
"context": context,
|
| 372 |
+
"context_mask": context_mask,
|
| 373 |
+
"action_horizon": action_horizon,
|
| 374 |
+
}
|
| 375 |
+
|
| 376 |
+
@torch.no_grad()
|
| 377 |
+
def evaluate(self):
|
| 378 |
+
if self.val_dataset is None:
|
| 379 |
+
return None
|
| 380 |
+
|
| 381 |
+
model = self.accelerator.unwrap_model(self.model)
|
| 382 |
+
was_dit_training = model.dit.training
|
| 383 |
+
model.eval()
|
| 384 |
+
|
| 385 |
+
# eval_index = (self.global_step + self.accelerator.process_index) % len(self.val_dataset)
|
| 386 |
+
rng = torch.Generator(device="cpu").manual_seed(self.global_step + self.accelerator.process_index)
|
| 387 |
+
eval_index = torch.randint(0, len(self.val_dataset), (1,), generator=rng).item()
|
| 388 |
+
sample = self._to_batched_eval_sample(self.val_dataset[eval_index])
|
| 389 |
+
|
| 390 |
+
# 1. training loss
|
| 391 |
+
with self.accelerator.autocast():
|
| 392 |
+
val_loss, _ = model.training_loss(sample)
|
| 393 |
+
val_loss = val_loss.float().item()
|
| 394 |
+
|
| 395 |
+
prompt = sample["prompt"][0]
|
| 396 |
+
video0 = sample["video"][0] # Tensor [3, T, H, W] in (-1, 1)
|
| 397 |
+
action = sample["action"][0] if "action" in sample and sample["action"] is not None else None
|
| 398 |
+
proprio = sample["proprio"][0, 0] if "proprio" in sample and sample["proprio"] is not None else None # from [1, T, d] to [d]
|
| 399 |
+
input_image = video0[:, 0].unsqueeze(0)
|
| 400 |
+
_, num_frames, _, _ = video0.shape
|
| 401 |
+
|
| 402 |
+
# 2. inference and video saving
|
| 403 |
+
infer_kwargs = {
|
| 404 |
+
"input_image": input_image,
|
| 405 |
+
"num_frames": num_frames,
|
| 406 |
+
"action": action,
|
| 407 |
+
"action_horizon": sample['action_horizon'],
|
| 408 |
+
"proprio": proprio,
|
| 409 |
+
"text_cfg_scale": 1.0,
|
| 410 |
+
"action_cfg_scale": 1.0,
|
| 411 |
+
"num_inference_steps": self.eval_num_inference_steps,
|
| 412 |
+
"seed": 42,
|
| 413 |
+
"tiled": False,
|
| 414 |
+
}
|
| 415 |
+
if sample["context"] is not None:
|
| 416 |
+
infer_kwargs["prompt"] = None
|
| 417 |
+
infer_kwargs["context"] = sample["context"][0]
|
| 418 |
+
infer_kwargs["context_mask"] = sample["context_mask"][0]
|
| 419 |
+
else:
|
| 420 |
+
infer_kwargs["prompt"] = prompt
|
| 421 |
+
|
| 422 |
+
pred = model.infer(
|
| 423 |
+
**infer_kwargs,
|
| 424 |
+
)
|
| 425 |
+
|
| 426 |
+
pred_video = pred["video"]
|
| 427 |
+
pred_action = pred.get("action", None)
|
| 428 |
+
|
| 429 |
+
# 3. inference metrics against GT video
|
| 430 |
+
pred_video_tensor = pil_frames_to_video_tensor(pred_video)
|
| 431 |
+
gt_video_tensor = ((video0.detach().float().cpu().clamp(-1.0, 1.0) + 1.0) * 0.5).contiguous()
|
| 432 |
+
|
| 433 |
+
assert pred_video_tensor.shape == gt_video_tensor.shape, (
|
| 434 |
+
"Eval infer prediction/GT shape mismatch: "
|
| 435 |
+
f"pred={tuple(pred_video_tensor.shape)} vs gt={tuple(gt_video_tensor.shape)}"
|
| 436 |
+
)
|
| 437 |
+
|
| 438 |
+
psnr_rollout_vs_gt = video_psnr(pred=pred_video_tensor, target=gt_video_tensor)
|
| 439 |
+
ssim_rollout_vs_gt = video_ssim(pred=pred_video_tensor, target=gt_video_tensor)
|
| 440 |
+
|
| 441 |
+
action_l1 = None
|
| 442 |
+
action_l2 = None
|
| 443 |
+
if action is not None and pred_action is not None:
|
| 444 |
+
if sample["proprio"] is None:
|
| 445 |
+
raise ValueError("Eval sample must contain `proprio` for action denormalization.")
|
| 446 |
+
proprio = sample["proprio"].detach().to(device="cpu", dtype=torch.float32)
|
| 447 |
+
|
| 448 |
+
processor = self.val_dataset.lerobot_dataset.processor
|
| 449 |
+
|
| 450 |
+
denorm_actions = {}
|
| 451 |
+
action_meta = processor.shape_meta["action"]
|
| 452 |
+
state_meta = processor.shape_meta["state"]
|
| 453 |
+
for action_name, raw_action in (("pred", pred_action), ("gt", action)):
|
| 454 |
+
if not isinstance(raw_action, torch.Tensor):
|
| 455 |
+
raise TypeError(f"{action_name} action must be a torch.Tensor, got {type(raw_action)}")
|
| 456 |
+
if raw_action.ndim == 2:
|
| 457 |
+
action_btd = raw_action.unsqueeze(0)
|
| 458 |
+
elif raw_action.ndim == 3 and raw_action.shape[0] == 1:
|
| 459 |
+
action_btd = raw_action
|
| 460 |
+
else:
|
| 461 |
+
raise ValueError(
|
| 462 |
+
f"{action_name} action must have shape [T, D] or [1, T, D], got {tuple(raw_action.shape)}"
|
| 463 |
+
)
|
| 464 |
+
action_btd = action_btd.detach().to(device="cpu", dtype=torch.float32)
|
| 465 |
+
|
| 466 |
+
batch = {
|
| 467 |
+
"action": action_btd,
|
| 468 |
+
"state": proprio,
|
| 469 |
+
}
|
| 470 |
+
batch = processor.action_state_merger.backward(batch)
|
| 471 |
+
batch = processor.normalizer.backward(batch)
|
| 472 |
+
merged_batch = {
|
| 473 |
+
"action": {meta["key"]: batch["action"][meta["key"]].squeeze(0) for meta in action_meta},
|
| 474 |
+
"state": {meta["key"]: batch["state"][meta["key"]].squeeze(0) for meta in state_meta},
|
| 475 |
+
}
|
| 476 |
+
merged_batch = processor.action_state_merger.forward(merged_batch)
|
| 477 |
+
denorm_action = merged_batch["action"].unsqueeze(0)
|
| 478 |
+
if denorm_action.ndim != 3 or denorm_action.shape[0] != 1:
|
| 479 |
+
raise ValueError(
|
| 480 |
+
f"Denormalized {action_name} action must have shape [1, T, D], got {tuple(denorm_action.shape)}"
|
| 481 |
+
)
|
| 482 |
+
denorm_actions[action_name] = denorm_action
|
| 483 |
+
|
| 484 |
+
pred_action_denorm = denorm_actions["pred"]
|
| 485 |
+
gt_action_denorm = denorm_actions["gt"]
|
| 486 |
+
|
| 487 |
+
if pred_action_denorm.shape != gt_action_denorm.shape:
|
| 488 |
+
raise ValueError(
|
| 489 |
+
"Predicted action/GT action shape mismatch after denormalization: "
|
| 490 |
+
f"pred={tuple(pred_action_denorm.shape)} vs gt={tuple(gt_action_denorm.shape)}"
|
| 491 |
+
)
|
| 492 |
+
action_diff = pred_action_denorm - gt_action_denorm
|
| 493 |
+
action_l1 = action_diff.abs().mean().item()
|
| 494 |
+
action_l2 = action_diff.pow(2).mean().item()
|
| 495 |
+
|
| 496 |
+
# 4. VAE reconstruction metrics against GT video
|
| 497 |
+
gt_video_batch = video0.unsqueeze(0).to(device=model.device, dtype=model.torch_dtype)
|
| 498 |
+
vae_latents = model._encode_video_latents(gt_video_batch, tiled=False)
|
| 499 |
+
vae_recon_video = model._decode_latents(vae_latents, tiled=False)
|
| 500 |
+
vae_video_tensor = pil_frames_to_video_tensor(vae_recon_video)
|
| 501 |
+
|
| 502 |
+
assert vae_video_tensor.shape == gt_video_tensor.shape, (
|
| 503 |
+
"Eval VAE reconstruction/GT shape mismatch: "
|
| 504 |
+
f"vae={tuple(vae_video_tensor.shape)} vs gt={tuple(gt_video_tensor.shape)}"
|
| 505 |
+
)
|
| 506 |
+
|
| 507 |
+
psnr_decode_vs_gt = video_psnr(pred=vae_video_tensor, target=gt_video_tensor)
|
| 508 |
+
ssim_decode_vs_gt = video_ssim(pred=vae_video_tensor, target=gt_video_tensor)
|
| 509 |
+
|
| 510 |
+
psnr_rollout_vs_decode = video_psnr(pred=pred_video_tensor, target=vae_video_tensor)
|
| 511 |
+
ssim_rollout_vs_decode = video_ssim(pred=pred_video_tensor, target=vae_video_tensor)
|
| 512 |
+
|
| 513 |
+
stitched_video_tensor = torch.cat(
|
| 514 |
+
[pred_video_tensor, vae_video_tensor, gt_video_tensor],
|
| 515 |
+
dim=2,
|
| 516 |
+
).contiguous()
|
| 517 |
+
stitched_frames = []
|
| 518 |
+
for t in range(stitched_video_tensor.shape[1]):
|
| 519 |
+
frame = (stitched_video_tensor[:, t].permute(1, 2, 0).clamp(0.0, 1.0).numpy() * 255.0).astype(np.uint8)
|
| 520 |
+
stitched_frames.append(Image.fromarray(frame))
|
| 521 |
+
|
| 522 |
+
video_path = os.path.join(
|
| 523 |
+
self.eval_dir,
|
| 524 |
+
f"step_{self.global_step:06d}_rank_{self.accelerator.process_index:03d}.mp4",
|
| 525 |
+
)
|
| 526 |
+
save_mp4(stitched_frames, video_path, fps=8)
|
| 527 |
+
|
| 528 |
+
local_metrics = torch.tensor(
|
| 529 |
+
[
|
| 530 |
+
float(val_loss),
|
| 531 |
+
float(psnr_rollout_vs_gt),
|
| 532 |
+
float(ssim_rollout_vs_gt),
|
| 533 |
+
float(psnr_rollout_vs_decode),
|
| 534 |
+
float(ssim_rollout_vs_decode),
|
| 535 |
+
float(psnr_decode_vs_gt),
|
| 536 |
+
float(ssim_decode_vs_gt),
|
| 537 |
+
float(action_l2) if action_l2 is not None else -1.0,
|
| 538 |
+
float(action_l1) if action_l1 is not None else -1.0,
|
| 539 |
+
],
|
| 540 |
+
device=self.accelerator.device,
|
| 541 |
+
dtype=torch.float32,
|
| 542 |
+
).unsqueeze(0)
|
| 543 |
+
gathered_metrics = self.accelerator.gather_for_metrics(local_metrics)
|
| 544 |
+
mean_metrics = gathered_metrics[:, :7].mean(dim=0)
|
| 545 |
+
action_l2_mean = gathered_metrics[:, 7].mean().item() if action_l2 is not None else None
|
| 546 |
+
action_l1_mean = gathered_metrics[:, 8].mean().item() if action_l1 is not None else None
|
| 547 |
+
|
| 548 |
+
if was_dit_training:
|
| 549 |
+
self._set_dit_only_train_mode()
|
| 550 |
+
|
| 551 |
+
result = {
|
| 552 |
+
"val_loss": float(mean_metrics[0].item()),
|
| 553 |
+
"psnr_rg": float(mean_metrics[1].item()),
|
| 554 |
+
"ssim_rg": float(mean_metrics[2].item()),
|
| 555 |
+
"psnr_rd": float(mean_metrics[3].item()),
|
| 556 |
+
"ssim_rd": float(mean_metrics[4].item()),
|
| 557 |
+
"psnr_dg": float(mean_metrics[5].item()),
|
| 558 |
+
"ssim_dg": float(mean_metrics[6].item()),
|
| 559 |
+
"video_path": video_path,
|
| 560 |
+
}
|
| 561 |
+
if action_l2_mean is not None:
|
| 562 |
+
result["action_l2"] = float(action_l2_mean)
|
| 563 |
+
if action_l1_mean is not None:
|
| 564 |
+
result["action_l1"] = float(action_l1_mean)
|
| 565 |
+
return result
|
| 566 |
+
|
| 567 |
+
def _save_weights_checkpoint(self, step_tag: str):
|
| 568 |
+
model = self.accelerator.unwrap_model(self.model)
|
| 569 |
+
ckpt_path = os.path.join(self.weights_dir, f"{step_tag}.pt")
|
| 570 |
+
model.save_checkpoint(ckpt_path, optimizer=None, step=self.global_step)
|
| 571 |
+
return ckpt_path
|
| 572 |
+
|
| 573 |
+
def _save_trainer_state(self, state_path: str):
|
| 574 |
+
state_file = os.path.join(state_path, "trainer_state.json")
|
| 575 |
+
payload = {
|
| 576 |
+
"global_step": int(self.global_step),
|
| 577 |
+
"epoch": int(self.epoch),
|
| 578 |
+
"batch_in_epoch": int(self.batch_in_epoch),
|
| 579 |
+
}
|
| 580 |
+
with open(state_file, "w", encoding="utf-8") as f:
|
| 581 |
+
json.dump(payload, f, ensure_ascii=True, indent=2)
|
| 582 |
+
|
| 583 |
+
def save_checkpoint(self):
|
| 584 |
+
step_tag = f"step_{self.global_step:06d}"
|
| 585 |
+
|
| 586 |
+
self.accelerator.wait_for_everyone()
|
| 587 |
+
ckpt_path = None
|
| 588 |
+
if self.accelerator.is_main_process:
|
| 589 |
+
ckpt_path = self._save_weights_checkpoint(step_tag=step_tag)
|
| 590 |
+
self.accelerator.wait_for_everyone()
|
| 591 |
+
|
| 592 |
+
state_path = os.path.join(self.state_dir, step_tag)
|
| 593 |
+
ensure_dir(state_path)
|
| 594 |
+
self.accelerator.save_state(output_dir=state_path)
|
| 595 |
+
if self.accelerator.is_main_process:
|
| 596 |
+
self._save_trainer_state(state_path)
|
| 597 |
+
self.accelerator.wait_for_everyone()
|
| 598 |
+
|
| 599 |
+
return {"weights_path": ckpt_path, "state_path": state_path}
|
| 600 |
+
|
| 601 |
+
def load_training_state(self, state_dir: str):
|
| 602 |
+
self.accelerator.load_state(input_dir=state_dir)
|
| 603 |
+
state_file = Path(state_dir) / "trainer_state.json"
|
| 604 |
+
if state_file.exists():
|
| 605 |
+
with open(state_file, "r", encoding="utf-8") as f:
|
| 606 |
+
payload = json.load(f)
|
| 607 |
+
self.global_step = int(payload["global_step"])
|
| 608 |
+
|
| 609 |
+
if "epoch" in payload and "batch_in_epoch" in payload:
|
| 610 |
+
self.epoch = int(payload["epoch"])
|
| 611 |
+
self.batch_in_epoch = int(payload["batch_in_epoch"])
|
| 612 |
+
self.train_sampler.set_epoch_offset(self.epoch)
|
| 613 |
+
self.train_sampler.set_resume_batch_offset(self.batch_in_epoch)
|
| 614 |
+
logger.info(
|
| 615 |
+
"Restored dataloader progress: epoch=%d batch_in_epoch=%d sample_offset=%d",
|
| 616 |
+
self.epoch,
|
| 617 |
+
self.batch_in_epoch,
|
| 618 |
+
self.batch_in_epoch * self.batch_size * self.accelerator.num_processes,
|
| 619 |
+
)
|
| 620 |
+
else:
|
| 621 |
+
self.epoch = 0
|
| 622 |
+
self.batch_in_epoch = 0
|
| 623 |
+
self.train_sampler.clear_resume_batch_offset()
|
| 624 |
+
logger.warning(
|
| 625 |
+
"State file does not contain `epoch`/`batch_in_epoch`; "
|
| 626 |
+
"optimizer/scheduler were restored, but dataloader progress resume is skipped."
|
| 627 |
+
)
|
| 628 |
+
self.accelerator.wait_for_everyone()
|
| 629 |
+
return
|
| 630 |
+
|
| 631 |
+
match = re.search(r"step[_-](\d+)$", str(state_dir).rstrip("/"))
|
| 632 |
+
if match:
|
| 633 |
+
self.global_step = int(match.group(1))
|
| 634 |
+
else:
|
| 635 |
+
self.global_step = 0
|
| 636 |
+
self.epoch = 0
|
| 637 |
+
self.batch_in_epoch = 0
|
| 638 |
+
self.train_sampler.clear_resume_batch_offset()
|
| 639 |
+
self.accelerator.wait_for_everyone()
|
| 640 |
+
logger.info("Loaded accelerate training state from %s at step=%d", state_dir, self.global_step)
|
| 641 |
+
logger.warning(
|
| 642 |
+
"State file `%s` is missing; dataloader progress resume is skipped.",
|
| 643 |
+
state_file,
|
| 644 |
+
)
|
| 645 |
+
|
| 646 |
+
def train(self):
|
| 647 |
+
self._set_dit_only_train_mode()
|
| 648 |
+
|
| 649 |
+
unwrapped_model = self.accelerator.unwrap_model(self.model)
|
| 650 |
+
|
| 651 |
+
if self.max_steps is None:
|
| 652 |
+
raise ValueError("`max_steps` must be set before entering the while-step training loop.")
|
| 653 |
+
|
| 654 |
+
logger.info("Starting training with max_steps=%d.", self.max_steps)
|
| 655 |
+
data_iter = iter(self.train_loader)
|
| 656 |
+
self.run_start_step = self.global_step
|
| 657 |
+
self.run_start_time = time.perf_counter()
|
| 658 |
+
|
| 659 |
+
while self.global_step < self.max_steps:
|
| 660 |
+
try:
|
| 661 |
+
sample = next(data_iter)
|
| 662 |
+
self.batch_in_epoch += 1
|
| 663 |
+
except StopIteration:
|
| 664 |
+
self.epoch += 1
|
| 665 |
+
self.batch_in_epoch = 0
|
| 666 |
+
self.train_sampler.clear_resume_batch_offset()
|
| 667 |
+
data_iter = iter(self.train_loader)
|
| 668 |
+
continue
|
| 669 |
+
|
| 670 |
+
with self.accelerator.accumulate(self.model):
|
| 671 |
+
train_model = self.model if hasattr(self.model, "training_loss") else self.accelerator.unwrap_model(self.model)
|
| 672 |
+
|
| 673 |
+
with self.accelerator.autocast():
|
| 674 |
+
loss, loss_dict = train_model.training_loss(sample)
|
| 675 |
+
self.accelerator.backward(loss)
|
| 676 |
+
|
| 677 |
+
if self.accelerator.sync_gradients:
|
| 678 |
+
grad_norm = self.accelerator.clip_grad_norm_(self.model.parameters(), self.max_grad_norm)
|
| 679 |
+
self.optimizer.step()
|
| 680 |
+
if not self.accelerator.optimizer_step_was_skipped:
|
| 681 |
+
self.scheduler.step()
|
| 682 |
+
self.optimizer.zero_grad(set_to_none=True)
|
| 683 |
+
self.global_step += 1
|
| 684 |
+
global_loss = float(
|
| 685 |
+
self.accelerator.gather(loss.detach().float().reshape(1)).mean().item()
|
| 686 |
+
)
|
| 687 |
+
global_loss_metrics = {}
|
| 688 |
+
for key, value in loss_dict.items():
|
| 689 |
+
metric_tensor = torch.tensor(float(value), device=loss.device, dtype=torch.float32).reshape(1)
|
| 690 |
+
global_loss_metrics[key] = float(
|
| 691 |
+
self.accelerator.gather(metric_tensor).mean().item()
|
| 692 |
+
)
|
| 693 |
+
grad_norm_tensor = torch.tensor(grad_norm, device=loss.device, dtype=torch.float32)
|
| 694 |
+
global_grad_norm = float(self.accelerator.gather(grad_norm_tensor).mean().item())
|
| 695 |
+
|
| 696 |
+
current_lr = float(self.optimizer.param_groups[0]["lr"])
|
| 697 |
+
|
| 698 |
+
if self.log_every > 0 and self.global_step % self.log_every == 0 and self.accelerator.is_main_process:
|
| 699 |
+
eta_str, steps_per_sec = self._estimate_eta()
|
| 700 |
+
description = "[train] epoch=%d step=%d/%d loss=%.4f " % (
|
| 701 |
+
self.epoch,
|
| 702 |
+
self.global_step,
|
| 703 |
+
self.max_steps,
|
| 704 |
+
global_loss,
|
| 705 |
+
)
|
| 706 |
+
if global_loss_metrics:
|
| 707 |
+
detail_str = " ".join([f"{k}={v:.4f}" for k, v in sorted(global_loss_metrics.items())])
|
| 708 |
+
description += detail_str + " "
|
| 709 |
+
description += "lr=%.2e speed=%.2f step/s, %.2f samples/s eta=%s" % (
|
| 710 |
+
current_lr,
|
| 711 |
+
steps_per_sec,
|
| 712 |
+
steps_per_sec * self.batch_size * self.accelerator.num_processes,
|
| 713 |
+
eta_str,
|
| 714 |
+
)
|
| 715 |
+
logger.info(description)
|
| 716 |
+
|
| 717 |
+
wandb_payload = {
|
| 718 |
+
"train/loss": global_loss,
|
| 719 |
+
"train/grad_norm": global_grad_norm,
|
| 720 |
+
"train/lr": current_lr,
|
| 721 |
+
"performance/steps_per_sec": steps_per_sec,
|
| 722 |
+
"performance/samples_per_sec": steps_per_sec * self.batch_size * self.accelerator.num_processes,
|
| 723 |
+
}
|
| 724 |
+
for key, value in global_loss_metrics.items():
|
| 725 |
+
wandb_payload[f"train/{key}"] = value
|
| 726 |
+
self._wandb_log(wandb_payload)
|
| 727 |
+
|
| 728 |
+
if (
|
| 729 |
+
self.eval_every > 0
|
| 730 |
+
and self.val_dataset is not None
|
| 731 |
+
and self.global_step % self.eval_every == 0
|
| 732 |
+
):
|
| 733 |
+
metrics = self.evaluate()
|
| 734 |
+
self.accelerator.wait_for_everyone()
|
| 735 |
+
if metrics is not None and self.accelerator.is_main_process:
|
| 736 |
+
description = "[eval] step=%d val_loss=%.4f infer_psnr=%.4f infer_ssim=%.4f" % (
|
| 737 |
+
self.global_step,
|
| 738 |
+
metrics["val_loss"],
|
| 739 |
+
metrics["psnr_rd"],
|
| 740 |
+
metrics["ssim_rd"],
|
| 741 |
+
)
|
| 742 |
+
if "action_l2" in metrics:
|
| 743 |
+
description += " action_l2=%.4f" % metrics["action_l2"]
|
| 744 |
+
if "action_l1" in metrics:
|
| 745 |
+
description += " action_l1=%.4f" % metrics["action_l1"]
|
| 746 |
+
logger.info(description)
|
| 747 |
+
eval_payload = {
|
| 748 |
+
"eval/val_loss": float(metrics["val_loss"]),
|
| 749 |
+
"eval/psnr_rg": float(metrics["psnr_rg"]),
|
| 750 |
+
"eval/ssim_rg": float(metrics["ssim_rg"]),
|
| 751 |
+
"eval/psnr_rd": float(metrics["psnr_rd"]),
|
| 752 |
+
"eval/ssim_rd": float(metrics["ssim_rd"]),
|
| 753 |
+
"eval/psnr_dg": float(metrics["psnr_dg"]),
|
| 754 |
+
"eval/ssim_dg": float(metrics["ssim_dg"]),
|
| 755 |
+
}
|
| 756 |
+
if "action_l2" in metrics:
|
| 757 |
+
eval_payload["eval/action_l2"] = float(metrics["action_l2"])
|
| 758 |
+
if "action_l1" in metrics:
|
| 759 |
+
eval_payload["eval/action_l1"] = float(metrics["action_l1"])
|
| 760 |
+
self._wandb_log(eval_payload)
|
| 761 |
+
|
| 762 |
+
if self.save_every > 0 and self.global_step % self.save_every == 0:
|
| 763 |
+
ckpt_info = self.save_checkpoint()
|
| 764 |
+
if self.accelerator.is_main_process:
|
| 765 |
+
logger.info(
|
| 766 |
+
"[ckpt] step=%d weights=%s state=%s",
|
| 767 |
+
self.global_step,
|
| 768 |
+
ckpt_info["weights_path"],
|
| 769 |
+
ckpt_info["state_path"],
|
| 770 |
+
)
|
| 771 |
+
|
| 772 |
+
if self.global_step >= self.max_steps:
|
| 773 |
+
ckpt_info = self.save_checkpoint()
|
| 774 |
+
if self.accelerator.is_main_process:
|
| 775 |
+
logger.info(
|
| 776 |
+
"[done] max_steps reached step=%d weights=%s state=%s",
|
| 777 |
+
self.global_step,
|
| 778 |
+
ckpt_info["weights_path"],
|
| 779 |
+
ckpt_info["state_path"],
|
| 780 |
+
)
|
| 781 |
+
return
|
| 782 |
+
|
| 783 |
+
ckpt_info = self.save_checkpoint()
|
| 784 |
+
if self.accelerator.is_main_process:
|
| 785 |
+
logger.info(
|
| 786 |
+
"[done] training finished step=%d weights=%s state=%s",
|
| 787 |
+
self.global_step,
|
| 788 |
+
ckpt_info["weights_path"],
|
| 789 |
+
ckpt_info["state_path"],
|
| 790 |
+
)
|
| 791 |
+
|