diff --git a/baseline/fastwam_zipper/README.md b/baseline/fastwam_zipper/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e295dfbe6b8d93956652f6f3a369474635c27860 --- /dev/null +++ b/baseline/fastwam_zipper/README.md @@ -0,0 +1,44 @@ +# FastWAM — zipper_pulling + +Final checkpoint from `bifrost-2026102616142400-shouqx`: **30 epochs**, **16,380 optimizer steps**. The checkpoint is uploaded unchanged; its SHA-256 is recorded in `provenance.json`. + +## Included inference assets + +- `step_016380.pt`: final MoT (video/action) and proprioception encoder weights. +- `models/Wan2.2-TI2V-5B/Wan2.2_VAE.pth`: the frozen VAE actually used in training. +- `context.pt`: exact cached native UMT5 embedding for **Pull the zipper.**. +- `norm_stats.json`: this task's original state/action q01/q99 statistics, byte-identical to training. +- `inference_config.json`, `inference.py`, `requirements-inference.txt`, and exact training source in `source/`. + +The fixed-task path works offline after downloading this entire directory and installing Python dependencies. No external model checkpoint, tokenizer, or text encoder is required: it uses the included text cache. Changing the instruction requires a newly computed matching native UMT5 cache; editing the prompt string alone does not change conditioning. + +## Download + +Download the complete task directory (including the VAE and source): + +```python +from huggingface_hub import snapshot_download +snapshot_download("qshou-coder/SpaceDreamer_Galexea", allow_patterns=["baseline/fastwam_zipper/**"], local_dir="fastwam_export") +``` + +Then enter `fastwam_export/baseline/fastwam_zipper/`. + +## Setup and inference + +Use Python 3.10+ and a CUDA GPU with BF16 support and sufficient free VRAM (training used PyTorch 2.7.1 / CUDA 12.8). From this directory: + +```bash +python -m pip install torch==2.7.1 torchvision==0.22.1 --index-url https://download.pytorch.org/whl/cu128 +python -m pip install -r requirements-inference.txt +python inference.py --input observation.npz --output prediction.npz --device cuda:0 --steps 20 --seed 0 +``` + +`observation.npz` must contain `head_left`, `wrist_left`, `wrist_right` (HxWx3 **RGB uint8**, no BGR conversion), and `state` (float32, shape `[14]`, original physical state units). The order is **left arm 6, left gripper, right arm 6, right gripper**. + +The head image is resized to 256x320 and the two wrists to 128x160, then packed as head on top, left/right wrists below, giving 384x320. Inference applies the training q01/q99 transform and no random crop/color augmentation. `prediction.npz` contains `actions` `[40,14]` in original physical target units and `normalized_actions`. Actions are **absolute original `motion_target.position` labels**, not deltas, Cartesian commands, or velocity. Keep the native gripper encoding and coordinate conventions. The data rate was 20 Hz; execution/replanning is controlled by the downstream robot integration, not this file-based runner. + +Python API: `from inference import Policy; policy = Policy(); result = policy.predict(images, state)`. `predict_video=True` additionally uses the native joint sampler to return nine video frames; default inference uses the native cached action-only path. Default 20 denoising steps is the native inference default, not a claimed evaluation optimum. + +## Validation scope + +See `validation.json` for actual checks. Packaging checks cover source identity, checkpoint step and architecture, cached text, training preprocessing and normalization parity, inverse normalization, and relative asset resolution. They do not constitute a new real-robot rollout or GPU performance measurement. Training completion receipts confirm final save and strict reload; the original training smoke used an earlier smoke checkpoint. diff --git a/baseline/fastwam_zipper/THIRD_PARTY.md b/baseline/fastwam_zipper/THIRD_PARTY.md new file mode 100644 index 0000000000000000000000000000000000000000..425c856b1d156dc569e395e4fb674fb0309c1107 --- /dev/null +++ b/baseline/fastwam_zipper/THIRD_PARTY.md @@ -0,0 +1,3 @@ +# Third-party components + +The exact FastWAM training source is included under `source/`, with its original MIT license and source headers. The frozen `Wan2.2_VAE.pth` is the original Wan-AI/Wan2.2-TI2V-5B component, SHA-256 `20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36`. Wan2.2 is distributed under Apache-2.0; see https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B and https://github.com/Wan-Video/Wan2.2 . The native task text cache was produced with the frozen Wan UMT5 encoder. diff --git a/baseline/fastwam_zipper/context.pt b/baseline/fastwam_zipper/context.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ada6c10dd6fe2ffec8b3eb6a07338738a76ecab --- /dev/null +++ b/baseline/fastwam_zipper/context.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24c9c95ba634779ba7aeb33743ee38b3cf688805471889400a2e841fd4ea4ad1 +size 1050977 diff --git a/baseline/fastwam_zipper/inference.py b/baseline/fastwam_zipper/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..d4e5ebc78a09ad3ee738cba8baf602e980b1b20e --- /dev/null +++ b/baseline/fastwam_zipper/inference.py @@ -0,0 +1,107 @@ +"""Portable, fixed-task FastWAM inference using the exact training source.""" +import argparse +import json +import os +from pathlib import Path +import sys +import numpy as np +import torch +from torchvision.transforms import functional as F, InterpolationMode + +CAMERAS=('head_left','wrist_left','wrist_right') + +def pack_images(images): + views=[] + for name in CAMERAS: + image=np.asarray(images[name]) + if image.ndim!=3 or image.shape[-1]!=3 or image.dtype!=np.uint8: + raise ValueError(f'{name} must be HxWx3 uint8 RGB (not BGR).') + v=torch.from_numpy(image.copy()).permute(2,0,1).float()/255. + size=[256,320] if name=='head_left' else [128,160] + views.append(F.resize(v,size,InterpolationMode.BILINEAR,antialias=True)) + return torch.cat([views[0],torch.cat(views[1:],dim=-1)],dim=-2).contiguous()*2-1 + +def normalize(x,stats): + x=np.asarray(x,dtype=np.float32) + lo,hi=np.array(stats['q01']),np.array(stats['q99']) + if x.shape[-1]!=14 or lo.shape!=(14,) or hi.shape!=(14,) or not np.isfinite(x).all(): + raise ValueError('Expected finite 14D state/actions and 14D quantile bounds.') + result=np.zeros_like(x,dtype=np.float32);mask=~np.isclose(hi,lo) + result[...,mask]=(x[...,mask]-lo[mask])/(hi[mask]-lo[mask]) + result[...,mask]=2*result[...,mask]-1 + return np.clip(result,-1,1) + +def denormalize(x,stats): + x=np.asarray(x,dtype=np.float32) + lo,hi=np.array(stats['q01']),np.array(stats['q99']) + if x.shape[-1]!=14 or not np.isfinite(x).all():raise ValueError('Invalid model actions.') + result=(x+1)/2*(hi-lo)+lo + result[...,np.isclose(hi,lo)]=lo[np.isclose(hi,lo)] + return result.astype(np.float32) + +class Policy: + def __init__(self,bundle=None,device='cuda:0'): + self.root=Path(bundle or Path(__file__).parent).resolve() + cfg=json.loads((self.root/'inference_config.json').read_text()) + self.config=cfg + self.stats=json.loads((self.root/'norm_stats.json').read_text())['norm_stats'] + context=torch.load(self.root/'context.pt',map_location='cpu',weights_only=True) + if context['prompt']!=cfg['encoded_prompt']:raise ValueError('Task/context mismatch.') + if context['context'].shape!=(128,4096) or context['mask'].shape!=(128,):raise ValueError('Invalid text cache.') + self.context=context + sys.path.insert(0,str(self.root/'source/src')) + os.environ['DIFFSYNTH_MODEL_BASE_PATH']=str(self.root/'models') + os.environ['DIFFSYNTH_SKIP_DOWNLOAD']='true' + if not (self.root/'models/Wan2.2-TI2V-5B/Wan2.2_VAE.pth').is_file():raise FileNotFoundError('Missing bundled VAE.') + from fastwam.models.wan22.fastwam import FastWAM + m=cfg['model'] + self.model=FastWAM.from_wan22_pretrained( + device=device,torch_dtype=torch.bfloat16, + model_id=m['model_id'],tokenizer_model_id=m['tokenizer_model_id'], + tokenizer_max_len=m['tokenizer_max_len'],load_text_encoder=False, + proprio_dim=m['proprio_dim'],redirect_common_files=False, + video_dit_config=m['video_dit_config'],action_dit_config=m['action_dit_config'], + skip_dit_load_from_pretrain=True,action_dit_pretrained_path=None, + mot_checkpoint_mixed_attn=False,compile_training_denoise=False, + video_train_shift=m['video_scheduler']['train_shift'],video_infer_shift=m['video_scheduler']['infer_shift'], + video_num_train_timesteps=m['video_scheduler']['num_train_timesteps'], + action_train_shift=m['action_scheduler']['train_shift'],action_infer_shift=m['action_scheduler']['infer_shift'], + action_num_train_timesteps=m['action_scheduler']['num_train_timesteps']) + checkpoint=torch.load(self.root/cfg['checkpoint'],map_location='cpu',weights_only=True,mmap=True) + if checkpoint['step']!=cfg['final_step']:raise ValueError('Checkpoint step mismatch.') + self.model.mot.load_state_dict(checkpoint['mot'],strict=True) + self.model.proprio_encoder.load_state_dict(checkpoint['proprio_encoder'],strict=True) + self.model.eval() + + @torch.inference_mode() + def predict(self,images,state,steps=20,seed=None,predict_video=False): + state=np.asarray(state,dtype=np.float32) + if state.shape!=(14,):raise ValueError('state must be shape (14,), in the original physical units.') + if steps<1:raise ValueError('steps must be positive') + args=dict(prompt=None,input_image=pack_images(images),action_horizon=40, + proprio=torch.from_numpy(normalize(state,self.stats['state'])), + context=self.context['context'],context_mask=self.context['mask'], + num_inference_steps=steps,seed=seed,compile_action_infer=False) + if predict_video: + result=self.model.infer_joint(**args,num_video_frames=9,test_action_with_infer_action=False) + else:result=self.model.infer_action(**args) + normalized=result['action'].detach().float().cpu().numpy() + if normalized.shape!=(40,14):raise ValueError(f'Unexpected action shape {normalized.shape}') + output={'actions':denormalize(normalized,self.stats['actions']),'normalized_actions':normalized} + if predict_video:output['video']=result['video'] + return output + +def main(): + p=argparse.ArgumentParser(description=__doc__) + p.add_argument('--bundle',type=Path,default=Path(__file__).parent) + p.add_argument('--input',type=Path,required=True,help='NPZ: head_left, wrist_left, wrist_right RGB uint8 arrays; state float32[14].') + p.add_argument('--output',type=Path,required=True) + p.add_argument('--device',default='cuda:0');p.add_argument('--steps',type=int,default=20);p.add_argument('--seed',type=int,default=None) + a=p.parse_args() + with np.load(a.input,allow_pickle=False) as data: + images={name:data[name] for name in CAMERAS};state=data['state'] + result=Policy(a.bundle,a.device).predict(images,state,a.steps,a.seed) + np.savez(a.output,**result) + print(json.dumps({'output':str(a.output),'action_shape':list(result['actions'].shape),'action_semantics':'absolute motion_target.position'})) + +if __name__=='__main__':main() diff --git a/baseline/fastwam_zipper/inference_config.json b/baseline/fastwam_zipper/inference_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b8d5fd354a5e911243ba6d9359b6df009f791840 --- /dev/null +++ b/baseline/fastwam_zipper/inference_config.json @@ -0,0 +1,78 @@ +{ + "task": "zipper_pulling", + "training_job": "bifrost-2026102616142400-shouqx", + "final_step": 16380, + "checkpoint": "step_016380.pt", + "prompt": "Pull the zipper.", + "encoded_prompt": "A video recorded from a robot's point of view executing the following instruction: Pull the zipper.", + "fps": 20, + "action_horizon": 40, + "video_frames": 9, + "model": { + "model_id": "Wan2.2-TI2V-5B", + "tokenizer_model_id": "Wan2.1-T2V-1.3B", + "tokenizer_max_len": 128, + "load_text_encoder": false, + "proprio_dim": 14, + "redirect_common_files": false, + "mot_checkpoint_mixed_attn": false, + "compile_training_denoise": false, + "action_dit_pretrained_path": null, + "skip_dit_load_from_pretrain": true, + "video_dit_config": { + "has_image_input": false, + "patch_size": [ + 1, + 2, + 2 + ], + "in_dim": 48, + "hidden_dim": 3072, + "ffn_dim": 14336, + "freq_dim": 256, + "text_dim": 4096, + "out_dim": 48, + "num_heads": 24, + "attn_head_dim": 128, + "num_layers": 30, + "eps": 1e-06, + "seperated_timestep": true, + "require_clip_embedding": false, + "require_vae_embedding": false, + "fuse_vae_embedding_in_latents": true, + "use_gradient_checkpointing": false, + "video_attention_mask_mode": "first_frame_causal", + "action_conditioned": false, + "action_dim": 14, + "action_group_causal_mask_mode": "group_diagonal" + }, + "action_dit_config": { + "action_dim": 14, + "hidden_dim": 1024, + "ffn_dim": 4096, + "num_heads": 24, + "attn_head_dim": 128, + "num_layers": 30, + "text_dim": 4096, + "freq_dim": 256, + "eps": 1e-06, + "use_gradient_checkpointing": false + }, + "video_scheduler": { + "train_shift": 5.0, + "infer_shift": 5.0, + "num_train_timesteps": 1000 + }, + "action_scheduler": { + "train_shift": 1.0, + "infer_shift": 1.0, + "num_train_timesteps": 1000 + }, + "loss": { + "lambda_action": 1.0 + } + }, + "action_semantics": "absolute_original_motion_target_position", + "joint_order": "left_arm6,left_gripper,right_arm6,right_gripper", + "default_denoising_steps": 20 +} diff --git a/baseline/fastwam_zipper/inference_manifest.json b/baseline/fastwam_zipper/inference_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..0bd9fe7a3af9738a0dab3ae76301814d479af362 --- /dev/null +++ b/baseline/fastwam_zipper/inference_manifest.json @@ -0,0 +1,539 @@ +{ + "files": [ + { + "path": "step_016380.pt", + "bytes": 12041813369, + "sha256": "5e574c8cc7bb5747852cd815cfed9896d8d0f1db83eee28a367bd21beb25f348" + }, + { + "path": "README.md", + "bytes": 3521, + "sha256": "0fe1530fc1195d0f4ba393fb747a97485d762aefc4f26831ff7ae709ddceb641" + }, + { + "path": "THIRD_PARTY.md", + "bytes": 497, + "sha256": "085e845844811f3cbf63a07766841a2af78ea80f337753353af868a1a243fc55" + }, + { + "path": "context.pt", + "bytes": 1050977, + "sha256": "24c9c95ba634779ba7aeb33743ee38b3cf688805471889400a2e841fd4ea4ad1" + }, + { + "path": "inference.py", + "bytes": 6194, + "sha256": "12e401a2400cc7fdb54560b08c30a1401ef3680261a7bee6fd5912443dadb961" + }, + { + "path": "inference_config.json", + "bytes": 2206, + "sha256": "e207f5427b3c9a25fc9f1d61074138c6c04dd717f1d253794d2771345cd69f8f" + }, + { + "path": "models/LICENSE-WAN-APACHE-2.0.txt", + "bytes": 11358, + "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30" + }, + { + "path": "models/Wan2.2-TI2V-5B/Wan2.2_VAE.pth", + "bytes": 2818839170, + "sha256": "20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36" + }, + { + "path": "norm_stats.json", + "bytes": 3306, + "sha256": "10a5eed0657164c56f13efb0d99434824b201f07e450263f564c828944aa5231" + }, + { + "path": "provenance.json", + "bytes": 563, + "sha256": "3858af2efaf9c8646d791137a5738db3cb082c7bcb74d9c6f92c2ba5b9573c9a" + }, + { + "path": "requirements-inference.txt", + "bytes": 196, + "sha256": "87f014a6694ed6b262d730652c68878ea4297a420c6309711346891920a0d4dc" + }, + { + "path": "source/LICENSE", + "bytes": 1210, + "sha256": "8379dddfbbb1154ad9cc57f008f1f8d4a8d26c3de99bd6b36aa4d8067888aa92" + }, + { + "path": "source/README.md", + "bytes": 15889, + "sha256": "1fbad9ad9407d54a224864a10ec9576fa731e4a8f291bc9a7e28f545647fef2a" + }, + { + "path": "source/configs/data/galaxea_r1_lite_3cam.yaml", + "bytes": 3145, + "sha256": "e83ab2ece05ccea8a352404a7cdd38b361325ecf720f64a79e94c6dccb36cc57" + }, + { + "path": "source/configs/data/libero_2cam.yaml", + "bytes": 2448, + "sha256": "d2ba368bd906992a2f3ca6229da9a5abd58163e2e64a8c4569d012efef039008" + }, + { + "path": "source/configs/data/libero_2cam_lerobot_v30.yaml", + "bytes": 1935, + "sha256": "9c243358de9e978f210d827569cf8bf074fec6b7a3d47b079e8827342562edc1" + }, + { + "path": "source/configs/data/robotwin.yaml", + "bytes": 4110, + "sha256": "f1ba509fac21fc59d5a210d6f68bcf94640729ce00dce695b7ca6df24c7b5aa7" + }, + { + "path": "source/configs/model/fastwam.yaml", + "bytes": 1536, + "sha256": "0c9378106366d35f99ea3a82ef30afdfbab9a1d330af3e6de6c94c4490d24868" + }, + { + "path": "source/configs/model/fastwam_idm.yaml", + "bytes": 1567, + "sha256": "7c95ab549c677ade48c7664beb85202aae15ad8c86dba99fbe4b14f2047aeb76" + }, + { + "path": "source/configs/model/fastwam_joint.yaml", + "bytes": 1510, + "sha256": "ab2a243f7452478b6ba779cc2507e08d503874e7e7ae6ddcc3f081055b0730ad" + }, + { + "path": "source/configs/model/fastwam_optional_idm.yaml", + "bytes": 1607, + "sha256": "f5c01951f483d453f7a0ef01bc7026eb8c2922b32361230a99e0d09d898904db" + }, + { + "path": "source/configs/sim_libero.yaml", + "bytes": 1082, + "sha256": "005c4da6d40abc2304ff6f9a06feca1204f330c1ac7c3919f807d1b58dfd016c" + }, + { + "path": "source/configs/sim_robotwin.yaml", + "bytes": 975, + "sha256": "b96b579a8f243c935f1e17f8f3d6e9b7cb6bf2f2a6823272f626ecc33d68056f" + }, + { + "path": "source/configs/task/galaxea_r1_lite_3cam_1e-4.yaml", + "bytes": 1263, + "sha256": "6553c8c22fafb7b3a563eb81fb310bf1b36d6b4d5ba7451c89b63d722a2543be" + }, + { + "path": "source/configs/task/libero_idm_2cam224_1e-4.yaml", + "bytes": 409, + "sha256": "5df6cf62843806b0d9d2fb48b873945865bbfa74d135c58032651d54d6992c4e" + }, + { + "path": "source/configs/task/libero_joint_2cam224_1e-4.yaml", + "bytes": 411, + "sha256": "1c46f34058c9c5363bab6814a0b86eb13be3a0077e944db7d39cf151da07636e" + }, + { + "path": "source/configs/task/libero_optional_idm_2cam224_1e-4.yaml", + "bytes": 381, + "sha256": "367794491e728cba5fdb63929e69ab19f3c75b89eb6631754ae6a67d114d542c" + }, + { + "path": "source/configs/task/libero_uncond_2cam224_1e-4.yaml", + "bytes": 405, + "sha256": "27bdc30545aa06bca81137690a241d469d5a67a433881976bd79da3d4171d1a0" + }, + { + "path": "source/configs/task/robotwin_idm_3cam_384_1e-4.yaml", + "bytes": 405, + "sha256": "2e67c9f301105752962af825e57e5b46ef5e6ba4290e397e5efc7abb88652619" + }, + { + "path": "source/configs/task/robotwin_joint_3cam_384_1e-4.yaml", + "bytes": 407, + "sha256": "daf4cb5246728ba60ae6903fbfd9d5e029d7172f30916b8143f81783450fa874" + }, + { + "path": "source/configs/task/robotwin_uncond_3cam_384_1e-4.yaml", + "bytes": 401, + "sha256": "2f78a207415b452b05e0238f3e00eb9942d7960413076b5b3cdcf1ce09caef9f" + }, + { + "path": "source/configs/train.yaml", + "bytes": 697, + "sha256": "42a60765bfdd842af01aacdde6066ce63be00698377fe7729110b795be684016" + }, + { + "path": "source/pyproject.toml", + "bytes": 1049, + "sha256": "161e4be4e5a0c3f9888aa7b8b8416c45441eabf0aed4374ac193a08de02c043a" + }, + { + "path": "source/scripts/__init__.py", + "bytes": 37, + "sha256": "5a71e51c716ea7579151f36996bebb1ff82c6fedd0c45c68f2918cf5812f1053" + }, + { + "path": "source/scripts/accelerate_configs/accelerate_zero0.yaml", + "bytes": 489, + "sha256": "dacdf1cee3675246567f1f61e943c6762a4700b8b40c888af8fdf89773adc35b" + }, + { + "path": "source/scripts/accelerate_configs/accelerate_zero1_ds.yaml", + "bytes": 496, + "sha256": "5b9e24f38c199fdff6bbd17981d86e2624f855f707660094c99f711efc8e0833" + }, + { + "path": "source/scripts/accelerate_configs/accelerate_zero2_ds.yaml", + "bytes": 496, + "sha256": "3b838fbf44cd279613797e9ac3265f306806391963be593cb47787d38240a2e1" + }, + { + "path": "source/scripts/check_galaxea_r1_lite_contract.py", + "bytes": 3820, + "sha256": "585200c8bea8e37a832381a00ec86cdb0fc3176840b35cd0df26e6700819c522" + }, + { + "path": "source/scripts/check_galaxea_text_cache.py", + "bytes": 1749, + "sha256": "2ef7e6dcfc821b8a65ad75ca1af3e70d58100a2e129105d9dd0fcf389e1c7689" + }, + { + "path": "source/scripts/dryrun_fastwam.py", + "bytes": 7168, + "sha256": "102e89470ae21392ab0e868f4df20eef19ee902cecebf4c0fcf9d937d6c29c50" + }, + { + "path": "source/scripts/ds_configs/ds_zero1_config.json", + "bytes": 400, + "sha256": "8b65ebcb8de8c3c89113695a61783cdb97c2aa5003477f0ca743aa511bc68312" + }, + { + "path": "source/scripts/ds_configs/ds_zero2_config.json", + "bytes": 428, + "sha256": "93928339afe3946e603128ed42aae1f44279f694e20be911527e6e02ab4f9c72" + }, + { + "path": "source/scripts/precompute_text_embeds.py", + "bytes": 14454, + "sha256": "6be7ade7c56287ee640683a2dceedaf0bb5fc1b783802ca6a81fe27a53017edc" + }, + { + "path": "source/scripts/prepare_galaxea_r1_lite_overlay.py", + "bytes": 5255, + "sha256": "dcab484f9105cde123d2b240d764082a7c165e2fc73af1948039bb837c51726d" + }, + { + "path": "source/scripts/preprocess_action_dit_backbone.py", + "bytes": 9948, + "sha256": "0fb11f04756d71e6b1b1d26cdf9e5831dfead339da38aa25c02f171d7f484a94" + }, + { + "path": "source/scripts/train.py", + "bytes": 358, + "sha256": "9fb7bff4bcc0eb47294b6815c547e8cf47c02532da4d58bcb072e1f18ba67ef8" + }, + { + "path": "source/scripts/train_galaxea_r1_lite.sh", + "bytes": 6889, + "sha256": "d9e6ec7442afb4346d3a53ea97553262001fe84bed21c59a00a1c8a1e1dc912a" + }, + { + "path": "source/scripts/train_zero1.sh", + "bytes": 3241, + "sha256": "0ee0031b65193953ab2c449201f9ff64574d93d7956a9d5fd6644317bdaf7654" + }, + { + "path": "source/scripts/train_zero2.sh", + "bytes": 3235, + "sha256": "b75455a0c6519b6b7339ab6a3b4c8e6b6aa882c599a6029aff2d349620bdf43a" + }, + { + "path": "source/src/fastwam/__init__.py", + "bytes": 23, + "sha256": "c4149c5ff0f9abd1517360daf761e692feb8570eb7a0caa2dfde52d4412c3e23" + }, + { + "path": "source/src/fastwam/datasets/__init__.py", + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "source/src/fastwam/datasets/dataset_utils.py", + "bytes": 3735, + "sha256": "3bc17590049c7cb54061acf8d731b07892e7f530c5d7eeac2613fa77a285a392" + }, + { + "path": "source/src/fastwam/datasets/lerobot/base_lerobot_dataset.py", + "bytes": 19162, + "sha256": "c076b4028c5414d3c08199f1501698c3a4e15ac4fe09c6c7a67fa7f462a13b4a" + }, + { + "path": "source/src/fastwam/datasets/lerobot/constants.py", + "bytes": 1915, + "sha256": "11ac9260341e1560c5eb0dd7acb79ed63c74180b8ab4113c27622ff213db417f" + }, + { + "path": "source/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py", + "bytes": 7067, + "sha256": "7d7daab219332b5abdfd5ec67ccaf3502e8eb583a17a4e482c3f3d4f7a691e89" + }, + { + "path": "source/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py", + "bytes": 33439, + "sha256": "40f1b94e6ceb50765fb8dfa3d498887c2a0fc2eb49f09770c5f36190e7110c98" + }, + { + "path": "source/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py", + "bytes": 19421, + "sha256": "cb647a1e955c3279e39ce8d7af7a37ea7432bbd71534c183f9ceebc8557dd89f" + }, + { + "path": "source/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py", + "bytes": 55233, + "sha256": "a71b26a73a0e460198b8316db2acb77b43a719d4ab9706150e2ba53773ffda64" + }, + { + "path": "source/src/fastwam/datasets/lerobot/processors/base_processor.py", + "bytes": 10811, + "sha256": "1f47bc85fee11e3d16714b2f1e3a2dc874f4609a1706a788617156d34b48b58e" + }, + { + "path": "source/src/fastwam/datasets/lerobot/processors/fastwam_processor.py", + "bytes": 13458, + "sha256": "3aa0e275b3815b67785c6e38ac5fc400472f9d09ac18b2ab269ea77bf574ade4" + }, + { + "path": "source/src/fastwam/datasets/lerobot/robot_video_dataset.py", + "bytes": 13058, + "sha256": "04f830e4c7dcc93be8e1d3514e502719aed94e62074364edbc1257e533a27de9" + }, + { + "path": "source/src/fastwam/datasets/lerobot/transforms/action_state_merger.py", + "bytes": 2522, + "sha256": "669d6991415fc5eca44b32b129d15fee1e06e2f17d59d90350c252598033b427" + }, + { + "path": "source/src/fastwam/datasets/lerobot/transforms/image.py", + "bytes": 725, + "sha256": "4d7ab61d0844918525244db653831f0f632730046b2714a6163ff57c441a16fa" + }, + { + "path": "source/src/fastwam/datasets/lerobot/transforms/misc.py", + "bytes": 422, + "sha256": "8b505bd9e3a54ab2db7efcd64f0929d2e0303f5db1123de1074fd01fa9887e2b" + }, + { + "path": "source/src/fastwam/datasets/lerobot/transforms/relative_action.py", + "bytes": 5736, + "sha256": "2b4f05297de6cd364e507b5874a752003a9dfb770a528ea73d2781c8a1a12059" + }, + { + "path": "source/src/fastwam/datasets/lerobot/transforms/rotation.py", + "bytes": 5603, + "sha256": "0de34b252d60b956cda07c0c19baebbd3c8a28916face700c6664c5815e11e91" + }, + { + "path": "source/src/fastwam/datasets/lerobot/utils/normalizer.py", + "bytes": 11274, + "sha256": "5ac89a646cd6395e98feaa832f8fdf7c7e00857080af3d9c15710e243c43ad43" + }, + { + "path": "source/src/fastwam/datasets/lerobot/utils/rotation.py", + "bytes": 21110, + "sha256": "6169162f98faf02a3fe5201e2b8a1069fe299840d036ee7cad8af5fa8f561542" + }, + { + "path": "source/src/fastwam/datasets/lerobot3/__init__.py", + "bytes": 389, + "sha256": "26f0c3a4afdbd54f723cab81d2288781844e61ee367fd88bc0a821c7929cb79f" + }, + { + "path": "source/src/fastwam/datasets/lerobot3/base_lerobot_dataset.py", + "bytes": 389, + "sha256": "7dc96f86caf8239795a0dce0161e25969ac002ebf037dfb9695608c40d89d769" + }, + { + "path": "source/src/fastwam/datasets/lerobot3/lerobot_dataset.py", + "bytes": 14998, + "sha256": "4acae53b736c47c1ed6cb42ad41a68cb0a64cf6578791c5f9e50a8dd9f5541e1" + }, + { + "path": "source/src/fastwam/datasets/lerobot3/robot_video_dataset.py", + "bytes": 310, + "sha256": "62230146222dfe0719c43f72d9f8e913cd4998d8f68a3f9d09b9b8d3db7441d5" + }, + { + "path": "source/src/fastwam/models/__init__.py", + "bytes": 35, + "sha256": "ae4c5733f4bbf7caf31df2ae6f789516f6a10cd84a321f7d0375a7034253ac6d" + }, + { + "path": "source/src/fastwam/models/wan22/__init__.py", + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "source/src/fastwam/models/wan22/action_dit.py", + "bytes": 14329, + "sha256": "631fdf598fbe06a95a41c1a27acd9de16f2187f3a2009e10690197527d18283b" + }, + { + "path": "source/src/fastwam/models/wan22/fastwam.py", + "bytes": 54376, + "sha256": "154c339efd33fae119fe902d6efb91cde7475e3cfd0e7b50e606278c1c3eac09" + }, + { + "path": "source/src/fastwam/models/wan22/fastwam_idm.py", + "bytes": 25404, + "sha256": "5ebdaabb42d4e6c0c6543760ad2ea9b3c8ef9415ebab44fbf845f73f52280bba" + }, + { + "path": "source/src/fastwam/models/wan22/fastwam_joint.py", + "bytes": 10242, + "sha256": "a61aab79cfc607af02ab816849329a13c1dcd0bbcbf63c6584a345d8a67bb2b6" + }, + { + "path": "source/src/fastwam/models/wan22/fastwam_optional_idm.py", + "bytes": 4420, + "sha256": "bd508293545895bae159a42998992cece9c1f1aca373e6f03521ea05f55749b2" + }, + { + "path": "source/src/fastwam/models/wan22/helpers/__init__.py", + "bytes": 415, + "sha256": "f26116de229557a37a4812dc4ee75971f5f9e8b80240786ea4d0e5b0ff703cbf" + }, + { + "path": "source/src/fastwam/models/wan22/helpers/gradient.py", + "bytes": 565, + "sha256": "ba5d8f7272eb029dc6cd2849ca99b70f6ad5abb838d21c818beb0590620dc793" + }, + { + "path": "source/src/fastwam/models/wan22/helpers/io.py", + "bytes": 7751, + "sha256": "1c6c1bc8a975b47092912833194fce25e2e3ac886fbab72d73af3ff5a00b3c62" + }, + { + "path": "source/src/fastwam/models/wan22/helpers/loader.py", + "bytes": 8279, + "sha256": "792f66bd0609c1365256aa24d43301340f81260c1651d89b04d2ae8224a08726" + }, + { + "path": "source/src/fastwam/models/wan22/helpers/state_dict_converters.py", + "bytes": 5226, + "sha256": "81d213373e5056a8d8bf31768b9b04164138ea78d0b306e316334841499f4bf9" + }, + { + "path": "source/src/fastwam/models/wan22/mot.py", + "bytes": 31234, + "sha256": "2383e4cfa00efa9cde889493e0b6c861bc7b17d24d731f5ff38d3ebf8ab7f541" + }, + { + "path": "source/src/fastwam/models/wan22/schedulers/scheduler_continuous.py", + "bytes": 4068, + "sha256": "2ececce284f7b9fe876965fbf5e11ca7b69caaf693979266b0f63062a589f181" + }, + { + "path": "source/src/fastwam/models/wan22/wan22.py", + "bytes": 16925, + "sha256": "b85349beeeb018d5be3f5b932bee3d640274d09c5e1e0fa0fa01e8ad0e02356e" + }, + { + "path": "source/src/fastwam/models/wan22/wan_video_dit.py", + "bytes": 34764, + "sha256": "578712e83fac2815d3e3a3fd9134178028082deba361433122fe3c5764982dd6" + }, + { + "path": "source/src/fastwam/models/wan22/wan_video_text_encoder.py", + "bytes": 11459, + "sha256": "aedf7f131556a39b99a273deec3a85d67f41b5367cef41f310f33a807ae60a87" + }, + { + "path": "source/src/fastwam/models/wan22/wan_video_vae.py", + "bytes": 49604, + "sha256": "cb2b1b53f35a1a201c640ec52d12a7c4cb4ba1df322155addd3dc409e04dc07c" + }, + { + "path": "source/src/fastwam/runtime.py", + "bytes": 22779, + "sha256": "230a814b7a42a3f74145513f5775a3c6cd56ffa244635c536b3d1f384c5f82d0" + }, + { + "path": "source/src/fastwam/trainer.py", + "bytes": 37779, + "sha256": "00f9a8b7c7c222fe8572ec2a8f26ffe5fc5779a3fe07afd77369deab6253ddd3" + }, + { + "path": "source/src/fastwam/utils/__init__.py", + "bytes": 96, + "sha256": "2893c05b99b17162941c70a41859c2808f8ae390be367a26cac7d5ab93ee197b" + }, + { + "path": "source/src/fastwam/utils/config_resolvers.py", + "bytes": 2526, + "sha256": "01c48cbd42b86e636b70ccc3e193a81147162ba890777102b2c61086fa415340" + }, + { + "path": "source/src/fastwam/utils/fs.py", + "bytes": 76, + "sha256": "69e6df236b43360428a613686c8f37c531c0d16a3a2c5b06a725c46b49d657c5" + }, + { + "path": "source/src/fastwam/utils/logging_config.py", + "bytes": 4679, + "sha256": "cadc4f6ad487c3f37171a6defad36d23d9ab8bf262f73a461cd9ee9b03b757e7" + }, + { + "path": "source/src/fastwam/utils/misc.py", + "bytes": 1223, + "sha256": "05aa51247936a8d3faac27f90b78b65ff83a688259bed55d114b6a617ced532d" + }, + { + "path": "source/src/fastwam/utils/pytorch_utils.py", + "bytes": 5051, + "sha256": "24de921fe7305b51f94222f4163c6f81eb5e077e46272314d83eb44f9f4bde71" + }, + { + "path": "source/src/fastwam/utils/samplers.py", + "bytes": 1339, + "sha256": "4d3bd9b2d498f4ad8f2cf9765d60c4bf5368fa6fa2a7d42d0db6210f66899652" + }, + { + "path": "source/src/fastwam/utils/video_io.py", + "bytes": 819, + "sha256": "9e9898b021a44e87aa1b4e6d13e3755a9869a2e27a7eb53c72a6802808d5bd80" + }, + { + "path": "source/src/fastwam/utils/video_metrics.py", + "bytes": 3446, + "sha256": "aaa37e458cc9e4bb79a17923b5ff5900c97b4309f7abd50db44cf76e9143ffd6" + }, + { + "path": "source/src/fastwam.egg-info/PKG-INFO", + "bytes": 1161, + "sha256": "e129d204188c26725819c3994455d55762913c8345b8ae1b2fdabc469a1c3420" + }, + { + "path": "source/src/fastwam.egg-info/SOURCES.txt", + "bytes": 2462, + "sha256": "dbb1f8a6580463d20f8bd783f10ba5b7a29e020ea86832f976691b855f250105" + }, + { + "path": "source/src/fastwam.egg-info/dependency_links.txt", + "bytes": 1, + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b" + }, + { + "path": "source/src/fastwam.egg-info/requires.txt", + "bytes": 521, + "sha256": "c82af4bdebdaa050271d8eecd4aca96dede56d6a42bb2cf6db0372023503a1c8" + }, + { + "path": "source/src/fastwam.egg-info/top_level.txt", + "bytes": 8, + "sha256": "8a42966a9145ab062768faad7503d31d40a654dcd024e8a8fb1885fe4bddeedb" + }, + { + "path": "validation.json", + "bytes": 700, + "sha256": "95d3f67742d93369c73ed436ad412dab078ec8094d7dab2330a4a0f0cca1bc9c" + } + ] +} diff --git a/baseline/fastwam_zipper/models/LICENSE-WAN-APACHE-2.0.txt b/baseline/fastwam_zipper/models/LICENSE-WAN-APACHE-2.0.txt new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/baseline/fastwam_zipper/models/LICENSE-WAN-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/baseline/fastwam_zipper/models/Wan2.2-TI2V-5B/Wan2.2_VAE.pth b/baseline/fastwam_zipper/models/Wan2.2-TI2V-5B/Wan2.2_VAE.pth new file mode 100644 index 0000000000000000000000000000000000000000..2264ad59a1f3be73cf56ff0dcf7d0a8e06efe8ed --- /dev/null +++ b/baseline/fastwam_zipper/models/Wan2.2-TI2V-5B/Wan2.2_VAE.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36 +size 2818839170 diff --git a/baseline/fastwam_zipper/norm_stats.json b/baseline/fastwam_zipper/norm_stats.json new file mode 100644 index 0000000000000000000000000000000000000000..9a6a0baccfdc6069609d871c68801b6d26a76955 --- /dev/null +++ b/baseline/fastwam_zipper/norm_stats.json @@ -0,0 +1,136 @@ +{ + "norm_stats": { + "state": { + "mean": [ + -0.32184191477392804, + 1.5207721980826827, + -0.7150242090298449, + -0.28362794490829607, + -0.0326558641526937, + -0.013098741542747098, + 64.66151715104193, + 0.21104983370396177, + 1.7986193640856323, + -1.03603124575021, + -0.2070677500803398, + 0.49687352638322985, + 0.6771545717346843, + 62.77788487973063 + ], + "std": [ + 0.3021247081483893, + 1.0365802698172673, + 0.6522747017660155, + 0.4776692517096447, + 0.17428160840573853, + 0.21532609078838216, + 43.57897586492507, + 0.1853564774548537, + 1.0606944989455414, + 0.7407364588573183, + 0.45290236233056375, + 0.41556147349584566, + 0.6096365556465444, + 43.38786400775888 + ], + "q01": [ + -0.9572340250015259, + 0.2140425592660904, + -2.039149045944214, + -1.4893616437911987, + -0.5359574556350708, + -0.7127659320831299, + 3.0947201251983643, + -0.0908510610461235, + 0.21531914174556732, + -2.0451064109802246, + -1.52042555809021, + -0.1923404186964035, + -0.08297872543334961, + 3.2300286293029785 + ], + "q99": [ + 0.10489361733198166, + 3.0346808433532715, + -0.0019148936262354255, + 0.6214893460273743, + 0.31340426206588745, + 0.7790850949287501, + 100.67488861083984, + 0.6489361524581909, + 3.085106372833252, + -0.001276595750823617, + 0.7149106299877201, + 1.212978720664978, + 1.943404197692871, + 101.49044036865234 + ] + }, + "actions": { + "mean": [ + -0.3302394307611646, + 1.3351088878676973, + -0.730087379438977, + -0.3051273035432853, + -0.05725393898553614, + -0.03795729405196406, + 63.83399157797273, + 0.1917094052418155, + 1.6079172755673306, + -1.0522424038321934, + -0.22567802180755175, + 0.4700642734741484, + 0.6497894015838603, + 59.816608312332825 + ], + "std": [ + 0.2976434695253568, + 1.074422013023125, + 0.6661020136243821, + 0.4836939282841582, + 0.17474814803096714, + 0.2161063544717659, + 47.573759196581726, + 0.18222145953683355, + 1.1005178847180743, + 0.7551834604102067, + 0.4582503537690994, + 0.4172776054469184, + 0.6124397993031185, + 48.446447570698396 + ], + "q01": [ + -0.9526020884513855, + 0.0, + -2.0, + -1.5201749801635742, + -0.5614369511604309, + -0.7378447651863098, + 0.0, + -0.12118448317050934, + 0.0, + -2.0, + -1.5707999467849731, + -0.22856314480304718, + -0.13499030470848083, + 0.0 + ], + "q99": [ + 0.10124272853136063, + 3.0096702575683594, + 0.0, + 0.607456386089325, + 0.29452431201934814, + 0.754718542098999, + 100.0, + 0.6181942820549011, + 3.0142722129821777, + 0.0, + 0.7148350477218628, + 1.1888351440429688, + 1.9190099239349365, + 100.27293395996094 + ] + } + } +} diff --git a/baseline/fastwam_zipper/provenance.json b/baseline/fastwam_zipper/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..d9bf8b51a3b3d59b2917250924d39ef70b461227 --- /dev/null +++ b/baseline/fastwam_zipper/provenance.json @@ -0,0 +1,20 @@ +{ + "training_job": "bifrost-2026102616142400-shouqx", + "task": "zipper_pulling", + "final_step": 16380, + "epochs": 30, + "filename": "step_016380.pt", + "bytes": 12041813369, + "sha256": "5e574c8cc7bb5747852cd815cfed9896d8d0f1db83eee28a367bd21beb25f348", + "checkpoint_keys": [ + "mot", + "step", + "torch_dtype", + "proprio_encoder" + ], + "torch_dtype": "torch.bfloat16", + "tensor_elements": 6020749518, + "training_status": "JOB_COMPLETE", + "uploaded_as": "original unchanged final weights", + "verified_utc": "2026-10-27T09:34:00.717440+00:00" +} diff --git a/baseline/fastwam_zipper/requirements-inference.txt b/baseline/fastwam_zipper/requirements-inference.txt new file mode 100644 index 0000000000000000000000000000000000000000..dd6ede8d6b34b8107241f5996a1e254d6a833113 --- /dev/null +++ b/baseline/fastwam_zipper/requirements-inference.txt @@ -0,0 +1,12 @@ +torch==2.7.1 +torchvision==0.22.1 +numpy==2.2.6 +pillow==12.0.0 +einops==0.8.1 +safetensors==0.5.3 +transformers==4.49.0 +regex==2025.11.3 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +tqdm==4.68.3 +rich==14.2.0 diff --git a/baseline/fastwam_zipper/source/LICENSE b/baseline/fastwam_zipper/source/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..90124d129334cd5de11aa0fd606182b3ffd442cb --- /dev/null +++ b/baseline/fastwam_zipper/source/LICENSE @@ -0,0 +1,24 @@ +MIT License + +Copyright (c) 2026 The FastWAM Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +Except for third-party code and files that retain their original license +headers, this repository is released under the MIT License. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/baseline/fastwam_zipper/source/README.md b/baseline/fastwam_zipper/source/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b691cbb230c72a6b0220afc99e9d89992c5ab07b --- /dev/null +++ b/baseline/fastwam_zipper/source/README.md @@ -0,0 +1,437 @@ +# FastWAM + +Official codebase for **Fast-WAM: Do World Action Models Need Test-time Future Imagination?** + +[![English](https://img.shields.io/badge/README-English-111111.svg)](./README.md) +[![中文](https://img.shields.io/badge/README-%E4%B8%AD%E6%96%87-d14836.svg)](./README_zh.md) + +[![arXiv](https://img.shields.io/badge/arXiv-2603.16666-b31b1b.svg)](https://arxiv.org/abs/2603.16666) +[![Project Page](https://img.shields.io/badge/Project_Page-Fast--WAM-2ea44f.svg)](https://yuantianyuan01.github.io/FastWAM/) +[![Hugging Face Model](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Model-f7c843)](https://huggingface.co/yuanty/fastwam) +[![Hugging Face Dataset - LIBERO](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Dataset%20LIBERO-f7c843)](https://huggingface.co/datasets/yuanty/LIBERO-fastwam) +[![Hugging Face Dataset - RoboTwin](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Dataset%20RoboTwin-f7c843)](https://huggingface.co/datasets/yuanty/robotwin2.0-fastwam) + +This repository contains the training and evaluation code for FastWAM on LIBERO / RoboTwin. + +## What's New + +FastWAM is now faster, better suited to large-scale datasets, and more flexible +for research. This update brings substantially faster training and inference, +native LeRobot v3.0 support, and a new model that can switch between acting with +and without future imagination. + +### ⚡ Approximately 2x faster inference + +End-to-end FastWAM inference is now approximately **2x faster**, including text +encoding and VAE encoding: + +- **NVIDIA H20:** 470 ms → 210 ms +- **NVIDIA RTX 4090:** 190 ms → 110 ms + +The accelerated path is enabled by default for LIBERO with +`EVALUATION.compile_action_infer=true`. We gratefully acknowledge +[PR #43](https://github.com/yuantianyuan01/FastWAM/pull/43) for proposing the +optimization ideas that inspired this work. Existing checkpoints remain fully +compatible with the accelerated inference path. + +### 🚀 Approximately 10% faster training + +FastWAM training is approximately **10% faster on NVIDIA H20 GPUs**. The new +training path combines a compiled denoising core with batched VAE encoding and +a lightweight CUDA Graph backend. Enable denoising compilation with: + +```bash +bash scripts/train_zero1.sh 8 task=libero_uncond_2cam224_1e-4 \ + model.compile_training_denoise=true +``` + +Both cached text embeddings and on-the-fly T5 encoding are supported. The +latter skips text-cache preprocessing and is more convenient, at the cost of +approximately 10% lower training throughput. + +### 📦 Native LeRobot 2.1 and 3.0 support + +FastWAM now supports both **LeRobot 2.1 and LeRobot 3.0** datasets. LeRobot 3.0's +chunked parquet and video layout scales better to large datasets, with faster +data loading and dataset-statistics computation as the dataset grows. + +Download the released LeRobot 3.0 LIBERO dataset from +[Hugging Face](https://huggingface.co/datasets/yuanty/LIBERO-fastwam) and select +the v3.0 data config: + +```bash +huggingface-cli download yuanty/LIBERO-fastwam \ + --repo-type dataset \ + --include "lerobot_v30/**" \ + --local-dir ./data + +python scripts/train.py task=libero_uncond_2cam224_1e-4 \ + data=libero_2cam_lerobot_v30 +``` + +For another LeRobot 3.0 dataset, copy +`configs/data/libero_2cam_lerobot_v30.yaml`, update `train.dataset_dirs`, and +select the new config with `data=`. Existing LeRobot 2.1 configs +continue to work unchanged. + +### 🧠 Optional IDM: one model, two thinking modes + +Optional IDM is a new FastWAM variant that supports **two inference modes in a +single model**: + +- **IDM mode:** imagine the future video first, then predict actions. +- **First-frame mode (Fast-WAM):** skip test-time future imagination and predict + actions directly from the current observation. + +Download the released Optional IDM checkpoint from +[Hugging Face](https://huggingface.co/yuanty/fastwam): + +```bash +huggingface-cli download yuanty/fastwam \ + libero_optional_idm_2cam224.pt \ + libero_optional_idm_2cam224_dataset_stats.json \ + --local-dir ./checkpoints/fastwam_release +``` + +Train the optional-IDM variant once: + +```bash +bash scripts/train_zero1.sh 8 task=libero_optional_idm_2cam224_1e-4 +``` + +Then choose either inference mode at evaluation time without retraining, making +it easy to study when future imagination helps: + +```bash +python experiments/libero/run_libero_manager.py \ + task=libero_optional_idm_2cam224_1e-4 \ + ckpt=./checkpoints/fastwam_release/libero_optional_idm_2cam224.pt \ + EVALUATION.dataset_stats_path=./checkpoints/fastwam_release/libero_optional_idm_2cam224_dataset_stats.json \ + EVALUATION.sigma_shift=1.0 \ + +EVALUATION.action_infer_mode=idm \ + MULTIRUN.num_gpus=8 +``` + +Replace `idm` with `first_frame` to use the Fast-WAM inference mode. + +The released checkpoint, trained with action scheduler shift `1.0`, achieves +the following success rates on the full LIBERO benchmark (40 tasks, 50 episodes +per task): + +| Inference mode | Spatial | Goal | Object | Long | Average | +| --- | ---: | ---: | ---: | ---: | ---: | +| IDM | 99.0% | 98.6% | 99.6% | 97.0% | **98.55%** | +| First-frame (Fast-WAM) | 98.2% | 97.8% | 99.2% | 95.8% | **97.75%** | + +### Other improvements + +- The action scheduler shift now defaults to `1.0` for both training and + evaluation; shifts from `1.0` to `3.0` perform similarly in our experiments. + When evaluating the original released checkpoints, set + `EVALUATION.sigma_shift=5.0` to reproduce the original setting. +- Upgraded LIBERO evaluation with persistent model workers, dynamic task + scheduling, bad-GPU quarantine, failure recovery, and resumable results. +- Optimized action-only inference for IDM and Optional IDM: `infer_action` + returns actions and video latents directly, while VAE decoding is performed + only by `infer_joint` when video output is requested, eliminating redundant + computation in action-only deployments. + +## Index + +- [File Structure](#file-structure) +- [Environment Setup](#environment-setup) +- [Model Preparation](#model-preparation) +- [Dataset Download](#dataset-download) +- [Inference with Released Checkpoints](#inference-with-released-checkpoints) +- [Training](#training) +- [Inference with Your Trained Checkpoints](#inference-with-your-trained-checkpoints) +- [Acknowledgements](#acknowledgements) +- [BibTeX](#bibtex) + +## File Structure + +```text +FastWAM/ +├── configs/ +│ ├── data/ # Dataset configs (LIBERO, RoboTwin, etc.) +│ ├── model/ # Model architecture and component configs +│ └── task/ # Task-level configs (training task names) +├── scripts/ +│ ├── train.py +│ ├── train_zero1.sh # Deepspeed zero1 training entrypoint +│ ├── preprocess_action_dit_backbone.py # Preprocess ActionDiT backbone before training +│ └── precompute_text_embeds.py # Precompute T5 text embedding cache before training +├── experiments/ +│ ├── libero/ +│ │ └── run_libero_manager.py +│ └── robotwin/ +│ └── run_robotwin_manager.py +├── src/fastwam/ # Core code +├── runs/ # Training outputs (ckpt, logs) +├── checkpoints/ # Pretrained or external checkpoints +├── data/ # Data directory +└── evaluate_results/ # Inference / evaluation results +``` + +## Environment Setup + +```bash +conda create -n fastwam python=3.10 -y +conda activate fastwam +pip install -U pip +pip install torch==2.7.1+cu128 torchvision==0.22.1+cu128 --extra-index-url https://download.pytorch.org/whl/cu128 +pip install -e . +``` + +## Model Preparation + +This step is required before both training and inference. + +Step 1: set the Wan model directory first (opional, default `./checkpoints`): + +```bash +mkdir -p checkpoints +export DIFFSYNTH_MODEL_BASE_PATH="$(pwd)/checkpoints" +``` + +Step 2: pre-generate the ActionDiT backbone (interpolated from Wan22 DiT): + +```bash +# uncond (fastwam) +python scripts/preprocess_action_dit_backbone.py \ + --model-config configs/model/fastwam.yaml \ + --output checkpoints/ActionDiT_linear_interp_Wan22_alphascale_1024hdim.pt \ + --device cuda \ + --dtype bfloat16 +``` + +## Dataset Download + +### LIBERO + +The preprocessed LIBERO dataset used by Fast-WAM is available at: + +- https://huggingface.co/datasets/yuanty/LIBERO-fastwam + +Download all compressed files first, then extract them all: + +```bash +mkdir -p data/libero_mujoco3.3.2 +cd data/libero_mujoco3.3.2 + +# Run after downloading all 4 tar.gz files +for f in *.tar.gz; do + tar -xzf "$f" +done +``` + +The extracted directory structure should be: + +```text +data/libero_mujoco3.3.2/ +├── libero_10_no_noops_lerobot/ +├── libero_goal_no_noops_lerobot/ +├── libero_object_no_noops_lerobot/ +└── libero_spatial_no_noops_lerobot/ +``` + +### RoboTwin + +The preprocessed RoboTwin dataset used by Fast-WAM is available at: + +- https://huggingface.co/datasets/yuanty/robotwin2.0-fastwam + +Download all split archive files first, then concatenate and extract: + +```bash +mkdir -p data/robotwin2.0 +cd data/robotwin2.0 + +# Run after downloading all robotwin2.0.tar.gz.part-* files +cat robotwin2.0.tar.gz.part-* | tar -xzf - +``` + +The extracted directory structure should be: + +```text +data/robotwin2.0/ +└── robotwin2.0/ + ├── data/ + ├── meta/ + └── videos/ +``` + +If you also keep: + +```text +data/robotwin2.0/dataset_stats.json +``` + +in the root directory, it can be used directly as the statistics file for the current configs in this repo. You can also recompute it. + +## Inference with Released Checkpoints + +The released checkpoints and their corresponding dataset stats are available on [Hugging Face](https://huggingface.co/yuanty/fastwam). + +Optional: download released checkpoints and dataset stats from Hugging Face: + +```bash +pip install -U huggingface_hub + +huggingface-cli download yuanty/fastwam \ + libero_uncond_2cam224.pt \ + libero_uncond_2cam224_dataset_stats.json \ + libero_optional_idm_2cam224.pt \ + libero_optional_idm_2cam224_dataset_stats.json \ + robotwin_uncond_3cam_384.pt \ + robotwin_uncond_3cam_384_dataset_stats.json \ + --local-dir ./checkpoints/fastwam_release +``` + +After downloading, the local directory is expected to contain: + +```text +checkpoints/fastwam_release/ +├── libero_uncond_2cam224.pt +├── libero_uncond_2cam224_dataset_stats.json +├── libero_optional_idm_2cam224.pt +├── libero_optional_idm_2cam224_dataset_stats.json +├── robotwin_uncond_3cam_384.pt +└── robotwin_uncond_3cam_384_dataset_stats.json +``` + +Before running the `LIBERO` benchmark, install the official LIBERO environment first +from the [LIBERO repository](https://github.com/Lifelong-Robot-Learning/LIBERO). +Then run this final step: + +```bash +pip install mujoco==3.3.2 +``` + +The `mujoco` environment should ideally stay consistent with the LIBERO data version. + +We have already copied the `RoboTwin` evaluation-related code into `third_party/RoboTwin`. +You still need to follow the official RoboTwin instructions from the +[RoboTwin repository](https://github.com/RoboTwin-Platform/RoboTwin) to finish environment installation and download the required assets, then create the policy symlink: + +```bash +ln -sfn "$(pwd)/experiments/robotwin/fastwam_policy" "$(pwd)/third_party/RoboTwin/policy/fastwam_policy" +``` + +Optional: evaluate released LIBERO checkpoint: + +The released `LIBERO` / `RoboTwin` evaluation managers default to `8` GPUs +(`MULTIRUN.num_gpus=8` in `configs/sim_libero.yaml` and `configs/sim_robotwin.yaml`). +If you want to evaluate with fewer GPUs, pass a smaller value such as +`MULTIRUN.num_gpus=4`. + +```bash +python experiments/libero/run_libero_manager.py \ + task=libero_uncond_2cam224_1e-4 \ + ckpt=./checkpoints/fastwam_release/libero_uncond_2cam224.pt \ + EVALUATION.dataset_stats_path=./checkpoints/fastwam_release/libero_uncond_2cam224_dataset_stats.json \ + EVALUATION.sigma_shift=5.0 \ + MULTIRUN.num_gpus=8 +``` + +Optional: evaluate released RoboTwin checkpoint: + +```bash +python experiments/robotwin/run_robotwin_manager.py \ + task=robotwin_uncond_3cam_384_1e-4 \ + ckpt=./checkpoints/fastwam_release/robotwin_uncond_3cam_384.pt \ + EVALUATION.dataset_stats_path=./checkpoints/fastwam_release/robotwin_uncond_3cam_384_dataset_stats.json \ + EVALUATION.sigma_shift=5.0 \ + MULTIRUN.num_gpus=8 +``` + +For faster RoboTwin evaluation, we have enabled `EVALUATION.skip_get_obs_within_replan=true` in [`configs/sim_robotwin.yaml`](./configs/sim_robotwin.yaml). +This skips RGB rendering while consecutively executing an action chunk within one replan window, which speeds up evaluation but makes the saved video look very low-FPS. +Set it to `false` if you want to save a fully rendered video. + +**Note:** We evaluate with **unseen** instructions, following Motus. [Lingbot-VA](https://github.com/Robbyant/lingbot-va/blob/661d52a59dc634a650efcd10a79d06bbb17ea81f/evaluation/robotwin/eval_polict_client_openpi.py#L308) uses **seen** instructions instead. You can try `EVALUATION.instruction_type=seen` to use **seen** instructions, which should theoretically improve performance by one or two points. + +## Training + +### 1) Precompute T5 embedding cache before training + +Use `scripts/precompute_text_embeds.py` to precompute embeddings for each training task: + +```bash +# LIBERO +python scripts/precompute_text_embeds.py task=libero_uncond_2cam224_1e-4 + +# RoboTwin +python scripts/precompute_text_embeds.py task=robotwin_uncond_3cam_384_1e-4 +``` + +For multi-GPU: + +```bash +torchrun --standalone --nproc_per_node=8 scripts/precompute_text_embeds.py task=libero_uncond_2cam224_1e-4 +``` + +### 2) Training (using `fastwam` as an example) + +When running a new task for the first time, set `pretrained_norm_stats` in the corresponding `configs/data/*.yaml` to `null` first. +After one training run, a `dataset_stats.json` file will be generated in the current run directory (for example, `runs/{task_name}/{run_id}/dataset_stats.json`). +You can then update `pretrained_norm_stats` to that file path for subsequent runs. + +```bash +# LIBERO +bash scripts/train_zero1.sh 8 task=libero_uncond_2cam224_1e-4 + +# RoboTwin +bash scripts/train_zero1.sh 8 task=robotwin_uncond_3cam_384_1e-4 +``` + +For LIBERO, we train on a single node with 8 GPUs. For RoboTwin, we use 64 GPUs to accelerate training. You can try reducing the GPU count or training epochs. + +## Inference with Your Trained Checkpoints + +The `mujoco` environment should ideally stay consistent with the LIBERO data version. Then run LIBERO evaluation: + +```bash +# LIBERO +python experiments/libero/run_libero_manager.py task={task_name} ckpt={ckpt_path} +``` + +We have already copied the `RoboTwin` evaluation-related code into `third_party/RoboTwin`. +You still need to follow the official RoboTwin instructions from the +[RoboTwin repository](https://github.com/RoboTwin-Platform/RoboTwin). +Finish installation and download the required assets, then create the policy symlink: + +```bash +ln -sfn "$(pwd)/experiments/robotwin/fastwam_policy" "$(pwd)/third_party/RoboTwin/policy/fastwam_policy" +``` + +Then run RoboTwin evaluation: + +```bash +python experiments/robotwin/run_robotwin_manager.py task={task_name} ckpt={ckpt_path} +``` + +Common `task_name` examples: + +```text +libero_uncond_2cam224_1e-4 +robotwin_uncond_3cam_384_1e-4 +``` + +## Acknowledgements + +The RoboTwin evaluation code in this repository is adapted from the official [RoboTwin repository](https://github.com/RoboTwin-Platform/RoboTwin). We thank the RoboTwin team for releasing their codebase and assets. + +## BibTeX + +If you find our work helpful, please consider citing: + +```bibtex +@article{yuan2026fastwam, + title={Fast-WAM: Do World Action Models Need Test-time Future Imagination?}, + author={Tianyuan Yuan and Zibin Dong and Yicheng Liu and Hang Zhao}, + journal={arXiv preprint arXiv:2603.16666}, + year={2026}, + url={https://arxiv.org/abs/2603.16666} +} +``` diff --git a/baseline/fastwam_zipper/source/configs/data/galaxea_r1_lite_3cam.yaml b/baseline/fastwam_zipper/source/configs/data/galaxea_r1_lite_3cam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c78202c2ae4c73050e8483d0ec22278bfc1b5c9d --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/data/galaxea_r1_lite_3cam.yaml @@ -0,0 +1,87 @@ +train: + _target_: fastwam.datasets.lerobot.robot_video_dataset.RobotVideoDataset + dataset_dirs: + - ${oc.env:GALAXEA_FASTWAM_TRAIN_DATA,/dataset_rc_o1_p1/qshou@xiaopeng.com/dataset/galaxea_real_baselines_derived/fastwam_lerobot_v2/train} + shape_meta: &shape_meta + images: + - key: head_left + raw_shape: [3, 480, 640] + shape: [3, 240, 320] + - key: wrist_left + raw_shape: [3, 480, 640] + shape: [3, 240, 320] + - key: wrist_right + raw_shape: [3, 480, 640] + shape: [3, 240, 320] + action: + - key: default + raw_shape: 14 + shape: 14 + state: + - key: default + raw_shape: 14 + shape: 14 + # 25 native 20 Hz steps give the shared 24-action horizon. Sampling every + # third image yields 9 video frames, satisfying Wan's 4n+1 contract. + num_frames: 25 + global_sample_stride: 1 + action_video_freq_ratio: 3 + video_size: [384, 320] + camera_key: null + val_set_proportion: 0.0 + is_training_set: true + skip_padding_as_possible: false + concat_multi_camera: robotwin + tolerance_s: 0.001 + video_backend: pyav + processor: &processor + _target_: fastwam.datasets.lerobot.processors.fastwam_processor.FastWAMProcessor + shape_meta: ${data.train.shape_meta} + num_obs_steps: ${data.train.num_frames} + num_output_cameras: 3 + action_output_dim: 14 + proprio_output_dim: 14 + action_state_transforms: + - _target_: fastwam.datasets.lerobot.transforms.relative_action.MaskedRelativeJointTransform + masks: + default: [true, true, true, true, true, true, false, true, true, true, true, true, true, false] + use_stepwise_action_norm: false + norm_default_mode: z-score + norm_exception_mode: null + action_state_merger: + _target_: fastwam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign + train_transforms: + - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor + - _target_: torchvision.transforms.Resize + size: [240, 320] + val_transforms: + - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor + - _target_: torchvision.transforms.Resize + size: [240, 320] + text_embedding_cache_dir: ${oc.env:GALAXEA_FASTWAM_TEXT_CACHE,/dataset_rc_o1_p1/qshou@xiaopeng.com/dataset/galaxea_real_baselines_derived/fastwam_text_cache} + use_text_embed_cache: true + context_len: 128 + +val: + _target_: fastwam.datasets.lerobot.robot_video_dataset.RobotVideoDataset + dataset_dirs: + - ${oc.env:GALAXEA_FASTWAM_VAL_DATA,/dataset_rc_o1_p1/qshou@xiaopeng.com/dataset/galaxea_real_baselines_derived/fastwam_lerobot_v2/validation} + shape_meta: *shape_meta + num_frames: 25 + global_sample_stride: 1 + action_video_freq_ratio: 3 + video_size: [384, 320] + camera_key: null + val_set_proportion: 0.0 + is_training_set: false + skip_padding_as_possible: false + concat_multi_camera: robotwin + tolerance_s: 0.001 + video_backend: pyav + processor: + <<: *processor + shape_meta: ${data.val.shape_meta} + num_obs_steps: ${data.val.num_frames} + text_embedding_cache_dir: ${data.train.text_embedding_cache_dir} + use_text_embed_cache: true + context_len: 128 diff --git a/baseline/fastwam_zipper/source/configs/data/libero_2cam.yaml b/baseline/fastwam_zipper/source/configs/data/libero_2cam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13976f9ebf004fe9f2c63663deadc9da88d7b5d0 --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/data/libero_2cam.yaml @@ -0,0 +1,68 @@ +train: + _target_: fastwam.datasets.lerobot.robot_video_dataset.RobotVideoDataset + dataset_dirs: + - ./data/libero_mujoco3.3.2/libero_spatial_no_noops_lerobot + - ./data/libero_mujoco3.3.2/libero_object_no_noops_lerobot + - ./data/libero_mujoco3.3.2/libero_goal_no_noops_lerobot + - ./data/libero_mujoco3.3.2/libero_10_no_noops_lerobot + shape_meta: + images: + - key: image + # - key: observation.images.image + raw_shape: [3, 512, 512] + shape: [3, 224, 224] + - key: wrist_image + # - key: observation.images.wrist_image + raw_shape: [3, 512, 512] + shape: [3, 224, 224] + action: + - key: default + # - key: action + raw_shape: 7 # before transform + shape: 7 # after transform + state: + - key: default + # - key: observation.state + raw_shape: 8 # before transform + shape: 8 # after transform + num_frames: 33 + global_sample_stride: 1 + action_video_freq_ratio: 4 # 32 action, 9 video frames + video_size: [224, 448] # final resize video + camera_key: null + val_set_proportion: 0.0 + is_training_set: true + skip_padding_as_possible: false + concat_multi_camera: "horizontal" + processor: + _target_: fastwam.datasets.lerobot.processors.fastwam_processor.FastWAMProcessor + shape_meta: ${data.train.shape_meta} + num_obs_steps: ${data.train.num_frames} + num_output_cameras: 2 + action_output_dim: 7 # eef_pose (6) + gripper (1) + proprio_output_dim: 8 # eef_pose (6) + gripper (2) + delta_action_dim_mask: + default: [true, true, true, true, true, true, false] # eef poses are delta, gripper is not + + # action & state normalization + action_state_transforms: null + # action_state_transforms: + # - _target_: fastwam.datasets.lerobot.transforms.libero.LiberoTransform + use_stepwise_action_norm: False + norm_default_mode: min/max + norm_exception_mode: null + + action_state_merger: + _target_: fastwam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign + + train_transforms: + - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor + - _target_: torchvision.transforms.Resize + size: [224, 224] + val_transforms: + - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor + - _target_: torchvision.transforms.Resize + size: [224, 224] + text_embedding_cache_dir: ./data/text_embeds_cache/libero + use_text_embed_cache: true + context_len: 128 diff --git a/baseline/fastwam_zipper/source/configs/data/libero_2cam_lerobot_v30.yaml b/baseline/fastwam_zipper/source/configs/data/libero_2cam_lerobot_v30.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83c517b4aa6c26f0fb8acb9f3804d35a5139286e --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/data/libero_2cam_lerobot_v30.yaml @@ -0,0 +1,59 @@ +train: + _target_: fastwam.datasets.lerobot3.robot_video_dataset.RobotVideoDataset + dataset_dirs: + - ./data/lerobot_v30/libero_spatial_no_noops_lerobot + - ./data/lerobot_v30/libero_object_no_noops_lerobot + - ./data/lerobot_v30/libero_goal_no_noops_lerobot + - ./data/lerobot_v30/libero_10_no_noops_lerobot + shape_meta: + images: + - key: image + raw_shape: [3, 512, 512] + shape: [3, 224, 224] + - key: wrist_image + raw_shape: [3, 512, 512] + shape: [3, 224, 224] + action: + - key: default + raw_shape: 7 + shape: 7 + state: + - key: default + raw_shape: 8 + shape: 8 + num_frames: 33 + global_sample_stride: 1 + action_video_freq_ratio: 4 + video_size: [224, 448] + camera_key: null + val_set_proportion: 0.0 + is_training_set: true + skip_padding_as_possible: false + concat_multi_camera: "horizontal" + processor: + _target_: fastwam.datasets.lerobot.processors.fastwam_processor.FastWAMProcessor + shape_meta: ${data.train.shape_meta} + num_obs_steps: ${data.train.num_frames} + num_output_cameras: 2 + action_output_dim: 7 + proprio_output_dim: 8 + delta_action_dim_mask: + default: [true, true, true, true, true, true, false] + action_state_transforms: null + use_stepwise_action_norm: false + norm_default_mode: min/max + norm_exception_mode: null + action_state_merger: + _target_: fastwam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign + train_transforms: + - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor + - _target_: torchvision.transforms.Resize + size: [224, 224] + val_transforms: + - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor + - _target_: torchvision.transforms.Resize + size: [224, 224] + text_embedding_cache_dir: ./data/text_embeds_cache/libero + use_text_embed_cache: true + context_len: 128 + video_backend: pyav diff --git a/baseline/fastwam_zipper/source/configs/data/robotwin.yaml b/baseline/fastwam_zipper/source/configs/data/robotwin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35ab22fa56da0e983e4db0fb423dacce1e5b4242 --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/data/robotwin.yaml @@ -0,0 +1,125 @@ +train: + _target_: fastwam.datasets.lerobot.robot_video_dataset.RobotVideoDataset + dataset_dirs: + - ./data/robotwin2.0/robotwin2.0 + shape_meta: + images: + - key: cam_high + raw_shape: [3, 480, 640] + shape: [3, 240, 320] + - key: cam_left_wrist + raw_shape: [3, 480, 640] + shape: [3, 240, 320] + - key: cam_right_wrist + raw_shape: [3, 480, 640] + shape: [3, 240, 320] + action: + - key: default + raw_shape: 14 # before transform + shape: 14 # after transform + state: + - key: default + raw_shape: 14 # before transform + shape: 14 # after transform + num_frames: 33 + global_sample_stride: 1 + action_video_freq_ratio: 4 # 32 action, 9 video frames + video_size: [384, 320] # final resize video + camera_key: null + val_set_proportion: 0.01 + is_training_set: true + pretrained_norm_stats: ./data/robotwin2.0/dataset_stats.json + skip_padding_as_possible: false + concat_multi_camera: "robotwin" + processor: + _target_: fastwam.datasets.lerobot.processors.fastwam_processor.FastWAMProcessor + shape_meta: ${data.train.shape_meta} + num_obs_steps: ${data.train.num_frames} + num_output_cameras: 3 + action_output_dim: 14 + proprio_output_dim: 14 + + # action & state normalization + action_state_transforms: null + # action_state_transforms: + # - _target_: fastwam.datasets.lerobot.transforms.libero.LiberoTransform + use_stepwise_action_norm: False + norm_default_mode: "z-score" + norm_exception_mode: null + + action_state_merger: + _target_: fastwam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign + + train_transforms: + - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor + - _target_: torchvision.transforms.Resize + size: [240, 320] + val_transforms: + - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor + - _target_: torchvision.transforms.Resize + size: [240, 320] + text_embedding_cache_dir: ./data/text_embeds_cache/robotwin + context_len: 128 + +val: + _target_: fastwam.datasets.lerobot.robot_video_dataset.RobotVideoDataset + dataset_dirs: + - ./data/robotwin2.0/robotwin2.0/ + shape_meta: + images: + - key: cam_high + raw_shape: [3, 480, 640] + shape: [3, 240, 320] + - key: cam_left_wrist + raw_shape: [3, 480, 640] + shape: [3, 240, 320] + - key: cam_right_wrist + raw_shape: [3, 480, 640] + shape: [3, 240, 320] + action: + - key: default + raw_shape: 14 # before transform + shape: 14 # after transform + state: + - key: default + raw_shape: 14 # before transform + shape: 14 # after transform + num_frames: 33 + global_sample_stride: 1 + action_video_freq_ratio: 4 # 32 action, 9 video frames + video_size: [384, 320] # final resize video + camera_key: null + val_set_proportion: 0.01 + is_training_set: false + pretrained_norm_stats: ./data/robotwin2.0/dataset_stats.json + skip_padding_as_possible: false + concat_multi_camera: "robotwin" + processor: + _target_: fastwam.datasets.lerobot.processors.fastwam_processor.FastWAMProcessor + shape_meta: ${data.train.shape_meta} + num_obs_steps: ${data.train.num_frames} + num_output_cameras: 3 + action_output_dim: 14 + proprio_output_dim: 14 + + # action & state normalization + action_state_transforms: null + # action_state_transforms: + # - _target_: fastwam.datasets.lerobot.transforms.libero.LiberoTransform + use_stepwise_action_norm: False + norm_default_mode: "z-score" + norm_exception_mode: null + + action_state_merger: + _target_: fastwam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign + + train_transforms: + - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor + - _target_: torchvision.transforms.Resize + size: [240, 320] + val_transforms: + - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor + - _target_: torchvision.transforms.Resize + size: [240, 320] + text_embedding_cache_dir: ./data/text_embeds_cache/robotwin + context_len: 128 \ No newline at end of file diff --git a/baseline/fastwam_zipper/source/configs/model/fastwam.yaml b/baseline/fastwam_zipper/source/configs/model/fastwam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..296f9e139befbcd8d9069789b83f93a21be4030f --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/model/fastwam.yaml @@ -0,0 +1,59 @@ +_target_: fastwam.runtime.create_fastwam +model_id: Wan-AI/Wan2.2-TI2V-5B +tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B +tokenizer_max_len: 128 +load_text_encoder: false +proprio_dim: ${data.train.processor.proprio_output_dim} +redirect_common_files: true +mot_checkpoint_mixed_attn: false +compile_training_denoise: false +action_dit_pretrained_path: checkpoints/ActionDiT_linear_interp_Wan22_alphascale_1024hdim.pt +skip_dit_load_from_pretrain: false + +video_dit_config: + has_image_input: false + patch_size: [1, 2, 2] + in_dim: 48 + hidden_dim: 3072 + ffn_dim: 14336 + freq_dim: 256 + text_dim: 4096 + out_dim: 48 + num_heads: 24 + attn_head_dim: 128 + num_layers: 30 + eps: 1.0e-06 + seperated_timestep: true + require_clip_embedding: false + require_vae_embedding: false + fuse_vae_embedding_in_latents: true + use_gradient_checkpointing: ${model.mot_checkpoint_mixed_attn} + video_attention_mask_mode: "first_frame_causal" # default + action_conditioned: false + action_dim: ${data.train.processor.action_output_dim} + action_group_causal_mask_mode: "group_diagonal" + +action_dit_config: + action_dim: ${data.train.processor.action_output_dim} + hidden_dim: 1024 + ffn_dim: 4096 + num_heads: 24 + attn_head_dim: 128 + num_layers: 30 + text_dim: 4096 + freq_dim: 256 + eps: 1.0e-06 + use_gradient_checkpointing: ${model.mot_checkpoint_mixed_attn} + +video_scheduler: + train_shift: 5.0 + infer_shift: 5.0 + num_train_timesteps: 1000 + +action_scheduler: + train_shift: 1.0 + infer_shift: 1.0 + num_train_timesteps: 1000 + +loss: + lambda_action: 1.0 diff --git a/baseline/fastwam_zipper/source/configs/model/fastwam_idm.yaml b/baseline/fastwam_zipper/source/configs/model/fastwam_idm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..403c245fa57bb075d7b80be72fe1c94dd958ad3e --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/model/fastwam_idm.yaml @@ -0,0 +1,60 @@ +_target_: fastwam.runtime.create_fastwam_idm +model_id: Wan-AI/Wan2.2-TI2V-5B +tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B +tokenizer_max_len: 128 +load_text_encoder: false +proprio_dim: ${data.train.processor.proprio_output_dim} +redirect_common_files: true +mot_checkpoint_mixed_attn: false +compile_training_denoise: false +video_cond_noise_prob: 0.5 +action_dit_pretrained_path: checkpoints/ActionDiT_linear_interp_Wan22_alphascale_1024hdim.pt +skip_dit_load_from_pretrain: false + +video_dit_config: + has_image_input: false + patch_size: [1, 2, 2] + in_dim: 48 + hidden_dim: 3072 + ffn_dim: 14336 + freq_dim: 256 + text_dim: 4096 + out_dim: 48 + num_heads: 24 + attn_head_dim: 128 + num_layers: 30 + eps: 1.0e-06 + seperated_timestep: true + require_clip_embedding: false + require_vae_embedding: false + fuse_vae_embedding_in_latents: true + use_gradient_checkpointing: ${model.mot_checkpoint_mixed_attn} + video_attention_mask_mode: "first_frame_causal" # default + action_conditioned: false + action_dim: ${data.train.processor.action_output_dim} + action_group_causal_mask_mode: "group_diagonal" + +action_dit_config: + action_dim: ${data.train.processor.action_output_dim} + hidden_dim: 1024 + ffn_dim: 4096 + num_heads: 24 + attn_head_dim: 128 + num_layers: 30 + text_dim: 4096 + freq_dim: 256 + eps: 1.0e-06 + use_gradient_checkpointing: ${model.mot_checkpoint_mixed_attn} + +video_scheduler: + train_shift: 5.0 + infer_shift: 5.0 + num_train_timesteps: 1000 + +action_scheduler: + train_shift: 1.0 + infer_shift: 1.0 + num_train_timesteps: 1000 + +loss: + lambda_action: 1.0 diff --git a/baseline/fastwam_zipper/source/configs/model/fastwam_joint.yaml b/baseline/fastwam_zipper/source/configs/model/fastwam_joint.yaml new file mode 100644 index 0000000000000000000000000000000000000000..426ba154400178c0cff116e1378705d1dfa169b9 --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/model/fastwam_joint.yaml @@ -0,0 +1,58 @@ +_target_: fastwam.runtime.create_fastwam_joint +model_id: Wan-AI/Wan2.2-TI2V-5B +tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B +tokenizer_max_len: 128 +load_text_encoder: false +proprio_dim: ${data.train.processor.proprio_output_dim} +redirect_common_files: true +mot_checkpoint_mixed_attn: false +action_dit_pretrained_path: checkpoints/ActionDiT_linear_interp_Wan22_alphascale_1024hdim.pt +skip_dit_load_from_pretrain: false + +video_dit_config: + has_image_input: false + patch_size: [1, 2, 2] + in_dim: 48 + hidden_dim: 3072 + ffn_dim: 14336 + freq_dim: 256 + text_dim: 4096 + out_dim: 48 + num_heads: 24 + attn_head_dim: 128 + num_layers: 30 + eps: 1.0e-06 + seperated_timestep: true + require_clip_embedding: false + require_vae_embedding: false + fuse_vae_embedding_in_latents: true + use_gradient_checkpointing: ${model.mot_checkpoint_mixed_attn} + video_attention_mask_mode: "first_frame_causal" # default + action_conditioned: false + action_dim: ${data.train.processor.action_output_dim} + action_group_causal_mask_mode: "group_diagonal" + +action_dit_config: + action_dim: ${data.train.processor.action_output_dim} + hidden_dim: 1024 + ffn_dim: 4096 + num_heads: 24 + attn_head_dim: 128 + num_layers: 30 + text_dim: 4096 + freq_dim: 256 + eps: 1.0e-06 + use_gradient_checkpointing: ${model.mot_checkpoint_mixed_attn} + +video_scheduler: + train_shift: 5.0 + infer_shift: 5.0 + num_train_timesteps: 1000 + +action_scheduler: + train_shift: 1.0 + infer_shift: 1.0 + num_train_timesteps: 1000 + +loss: + lambda_action: 1.0 diff --git a/baseline/fastwam_zipper/source/configs/model/fastwam_optional_idm.yaml b/baseline/fastwam_zipper/source/configs/model/fastwam_optional_idm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..314d327b87ebc85fbba06e085de128cb61c8543c --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/model/fastwam_optional_idm.yaml @@ -0,0 +1,62 @@ +_target_: fastwam.runtime.create_fastwam_optional_idm +model_id: Wan-AI/Wan2.2-TI2V-5B +tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B +tokenizer_max_len: 128 +load_text_encoder: false +proprio_dim: ${data.train.processor.proprio_output_dim} +redirect_common_files: true +mot_checkpoint_mixed_attn: false +compile_training_denoise: false +action_idm_prob: 0.5 +video_cond_noise_prob: 0.0 +action_dit_pretrained_path: checkpoints/ActionDiT_linear_interp_Wan22_alphascale_1024hdim.pt +skip_dit_load_from_pretrain: false + +video_dit_config: + has_image_input: false + patch_size: [1, 2, 2] + in_dim: 48 + hidden_dim: 3072 + ffn_dim: 14336 + freq_dim: 256 + text_dim: 4096 + out_dim: 48 + num_heads: 24 + attn_head_dim: 128 + num_layers: 30 + eps: 1.0e-06 + seperated_timestep: true + require_clip_embedding: false + require_vae_embedding: false + fuse_vae_embedding_in_latents: true + use_gradient_checkpointing: ${model.mot_checkpoint_mixed_attn} + video_attention_mask_mode: "first_frame_causal" + action_conditioned: false + action_dim: ${data.train.processor.action_output_dim} + action_group_causal_mask_mode: "group_diagonal" + +action_dit_config: + action_dim: ${data.train.processor.action_output_dim} + hidden_dim: 1024 + ffn_dim: 4096 + num_heads: 24 + attn_head_dim: 128 + num_layers: 30 + text_dim: 4096 + freq_dim: 256 + eps: 1.0e-06 + use_gradient_checkpointing: ${model.mot_checkpoint_mixed_attn} + +video_scheduler: + train_shift: 5.0 + infer_shift: 5.0 + num_train_timesteps: 1000 + +action_scheduler: + train_shift: 1.0 + infer_shift: 1.0 + num_train_timesteps: 1000 + +loss: + lambda_video: 1.0 + lambda_action: 1.0 diff --git a/baseline/fastwam_zipper/source/configs/sim_libero.yaml b/baseline/fastwam_zipper/source/configs/sim_libero.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc44f89d7f18df778942b59be280fa3bbca8f405 --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/sim_libero.yaml @@ -0,0 +1,59 @@ +# @package _global_ + +defaults: + - train + - override /task: libero_uncond_2cam224_1e-4 + - _self_ + +ckpt: null +gpu_id: 0 + +model: + load_text_encoder: true + skip_dit_load_from_pretrain: true + action_dit_pretrained_path: null + +EVALUATION: + task_suite_name: libero_spatial + task_id: 0 + num_trials: 50 + output_dir: ./evaluate_results/libero/${hydra:runtime.choices.task}/${now:%Y%m%d_%H%M%S} + + # Runtime behavior + env_num: 1 + num_steps_wait: 30 + replan_steps: 10 + binarize_gripper: true + use_action_ensembler: false + visualize_future_video: false + + # Action inference + action_horizon: null + num_inference_steps: ${eval_num_inference_steps} + sigma_shift: null + text_cfg_scale: 1.0 + negative_prompt: "" + rand_device: cpu + tiled: false + compile_action_infer: true + + # Optional external inputs + dataset_stats_path: null + device: cuda + +MULTIRUN: + create_only: false + task_file: null + task_suite_names: + - libero_10 + - libero_goal + - libero_spatial + - libero_object + num_gpus: 8 + +hydra: + job: + chdir: false + run: + dir: . + output_subdir: null diff --git a/baseline/fastwam_zipper/source/configs/sim_robotwin.yaml b/baseline/fastwam_zipper/source/configs/sim_robotwin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2121b6bfd2eba1392ed13ff373e1ef010762722 --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/sim_robotwin.yaml @@ -0,0 +1,51 @@ +# @package _global_ + +defaults: + - train + - override /task: robotwin_uncond_3cam_384_1e-4 + - _self_ + +ckpt: null +gpu_id: 0 + +model: + load_text_encoder: true + skip_dit_load_from_pretrain: true + action_dit_pretrained_path: null + +EVALUATION: + robotwin_root: third_party/RoboTwin + policy_name: fastwam_policy + task_name: null + task_config: demo_randomized + instruction_type: unseen + eval_num_episodes: 100 + output_dir: ./evaluate_results/robotwin/${hydra:runtime.choices.task}/${now:%Y%m%d_%H%M%S} + + # Action inference + action_horizon: null + replan_steps: 24 + num_inference_steps: ${eval_num_inference_steps} + sigma_shift: null + text_cfg_scale: 1.0 + negative_prompt: "" + rand_device: cpu + tiled: false + timing_enabled: false + skip_get_obs_within_replan: true + + # Runtime inputs + dataset_stats_path: null + device: cuda + +MULTIRUN: + enabled: false + num_gpus: 8 + max_tasks_per_gpu: 2 + +hydra: + job: + chdir: false + run: + dir: . + output_subdir: null diff --git a/baseline/fastwam_zipper/source/configs/task/galaxea_r1_lite_3cam_1e-4.yaml b/baseline/fastwam_zipper/source/configs/task/galaxea_r1_lite_3cam_1e-4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cacde398f9e591e0640f6e09df7dfcef867f8f6 --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/task/galaxea_r1_lite_3cam_1e-4.yaml @@ -0,0 +1,46 @@ +# @package _global_ + +defaults: + - override /data: galaxea_r1_lite_3cam + - override /model: fastwam + - _self_ + +# Formal launcher: 8 samples/GPU x 8 GPUs x 2 accumulation = GBS 128. +batch_size: 8 +num_workers: 8 +gradient_accumulation_steps: 2 + +model: + # ModelConfig joins these ids to DIFFSYNTH_MODEL_BASE_PATH. The validated + # local snapshots are stored without a redundant `Wan-AI/` directory level. + model_id: ${oc.env:FASTWAM_WAN_MODEL_ID,Wan2.2-TI2V-5B} + tokenizer_model_id: ${oc.env:FASTWAM_TOKENIZER_MODEL_ID,Wan2.1-T2V-1.3B} + # The released RobotWin 3-camera checkpoint supplies both experts. Avoid a + # redundant 5B DiT load before the weight-only resume is applied. + skip_dit_load_from_pretrain: true + action_dit_pretrained_path: null + redirect_common_files: false + mot_checkpoint_mixed_attn: false + compile_training_denoise: false + +learning_rate: 1.0e-4 +lr_scheduler_type: cosine +num_epochs: 10 +max_steps: null +log_every: 10 +save_every: 1000 +eval_every: 500 +eval_num_inference_steps: 10 +weight_decay: 1.0e-2 +mixed_precision: bf16 +max_grad_norm: 1.0 +seed: 20261002 +resume: null + +wandb: + enabled: true + workspace: null + project: galaxea-real-baselines + name: fastwam-galaxea-r1-lite-3cam + group: galaxea-real-baselines-v1 + mode: online diff --git a/baseline/fastwam_zipper/source/configs/task/libero_idm_2cam224_1e-4.yaml b/baseline/fastwam_zipper/source/configs/task/libero_idm_2cam224_1e-4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..376b5701e3d411ecc0525da83325b5fa6e12e884 --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/task/libero_idm_2cam224_1e-4.yaml @@ -0,0 +1,27 @@ +# @package _global_ + +defaults: + - override /data: libero_2cam + - override /model: fastwam_idm + - _self_ + +# dataloading +batch_size: 16 +num_workers: 8 + +model: + mot_checkpoint_mixed_attn: false + +# scheduler +lr_scheduler_type: "cosine" +learning_rate: 1e-4 +num_epochs: 10 +max_steps: null +log_every: 10 +save_every: 2000 +eval_every: 200 + +# training +gradient_accumulation_steps: 1 +weight_decay: 1e-2 +resume: null diff --git a/baseline/fastwam_zipper/source/configs/task/libero_joint_2cam224_1e-4.yaml b/baseline/fastwam_zipper/source/configs/task/libero_joint_2cam224_1e-4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6864ab26856e9d69898674935719bc0f51cf717 --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/task/libero_joint_2cam224_1e-4.yaml @@ -0,0 +1,27 @@ +# @package _global_ + +defaults: + - override /data: libero_2cam + - override /model: fastwam_joint + - _self_ + +# dataloading +batch_size: 16 +num_workers: 8 + +model: + mot_checkpoint_mixed_attn: false + +# scheduler +lr_scheduler_type: "cosine" +learning_rate: 1e-4 +num_epochs: 10 +max_steps: null +log_every: 10 +save_every: 2000 +eval_every: 200 + +# training +gradient_accumulation_steps: 1 +weight_decay: 1e-2 +resume: null diff --git a/baseline/fastwam_zipper/source/configs/task/libero_optional_idm_2cam224_1e-4.yaml b/baseline/fastwam_zipper/source/configs/task/libero_optional_idm_2cam224_1e-4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee4dcbbffeed14672ecb2934a958cd440ab394de --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/task/libero_optional_idm_2cam224_1e-4.yaml @@ -0,0 +1,24 @@ +# @package _global_ + +defaults: + - override /data: libero_2cam + - override /model: fastwam_optional_idm + - _self_ + +batch_size: 16 +num_workers: 8 + +model: + mot_checkpoint_mixed_attn: false + +lr_scheduler_type: "cosine" +learning_rate: 1e-4 +num_epochs: 10 +max_steps: null +log_every: 10 +save_every: 2000 +eval_every: 200 + +gradient_accumulation_steps: 1 +weight_decay: 1e-2 +resume: null diff --git a/baseline/fastwam_zipper/source/configs/task/libero_uncond_2cam224_1e-4.yaml b/baseline/fastwam_zipper/source/configs/task/libero_uncond_2cam224_1e-4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37c27d725b82355656a98b435f8ef12c7262daad --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/task/libero_uncond_2cam224_1e-4.yaml @@ -0,0 +1,27 @@ +# @package _global_ + +defaults: + - override /data: libero_2cam + - override /model: fastwam + - _self_ + +# dataloading +batch_size: 16 +num_workers: 8 + +model: + mot_checkpoint_mixed_attn: false + +# scheduler +lr_scheduler_type: "cosine" +learning_rate: 1e-4 +num_epochs: 10 +max_steps: null +log_every: 10 +save_every: 2000 +eval_every: 200 + +# training +gradient_accumulation_steps: 1 +weight_decay: 1e-2 +resume: null diff --git a/baseline/fastwam_zipper/source/configs/task/robotwin_idm_3cam_384_1e-4.yaml b/baseline/fastwam_zipper/source/configs/task/robotwin_idm_3cam_384_1e-4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c781838b01a2b0e9be4c720d11a06597929e497c --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/task/robotwin_idm_3cam_384_1e-4.yaml @@ -0,0 +1,27 @@ +# @package _global_ + +defaults: + - override /data: robotwin + - override /model: fastwam_idm + - _self_ + +# dataloading +batch_size: 16 +num_workers: 8 + +model: + mot_checkpoint_mixed_attn: false + +# scheduler +lr_scheduler_type: "cosine" +learning_rate: 1e-4 +num_epochs: 5 +max_steps: null +log_every: 10 +save_every: 2500 +eval_every: 500 + +# training +gradient_accumulation_steps: 1 +weight_decay: 1e-2 +resume: null diff --git a/baseline/fastwam_zipper/source/configs/task/robotwin_joint_3cam_384_1e-4.yaml b/baseline/fastwam_zipper/source/configs/task/robotwin_joint_3cam_384_1e-4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c0211fbea082943f95b10abef3ef92e4390f395 --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/task/robotwin_joint_3cam_384_1e-4.yaml @@ -0,0 +1,27 @@ +# @package _global_ + +defaults: + - override /data: robotwin + - override /model: fastwam_joint + - _self_ + +# dataloading +batch_size: 16 +num_workers: 8 + +model: + mot_checkpoint_mixed_attn: false + +# scheduler +lr_scheduler_type: "cosine" +learning_rate: 1e-4 +num_epochs: 5 +max_steps: null +log_every: 10 +save_every: 2500 +eval_every: 500 + +# training +gradient_accumulation_steps: 1 +weight_decay: 1e-2 +resume: null diff --git a/baseline/fastwam_zipper/source/configs/task/robotwin_uncond_3cam_384_1e-4.yaml b/baseline/fastwam_zipper/source/configs/task/robotwin_uncond_3cam_384_1e-4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d01f33c8394ce36764971935e1d5c15ed5bec6d --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/task/robotwin_uncond_3cam_384_1e-4.yaml @@ -0,0 +1,27 @@ +# @package _global_ + +defaults: + - override /data: robotwin + - override /model: fastwam + - _self_ + +# dataloading +batch_size: 16 +num_workers: 8 + +model: + mot_checkpoint_mixed_attn: false + +# scheduler +lr_scheduler_type: "cosine" +learning_rate: 1e-4 +num_epochs: 5 +max_steps: null +log_every: 10 +save_every: 2500 +eval_every: 500 + +# training +gradient_accumulation_steps: 1 +weight_decay: 1e-2 +resume: null diff --git a/baseline/fastwam_zipper/source/configs/train.yaml b/baseline/fastwam_zipper/source/configs/train.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb1f7889e4fc07aac7c1f7201169e4e24e18d333 --- /dev/null +++ b/baseline/fastwam_zipper/source/configs/train.yaml @@ -0,0 +1,48 @@ +# @package _global_ + +defaults: + - _self_ + - data: null + - model: null + - task: null + + +output_dir: ./runs/train/${now:%Y-%m-%d}_${now:%H-%M-%S} + +# dataloading +batch_size: 2 +num_workers: 4 + +# scheduler +lr_scheduler_type: "cosine" +learning_rate: 1.0e-4 +num_epochs: 1 +max_steps: null +log_every: 10 +save_every: 100 +eval_every: 50 +eval_num_inference_steps: 10 + +# training +gradient_accumulation_steps: 1 +mixed_precision: "bf16" # one of: [no, fp16, bf16] +seed: 42 +max_grad_norm: 1.0 +weight_decay: 0.0 +resume: null + +# wandb +wandb: + enabled: false + workspace: null + project: fast-wam + name: train + group: null + mode: online + +hydra: + job: + chdir: false + run: + dir: . + output_subdir: null diff --git a/baseline/fastwam_zipper/source/pyproject.toml b/baseline/fastwam_zipper/source/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..71a05cdc856186683a255f23b0f155ed26e4a951 --- /dev/null +++ b/baseline/fastwam_zipper/source/pyproject.toml @@ -0,0 +1,48 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "fastwam" +version = "0.1.0" +description = "FastWAM: standalone Wan2.2-TI2V-5B training/inference package" +requires-python = ">=3.10" +dependencies = [ + "accelerate==1.12.0", + "av==15.1.0", + "boto3==1.35.99", + "datasets==4.8.5", + "deepspeed==0.18.7", + "einops==0.8.1", + "gitpython==3.1.45", + "huggingface-hub==0.29.2", + "hydra-core==1.3.2", + "imageio==2.37.0", + "imageio-ffmpeg==0.6.0", + "jsonlines==4.0.0", + "modelscope==1.34.0", + "numpy==2.2.6", + "omegaconf==2.3.0", + "packaging==25.0", + "pandas==2.2.3", + "pillow==12.0.0", + "pyarrow==24.0.0", + "regex==2025.11.3", + "rich==14.2.0", + "safetensors==0.5.3", + "termcolor==2.5.0", + "torch==2.7.1+cu128", + "torchcodec==0.4.0", + "torchvision==0.22.1+cu128", + "tqdm==4.68.3", + "transformers==4.49.0", + "typing-extensions==4.15.0", + "wandb==0.23.1", +] + +[tool.setuptools.package-dir] +"" = "src" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["fastwam*"] diff --git a/baseline/fastwam_zipper/source/scripts/__init__.py b/baseline/fastwam_zipper/source/scripts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b08f6f9860af34c2073f53bf712d63c6b6bd6df2 --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/__init__.py @@ -0,0 +1 @@ +"""CLI entry scripts for fastwam.""" diff --git a/baseline/fastwam_zipper/source/scripts/accelerate_configs/accelerate_zero0.yaml b/baseline/fastwam_zipper/source/scripts/accelerate_configs/accelerate_zero0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea85b0480fe7bac8fdfe115ee8a20a88c6bf1eb4 --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/accelerate_configs/accelerate_zero0.yaml @@ -0,0 +1,22 @@ +compute_environment: LOCAL_MACHINE +debug: false +distributed_type: DEEPSPEED +deepspeed_config: + gradient_accumulation_steps: 1 + offload_optimizer_device: none + offload_param_device: none + zero3_init_flag: false + zero_stage: 0 +downcast_bf16: "no" +enable_cpu_affinity: false +machine_rank: 0 +main_training_function: main +mixed_precision: "bf16" +num_machines: 1 +num_processes: 1 +rdzv_backend: static +same_network: true +tpu_env: [] +tpu_use_cluster: false +tpu_use_sudo: false +use_cpu: false diff --git a/baseline/fastwam_zipper/source/scripts/accelerate_configs/accelerate_zero1_ds.yaml b/baseline/fastwam_zipper/source/scripts/accelerate_configs/accelerate_zero1_ds.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3183b8aec454992ab9180890f6da2a1493bbb8f --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/accelerate_configs/accelerate_zero1_ds.yaml @@ -0,0 +1,18 @@ +compute_environment: LOCAL_MACHINE +debug: false +distributed_type: DEEPSPEED +deepspeed_config: + deepspeed_config_file: scripts/ds_configs/ds_zero1_config.json + zero3_init_flag: false + +# Keep this null when using deepspeed_config_file; precision is controlled by +# training config (Trainer -> Accelerator(mixed_precision=...)) and DS JSON. +mixed_precision: null + +machine_rank: 0 +main_training_function: main +num_machines: 1 +num_processes: 1 +rdzv_backend: static +same_network: true +use_cpu: false diff --git a/baseline/fastwam_zipper/source/scripts/accelerate_configs/accelerate_zero2_ds.yaml b/baseline/fastwam_zipper/source/scripts/accelerate_configs/accelerate_zero2_ds.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be2f8e34c4785ba8611af88a2f5a88d48e55aaf8 --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/accelerate_configs/accelerate_zero2_ds.yaml @@ -0,0 +1,18 @@ +compute_environment: LOCAL_MACHINE +debug: false +distributed_type: DEEPSPEED +deepspeed_config: + deepspeed_config_file: scripts/ds_configs/ds_zero2_config.json + zero3_init_flag: false + +# Keep this null when using deepspeed_config_file; precision is controlled by +# training config (Trainer -> Accelerator(mixed_precision=...)) and DS JSON. +mixed_precision: null + +machine_rank: 0 +main_training_function: main +num_machines: 1 +num_processes: 1 +rdzv_backend: static +same_network: true +use_cpu: false diff --git a/baseline/fastwam_zipper/source/scripts/check_galaxea_r1_lite_contract.py b/baseline/fastwam_zipper/source/scripts/check_galaxea_r1_lite_contract.py new file mode 100644 index 0000000000000000000000000000000000000000..93082c1c2c63712923b632f772d9cfb54c37c357 --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/check_galaxea_r1_lite_contract.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Fail-closed checks for the shared Galaxea LeRobot-v2 FastWAM input.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +EXPECTED_MANIFEST_SHA256 = "36dcbf10695ec1f6c79a90375f396e59e70ae7643595f51dbb3e1299765be5b2" +EXPECTED_IMAGE_KEYS = { + "observation.images.head_left", + "observation.images.wrist_left", + "observation.images.wrist_right", +} + + +def read_json(path: Path) -> dict: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError(f"expected mapping: {path}") + return value + + +def check_pack(root: Path, split: str, expected_episodes: int) -> dict: + info = read_json(root / "meta/info.json") + qc = read_json(root / "meta/alignment_qc.json") + features = info.get("features", {}) + if str(info.get("codebase_version")) != "v2.0": + raise ValueError(f"{split}: expected LeRobot v2.0") + if int(info.get("fps", -1)) != 20: + raise ValueError(f"{split}: expected 20 Hz") + if int(info.get("total_episodes", -1)) != expected_episodes: + raise ValueError(f"{split}: episode count mismatch") + if not EXPECTED_IMAGE_KEYS.issubset(features): + raise ValueError(f"{split}: missing camera features") + for key in ("observation.state", "action"): + if features.get(key, {}).get("shape") != [14]: + raise ValueError(f"{split}: {key} must be 14D") + if qc.get("split") != split: + raise ValueError(f"{split}: alignment report split mismatch") + if qc.get("pass") is not True or qc.get("strictly_causal") is not True: + raise ValueError(f"{split}: alignment QC did not pass") + if qc.get("all_required_timestamps_from_ros_header") is not True: + raise ValueError(f"{split}: timestamp fallback is forbidden") + if float(qc.get("max_source_minus_anchor_seconds", 1.0)) > 0.0: + raise ValueError(f"{split}: future selections are forbidden") + for episode in qc.get("episodes", []): + for stream in episode.get("streams", {}).values(): + if int(stream.get("future_selected_frames", -1)) != 0: + raise ValueError(f"{split}: future-selected camera/state sample") + for side in episode.get("gripper_latching", {}).values(): + if int(side.get("future_selected_frames", -1)) != 0: + raise ValueError(f"{split}: future-selected gripper sample") + return { + "root": str(root.resolve()), + "episodes": expected_episodes, + "frames": int(info["total_frames"]), + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--train", type=Path, required=True) + parser.add_argument("--validation", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--expected-train-episodes", type=int, default=511) + parser.add_argument("--expected-validation-episodes", type=int, default=57) + parser.add_argument("--allow-smoke-manifest", action="store_true") + args = parser.parse_args() + digest = hashlib.sha256(args.manifest.read_bytes()).hexdigest() + if digest != EXPECTED_MANIFEST_SHA256 and not args.allow_smoke_manifest: + raise ValueError(f"source manifest SHA256 mismatch: {digest}") + result = { + "status": "galaxea_fastwam_contract_ok", + "manifest_sha256": digest, + "action_horizon": 24, + "video_frames": 9, + "views": ["head_left", "wrist_left", "wrist_right"], + "train": check_pack(args.train, "train", args.expected_train_episodes), + "validation": check_pack( + args.validation, "validation", args.expected_validation_episodes + ), + } + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/baseline/fastwam_zipper/source/scripts/check_galaxea_text_cache.py b/baseline/fastwam_zipper/source/scripts/check_galaxea_text_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..1261c7b7251fe802bdc8c0e85669b09dfd5119c0 --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/check_galaxea_text_cache.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Validate all fixed Galaxea prompt embeddings without loading UMT5.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import torch + + +PROMPT = "A video recorded from a robot's point of view executing the following instruction: {task}" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--train", type=Path, required=True) + parser.add_argument("--validation", type=Path, required=True) + parser.add_argument("--cache", type=Path, required=True) + args = parser.parse_args() + + prompts: set[str] = set() + for dataset in (args.train, args.validation): + tasks = dataset / "meta/tasks.jsonl" + for line in tasks.read_text(encoding="utf-8").splitlines(): + if line.strip(): + prompts.add(PROMPT.format(task=json.loads(line)["task"])) + if len(prompts) != 5: + raise RuntimeError(f"expected five fixed task prompts, got {len(prompts)}") + + for prompt in sorted(prompts): + digest = hashlib.sha256(prompt.encode("utf-8")).hexdigest() + path = args.cache / f"{digest}.t5_len128.wan22ti2v5b.pt" + payload = torch.load(path, map_location="cpu", weights_only=True) + context = payload["context"] + mask = payload["mask"] + if context.ndim != 2 or context.shape[0] != 128 or not torch.isfinite(context).all(): + raise RuntimeError(f"invalid context tensor in {path}: {tuple(context.shape)}") + if mask.shape != (128,): + raise RuntimeError(f"invalid mask tensor in {path}: {tuple(mask.shape)}") + print(f"FASTWAM_TEXT_CACHE_OK prompts={len(prompts)} cache={args.cache.resolve()}") + + +if __name__ == "__main__": + main() diff --git a/baseline/fastwam_zipper/source/scripts/dryrun_fastwam.py b/baseline/fastwam_zipper/source/scripts/dryrun_fastwam.py new file mode 100644 index 0000000000000000000000000000000000000000..b96bbb01ef68a335469eb4a4399942b322e9f21e --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/dryrun_fastwam.py @@ -0,0 +1,227 @@ +"""Benchmark FastWAM action inference with synthetic task-shaped inputs. + +Example: + python scripts/dryrun_fastwam.py \ + task=libero_uncond_2cam224_1e-4 \ + ckpt=./checkpoints/fastwam_release/libero_uncond_2cam224.pt \ + +DRYRUN.compile_action_infer=true \ + +DRYRUN.warmup=2 +DRYRUN.iters=10 +""" + +from __future__ import annotations + +import logging +import inspect +import statistics +import time + +import hydra +import torch +from hydra.utils import instantiate +from omegaconf import DictConfig, OmegaConf + + +logger = logging.getLogger(__name__) + + +def _dryrun_config(cfg: DictConfig) -> DictConfig: + defaults = OmegaConf.create( + { + "warmup": 2, + "iters": 10, + "seed": 42, + "action_horizon": None, + "num_inference_steps": None, + "compile_action_infer": False, + "benchmark_both": False, + "use_random_context": False, + "action_infer_mode": "idm", + } + ) + return OmegaConf.merge(defaults, cfg.get("DRYRUN", {})) + + +def _model_dtype(mixed_precision: str) -> torch.dtype: + return { + "no": torch.float32, + "fp16": torch.float16, + "bf16": torch.bfloat16, + }[str(mixed_precision)] + + +def _timed_infer_with_output( + model, + kwargs: dict, + device: torch.device, +) -> tuple[float, torch.Tensor]: + torch.cuda.synchronize(device) + start = time.perf_counter() + with torch.no_grad(): + output = model.infer_action(**kwargs) + torch.cuda.synchronize(device) + logger.info("action shape=%s", tuple(output["action"].shape)) + return time.perf_counter() - start, output["action"].detach().clone() + + +def _timed_infer(model, kwargs: dict, device: torch.device) -> float: + elapsed, _ = _timed_infer_with_output(model, kwargs, device) + return elapsed + + +def _benchmark_mode( + model, + infer_kwargs: dict, + device: torch.device, + *, + label: str, + warmup: int, + iters: int, +) -> tuple[list[float], float]: + for index in range(warmup): + elapsed = _timed_infer(model, infer_kwargs, device) + logger.info("%s warmup %d/%d: %.4fs", label, index + 1, warmup, elapsed) + + torch.cuda.reset_peak_memory_stats(device) + timings = [] + for index in range(iters): + elapsed = _timed_infer(model, infer_kwargs, device) + timings.append(elapsed) + logger.info("%s iteration %d/%d: %.4fs", label, index + 1, iters, elapsed) + return timings, torch.cuda.max_memory_allocated(device) / 1024**3 + + +@hydra.main(config_path="../configs", config_name="sim_libero.yaml", version_base="1.3") +def main(cfg: DictConfig) -> None: + logging.basicConfig(level=logging.INFO) + dryrun = _dryrun_config(cfg) + device = torch.device(str(cfg.EVALUATION.device)) + model = instantiate( + cfg.model, + model_dtype=_model_dtype(cfg.mixed_precision), + device=str(device), + ) + if cfg.ckpt is not None: + model.load_checkpoint(str(cfg.ckpt)) + model.eval() + logger.info( + "device=%s gpu=%s dtype=%s", + device, + torch.cuda.get_device_name(device), + next(model.parameters()).dtype, + ) + + height, width = (int(value) for value in cfg.data.train.video_size) + action_horizon = ( + int(cfg.data.train.num_frames) - 1 + if dryrun.action_horizon is None + else int(dryrun.action_horizon) + ) + num_inference_steps = ( + int(cfg.eval_num_inference_steps) + if dryrun.num_inference_steps is None + else int(dryrun.num_inference_steps) + ) + logger.info( + "action_horizon=%d num_inference_steps=%d warmup=%d iters=%d", + action_horizon, + num_inference_steps, + int(dryrun.warmup), + int(dryrun.iters), + ) + torch.manual_seed(int(dryrun.seed)) + input_image = torch.rand(1, 3, height, width) * 2 - 1 + proprio_dim = int(cfg.data.train.processor.proprio_output_dim) + proprio = torch.randn(1, proprio_dim) + if bool(dryrun.use_random_context): + context = torch.randn(1, int(cfg.data.train.context_len), model.text_dim) + context_mask = torch.ones(context.shape[:2], dtype=torch.bool) + prompt = None + else: + context = None + context_mask = None + prompt = "pick up the object" + + infer_kwargs = { + "prompt": prompt, + "input_image": input_image, + "action_horizon": action_horizon, + "proprio": proprio, + "context": context, + "context_mask": context_mask, + "num_inference_steps": num_inference_steps, + "seed": int(dryrun.seed), + "rand_device": str(cfg.EVALUATION.rand_device), + "compile_action_infer": bool(dryrun.compile_action_infer), + } + infer_parameters = inspect.signature(model.infer_action).parameters + if "num_video_frames" in infer_parameters: + infer_kwargs["num_video_frames"] = ( + (int(cfg.data.train.num_frames) - 1) + // int(cfg.data.train.action_video_freq_ratio) + + 1 + ) + if "action_infer_mode" in infer_parameters: + infer_kwargs["action_infer_mode"] = str(dryrun.action_infer_mode) + + warmup = int(dryrun.warmup) + iters = int(dryrun.iters) + if bool(dryrun.benchmark_both): + infer_kwargs["compile_action_infer"] = False + eager_times, eager_peak = _benchmark_mode( + model, + infer_kwargs, + device, + label="eager", + warmup=warmup, + iters=iters, + ) + + infer_kwargs["compile_action_infer"] = True + cold_compile_time = _timed_infer(model, infer_kwargs, device) + logger.info("compile cold start: %.4fs", cold_compile_time) + compile_times, compile_peak = _benchmark_mode( + model, + infer_kwargs, + device, + label="compile", + warmup=warmup, + iters=iters, + ) + eager_mean = statistics.mean(eager_times) + compile_mean = statistics.mean(compile_times) + logger.info( + "comparison | eager_mean=%.4fs compile_mean=%.4fs speedup=%.3fx " + "compile_cold_start=%.4fs eager_peak=%.3fGiB compile_peak=%.3fGiB", + eager_mean, + compile_mean, + eager_mean / compile_mean, + cold_compile_time, + eager_peak, + compile_peak, + ) + return + + torch.cuda.reset_peak_memory_stats(device) + for index in range(warmup): + elapsed = _timed_infer(model, infer_kwargs, device) + logger.info("warmup %d/%d: %.4fs", index + 1, warmup, elapsed) + + torch.cuda.reset_peak_memory_stats(device) + timings = [] + for index in range(iters): + elapsed = _timed_infer(model, infer_kwargs, device) + timings.append(elapsed) + logger.info("iteration %d/%d: %.4fs", index + 1, iters, elapsed) + peak_gib = torch.cuda.max_memory_allocated(device) / 1024**3 + logger.info( + "mean=%.4fs median=%.4fs min=%.4fs max=%.4fs peak_allocated=%.3fGiB", + statistics.mean(timings), + statistics.median(timings), + min(timings), + max(timings), + peak_gib, + ) + + +if __name__ == "__main__": + main() diff --git a/baseline/fastwam_zipper/source/scripts/ds_configs/ds_zero1_config.json b/baseline/fastwam_zipper/source/scripts/ds_configs/ds_zero1_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5c9bea2cfb527dcd130b416cebf54bfdef221165 --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/ds_configs/ds_zero1_config.json @@ -0,0 +1,18 @@ +{ + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "gradient_accumulation_steps": "auto", + "zero_optimization": { + "stage": 1, + "offload_optimizer": { + "device": "none" + }, + "offload_param": { + "device": "none" + }, + "overlap_comm": false, + "contiguous_gradients": false, + "reduce_bucket_size": 2e8, + "allgather_bucket_size": 2e8 + } +} diff --git a/baseline/fastwam_zipper/source/scripts/ds_configs/ds_zero2_config.json b/baseline/fastwam_zipper/source/scripts/ds_configs/ds_zero2_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9a7872c503be20e08715d58d62b9bd84bd46ac0a --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/ds_configs/ds_zero2_config.json @@ -0,0 +1,19 @@ +{ + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "gradient_accumulation_steps": "auto", + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": 2, + "offload_optimizer": { + "device": "none" + }, + "offload_param": { + "device": "none" + }, + "overlap_comm": false, + "contiguous_gradients": false, + "reduce_bucket_size": 2e8, + "allgather_bucket_size": 2e8 + } +} diff --git a/baseline/fastwam_zipper/source/scripts/precompute_text_embeds.py b/baseline/fastwam_zipper/source/scripts/precompute_text_embeds.py new file mode 100644 index 0000000000000000000000000000000000000000..0c1f8e2d9fd9e3aec7a22fd6f2645124bc531201 --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/precompute_text_embeds.py @@ -0,0 +1,389 @@ +import hashlib +import json +import logging +import os +import re +import uuid +from pathlib import Path +from typing import Any + +import hydra +import torch +import torch.distributed as dist +from omegaconf import DictConfig, ListConfig +from tqdm import tqdm + +from fastwam.datasets.lerobot.robot_video_dataset import DEFAULT_PROMPT +from fastwam.models.wan22.helpers.loader import _load_registered_model, _resolve_configs +from fastwam.models.wan22.wan_video_text_encoder import HuggingfaceTokenizer +from fastwam.utils.config_resolvers import register_default_resolvers +from fastwam.utils.logging_config import get_logger, setup_logging + +register_default_resolvers() +logger = get_logger(__name__) + +DEFAULT_MODEL_ID = "Wan-AI/Wan2.2-TI2V-5B" +DEFAULT_TOKENIZER_MODEL_ID = "Wan-AI/Wan2.1-T2V-1.3B" +DEFAULT_CONTEXT_LEN = 128 +DEFAULT_BATCH_SIZE = 16 + + +def _init_distributed(): + world_size = int(os.environ.get("WORLD_SIZE", "1")) + if world_size <= 1: + return False, 0, 1, 0 + + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + backend = "nccl" if torch.cuda.is_available() else "gloo" + if torch.cuda.is_available(): + torch.cuda.set_device(local_rank) + + if not dist.is_initialized(): + dist.init_process_group(backend=backend, init_method="env://") + + return True, dist.get_rank(), dist.get_world_size(), local_rank + + +def _to_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + text = value.strip().lower() + if text in {"1", "true", "yes", "y"}: + return True + if text in {"0", "false", "no", "n"}: + return False + raise ValueError(f"Cannot parse bool value: {value}") + + +def _iter_dataset_nodes(node: Any, path: str = "data"): + if isinstance(node, DictConfig): + if "dataset_dirs" in node and node.get("dataset_dirs") is not None: + yield path, node + for key, value in node.items(): + yield from _iter_dataset_nodes(value, f"{path}.{key}") + elif isinstance(node, ListConfig): + for idx, value in enumerate(node): + yield from _iter_dataset_nodes(value, f"{path}[{idx}]") + + +def _collect_dataset_settings(data_cfg: DictConfig): + dataset_dirs: list[str] = [] + cache_dirs: list[Path] = [] + context_lens = set() + + for node_path, node in _iter_dataset_nodes(data_cfg, path="data"): + raw_dirs = node.get("dataset_dirs") + if raw_dirs is None: + continue + + cache_dir = node.get("text_embedding_cache_dir") + if cache_dir is None or not str(cache_dir).strip(): + raise ValueError( + f"Missing `text_embedding_cache_dir` for dataset node `{node_path}` " + "(this node defines `dataset_dirs`)." + ) + + for ds in raw_dirs: + ds_str = str(ds) + if ds_str not in dataset_dirs: + dataset_dirs.append(ds_str) + + cache_dir_path = Path(str(cache_dir)).expanduser() + if cache_dir_path not in cache_dirs: + cache_dirs.append(cache_dir_path) + + context_len = node.get("context_len") + if context_len is not None: + context_lens.add(int(context_len)) + + logger.info("Discovered dataset node `%s` with %d dataset_dirs.", node_path, len(raw_dirs)) + + return dataset_dirs, cache_dirs, context_lens + + +def _resolve_context_len(context_lens: set[int]) -> int: + if len(context_lens) != 1: + raise ValueError( + f"Found multiple context_len values in data config: {sorted(context_lens)}. " + "Please keep them consistent." + ) + return next(iter(context_lens)) + + +def _read_unique_prompts(dataset_dirs: list[str]) -> list[str]: + prompts: list[str] = [] + seen = set() + total_task_rows = 0 + + for ds_dir in dataset_dirs: + tasks_path = Path(ds_dir) / "meta" / "tasks.jsonl" + if not tasks_path.exists(): + raise FileNotFoundError(f"Missing tasks file: {tasks_path}") + + with tasks_path.open("r", encoding="utf-8") as f: + for line_idx, line in enumerate(f, start=1): + line = line.strip() + if not line: + continue + record = json.loads(line) + if "task" not in record: + raise KeyError(f"Missing `task` field at {tasks_path}:{line_idx}") + task = str(record["task"]) + prompt = DEFAULT_PROMPT.format(task=task) + total_task_rows += 1 + if prompt not in seen: + seen.add(prompt) + prompts.append(prompt) + + logger.info( + "Loaded %d task rows from %d datasets, deduplicated to %d prompts.", + total_task_rows, + len(dataset_dirs), + len(prompts), + ) + return prompts + + +def _get_override_prompt(override_instruction: Any) -> str | None: + if override_instruction is None: + return None + task = str(override_instruction).strip() + if task == "": + return None + return DEFAULT_PROMPT.format(task=task) + + +def _model_id_to_enc_id(model_id: str) -> str: + base = str(model_id).split("/")[-1] + enc_id = re.sub(r"[^a-z0-9]+", "", base.lower()) + return enc_id or "textenc" + + +def _atomic_torch_save(payload: dict[str, torch.Tensor], output_path: Path): + output_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = output_path.parent / f".{output_path.name}.tmp.{uuid.uuid4().hex}" + torch.save(payload, str(tmp_path)) + os.replace(tmp_path, output_path) + + +@hydra.main(config_path="../configs", config_name="train", version_base="1.3") +def main(cfg: DictConfig): + setup_logging(log_level=logging.INFO) + + is_distributed, rank, world_size, local_rank = _init_distributed() + if is_distributed and rank == 0: + logger.info("Distributed enabled: world_size=%d", world_size) + if (not is_distributed) and torch.cuda.is_available() and torch.cuda.device_count() > 1: + logger.info( + "Multi-GPU available. To use it, run: torchrun --standalone --nproc_per_node=%d scripts/precompute_text_embeds.py", + torch.cuda.device_count(), + ) + + overwrite = _to_bool(cfg.get("overwrite", True)) + model_cfg = cfg.model + if model_cfg is None: + raise ValueError("`cfg.model` is required.") + if cfg.data is None: + raise ValueError("`cfg.data` is required.") + + dataset_dirs, cache_dirs, context_lens = _collect_dataset_settings(cfg.data) + if not cache_dirs: + raise ValueError("No `text_embedding_cache_dir` found under `cfg.data`.") + + context_len = _resolve_context_len(context_lens) + override_prompt = _get_override_prompt(cfg.get("override_instruction")) + if override_prompt is not None: + prompts = [override_prompt] + logger.info("Using override_instruction; skipping dataset scan and encoding exactly 1 prompt.") + else: + if not dataset_dirs: + raise ValueError("No `dataset_dirs` found under `cfg.data`.") + prompts = _read_unique_prompts(dataset_dirs) + if not prompts: + logger.warning("No prompts found from tasks.jsonl; nothing to do.") + return + + if torch.cuda.is_available(): + device = f"cuda:{local_rank}" if is_distributed else "cuda" + else: + device = "cpu" + torch_dtype = torch.bfloat16 + model_id = str(model_cfg.get("model_id", DEFAULT_MODEL_ID)) + tokenizer_model_id = str(model_cfg.get("tokenizer_model_id", DEFAULT_TOKENIZER_MODEL_ID)) + redirect_common_files = bool(model_cfg.get("redirect_common_files", True)) + enc_id = _model_id_to_enc_id(model_id) + + logger.info( + "Preparing text encoder with model_id=%s tokenizer_model_id=%s device=%s dtype=%s context_len=%d overwrite=%s", + model_id, + tokenizer_model_id, + device, + torch_dtype, + context_len, + overwrite, + ) + + _, text_config, _, tokenizer_config = _resolve_configs( + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + redirect_common_files=redirect_common_files, + ) + text_config.download_if_necessary() + tokenizer_config.download_if_necessary() + + text_encoder = _load_registered_model( + text_config.path, + "wan_video_text_encoder", + torch_dtype=torch_dtype, + device=device, + ).eval() + tokenizer = HuggingfaceTokenizer( + name=tokenizer_config.path, + seq_len=context_len, + clean="whitespace", + ) + + stats = { + str(cache_dir): {"new": 0, "overwrite": 0, "skip": 0} + for cache_dir in cache_dirs + } + + prompts = prompts[rank::world_size] if is_distributed else prompts + + if not overwrite: + fully_cached_local = 0 + prompts_to_encode: list[str] = [] + for prompt in prompts: + hashed = hashlib.sha256(prompt.encode("utf-8")).hexdigest() + filename = f"{hashed}.t5_len{context_len}.{enc_id}.pt" + fully_cached = True + for cache_dir in cache_dirs: + cache_path = cache_dir / filename + if not cache_path.exists(): + fully_cached = False + break + if fully_cached: + fully_cached_local += 1 + for cache_dir in cache_dirs: + stats[str(cache_dir)]["skip"] += 1 + else: + prompts_to_encode.append(prompt) + + prompts = prompts_to_encode + + fully_cached_global = fully_cached_local + to_encode_global = len(prompts) + if is_distributed: + reduce_device = torch.device(device) if device.startswith("cuda") else torch.device("cpu") + count_tensor = torch.tensor([fully_cached_local, len(prompts)], device=reduce_device, dtype=torch.long) + dist.all_reduce(count_tensor, op=dist.ReduceOp.SUM) + fully_cached_global = int(count_tensor[0].item()) + to_encode_global = int(count_tensor[1].item()) + + if (not is_distributed) or rank == 0: + logger.info( + "overwrite=false: fully cached prompts=%d, prompts to encode=%d", + fully_cached_global, + to_encode_global, + ) + + logger.info("Writing caches to %d directories.", len(cache_dirs)) + prompts_encoded_local = len(prompts) + prompts_encoded_global = prompts_encoded_local + if is_distributed: + reduce_device = torch.device(device) if device.startswith("cuda") else torch.device("cpu") + count_tensor = torch.tensor([prompts_encoded_local], device=reduce_device, dtype=torch.long) + dist.all_reduce(count_tensor, op=dist.ReduceOp.SUM) + prompts_encoded_global = int(count_tensor.item()) + + over_length_prompts = 0 + with tqdm( + total=len(prompts), + desc=f"Encoding prompts (rank {rank}/{world_size})" if is_distributed else "Encoding prompts", + unit="prompt", + dynamic_ncols=True, + disable=is_distributed and rank != 0, + ) as pbar: + with torch.no_grad(): + for start in range(0, len(prompts), DEFAULT_BATCH_SIZE): + batch_prompts = prompts[start : start + DEFAULT_BATCH_SIZE] + ids, mask = tokenizer(batch_prompts, return_mask=True, add_special_tokens=True) + ids = ids.to(device) + mask = mask.to(device=device, dtype=torch.bool) + over_length_prompts += int(mask.all(dim=1).sum().item()) + context = text_encoder(ids, mask) + + for i, prompt in enumerate(batch_prompts): + hashed = hashlib.sha256(prompt.encode("utf-8")).hexdigest() + context_i = context[i].detach().to(device="cpu", dtype=torch.bfloat16).contiguous() + mask_i = mask[i].detach().to(device="cpu", dtype=torch.bool).contiguous() + payload = { + "context": context_i, + "mask": mask_i, + } + + for cache_dir in cache_dirs: + cache_path = cache_dir / f"{hashed}.t5_len{context_len}.{enc_id}.pt" + key = str(cache_dir) + if cache_path.exists() and not overwrite: + stats[key]["skip"] += 1 + continue + + if cache_path.exists(): + stats[key]["overwrite"] += 1 + else: + stats[key]["new"] += 1 + + _atomic_torch_save(payload, cache_path) + + pbar.update(len(batch_prompts)) + + over_length_global = over_length_prompts + if is_distributed: + reduce_device = torch.device(device) if device.startswith("cuda") else torch.device("cpu") + over_tensor = torch.tensor([over_length_prompts], device=reduce_device, dtype=torch.long) + dist.all_reduce(over_tensor, op=dist.ReduceOp.SUM) + over_length_global = int(over_tensor.item()) + + counts_tensor = torch.tensor( + [ + [stats[str(cache_dir)]["new"], stats[str(cache_dir)]["overwrite"], stats[str(cache_dir)]["skip"]] + for cache_dir in cache_dirs + ], + device=reduce_device, + dtype=torch.long, + ) + dist.all_reduce(counts_tensor, op=dist.ReduceOp.SUM) + if rank == 0: + for idx, cache_dir in enumerate(cache_dirs): + key = str(cache_dir) + stats[key]["new"] = int(counts_tensor[idx, 0].item()) + stats[key]["overwrite"] = int(counts_tensor[idx, 1].item()) + stats[key]["skip"] = int(counts_tensor[idx, 2].item()) + + if (not is_distributed) or rank == 0: + logger.info("Finished precomputing text embeddings.") + logger.info( + "Over-length prompts (mask all True, i.e. no padding after truncation/max_length=%d): %d/%d", + context_len, + over_length_global, + prompts_encoded_global, + ) + for cache_dir in cache_dirs: + key = str(cache_dir) + logger.info( + "Cache dir: %s | new=%d overwrite=%d skip=%d", + key, + stats[key]["new"], + stats[key]["overwrite"], + stats[key]["skip"], + ) + + if is_distributed and dist.is_initialized(): + dist.barrier() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/baseline/fastwam_zipper/source/scripts/prepare_galaxea_r1_lite_overlay.py b/baseline/fastwam_zipper/source/scripts/prepare_galaxea_r1_lite_overlay.py new file mode 100644 index 0000000000000000000000000000000000000000..3bd693dea018ceaa4b6fcae8f5ab86955dd7d419 --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/prepare_galaxea_r1_lite_overlay.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Build a metadata-only FastWAM view of a canonical Galaxea LeRobot-v2 pack. + +FastWAM vendors a LeRobot v2.1 loader which requires every statistics ``count`` +to have shape ``[1]``. The shared converter stores an equivalent count once per +vector component. This script collapses only that redundant field in a private +overlay; parquet and video payloads remain read-only symlinks to the canonical +pack. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import tempfile +from pathlib import Path + + +MARKER_NAME = "fastwam_overlay.json" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def metadata_hashes(meta: Path) -> dict[str, str]: + return { + path.relative_to(meta).as_posix(): sha256(path) + for path in sorted(meta.rglob("*")) + if path.is_file() and path.name != MARKER_NAME + } + + +def normalized_stats(source: Path) -> tuple[dict, list[str]]: + stats = json.loads(source.read_text(encoding="utf-8")) + changed: list[str] = [] + for feature, values in stats.items(): + if not isinstance(values, dict) or "count" not in values: + continue + count = values["count"] + if not isinstance(count, list) or not count: + raise ValueError(f"{source}: {feature}.count must be a nonempty list") + if len(count) == 1: + continue + if any(value != count[0] for value in count[1:]): + raise ValueError(f"{source}: {feature}.count is not redundant: {count}") + values["count"] = [count[0]] + changed.append(feature) + return stats, changed + + +def validate_existing(source: Path, destination: Path) -> bool: + marker_path = destination / "meta" / MARKER_NAME + if not marker_path.is_file(): + return False + marker = json.loads(marker_path.read_text(encoding="utf-8")) + expected = { + "format": "fastwam-galaxea-metadata-overlay-v1", + "source": str(source.resolve()), + "source_meta_sha256": metadata_hashes(source / "meta"), + } + if any(marker.get(key) != value for key, value in expected.items()): + return False + for payload in ("data", "videos"): + link = destination / payload + if not link.is_symlink() or link.resolve() != (source / payload).resolve(): + return False + return marker.get("overlay_meta_sha256") == metadata_hashes(destination / "meta") + + +def build_overlay(source: Path, destination: Path) -> dict: + source = source.resolve() + if not (source / "meta" / "stats.json").is_file(): + raise FileNotFoundError(f"missing canonical metadata: {source / 'meta/stats.json'}") + for payload in ("data", "videos"): + if not (source / payload).exists(): + raise FileNotFoundError(f"missing canonical payload: {source / payload}") + + if destination.exists(): + if validate_existing(source, destination): + return json.loads( + (destination / "meta" / MARKER_NAME).read_text(encoding="utf-8") + ) + raise FileExistsError( + f"refusing stale or foreign overlay: {destination}; remove that exact " + "FastWAM overlay before rebuilding" + ) + + destination.parent.mkdir(parents=True, exist_ok=True) + staging = Path( + tempfile.mkdtemp(prefix=f".{destination.name}.tmp.", dir=destination.parent) + ) + try: + shutil.copytree(source / "meta", staging / "meta") + stats, changed = normalized_stats(source / "meta" / "stats.json") + (staging / "meta" / "stats.json").write_text( + json.dumps(stats, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + for payload in ("data", "videos"): + os.symlink(source / payload, staging / payload, target_is_directory=True) + + marker = { + "format": "fastwam-galaxea-metadata-overlay-v1", + "source": str(source), + "source_meta_sha256": metadata_hashes(source / "meta"), + "overlay_meta_sha256": metadata_hashes(staging / "meta"), + "transformation": { + "file": "meta/stats.json", + "operation": "collapse identical per-dimension count values to one scalar count", + "features": changed, + }, + "payload_storage": "absolute read-only symlinks; no data or videos copied", + } + (staging / "meta" / MARKER_NAME).write_text( + json.dumps(marker, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + os.replace(staging, destination) + return marker + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--destination", type=Path, required=True) + args = parser.parse_args() + marker = build_overlay(args.source, args.destination) + print(json.dumps({"status": "fastwam_overlay_ok", **marker}, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/baseline/fastwam_zipper/source/scripts/preprocess_action_dit_backbone.py b/baseline/fastwam_zipper/source/scripts/preprocess_action_dit_backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..737f9c05779518d976550727251fa0fa9a1ecd48 --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/preprocess_action_dit_backbone.py @@ -0,0 +1,240 @@ +import argparse +from pathlib import Path +from typing import Any + +import torch +import torch.nn.functional as F +from omegaconf import OmegaConf + +from fastwam.models.wan22.action_dit import ActionDiT +from fastwam.models.wan22.helpers.loader import load_wan22_ti2v_5b_components + + +def _parse_dtype(name: str) -> torch.dtype: + value = str(name).strip().lower() + if value == "float32": + return torch.float32 + if value == "float16": + return torch.float16 + if value == "bfloat16": + return torch.bfloat16 + raise ValueError(f"Unsupported dtype: {name}. Expected one of: float32, float16, bfloat16.") + + +def _parse_bool(name: str) -> bool: + value = str(name).strip().lower() + if value in {"1", "true", "yes", "y"}: + return True + if value in {"0", "false", "no", "n"}: + return False + raise ValueError(f"Cannot parse bool value: {name}") + + +def _is_unresolved_interpolation(value: Any) -> bool: + return isinstance(value, str) and "${" in value and "}" in value + + +def _resolve_from_video_cfg(value: Any, video_cfg: dict[str, Any]) -> Any: + if not _is_unresolved_interpolation(value): + return value + text = str(value).strip() + if not (text.startswith("${") and text.endswith("}")): + return value + expr = text[2:-1] + if not expr.startswith("video_dit_config."): + return value + key = expr.split(".", 1)[1] + if key not in video_cfg: + return value + resolved = video_cfg[key] + return value if _is_unresolved_interpolation(resolved) else resolved + + +def _interpolate_last_dim(tensor: torch.Tensor, new_size: int) -> torch.Tensor: + if tensor.shape[-1] == new_size: + return tensor + flat = tensor.reshape(-1, 1, tensor.shape[-1]).to(torch.float32) + flat = F.interpolate(flat, size=new_size, mode="linear", align_corners=True) + return flat.reshape(*tensor.shape[:-1], new_size) + + +def _resize_tensor_to_shape(src: torch.Tensor, target_shape: tuple[int, ...]) -> torch.Tensor: + if tuple(src.shape) == tuple(target_shape): + return src + + out = src.to(torch.float32) + while out.ndim < len(target_shape): + out = out.unsqueeze(0) + while out.ndim > len(target_shape): + if out.shape[0] != 1: + raise ValueError( + f"Cannot reduce tensor rank for resize: src shape={tuple(src.shape)}, target={target_shape}" + ) + out = out.squeeze(0) + + for dim, new_size in enumerate(target_shape): + current_size = out.shape[dim] + if current_size == new_size: + continue + # Permute the target dimension to the end for interpolation + perm = [i for i in range(out.ndim) if i != dim] + [dim] + # Construct inverse permutation to restore original order + inv_perm = [0] * out.ndim + for i, p in enumerate(perm): + inv_perm[p] = i + # Permute, interpolate, and restore original order + out_perm = out.permute(*perm).contiguous() + prefix_shape = out_perm.shape[:-1] + out_perm = _interpolate_last_dim(out_perm, new_size) + out_perm = out_perm.reshape(*prefix_shape, new_size) + out = out_perm.permute(*inv_perm).contiguous() + + if tuple(out.shape) != tuple(target_shape): + raise ValueError( + f"Resize produced wrong shape for tensor. src={tuple(src.shape)}, target={target_shape}, got={tuple(out.shape)}" + ) + return out.to(dtype=src.dtype) + + +def _load_model_config(path: Path) -> tuple[dict[str, Any], dict[str, Any]]: + cfg = OmegaConf.load(str(path)) + if "video_dit_config" not in cfg or "action_dit_config" not in cfg: + raise ValueError( + f"`{path}` must contain both `video_dit_config` and `action_dit_config` at top level." + ) + + video_cfg = OmegaConf.to_container(cfg.video_dit_config, resolve=False) + action_cfg = OmegaConf.to_container(cfg.action_dit_config, resolve=False) + if not isinstance(video_cfg, dict) or not isinstance(action_cfg, dict): + raise ValueError("`video_dit_config` and `action_dit_config` must resolve to dicts.") + + if _is_unresolved_interpolation(video_cfg.get("action_dim")): + print("[WARN] `video_dit_config.action_dim` is unresolved; defaulting to 7 for preprocessing.") + video_cfg["action_dim"] = 7 + + if _is_unresolved_interpolation(action_cfg.get("action_dim")): + print("[WARN] `action_dit_config.action_dim` is unresolved; defaulting to 7 for preprocessing.") + action_cfg["action_dim"] = 7 + + for key in ["num_heads", "attn_head_dim", "num_layers", "text_dim", "freq_dim"]: + action_cfg[key] = _resolve_from_video_cfg(action_cfg.get(key), video_cfg) + + return video_cfg, action_cfg, cfg + + +def _require_int_config(cfg: dict[str, Any], key: str) -> int: + value = cfg.get(key) + if _is_unresolved_interpolation(value): + raise ValueError(f"`{key}` is unresolved interpolation: {value}") + return int(value) + + +def _require_float_config(cfg: dict[str, Any], key: str) -> float: + value = cfg.get(key) + if _is_unresolved_interpolation(value): + raise ValueError(f"`{key}` is unresolved interpolation: {value}") + return float(value) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Preprocess ActionDiT backbone weights from WanVideoDiT and save as .pt payload." + ) + parser.add_argument("--model-config", required=True, help="Path to model yaml, e.g. configs/model/fastwam.yaml") + parser.add_argument("--output", required=True, help="Output .pt path for preprocessed ActionDiT backbone.") + parser.add_argument("--device", default="cpu", help="Device for loading model and preprocessing.") + parser.add_argument("--dtype", default="float32", choices=["float32", "float16", "bfloat16"]) + parser.add_argument( + "--apply-alpha-scaling", + default="true", + help="Whether to apply alpha=sqrt(dv/da) when the last dimension is resized (true/false). Default: true.", + ) + args = parser.parse_args() + + model_config_path = Path(args.model_config) + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + apply_alpha_scaling = _parse_bool(args.apply_alpha_scaling) + + video_cfg, action_cfg, cfg = _load_model_config(model_config_path) + torch_dtype = _parse_dtype(args.dtype) + redirect_common_files = _parse_bool(cfg.get("redirect_common_files", False)) + + int_fields = ["hidden_dim", "action_dim", "ffn_dim", "num_layers", "num_heads", "attn_head_dim", "text_dim", "freq_dim"] + for key in int_fields: + action_cfg[key] = _require_int_config(action_cfg, key) + action_cfg["eps"] = _require_float_config(action_cfg, "eps") + + print(f"[INFO] Loaded model config from {model_config_path}. " + f"Preprocessing ActionDiT backbone with dtype={torch_dtype} on device={args.device}, " + f"apply_alpha_scaling={apply_alpha_scaling}.") + components = load_wan22_ti2v_5b_components( + device=args.device, + torch_dtype=torch_dtype, + model_id=cfg.get("model_id", "Wan-AI/Wan2.2-TI2V-5B"), + tokenizer_model_id=cfg.get("tokenizer_model_id", "Wan-AI/Wan2.1-T2V-1.3B"), + redirect_common_files=redirect_common_files, + dit_config=video_cfg, + ) + video_expert = components.dit + + action_expert = ActionDiT(**action_cfg).to(device=args.device, dtype=torch_dtype) + if int(action_cfg["num_heads"]) != int(video_expert.num_heads): + raise ValueError("ActionDiT `num_heads` must match video expert for MoT mixed attention.") + if int(action_cfg["attn_head_dim"]) != int(video_expert.attn_head_dim): + raise ValueError("ActionDiT `attn_head_dim` must match video expert for MoT mixed attention.") + if int(action_cfg["num_layers"]) != int(len(video_expert.blocks)): + raise ValueError("ActionDiT `num_layers` must match video expert.") + + action_state = action_expert.state_dict() + video_state = video_expert.state_dict() + backbone_keys = ActionDiT.backbone_key_set(action_state.keys()) + + backbone_state_dict: dict[str, torch.Tensor] = {} + copied = 0 + interpolated = 0 + for key in sorted(backbone_keys): + if key not in video_state: + raise ValueError(f"Key `{key}` not found in video expert state dict.") + src = video_state[key] + target = action_state[key] + if tuple(src.shape) == tuple(target.shape): + value = src + copied += 1 + else: + value = _resize_tensor_to_shape(src, tuple(target.shape)) + if apply_alpha_scaling and src.ndim >= 2 and src.shape[-1] != target.shape[-1]: + alpha = (float(src.shape[-1]) / float(target.shape[-1])) ** 0.5 + value = value.to(torch.float32) * alpha + interpolated += 1 + backbone_state_dict[key] = value.detach().to(dtype=target.dtype, device="cpu").contiguous() + + payload = { + "policy": { + "skip_prefixes": list(ActionDiT.ACTION_BACKBONE_SKIP_PREFIXES), + "alpha_scaling": bool(apply_alpha_scaling), + "interpolation": "sequential_1d_linear_align_corners_true", + }, + "backbone_state_dict": backbone_state_dict, + "meta": { + "hidden_dim": int(action_cfg["hidden_dim"]), + "ffn_dim": int(action_cfg["ffn_dim"]), + "num_layers": int(action_cfg["num_layers"]), + "num_heads": int(action_cfg["num_heads"]), + "attn_head_dim": int(action_cfg["attn_head_dim"]), + "text_dim": int(action_cfg["text_dim"]), + "freq_dim": int(action_cfg["freq_dim"]), + "eps": float(action_cfg["eps"]), + }, + } + torch.save(payload, str(output_path)) + + skipped = len(action_state) - len(backbone_keys) + print( + "[INFO] Saved ActionDiT backbone payload to " + f"{output_path} (copied={copied}, interpolated={interpolated}, skipped={skipped})." + ) + + +if __name__ == "__main__": + main() diff --git a/baseline/fastwam_zipper/source/scripts/train.py b/baseline/fastwam_zipper/source/scripts/train.py new file mode 100644 index 0000000000000000000000000000000000000000..9f91819a390da26147b9e0fa3f634aecea78f85d --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/train.py @@ -0,0 +1,16 @@ +import hydra +from omegaconf import DictConfig + +from fastwam.runtime import run_training +from fastwam.utils.config_resolvers import register_default_resolvers + +register_default_resolvers() + + +@hydra.main(config_path="../configs", config_name="train", version_base="1.3") +def main(cfg: DictConfig): + run_training(cfg) + + +if __name__ == "__main__": + main() diff --git a/baseline/fastwam_zipper/source/scripts/train_galaxea_r1_lite.sh b/baseline/fastwam_zipper/source/scripts/train_galaxea_r1_lite.sh new file mode 100644 index 0000000000000000000000000000000000000000..d74724d51de65fcd60e7373c045d30bc2dbdc086 --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/train_galaxea_r1_lite.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RUNTIME_ROOT="${FASTWAM_RUNTIME_ROOT:-/code/fastwam-runtime-v1}" +PYTHON="${RUNTIME_ROOT}/bin/python" +NPROC_PER_NODE="${1:?usage: scripts/train_galaxea_r1_lite.sh NPROC_PER_NODE RUN_NAME [hydra overrides...]}" +RUN_NAME="${2:?usage: scripts/train_galaxea_r1_lite.sh NPROC_PER_NODE RUN_NAME [hydra overrides...]}" +shift 2 + +CANONICAL_TRAIN_DATA="${GALAXEA_CANONICAL_TRAIN_DATA:-/dataset_rc_o1_p1/qshou@xiaopeng.com/dataset/galaxea_real_baselines_derived/pi05_lerobot_v2/train}" +CANONICAL_VAL_DATA="${GALAXEA_CANONICAL_VAL_DATA:-/dataset_rc_o1_p1/qshou@xiaopeng.com/dataset/galaxea_real_baselines_derived/pi05_lerobot_v2/validation}" +TRAIN_DATA="${GALAXEA_FASTWAM_TRAIN_DATA:-/dataset_rc_o1_p1/qshou@xiaopeng.com/dataset/galaxea_real_baselines_derived/fastwam_lerobot_v2/train}" +VAL_DATA="${GALAXEA_FASTWAM_VAL_DATA:-/dataset_rc_o1_p1/qshou@xiaopeng.com/dataset/galaxea_real_baselines_derived/fastwam_lerobot_v2/validation}" +MANIFEST="${GALAXEA_MANIFEST:-${ROOT}/../reports/galaxea_baselines/source_protocol/episodes_final.jsonl}" +BASE_CKPT="${FASTWAM_BASE_CHECKPOINT:-/dataset_rc_o1_p1/qshou@xiaopeng.com/weights/real_world_baselines/fastwam/robotwin_uncond_3cam_384.pt}" +CKPT_MARKER="$(dirname "${BASE_CKPT}")/DOWNLOAD_VERIFIED.json" +RUN_ROOT="${GALAXEA_FASTWAM_RUN_ROOT:-/dataset_rc_o1_p1/qshou@xiaopeng.com/training_runs/galaxea_real_baselines/fastwam}" +OUTPUT_DIR="${RUN_ROOT}/${RUN_NAME}" +EXPECTED_CKPT_SIZE=12041813092 +EXPECTED_CKPT_SHA256="776475b22566a791854ecf31cf3b50f25e7d8d94c343132ec16eb94994aa9e63" +EXPECTED_TRAIN_EPISODES="${GALAXEA_EXPECTED_TRAIN_EPISODES:-511}" +EXPECTED_VAL_EPISODES="${GALAXEA_EXPECTED_VAL_EPISODES:-57}" +NUM_EPOCHS="${NUM_EPOCHS:-10}" +PER_DEVICE_BATCH_SIZE="${PER_DEVICE_BATCH_SIZE:-8}" +GRADIENT_ACCUMULATION_STEPS="${GRADIENT_ACCUMULATION_STEPS:-2}" +MIN_FREE_GIB="${GALAXEA_RUN_MIN_FREE_GIB:-2048}" + +[[ "${NPROC_PER_NODE}" =~ ^[1-9][0-9]*$ ]] || { echo "invalid process count" >&2; exit 2; } +[[ "${RUN_NAME}" =~ ^[A-Za-z0-9._-]+$ ]] || { echo "unsafe run name" >&2; exit 2; } +[[ "${NUM_EPOCHS}" =~ ^[1-9][0-9]*$ ]] || { echo "invalid NUM_EPOCHS" >&2; exit 2; } +[[ "${PER_DEVICE_BATCH_SIZE}" =~ ^[1-9][0-9]*$ ]] || { echo "invalid PER_DEVICE_BATCH_SIZE" >&2; exit 2; } +[[ "${GRADIENT_ACCUMULATION_STEPS}" =~ ^[1-9][0-9]*$ ]] || { echo "invalid GRADIENT_ACCUMULATION_STEPS" >&2; exit 2; } +[[ -x "${PYTHON}" ]] || { echo "missing FastWAM runtime: ${PYTHON}" >&2; exit 2; } +[[ -f "${BASE_CKPT}" ]] || { echo "missing FastWAM checkpoint: ${BASE_CKPT}" >&2; exit 2; } +[[ "$(stat -c %s "${BASE_CKPT}")" = "${EXPECTED_CKPT_SIZE}" ]] || { echo "FastWAM checkpoint size mismatch" >&2; exit 2; } +[[ "$(sha256sum "${BASE_CKPT}" | awk '{print $1}')" = "${EXPECTED_CKPT_SHA256}" ]] || { echo "FastWAM checkpoint SHA256 mismatch" >&2; exit 2; } +[[ -f "${CKPT_MARKER}" ]] || { echo "missing FastWAM checkpoint verification marker" >&2; exit 2; } +"${PYTHON}" - "${BASE_CKPT}" "${CKPT_MARKER}" <<'PY' +import json +import pathlib +import sys + +checkpoint = pathlib.Path(sys.argv[1]) +marker = json.loads(pathlib.Path(sys.argv[2]).read_text(encoding="utf-8")) +if marker.get("file") != checkpoint.name: + raise SystemExit("FastWAM checkpoint marker filename mismatch") +if int(marker.get("size_bytes", -1)) != checkpoint.stat().st_size: + raise SystemExit("FastWAM checkpoint marker size mismatch") +if marker.get("sha256") != "776475b22566a791854ecf31cf3b50f25e7d8d94c343132ec16eb94994aa9e63": + raise SystemExit("FastWAM checkpoint marker SHA256 mismatch") +PY +mkdir -p "${RUN_ROOT}" +available_kib="$(df -Pk "${RUN_ROOT}" | awk 'NR==2 {print $4}')" +required_kib="$((MIN_FREE_GIB * 1024 * 1024))" +(( available_kib >= required_kib )) || { + echo "insufficient run space: available_kib=${available_kib} required_kib=${required_kib}" >&2; exit 4; +} + +resume_path="${BASE_CKPT}" +if [[ -e "${OUTPUT_DIR}" ]]; then + latest_state="$(find "${OUTPUT_DIR}/checkpoints/state" -mindepth 1 -maxdepth 1 -type d -name 'step_[0-9]*' -printf '%f\n' 2>/dev/null | sort | tail -n 1)" + [[ -n "${latest_state}" ]] || { + echo "existing output has no resumable state: ${OUTPUT_DIR}" >&2; exit 3; + } + resume_path="${OUTPUT_DIR}/checkpoints/state/${latest_state}" +fi + +export GALAXEA_FASTWAM_TRAIN_DATA="${TRAIN_DATA}" +export GALAXEA_FASTWAM_VAL_DATA="${VAL_DATA}" +export DIFFSYNTH_MODEL_BASE_PATH="${DIFFSYNTH_MODEL_BASE_PATH:-/workspace-wl/shouqx@xiaopeng.com/model}" +export DIFFSYNTH_SKIP_DOWNLOAD=true +export PATH="${RUNTIME_ROOT}/bin:${PATH}" +export WANDB_MODE="${WANDB_MODE:-online}" +[[ "${WANDB_MODE}" != online || -n "${WANDB_API_KEY:-}" ]] || { + echo "WANDB_MODE=online requires WANDB_API_KEY" >&2; exit 2; +} + +cd "${ROOT}" +"${PYTHON}" scripts/prepare_galaxea_r1_lite_overlay.py \ + --source "${CANONICAL_TRAIN_DATA}" --destination "${TRAIN_DATA}" +"${PYTHON}" scripts/prepare_galaxea_r1_lite_overlay.py \ + --source "${CANONICAL_VAL_DATA}" --destination "${VAL_DATA}" +"${PYTHON}" scripts/check_galaxea_r1_lite_contract.py \ + --train "${TRAIN_DATA}" --validation "${VAL_DATA}" --manifest "${MANIFEST}" \ + --expected-train-episodes "${EXPECTED_TRAIN_EPISODES}" \ + --expected-validation-episodes "${EXPECTED_VAL_EPISODES}" + +effective_global_batch_size="$((PER_DEVICE_BATCH_SIZE * NPROC_PER_NODE * GRADIENT_ACCUMULATION_STEPS))" +steps_per_epoch="$("${PYTHON}" "${ROOT}/../scripts/plan_galaxea_epochs.py" \ + "${TRAIN_DATA}" --epochs "${NUM_EPOCHS}" \ + --effective-global-batch-size "${effective_global_batch_size}" \ + --field steps_per_epoch)" + +if [[ "${FASTWAM_PREFLIGHT_ONLY:-0}" = "1" ]]; then + echo "GALEXEA_FASTWAM_PREFLIGHT_OK" + exit 0 +fi + +echo "GALAXEA_FASTWAM_LAUNCH gpus=${NPROC_PER_NODE} effective_gbs=${effective_global_batch_size} epochs=${NUM_EPOCHS} steps_per_epoch=${steps_per_epoch} dataset=${TRAIN_DATA}" + +if [[ "${FASTWAM_SKIP_TEXT_PRECOMPUTE:-0}" == "1" ]]; then + "${PYTHON}" scripts/check_galaxea_text_cache.py \ + --train "${TRAIN_DATA}" --validation "${VAL_DATA}" \ + --cache "${GALAXEA_FASTWAM_TEXT_CACHE:-/dataset_rc_o1_p1/qshou@xiaopeng.com/dataset/galaxea_real_baselines_derived/fastwam_text_cache}" +else + "${PYTHON}" -m torch.distributed.run --standalone --nproc_per_node="${NPROC_PER_NODE}" \ + scripts/precompute_text_embeds.py task=galaxea_r1_lite_3cam_1e-4 +overwrite=false +fi + +RUN_ID="${RUN_NAME}" bash scripts/train_zero1.sh "${NPROC_PER_NODE}" \ + task=galaxea_r1_lite_3cam_1e-4 \ + "output_dir=${OUTPUT_DIR}" \ + "resume=${resume_path}" \ + "num_epochs=${NUM_EPOCHS}" \ + "batch_size=${PER_DEVICE_BATCH_SIZE}" \ + "gradient_accumulation_steps=${GRADIENT_ACCUMULATION_STEPS}" \ + "save_every=${steps_per_epoch}" \ + "eval_every=${steps_per_epoch}" \ + "wandb.enabled=true" \ + "wandb.mode=online" \ + "wandb.project=galaxea-real-baselines" \ + "wandb.group=${GALAXEA_WANDB_GROUP:-galaxea-real-baselines-10ep}" \ + "wandb.name=${RUN_NAME}" \ + "$@" diff --git a/baseline/fastwam_zipper/source/scripts/train_zero1.sh b/baseline/fastwam_zipper/source/scripts/train_zero1.sh new file mode 100644 index 0000000000000000000000000000000000000000..66e0ca312826433db3bfb854187d6f081875ed65 --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/train_zero1.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +set -euo pipefail + +NPROC_PER_NODE="${1:?Usage: bash scripts/train_zero1.sh [hydra_overrides...]}" +shift + +EXTRA_ARGS=("$@") +NUM_MACHINES="${NNODES:-1}" +MACHINE_RANK="${NODE_RANK:-0}" +MAIN_PROCESS_IP="${MASTER_ADDR:-127.0.0.1}" +MAIN_PROCESS_PORT="${MASTER_PORT:-29500}" + +is_integer() { + [[ "${1}" =~ ^[0-9]+$ ]] +} + +if ! is_integer "${NUM_MACHINES}" || ! is_integer "${MACHINE_RANK}"; then + echo "Error: NUM_MACHINES (${NUM_MACHINES}) and MACHINE_RANK (${MACHINE_RANK}) must be integers." >&2 + exit 1 +fi + +extract_task_basename() { + local cfg="$1" + if [[ "${cfg}" == task/* ]]; then + local name="${cfg#task/}" + name="${name%.yaml}" + echo "${name}" + return 0 + fi + return 1 +} + +TASK_BASENAME="train" +for ((i = 0; i < ${#EXTRA_ARGS[@]}; i++)); do + arg="${EXTRA_ARGS[$i]}" + case "${arg}" in + --config-name) + if ((i + 1 < ${#EXTRA_ARGS[@]})); then + next="${EXTRA_ARGS[$((i + 1))]}" + if parsed="$(extract_task_basename "${next}")"; then + TASK_BASENAME="${parsed}" + fi + fi + ;; + --config-name=*) + cfg="${arg#--config-name=}" + if parsed="$(extract_task_basename "${cfg}")"; then + TASK_BASENAME="${parsed}" + fi + ;; + task=*) + cfg="${arg#task=}" + cfg="${cfg%.yaml}" + TASK_BASENAME="${cfg}" + ;; + esac +done + +if [[ -z "${RUN_ID:-}" ]]; then + if (( NUM_MACHINES <= 1 )); then + RUN_ID="$(date +%Y-%m-%d_%H-%M-%S)" + else + RUN_ID_SYNC_TIMEOUT="${RUN_ID_SYNC_TIMEOUT:-180}" + RUN_ID_SYNC_PORT="${RUN_ID_SYNC_PORT:-$((MAIN_PROCESS_PORT + 11))}" + + export RUN_ID_SYNC_HOST="${MAIN_PROCESS_IP}" + export RUN_ID_SYNC_PORT + export RUN_ID_SYNC_TIMEOUT + export RUN_ID_SYNC_MACHINE_RANK="${MACHINE_RANK}" + export RUN_ID_SYNC_NUM_MACHINES="${NUM_MACHINES}" + export RUN_ID_SYNC_TASK_BASENAME="${TASK_BASENAME}" + + RUN_ID="$( + python - <<'PY' +import datetime +import os +from datetime import timedelta + +import torch.distributed as dist + +host = os.environ["RUN_ID_SYNC_HOST"] +port = int(os.environ["RUN_ID_SYNC_PORT"]) +timeout_s = int(os.environ["RUN_ID_SYNC_TIMEOUT"]) +machine_rank = int(os.environ["RUN_ID_SYNC_MACHINE_RANK"]) +num_machines = int(os.environ["RUN_ID_SYNC_NUM_MACHINES"]) +task_basename = os.environ.get("RUN_ID_SYNC_TASK_BASENAME", "train") + +store = dist.TCPStore( + host_name=host, + port=port, + world_size=num_machines, + is_master=(machine_rank == 0), + timeout=timedelta(seconds=timeout_s), +) +key = f"run_id::{task_basename}" +if machine_rank == 0: + run_id = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + store.set(key, run_id) +run_id = store.get(key).decode("utf-8") +print(run_id) +PY + )" + + echo "[run_id_sync] mode=tcpstore host=${RUN_ID_SYNC_HOST} port=${RUN_ID_SYNC_PORT} timeout_s=${RUN_ID_SYNC_TIMEOUT} run_id=${RUN_ID}" + fi +fi + +echo "[launch] nproc_per_node=${NPROC_PER_NODE} num_machines=${NUM_MACHINES} machine_rank=${MACHINE_RANK} run_id=${RUN_ID}" + +python -m accelerate.commands.launch \ + --config_file scripts/accelerate_configs/accelerate_zero1_ds.yaml \ + --num_processes "${NPROC_PER_NODE}" \ + scripts/train.py \ + "output_dir=./runs/${TASK_BASENAME}/${RUN_ID}" \ + "wandb.name=${TASK_BASENAME}" \ + "${EXTRA_ARGS[@]}" diff --git a/baseline/fastwam_zipper/source/scripts/train_zero2.sh b/baseline/fastwam_zipper/source/scripts/train_zero2.sh new file mode 100644 index 0000000000000000000000000000000000000000..904f012ca3116e4ffb974be35cc0682350a41f09 --- /dev/null +++ b/baseline/fastwam_zipper/source/scripts/train_zero2.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +set -euo pipefail + +NPROC_PER_NODE="${1:?Usage: bash scripts/train.sh [hydra_overrides...]}" +shift + +EXTRA_ARGS=("$@") +NUM_MACHINES="${NNODES:-1}" +MACHINE_RANK="${NODE_RANK:-0}" +MAIN_PROCESS_IP="${MASTER_ADDR:-127.0.0.1}" +MAIN_PROCESS_PORT="${MASTER_PORT:-29500}" + +is_integer() { + [[ "${1}" =~ ^[0-9]+$ ]] +} + +if ! is_integer "${NUM_MACHINES}" || ! is_integer "${MACHINE_RANK}"; then + echo "Error: NUM_MACHINES (${NUM_MACHINES}) and MACHINE_RANK (${MACHINE_RANK}) must be integers." >&2 + exit 1 +fi + +extract_task_basename() { + local cfg="$1" + if [[ "${cfg}" == task/* ]]; then + local name="${cfg#task/}" + name="${name%.yaml}" + echo "${name}" + return 0 + fi + return 1 +} + +TASK_BASENAME="train" +for ((i = 0; i < ${#EXTRA_ARGS[@]}; i++)); do + arg="${EXTRA_ARGS[$i]}" + case "${arg}" in + --config-name) + if ((i + 1 < ${#EXTRA_ARGS[@]})); then + next="${EXTRA_ARGS[$((i + 1))]}" + if parsed="$(extract_task_basename "${next}")"; then + TASK_BASENAME="${parsed}" + fi + fi + ;; + --config-name=*) + cfg="${arg#--config-name=}" + if parsed="$(extract_task_basename "${cfg}")"; then + TASK_BASENAME="${parsed}" + fi + ;; + task=*) + cfg="${arg#task=}" + cfg="${cfg%.yaml}" + TASK_BASENAME="${cfg}" + ;; + esac +done + +if [[ -z "${RUN_ID:-}" ]]; then + if (( NUM_MACHINES <= 1 )); then + RUN_ID="$(date +%Y-%m-%d_%H-%M-%S)" + else + RUN_ID_SYNC_TIMEOUT="${RUN_ID_SYNC_TIMEOUT:-180}" + RUN_ID_SYNC_PORT="${RUN_ID_SYNC_PORT:-$((MAIN_PROCESS_PORT + 11))}" + + export RUN_ID_SYNC_HOST="${MAIN_PROCESS_IP}" + export RUN_ID_SYNC_PORT + export RUN_ID_SYNC_TIMEOUT + export RUN_ID_SYNC_MACHINE_RANK="${MACHINE_RANK}" + export RUN_ID_SYNC_NUM_MACHINES="${NUM_MACHINES}" + export RUN_ID_SYNC_TASK_BASENAME="${TASK_BASENAME}" + + RUN_ID="$( + python - <<'PY' +import datetime +import os +from datetime import timedelta + +import torch.distributed as dist + +host = os.environ["RUN_ID_SYNC_HOST"] +port = int(os.environ["RUN_ID_SYNC_PORT"]) +timeout_s = int(os.environ["RUN_ID_SYNC_TIMEOUT"]) +machine_rank = int(os.environ["RUN_ID_SYNC_MACHINE_RANK"]) +num_machines = int(os.environ["RUN_ID_SYNC_NUM_MACHINES"]) +task_basename = os.environ.get("RUN_ID_SYNC_TASK_BASENAME", "train") + +store = dist.TCPStore( + host_name=host, + port=port, + world_size=num_machines, + is_master=(machine_rank == 0), + timeout=timedelta(seconds=timeout_s), +) +key = f"run_id::{task_basename}" +if machine_rank == 0: + run_id = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + store.set(key, run_id) +run_id = store.get(key).decode("utf-8") +print(run_id) +PY + )" + + echo "[run_id_sync] mode=tcpstore host=${RUN_ID_SYNC_HOST} port=${RUN_ID_SYNC_PORT} timeout_s=${RUN_ID_SYNC_TIMEOUT} run_id=${RUN_ID}" + fi +fi + +echo "[launch] nproc_per_node=${NPROC_PER_NODE} num_machines=${NUM_MACHINES} machine_rank=${MACHINE_RANK} run_id=${RUN_ID}" + +python -m accelerate.commands.launch \ + --config_file scripts/accelerate_configs/accelerate_zero2_ds.yaml \ + --num_processes "${NPROC_PER_NODE}" \ + scripts/train.py \ + "output_dir=./runs/${TASK_BASENAME}/${RUN_ID}" \ + "wandb.name=${TASK_BASENAME}" \ + "${EXTRA_ARGS[@]}" diff --git a/baseline/fastwam_zipper/source/src/fastwam.egg-info/PKG-INFO b/baseline/fastwam_zipper/source/src/fastwam.egg-info/PKG-INFO new file mode 100644 index 0000000000000000000000000000000000000000..e31946f999ad5fe53c477f732cfcec7e469df1a0 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam.egg-info/PKG-INFO @@ -0,0 +1,37 @@ +Metadata-Version: 2.4 +Name: fastwam +Version: 0.1.0 +Summary: FastWAM: standalone Wan2.2-TI2V-5B training/inference package +Requires-Python: >=3.10 +License-File: LICENSE +Requires-Dist: accelerate==1.12.0 +Requires-Dist: av==15.1.0 +Requires-Dist: boto3==1.35.99 +Requires-Dist: datasets==4.8.5 +Requires-Dist: deepspeed==0.18.7 +Requires-Dist: einops==0.8.1 +Requires-Dist: gitpython==3.1.45 +Requires-Dist: huggingface-hub==0.29.2 +Requires-Dist: hydra-core==1.3.2 +Requires-Dist: imageio==2.37.0 +Requires-Dist: imageio-ffmpeg==0.6.0 +Requires-Dist: jsonlines==4.0.0 +Requires-Dist: modelscope==1.34.0 +Requires-Dist: numpy==2.2.6 +Requires-Dist: omegaconf==2.3.0 +Requires-Dist: packaging==25.0 +Requires-Dist: pandas==2.2.3 +Requires-Dist: pillow==12.0.0 +Requires-Dist: pyarrow==24.0.0 +Requires-Dist: regex==2025.11.3 +Requires-Dist: rich==14.2.0 +Requires-Dist: safetensors==0.5.3 +Requires-Dist: termcolor==2.5.0 +Requires-Dist: torch==2.7.1+cu128 +Requires-Dist: torchcodec==0.4.0 +Requires-Dist: torchvision==0.22.1+cu128 +Requires-Dist: tqdm==4.68.3 +Requires-Dist: transformers==4.49.0 +Requires-Dist: typing-extensions==4.15.0 +Requires-Dist: wandb==0.23.1 +Dynamic: license-file diff --git a/baseline/fastwam_zipper/source/src/fastwam.egg-info/SOURCES.txt b/baseline/fastwam_zipper/source/src/fastwam.egg-info/SOURCES.txt new file mode 100644 index 0000000000000000000000000000000000000000..5dcb70a30388accc43b23ec5c931cacc62b30474 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam.egg-info/SOURCES.txt @@ -0,0 +1,60 @@ +LICENSE +README.md +pyproject.toml +src/fastwam/__init__.py +src/fastwam/runtime.py +src/fastwam/trainer.py +src/fastwam.egg-info/PKG-INFO +src/fastwam.egg-info/SOURCES.txt +src/fastwam.egg-info/dependency_links.txt +src/fastwam.egg-info/requires.txt +src/fastwam.egg-info/top_level.txt +src/fastwam/datasets/__init__.py +src/fastwam/datasets/dataset_utils.py +src/fastwam/datasets/lerobot/base_lerobot_dataset.py +src/fastwam/datasets/lerobot/constants.py +src/fastwam/datasets/lerobot/robot_video_dataset.py +src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py +src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py +src/fastwam/datasets/lerobot/lerobot/datasets/utils.py +src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py +src/fastwam/datasets/lerobot/processors/base_processor.py +src/fastwam/datasets/lerobot/processors/fastwam_processor.py +src/fastwam/datasets/lerobot/transforms/action_state_merger.py +src/fastwam/datasets/lerobot/transforms/image.py +src/fastwam/datasets/lerobot/transforms/misc.py +src/fastwam/datasets/lerobot/transforms/relative_action.py +src/fastwam/datasets/lerobot/transforms/rotation.py +src/fastwam/datasets/lerobot/utils/normalizer.py +src/fastwam/datasets/lerobot/utils/rotation.py +src/fastwam/datasets/lerobot3/__init__.py +src/fastwam/datasets/lerobot3/base_lerobot_dataset.py +src/fastwam/datasets/lerobot3/lerobot_dataset.py +src/fastwam/datasets/lerobot3/robot_video_dataset.py +src/fastwam/models/__init__.py +src/fastwam/models/wan22/__init__.py +src/fastwam/models/wan22/action_dit.py +src/fastwam/models/wan22/fastwam.py +src/fastwam/models/wan22/fastwam_idm.py +src/fastwam/models/wan22/fastwam_joint.py +src/fastwam/models/wan22/fastwam_optional_idm.py +src/fastwam/models/wan22/mot.py +src/fastwam/models/wan22/wan22.py +src/fastwam/models/wan22/wan_video_dit.py +src/fastwam/models/wan22/wan_video_text_encoder.py +src/fastwam/models/wan22/wan_video_vae.py +src/fastwam/models/wan22/helpers/__init__.py +src/fastwam/models/wan22/helpers/gradient.py +src/fastwam/models/wan22/helpers/io.py +src/fastwam/models/wan22/helpers/loader.py +src/fastwam/models/wan22/helpers/state_dict_converters.py +src/fastwam/models/wan22/schedulers/scheduler_continuous.py +src/fastwam/utils/__init__.py +src/fastwam/utils/config_resolvers.py +src/fastwam/utils/fs.py +src/fastwam/utils/logging_config.py +src/fastwam/utils/misc.py +src/fastwam/utils/pytorch_utils.py +src/fastwam/utils/samplers.py +src/fastwam/utils/video_io.py +src/fastwam/utils/video_metrics.py \ No newline at end of file diff --git a/baseline/fastwam_zipper/source/src/fastwam.egg-info/dependency_links.txt b/baseline/fastwam_zipper/source/src/fastwam.egg-info/dependency_links.txt new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/baseline/fastwam_zipper/source/src/fastwam.egg-info/requires.txt b/baseline/fastwam_zipper/source/src/fastwam.egg-info/requires.txt new file mode 100644 index 0000000000000000000000000000000000000000..ef872f7d508661547da42709a53be726574e7a72 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam.egg-info/requires.txt @@ -0,0 +1,30 @@ +accelerate==1.12.0 +av==15.1.0 +boto3==1.35.99 +datasets==4.8.5 +deepspeed==0.18.7 +einops==0.8.1 +gitpython==3.1.45 +huggingface-hub==0.29.2 +hydra-core==1.3.2 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +jsonlines==4.0.0 +modelscope==1.34.0 +numpy==2.2.6 +omegaconf==2.3.0 +packaging==25.0 +pandas==2.2.3 +pillow==12.0.0 +pyarrow==24.0.0 +regex==2025.11.3 +rich==14.2.0 +safetensors==0.5.3 +termcolor==2.5.0 +torch==2.7.1+cu128 +torchcodec==0.4.0 +torchvision==0.22.1+cu128 +tqdm==4.68.3 +transformers==4.49.0 +typing-extensions==4.15.0 +wandb==0.23.1 diff --git a/baseline/fastwam_zipper/source/src/fastwam.egg-info/top_level.txt b/baseline/fastwam_zipper/source/src/fastwam.egg-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..85d77647bac60aff1118e18e3b83392bebcea53b --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam.egg-info/top_level.txt @@ -0,0 +1 @@ +fastwam diff --git a/baseline/fastwam_zipper/source/src/fastwam/__init__.py b/baseline/fastwam_zipper/source/src/fastwam/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f40dcb36ffb7cb5999600c8d9520cd72e3f60af0 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/__init__.py @@ -0,0 +1 @@ +"""FastWAM package.""" diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/__init__.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/dataset_utils.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/dataset_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e7d5830119dd42c492ec3ba5c01081c26b836d71 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/dataset_utils.py @@ -0,0 +1,109 @@ + +from typing import Optional + +import torch +import torchvision.transforms.functional as transforms_F +from PIL import Image + + +def obtain_image_size(data: torch.Tensor | Image.Image) -> tuple[int, int]: + r"""Return spatial size from a PIL image or image/video tensor. + + Args: + data (torch.Tensor | Image.Image): Input image or video tensor. + Returns: + width (int): Input width. + height (int): Input height. + """ + + if isinstance(data, Image.Image): + width, height = data.size + elif isinstance(data, torch.Tensor): + height, width = data.size()[-2:] + else: + raise ValueError("data to random crop should be PIL Image or tensor") + + return width, height + +class ResizeSmallestSideAspectPreserving: + def __init__(self, args: Optional[dict] = None) -> None: + self.args = args + + def __call__(self, video: torch.Tensor | Image.Image) -> torch.Tensor | Image.Image: + r"""Resize while preserving aspect ratio. + + The output is scaled so both spatial dimensions are at least the + requested target size. + + Args: + video (torch.Tensor | Image.Image): Input image or video tensor. + Returns: + torch.Tensor | Image.Image: Resized image or video tensor. + """ + + assert self.args is not None, "Please specify args in augmentations" + + img_w, img_h = self.args["img_w"], self.args["img_h"] + + orig_w, orig_h = obtain_image_size(video) + scaling_ratio = max((img_w / orig_w), (img_h / orig_h)) + target_size = (int(scaling_ratio * orig_h + 0.5), int(scaling_ratio * orig_w + 0.5)) + + assert ( + target_size[0] >= img_h and target_size[1] >= img_w + ), f"Resize error. orig {(orig_w, orig_h)} desire {(img_w, img_h)} compute {target_size}" + + return transforms_F.resize( + video, + size=target_size, # type: ignore + interpolation=self.args.get("interpolation", transforms_F.InterpolationMode.BICUBIC), + antialias=True, + ) + + +class CenterCrop: + def __init__(self, args: Optional[dict] = None) -> None: + self.args = args + + def __call__(self, video: torch.Tensor | Image.Image) -> torch.Tensor | Image.Image: + r"""Center crop to the requested spatial size. + + Args: + video (torch.Tensor | Image.Image): Input image or video tensor. + Returns: + torch.Tensor | Image.Image: Center cropped image or video tensor. + """ + assert ( + (self.args is not None) and ("img_w" in self.args) and ("img_h" in self.args) + ), "Please specify size in args" + + img_w, img_h = self.args["img_w"], self.args["img_h"] + return transforms_F.center_crop(video, [img_h, img_w]) + + +class Normalize: + def __init__(self, args: Optional[dict] = None) -> None: + self.args = args + + def __call__(self, video: torch.Tensor | Image.Image) -> torch.Tensor: + r"""Convert to tensor if needed and normalize by mean/std. + + Args: + video (torch.Tensor | Image.Image): Input image or video tensor. + Returns: + torch.Tensor: Normalized image or video tensor. + """ + assert self.args is not None, "Please specify args" + + mean = self.args["mean"] + std = self.args["std"] + + if isinstance(video, torch.Tensor): + data = video.to(dtype=torch.float32) + if video.dtype == torch.uint8: + data = data / 255.0 + data = data.to(dtype=torch.get_default_dtype()) + else: + data = transforms_F.to_tensor(video) # division by 255 is applied in to_tensor() + + return transforms_F.normalize(tensor=data, mean=mean, std=std) diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/base_lerobot_dataset.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/base_lerobot_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..b59004215bd09028f52199f44732c7dfee958db9 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/base_lerobot_dataset.py @@ -0,0 +1,425 @@ +import torch +import numpy as np +from pathlib import Path +from typing import List, Literal, Dict, Optional, Any, DefaultDict +from tqdm import tqdm +from .lerobot.lerobot_dataset import LeRobotDatasetMetadata, MultiLeRobotDataset + +from concurrent.futures import ThreadPoolExecutor, as_completed +import traceback +from fastwam.utils.logging_config import get_logger +from .processors.base_processor import BaseProcessor + +logger = get_logger(__name__) + +MAX_GETITEM_ATTEMPT = 5 + +class BaseLerobotDataset(torch.utils.data.Dataset): + metadata_cls = LeRobotDatasetMetadata + multi_dataset_cls = MultiLeRobotDataset + presample_images = False + + def __init__( + self, + dataset_dirs: List[str], + + # shapes + shape_meta: Dict[str, Any], + action_size: int = 1, + past_action_size: int = 0, # Excludes the current frame + obs_size: int = 1, # should be + past_obs_size: int = 0, + + # train vs val + val_set_proportion: float = 0.05, + is_training_set: bool = False, + seed: int = 42, + + # sampling + global_sample_stride: int = 1, + image_subsample_stride: int = 1, + tolerance_s: Optional[float] = None, + video_backend: Optional[str] = None, + ): + assert len(dataset_dirs) > 0, "At least one dataset directory is required" + assert past_action_size == 0 + assert past_obs_size == 0 + assert action_size == obs_size - 1, "In this dataset, action_size should be obs_size - 1" + + self.dataset_dirs = dataset_dirs + self.shape_meta = shape_meta + self.action_size = action_size + self.past_action_size = past_action_size + self.obs_size = obs_size + self.image_subsample_stride = image_subsample_stride + self.processor = None # Will be set externally + metas = [] + for ds_dir in dataset_dirs: + ds_root = Path(ds_dir) + repo_id = ds_dir + meta = self.metadata_cls(repo_id=repo_id, root=ds_root) + metas.append(meta) + + fps_list = [m.fps for m in metas] + assert len(set(fps_list)) == 1, f"All dataset_dirs must have the same fps, got {fps_list}" + fps = fps_list[0] + + self.global_sample_stride = global_sample_stride + + self.val_set_proportion = val_set_proportion + self.is_training_set = is_training_set + + self.image_meta = shape_meta["images"] + self.state_meta = shape_meta["state"] + self.action_meta = shape_meta["action"] + + delta_timestamps = {} + for meta in self.image_meta: + key = meta["key"] + meta["lerobot_key"] = f"observation.images.{key}" if key != "default" else "observation.images" + image_stride = self.image_subsample_stride if self.presample_images else 1 + delta_timestamps[meta["lerobot_key"]] = [ + (t * global_sample_stride) / fps + for t in range(-past_obs_size, -past_obs_size + obs_size, image_stride) + ] + + for meta in self.state_meta: + key = meta["key"] + meta["lerobot_key"] = f"observation.state.{key}" if key != "default" else "observation.state" + delta_timestamps[meta["lerobot_key"]] = [ + (t * global_sample_stride) / fps for t in range(-past_obs_size, -past_obs_size + obs_size) + ] + + for meta in self.action_meta: + key = meta["key"] + meta["lerobot_key"] = f"action.{key}" if key != "default" else "action" + delta_timestamps[meta["lerobot_key"]] = [(t * global_sample_stride) / fps for t in range(-past_action_size, -past_action_size + action_size)] + + episodes = {} + if val_set_proportion < 1e-6: + for meta in metas: + episodes.update({meta.repo_id: list(range(meta.total_episodes))}) + else: + for meta in metas: + split_idx = int(meta.total_episodes * (1 - val_set_proportion)) + # random shuffle episode indices before splitting + episode_indices = list(range(meta.total_episodes)) + rng = np.random.default_rng(seed) + rng.shuffle(episode_indices) + if self.is_training_set: + episodes.update({meta.repo_id: [episode_indices[i] for i in range(split_idx)]}) + else: + episodes.update({meta.repo_id: [episode_indices[i] for i in range(split_idx, meta.total_episodes)]}) + + tolerances_s = None if tolerance_s is None else dict.fromkeys(self.dataset_dirs, tolerance_s) + self.multi_dataset = self.multi_dataset_cls( + dataset_dirs=self.dataset_dirs, + episodes=episodes, + delta_timestamps=delta_timestamps, + tolerances_s=tolerances_s, + video_backend=video_backend, + ) + + # HACK: lerobot 3.0 will fix this + episode_data_index = [] + end_index = 0 + for dataset in self.multi_dataset._datasets: + multi_episode_data_index = { + "from": dataset.episode_data_index["from"] + end_index, + "to": dataset.episode_data_index["to"] + end_index, + } + episode_data_index.append(multi_episode_data_index) + end_index = multi_episode_data_index["to"][-1] + + self.episode_data_index = { + "from": torch.cat([dataset["from"] for dataset in episode_data_index]), + "to": torch.cat([dataset["to"] for dataset in episode_data_index]), + } + + def _get_action(self, meta, lerobot_sample) -> torch.Tensor: + key, lerobot_key, raw_shape = meta["key"], meta["lerobot_key"], meta["raw_shape"] + action: torch.Tensor = lerobot_sample[lerobot_key] # [T, action_dim] + if action.ndim == 1: # for shape of 1, like gripper + action = action.unsqueeze(-1) + assert action.shape[-1] == raw_shape, f"Action '{key}' shape {action.shape[-1]} mismatch with meta {raw_shape}." + return action + + def _get_state(self, meta, lerobot_sample) -> torch.Tensor: + key, lerobot_key, raw_shape = meta["key"], meta["lerobot_key"], meta["raw_shape"] + state: torch.Tensor = lerobot_sample[lerobot_key] + if state.ndim == 1: # for shape of 1, like gripper + state = state.unsqueeze(-1) + # state = state[..., :-1, :] # use state_{t} as observation_t + assert state.shape[-1] == raw_shape, f"State '{key}' shape {state.shape[-1]} mismatch with meta {raw_shape}." + return state + + def _get_image(self, meta, lerobot_sample) -> torch.Tensor: + key, lerobot_key, raw_shape = meta["key"], meta["lerobot_key"], meta["raw_shape"] + image: torch.Tensor = lerobot_sample[lerobot_key] + if image.ndim == 3: # time dim will lost when obs_size is 1 + image = image.unsqueeze(0) + image = (image * 255).to(torch.uint8) # (1, 3, H, W) + # For config simplication + # assert image.shape[1:] == raw_shape, f"Image '{key}' shape {image.shape[1:]} mismatch with {raw_shape}." + return image + + def _split_lerobot_sample(self, lerobot_sample) -> Dict[str, Any]: + return lerobot_sample + + def _get_episode_data(self, episode_idx): + lerobot_sample = self.multi_dataset.get_episode_data(episode_idx) + lerobot_sample = self._split_lerobot_sample(lerobot_sample) + state, action = {}, {} + for meta in self.state_meta: + s = self._get_state(meta, lerobot_sample) + state[meta["key"]] = s.unsqueeze(1).float() + for meta in self.action_meta: + a = self._get_action(meta, lerobot_sample) + a = sliding_window_with_replication(a, self.action_size) + action[meta["key"]] = a.float() + return {"action": action, "state": state} + + def _set_return_images(self, flag: bool): + self.return_images = flag + self.multi_dataset.set_during_training(flag) + + def __len__(self): + return self.multi_dataset.num_frames + + def _get_additional_data(self, sample, lerobot_sample): + return sample + + def __getitem__(self, idx): + if idx >= len(self): + raise IndexError(f"Index {idx} out of bounds {len(self)}.") + + # Retry with random indices until we successfully load a frame. + sample_idx = idx + attempt = 0 + last_exception: Optional[Exception] = None + while attempt < MAX_GETITEM_ATTEMPT: + try: + lerobot_sample = self.multi_dataset[sample_idx] + lerobot_sample = self._split_lerobot_sample(lerobot_sample) + break + except Exception as err: + attempt += 1 + last_exception = err + logger.warning( + f"Error loading sample {sample_idx} (attempt {attempt}). " + "Retrying with a random index. " + f"Error: {err}" + ) + sample_idx = np.random.randint(len(self)) + print(traceback.format_exc()) + else: + raise RuntimeError( + f"Failed to load a valid sample after {MAX_GETITEM_ATTEMPT} attempts " + f"for index {idx}." + ) from last_exception + + # Get data from lerobot, organized in nested dict + sample = { + "idx": sample_idx, + "task": lerobot_sample["task"], + "action": {}, + "state": {}, + "images": {}, + } + for meta in self.state_meta: + sample["state"][meta["key"]] = self._get_state(meta, lerobot_sample) + + for meta in self.action_meta: + sample["action"][meta["key"]] = self._get_action(meta, lerobot_sample) + + for meta in self.image_meta: + sample["images"][meta["key"]] = self._get_image(meta, lerobot_sample) + + sample["action_is_pad"] = lerobot_sample[f"{self.action_meta[0]['lerobot_key']}_is_pad"] + sample["state_is_pad"] = lerobot_sample[f"{self.state_meta[0]['lerobot_key']}_is_pad"] + sample["image_is_pad"] = lerobot_sample[f"{self.image_meta[0]['lerobot_key']}_is_pad"] + + sample = self._get_additional_data(sample, lerobot_sample) + + for key in lerobot_sample: + if key not in sample and "observation" not in key and "action" not in key: + sample[key] = lerobot_sample[key] + + # Preprocess the sample using the processor + # for quick data loading + if self.processor is not None: + sample = self.processor.preprocess(sample) + + return sample + + def set_processor(self, processor: BaseProcessor): + """Set processor instance from external initialization.""" + self.processor = processor + if self.is_training_set: + self.processor.train() + else: + self.processor.eval() + return self + + def get_dataset_stats(self, preprocessor: BaseProcessor): + state_min = DefaultDict(list) + state_max = DefaultDict(list) + state_mean = DefaultDict(list) + state_var = DefaultDict(list) + state_q01 = DefaultDict(list) + state_q99 = DefaultDict(list) + + action_min = DefaultDict(list) + action_max = DefaultDict(list) + action_mean = DefaultDict(list) + action_var = DefaultDict(list) + action_q01 = DefaultDict(list) + action_q99 = DefaultDict(list) + + episodes_num = self.multi_dataset.num_episodes + + def process_episode(episode_idx): + batch = self._get_episode_data(episode_idx) + batch = preprocessor.action_state_transform(batch) + return batch + + multi_thread = True + if not multi_thread: + for episode_idx in tqdm(range(episodes_num), desc="Iterating dataset to get normalization"): + batch = process_episode(episode_idx) + for meta in self.state_meta: + key = meta["key"] + cur_state: torch.Tensor = batch["state"][key] # (B, T, dim) + state_min[key].append(cur_state.amin(0)) + state_max[key].append(cur_state.amax(0)) + state_mean[key].append(cur_state.mean(0)) + state_var[key].append(cur_state.var(0)) + state_q01[key].append(torch.quantile(cur_state, 0.01, dim=0, keepdim=False)) + state_q99[key].append(torch.quantile(cur_state, 0.99, dim=0, keepdim=False)) + for meta in self.action_meta: + key = meta["key"] + cur_action: torch.Tensor = batch["action"][key] # (B, T, dim) + action_min[key].append(cur_action.amin(0)) + action_max[key].append(cur_action.amax(0)) + action_mean[key].append(cur_action.mean(0)) + action_var[key].append(cur_action.var(0)) + action_q01[key].append(torch.quantile(cur_action, 0.01, dim=0, keepdim=False)) + action_q99[key].append(torch.quantile(cur_action, 0.99, dim=0, keepdim=False)) + + else: + with ThreadPoolExecutor() as executor: + futures = [executor.submit(process_episode, num) for num in range(episodes_num)] + + for future in tqdm(as_completed(futures), total=episodes_num, desc="Iterating dataset to get normalization"): + try: + batch = future.result() + for meta in self.state_meta: + key = meta["key"] + cur_state: torch.Tensor = batch["state"][key] # (B, T, dim) + state_min[key].append(cur_state.amin(0)) + state_max[key].append(cur_state.amax(0)) + state_mean[key].append(cur_state.mean(0)) + state_var[key].append(cur_state.var(0)) + state_q01[key].append(torch.quantile(cur_state, 0.01, dim=0, keepdim=False)) + state_q99[key].append(torch.quantile(cur_state, 0.99, dim=0, keepdim=False)) + + for meta in self.action_meta: + key = meta["key"] + cur_action: torch.Tensor = batch["action"][key] # (B, T, dim) + action_min[key].append(cur_action.amin(0)) + action_max[key].append(cur_action.amax(0)) + action_mean[key].append(cur_action.mean(0)) + action_var[key].append(cur_action.var(0)) + action_q01[key].append(torch.quantile(cur_action, 0.01, dim=0, keepdim=False)) + action_q99[key].append(torch.quantile(cur_action, 0.99, dim=0, keepdim=False)) + + except Exception as e: + logger.error(f"Error processing episode: {e}") + print(traceback.format_exc()) + raise e + + # assume that each minibatch has equal number of samples + def get_mean_std(means, vars): + means = torch.stack(means) + vars = torch.stack(vars) + stepwise_mean = means.mean(0) + stepwise_std = (vars + (means - stepwise_mean) ** 2).mean(0).sqrt() + global_mean = means.mean((0, 1)) + global_std = (vars + (means - global_mean) ** 2).mean((0, 1)).sqrt() + return stepwise_mean, stepwise_std, global_mean, global_std + + stats = {"state": DefaultDict(dict), "action": DefaultDict(dict), "num_episodes": episodes_num, "num_transition": self.multi_dataset.num_frames} + for meta in self.state_meta: + key = meta["key"] + stats["state"][key]["stepwise_min"] = torch.stack(state_min[key]).amin(0) + stats["state"][key]["stepwise_max"] = torch.stack(state_max[key]).amax(0) + stats["state"][key]["global_min"] = stats["state"][key]["stepwise_min"].amin(0) + stats["state"][key]["global_max"] = stats["state"][key]["stepwise_max"].amax(0) + stats["state"][key]["stepwise_q01"] = torch.stack(state_q01[key]).amin(0) + stats["state"][key]["stepwise_q99"] = torch.stack(state_q99[key]).amax(0) + stats["state"][key]["global_q01"] = stats["state"][key]["stepwise_q01"].amin(0) + stats["state"][key]["global_q99"] = stats["state"][key]["stepwise_q99"].amax(0) + ( + stats["state"][key]["stepwise_mean"], + stats["state"][key]["stepwise_std"], + stats["state"][key]["global_mean"], + stats["state"][key]["global_std"], + ) = get_mean_std(state_mean[key], state_var[key]) + + for meta in self.action_meta: + key = meta["key"] + stats["action"][key]["stepwise_min"] = torch.stack(action_min[key]).amin(0) + stats["action"][key]["stepwise_max"] = torch.stack(action_max[key]).amax(0) + stats["action"][key]["global_min"] = stats["action"][key]["stepwise_min"].amin(0) + stats["action"][key]["global_max"] = stats["action"][key]["stepwise_max"].amax(0) + stats["action"][key]["stepwise_q01"] = torch.stack(action_q01[key]).amin(0) + stats["action"][key]["stepwise_q99"] = torch.stack(action_q99[key]).amax(0) + stats["action"][key]["global_q01"] = stats["action"][key]["stepwise_q01"].amin(0) + stats["action"][key]["global_q99"] = stats["action"][key]["stepwise_q99"].amax(0) + ( + stats["action"][key]["stepwise_mean"], + stats["action"][key]["stepwise_std"], + stats["action"][key]["global_mean"], + stats["action"][key]["global_std"], + ) = get_mean_std(action_mean[key], action_var[key]) + + return stats + + +def sliding_window_with_replication(x: torch.Tensor, window_size: int) -> torch.Tensor: + """ + Construct a sliding-window tensor from the input tensor x (shape: [N, D]). + The output shape is [N, window_size, D]. + + For each starting index i: + out[i, j, :] = + x[i + j, :] if i + j < N + x[-1, :] otherwise (replicate the last row when out of bounds) + + Args: + x (torch.Tensor): Input tensor of shape [N, D] + window_size (int): Size of the sliding window + + Returns: + torch.Tensor: Tensor of shape [N, window_size, D] + """ + assert x.dim() == 2 + assert window_size > 0 + + N, D = x.shape + + # shape [N, window_size] + # indices[i, j] = i + j + i_indices = torch.arange(N).unsqueeze(1) # [N, 1] + j_indices = torch.arange(window_size).unsqueeze(0) # [1, window_size] + indices = i_indices + j_indices # [N, window_size] + + # N-1 + # torch.clamp [0, N-1] + clamped_indices = torch.clamp(indices, min=0, max=N - 1) + + # clamped_indices [N, window_size],x [N, D] + # out[i, j, :] = x[clamped_indices[i, j], :] + out = x[clamped_indices] # [N, window_size, D] + + return out diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/constants.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..30777239efdfa096bfefbc0f2d729e8f36da90ca --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/constants.py @@ -0,0 +1,54 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# keys +import os +from pathlib import Path + +from huggingface_hub.constants import HF_HOME + +OBS_ENV_STATE = "observation.environment_state" +OBS_STATE = "observation.state" +OBS_IMAGE = "observation.image" +OBS_IMAGES = "observation.images" +ACTION = "action" +REWARD = "next.reward" + +ROBOTS = "robots" +ROBOT_TYPE = "robot_type" +TELEOPERATORS = "teleoperators" + +# files & directories +CHECKPOINTS_DIR = "checkpoints" +LAST_CHECKPOINT_LINK = "last" +PRETRAINED_MODEL_DIR = "pretrained_model" +TRAINING_STATE_DIR = "training_state" +RNG_STATE = "rng_state.safetensors" +TRAINING_STEP = "training_step.json" +OPTIMIZER_STATE = "optimizer_state.safetensors" +OPTIMIZER_PARAM_GROUPS = "optimizer_param_groups.json" +SCHEDULER_STATE = "scheduler_state.json" + +if "LEROBOT_HOME" in os.environ: + raise ValueError( + f"You have a 'LEROBOT_HOME' environment variable set to '{os.getenv('LEROBOT_HOME')}'.\n" + "'LEROBOT_HOME' is deprecated, please use 'HF_LEROBOT_HOME' instead." + ) + +# cache dir +default_cache_path = Path(HF_HOME) / "lerobot" +HF_LEROBOT_HOME = Path(os.getenv("HF_LEROBOT_HOME", default_cache_path)).expanduser() + +# calibration dir +default_calibration_path = HF_LEROBOT_HOME / "calibration" +HF_LEROBOT_CALIBRATION = Path(os.getenv("HF_LEROBOT_CALIBRATION", default_calibration_path)).expanduser() diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py new file mode 100644 index 0000000000000000000000000000000000000000..df26818f654de1410ccdab2306a85ca4070b761b --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/lerobot/datasets/compute_stats.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python + +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +from .utils import load_image_as_numpy + + +def estimate_num_samples( + dataset_len: int, min_num_samples: int = 100, max_num_samples: int = 10_000, power: float = 0.75 +) -> int: + """Heuristic to estimate the number of samples based on dataset size. + The power controls the sample growth relative to dataset size. + Lower the power for less number of samples. + + For default arguments, we have: + - from 1 to ~500, num_samples=100 + - at 1000, num_samples=177 + - at 2000, num_samples=299 + - at 5000, num_samples=594 + - at 10000, num_samples=1000 + - at 20000, num_samples=1681 + """ + if dataset_len < min_num_samples: + min_num_samples = dataset_len + return max(min_num_samples, min(int(dataset_len**power), max_num_samples)) + + +def sample_indices(data_len: int) -> list[int]: + num_samples = estimate_num_samples(data_len) + return np.round(np.linspace(0, data_len - 1, num_samples)).astype(int).tolist() + + +def auto_downsample_height_width(img: np.ndarray, target_size: int = 150, max_size_threshold: int = 300): + _, height, width = img.shape + + if max(width, height) < max_size_threshold: + # no downsampling needed + return img + + downsample_factor = int(width / target_size) if width > height else int(height / target_size) + return img[:, ::downsample_factor, ::downsample_factor] + + +def sample_images(image_paths: list[str]) -> np.ndarray: + sampled_indices = sample_indices(len(image_paths)) + + images = None + for i, idx in enumerate(sampled_indices): + path = image_paths[idx] + # we load as uint8 to reduce memory usage + img = load_image_as_numpy(path, dtype=np.uint8, channel_first=True) + img = auto_downsample_height_width(img) + + if images is None: + images = np.empty((len(sampled_indices), *img.shape), dtype=np.uint8) + + images[i] = img + + return images + + +def get_feature_stats(array: np.ndarray, axis: tuple, keepdims: bool) -> dict[str, np.ndarray]: + return { + "min": np.min(array, axis=axis, keepdims=keepdims), + "max": np.max(array, axis=axis, keepdims=keepdims), + "mean": np.mean(array, axis=axis, keepdims=keepdims), + "std": np.std(array, axis=axis, keepdims=keepdims), + "count": np.array([len(array)]), + } + + +def compute_episode_stats(episode_data: dict[str, list[str] | np.ndarray], features: dict, is_compute_episode_stats_image : bool) -> dict: + ep_stats = {} + for key, data in episode_data.items(): + if features[key]["dtype"] == "string": + continue # HACK: we should receive np.arrays of strings + elif features[key]["dtype"] in ["image", "video"]: + if is_compute_episode_stats_image: + ep_ft_array = sample_images(data) # data is a list of image paths + axes_to_reduce = (0, 2, 3) # keep channel dim + keepdims = True + else: + continue + else: + ep_ft_array = data # data is already a np.ndarray + axes_to_reduce = 0 # compute stats over the first axis + keepdims = data.ndim == 1 # keep as np.array + + ep_stats[key] = get_feature_stats(ep_ft_array, axis=axes_to_reduce, keepdims=keepdims) + + # finally, we normalize and remove batch dim for images + if features[key]["dtype"] in ["image", "video"]: + ep_stats[key] = { + k: v if k == "count" else np.squeeze(v / 255.0, axis=0) for k, v in ep_stats[key].items() + } + + return ep_stats + + +def _assert_type_and_shape(stats_list: list[dict[str, dict]]): + for i in range(len(stats_list)): + for fkey in stats_list[i]: + for k, v in stats_list[i][fkey].items(): + if not isinstance(v, np.ndarray): + raise ValueError( + f"Stats must be composed of numpy array, but key '{k}' of feature '{fkey}' is of type '{type(v)}' instead." + ) + if v.ndim == 0: + raise ValueError("Number of dimensions must be at least 1, and is 0 instead.") + if k == "count" and v.shape != (1,): + raise ValueError(f"Shape of 'count' must be (1), but is {v.shape} instead.") + if "image" in fkey and k != "count" and v.shape != (3, 1, 1): + raise ValueError(f"Shape of '{k}' must be (3,1,1), but is {v.shape} instead.") + + +def aggregate_feature_stats(stats_ft_list: list[dict[str, dict]]) -> dict[str, dict[str, np.ndarray]]: + """Aggregates stats for a single feature.""" + means = np.stack([s["mean"] for s in stats_ft_list]) + variances = np.stack([s["std"] ** 2 for s in stats_ft_list]) + counts = np.stack([s["count"] for s in stats_ft_list]) + total_count = counts.sum(axis=0) + + # Prepare weighted mean by matching number of dimensions + while counts.ndim < means.ndim: + counts = np.expand_dims(counts, axis=-1) + + # Compute the weighted mean + weighted_means = means * counts + total_mean = weighted_means.sum(axis=0) / total_count + + # Compute the variance using the parallel algorithm + delta_means = means - total_mean + weighted_variances = (variances + delta_means**2) * counts + total_variance = weighted_variances.sum(axis=0) / total_count + + return { + "min": np.min(np.stack([s["min"] for s in stats_ft_list]), axis=0), + "max": np.max(np.stack([s["max"] for s in stats_ft_list]), axis=0), + "mean": total_mean, + "std": np.sqrt(total_variance), + "count": total_count, + } + + +def aggregate_stats(stats_list: list[dict[str, dict]]) -> dict[str, dict[str, np.ndarray]]: + """Aggregate stats from multiple compute_stats outputs into a single set of stats. + + The final stats will have the union of all data keys from each of the stats dicts. + + For instance: + - new_min = min(min_dataset_0, min_dataset_1, ...) + - new_max = max(max_dataset_0, max_dataset_1, ...) + - new_mean = (mean of all data, weighted by counts) + - new_std = (std of all data) + """ + + _assert_type_and_shape(stats_list) + + data_keys = {key for stats in stats_list for key in stats} + aggregated_stats = {key: {} for key in data_keys} + + for key in data_keys: + stats_with_key = [stats[key] for stats in stats_list if key in stats] + aggregated_stats[key] = aggregate_feature_stats(stats_with_key) + + return aggregated_stats diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6644945af39d90525013ef6f998bec1a35d5a7a6 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/lerobot/datasets/utils.py @@ -0,0 +1,903 @@ +#!/usr/bin/env python + +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import contextlib +import importlib.resources +import io +import json +import logging +from collections.abc import Iterator +from itertools import accumulate +from pathlib import Path +from pprint import pformat +from types import SimpleNamespace +from typing import Any +import array + +import datasets +import jsonlines +import numpy as np +import packaging.version +import torch +from datasets.table import embed_table_storage +from huggingface_hub import DatasetCard, DatasetCardData, HfApi +from huggingface_hub.errors import RevisionNotFoundError +from PIL import Image as PILImage +from torchvision import transforms + +from functools import partial + +# from lerobot.configs.types import DictLike, FeatureType, PolicyFeature +# from lerobot.datasets.backward_compatibility import ( +# V21_MESSAGE, +# BackwardCompatibilityError, +# ForwardCompatibilityError, +# ) +# from lerobot.robots import Robot +# from lerobot.utils.utils import is_valid_numpy_dtype_string + +DEFAULT_CHUNK_SIZE = 1000 # Max number of episodes per chunk + +INFO_PATH = "meta/info.json" +EPISODES_PATH = "meta/episodes.jsonl" +STATS_PATH = "meta/stats.json" +EPISODES_STATS_PATH = "meta/episodes_stats.jsonl" +TASKS_PATH = "meta/tasks.jsonl" + +ANNOTATION_PATHS = { + "subtask": "annotations/subtask_annotations.jsonl", + "scene": "annotations/scene_annotations.jsonl", + "gripper_mode": "annotations/gripper_mode_annotation.jsonl", + "gripper_activity": "annotations/gripper_activity_annotation.jsonl", + "eef_velocity": "annotations/eef_velocity_annotation.jsonl", + "eef_acc_mag": "annotations/eef_acc_mag_annotation.jsonl", + "eef_direction": "annotations/eef_direction_annotation.jsonl", +} + +DEFAULT_VIDEO_PATH = "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4" +DEFAULT_PARQUET_PATH = "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet" +DEFAULT_IMAGE_PATH = "images/{image_key}/episode_{episode_index:06d}/frame_{frame_index:06d}.jpeg" + +DATASET_CARD_TEMPLATE = """ +--- +# Metadata will go there +--- +This dataset was created using [LeRobot](https://github.com/huggingface/lerobot). + +## {} + +""" + +DEFAULT_FEATURES = { + "timestamp": {"dtype": "float32", "shape": (1,), "names": None}, + "frame_index": {"dtype": "int64", "shape": (1,), "names": None}, + "episode_index": {"dtype": "int64", "shape": (1,), "names": None}, + "index": {"dtype": "int64", "shape": (1,), "names": None}, + "coarse_task_index": {"dtype": "int64", "shape": (1,), "names": None}, + "task_index": {"dtype": "int64", "shape": (1,), "names": None}, + "coarse_quality_index": {"dtype": "int64", "shape": (1,), "names": None}, + "quality_index": {"dtype": "int64", "shape": (1,), "names": None}, +} + + +def flatten_dict(d: dict, parent_key: str = "", sep: str = "/") -> dict: + """Flatten a nested dictionary structure by collapsing nested keys into one key with a separator. + + For example: + ``` + >>> dct = {"a": {"b": 1, "c": {"d": 2}}, "e": 3}` + >>> print(flatten_dict(dct)) + {"a/b": 1, "a/c/d": 2, "e": 3} + """ + items = [] + for k, v in d.items(): + new_key = f"{parent_key}{sep}{k}" if parent_key else k + if isinstance(v, dict): + items.extend(flatten_dict(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + +def unflatten_dict(d: dict, sep: str = "/") -> dict: + outdict = {} + for key, value in d.items(): + parts = key.split(sep) + d = outdict + for part in parts[:-1]: + if part not in d: + d[part] = {} + d = d[part] + d[parts[-1]] = value + return outdict + + +# def get_nested_item(obj: DictLike, flattened_key: str, sep: str = "/") -> Any: +# split_keys = flattened_key.split(sep) +# getter = obj[split_keys[0]] +# if len(split_keys) == 1: +# return getter + +# for key in split_keys[1:]: +# getter = getter[key] + +# return getter + + +def serialize_dict(stats: dict[str, torch.Tensor | np.ndarray | dict]) -> dict: + serialized_dict = {} + for key, value in flatten_dict(stats).items(): + if isinstance(value, (torch.Tensor, np.ndarray)): + serialized_dict[key] = value.tolist() + elif isinstance(value, np.generic): + serialized_dict[key] = value.item() + elif isinstance(value, (int, float)): + serialized_dict[key] = value + else: + raise NotImplementedError(f"The value '{value}' of type '{type(value)}' is not supported.") + return unflatten_dict(serialized_dict) + + +def embed_images(dataset: datasets.Dataset) -> datasets.Dataset: + # Embed image bytes into the table before saving to parquet + format = dataset.format + dataset = dataset.with_format("arrow") + dataset = dataset.map(embed_table_storage, batched=False) + dataset = dataset.with_format(**format) + return dataset + + +def load_json(fpath: Path) -> Any: + with open(fpath) as f: + return json.load(f) + + +def write_json(data: dict, fpath: Path) -> None: + fpath.parent.mkdir(exist_ok=True, parents=True) + with open(fpath, "w") as f: + json.dump(data, f, indent=4, ensure_ascii=False) + + +def load_jsonlines(fpath: Path) -> list[Any]: + # allow \t and \n + loose_loader = partial(json.loads, strict=False) + with jsonlines.open(fpath, "r", loads=loose_loader) as reader: + return list(reader) + + +def write_jsonlines(data: dict, fpath: Path) -> None: + fpath.parent.mkdir(exist_ok=True, parents=True) + with jsonlines.open(fpath, "w") as writer: + writer.write_all(data) + + +def append_jsonlines(data: dict, fpath: Path) -> None: + fpath.parent.mkdir(exist_ok=True, parents=True) + with jsonlines.open(fpath, "a") as writer: + writer.write(data) + + +def write_info(info: dict, local_dir: Path): + write_json(info, local_dir / INFO_PATH) + + +def load_info(local_dir: Path) -> dict: + info = load_json(local_dir / INFO_PATH) + for ft in info["features"].values(): + ft["shape"] = tuple(ft["shape"]) + return info + + +def write_stats(stats: dict, local_dir: Path): + serialized_stats = serialize_dict(stats) + write_json(serialized_stats, local_dir / STATS_PATH) + + +def cast_stats_to_numpy(stats) -> dict[str, dict[str, np.ndarray]]: + stats = {key: np.array(value) for key, value in flatten_dict(stats).items()} + return unflatten_dict(stats) + + +def load_stats(local_dir: Path) -> dict[str, dict[str, np.ndarray]]: + if not (local_dir / STATS_PATH).exists(): + return None + stats = load_json(local_dir / STATS_PATH) + return cast_stats_to_numpy(stats) + + +def write_task(task_index: int, task: dict, local_dir: Path): + task_dict = { + "task_index": task_index, + "task": task, + } + append_jsonlines(task_dict, local_dir / TASKS_PATH) + + +def load_tasks(local_dir: Path) -> tuple[dict, dict]: + tasks = load_jsonlines(local_dir / TASKS_PATH) + tasks = {item["task_index"]: item["task"] for item in sorted(tasks, key=lambda x: x["task_index"])} + task_to_task_index = {task: task_index for task_index, task in tasks.items()} + return tasks, task_to_task_index + +def load_annotations(local_dir: Path) -> dict[str, dict[int, str]]: + annotations = {} + for key, path in ANNOTATION_PATHS.items(): + anno = load_jsonlines(local_dir / path) + anno = {item[f"{key}_index"]: item[key] for item in sorted(anno, key=lambda x: x[f"{key}_index"])} + annotations[key] = anno + return annotations + +def write_episode(episode: dict, local_dir: Path): + append_jsonlines(episode, local_dir / EPISODES_PATH) + + +def load_episodes(local_dir: Path) -> dict: + episodes = load_jsonlines(local_dir / EPISODES_PATH) + return {item["episode_index"]: item for item in sorted(episodes, key=lambda x: x["episode_index"])} + + +def write_episode_stats(episode_index: int, episode_stats: dict, local_dir: Path): + # We wrap episode_stats in a dictionary since `episode_stats["episode_index"]` + # is a dictionary of stats and not an integer. + episode_stats = {"episode_index": episode_index, "stats": serialize_dict(episode_stats)} + append_jsonlines(episode_stats, local_dir / EPISODES_STATS_PATH) + + +def load_episodes_stats(local_dir: Path) -> dict: + episodes_stats = load_jsonlines(local_dir / EPISODES_STATS_PATH) + return { + item["episode_index"]: cast_stats_to_numpy(item["stats"]) + for item in sorted(episodes_stats, key=lambda x: x["episode_index"]) + } + + +def backward_compatible_episodes_stats( + stats: dict[str, dict[str, np.ndarray]], episodes: list[int] +) -> dict[str, dict[str, np.ndarray]]: + return dict.fromkeys(episodes, stats) + + +def load_image_as_numpy( + fpath: str | Path, dtype: np.dtype = np.float32, channel_first: bool = True +) -> np.ndarray: + img = PILImage.open(fpath).convert("RGB") + img_array = np.array(img, dtype=dtype) + if channel_first: # (H, W, C) -> (C, H, W) + img_array = np.transpose(img_array, (2, 0, 1)) + if np.issubdtype(dtype, np.floating): + img_array /= 255.0 + return img_array + + +def hf_transform_to_torch(items_dict: dict[torch.Tensor | None]): + """Get a transform function that convert items from Hugging Face dataset (pyarrow) + to torch tensors. Importantly, images are converted from PIL, which corresponds to + a channel last representation (h w c) of uint8 type, to a torch image representation + with channel first (c h w) of float32 type in range [0,1]. + """ + for key in items_dict: + first_item = items_dict[key][0] + if type(first_item) == dict and 'bytes' in first_item and first_item['bytes'] is not None: + to_pil = lambda img: PILImage.open(io.BytesIO(img['bytes'])) + items_dict[key] = [to_pil(img) for img in items_dict[key]] + first_item = items_dict[key][0] + if isinstance(first_item, PILImage.Image): + to_tensor = transforms.ToTensor() + items_dict[key] = [to_tensor(img) for img in items_dict[key]] + elif first_item is None: + pass + else: + items_dict[key] = [x if isinstance(x, str) else torch.tensor(x) for x in items_dict[key]] + return items_dict + + +def is_valid_version(version: str) -> bool: + try: + packaging.version.parse(version) + return True + except packaging.version.InvalidVersion: + return False + + +# def check_version_compatibility( +# repo_id: str, +# version_to_check: str | packaging.version.Version, +# current_version: str | packaging.version.Version, +# enforce_breaking_major: bool = True, +# ) -> None: +# v_check = ( +# packaging.version.parse(version_to_check) +# if not isinstance(version_to_check, packaging.version.Version) +# else version_to_check +# ) +# v_current = ( +# packaging.version.parse(current_version) +# if not isinstance(current_version, packaging.version.Version) +# else current_version +# ) +# if v_check.major < v_current.major and enforce_breaking_major: +# raise BackwardCompatibilityError(repo_id, v_check) +# elif v_check.minor < v_current.minor: +# logging.warning(V21_MESSAGE.format(repo_id=repo_id, version=v_check)) + + +def get_repo_versions(repo_id: str) -> list[packaging.version.Version]: + """Returns available valid versions (branches and tags) on given repo.""" + api = HfApi() + repo_refs = api.list_repo_refs(repo_id, repo_type="dataset") + repo_refs = [b.name for b in repo_refs.branches + repo_refs.tags] + repo_versions = [] + for ref in repo_refs: + with contextlib.suppress(packaging.version.InvalidVersion): + repo_versions.append(packaging.version.parse(ref)) + + return repo_versions + + +# def get_safe_version(repo_id: str, version: str | packaging.version.Version) -> str: +# """ +# Returns the version if available on repo or the latest compatible one. +# Otherwise, will throw a `CompatibilityError`. +# """ +# target_version = ( +# packaging.version.parse(version) if not isinstance(version, packaging.version.Version) else version +# ) +# hub_versions = get_repo_versions(repo_id) + +# if not hub_versions: +# raise RevisionNotFoundError( +# f"""Your dataset must be tagged with a codebase version. +# Assuming _version_ is the codebase_version value in the info.json, you can run this: +# ```python +# from huggingface_hub import HfApi + +# hub_api = HfApi() +# hub_api.create_tag("{repo_id}", tag="_version_", repo_type="dataset") +# ``` +# """ +# ) + +# if target_version in hub_versions: +# return f"v{target_version}" + +# compatibles = [ +# v for v in hub_versions if v.major == target_version.major and v.minor <= target_version.minor +# ] +# if compatibles: +# return_version = max(compatibles) +# if return_version < target_version: +# logging.warning(f"Revision {version} for {repo_id} not found, using version v{return_version}") +# return f"v{return_version}" + +# lower_major = [v for v in hub_versions if v.major < target_version.major] +# if lower_major: +# raise BackwardCompatibilityError(repo_id, max(lower_major)) + +# upper_versions = [v for v in hub_versions if v > target_version] +# assert len(upper_versions) > 0 +# raise ForwardCompatibilityError(repo_id, min(upper_versions)) + + +def get_hf_features_from_features(features: dict) -> datasets.Features: + hf_features = {} + for key, ft in features.items(): + if ft["dtype"] == "video": + continue + elif ft["dtype"] == "image": + hf_features[key] = datasets.Image() + elif ft["shape"] == (1,): + hf_features[key] = datasets.Value(dtype=ft["dtype"]) + elif len(ft["shape"]) == 1: + hf_features[key] = datasets.Sequence( + length=ft["shape"][0], feature=datasets.Value(dtype=ft["dtype"]) + ) + elif len(ft["shape"]) == 2: + hf_features[key] = datasets.Array2D(shape=ft["shape"], dtype=ft["dtype"]) + elif len(ft["shape"]) == 3: + hf_features[key] = datasets.Array3D(shape=ft["shape"], dtype=ft["dtype"]) + elif len(ft["shape"]) == 4: + hf_features[key] = datasets.Array4D(shape=ft["shape"], dtype=ft["dtype"]) + elif len(ft["shape"]) == 5: + hf_features[key] = datasets.Array5D(shape=ft["shape"], dtype=ft["dtype"]) + else: + raise ValueError(f"Corresponding feature is not valid: {ft}") + + return datasets.Features(hf_features) + + +def _validate_feature_names(features: dict[str, dict]) -> None: + invalid_features = {name: ft for name, ft in features.items() if "/" in name} + if invalid_features: + raise ValueError(f"Feature names should not contain '/'. Found '/' in '{invalid_features}'.") + + +def hw_to_dataset_features( + hw_features: dict[str, type | tuple], prefix: str, use_video: bool = True +) -> dict[str, dict]: + features = {} + joint_fts = {key: ftype for key, ftype in hw_features.items() if ftype is float} + cam_fts = {key: shape for key, shape in hw_features.items() if isinstance(shape, tuple)} + + if joint_fts and prefix == "action": + features[prefix] = { + "dtype": "float32", + "shape": (len(joint_fts),), + "names": list(joint_fts), + } + + if joint_fts and prefix == "observation": + features[f"{prefix}.state"] = { + "dtype": "float32", + "shape": (len(joint_fts),), + "names": list(joint_fts), + } + + for key, shape in cam_fts.items(): + features[f"{prefix}.images.{key}"] = { + "dtype": "video" if use_video else "image", + "shape": shape, + "names": ["height", "width", "channels"], + } + + _validate_feature_names(features) + return features + + +def build_dataset_frame( + ds_features: dict[str, dict], values: dict[str, Any], prefix: str +) -> dict[str, np.ndarray]: + frame = {} + for key, ft in ds_features.items(): + if key in DEFAULT_FEATURES or not key.startswith(prefix): + continue + elif ft["dtype"] == "float32" and len(ft["shape"]) == 1: + frame[key] = np.array([values[name] for name in ft["names"]], dtype=np.float32) + elif ft["dtype"] in ["image", "video"]: + frame[key] = values[key.removeprefix(f"{prefix}.images.")] + + return frame + + +# def get_features_from_robot(robot: Robot, use_videos: bool = True) -> dict: +# camera_ft = {} +# if robot.cameras: +# camera_ft = { +# key: {"dtype": "video" if use_videos else "image", **ft} +# for key, ft in robot.camera_features.items() +# } +# return {**robot.motor_features, **camera_ft, **DEFAULT_FEATURES} + + +# def dataset_to_policy_features(features: dict[str, dict]) -> dict[str, PolicyFeature]: +# # TODO(aliberts): Implement "type" in dataset features and simplify this +# policy_features = {} +# for key, ft in features.items(): +# shape = ft["shape"] +# if ft["dtype"] in ["image", "video"]: +# type = FeatureType.VISUAL +# if len(shape) != 3: +# raise ValueError(f"Number of dimensions of {key} != 3 (shape={shape})") + +# names = ft["names"] +# # Backward compatibility for "channel" which is an error introduced in LeRobotDataset v2.0 for ported datasets. +# if names[2] in ["channel", "channels"]: # (h, w, c) -> (c, h, w) +# shape = (shape[2], shape[0], shape[1]) +# elif key == "observation.environment_state": +# type = FeatureType.ENV +# elif key.startswith("observation"): +# type = FeatureType.STATE +# elif key.startswith("action"): +# type = FeatureType.ACTION +# else: +# continue + +# policy_features[key] = PolicyFeature( +# type=type, +# shape=shape, +# ) + +# return policy_features + + +def create_empty_dataset_info( + codebase_version: str, + fps: int, + features: dict, + use_videos: bool, + robot_type: str | None = None, +) -> dict: + return { + "codebase_version": codebase_version, + "robot_type": robot_type, + "total_episodes": 0, + "total_frames": 0, + "total_tasks": 0, + "total_videos": 0, + "total_chunks": 0, + "chunks_size": DEFAULT_CHUNK_SIZE, + "fps": fps, + "splits": {}, + "data_path": DEFAULT_PARQUET_PATH, + "video_path": DEFAULT_VIDEO_PATH if use_videos else None, + "features": features, + } + + +def get_episode_data_index( + episode_dicts: dict[dict], episodes: list[int] | None = None +) -> dict[str, torch.Tensor]: + episode_lengths = {ep_idx: ep_dict["length"] for ep_idx, ep_dict in episode_dicts.items()} + if episodes is not None: + episode_lengths = {ep_idx: episode_lengths[ep_idx] for ep_idx in episodes} + + cumulative_lengths = list(accumulate(episode_lengths.values())) + return { + "from": torch.LongTensor([0] + cumulative_lengths[:-1]), + "to": torch.LongTensor(cumulative_lengths), + } + + +def check_timestamps_sync( + timestamps: np.ndarray, + episode_indices: np.ndarray, + episode_data_index: dict[str, np.ndarray], + fps: int, + tolerance_s: float, + raise_value_error: bool = True, +) -> bool: + """ + This check is to make sure that each timestamp is separated from the next by (1/fps) +/- tolerance + to account for possible numerical error. + + Args: + timestamps (np.ndarray): Array of timestamps in seconds. + episode_indices (np.ndarray): Array indicating the episode index for each timestamp. + episode_data_index (dict[str, np.ndarray]): A dictionary that includes 'to', + which identifies indices for the end of each episode. + fps (int): Frames per second. Used to check the expected difference between consecutive timestamps. + tolerance_s (float): Allowed deviation from the expected (1/fps) difference. + raise_value_error (bool): Whether to raise a ValueError if the check fails. + + Returns: + bool: True if all checked timestamp differences lie within tolerance, False otherwise. + + Raises: + ValueError: If the check fails and `raise_value_error` is True. + """ + if timestamps.shape != episode_indices.shape: + raise ValueError( + "timestamps and episode_indices should have the same shape. " + f"Found {timestamps.shape=} and {episode_indices.shape=}." + ) + + # Consecutive differences + diffs = np.diff(timestamps) + within_tolerance = np.abs(diffs - (1.0 / fps)) <= tolerance_s + + # Mask to ignore differences at the boundaries between episodes + mask = np.ones(len(diffs), dtype=bool) + ignored_diffs = episode_data_index["to"][:-1] - 1 # indices at the end of each episode + mask[ignored_diffs] = False + filtered_within_tolerance = within_tolerance[mask] + + # Check if all remaining diffs are within tolerance + if not np.all(filtered_within_tolerance): + # Track original indices before masking + original_indices = np.arange(len(diffs)) + filtered_indices = original_indices[mask] + outside_tolerance_filtered_indices = np.nonzero(~filtered_within_tolerance)[0] + outside_tolerance_indices = filtered_indices[outside_tolerance_filtered_indices] + + outside_tolerances = [] + for idx in outside_tolerance_indices: + entry = { + "timestamps": [timestamps[idx], timestamps[idx + 1]], + "diff": diffs[idx], + "episode_index": episode_indices[idx].item() + if hasattr(episode_indices[idx], "item") + else episode_indices[idx], + } + outside_tolerances.append(entry) + + if raise_value_error: + raise ValueError( + f"""One or several timestamps unexpectedly violate the tolerance inside episode range. + This might be due to synchronization issues during data collection. + \n{pformat(outside_tolerances)}""" + ) + return False + + return True + + +def check_delta_timestamps( + delta_timestamps: dict[str, list[float]], fps: int, tolerance_s: float, raise_value_error: bool = True +) -> bool: + """This will check if all the values in delta_timestamps are multiples of 1/fps +/- tolerance. + This is to ensure that these delta_timestamps added to any timestamp from a dataset will themselves be + actual timestamps from the dataset. + """ + outside_tolerance = {} + for key, delta_ts in delta_timestamps.items(): + within_tolerance = [abs(ts * fps - round(ts * fps)) / fps <= tolerance_s for ts in delta_ts] + if not all(within_tolerance): + outside_tolerance[key] = [ + ts for ts, is_within in zip(delta_ts, within_tolerance, strict=True) if not is_within + ] + + if len(outside_tolerance) > 0: + if raise_value_error: + raise ValueError( + f""" + The following delta_timestamps are found outside of tolerance range. + Please make sure they are multiples of 1/{fps} +/- tolerance and adjust + their values accordingly. + \n{pformat(outside_tolerance)} + """ + ) + return False + + return True + + +def get_delta_indices(delta_timestamps: dict[str, list[float]], fps: int) -> dict[str, list[int]]: + delta_indices = {} + for key, delta_ts in delta_timestamps.items(): + delta_indices[key] = [round(d * fps) for d in delta_ts] + + return delta_indices + + +def cycle(iterable): + """The equivalent of itertools.cycle, but safe for Pytorch dataloaders. + + See https://github.com/pytorch/pytorch/issues/23900 for information on why itertools.cycle is not safe. + """ + iterator = iter(iterable) + while True: + try: + yield next(iterator) + except StopIteration: + iterator = iter(iterable) + + +def create_branch(repo_id, *, branch: str, repo_type: str | None = None) -> None: + """Create a branch on a existing Hugging Face repo. Delete the branch if it already + exists before creating it. + """ + api = HfApi() + + branches = api.list_repo_refs(repo_id, repo_type=repo_type).branches + refs = [branch.ref for branch in branches] + ref = f"refs/heads/{branch}" + if ref in refs: + api.delete_branch(repo_id, repo_type=repo_type, branch=branch) + + api.create_branch(repo_id, repo_type=repo_type, branch=branch) + + +def create_lerobot_dataset_card( + tags: list | None = None, + dataset_info: dict | None = None, + **kwargs, +) -> DatasetCard: + """ + Keyword arguments will be used to replace values in src/lerobot/datasets/card_template.md. + Note: If specified, license must be one of https://huggingface.co/docs/hub/repositories-licenses. + """ + card_tags = ["LeRobot"] + + if tags: + card_tags += tags + if dataset_info: + dataset_structure = "[meta/info.json](meta/info.json):\n" + dataset_structure += f"```json\n{json.dumps(dataset_info, indent=4)}\n```\n" + kwargs = {**kwargs, "dataset_structure": dataset_structure} + card_data = DatasetCardData( + license=kwargs.get("license"), + tags=card_tags, + task_categories=["robotics"], + configs=[ + { + "config_name": "default", + "data_files": "data/*/*.parquet", + } + ], + ) + + card_template = (importlib.resources.files("lerobot.datasets") / "card_template.md").read_text() + + return DatasetCard.from_template( + card_data=card_data, + template_str=card_template, + **kwargs, + ) + + +class IterableNamespace(SimpleNamespace): + """ + A namespace object that supports both dictionary-like iteration and dot notation access. + Automatically converts nested dictionaries into IterableNamespaces. + + This class extends SimpleNamespace to provide: + - Dictionary-style iteration over keys + - Access to items via both dot notation (obj.key) and brackets (obj["key"]) + - Dictionary-like methods: items(), keys(), values() + - Recursive conversion of nested dictionaries + + Args: + dictionary: Optional dictionary to initialize the namespace + **kwargs: Additional keyword arguments passed to SimpleNamespace + + Examples: + >>> data = {"name": "Alice", "details": {"age": 25}} + >>> ns = IterableNamespace(data) + >>> ns.name + 'Alice' + >>> ns.details.age + 25 + >>> list(ns.keys()) + ['name', 'details'] + >>> for key, value in ns.items(): + ... print(f"{key}: {value}") + name: Alice + details: IterableNamespace(age=25) + """ + + def __init__(self, dictionary: dict[str, Any] = None, **kwargs): + super().__init__(**kwargs) + if dictionary is not None: + for key, value in dictionary.items(): + if isinstance(value, dict): + setattr(self, key, IterableNamespace(value)) + else: + setattr(self, key, value) + + def __iter__(self) -> Iterator[str]: + return iter(vars(self)) + + def __getitem__(self, key: str) -> Any: + return vars(self)[key] + + def items(self): + return vars(self).items() + + def values(self): + return vars(self).values() + + def keys(self): + return vars(self).keys() + + +def validate_frame(frame: dict, features: dict): + expected_features = set(features) - set(DEFAULT_FEATURES) + actual_features = set(frame) + + error_message = validate_features_presence(actual_features, expected_features) + + common_features = actual_features & expected_features + for name in common_features - {"task"}: + error_message += validate_feature_dtype_and_shape(name, features[name], frame[name]) + + if error_message: + raise ValueError(error_message) + + +def validate_features_presence(actual_features: set[str], expected_features: set[str]): + error_message = "" + missing_features = expected_features - actual_features + extra_features = actual_features - expected_features + + if missing_features or extra_features: + error_message += "Feature mismatch in `frame` dictionary:\n" + if missing_features: + error_message += f"Missing features: {missing_features}\n" + if extra_features: + error_message += f"Extra features: {extra_features}\n" + + return error_message + +def is_valid_numpy_dtype_string(dtype_str: str) -> bool: + """ + Return True if a given string can be converted to a numpy dtype. + """ + try: + # Attempt to convert the string to a numpy dtype + np.dtype(dtype_str) + return True + except TypeError: + # If a TypeError is raised, the string is not a valid dtype + return False +def validate_feature_dtype_and_shape(name: str, feature: dict, value: np.ndarray | PILImage.Image | str | bytes): + if isinstance(value, bytes) or isinstance(value, array.array): # ROS 1 and 2 + #TODO fix bytes + return "" + expected_dtype = feature["dtype"] + expected_shape = feature["shape"] + if is_valid_numpy_dtype_string(expected_dtype): + return validate_feature_numpy_array(name, expected_dtype, expected_shape, value) + elif expected_dtype in ["image", "video"]: + return validate_feature_image_or_video(name, expected_shape, value) + elif expected_dtype == "string": + return validate_feature_string(name, value) + else: + raise NotImplementedError(f"The feature dtype '{expected_dtype}' is not implemented yet.") + + +def validate_feature_numpy_array( + name: str, expected_dtype: str, expected_shape: list[int], value: np.ndarray +): + error_message = "" + if isinstance(value, np.ndarray): + actual_dtype = value.dtype + actual_shape = value.shape + + if actual_dtype != np.dtype(expected_dtype): + error_message += f"The feature '{name}' of dtype '{actual_dtype}' is not of the expected dtype '{expected_dtype}'.\n" + + if actual_shape != expected_shape: + error_message += f"The feature '{name}' of shape '{actual_shape}' does not have the expected shape '{expected_shape}'.\n" + else: + error_message += f"The feature '{name}' is not a 'np.ndarray'. Expected type is '{expected_dtype}', but type '{type(value)}' provided instead.\n" + + return error_message + + +def validate_feature_image_or_video(name: str, expected_shape: list[str], value: np.ndarray | PILImage.Image): + # Note: The check of pixels range ([0,1] for float and [0,255] for uint8) is done by the image writer threads. + error_message = "" + if isinstance(value, np.ndarray): + actual_shape = value.shape + c, h, w = expected_shape + if len(actual_shape) != 3 or (actual_shape != (c, h, w) and actual_shape != (h, w, c)): + error_message += f"The feature '{name}' of shape '{actual_shape}' does not have the expected shape '{(c, h, w)}' or '{(h, w, c)}'.\n" + elif isinstance(value, PILImage.Image): + pass + else: + error_message += f"The feature '{name}' is expected to be of type 'PIL.Image' or 'np.ndarray' channel first or channel last, but type '{type(value)}' provided instead.\n" + + return error_message + + +def validate_feature_string(name: str, value: str): + if not isinstance(value, str): + return f"The feature '{name}' is expected to be of type 'str', but type '{type(value)}' provided instead.\n" + return "" + + +def validate_episode_buffer(episode_buffer: dict, total_episodes: int, features: dict): + if "size" not in episode_buffer: + raise ValueError("size key not found in episode_buffer") + + if "task" not in episode_buffer: + raise ValueError("task key not found in episode_buffer") + + if episode_buffer["episode_index"] != total_episodes: + # TODO(aliberts): Add option to use existing episode_index + raise NotImplementedError( + "You might have manually provided the episode_buffer with an episode_index that doesn't " + "match the total number of episodes already in the dataset. This is not supported for now." + ) + + if episode_buffer["size"] == 0: + raise ValueError("You must add one or several frames with `add_frame` before calling `add_episode`.") + + buffer_keys = set(episode_buffer.keys()) - {"task", "size"} + if not buffer_keys == set(features): + raise ValueError( + f"Features from `episode_buffer` don't match the ones in `features`." + f"In episode_buffer not in features: {buffer_keys - set(features)}" + f"In features not in episode_buffer: {set(features) - buffer_keys}" + ) diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9dd6fee91c9d867d5a0f4540e5ae0713b7f753dc --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/lerobot/datasets/video_utils.py @@ -0,0 +1,527 @@ +#!/usr/bin/env python + +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import glob +import importlib +import logging +import warnings +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, ClassVar + +import av +import pyarrow as pa +import torch +import torchvision +from datasets.features.features import register_feature +from PIL import Image + + +def get_safe_default_codec(): + if importlib.util.find_spec("torchcodec"): + return "torchcodec" + else: + logging.warning( + "'torchcodec' is not available in your platform, falling back to 'pyav' as a default decoder" + ) + return "pyav" + + +def decode_video_frames( + video_path: Path | str, + timestamps: list[float], + tolerance_s: float, + backend: str | None = None, +) -> torch.Tensor: + """ + Decodes video frames using the specified backend. + + Args: + video_path (Path): Path to the video file. + timestamps (list[float]): List of timestamps to extract frames. + tolerance_s (float): Allowed deviation in seconds for frame retrieval. + backend (str, optional): Backend to use for decoding. Defaults to "torchcodec" when available in the platform; otherwise, defaults to "pyav".. + + Returns: + torch.Tensor: Decoded frames. + + Currently supports torchcodec on cpu and pyav. + """ + if backend is None: + backend = get_safe_default_codec() + if backend == "torchcodec": + try: + return decode_video_frames_torchcodec(video_path, timestamps, tolerance_s) + except Exception as err: + warnings.warn( + f"torchcodec video decode failed ({type(err).__name__}: {err}); falling back to torchvision/pyav." + ) + return decode_video_frames_torchvision(video_path, timestamps, tolerance_s, backend="pyav") + elif backend in ["pyav", "video_reader"]: + return decode_video_frames_torchvision(video_path, timestamps, tolerance_s, backend) + else: + raise ValueError(f"Unsupported video backend: {backend}") + + +def decode_video_frames_torchvision( + video_path: Path | str, + timestamps: list[float], + tolerance_s: float, + backend: str = "pyav", + log_loaded_timestamps: bool = False, +) -> torch.Tensor: + """Loads frames associated to the requested timestamps of a video + + The backend can be either "pyav" (default) or "video_reader". + "video_reader" requires installing torchvision from source, see: + https://github.com/pytorch/vision/blob/main/torchvision/csrc/io/decoder/gpu/README.rst + (note that you need to compile against ffmpeg<4.3) + + While both use cpu, "video_reader" is supposedly faster than "pyav" but requires additional setup. + For more info on video decoding, see `benchmark/video/README.md` + + See torchvision doc for more info on these two backends: + https://pytorch.org/vision/0.18/index.html?highlight=backend#torchvision.set_video_backend + + Note: Video benefits from inter-frame compression. Instead of storing every frame individually, + the encoder stores a reference frame (or a key frame) and subsequent frames as differences relative to + that key frame. As a consequence, to access a requested frame, we need to load the preceding key frame, + and all subsequent frames until reaching the requested frame. The number of key frames in a video + can be adjusted during encoding to take into account decoding time and video size in bytes. + """ + video_path = str(video_path) + + # set backend + keyframes_only = False + torchvision.set_video_backend(backend) + if backend == "pyav": + keyframes_only = True # pyav doesn't support accurate seek + + # set a video stream reader + # TODO(rcadene): also load audio stream at the same time + reader = torchvision.io.VideoReader(video_path, "video") + + # set the first and last requested timestamps + # Note: previous timestamps are usually loaded, since we need to access the previous key frame + first_ts = min(timestamps) + last_ts = max(timestamps) + + # access closest key frame of the first requested frame + # Note: closest key frame timestamp is usually smaller than `first_ts` (e.g. key frame can be the first frame of the video) + # for details on what `seek` is doing see: https://pyav.basswood-io.com/docs/stable/api/container.html?highlight=inputcontainer#av.container.InputContainer.seek + reader.seek(first_ts, keyframes_only=keyframes_only) + + # load all frames until last requested frame + loaded_frames = [] + loaded_ts = [] + for frame in reader: + current_ts = frame["pts"] + if log_loaded_timestamps: + logging.info(f"frame loaded at timestamp={current_ts:.4f}") + loaded_frames.append(frame["data"]) + loaded_ts.append(current_ts) + if current_ts >= last_ts: + break + + if backend == "pyav": + reader.container.close() + + reader = None + + # Use float32 for timestamp distance computation (torch.cdist doesn't support bfloat16). + query_ts = torch.tensor(timestamps, dtype=torch.float32) + loaded_ts = torch.tensor(loaded_ts, dtype=torch.float32) + + # compute distances between each query timestamp and timestamps of all loaded frames + dist = torch.cdist(query_ts[:, None], loaded_ts[:, None], p=1) + min_, argmin_ = dist.min(1) + + is_within_tol = min_ < tolerance_s + assert is_within_tol.all(), ( + f"One or several query timestamps unexpectedly violate the tolerance ({min_[~is_within_tol]} > {tolerance_s=})." + "It means that the closest frame that can be loaded from the video is too far away in time." + "This might be due to synchronization issues with timestamps during data collection." + "To be safe, we advise to ignore this item during training." + f"\nqueried timestamps: {query_ts}" + f"\nloaded timestamps: {loaded_ts}" + f"\nvideo: {video_path}" + f"\nbackend: {backend}" + ) + + # get closest frames to the query timestamps + closest_frames = torch.stack([loaded_frames[idx] for idx in argmin_]) + closest_ts = loaded_ts[argmin_] + + if log_loaded_timestamps: + logging.info(f"{closest_ts=}") + + # convert to the pytorch format which is float32 in [0,1] range (and channel first) + closest_frames = closest_frames.type(torch.float32) / 255 + + assert len(timestamps) == len(closest_frames) + return closest_frames + + +def decode_video_frames_torchcodec( + video_path: Path | str, + timestamps: list[float], + tolerance_s: float, + device: str = "cpu", + log_loaded_timestamps: bool = False, +) -> torch.Tensor: + """Loads frames associated with the requested timestamps of a video using torchcodec. + + Note: Setting device="cuda" outside the main process, e.g. in data loader workers, will lead to CUDA initialization errors. + + Note: Video benefits from inter-frame compression. Instead of storing every frame individually, + the encoder stores a reference frame (or a key frame) and subsequent frames as differences relative to + that key frame. As a consequence, to access a requested frame, we need to load the preceding key frame, + and all subsequent frames until reaching the requested frame. The number of key frames in a video + can be adjusted during encoding to take into account decoding time and video size in bytes. + """ + + if importlib.util.find_spec("torchcodec"): + from torchcodec.decoders import VideoDecoder + else: + raise ImportError("torchcodec is required but not available.") + + # initialize video decoder + decoder = VideoDecoder(video_path, device=device, seek_mode="approximate") + loaded_frames = [] + loaded_ts = [] + # get metadata for frame information + metadata = decoder.metadata + average_fps = metadata.average_fps + + # convert timestamps to frame indices + frame_indices = [round(ts * average_fps) for ts in timestamps] + + # retrieve frames based on indices + frames_batch = decoder.get_frames_at(indices=frame_indices) + + for frame, pts in zip(frames_batch.data, frames_batch.pts_seconds, strict=False): + loaded_frames.append(frame) + loaded_ts.append(pts.item()) + if log_loaded_timestamps: + logging.info(f"Frame loaded at timestamp={pts:.4f}") + + # Use float32 for timestamp distance computation (torch.cdist doesn't support bfloat16). + query_ts = torch.tensor(timestamps, dtype=torch.float32) + loaded_ts = torch.tensor(loaded_ts, dtype=torch.float32) + + # compute distances between each query timestamp and loaded timestamps + dist = torch.cdist(query_ts[:, None], loaded_ts[:, None], p=1) + min_, argmin_ = dist.min(1) + + is_within_tol = min_ < tolerance_s + assert is_within_tol.all(), ( + f"One or several query timestamps unexpectedly violate the tolerance ({min_[~is_within_tol]} > {tolerance_s=})." + "It means that the closest frame that can be loaded from the video is too far away in time." + "This might be due to synchronization issues with timestamps during data collection." + "To be safe, we advise to ignore this item during training." + f"\nqueried timestamps: {query_ts}" + f"\nloaded timestamps: {loaded_ts}" + f"\nvideo: {video_path}" + ) + + # get closest frames to the query timestamps + closest_frames = torch.stack([loaded_frames[idx] for idx in argmin_]) + closest_ts = loaded_ts[argmin_] + + if log_loaded_timestamps: + logging.info(f"{closest_ts=}") + + # convert to float32 in [0,1] range (channel first) + closest_frames = closest_frames.type(torch.float32) / 255 + + assert len(timestamps) == len(closest_frames) + return closest_frames + +def encode_video_frames( + imgs_dir: Path | str, + video_path: Path | str, + fps: int, + vcodec: str = "libsvtav1", + pix_fmt: str = "yuv420p", + g: int | None = 2, + crf: int | None = 30, + fast_decode: int = 0, + log_level: int | None = av.logging.ERROR, + overwrite: bool = False, +) -> None: + """More info on ffmpeg arguments tuning on `benchmark/video/README.md`""" + # Check encoder availability + if vcodec == "h264_nvenc" : + return encode_video_frames_ffmpeg( + imgs_dir, video_path, fps, pix_fmt = pix_fmt, overwrite=overwrite + ) + + if vcodec not in ["h264", "hevc", "libsvtav1"]: + raise ValueError(f"Unsupported video codec: {vcodec}. Supported codecs are: h264, hevc, libsvtav1.") + + video_path = Path(video_path) + imgs_dir = Path(imgs_dir) + + video_path.parent.mkdir(parents=True, exist_ok=overwrite) + + # Encoders/pixel formats incompatibility check + if (vcodec == "libsvtav1" or vcodec == "hevc") and pix_fmt == "yuv444p": + logging.warning( + f"Incompatible pixel format 'yuv444p' for codec {vcodec}, auto-selecting format 'yuv420p'" + ) + pix_fmt = "yuv420p" + + # Get input frames + template = "frame_" + ("[0-9]" * 6) + ".jpeg" + input_list = sorted( + glob.glob(str(imgs_dir / template)), key=lambda x: int(x.split("_")[-1].split(".")[0]) + ) + + # Define video output frame size (assuming all input frames are the same size) + if len(input_list) == 0: + raise FileNotFoundError(f"No images found in {imgs_dir}.") + dummy_image = Image.open(input_list[0]) + width, height = dummy_image.size + + # Define video codec options + video_options = {} + + if g is not None: + video_options["g"] = str(g) + + if crf is not None: + video_options["crf"] = str(crf) + + if fast_decode: + key = "svtav1-params" if vcodec == "libsvtav1" else "tune" + value = f"fast-decode={fast_decode}" if vcodec == "libsvtav1" else "fastdecode" + video_options[key] = value + + # Set logging level + if log_level is not None: + # "While less efficient, it is generally preferable to modify logging with Python’s logging" + logging.getLogger("libav").setLevel(log_level) + + # Create and open output file (overwrite by default) + with av.open(str(video_path), "w") as output: + output_stream = output.add_stream(vcodec, fps, options=video_options) + output_stream.pix_fmt = pix_fmt + output_stream.width = width + output_stream.height = height + + # Loop through input frames and encode them + for input_data in input_list: + input_image = Image.open(input_data).convert("RGB") + input_frame = av.VideoFrame.from_image(input_image) + packet = output_stream.encode(input_frame) + if packet: + output.mux(packet) + + # Flush the encoder + packet = output_stream.encode() + if packet: + output.mux(packet) + + # Reset logging level + if log_level is not None: + av.logging.restore_default_callback() + + if not video_path.exists(): + raise OSError(f"Video encoding did not work. File not found: {video_path}.") + +# GPU speed up video encoding +def encode_video_frames_ffmpeg( + imgs_dir: Path | str, + video_path: Path | str, + fps: int, + vcodec: str = "h264_nvenc", + pix_fmt: str = "yuv420p", + g: int | None = 4, + crf: int | None = 30, + overwrite: bool = False, +) -> None: + import subprocess + import shutil + + video_path = Path(video_path) + imgs_dir = Path(imgs_dir) + + video_path.parent.mkdir(parents=True, exist_ok=True) + + # Check if ffmpeg is available + ffmpeg_path = shutil.which("ffmpeg") + if ffmpeg_path is None: + raise RuntimeError("ffmpeg is not installed or not in system PATH") + + # Build ffmpeg command + cmd = [ + ffmpeg_path, + "-y" if overwrite else "-n", + "-framerate", str(fps), + "-pattern_type", "sequence", + "-i", str(imgs_dir / "frame_%06d.jpeg"), + "-vcodec", vcodec, + "-pix_fmt", pix_fmt, + ] + + if g is not None: + cmd.extend(["-g", str(g)]) + + if crf is not None: + cmd.extend(["-crf", str(crf)]) + + cmd.append(str(video_path)) + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + check=True + ) + except subprocess.CalledProcessError as e: + error_msg = f"FFmpeg failed:\nCommand: {' '.join(cmd)}\nStdout: {e.stdout}\nStderr: {e.stderr}" + logging.error(error_msg) + logging.error(video_path) + logging.error(vcodec) + print(f"ffmpeg {' '.join(cmd)}") + raise RuntimeError(error_msg) + except Exception as e: + error_msg = f"Failed to run FFmpeg command: {' '.join(cmd)}\nError: {str(e)}" + logging.error(error_msg) + raise RuntimeError(error_msg) + + +@dataclass +class VideoFrame: + # TODO(rcadene, lhoestq): move to Hugging Face `datasets` repo + """ + Provides a type for a dataset containing video frames. + + Example: + + ```python + data_dict = [{"image": {"path": "videos/episode_0.mp4", "timestamp": 0.3}}] + features = {"image": VideoFrame()} + Dataset.from_dict(data_dict, features=Features(features)) + ``` + """ + + pa_type: ClassVar[Any] = pa.struct({"path": pa.string(), "timestamp": pa.float32()}) + _type: str = field(default="VideoFrame", init=False, repr=False) + + def __call__(self): + return self.pa_type + + +with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + "'register_feature' is experimental and might be subject to breaking changes in the future.", + category=UserWarning, + ) + # to make VideoFrame available in HuggingFace `datasets` + register_feature(VideoFrame, "VideoFrame") + + +def get_audio_info(video_path: Path | str) -> dict: + # Set logging level + logging.getLogger("libav").setLevel(av.logging.ERROR) + + # Getting audio stream information + audio_info = {} + with av.open(str(video_path), "r") as audio_file: + try: + audio_stream = audio_file.streams.audio[0] + except IndexError: + # Reset logging level + av.logging.restore_default_callback() + return {"has_audio": False} + + audio_info["audio.channels"] = audio_stream.channels + audio_info["audio.codec"] = audio_stream.codec.canonical_name + # In an ideal loseless case : bit depth x sample rate x channels = bit rate. + # In an actual compressed case, the bit rate is set according to the compression level : the lower the bit rate, the more compression is applied. + audio_info["audio.bit_rate"] = audio_stream.bit_rate + audio_info["audio.sample_rate"] = audio_stream.sample_rate # Number of samples per second + # In an ideal loseless case : fixed number of bits per sample. + # In an actual compressed case : variable number of bits per sample (often reduced to match a given depth rate). + audio_info["audio.bit_depth"] = audio_stream.format.bits + audio_info["audio.channel_layout"] = audio_stream.layout.name + audio_info["has_audio"] = True + + # Reset logging level + av.logging.restore_default_callback() + + return audio_info + + +def get_video_info(video_path: Path | str) -> dict: + # Set logging level + logging.getLogger("libav").setLevel(av.logging.ERROR) + + # Getting video stream information + video_info = {} + with av.open(str(video_path), "r") as video_file: + try: + video_stream = video_file.streams.video[0] + except IndexError: + # Reset logging level + av.logging.restore_default_callback() + return {} + + video_info["video.height"] = video_stream.height + video_info["video.width"] = video_stream.width + video_info["video.codec"] = video_stream.codec.canonical_name + video_info["video.pix_fmt"] = video_stream.pix_fmt + video_info["video.is_depth_map"] = False + + # Calculate fps from r_frame_rate + video_info["video.fps"] = int(video_stream.base_rate) + + pixel_channels = get_video_pixel_channels(video_stream.pix_fmt) + video_info["video.channels"] = pixel_channels + + # Reset logging level + av.logging.restore_default_callback() + + # Adding audio stream information + video_info.update(**get_audio_info(video_path)) + + return video_info + + +def get_video_pixel_channels(pix_fmt: str) -> int: + if "gray" in pix_fmt or "depth" in pix_fmt or "monochrome" in pix_fmt: + return 1 + elif "rgba" in pix_fmt or "yuva" in pix_fmt: + return 4 + elif "rgb" in pix_fmt or "yuv" in pix_fmt: + return 3 + else: + raise ValueError("Unknown format") + + +def get_image_pixel_channels(image: Image): + if image.mode == "L": + return 1 # Grayscale + elif image.mode == "LA": + return 2 # Grayscale + Alpha + elif image.mode == "RGB": + return 3 # RGB + elif image.mode == "RGBA": + return 4 # RGBA + else: + raise ValueError("Unknown format") diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..bc36bdba8638c23584d56736c35c494a4643b97b --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/lerobot/lerobot_dataset.py @@ -0,0 +1,1298 @@ +#!/usr/bin/env python + +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import contextlib +import logging +import shutil +from pathlib import Path +from typing import Callable, List, Literal +import warnings + +import datasets +import numpy as np +import packaging.version +import PIL.Image +import torch +import torch.utils +import pyarrow.parquet as pq +from datasets import concatenate_datasets, load_dataset +from huggingface_hub import HfApi, snapshot_download +from huggingface_hub.constants import REPOCARD_NAME +from huggingface_hub.errors import RevisionNotFoundError + +# adapte to own path +from ..constants import HF_LEROBOT_HOME +from .datasets.compute_stats import aggregate_stats, compute_episode_stats +from .datasets.utils import ( + DEFAULT_FEATURES, + DEFAULT_IMAGE_PATH, + INFO_PATH, + TASKS_PATH, + _validate_feature_names, + append_jsonlines, + backward_compatible_episodes_stats, + check_delta_timestamps, + check_timestamps_sync, + # check_version_compatibility, + create_empty_dataset_info, + create_lerobot_dataset_card, + embed_images, + get_delta_indices, + get_episode_data_index, + get_hf_features_from_features, + # get_safe_version, + hf_transform_to_torch, + is_valid_version, + load_episodes, + load_episodes_stats, + load_info, + load_stats, + load_tasks, + load_annotations, + validate_episode_buffer, + validate_frame, + write_episode, + write_episode_stats, + write_info, + write_json, +) +from .datasets.video_utils import ( + VideoFrame, + decode_video_frames, + encode_video_frames, + get_safe_default_codec, + get_video_info, +) +import traceback + +CODEBASE_VERSION = "v2.1" + + +class LeRobotDatasetMetadata: + def __init__( + self, + repo_id: str, + root: str | Path | None = None, + revision: str | None = None, + force_cache_sync: bool = False, + ): + self.repo_id = repo_id + self.revision = revision if revision else CODEBASE_VERSION + self.root = Path(root) if root is not None else HF_LEROBOT_HOME / repo_id + + try: + if force_cache_sync: + raise FileNotFoundError + self.load_metadata() + except (FileNotFoundError, NotADirectoryError): + # if is_valid_version(self.revision): + # self.revision = get_safe_version(self.repo_id, self.revision) + + (self.root / "meta").mkdir(exist_ok=True, parents=True) + self.pull_from_repo(allow_patterns="meta/") + self.load_metadata() + + def load_metadata(self): + self.info = load_info(self.root) + # TODO add new check + # check_version_compatibility(self.repo_id, self._version, CODEBASE_VERSION) + self.tasks, self.task_to_task_index = load_tasks(self.root) + if (self.root / "annotations").exists(): + self.annotations = load_annotations(self.root) + self.episodes = load_episodes(self.root) + if self._version < packaging.version.parse("v2.1"): + self.stats = load_stats(self.root) + self.episodes_stats = backward_compatible_episodes_stats(self.stats, self.episodes) + else: + self.episodes_stats = load_episodes_stats(self.root) + self.stats = aggregate_stats(list(self.episodes_stats.values())) + + def pull_from_repo( + self, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + ) -> None: + snapshot_download( + self.repo_id, + repo_type="dataset", + revision=self.revision, + local_dir=self.root, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + ) + + @property + def _version(self) -> packaging.version.Version: + """Codebase version used to create this dataset.""" + return packaging.version.parse(self.info["codebase_version"]) + + def get_data_file_path(self, ep_index: int) -> Path: + ep_chunk = self.get_episode_chunk(ep_index) + fpath = self.data_path.format(episode_chunk=ep_chunk, episode_index=ep_index) + return Path(fpath) + + def get_video_file_path(self, ep_index: int, vid_key: str) -> Path: + ep_chunk = self.get_episode_chunk(ep_index) + fpath = self.video_path.format(episode_chunk=ep_chunk, video_key=vid_key, episode_index=ep_index) + return Path(fpath) + + def get_episode_chunk(self, ep_index: int) -> int: + return ep_index // self.chunks_size + + @property + def data_path(self) -> str: + """Formattable string for the parquet files.""" + return self.info["data_path"] + + @property + def video_path(self) -> str | None: + """Formattable string for the video files.""" + return self.info["video_path"] + + @property + def robot_type(self) -> str | None: + """Robot type used in recording this dataset.""" + return self.info["robot_type"] + + @property + def fps(self) -> int: + """Frames per second used during data collection.""" + return self.info["fps"] + + @property + def features(self) -> dict[str, dict]: + """All features contained in the dataset.""" + return self.info["features"] + + @property + def image_keys(self) -> list[str]: + """Keys to access visual modalities stored as images.""" + return [key for key, ft in self.features.items() if ft["dtype"] == "image"] + + @property + def video_keys(self) -> list[str]: + """Keys to access visual modalities stored as videos.""" + return [key for key, ft in self.features.items() if ft["dtype"] == "video"] + + @property + def camera_keys(self) -> list[str]: + """Keys to access visual modalities (regardless of their storage method).""" + return [key for key, ft in self.features.items() if ft["dtype"] in ["video", "image"]] + + @property + def names(self) -> dict[str, list | dict]: + """Names of the various dimensions of vector modalities.""" + return {key: ft["names"] for key, ft in self.features.items()} + + @property + def shapes(self) -> dict: + """Shapes for the different features.""" + return {key: tuple(ft["shape"]) for key, ft in self.features.items()} + + @property + def total_episodes(self) -> int: + """Total number of episodes available.""" + return self.info["total_episodes"] + + @property + def total_frames(self) -> int: + """Total number of frames saved in this dataset.""" + return self.info["total_frames"] + + @property + def total_tasks(self) -> int: + """Total number of different tasks performed in this dataset.""" + return self.info["total_tasks"] + + @property + def total_chunks(self) -> int: + """Total number of chunks (groups of episodes).""" + return self.info["total_chunks"] + + @property + def chunks_size(self) -> int: + """Max number of episodes per chunk.""" + return self.info["chunks_size"] + + def get_task_index(self, task: str) -> int | None: + """ + Given a task in natural language, returns its task_index if the task already exists in the dataset, + otherwise return None. + """ + return self.task_to_task_index.get(task, None) + + def add_task(self, task: str): + """ + Given a task in natural language, add it to the dictionary of tasks. + """ + if task in self.task_to_task_index: + raise ValueError(f"The task '{task}' already exists and can't be added twice.") + + task_index = self.info["total_tasks"] + self.task_to_task_index[task] = task_index + self.tasks[task_index] = task + self.info["total_tasks"] += 1 + + task_dict = { + "task_index": task_index, + "task": task, + } + append_jsonlines(task_dict, self.root / TASKS_PATH) + + def save_episode( + self, + episode_index: int, + episode_length: int, + episode_tasks: list[str], + episode_stats: dict[str, dict], + raw_file_name: str | None = None, + ) -> None: + self.info["total_episodes"] += 1 + self.info["total_frames"] += episode_length + + chunk = self.get_episode_chunk(episode_index) + if chunk >= self.total_chunks: + self.info["total_chunks"] += 1 + + self.info["splits"] = {"train": f"0:{self.info['total_episodes']}"} + self.info["total_videos"] += len(self.video_keys) + if len(self.video_keys) > 0: + self.update_video_info() + + write_info(self.info, self.root) + + episode_dict = { + "episode_index": episode_index, + "tasks": episode_tasks, + "length": episode_length, + } + if raw_file_name is not None: + episode_dict["raw_file_name"] = raw_file_name + + self.episodes[episode_index] = episode_dict + write_episode(episode_dict, self.root) + + self.episodes_stats[episode_index] = episode_stats + self.stats = aggregate_stats([self.stats, episode_stats]) if self.stats else episode_stats + write_episode_stats(episode_index, episode_stats, self.root) + + def update_video_info(self) -> None: + """ + Warning: this function writes info from first episode videos, implicitly assuming that all videos have + been encoded the same way. Also, this means it assumes the first episode exists. + """ + for key in self.video_keys: + if not self.features[key].get("info", None): + video_path = self.root / self.get_video_file_path(ep_index=0, vid_key=key) + self.info["features"][key]["info"] = get_video_info(video_path) + + def __repr__(self): + feature_keys = list(self.features) + return ( + f"{self.__class__.__name__}({{\n" + f" Repository ID: '{self.repo_id}',\n" + f" Total episodes: '{self.total_episodes}',\n" + f" Total frames: '{self.total_frames}',\n" + f" Features: '{feature_keys}',\n" + "})',\n" + ) + + @classmethod + def create( + cls, + repo_id: str, + fps: int, + features: dict, + robot_type: str | None = None, + root: str | Path | None = None, + use_videos: bool = True, + ) -> "LeRobotDatasetMetadata": + """Creates metadata for a LeRobotDataset.""" + obj = cls.__new__(cls) + obj.repo_id = repo_id + obj.root = Path(root) if root is not None else HF_LEROBOT_HOME / repo_id + + obj.root.mkdir(parents=True, exist_ok=False) + + # TODO(aliberts, rcadene): implement sanity check for features + features = {**features, **DEFAULT_FEATURES} + _validate_feature_names(features) + + obj.tasks, obj.task_to_task_index = {}, {} + obj.episodes_stats, obj.stats, obj.episodes = {}, {}, {} + obj.info = create_empty_dataset_info(CODEBASE_VERSION, fps, features, use_videos, robot_type) + if len(obj.video_keys) > 0 and not use_videos: + raise ValueError() + write_json(obj.info, obj.root / INFO_PATH) + obj.revision = None + return obj + + +class LeRobotDataset(torch.utils.data.Dataset): + def __init__( + self, + repo_id: str, + root: str | Path | None = None, + episodes: list[int] | None = None, + image_transforms: Callable | None = None, + delta_timestamps: dict[list[float]] | None = None, + tolerance_s: float = 1e-4, + revision: str | None = None, + force_cache_sync: bool = False, + download_videos: bool = True, + video_backend: str | None = None, + video_codec: Literal["h264", "hevc", "libsvtav1", "h264_nvenc"] = "libsvtav1", + is_compute_episode_stats_image: bool = True, + ): + """ + 2 modes are available for instantiating this class, depending on 2 different use cases: + + 1. Your dataset already exists: + - On your local disk in the 'root' folder. This is typically the case when you recorded your + dataset locally and you may or may not have pushed it to the hub yet. Instantiating this class + with 'root' will load your dataset directly from disk. This can happen while you're offline (no + internet connection). + + - On the Hugging Face Hub at the address https://huggingface.co/datasets/{repo_id} and not on + your local disk in the 'root' folder. Instantiating this class with this 'repo_id' will download + the dataset from that address and load it, pending your dataset is compliant with + codebase_version v2.0. If your dataset has been created before this new format, you will be + prompted to convert it using our conversion script from v1.6 to v2.0, which you can find at + lerobot/datasets/v2/convert_dataset_v1_to_v2.py. + + + 2. Your dataset doesn't already exists (either on local disk or on the Hub): you can create an empty + LeRobotDataset with the 'create' classmethod. This can be used for recording a dataset or port an + existing dataset to the LeRobotDataset format. + + + In terms of files, LeRobotDataset encapsulates 3 main things: + - metadata: + - info contains various information about the dataset like shapes, keys, fps etc. + - stats stores the dataset statistics of the different modalities for normalization + - tasks contains the prompts for each task of the dataset, which can be used for + task-conditioned training. + - hf_dataset (from datasets.Dataset), which will read any values from parquet files. + - videos (optional) from which frames are loaded to be synchronous with data from parquet files. + + A typical LeRobotDataset looks like this from its root path: + . + ├── data + │ ├── chunk-000 + │ │ ├── episode_000000.parquet + │ │ ├── episode_000001.parquet + │ │ ├── episode_000002.parquet + │ │ └── ... + │ ├── chunk-001 + │ │ ├── episode_001000.parquet + │ │ ├── episode_001001.parquet + │ │ ├── episode_001002.parquet + │ │ └── ... + │ └── ... + ├── meta + │ ├── episodes.jsonl + │ ├── info.json + │ ├── stats.json + │ └── tasks.jsonl + └── videos + ├── chunk-000 + │ ├── observation.images.laptop + │ │ ├── episode_000000.mp4 + │ │ ├── episode_000001.mp4 + │ │ ├── episode_000002.mp4 + │ │ └── ... + │ ├── observation.images.phone + │ │ ├── episode_000000.mp4 + │ │ ├── episode_000001.mp4 + │ │ ├── episode_000002.mp4 + │ │ └── ... + ├── chunk-001 + └── ... + + Note that this file-based structure is designed to be as versatile as possible. The files are split by + episodes which allows a more granular control over which episodes one wants to use and download. The + structure of the dataset is entirely described in the info.json file, which can be easily downloaded + or viewed directly on the hub before downloading any actual data. The type of files used are very + simple and do not need complex tools to be read, it only uses .parquet, .json and .mp4 files (and .md + for the README). + + Args: + repo_id (str): This is the repo id that will be used to fetch the dataset. Locally, the dataset + will be stored under root/repo_id. + root (Path | None, optional): Local directory to use for downloading/writing files. You can also + set the LEROBOT_HOME environment variable to point to a different location. Defaults to + '~/.cache/huggingface/lerobot'. + episodes (list[int] | None, optional): If specified, this will only load episodes specified by + their episode_index in this list. Defaults to None. + image_transforms (Callable | None, optional): You can pass standard v2 image transforms from + torchvision.transforms.v2 here which will be applied to visual modalities (whether they come + from videos or images). Defaults to None. + delta_timestamps (dict[list[float]] | None, optional): _description_. Defaults to None. + tolerance_s (float, optional): Tolerance in seconds used to ensure data timestamps are actually in + sync with the fps value. It is used at the init of the dataset to make sure that each + timestamps is separated to the next by 1/fps +/- tolerance_s. This also applies to frames + decoded from video files. It is also used to check that `delta_timestamps` (when provided) are + multiples of 1/fps. Defaults to 1e-4. + revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a + commit hash. Defaults to current codebase version tag. + sync_cache_first (bool, optional): Flag to sync and refresh local files first. If True and files + are already present in the local cache, this will be faster. However, files loaded might not + be in sync with the version on the hub, especially if you specified 'revision'. Defaults to + False. + download_videos (bool, optional): Flag to download the videos. Note that when set to True but the + video files are already present on local disk, they won't be downloaded again. Defaults to + True. + video_backend (str | None, optional): Video backend to use for decoding videos. Defaults to torchcodec when available int the platform; otherwise, defaults to 'pyav'. + You can also use the 'pyav' decoder used by Torchvision, which used to be the default option, or 'video_reader' which is another decoder of Torchvision. + """ + super().__init__() + self.repo_id = repo_id + self.root = Path(root) if root else HF_LEROBOT_HOME / repo_id + self.image_transforms = image_transforms + self.delta_timestamps = delta_timestamps + self.episodes = episodes + self.tolerance_s = tolerance_s + self.revision = revision if revision else CODEBASE_VERSION + self.video_backend = video_backend if video_backend else get_safe_default_codec() + self.video_codec = video_codec + self.is_compute_episode_stats_image = is_compute_episode_stats_image + self.delta_indices = None + self.during_training = True + + # Unused attributes + self.image_writer = None + self.episode_buffer = None + + self.root.mkdir(exist_ok=True, parents=True) + + # Load metadata + self.meta = LeRobotDatasetMetadata( + self.repo_id, self.root, self.revision, force_cache_sync=force_cache_sync + ) + if self.episodes is not None and self.meta._version >= packaging.version.parse("v2.1"): + episodes_stats = [self.meta.episodes_stats[ep_idx] for ep_idx in self.episodes] + self.stats = aggregate_stats(episodes_stats) + + # Load actual data + try: + if force_cache_sync: + raise FileNotFoundError + assert all((self.root / fpath).is_file() for fpath in self.get_episodes_file_paths()) + self.hf_dataset = self.load_hf_dataset() + except (AssertionError, FileNotFoundError, NotADirectoryError): + # self.revision = get_safe_version(self.repo_id, self.revision) + self.revision = CODEBASE_VERSION + self.download_episodes(download_videos) + self.hf_dataset = self.load_hf_dataset() + + self.episode_data_index = get_episode_data_index(self.meta.episodes, self.episodes) + + # Setup delta_indices + if self.delta_timestamps is not None: + # check_delta_timestamps(self.delta_timestamps, self.fps, self.tolerance_s) + self.delta_indices = get_delta_indices(self.delta_timestamps, self.fps) + + def push_to_hub( + self, + branch: str | None = None, + tags: list | None = None, + license: str | None = "apache-2.0", + tag_version: bool = True, + push_videos: bool = True, + private: bool = False, + allow_patterns: list[str] | str | None = None, + upload_large_folder: bool = False, + **card_kwargs, + ) -> None: + ignore_patterns = ["images/"] + if not push_videos: + ignore_patterns.append("videos/") + + hub_api = HfApi() + hub_api.create_repo( + repo_id=self.repo_id, + private=private, + repo_type="dataset", + exist_ok=True, + ) + if branch: + hub_api.create_branch( + repo_id=self.repo_id, + branch=branch, + revision=self.revision, + repo_type="dataset", + exist_ok=True, + ) + + upload_kwargs = { + "repo_id": self.repo_id, + "folder_path": self.root, + "repo_type": "dataset", + "revision": branch, + "allow_patterns": allow_patterns, + "ignore_patterns": ignore_patterns, + } + if upload_large_folder: + hub_api.upload_large_folder(**upload_kwargs) + else: + hub_api.upload_folder(**upload_kwargs) + + if not hub_api.file_exists(self.repo_id, REPOCARD_NAME, repo_type="dataset", revision=branch): + card = create_lerobot_dataset_card( + tags=tags, dataset_info=self.meta.info, license=license, **card_kwargs + ) + card.push_to_hub(repo_id=self.repo_id, repo_type="dataset", revision=branch) + + if tag_version: + with contextlib.suppress(RevisionNotFoundError): + hub_api.delete_tag(self.repo_id, tag=CODEBASE_VERSION, repo_type="dataset") + hub_api.create_tag(self.repo_id, tag=CODEBASE_VERSION, revision=branch, repo_type="dataset") + + def pull_from_repo( + self, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + ) -> None: + snapshot_download( + self.repo_id, + repo_type="dataset", + revision=self.revision, + local_dir=self.root, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + ) + + def download_episodes(self, download_videos: bool = True) -> None: + """Downloads the dataset from the given 'repo_id' at the provided version. If 'episodes' is given, this + will only download those episodes (selected by their episode_index). If 'episodes' is None, the whole + dataset will be downloaded. Thanks to the behavior of snapshot_download, if the files are already present + in 'local_dir', they won't be downloaded again. + """ + # TODO(rcadene, aliberts): implement faster transfer + # https://huggingface.co/docs/huggingface_hub/en/guides/download#faster-downloads + files = None + ignore_patterns = None if download_videos else "videos/" + if self.episodes is not None: + files = self.get_episodes_file_paths() + + self.pull_from_repo(allow_patterns=files, ignore_patterns=ignore_patterns) + + def get_episodes_file_paths(self) -> list[Path]: + episodes = self.episodes if self.episodes is not None else list(range(self.meta.total_episodes)) + fpaths = [str(self.meta.get_data_file_path(ep_idx)) for ep_idx in episodes] + if len(self.meta.video_keys) > 0: + video_files = [ + str(self.meta.get_video_file_path(ep_idx, vid_key)) + for vid_key in self.meta.video_keys + for ep_idx in episodes + ] + fpaths += video_files + + return fpaths + + def load_hf_dataset(self) -> datasets.Dataset: + """hf_dataset contains all the observations, states, actions, rewards, etc.""" + if self.episodes is None: + path = str(self.root / "data") + hf_dataset = load_dataset("parquet", data_dir=path, split="train") + else: + files = [str(self.root / self.meta.get_data_file_path(ep_idx)) for ep_idx in self.episodes] + hf_dataset = load_dataset("parquet", data_files=files, split="train") + + # TODO(aliberts): hf_dataset.set_format("torch") + hf_dataset.set_transform(hf_transform_to_torch) + return hf_dataset + + def create_hf_dataset(self) -> datasets.Dataset: + features = get_hf_features_from_features(self.features) + ft_dict = {col: [] for col in features} + hf_dataset = datasets.Dataset.from_dict(ft_dict, features=features, split="train") + + # TODO(aliberts): hf_dataset.set_format("torch") + hf_dataset.set_transform(hf_transform_to_torch) + return hf_dataset + + @property + def fps(self) -> int: + """Frames per second used during data collection.""" + return self.meta.fps + + @property + def num_frames(self) -> int: + """Number of frames in selected episodes.""" + return len(self.hf_dataset) if self.hf_dataset is not None else self.meta.total_frames + + @property + def num_episodes(self) -> int: + """Number of episodes selected.""" + return len(self.episodes) if self.episodes is not None else self.meta.total_episodes + + @property + def features(self) -> dict[str, dict]: + return self.meta.features + + @property + def hf_features(self) -> datasets.Features: + """Features of the hf_dataset.""" + if self.hf_dataset is not None: + return self.hf_dataset.features + else: + return get_hf_features_from_features(self.features) + + def _get_query_indices(self, idx: int, ep_idx: int) -> tuple[dict[str, list[int | bool]]]: + ep_start = self.episode_data_index["from"][ep_idx] + ep_end = self.episode_data_index["to"][ep_idx] + query_indices = { + key: [max(ep_start.item(), min(ep_end.item() - 1, idx + delta)) for delta in delta_idx] + for key, delta_idx in self.delta_indices.items() + } + padding = { # Pad values outside of current episode range + f"{key}_is_pad": torch.BoolTensor( + [(idx + delta < ep_start.item()) | (idx + delta >= ep_end.item()) for delta in delta_idx] + ) + for key, delta_idx in self.delta_indices.items() + } + return query_indices, padding + + def _get_query_timestamps( + self, + current_ts: float, + query_indices: dict[str, list[int]] | None = None, + ) -> dict[str, list[float]]: + query_timestamps = {} + for key in self.meta.video_keys: + if query_indices is not None and key in query_indices: + timestamps = self.hf_dataset.select(query_indices[key])["timestamp"] + query_timestamps[key] = torch.stack(list(timestamps)).tolist() + else: + query_timestamps[key] = [current_ts] + + return query_timestamps + + def _query_hf_dataset(self, query_indices: dict[str, list[int]]) -> dict: + return { + key: torch.stack(list(self.hf_dataset.select(q_idx)[key])) + for key, q_idx in query_indices.items() + if key not in self.meta.video_keys + } + + def _query_hf_dataset_fast(self, query_indices: dict[str, list[int]]) -> dict: + result = {} + processed_indices = set() + index_to_selected = {} + for key, q_idx in query_indices.items(): + if key not in self.meta.video_keys : + if 'images' in key and not self.during_training: + continue + q_idx_tuple = tuple(q_idx) + if q_idx_tuple not in processed_indices: + selected_data = self.hf_dataset.select(q_idx) + index_to_selected[q_idx_tuple] = selected_data + processed_indices.add(q_idx_tuple) + else: + selected_data = index_to_selected[q_idx_tuple] + result[key] = torch.stack(list(selected_data[key])) + return result + + # no videos + def get_episode_data(self, episode_id: int) -> dict: + ep_start = self.episode_data_index["from"][episode_id].item() + ep_end = self.episode_data_index["to"][episode_id].item() + q_idx = list(range(ep_start, ep_end)) + # selected_data = self.hf_dataset.select(q_idx) + selected_data = self.hf_dataset[q_idx] + res_keys = self.meta.features.keys() - set(self.meta.video_keys) + res = {key : torch.stack(selected_data[key]) for key in res_keys} + return res + + def _query_videos(self, query_timestamps: dict[str, list[float]], ep_idx: int) -> dict[str, torch.Tensor]: + """Note: When using data workers (e.g. DataLoader with num_workers>0), do not call this function + in the main process (e.g. by using a second Dataloader with num_workers=0). It will result in a + Segmentation Fault. This probably happens because a memory reference to the video loader is created in + the main process and a subprocess fails to access it. + """ + item = {} + for vid_key, query_ts in query_timestamps.items(): + video_path = self.root / self.meta.get_video_file_path(ep_idx, vid_key) + frames = decode_video_frames(video_path, query_ts, self.tolerance_s, self.video_backend) + item[vid_key] = frames.squeeze(0) + + return item + + def _add_padding_keys(self, item: dict, padding: dict[str, list[bool]]) -> dict: + for key, val in padding.items(): + item[key] = torch.BoolTensor(val) + return item + + def __len__(self): + return self.num_frames + + def __getitem__(self, idx) -> dict: + item = self.hf_dataset[idx] + ep_idx = item["episode_index"].item() + + query_indices = None + if self.delta_indices is not None: + current_ep_idx = self.episodes.index(ep_idx) if self.episodes is not None else ep_idx + query_indices, padding = self._get_query_indices(idx, current_ep_idx) + query_result = self._query_hf_dataset_fast(query_indices) + item = {**item, **padding} + for key, val in query_result.items(): + item[key] = val + + if len(self.meta.video_keys) > 0 and self.during_training: + current_ts = item["timestamp"].item() + query_timestamps = self._get_query_timestamps(current_ts, query_indices) + video_frames = self._query_videos(query_timestamps, ep_idx) + item = {**video_frames, **item} + + if self.image_transforms is not None: + image_keys = self.meta.camera_keys + for cam in image_keys: + item[cam] = self.image_transforms(item[cam]) + + # Add task as a string + task_idx = item["task_index"].item() + item["task"] = self.meta.tasks[task_idx] + if "coarse_task_index" in item: + coarse_task_index = item["coarse_task_index"].item() + item["coarse_task"] = self.meta.tasks[coarse_task_index] + + if "operating_hand_index" in item: + operating_hand_index = item["operating_hand_index"].item() + item["operating_hand"] = self.meta.tasks[operating_hand_index] + + if "subtask_annotation" in item and hasattr(self.meta, "annotations"): + index = item["subtask_annotation"][0].item() + item["subtask"] = self.meta.annotations["subtask"][index] + + if "atomic_task_index" in item and item["atomic_task_index"] is not None: + atomic_task_index = item["atomic_task_index"].item() + item["subtask"] = self.meta.tasks[int(atomic_task_index)] + + return item + + def __repr__(self): + feature_keys = list(self.features) + return ( + f"{self.__class__.__name__}({{\n" + f" Repository ID: '{self.repo_id}',\n" + f" Number of selected episodes: '{self.num_episodes}',\n" + f" Number of selected samples: '{self.num_frames}',\n" + f" Features: '{feature_keys}',\n" + "})',\n" + ) + + def create_episode_buffer(self, episode_index: int | None = None) -> dict: + current_ep_idx = self.meta.total_episodes if episode_index is None else episode_index + ep_buffer = {} + # size and task are special cases that are not in self.features + ep_buffer["size"] = 0 + ep_buffer["task"] = [] + for key in self.features: + ep_buffer[key] = current_ep_idx if key == "episode_index" else [] + return ep_buffer + + def _get_image_file_path(self, episode_index: int, image_key: str, frame_index: int) -> Path: + fpath = DEFAULT_IMAGE_PATH.format( + image_key=image_key, episode_index=episode_index, frame_index=frame_index + ) + return self.root / fpath + + # def _save_image(self, image: torch.Tensor | np.ndarray | PIL.Image.Image | bytes, fpath: Path) -> None: + # if self.image_writer is None: + # if isinstance(image, torch.Tensor): + # image = image.cpu().numpy() + # write_image(image, fpath) + # else: + # self.image_writer.save_image(image=image, fpath=fpath) + + def add_frame(self, frame: dict, task: List[str], timestamp: float | None = None) -> None: + """ + This function only adds the frame to the episode_buffer. Apart from images — which are written in a + temporary directory — nothing is written to disk. To save those frames, the 'save_episode()' method + then needs to be called. + """ + # Convert torch to numpy if needed + assert len(task) == 4, "Task frame must be of two elements" + for name in frame: + if isinstance(frame[name], torch.Tensor): + frame[name] = frame[name].numpy() + + validate_frame(frame, self.features) + + if self.episode_buffer is None: + self.episode_buffer = self.create_episode_buffer() + + # Automatically add frame_index and timestamp to episode buffer + frame_index = self.episode_buffer["size"] + if timestamp is None: + timestamp = frame_index / self.fps + self.episode_buffer["frame_index"].append(frame_index) + self.episode_buffer["timestamp"].append(timestamp) + self.episode_buffer["task"].append(task) + + # Add frame features to episode_buffer + for key in frame: + if key not in self.features: + raise ValueError( + f"An element of the frame is not in the features. '{key}' not in '{self.features.keys()}'." + ) + + if self.features[key]["dtype"] in ["image", "video"]: + img_path = self._get_image_file_path( + episode_index=self.episode_buffer["episode_index"], image_key=key, frame_index=frame_index + ) + if frame_index == 0: + img_path.parent.mkdir(parents=True, exist_ok=True) + # self._save_image(frame[key], img_path) + self.episode_buffer[key].append(str(img_path)) + else: + self.episode_buffer[key].append(frame[key]) + + self.episode_buffer["size"] += 1 + + def save_episode(self, episode_data: dict | None = None, raw_file_name: str | None = None) -> None: + """ + This will save to disk the current episode in self.episode_buffer. + + Args: + episode_data (dict | None, optional): Dict containing the episode data to save. If None, this will + save the current episode in self.episode_buffer, which is filled with 'add_frame'. Defaults to + None. + """ + if not episode_data: + episode_buffer = self.episode_buffer + + validate_episode_buffer(episode_buffer, self.meta.total_episodes, self.features) + + # size and task are special cases that won't be added to hf_dataset + episode_length = episode_buffer.pop("size") + tasks = episode_buffer.pop("task") + episode_tasks = list(set([item for sublist in tasks for item in sublist])) + episode_index = episode_buffer["episode_index"] + + episode_buffer["index"] = np.arange(self.meta.total_frames, self.meta.total_frames + episode_length) + episode_buffer["episode_index"] = np.full((episode_length,), episode_index) + + # Add new tasks to the tasks dictionary + for task in episode_tasks: + task_index = self.meta.get_task_index(task) + if task_index is None: + self.meta.add_task(task) + + # Given tasks in natural language, find their corresponding task indices + episode_buffer["coarse_task_index"] = np.array([self.meta.get_task_index(task[0]) for task in tasks]) + episode_buffer["task_index"] = np.array([self.meta.get_task_index(task[1]) for task in tasks]) + episode_buffer["coarse_quality_index"] = np.array([self.meta.get_task_index(task[2]) for task in tasks]) + episode_buffer["quality_index"] = np.array([self.meta.get_task_index(task[3]) for task in tasks]) + + for key, ft in self.features.items(): + # index, episode_index, task_index are already processed above, and image and video + # are processed separately by storing image path and frame info as meta data + if key in ["index", "episode_index", "coarse_task_index", "task_index", "coarse_quality_index", "quality_index"] or ft["dtype"] in ["image", "video"]: + continue + episode_buffer[key] = np.stack(episode_buffer[key]) + + self._wait_image_writer() + self._save_episode_table(episode_buffer, episode_index) + ep_stats = compute_episode_stats(episode_buffer, self.features, self.is_compute_episode_stats_image) + + if len(self.meta.video_keys) > 0: + video_paths = self.encode_episode_videos(episode_index) + for key in self.meta.video_keys: + episode_buffer[key] = video_paths[key] + + # `meta.save_episode` be executed after encoding the videos + self.meta.save_episode(episode_index, episode_length, episode_tasks, ep_stats, raw_file_name) + + ep_data_index = get_episode_data_index(self.meta.episodes, [episode_index]) + ep_data_index_np = {k: t.numpy() for k, t in ep_data_index.items()} + check_timestamps_sync( + episode_buffer["timestamp"], + episode_buffer["episode_index"], + ep_data_index_np, + self.fps, + self.tolerance_s, + ) + + video_files = list(self.root.rglob("*.mp4")) + assert len(video_files) == self.num_episodes * len(self.meta.video_keys) + + parquet_files = list(self.root.rglob("*.parquet")) + assert len(parquet_files) == self.num_episodes + + # delete images + img_dir = self.root / "images" + if img_dir.is_dir(): + shutil.rmtree(self.root / "images") + + if not episode_data: # Reset the buffer + self.episode_buffer = self.create_episode_buffer() + + def _save_episode_table(self, episode_buffer: dict, episode_index: int) -> None: + episode_dict = {key: episode_buffer[key] for key in self.hf_features} + ep_dataset = datasets.Dataset.from_dict(episode_dict, features=self.hf_features, split="train") + ep_dataset = embed_images(ep_dataset) + self.hf_dataset = concatenate_datasets([self.hf_dataset, ep_dataset]) + self.hf_dataset.set_transform(hf_transform_to_torch) + ep_data_path = self.root / self.meta.get_data_file_path(ep_index=episode_index) + ep_data_path.parent.mkdir(parents=True, exist_ok=True) + ep_dataset.to_parquet(ep_data_path) + + def clear_episode_buffer(self) -> None: + episode_index = self.episode_buffer["episode_index"] + if self.image_writer is not None: + for cam_key in self.meta.camera_keys: + img_dir = self._get_image_file_path( + episode_index=episode_index, image_key=cam_key, frame_index=0 + ).parent + if img_dir.is_dir(): + shutil.rmtree(img_dir) + + # Reset the buffer + self.episode_buffer = self.create_episode_buffer() + + # def start_image_writer(self, num_processes: int = 0, num_threads: int = 4) -> None: + # if isinstance(self.image_writer, AsyncImageWriter): + # logging.warning( + # "You are starting a new AsyncImageWriter that is replacing an already existing one in the dataset." + # ) + + # self.image_writer = AsyncImageWriter( + # num_processes=num_processes, + # num_threads=num_threads, + # ) + + def stop_image_writer(self) -> None: + """ + Whenever wrapping this dataset inside a parallelized DataLoader, this needs to be called first to + remove the image_writer in order for the LeRobotDataset object to be picklable and parallelized. + """ + if self.image_writer is not None: + self.image_writer.stop() + self.image_writer = None + + def _wait_image_writer(self) -> None: + """Wait for asynchronous image writer to finish.""" + if self.image_writer is not None: + self.image_writer.wait_until_done() + + def encode_videos(self) -> None: + """ + Use ffmpeg to convert frames stored as png into mp4 videos. + Note: `encode_video_frames` is a blocking call. Making it asynchronous shouldn't speedup encoding, + since video encoding with ffmpeg is already using multithreading. + """ + for ep_idx in range(self.meta.total_episodes): + self.encode_episode_videos(ep_idx) + + def encode_episode_videos(self, episode_index: int) -> dict: + """ + Use ffmpeg to convert frames stored as png into mp4 videos. + Note: `encode_video_frames` is a blocking call. Making it asynchronous shouldn't speedup encoding, + since video encoding with ffmpeg is already using multithreading. + """ + video_paths = {} + for key in self.meta.video_keys: + video_path = self.root / self.meta.get_video_file_path(episode_index, key) + video_paths[key] = str(video_path) + if video_path.is_file(): + # Skip if video is already encoded. Could be the case when resuming data recording. + continue + img_dir = self._get_image_file_path( + episode_index=episode_index, image_key=key, frame_index=0 + ).parent + encode_video_frames(img_dir, video_path, self.fps, overwrite=True, vcodec=self.video_codec) + + return video_paths + + @classmethod + def create( + cls, + repo_id: str, + fps: int, + features: dict, + root: str | Path | None = None, + robot_type: str | None = None, + use_videos: bool = True, + tolerance_s: float = 1e-4, + image_writer_processes: int = 0, + image_writer_threads: int = 0, + video_backend: str | None = None, + video_codec: Literal["h264", "hevc", "libsvtav1", "h264_nvenc"] = "libsvtav1", + is_compute_episode_stats_image = True, + ) -> "LeRobotDataset": + """Create a LeRobot Dataset from scratch in order to record data.""" + obj = cls.__new__(cls) + obj.meta = LeRobotDatasetMetadata.create( + repo_id=repo_id, + fps=fps, + robot_type=robot_type, + features=features, + root=root, + use_videos=use_videos, + ) + obj.repo_id = obj.meta.repo_id + obj.root = obj.meta.root + obj.revision = None + obj.tolerance_s = tolerance_s + obj.image_writer = None + + # if image_writer_processes or image_writer_threads: + # obj.start_image_writer(image_writer_processes, image_writer_threads) + + # TODO(aliberts, rcadene, alexander-soare): Merge this with OnlineBuffer/DataBuffer + obj.episode_buffer = obj.create_episode_buffer() + + obj.episodes = None + obj.hf_dataset = obj.create_hf_dataset() + obj.image_transforms = None + obj.delta_timestamps = None + obj.delta_indices = None + obj.episode_data_index = None + obj.video_backend = video_backend if video_backend is not None else get_safe_default_codec() + obj.video_codec = video_codec + obj.is_compute_episode_stats_image = is_compute_episode_stats_image + return obj + + +class MultiLeRobotDataset(torch.utils.data.Dataset): + """A dataset consisting of multiple underlying `LeRobotDataset`s. + + The underlying `LeRobotDataset`s are effectively concatenated, and this class adopts much of the API + structure of `LeRobotDataset`. + """ + + def __init__( + self, + dataset_dirs: list[str], + episodes: dict | None = None, + image_transforms: Callable | None = None, + delta_timestamps: dict[list[float]] | None = None, + tolerances_s: dict | None = None, + download_videos: bool = True, + video_backend: str | None = None, + ): + super().__init__() + self.dataset_dirs = dataset_dirs + ds_roots = [Path(ds_dir) for ds_dir in dataset_dirs] + ds_names = [ds_dir for ds_dir in dataset_dirs] + self.ds_names = ds_names + self.ds_roots = ds_roots + self.tolerances_s = tolerances_s if tolerances_s else dict.fromkeys(ds_names, 0.0001) + # Construct the underlying datasets passing everything but `transform` and `delta_timestamps` which + # are handled by this class. + self._datasets = [] + for ds_root, ds_name in zip(ds_roots, ds_names, strict=True): + try: + _dataset = LeRobotDataset( + ds_name, + root=ds_root, + episodes=episodes[ds_name] if episodes else None, + image_transforms=image_transforms, + delta_timestamps=delta_timestamps, + tolerance_s=self.tolerances_s[ds_name], + download_videos=download_videos, + video_backend=video_backend, + ) + self._datasets.append(_dataset) + except Exception as e: + # logging.error(e) + logging.error(f"Exception while process ds_root: {ds_root}, ds_name: {ds_name}") + traceback.print_exc() + raise e + + # Disable any data keys that are not common across all of the datasets. Note: we may relax this + # restriction in future iterations of this class. For now, this is necessary at least for being able + # to use PyTorch's default DataLoader collate function. + self.disabled_features = set() + intersection_features = set(self._datasets[0].features) + for ds in self._datasets: + intersection_features.intersection_update(ds.features) + if len(intersection_features) == 0: + raise RuntimeError( + "Multiple datasets were provided but they had no keys common to all of them. " + "The multi-dataset functionality currently only keeps common keys." + ) + for ds_name, ds in zip(self.ds_names, self._datasets, strict=True): + # Disable warning of empty extra keys do + extra_keys = set(ds.features).difference(intersection_features) + if extra_keys: + logging.warning( + f"keys {extra_keys} of {ds_name} were disabled as they are not contained in all the " + "other datasets." + ) + self.disabled_features.update(extra_keys) + + self.image_transforms = image_transforms + self.delta_timestamps = delta_timestamps + # TODO(rcadene, aliberts): We should not perform this aggregation for datasets + # with multiple robots of different ranges. Instead we should have one normalization + # per robot. + self.stats = aggregate_stats([dataset.meta.stats for dataset in self._datasets]) + + def set_during_training(self, during_training: bool): + for dataset in self._datasets: + dataset.during_training = during_training + + @property + def repo_id_to_index(self): + """Return a mapping from dataset repo_id to a dataset index automatically created by this class. + + This index is incorporated as a data key in the dictionary returned by `__getitem__`. + """ + return {repo_id: i for i, repo_id in enumerate(self.ds_names)} + + @property + def repo_index_to_id(self): + """Return the inverse mapping if repo_id_to_index.""" + return {v: k for k, v in self.repo_id_to_index} + + @property + def fps(self) -> int: + """Frames per second used during data collection. + + NOTE: Fow now, this relies on a check in __init__ to make sure all sub-datasets have the same info. + """ + return self._datasets[0].meta.info["fps"] + + @property + def video(self) -> bool: + """Returns True if this dataset loads video frames from mp4 files. + + Returns False if it only loads images from png files. + + NOTE: Fow now, this relies on a check in __init__ to make sure all sub-datasets have the same info. + """ + return self._datasets[0].meta.info.get("video", False) + + @property + def features(self) -> datasets.Features: + features = {} + for dataset in self._datasets: + features.update({k: v for k, v in dataset.hf_features.items() if k not in self.disabled_features}) + return features + + @property + def camera_keys(self) -> list[str]: + """Keys to access image and video stream from cameras.""" + keys = [] + for key, feats in self.features.items(): + if isinstance(feats, (datasets.Image, VideoFrame)): + keys.append(key) + return keys + + @property + def video_frame_keys(self) -> list[str]: + """Keys to access video frames that requires to be decoded into images. + + Note: It is empty if the dataset contains images only, + or equal to `self.cameras` if the dataset contains videos only, + or can even be a subset of `self.cameras` in a case of a mixed image/video dataset. + """ + video_frame_keys = [] + for key, feats in self.features.items(): + if isinstance(feats, VideoFrame): + video_frame_keys.append(key) + return video_frame_keys + + @property + def num_frames(self) -> int: + """Number of samples/frames.""" + return sum(d.num_frames for d in self._datasets) + + @property + def num_episodes(self) -> int: + """Number of episodes.""" + return sum(d.num_episodes for d in self._datasets) + + @property + def tolerance_s(self) -> float: + """Tolerance in seconds used to discard loaded frames when their timestamps + are not close enough from the requested frames. It is only used when `delta_timestamps` + is provided or when loading video frames from mp4 files. + """ + # 1e-4 to account for possible numerical error + return 1 / self.fps - 1e-4 + + def get_episode_data(self, episode_idx: int) -> dict: + for dataset in self._datasets: + if episode_idx < dataset.num_episodes: + file = str(dataset.root / dataset.meta.get_data_file_path(dataset.episodes[episode_idx])) + table = pq.read_table(str(file)) + + result_dict = {} + for col_name in table.column_names: + col = table[col_name] + try: + np_arr = col.to_numpy(zero_copy_only=True) + except Exception: + raw = col.to_numpy() + np_arr = np.stack(raw) if raw.dtype == object else raw + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="The given NumPy array is not writable", + category=UserWarning, + ) + # deal with string in parquet file + if np_arr.dtype == 'O': + result_dict[col_name] = np_arr + else: + result_dict[col_name] = torch.from_numpy(np_arr) + return result_dict + else: + episode_idx -= dataset.num_episodes + raise IndexError(f"Episode index {episode_idx} out of bounds.") + + def __len__(self): + return self.num_frames + + def __getitem__(self, idx: int) -> dict[str, torch.Tensor]: + if idx >= len(self): + raise IndexError(f"Index {idx} out of bounds.") + # Determine which dataset to get an item from based on the index. + start_idx = 0 + dataset_idx = 0 + for dataset in self._datasets: + if idx >= start_idx + dataset.num_frames: + start_idx += dataset.num_frames + dataset_idx += 1 + continue + break + else: + raise AssertionError("We expect the loop to break out as long as the index is within bounds.") + item = self._datasets[dataset_idx][idx - start_idx] + item["dataset_index"] = torch.tensor(dataset_idx) + for data_key in self.disabled_features: + if data_key in item: + del item[data_key] + + return item + + def __repr__(self): + return ( + f"{self.__class__.__name__}(\n" + f" Dataset Names: '{self.ds_names}',\n" + f" Number of Samples: {self.num_frames},\n" + f" Number of Episodes: {self.num_episodes},\n" + f" Type: {'video (.mp4)' if self.video else 'image (.png)'},\n" + f" Recorded Frames per Second: {self.fps},\n" + f" Camera Keys: {self.camera_keys},\n" + f" Video Frame Keys: {self.video_frame_keys if self.video else 'N/A'},\n" + f" Transformations: {self.image_transforms},\n" + f")" + ) diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/processors/base_processor.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/processors/base_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..39d1665b6f0da68060fad7a120241577301a6139 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/processors/base_processor.py @@ -0,0 +1,263 @@ +from abc import ABC, abstractmethod +from typing import Dict, Any, Optional, List, Literal + +import torch +import numpy as np +from copy import deepcopy +from ..utils.normalizer import LinearNormalizer, NormMode +from fastwam.utils.pytorch_utils import dict_apply + + +class BaseProcessor(ABC): + def __init__( + self, + # keys + shape_meta: Dict[str, Any], + num_obs_steps: int, + num_output_cameras: int, + action_output_dim: int, + proprio_output_dim: int, + + action_state_transforms: Optional[List[Any]], + + # action & state normalization + use_stepwise_action_norm: bool, + norm_default_mode: NormMode, + norm_exception_mode: Dict[str, Dict[str, NormMode]], + + action_state_merger, + + # image transform + train_transforms: Dict[str, List[Any]] | None, + val_transforms: Dict[str, List[Any]] | None, + + # instruction transform + drop_high_level_prob: float, + use_zh_instruction: bool, + + tokenizer: Any + ): + self.shape_meta = shape_meta + self.num_obs_steps = num_obs_steps + self.num_output_cameras = num_output_cameras + self.action_output_dim = action_output_dim + self.proprio_output_dim = proprio_output_dim + + self.drop_high_level_prob = drop_high_level_prob + self.use_zh_instruction = use_zh_instruction + + # image + self.train_transforms = train_transforms + self.val_transforms = val_transforms + + self._is_train = None + + self.action_state_transforms = action_state_transforms + self.action_state_merger = action_state_merger + self.action_state_merger.set_shape_meta(self.shape_meta) + + self.use_stepwise_action_norm = use_stepwise_action_norm + self.norm_default_mode = norm_default_mode + self.norm_exception_mode = norm_exception_mode + self._normalizer = None + + self.tokenizer = tokenizer + + @property + def is_train(self): + if self._is_train is None: + raise ValueError("is_train has not been set. Please call train() and eval() first.") + return self._is_train + + @property + def normalizer(self) -> LinearNormalizer: + if self._normalizer is None: + raise ValueError("normalizer has not been set. Please call set_normalizer_from_stats() first.") + return self._normalizer + + def train(self): + self._is_train = True + return self + + def eval(self): + self._is_train = False + return self + + def set_normalizer_from_stats(self, dataset_stats: Dict[str, Any] = None): + self._normalizer = LinearNormalizer( + use_stepwise_action_norm=self.use_stepwise_action_norm, + shape_meta=self.shape_meta, + default_mode=self.norm_default_mode, + exception_mode=self.norm_exception_mode, + stats=dataset_stats, + ) + + def augment_instruction(self, data: Dict[str, str] | List[str]) -> List[str]: + """ + Args: + data: Dict[str, str] | List[str], lerobot sample in raw mcap + + Returns: + List[str], processed instructions + """ + # if single instruction, convert to list + if "coarse_task" in data: + high_level_instruction = data["coarse_task"] + else: + high_level_instruction = "" + if "task" not in data: + return f"[high] {high_level_instruction}" + + low_level_instruction = data["task"] + # Galaxea lerobot use @ to split Chinese and English instruction + if "@" in low_level_instruction: + zh, eng = low_level_instruction.split("@") + low_level_instruction = zh if self.use_zh_instruction else eng + + if np.random.rand() < self.drop_high_level_prob: + instruction = f"[Low]: {low_level_instruction}" + else: + instruction = f"[High]: {high_level_instruction}, [Low]: {low_level_instruction}" + + return instruction + + def action_state_transform(self, batch): + if "action" in batch: + for meta in self.shape_meta["action"]: + k, meta_shape = meta["key"], meta["raw_shape"] + actual_shape = batch["action"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"Action key {k} actual raw shape {actual_shape} mismatch with meta raw shape {meta_shape}." + + for meta in self.shape_meta["state"]: + k, meta_shape = meta["key"], meta["raw_shape"] + actual_shape = batch["state"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"State key {k} actual raw shape {actual_shape} mismatch with meta raw shape{meta_shape}." + + if self.action_state_transforms is not None: + for trans in self.action_state_transforms: + batch = trans.forward(batch) + + if "action" in batch: + for meta in self.shape_meta["action"]: + k, meta_shape = meta["key"], meta["shape"] + actual_shape = batch["action"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"Action key {k} actual transformed shape {actual_shape} mismatch with meta shape {meta_shape}." + + for meta in self.shape_meta["state"]: + k, meta_shape = meta["key"], meta["shape"] + actual_shape = batch["state"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"State key {k} actual transformed shape {actual_shape} mismatch with meta raw shape {meta_shape}." + + return batch + + def preprocess(self, data: Dict[str, Any]) -> Dict[str, Any]: + """ + Preprocess the data for the policy model. + + Args: + Data: Dict[str, Any], lerobot sample in raw mcap obtained from dataset __getitem__: + - "action": Optional, Dict[str, torch.Tensor] -> [action_horizon, action_dim] + - "state": Dict[str, torch.Tensor] -> [num_obs_steps, state_dim] + - "images": Dict[str, torch.Tensor] -> [num_obs_steps, C, H, W] + - "action_is_pad": Optional, torch.Tensor -> [action_horizon,] + - "state_is_pad": torch.Tensor -> [num_obs_steps,] + - "image_is_pad": torch.Tensor -> [num_obs_steps,] + - "idx": int, sample index + + Returns: + Sample: Dict[str, Any], which can collated: + - "input_ids": torch.Tensor -> [max_image_text_tokens,] + - "attention_mask": torch.Tensor -> [max_image_text_tokens,] + - "pixel_values": torch.Tensor -> [num_input_cameras, C, H, W] + - "image_is_pad": torch.Tensor -> [num_obs_steps,] + - "proprio": torch.Tensor -> [num_obs_steps, proprio_dim] + - "state_is_pad": torch.Tensor -> [num_obs_steps,] + - "action": Optional, torch.Tensor -> [action_horizon, action_dim] + - "action_is_pad": Optional, torch.Tensor -> [action_horizon,] + - "gt_action: Optional, deepcopy of input action for open loop eval, which is left untouched + - "idx": int, sample index + """ + sample = {} + # 1. instruction + sample["instruction"] = self.augment_instruction(data) + sample["image_is_pad"] = data["image_is_pad"] + + # 2. image + processed_images = [] + for meta in self.shape_meta["images"]: + key, shape = meta["key"], meta["shape"] + image = data["images"][key] # [num_obs_steps, C, H, W] + assert image.ndim == 4, f"Expected 4 dimensions (num_obs_steps, C, H, W), got shape {image.shape}" + + # Apply transforms efficiently on the merged batch + transforms = self.train_transforms if self.is_train else self.val_transforms + for trans in transforms[key]: + image = trans(image) + + meta_shape = [self.num_obs_steps] + shape + assert image.shape == meta_shape, \ + f"Expected shape {meta_shape}, got {image.shape} after transforms for key {key}" + + processed_images.append(image) + + pixel_values = torch.cat(processed_images, dim=0) # [num_input_cameras, C, H, W] + if self.num_output_cameras > pixel_values.shape[0]: + out = torch.zeros((self.num_output_cameras,) + pixel_values.shape[1:], device=pixel_values.device, dtype=pixel_values.dtype) + out[0: pixel_values.shape[0]] = pixel_values + sample["pixel_values"] = out + else: + sample["pixel_values"] = pixel_values + + # Copy action before transform for open-loop evaluation, + # disabled for training dataset as it may cause collating key problem. + if not self.is_train and "action" in data: + sample["gt_action"] = deepcopy(data["action"]) + + # 3. action & state + data = self.action_state_transform(data) + data = self.normalizer.forward(data) + data = self.action_state_merger.forward(data) + + if "action" in data: + sample["action"] = data["action"] # [action_horizon, action_dim] + sample["action_is_pad"] = data["action_is_pad"] # [action_horizon,] + sample["action_dim_is_pad"] = data["action_dim_is_pad"] # [action_dim,] + assert sample["action"].shape[-1] == self.action_output_dim + + # TODO: rename all "state" into "proprio" + sample["proprio"] = data["state"] # [num_obs_steps, proprio_dim] + sample["proprio_is_pad"] = data["state_is_pad"] # [num_obs_steps,] + sample["proprio_dim_is_pad"] = data["state_dim_is_pad"] # [proprio_dim,] + assert sample["proprio"].shape[-1] == self.proprio_output_dim + + sample["idx"] = data["idx"] + + sample = self.tokenizer(sample) + + return sample + + def postprocess(self, data: Dict[str, Any]) -> Dict[str, Any]: + """ + Postprocess the data for the policy model. + + Args: + data: Dict[str, Any], lerobot sample in raw mcap + + Returns: + data: Dict[str, Any], processed data including unnormalized action + """ + assert "action" in data, "Action is required in postprocess" + data["state"] = data.pop("proprio") + data = self.action_state_merger.backward(data) + data = self.normalizer.backward(data) + if self.action_state_transforms is not None: + for trans in reversed(self.action_state_transforms): + data = trans.backward(data) + + start_obs_step = self.num_obs_steps - 1 + data["action"] = dict_apply(data["action"], lambda x: x[:, start_obs_step:, :]) + return data diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/processors/fastwam_processor.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/processors/fastwam_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..1a021b8df30dac093bdc987f4a25121d95bba8e3 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/processors/fastwam_processor.py @@ -0,0 +1,306 @@ +from abc import ABC, abstractmethod +from typing import Dict, Any, Optional, List, Literal + +import torch +import numpy as np +from copy import deepcopy +from ..utils.normalizer import LinearNormalizer, NormMode +from fastwam.utils.pytorch_utils import dict_apply +from fastwam.utils.logging_config import get_logger +from .base_processor import BaseProcessor + +logger = get_logger(__name__) + +class FastWAMProcessor(BaseProcessor): + def __init__( + self, + # keys + shape_meta: Dict[str, Any], + num_obs_steps: int, + num_output_cameras: int, + action_output_dim: int, + proprio_output_dim: int, + + action_state_transforms: Optional[List[Any]], + + # action & state normalization + use_stepwise_action_norm: bool, + norm_default_mode: NormMode, + norm_exception_mode: Dict[str, Dict[str, NormMode]], + + action_state_merger, + + # image transform + train_transforms: Dict[str, List[Any]] | None, + val_transforms: Dict[str, List[Any]] | None, + + # instruction transform + drop_high_level_prob: float = 1.0, + use_zh_instruction: bool = False, + + tokenizer: Optional[Any] = None, + delta_action_dim_mask: Optional[Dict[str, List[bool]]] = None, + num_image_steps: Optional[int] = None, + ): + self.shape_meta = shape_meta + self.num_obs_steps = num_obs_steps + self.num_image_steps = num_obs_steps if num_image_steps is None else num_image_steps + self.num_output_cameras = num_output_cameras + self.action_output_dim = action_output_dim + self.proprio_output_dim = proprio_output_dim + + self.drop_high_level_prob = drop_high_level_prob + self.use_zh_instruction = use_zh_instruction + + # image + self.train_transforms = train_transforms + self.val_transforms = val_transforms + + self._is_train = None + + self.action_state_transforms = action_state_transforms + self.action_state_merger = action_state_merger + self.action_state_merger.set_shape_meta(self.shape_meta) + + self.use_stepwise_action_norm = use_stepwise_action_norm + self.norm_default_mode = norm_default_mode + self.norm_exception_mode = norm_exception_mode + self._normalizer = None + + self.tokenizer = tokenizer + if delta_action_dim_mask is None: + self.delta_action_dim_mask = None + else: + action_meta = self.shape_meta["action"] + expected_keys = [m["key"] for m in action_meta] + provided_keys = list(delta_action_dim_mask.keys()) + if set(provided_keys) != set(expected_keys): + raise ValueError( + f"`delta_action_dim_mask` keys mismatch. Expected {expected_keys}, got {provided_keys}." + ) + + self.delta_action_dim_mask = {} + for meta in action_meta: + key = meta["key"] + expected_dim = meta["shape"] + mask = delta_action_dim_mask[key] + if len(mask) != expected_dim: + raise ValueError( + f"`delta_action_dim_mask[{key}]` length must be {expected_dim}, got {len(mask)}." + ) + self.delta_action_dim_mask[key] = torch.as_tensor(mask, dtype=torch.bool) + + @property + def is_train(self): + if self._is_train is None: + raise ValueError("is_train has not been set. Please call train() and eval() first.") + return self._is_train + + @property + def normalizer(self) -> LinearNormalizer: + if self._normalizer is None: + raise ValueError("normalizer has not been set. Please call set_normalizer_from_stats() first.") + return self._normalizer + + def train(self): + self._is_train = True + return self + + def eval(self): + self._is_train = False + return self + + def set_normalizer_from_stats(self, dataset_stats: Dict[str, Any] = None): + self._normalizer = LinearNormalizer( + use_stepwise_action_norm=self.use_stepwise_action_norm, + shape_meta=self.shape_meta, + default_mode=self.norm_default_mode, + exception_mode=self.norm_exception_mode, + stats=dataset_stats, + ) + + def augment_instruction(self, data: Dict[str, str] | List[str]) -> List[str]: + """ + Args: + data: Dict[str, str] | List[str], lerobot sample in raw mcap + + Returns: + List[str], processed instructions + """ + # if single instruction, convert to list + if "coarse_task" in data: + high_level_instruction = data["coarse_task"] + else: + high_level_instruction = "" + if "task" not in data: + return f"[high] {high_level_instruction}" + + low_level_instruction = data["task"] + # Galaxea lerobot use @ to split Chinese and English instruction + if "@" in low_level_instruction: + zh, eng = low_level_instruction.split("@") + low_level_instruction = zh if self.use_zh_instruction else eng + + if np.random.rand() < self.drop_high_level_prob: + instruction = f"{low_level_instruction}" + else: + instruction = f"[High]: {high_level_instruction}, [Low]: {low_level_instruction}" + + return instruction + + def action_state_transform(self, batch): + if "action" in batch: + for meta in self.shape_meta["action"]: + k, meta_shape = meta["key"], meta["raw_shape"] + actual_shape = batch["action"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"Action key {k} actual raw shape {actual_shape} mismatch with meta raw shape {meta_shape}." + + for meta in self.shape_meta["state"]: + k, meta_shape = meta["key"], meta["raw_shape"] + actual_shape = batch["state"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"State key {k} actual raw shape {actual_shape} mismatch with meta raw shape {meta_shape}." + + if self.action_state_transforms is not None: + for trans in self.action_state_transforms: + batch = trans.forward(batch) + + if "action" in batch: + for meta in self.shape_meta["action"]: + k, meta_shape = meta["key"], meta["shape"] + actual_shape = batch["action"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"Action key {k} actual transformed shape {actual_shape} mismatch with meta shape {meta_shape}." + + for meta in self.shape_meta["state"]: + k, meta_shape = meta["key"], meta["shape"] + actual_shape = batch["state"][k].shape[-1] + assert actual_shape == meta_shape, \ + f"State key {k} actual transformed shape {actual_shape} mismatch with meta raw shape {meta_shape}." + + return batch + + def preprocess(self, data: Dict[str, Any]) -> Dict[str, Any]: + """ + Preprocess the data for the policy model. + + Args: + Data: Dict[str, Any], lerobot sample in raw mcap obtained from dataset __getitem__: + - "action": Optional, Dict[str, torch.Tensor] -> [action_horizon, action_dim] + - "state": Dict[str, torch.Tensor] -> [num_obs_steps, state_dim] + - "images": Dict[str, torch.Tensor] -> [num_obs_steps, C, H, W] + - "action_is_pad": Optional, torch.Tensor -> [action_horizon,] + - "state_is_pad": torch.Tensor -> [num_obs_steps,] + - "image_is_pad": torch.Tensor -> [num_obs_steps,] + - "idx": int, sample index + + Returns: + Sample: Dict[str, Any], which can collated: + - "input_ids": torch.Tensor -> [max_image_text_tokens,] + - "attention_mask": torch.Tensor -> [max_image_text_tokens,] + - "pixel_values": torch.Tensor -> [num_input_cameras, C, H, W] + - "image_is_pad": torch.Tensor -> [num_obs_steps,] + - "proprio": torch.Tensor -> [num_obs_steps, proprio_dim] + - "state_is_pad": torch.Tensor -> [num_obs_steps,] + - "action": Optional, torch.Tensor -> [action_horizon, action_dim] + - "action_is_pad": Optional, torch.Tensor -> [action_horizon,] + - "gt_action: Optional, deepcopy of input action for open loop eval, which is left untouched + - "idx": int, sample index + """ + sample = {} + # 1. instruction + sample["instruction"] = self.augment_instruction(data) + sample["image_is_pad"] = data["image_is_pad"] + + # 2. image + processed_images = [] + for meta in self.shape_meta["images"]: + key, shape = meta["key"], meta["shape"] + image = data["images"][key] # [num_obs_steps, C, H, W] + assert image.ndim == 4, f"Expected 4 dimensions (num_obs_steps, C, H, W), got shape {image.shape}" + + # Apply transforms efficiently on the merged batch + transforms = self.train_transforms if self.is_train else self.val_transforms + current_transforms = transforms[key] if isinstance(transforms, dict) else transforms + for trans in current_transforms: + image = trans(image) + + meta_shape = [self.num_image_steps] + shape + assert image.shape == meta_shape, \ + f"Expected shape {meta_shape}, got {image.shape} after transforms for key {key}" + + processed_images.append(image) + pixel_values = torch.stack(processed_images, dim=0) # [num_input_cameras, T, C, H, W] + + if self.num_output_cameras > pixel_values.shape[0]: + out = torch.zeros((self.num_output_cameras,) + pixel_values.shape[1:], device=pixel_values.device, dtype=pixel_values.dtype) + out[0: pixel_values.shape[0]] = pixel_values + sample["pixel_values"] = out + elif self.num_output_cameras < pixel_values.shape[0]: + logger.warning(f"num_output_cameras {self.num_output_cameras} is less than the number of cameras in data {pixel_values.shape[0]}, " + f"truncating the input to the first {self.num_output_cameras} cameras.") + sample["pixel_values"] = pixel_values[:self.num_output_cameras] + else: + sample["pixel_values"] = pixel_values + + # Copy action before transform for open-loop evaluation, + # disabled for training dataset as it may cause collating key problem. + if not self.is_train and "action" in data: + sample["gt_action"] = deepcopy(data["action"]) + + # 3. action & state + if "action" in data and self.delta_action_dim_mask is not None: + action_is_pad = torch.as_tensor(data["action_is_pad"], dtype=torch.bool) + if bool(action_is_pad.any().item()): + for key, dim_mask in self.delta_action_dim_mask.items(): + cur_action = data["action"][key] + cur_action_is_pad = action_is_pad.to(device=cur_action.device) + cur_dim_mask = dim_mask.to(device=cur_action.device) + pad_delta_mask = cur_action_is_pad.unsqueeze(1) & cur_dim_mask.unsqueeze(0) + cur_action[pad_delta_mask] = 0.0 + data = self.action_state_transform(data) + data = self.normalizer.forward(data) + data = self.action_state_merger.forward(data) + + if "action" in data: + sample["action"] = data["action"] # [action_horizon, action_dim] + sample["action_is_pad"] = data["action_is_pad"] # [action_horizon,] + sample["action_dim_is_pad"] = data["action_dim_is_pad"] # [action_dim,] + assert sample["action"].shape[-1] == self.action_output_dim + # sample["action"][sample["action_is_pad"], :-1] = 0.0 # NOTE: we assume use delta_eef_pose + gripper, so pad action is 0 + + + # TODO: rename all "state" into "proprio" + sample["proprio"] = data["state"] # [num_obs_steps, proprio_dim] + sample["proprio_is_pad"] = data["state_is_pad"] # [num_obs_steps,] + sample["proprio_dim_is_pad"] = data["state_dim_is_pad"] # [proprio_dim,] + assert sample["proprio"].shape[-1] == self.proprio_output_dim + + sample["idx"] = data["idx"] + + # sample = self.tokenizer(sample) + + return sample + + def postprocess(self, data: Dict[str, Any]) -> Dict[str, Any]: + """ + Postprocess the data for the policy model. + + Args: + data: Dict[str, Any], lerobot sample in raw mcap + + Returns: + data: Dict[str, Any], processed data including unnormalized action + """ + assert "action" in data, "Action is required in postprocess" + data["state"] = data.pop("proprio") + data = self.action_state_merger.backward(data) + data = self.normalizer.backward(data) + if self.action_state_transforms is not None: + for trans in reversed(self.action_state_transforms): + data = trans.backward(data) + + start_obs_step = self.num_obs_steps - 1 + data["action"] = dict_apply(data["action"], lambda x: x[:, start_obs_step:, :]) + return data diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/robot_video_dataset.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/robot_video_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..23da576217a1c5413de0714eca084968768299d2 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/robot_video_dataset.py @@ -0,0 +1,292 @@ +import hashlib +import os +from typing import Optional +import time +import numpy as np +import traceback +import torch +import torchvision.transforms.functional as transforms_F +from contextlib import contextmanager + +from omegaconf import DictConfig, OmegaConf + +from hydra.utils import instantiate +from .base_lerobot_dataset import BaseLerobotDataset +from .utils.normalizer import save_dataset_stats_to_json, load_dataset_stats_from_json +from ..dataset_utils import ResizeSmallestSideAspectPreserving, CenterCrop, Normalize +from fastwam.utils.logging_config import get_logger +from fastwam.utils import misc, pytorch_utils +from accelerate import PartialState +logger = get_logger(__name__) + + +DEFAULT_PROMPT = "A video recorded from a robot's point of view executing the following instruction: {task}" + +class RobotVideoDataset(torch.utils.data.Dataset): + base_dataset_cls = BaseLerobotDataset + + def __init__( + self, + dataset_dirs, + shape_meta, + num_frames=33, + video_size=[384, 640], + camera_key=None, + processor=None, + text_embedding_cache_dir=None, + use_text_embed_cache=True, + context_len=128, + pretrained_norm_stats=None, + val_set_proportion=0.05, + is_training_set=False, + global_sample_stride=1, + action_video_freq_ratio: int = 1, + skip_padding_as_possible: bool = False, + max_padding_retry: int = 3, + concat_multi_camera: str = "horizontal", # "horizontal", "vertical", "robotwin", or None + override_instruction: Optional[str] = None, # whether to hardcode a specific instruction for all samples, for debugging + tolerance_s: Optional[float] = None, + video_backend: Optional[str] = None, + ): + self.lerobot_dataset = self.base_dataset_cls( + dataset_dirs=dataset_dirs, + shape_meta=OmegaConf.to_container(shape_meta, resolve=True), + obs_size=num_frames, + action_size=num_frames - 1, + val_set_proportion=val_set_proportion, + is_training_set=is_training_set, + global_sample_stride=global_sample_stride, + image_subsample_stride=action_video_freq_ratio, + tolerance_s=tolerance_s, + video_backend=video_backend, + ) + + self.num_frames = num_frames + self.action_video_freq_ratio = action_video_freq_ratio + + assert (num_frames - 1) % self.action_video_freq_ratio == 0, \ + f"num_frames-1 must be divisible by action_video_freq_ratio, got {num_frames - 1} and {self.action_video_freq_ratio}" + assert ((num_frames - 1) // self.action_video_freq_ratio) % 4 == 0, \ + f"video frames must be divisible by 4 for tokenization, got {(num_frames - 1) // self.action_video_freq_ratio}" + self.video_sample_indices = list(range(0, num_frames, self.action_video_freq_ratio)) + + self.camera_key = camera_key + self.lerobot_dataset._set_return_images(True) + + self.video_size = video_size + self.text_embedding_cache_dir = text_embedding_cache_dir + self.use_text_embed_cache = bool(use_text_embed_cache) + self.context_len = context_len + self.skip_padding_as_possible = skip_padding_as_possible + self.max_padding_retry = max_padding_retry + self.concat_multi_camera = concat_multi_camera + self.override_instruction = override_instruction + + self.resize_transform = ResizeSmallestSideAspectPreserving( + args={"img_w": self.video_size[1], "img_h": self.video_size[0]}, + ) + self.crop_transform = CenterCrop( + args={"img_w": self.video_size[1], "img_h": self.video_size[0]}, + ) + self.normalize_transform = Normalize( + args={"mean": 0.5, "std": 0.5}, + ) + if processor is not None: + if isinstance(processor, DictConfig): + processor = instantiate(processor) + if self.lerobot_dataset.presample_images: + processor.num_image_steps = len(self.video_sample_indices) + if not pretrained_norm_stats: + if not is_training_set: + raise ValueError("pretrained_norm_stats must be provided for validation/test sets since we don't want to calculate stats on them.") + if PartialState().is_main_process: + logger.info("Calculating dataset stats for normalization...") + dataset_stats = self.lerobot_dataset.get_dataset_stats(processor) + work_dir = misc.get_work_dir() + save_dataset_stats_to_json(dataset_stats, os.path.join(work_dir, "dataset_stats.json")) + else: + dataset_stats = None + if torch.distributed.is_available() and torch.distributed.is_initialized(): + obj_list = [dataset_stats] + torch.distributed.broadcast_object_list(obj_list, src=0) + dataset_stats = obj_list[0] + else: + dataset_stats = load_dataset_stats_from_json(pretrained_norm_stats) + logger.info(f"Using dataset stats: {pretrained_norm_stats}") + if PartialState().is_main_process: + work_dir = misc.get_work_dir() + save_dataset_stats_to_json(dataset_stats, os.path.join(work_dir, "dataset_stats.json")) + + processor.set_normalizer_from_stats(dataset_stats) + self.lerobot_dataset.set_processor(processor) + + def __len__(self): + return len(self.lerobot_dataset) + + def _get(self, idx): + sample_idx = idx + sample = None + for attempt in range(self.max_padding_retry + 1): + sample = self.lerobot_dataset[sample_idx] + + if not self.skip_padding_as_possible: + break + + action_is_pad = sample["action_is_pad"] + image_is_pad = sample["image_is_pad"] + proprio_is_pad = sample["proprio_is_pad"] + has_pad = False + if bool(action_is_pad.any().item()): + has_pad = True + if bool(image_is_pad.any().item()): + has_pad = True + if bool(proprio_is_pad.any().item()): + has_pad = True + + if not has_pad or attempt >= self.max_padding_retry: + break + + sample_idx = np.random.randint(len(self.lerobot_dataset)) + + image_is_pad = sample["image_is_pad"] + + video = sample["pixel_values"] # [T, C, H, W] or [num_cameras, T, C, H, W] + num_cameras = 1 + if video.ndim == 5: + if not self.lerobot_dataset.presample_images: + video = video[:, self.video_sample_indices, :, :, :] # [num_cameras, T_video, C, H, W] + num_cameras, T_video, C, H, W = video.shape + else: + assert video.ndim == 4, f"Expected video to have shape [T, C, H, W], but got {video.shape}" + if not self.lerobot_dataset.presample_images: + video = video[self.video_sample_indices, :, :, :] # [T_video, C, H, W] + T_video, C, H, W = video.shape + if not self.lerobot_dataset.presample_images: + image_is_pad = image_is_pad[self.video_sample_indices] + + video = video.view(num_cameras, T_video, C, H, W) # [num_cameras, T_video, C, H, W] + if self.concat_multi_camera == "robotwin": + if num_cameras != 3: + raise ValueError( + f"`concat_multi_camera='robotwin'` requires exactly 3 cameras, got {num_cameras}" + ) + cam_top = transforms_F.resize( + video[0], + size=[256, 320], + interpolation=transforms_F.InterpolationMode.BILINEAR, + antialias=True, + ) # [T_video, C, 256, 320] + cam_left = transforms_F.resize( + video[1], + size=[128, 160], + interpolation=transforms_F.InterpolationMode.BILINEAR, + antialias=True, + ) # [T_video, C, 128, 160] + cam_right = transforms_F.resize( + video[2], + size=[128, 160], + interpolation=transforms_F.InterpolationMode.BILINEAR, + antialias=True, + ) # [T_video, C, 128, 160] + bottom = torch.cat([cam_left, cam_right], dim=-1) # [T_video, C, 128, 320] + video = torch.cat([cam_top, bottom], dim=-2) # [T_video, C, 384, 320] + elif num_cameras > 1: + if self.concat_multi_camera == "horizontal": + video = torch.cat([video[i] for i in range(num_cameras)], dim=-1) # [T_video, C, H, num_cameras*W] + elif self.concat_multi_camera == "vertical": + video = torch.cat([video[i] for i in range(num_cameras)], dim=-2) # [T_video, C, num_cameras*H, W] + else: + raise ValueError( + f"Invalid concat_multi_camera: {self.concat_multi_camera}. " + "Expected one of: horizontal, vertical, robotwin." + ) + else: + video = video.squeeze(0) # [T_video, C, H, W] + + # final resize and normalization + video = self.resize_transform(video) + video = self.crop_transform(video) + video = self.normalize_transform(video) # [T_video, C, H, W] + + video = video.permute(1, 0, 2, 3) # [C, T_video, H, W], range [-1, 1] + + # Proxy (from lerobot): + # action: [num_frames-1, action_dim] # start from t0, except the last frame + # proprio: [num_frames, proprio_dim] # start from t0 to the last frame, aligned with video frames + action = sample["action"] # [T-1, action_dim] + proprio = sample["proprio"][:-1, :] # [T-1, state_dim], to align with action + if video.shape[1] <= 1: + raise ValueError(f"`video` must have at least 2 frames, got shape {tuple(video.shape)}") + if action.shape[0] % (video.shape[1] - 1) != 0: + raise ValueError( + f"`action` horizon must be divisible by `video` transitions, got {action.shape[0]} and {video.shape[1] - 1}" + ) + + task = sample["instruction"] + + # FIXME + if self.override_instruction is not None: + task = self.override_instruction + instruction = DEFAULT_PROMPT.format(task=task) + + data = { + "video": video, + "action": action, + "proprio": proprio, + "prompt": instruction, + "image_is_pad": image_is_pad, + "action_is_pad": sample["action_is_pad"], + "proprio_is_pad": sample["proprio_is_pad"], + } + if self.use_text_embed_cache: + context, context_mask = self._get_cached_text_context(instruction) + # NOTE: to keep consistent with wan2.2's behavior + context[~context_mask] = 0.0 + data["context"] = context + data["context_mask"] = torch.ones_like(context_mask) + return data + + def _get_cached_text_context(self, prompt: str): + if self.text_embedding_cache_dir is None: + raise ValueError("text_embedding_cache_dir is not set.") + cache_dir = self.text_embedding_cache_dir + os.makedirs(cache_dir, exist_ok=True) + hashed = hashlib.sha256(prompt.encode("utf-8")).hexdigest() + cache_path = os.path.join(cache_dir, f"{hashed}.t5_len{self.context_len}.wan22ti2v5b.pt") + if not os.path.exists(cache_path): + raise FileNotFoundError( + f"Missing text embedding cache: {cache_path}. " + "Run scripts/precompute_text_embeds.py first." + ) + payload = torch.load(cache_path, map_location="cpu") + context = payload["context"] + context_mask = payload["mask"].bool() + if context.ndim != 2: + raise ValueError( + f"Cached `context` must be 2D [L, D], got shape {tuple(context.shape)} in {cache_path}" + ) + if context_mask.ndim != 1: + raise ValueError( + f"Cached `mask` must be 1D [L], got shape {tuple(context_mask.shape)} in {cache_path}" + ) + if context.shape[0] != self.context_len: + raise ValueError( + f"Cached context_len mismatch: expected {self.context_len}, got {context.shape[0]} in {cache_path}" + ) + if context_mask.shape[0] != self.context_len: + raise ValueError( + f"Cached mask_len mismatch: expected {self.context_len}, got {context_mask.shape[0]} in {cache_path}" + ) + + return context, context_mask + + def __getitem__(self, idx): + try: + data = self._get(idx) + except Exception as e: + print(f"Error processing sample idx {idx}: {e}. Returning a random sample instead.") + # trace back + print(traceback.format_exc()) + random_idx = np.random.randint(len(self)) + data = self._get(random_idx) + return data diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/action_state_merger.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/action_state_merger.py new file mode 100644 index 0000000000000000000000000000000000000000..0a35644a4fbc65a5cb6abe3f642f4af35a4e438b --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/action_state_merger.py @@ -0,0 +1,78 @@ +from typing import Dict + +import torch +from torch.nn.functional import pad + + +class ConcatLeftAlign: + def __init__( + self, + action_target_dim: int | None = None, + state_target_dim: int | None = None + ): + self.action_target_dim = action_target_dim + self.state_target_dim = state_target_dim + + def set_shape_meta(self, shape_meta): + self.action_meta = shape_meta["action"] + self.state_meta = shape_meta["state"] + + def forward(self, batch): + if "action" in batch: + batch["action"] = self._concat(batch["action"], self.action_meta) + batch["action"], batch["action_dim_is_pad"] = self._pad(batch["action"], self.action_target_dim) + + batch["state"] = self._concat(batch["state"], self.state_meta) + batch["state"], batch["state_dim_is_pad"] = self._pad(batch["state"], self.state_target_dim) + + return batch + + def backward(self, batch): + if self.state_target_dim is not None: + assert batch["state"].shape[-1] == self.state_target_dim + batch["state"] = self._crop(batch["state"], self.state_meta) + batch["state"] = self._split(batch["state"], self.state_meta) + + if self.action_target_dim is not None: + assert batch["action"].shape[-1] == self.action_target_dim + batch["action"] = self._crop(batch["action"], self.action_meta) + batch["action"] = self._split(batch["action"], self.action_meta) + + return batch + + @staticmethod + def _pad(x: torch.Tensor, dim: int): + if dim is None: + dim = x.shape[-1] + + assert x.ndim == 2 and x.shape[-1] <= dim + pad_dim = dim - x.shape[-1] + x_padded = pad(x, (0, pad_dim)) + mask = torch.zeros_like(x[0]).bool() + mask = pad(mask, (0, pad_dim), value=True) + return x_padded, mask + + @staticmethod + def _crop(x: torch.Tensor, meta: int): + assert x.ndim == 3 + dim = sum([m["shape"] for m in meta]) + x = x[:, :, :dim] + return x + + @staticmethod + def _concat(x: Dict[str, torch.Tensor], meta: Dict[str, Dict]): + x = torch.cat([x[m["key"]] for m in meta], dim=-1) + assert x.ndim == 2 + return x + + @staticmethod + def _split(x: torch.Tensor, meta: Dict[str, Dict]): + assert x.ndim == 3 + y = {} + idx = 0 + for m in meta: + key, dim = m["key"], m["shape"] + y[key] = x[:, :, idx: idx + dim] + idx += dim + + return y \ No newline at end of file diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/image.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/image.py new file mode 100644 index 0000000000000000000000000000000000000000..8dbeda01c09dd57bc772ad4d53cf8b49586a2d06 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/image.py @@ -0,0 +1,25 @@ +import torch +import torch.nn as nn +import torchvision.transforms as TF + + +class ToTensor(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x: torch.Tensor): + assert x.dtype == torch.uint8 + x = x.to(torch.float32) / 255.0 + return x + +class Pad(nn.Module): + def __init__(self, padding, fill=0, padding_mode='constant'): + super().__init__() + self.padding = padding + self.fill = fill + self.padding_mode = padding_mode + self.pad = TF.Pad(padding=tuple(padding), fill=fill, padding_mode=padding_mode) + + def forward(self, x: torch.Tensor): + assert x.ndim == 4, "Can only pad tensor of 4 dims." + return self.pad(x) diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/misc.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..49279d6888f565078aafbcc3a830584bb34515a5 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/misc.py @@ -0,0 +1,20 @@ +from typing import List + +import torch + + +class WrapStateAngle: + def __init__(self, keys: List[str]): + self.keys = keys + + @staticmethod + def _wrap(x): + return torch.atan2(torch.sin(x), torch.cos(x)) + + def forward(self, batch): + for k in self.keys: + batch["state"][k] = self._wrap(batch["state"][k]) + return batch + + def backward(self, batch): + return batch \ No newline at end of file diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/relative_action.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/relative_action.py new file mode 100644 index 0000000000000000000000000000000000000000..063c9b0a8dc5ccbc2a742c6d50f803dd055dfe15 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/relative_action.py @@ -0,0 +1,142 @@ +from typing import List, Dict + +import torch +from .rotation import quaternion_to_matrix, matrix_to_quaternion + +# pose: position and quaternion in x, y, z, i, j, k, r +# mat: homogeneous transformation matrix in 4×4 +# pos: position in x, y, z +# quat: quaternion in i, j, k, r + + +class RelativePoseTransform: + def __init__(self, keys: List[str]): + self.keys = keys + + def forward(self, batch: Dict): + # for close-loop eval, "action" may not be in batch + if "action" not in batch: + return batch + + for k in self.keys: + action = batch["action"][k] + state = batch["state"][k] + batch["action"][k] = self._forward(action, state[..., -1:, :]) + + return batch + + def backward(self, batch: Dict): + for k in self.keys: + action = batch["action"][k] + state = batch["state"][k] + batch["action"][k] = self._backward(action, state[..., -1:, :]) + + return batch + + def _forward(self, pose: torch.Tensor, base_pose: torch.Tensor): + # pose & base_pose: position and quaternion in x, y, z, i, j, k, r + assert pose.shape[-1] == 7, f"Pose shape must be (..., 7), but got {pose.shape}" + assert base_pose.shape[-1] == 7, f"Base pose shape must be (..., 7), but got {base_pose.shape}" + pose_matrix = self._pose_to_matrix(pose) + base_pose_matrix = self._pose_to_matrix(base_pose) + pose_matrix = self._absolute_to_relative(pose_matrix, base_pose_matrix) + pose = self._matrix_to_pose(pose_matrix) + return pose + + def _backward(self, pose: torch.Tensor, base_pose: torch.Tensor): + # pose & base_pose: position and quaternion in x, y, z, i, j, k, r + assert pose.shape[-1] == 7, f"Pose shape must be (..., 7), but got {pose.shape}" + assert base_pose.shape[-1] == 7, f"Base pose shape must be (..., 7), but got {base_pose.shape}" + pose_matrix = self._pose_to_matrix(pose) + base_pose_matrix = self._pose_to_matrix(base_pose) + pose_matrix = self._relative_to_absolute(pose_matrix, base_pose_matrix) + pose = self._matrix_to_pose(pose_matrix) + return pose + + @staticmethod + def _pose_to_matrix(pose: torch.Tensor): + position = pose[..., 0: 3] + quaternion = pose[..., [6, 3, 4, 5]] # (i j k r) to (r i j k) + rotation = quaternion_to_matrix(quaternion) + matrix = torch.zeros(pose.shape[:-1] + (4, 4), dtype=pose.dtype, device=pose.device) + matrix[..., 0: 3, 0: 3] = rotation + matrix[..., 0: 3, 3] = position + matrix[..., 3, 3] = 1 + return matrix + + @staticmethod + def _matrix_to_pose(matrix: torch.Tensor): + position = matrix[..., 0: 3, 3] / matrix[..., 3, 3][..., None] + rotation = matrix[..., 0: 3, 0: 3] + quaternion = matrix_to_quaternion(rotation) + quaternion = quaternion[..., [1, 2, 3, 0]] # (r i j k) to (i j k r) + pose = torch.cat([position, quaternion], dim=-1) + return pose + @staticmethod + def _absolute_to_relative(pose_matrix: torch.Tensor, base_pose_matrix: torch.Tensor): + return torch.linalg.inv(base_pose_matrix) @ pose_matrix + + @staticmethod + def _relative_to_absolute(pose_matrix: torch.Tensor, base_pose_matrix: torch.Tensor): + return base_pose_matrix @ pose_matrix + + +class RelativeJointTransform: + def __init__(self, keys: List[str]): + self.keys = keys + + def forward(self, batch: Dict): + # for close-loop eval, "action" may not be in batch + if "action" not in batch: + return batch + + for k in self.keys: + # NOTE: fixed to the first frame + batch["action"][k] = batch["action"][k] - batch["state"][k][..., :1, :] + + return batch + + def backward(self, batch: Dict): + for k in self.keys: + # NOTE: fixed to the first frame + batch["action"][k] = batch["action"][k] + batch["state"][k][..., :1, :] + + return batch + + +class MaskedRelativeJointTransform: + """Convert only selected absolute joint targets to first-state deltas.""" + + def __init__(self, masks: Dict[str, List[bool]]): + if not masks: + raise ValueError("masks must contain at least one action/state key") + self.masks = { + str(key): torch.as_tensor(mask, dtype=torch.bool) + for key, mask in masks.items() + } + if any(mask.ndim != 1 or not bool(mask.any()) for mask in self.masks.values()): + raise ValueError("each relative-joint mask must be a nonempty 1D mask") + + def _apply(self, batch: Dict, sign: float) -> Dict: + if "action" not in batch: + return batch + for key, cpu_mask in self.masks.items(): + if key not in batch["action"] or key not in batch["state"]: + raise KeyError(f"missing action/state key for relative transform: {key}") + action = batch["action"][key] + state = batch["state"][key] + if action.shape[-1] != state.shape[-1] or action.shape[-1] != cpu_mask.numel(): + raise ValueError( + f"relative-joint shape mismatch for {key}: action={action.shape}, " + f"state={state.shape}, mask={cpu_mask.numel()}" + ) + mask = cpu_mask.to(device=action.device) + reference = state[..., :1, :].to(device=action.device, dtype=action.dtype) + action[..., mask] = action[..., mask] + sign * reference[..., mask] + return batch + + def forward(self, batch: Dict) -> Dict: + return self._apply(batch, -1.0) + + def backward(self, batch: Dict) -> Dict: + return self._apply(batch, 1.0) diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/rotation.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/rotation.py new file mode 100644 index 0000000000000000000000000000000000000000..4f19924703f4274401f01537180aae11f9bc84c7 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/transforms/rotation.py @@ -0,0 +1,133 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Dict, List, Tuple +import torch + +from ..utils.rotation import ( + quaternion_to_matrix, + matrix_to_quaternion, + matrix_to_rotation_6d, + rotation_6d_to_matrix, + matrix_to_rotation_9d, + rotation_9d_to_matrix, + quaternion_to_axis_angle, + axis_angle_to_quaternion, + axis_angle_to_matrix, + matrix_to_axis_angle, + euler_angles_to_matrix, + matrix_to_euler_angles, +) + +class PoseRotationTransform: + def __init__(self, rotation_type: str, category_keys: Dict[str, List[str]]): + self.rotation_type = rotation_type + self.category_keys = category_keys + self.src_type, self.dst_type = self._parse_rotation_type(rotation_type) + + def forward(self, batch): + for cat, ks in self.category_keys.items(): + if cat == "action" and "action" not in batch: + continue + + for k in ks: + batch[cat][k] = self._forward(batch[cat][k]) + + return batch + + def backward(self, batch): + for cat, ks in self.category_keys.items(): + for k in ks: + batch[cat][k] = self._backward(batch[cat][k]) + + return batch + + + def _forward(self, pose): + assert pose.shape[-1] >= 6 + if self.src_type == self.dst_type: + return pose + position, rotation, others = self._split_pose(pose, self.src_type) + matrix = self._rotation_to_matrix(rotation, self.src_type) + rotation_out = self._matrix_to_rotation(matrix, self.dst_type) + return torch.cat([position, rotation_out, others], axis=-1) + + def _backward(self, pose: torch.Tensor): + if self.src_type == self.dst_type: + return pose + position, rotation, others = self._split_pose(pose, self.dst_type) + matrix = self._rotation_to_matrix(rotation, self.dst_type) + rotation_out = self._matrix_to_rotation(matrix, self.src_type) + return torch.cat([position, rotation_out, others], axis=-1) + + def add_noise(self, pose: torch.Tensor, std_position=0.05, std_angle=0.05): + position, rotation, others = self._split_pose(pose, self.src_type) + matrix = self._rotation_to_matrix(rotation, self.src_type) + axis_angles = matrix_to_axis_angle(matrix) + position = position + std_position * torch.randn_like(position) + axis_angles = axis_angles + std_angle * torch.randn_like(axis_angles) + matrix = axis_angle_to_matrix(axis_angles) + rotation_out = self._matrix_to_rotation(matrix, self.src_type) + return torch.cat([position, rotation_out, others], axis=-1) + + def _parse_rotation_type(self, rotation_type: str) -> Tuple[str, str]: + if "_to_" in rotation_type: + src, dst = rotation_type.split("_to_", 1) + else: + src, dst = "quaternion", rotation_type + return src, dst + + def _split_pose(self, pose: torch.Tensor, rotation_type: str) -> Tuple[torch.Tensor, torch.Tensor]: + rotation_dim = self._rotation_dim(rotation_type) + position = pose[..., 0:3] + rotation = pose[..., 3: 3 + rotation_dim] + others = pose[..., 3 + rotation_dim:] if pose.shape[-1] > 3 + rotation_dim else torch.empty_like(position[..., :0]) + assert rotation.shape[-1] == rotation_dim, f"Expected {rotation_dim} dims for {rotation_type}" + return position, rotation, others + + def _rotation_dim(self, rotation_type: str) -> int: + if rotation_type == "quaternion": + return 4 + if rotation_type == "axis_angle": + return 3 + if rotation_type in ("rotation_6d",): + return 6 + if rotation_type in ("rotation_9d", "matrix"): + return 9 + if rotation_type.startswith("euler_"): + return 3 + raise ValueError(f"Unsupported rotation type '{rotation_type}'") + + def _rotation_to_matrix(self, rotation: torch.Tensor, rotation_type: str) -> torch.Tensor: + if rotation_type == "quaternion": + quaternion = rotation[..., [3, 0, 1, 2]] + return quaternion_to_matrix(quaternion) + if rotation_type == "axis_angle": + return axis_angle_to_matrix(rotation) + if rotation_type == "rotation_6d": + return rotation_6d_to_matrix(rotation) + if rotation_type in ("rotation_9d", "matrix"): + return rotation_9d_to_matrix(rotation) + if rotation_type.startswith("euler_"): + convention = rotation_type.split("_", 1)[1].upper() + return euler_angles_to_matrix(rotation, convention) + raise ValueError(f"Unsupported rotation type '{rotation_type}'") + + def _matrix_to_rotation(self, matrix: torch.Tensor, rotation_type: str) -> torch.Tensor: + if rotation_type == "quaternion": + quaternion = matrix_to_quaternion(matrix) + return quaternion[..., [1, 2, 3, 0]] + if rotation_type == "axis_angle": + return matrix_to_axis_angle(matrix) + if rotation_type == "rotation_6d": + return matrix_to_rotation_6d(matrix) + if rotation_type in ("rotation_9d", "matrix"): + return matrix_to_rotation_9d(matrix) + if rotation_type.startswith("euler_"): + convention = rotation_type.split("_", 1)[1].upper() + return matrix_to_euler_angles(matrix, convention) + raise ValueError(f"Unsupported rotation type '{rotation_type}'") + diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/utils/normalizer.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/utils/normalizer.py new file mode 100644 index 0000000000000000000000000000000000000000..1c674b0234bcf3cf1a1bef0f8fc0523020449b4a --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/utils/normalizer.py @@ -0,0 +1,291 @@ +from typing import Literal, Dict, Annotated, Union, Any, List, Tuple, Optional +import torch +import json +from collections import defaultdict +import numpy as np +from omegaconf import DictConfig, OmegaConf +import hashlib +from pathlib import Path +from git import Repo +from fastwam.utils.logging_config import get_logger + +from fastwam.utils.pytorch_utils import dict_apply + +logger = get_logger(__name__) + +ConstConstStr = Annotated[str, "format: 'const_min/const_max', where const_min and const_max give the constant range"] +NormMode = Union[Literal["min/max", "q01/q99", "z-score"], ConstConstStr] + +class LinearNormalizer: + def __init__( + self, + shape_meta, + use_stepwise_action_norm, + default_mode: NormMode, + exception_mode: Dict[str, Dict[str, NormMode]], + stats: Dict[str, Dict[str, Dict[str, torch.Tensor]]] + ): + super().__init__() + self.normalizers = {"action": {}, "state": {}} + self.stats = stats + + for meta in shape_meta["action"]: + key = meta["key"] + + if use_stepwise_action_norm: + cur_stats = {k.removeprefix("stepwise_"): v for k, v in stats["action"][key].items() if k.startswith("stepwise_")} + else: + cur_stats = {k.removeprefix("global_"): v for k, v in stats["action"][key].items() if k.startswith("global_")} + + if exception_mode is not None and "action" in exception_mode and key in exception_mode["action"]: + cur_mode = exception_mode["action"][key] + else: + cur_mode = default_mode + + self.normalizers["action"][key] = SingleFieldLinearNormalizer( + stats=cur_stats, + mode=cur_mode, + ) + + for meta in shape_meta["state"]: + key = meta["key"] + cur_stats = {k.removeprefix("global_"): v for k, v in stats["state"][key].items() if k.startswith("global_")} + + if exception_mode is not None and "state" in exception_mode and key in exception_mode["state"]: + cur_mode = exception_mode["state"][key] + else: + cur_mode = default_mode + + self.normalizers["state"][key] = SingleFieldLinearNormalizer( + stats=cur_stats, + mode=cur_mode, + ) + + def get_stats(self): + stats = { + "action": {key: norm.get_stats() for key, norm in self.normalizers["action"].items()}, + "state": {key: norm.get_stats() for key, norm in self.normalizers["state"].items()} + } + return stats + + def forward(self, batch: Dict[str, Dict[str, torch.Tensor]]) -> torch.Tensor: + if "action" in batch: + for key, norm in self.normalizers["action"].items(): + batch["action"][key] = norm.forward(batch["action"][key]) + + for key, norm in self.normalizers["state"].items(): + batch["state"][key] = norm.forward(batch["state"][key]) + + return batch + + def backward(self, batch: Dict[str, Dict[str, torch.Tensor]]) -> torch.Tensor: + for key, norm in self.normalizers["action"].items(): + batch["action"][key] = norm.backward(batch["action"][key]) + + for key, norm in self.normalizers["state"].items(): + batch["state"][key] = norm.backward(batch["state"][key]) + + return batch + + +class SingleFieldLinearNormalizer: + std_reg = 1e-8 + range_tol = 1e-4 + output_max = 1.0 + output_min = -1.0 + def __init__(self, stats, mode: NormMode="min/max"): + self.stats = stats + self.mode = mode + + if mode == "z-score": + input_mean, input_std = stats["mean"], stats["std"] + scale = 1.0 / (input_std + self.std_reg) + offset = - input_mean / (input_std + self.std_reg) + else: + if mode == "min/max": + input_min, input_max = stats["min"], stats["max"] + elif mode == "q01/q99": + input_min, input_max = stats["q01"], stats["q99"] + else: + # parse const_min/const_max + input_min, input_max = map(float, mode.split("/")) + input_min = torch.full_like(stats["min"], input_min) + input_max = torch.full_like(stats["max"], input_max) + + input_range = input_max - input_min + ignore_dim = input_range < self.range_tol + input_range[ignore_dim] = self.output_max - self.output_min + scale = (self.output_max - self.output_min) / input_range + offset = self.output_min - scale * input_min + offset[ignore_dim] = (self.output_max + self.output_min) / 2 - input_min[ignore_dim] + + self.scale = scale + self.offset = offset + def get_stats(self): + return self.stats + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x * self.scale + self.offset + x = torch.clamp(x, -5.0, 5.0) + return x + def backward(self, x: torch.Tensor) -> torch.Tensor: + x = (x - self.offset) / self.scale + return x + +def save_dataset_stats_to_json(dataset_stats: dict, file_path: str): + + def convert_tensor(obj): + if isinstance(obj, torch.Tensor): + return obj.detach().cpu().numpy().tolist() + elif isinstance(obj, (defaultdict, dict)): + return {k: convert_tensor(v) for k, v in dict(obj).items()} + elif isinstance(obj, (list, tuple)): + return [convert_tensor(item) for item in obj] + elif isinstance(obj, (int, float, str, bool, type(None))): + return obj + else: + return str(obj) + + serializable_stats = convert_tensor(dataset_stats) + + with open(file_path, 'w', encoding='utf-8') as f: + json.dump(serializable_stats, f, ensure_ascii=False, indent=2) + +def load_dataset_stats_from_json(file_path: str, + try_convert_tensor: bool = True) -> Dict[str, Any]: + + def is_numeric_list(obj): + if isinstance(obj, list): + if not obj: + return True + first = obj[0] + if isinstance(first, (int, float)): + return all(isinstance(x, (int, float)) for x in obj) + elif isinstance(first, list): + return all(is_numeric_list(item) for item in obj) + else: + return False + return False + + def convert_back_to_tensor(obj): + if isinstance(obj, dict): + return {k: convert_back_to_tensor(v) for k, v in obj.items()} + elif isinstance(obj, list): + if is_numeric_list(obj): + try: + arr = np.array(obj) + return torch.from_numpy(arr) + except Exception: + return [convert_back_to_tensor(item) for item in obj] + else: + return [convert_back_to_tensor(item) for item in obj] + else: + return obj + + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + if try_convert_tensor: + data = convert_back_to_tensor(data) + + data = dict_apply( + data, + lambda x: x.to(torch.float32) if isinstance(x, torch.Tensor) else x, + ) + + return data + + +def search_dataset_stats_cache_json(cache_dir: str | Path, data_config: DictConfig) -> Tuple[bool, str | None]: + if isinstance(cache_dir, str): + cache_dir = Path(cache_dir) + cache_dir.mkdir(parents=True, exist_ok=True) + + def get_git_hash() -> Optional[str]: + repo = Repo(__file__, search_parent_directories=True) + return repo.head.commit.hexsha + + def to_plain(value: Any) -> Any: + if OmegaConf.is_config(value): + return OmegaConf.to_container(value, resolve=True) + return value + + def normalize_str_list(value: Any) -> List[str]: + value = to_plain(value) + if value is None: + return [] + if isinstance(value, str): + value = [value] + return [str(item) for item in value if item is not None] + + def normalize_transforms(value: Any) -> Any: + value = to_plain(value) + if isinstance(value, dict): + return [value] + return value + + def normalize_dataset_dirs(cfg: DictConfig) -> Any: + dataset_cfg = cfg.get("dataset") + if dataset_cfg is None: + return None + embodiment_datasets = dataset_cfg.get("embodiment_datasets") + if embodiment_datasets is not None: + emb_dirs: Dict[str, List[str]] = {} + for emb, emb_cfg in embodiment_datasets.items(): + dataset_groups = emb_cfg.get("dataset_groups") + if dataset_groups is None: + emb_dirs[emb] = [] + continue + dirs: List[str] = [] + for group in dataset_groups: + group_dirs = group.get("dataset_dirs") + if group_dirs is None: + continue + dirs.extend(normalize_str_list(group_dirs)) + emb_dirs[emb] = sorted(dirs) + return emb_dirs + + dataset_dirs = dataset_cfg.get("dataset_dirs") + return sorted(normalize_str_list(dataset_dirs)) + + def normalize_action_state_transforms(cfg: DictConfig) -> Any: + processor_cfg = cfg.get("processor") + if processor_cfg is None: + return None + embodiment_processors = processor_cfg.get("embodiment_processors") + if embodiment_processors is not None: + emb_transforms: Dict[str, Any] = {} + for emb, emb_cfg in embodiment_processors.items(): + transforms = emb_cfg.get("action_state_transforms") + emb_transforms[emb] = normalize_transforms(transforms) + return emb_transforms + + transforms = processor_cfg.get("action_state_transforms") + return normalize_transforms(transforms) + + signature = { + "action_size": data_config.dataset.action_size, + "dataset_dirs": normalize_dataset_dirs(data_config), + "action_state_transforms": normalize_action_state_transforms(data_config), + } + signature_json = json.dumps(signature, sort_keys=True, separators=(",", ":")) + dataset_hash = hashlib.sha256(signature_json.encode("utf-8"), usedforsecurity=False).hexdigest() + + git_hash = get_git_hash() + precise_name = f"dataset_stats_{dataset_hash}_{git_hash}.json" + precise = cache_dir / precise_name + if precise.exists(): + logger.info(f"Found dataset stats cache with precisely matching dataset and git hash: {precise_name}.") + return True, str(precise) + + candidates = sorted(cache_dir.glob(f"dataset_stats_{dataset_hash}_*.json")) + if not candidates: + logger.info(f"No dataset stats cache found for dataset hash {dataset_hash}") + return False, str(precise) # return precise cache path for saving cache + + picked = candidates[0] + prefix = f"dataset_stats_{dataset_hash}_" + picked_git_hash = picked.name[len(prefix):-5] + assert picked_git_hash != git_hash + logger.warning(f"Found substitute dataset stats cache {picked.name} which mismatch current git hash {git_hash}.") + return True, str(picked) diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/utils/rotation.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/utils/rotation.py new file mode 100644 index 0000000000000000000000000000000000000000..f468cd4f24092bfe8a601a85e2cb8e5e6b84cbbb --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot/utils/rotation.py @@ -0,0 +1,621 @@ + +from typing import Optional, Union, Literal, List +import torch +import torch.nn.functional as F + +Device = Union[str, torch.device] + + +# [Important] Below is adapted from pytorch3d, which uses (r, i, j, k) quat order + +""" +The transformation matrices returned from the functions in this file assume +the points on which the transformation will be applied are column vectors. +i.e. the R matrix is structured as + + R = [ + [Rxx, Rxy, Rxz], + [Ryx, Ryy, Ryz], + [Rzx, Rzy, Rzz], + ] # (3, 3) + +This matrix can be applied to column vectors by post multiplication +by the points e.g. + + points = [[0], [1], [2]] # (3 x 1) xyz coordinates of a point + transformed_points = R * points + +To apply the same matrix to points which are row vectors, the R matrix +can be transposed and pre multiplied by the points: + +e.g. + points = [[0, 1, 2]] # (1 x 3) xyz coordinates of a point + transformed_points = points * R.transpose(1, 0) +""" + + +def quaternion_to_matrix(quaternions: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as quaternions to rotation matrices. + + Args: + quaternions: quaternions with real part first, + as tensor of shape (..., 4). + + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + r, i, j, k = torch.unbind(quaternions, -1) + # pyre-fixme[58]: `/` is not supported for operand types `float` and `Tensor`. + two_s = 2.0 / (quaternions * quaternions).sum(-1) + + o = torch.stack( + ( + 1 - two_s * (j * j + k * k), + two_s * (i * j - k * r), + two_s * (i * k + j * r), + two_s * (i * j + k * r), + 1 - two_s * (i * i + k * k), + two_s * (j * k - i * r), + two_s * (i * k - j * r), + two_s * (j * k + i * r), + 1 - two_s * (i * i + j * j), + ), + -1, + ) + return o.reshape(quaternions.shape[:-1] + (3, 3)) + + +def _copysign(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + """ + Return a tensor where each element has the absolute value taken from the, + corresponding element of a, with sign taken from the corresponding + element of b. This is like the standard copysign floating-point operation, + but is not careful about negative 0 and NaN. + + Args: + a: source tensor. + b: tensor whose signs will be used, of the same shape as a. + + Returns: + Tensor of the same shape as a with the signs of b. + """ + signs_differ = (a < 0) != (b < 0) + return torch.where(signs_differ, -a, a) + +@torch.jit.script +def _sqrt_positive_part(x: torch.Tensor) -> torch.Tensor: + """ + Returns torch.sqrt(torch.max(0, x)) + but with a zero subgradient where x is 0. + """ + ret = torch.zeros_like(x) + positive_mask = x > 0 + ret[positive_mask] = torch.sqrt(x[positive_mask]) + return ret + + +def matrix_to_quaternion(matrix: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as rotation matrices to quaternions. + + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + + Returns: + quaternions with real part first, as tensor of shape (..., 4). + """ + if matrix.size(-1) != 3 or matrix.size(-2) != 3: + raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.") + + batch_dim = matrix.shape[:-2] + m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.unbind( + matrix.reshape(batch_dim + (9,)), dim=-1 + ) + + q_abs = _sqrt_positive_part( + torch.stack( + [ + 1.0 + m00 + m11 + m22, + 1.0 + m00 - m11 - m22, + 1.0 - m00 + m11 - m22, + 1.0 - m00 - m11 + m22, + ], + dim=-1, + ) + ) + + # we produce the desired quaternion multiplied by each of r, i, j, k + quat_by_rijk = torch.stack( + [ + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + torch.stack([q_abs[..., 0] ** 2, m21 - m12, m02 - m20, m10 - m01], dim=-1), + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + torch.stack([m21 - m12, q_abs[..., 1] ** 2, m10 + m01, m02 + m20], dim=-1), + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + torch.stack([m02 - m20, m10 + m01, q_abs[..., 2] ** 2, m12 + m21], dim=-1), + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + torch.stack([m10 - m01, m20 + m02, m21 + m12, q_abs[..., 3] ** 2], dim=-1), + ], + dim=-2, + ) + + # We floor here at 0.1 but the exact level is not important; if q_abs is small, + # the candidate won't be picked. + flr = torch.tensor(0.1).to(dtype=q_abs.dtype, device=q_abs.device) + quat_candidates = quat_by_rijk / (2.0 * q_abs[..., None].max(flr)) + + # if not for numerical problems, quat_candidates[i] should be same (up to a sign), + # forall i; we pick the best-conditioned one (with the largest denominator) + out = quat_candidates[ + F.one_hot(q_abs.argmax(dim=-1), num_classes=4) > 0.5, : + ].reshape(batch_dim + (4,)) + return standardize_quaternion(out) + + +def _axis_angle_rotation(axis: str, angle: torch.Tensor) -> torch.Tensor: + """ + Return the rotation matrices for one of the rotations about an axis + of which Euler angles describe, for each value of the angle given. + + Args: + axis: Axis label "X" or "Y or "Z". + angle: any shape tensor of Euler angles in radians + + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + + cos = torch.cos(angle) + sin = torch.sin(angle) + one = torch.ones_like(angle) + zero = torch.zeros_like(angle) + + if axis == "X": + R_flat = (one, zero, zero, zero, cos, -sin, zero, sin, cos) + elif axis == "Y": + R_flat = (cos, zero, sin, zero, one, zero, -sin, zero, cos) + elif axis == "Z": + R_flat = (cos, -sin, zero, sin, cos, zero, zero, zero, one) + else: + raise ValueError("letter must be either X, Y or Z.") + + return torch.stack(R_flat, -1).reshape(angle.shape + (3, 3)) + + +def euler_angles_to_matrix(euler_angles: torch.Tensor, convention: str) -> torch.Tensor: + """ + Convert rotations given as Euler angles in radians to rotation matrices. + + Args: + euler_angles: Euler angles in radians as tensor of shape (..., 3). + convention: Convention string of three uppercase letters from + {"X", "Y", and "Z"}. + + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + if euler_angles.dim() == 0 or euler_angles.shape[-1] != 3: + raise ValueError("Invalid input euler angles.") + if len(convention) != 3: + raise ValueError("Convention must have 3 letters.") + if convention[1] in (convention[0], convention[2]): + raise ValueError(f"Invalid convention {convention}.") + for letter in convention: + if letter not in ("X", "Y", "Z"): + raise ValueError(f"Invalid letter {letter} in convention string.") + matrices = [ + _axis_angle_rotation(c, e) + for c, e in zip(convention, torch.unbind(euler_angles, -1)) + ] + # return functools.reduce(torch.matmul, matrices) + return torch.matmul(torch.matmul(matrices[0], matrices[1]), matrices[2]) + + +def _angle_from_tan( + axis: str, other_axis: str, data, horizontal: bool, tait_bryan: bool +) -> torch.Tensor: + """ + Extract the first or third Euler angle from the two members of + the matrix which are positive constant times its sine and cosine. + + Args: + axis: Axis label "X" or "Y or "Z" for the angle we are finding. + other_axis: Axis label "X" or "Y or "Z" for the middle axis in the + convention. + data: Rotation matrices as tensor of shape (..., 3, 3). + horizontal: Whether we are looking for the angle for the third axis, + which means the relevant entries are in the same row of the + rotation matrix. If not, they are in the same column. + tait_bryan: Whether the first and third axes in the convention differ. + + Returns: + Euler Angles in radians for each matrix in data as a tensor + of shape (...). + """ + + i1, i2 = {"X": (2, 1), "Y": (0, 2), "Z": (1, 0)}[axis] + if horizontal: + i2, i1 = i1, i2 + even = (axis + other_axis) in ["XY", "YZ", "ZX"] + if horizontal == even: + return torch.atan2(data[..., i1], data[..., i2]) + if tait_bryan: + return torch.atan2(-data[..., i2], data[..., i1]) + return torch.atan2(data[..., i2], -data[..., i1]) + + +def _index_from_letter(letter: str) -> int: + if letter == "X": + return 0 + if letter == "Y": + return 1 + if letter == "Z": + return 2 + raise ValueError("letter must be either X, Y or Z.") + + +def matrix_to_euler_angles(matrix: torch.Tensor, convention: str) -> torch.Tensor: + """ + Convert rotations given as rotation matrices to Euler angles in radians. + + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + convention: Convention string of three uppercase letters. + + Returns: + Euler angles in radians as tensor of shape (..., 3). + """ + if len(convention) != 3: + raise ValueError("Convention must have 3 letters.") + if convention[1] in (convention[0], convention[2]): + raise ValueError(f"Invalid convention {convention}.") + for letter in convention: + if letter not in ("X", "Y", "Z"): + raise ValueError(f"Invalid letter {letter} in convention string.") + if matrix.size(-1) != 3 or matrix.size(-2) != 3: + raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.") + i0 = _index_from_letter(convention[0]) + i2 = _index_from_letter(convention[2]) + tait_bryan = i0 != i2 + if tait_bryan: + central_angle = torch.asin( + matrix[..., i0, i2] * (-1.0 if i0 - i2 in [-1, 2] else 1.0) + ) + else: + central_angle = torch.acos(matrix[..., i0, i0]) + + o = ( + _angle_from_tan( + convention[0], convention[1], matrix[..., i2], False, tait_bryan + ), + central_angle, + _angle_from_tan( + convention[2], convention[1], matrix[..., i0, :], True, tait_bryan + ), + ) + return torch.stack(o, -1) + + +def random_quaternions( + n: int, dtype: Optional[torch.dtype] = None, device: Optional[Device] = None +) -> torch.Tensor: + """ + Generate random quaternions representing rotations, + i.e. versors with nonnegative real part. + + Args: + n: Number of quaternions in a batch to return. + dtype: Type to return. + device: Desired device of returned tensor. Default: + uses the current device for the default tensor type. + + Returns: + Quaternions as tensor of shape (N, 4). + """ + if isinstance(device, str): + device = torch.device(device) + o = torch.randn((n, 4), dtype=dtype, device=device) + s = (o * o).sum(1) + o = o / _copysign(torch.sqrt(s), o[:, 0])[:, None] + return o + + +def random_rotations( + n: int, dtype: Optional[torch.dtype] = None, device: Optional[Device] = None +) -> torch.Tensor: + """ + Generate random rotations as 3x3 rotation matrices. + + Args: + n: Number of rotation matrices in a batch to return. + dtype: Type to return. + device: Device of returned tensor. Default: if None, + uses the current device for the default tensor type. + + Returns: + Rotation matrices as tensor of shape (n, 3, 3). + """ + quaternions = random_quaternions(n, dtype=dtype, device=device) + return quaternion_to_matrix(quaternions) + + +def random_rotation( + dtype: Optional[torch.dtype] = None, device: Optional[Device] = None +) -> torch.Tensor: + """ + Generate a single random 3x3 rotation matrix. + + Args: + dtype: Type to return + device: Device of returned tensor. Default: if None, + uses the current device for the default tensor type + + Returns: + Rotation matrix as tensor of shape (3, 3). + """ + return random_rotations(1, dtype, device)[0] + + +def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor: + """ + Convert a unit quaternion to a standard form: one in which the real + part is non negative. + + Args: + quaternions: Quaternions with real part first, + as tensor of shape (..., 4). + + Returns: + Standardized quaternions as tensor of shape (..., 4). + """ + return torch.where(quaternions[..., 0:1] < 0, -quaternions, quaternions) + + +def quaternion_raw_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + """ + Multiply two quaternions. + Usual torch rules for broadcasting apply. + + Args: + a: Quaternions as tensor of shape (..., 4), real part first. + b: Quaternions as tensor of shape (..., 4), real part first. + + Returns: + The product of a and b, a tensor of quaternions shape (..., 4). + """ + aw, ax, ay, az = torch.unbind(a, -1) + bw, bx, by, bz = torch.unbind(b, -1) + ow = aw * bw - ax * bx - ay * by - az * bz + ox = aw * bx + ax * bw + ay * bz - az * by + oy = aw * by - ax * bz + ay * bw + az * bx + oz = aw * bz + ax * by - ay * bx + az * bw + return torch.stack((ow, ox, oy, oz), -1) + + +def quaternion_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + """ + Multiply two quaternions representing rotations, returning the quaternion + representing their composition, i.e. the versor with nonnegative real part. + Usual torch rules for broadcasting apply. + + Args: + a: Quaternions as tensor of shape (..., 4), real part first. + b: Quaternions as tensor of shape (..., 4), real part first. + + Returns: + The product of a and b, a tensor of quaternions of shape (..., 4). + """ + ab = quaternion_raw_multiply(a, b) + return standardize_quaternion(ab) + + +def quaternion_invert(quaternion: torch.Tensor) -> torch.Tensor: + """ + Given a quaternion representing rotation, get the quaternion representing + its inverse. + + Args: + quaternion: Quaternions as tensor of shape (..., 4), with real part + first, which must be versors (unit quaternions). + + Returns: + The inverse, a tensor of quaternions of shape (..., 4). + """ + + scaling = torch.tensor([1, -1, -1, -1], device=quaternion.device) + return quaternion * scaling + + +def quaternion_apply(quaternion: torch.Tensor, point: torch.Tensor) -> torch.Tensor: + """ + Apply the rotation given by a quaternion to a 3D point. + Usual torch rules for broadcasting apply. + + Args: + quaternion: Tensor of quaternions, real part first, of shape (..., 4). + point: Tensor of 3D points of shape (..., 3). + + Returns: + Tensor of rotated points of shape (..., 3). + """ + if point.size(-1) != 3: + raise ValueError(f"Points are not in 3D, {point.shape}.") + real_parts = point.new_zeros(point.shape[:-1] + (1,)) + point_as_quaternion = torch.cat((real_parts, point), -1) + out = quaternion_raw_multiply( + quaternion_raw_multiply(quaternion, point_as_quaternion), + quaternion_invert(quaternion), + ) + return out[..., 1:] + + +def axis_angle_to_matrix(axis_angle: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as axis/angle to rotation matrices. + + Args: + axis_angle: Rotations given as a vector in axis angle form, + as a tensor of shape (..., 3), where the magnitude is + the angle turned anticlockwise in radians around the + vector's direction. + + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + return quaternion_to_matrix(axis_angle_to_quaternion(axis_angle)) + + +def matrix_to_axis_angle(matrix: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as rotation matrices to axis/angle. + + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + + Returns: + Rotations given as a vector in axis angle form, as a tensor + of shape (..., 3), where the magnitude is the angle + turned anticlockwise in radians around the vector's + direction. + """ + return quaternion_to_axis_angle(matrix_to_quaternion(matrix)) + + +def axis_angle_to_quaternion(axis_angle: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as axis/angle to quaternions. + + Args: + axis_angle: Rotations given as a vector in axis angle form, + as a tensor of shape (..., 3), where the magnitude is + the angle turned anticlockwise in radians around the + vector's direction. + + Returns: + quaternions with real part first, as tensor of shape (..., 4). + """ + angles = torch.norm(axis_angle, p=2, dim=-1, keepdim=True) + half_angles = angles * 0.5 + eps = 1e-6 + small_angles = angles.abs() < eps + sin_half_angles_over_angles = torch.empty_like(angles) + sin_half_angles_over_angles[~small_angles] = ( + torch.sin(half_angles[~small_angles]) / angles[~small_angles] + ) + # for x small, sin(x/2) is about x/2 - (x/2)^3/6 + # so sin(x/2)/x is about 1/2 - (x*x)/48 + sin_half_angles_over_angles[small_angles] = ( + 0.5 - (angles[small_angles] * angles[small_angles]) / 48 + ) + quaternions = torch.cat( + [torch.cos(half_angles), axis_angle * sin_half_angles_over_angles], dim=-1 + ) + return quaternions + + +def quaternion_to_axis_angle(quaternions: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as quaternions to axis/angle. + + Args: + quaternions: quaternions with real part first, + as tensor of shape (..., 4). + + Returns: + Rotations given as a vector in axis angle form, as a tensor + of shape (..., 3), where the magnitude is the angle + turned anticlockwise in radians around the vector's + direction. + """ + norms = torch.norm(quaternions[..., 1:], p=2, dim=-1, keepdim=True) + half_angles = torch.atan2(norms, quaternions[..., :1]) + angles = 2 * half_angles + eps = 1e-6 + small_angles = angles.abs() < eps + sin_half_angles_over_angles = torch.empty_like(angles) + sin_half_angles_over_angles[~small_angles] = ( + torch.sin(half_angles[~small_angles]) / angles[~small_angles] + ) + # for x small, sin(x/2) is about x/2 - (x/2)^3/6 + # so sin(x/2)/x is about 1/2 - (x*x)/48 + sin_half_angles_over_angles[small_angles] = ( + 0.5 - (angles[small_angles] * angles[small_angles]) / 48 + ) + return quaternions[..., 1:] / sin_half_angles_over_angles + + +def rotation_6d_to_matrix(d6: torch.Tensor) -> torch.Tensor: + """ + Converts 6D rotation representation by Zhou et al. [1] to rotation matrix + using Gram--Schmidt orthogonalization per Section B of [1]. + Args: + d6: 6D rotation representation, of size (*, 6) + + Returns: + batch of rotation matrices of size (*, 3, 3) + + [1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H. + On the Continuity of Rotation Representations in Neural Networks. + IEEE Conference on Computer Vision and Pattern Recognition, 2019. + Retrieved from http://arxiv.org/abs/1812.07035 + """ + + a1, a2 = d6[..., :3], d6[..., 3:] + b1 = F.normalize(a1, dim=-1) + b2 = a2 - (b1 * a2).sum(-1, keepdim=True) * b1 + b2 = F.normalize(b2, dim=-1) + b3 = torch.cross(b1, b2, dim=-1) + return torch.stack((b1, b2, b3), dim=-2) + + +def matrix_to_rotation_6d(matrix: torch.Tensor) -> torch.Tensor: + """ + Converts rotation matrices to 6D rotation representation by Zhou et al. [1] + by dropping the last row. Note that 6D representation is not unique. + Args: + matrix: batch of rotation matrices of size (*, 3, 3) + + Returns: + 6D rotation representation, of size (*, 6) + + [1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H. + On the Continuity of Rotation Representations in Neural Networks. + IEEE Conference on Computer Vision and Pattern Recognition, 2019. + Retrieved from http://arxiv.org/abs/1812.07035 + """ + batch_dim = matrix.size()[:-2] + return matrix[..., :2, :].clone().reshape(batch_dim + (6,)) + + +def rotation_9d_to_matrix(d9: torch.Tensor) -> torch.Tensor: + """" + Args: + d9: 9D rotaion representation of size (..., 9) + + Returns: + batch of rotation matrices of size (..., 3, 3) + """ + assert d9.shape[-1] == 9 + matrices = d9.reshape(-1, 3, 3).permute(0, 2, 1) + u, s, vt = torch.linalg.svd(matrices) # torch.svd returns v instead of vt + det = torch.det(torch.matmul(u, vt)) + vt[:, :, -1] *= det[:, None] + rot_matrices = torch.matmul(u, vt) + rot_matrices = rot_matrices.unflatten(0, d9.shape[:-1]) + return rot_matrices + + +def matrix_to_rotation_9d(matrix: torch.Tensor) -> torch.Tensor: + """" + Args: + matrix: batch of rotation matrices of size (..., 3, 3) + Returns: + 9D rotaion representation of size (..., 9) + """ + assert matrix.shape[-2:] == (3, 3) + return matrix.transpose(-2, -1).flatten(-2, -1) \ No newline at end of file diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot3/__init__.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot3/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5c25ebc2e060822140df41633f0a1b13575fe503 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot3/__init__.py @@ -0,0 +1,13 @@ +"""Read-only LeRobot v3.0 dataset integration.""" + +from .base_lerobot_dataset import BaseLerobotDataset +from .lerobot_dataset import LeRobotDataset, LeRobotDatasetMetadata, MultiLeRobotDataset +from .robot_video_dataset import RobotVideoDataset + +__all__ = [ + "BaseLerobotDataset", + "LeRobotDataset", + "LeRobotDatasetMetadata", + "MultiLeRobotDataset", + "RobotVideoDataset", +] diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot3/base_lerobot_dataset.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot3/base_lerobot_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..ae8b29037fb2a12c1ac02f7a40290c0fe89ca3d2 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot3/base_lerobot_dataset.py @@ -0,0 +1,11 @@ +"""FastWAM adapter for the LeRobot v3.0 reader.""" + +from fastwam.datasets.lerobot.base_lerobot_dataset import BaseLerobotDataset as _BaseLerobotDataset + +from .lerobot_dataset import LeRobotDatasetMetadata, MultiLeRobotDataset + + +class BaseLerobotDataset(_BaseLerobotDataset): + metadata_cls = LeRobotDatasetMetadata + multi_dataset_cls = MultiLeRobotDataset + presample_images = True diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot3/lerobot_dataset.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot3/lerobot_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..df05e2f4583159c3b484a12e054b1316fca6613b --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot3/lerobot_dataset.py @@ -0,0 +1,401 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# Licensed under the Apache License, Version 2.0. + +"""Minimal read-only reader for the LeRobot v3.0 file layout. + +LeRobot v3.0 stores many episodes in chunked parquet and video files. This +module intentionally exposes only the local-dataset read path needed by +FastWAM training. Dataset creation, Hub upload, conversion, and mixture logic +are out of scope. +""" + +import json +from itertools import accumulate +from pathlib import Path +from typing import Callable + +import datasets +import numpy as np +import packaging.version +import pandas as pd +import pyarrow.dataset as pa_dataset +import torch + +from fastwam.datasets.lerobot.constants import HF_LEROBOT_HOME +from fastwam.datasets.lerobot.lerobot.datasets.utils import ( + get_hf_features_from_features, + hf_transform_to_torch, +) +from fastwam.datasets.lerobot.lerobot.datasets.video_utils import ( + VideoFrame, + decode_video_frames, + get_safe_default_codec, +) + +CODEBASE_VERSION = packaging.version.parse("3.0") + + +def _load_json(path: Path): + with path.open() as f: + return json.load(f) + + +def _load_parquet_dataset(path: Path) -> datasets.Dataset: + paths = sorted(path.glob("*/*.parquet")) + if not paths: + raise FileNotFoundError(f"No parquet files found under {path}.") + return datasets.Dataset.from_parquet([str(item) for item in paths]) + + +def _episode_data_index(episode_rows: datasets.Dataset, episodes: list[int]): + lengths = [int(episode_rows[episode_idx]["length"]) for episode_idx in episodes] + cumulative = list(accumulate(lengths)) + return { + "from": torch.LongTensor([0] + cumulative[:-1]), + "to": torch.LongTensor(cumulative), + } + + +class LeRobotDatasetMetadata: + def __init__( + self, + repo_id: str, + root: str | Path | None = None, + ): + self.repo_id = repo_id + self.root = Path(root) if root is not None else HF_LEROBOT_HOME / repo_id + self.info = _load_json(self.root / "meta" / "info.json") + for feature in self.info["features"].values(): + feature["shape"] = tuple(feature["shape"]) + version = packaging.version.parse(self.info["codebase_version"]) + if version != CODEBASE_VERSION: + raise ValueError(f"Expected LeRobot codebase_version v3.0, got {version} at {self.root}.") + + self.tasks = pd.read_parquet(self.root / "meta" / "tasks.parquet") + episodes = _load_parquet_dataset(self.root / "meta" / "episodes") + columns = [name for name in episodes.column_names if not name.startswith("stats/")] + self.episodes = episodes.select_columns(columns) + stats_path = self.root / "meta" / "stats.json" + self.stats = _load_json(stats_path) if stats_path.exists() else None + + @property + def _version(self) -> packaging.version.Version: + return packaging.version.parse(self.info["codebase_version"]) + + def get_data_file_path(self, episode_index: int) -> Path: + episode = self.episodes[int(episode_index)] + return Path( + self.info["data_path"].format( + chunk_index=episode["data/chunk_index"], + file_index=episode["data/file_index"], + ) + ) + + def get_video_file_path(self, episode_index: int, video_key: str) -> Path: + episode = self.episodes[int(episode_index)] + return Path( + self.info["video_path"].format( + video_key=video_key, + chunk_index=episode[f"videos/{video_key}/chunk_index"], + file_index=episode[f"videos/{video_key}/file_index"], + ) + ) + + @property + def fps(self) -> int: + return int(self.info["fps"]) + + @property + def features(self) -> dict[str, dict]: + return self.info["features"] + + @property + def image_keys(self) -> list[str]: + return [key for key, feature in self.features.items() if feature["dtype"] == "image"] + + @property + def video_keys(self) -> list[str]: + return [key for key, feature in self.features.items() if feature["dtype"] == "video"] + + @property + def camera_keys(self) -> list[str]: + return [ + key for key, feature in self.features.items() if feature["dtype"] in {"image", "video"} + ] + + @property + def total_episodes(self) -> int: + return int(self.info["total_episodes"]) + + @property + def total_frames(self) -> int: + return int(self.info["total_frames"]) + + +class LeRobotDataset(torch.utils.data.Dataset): + def __init__( + self, + repo_id: str, + root: str | Path | None = None, + episodes: list[int] | None = None, + image_transforms: Callable | None = None, + delta_timestamps: dict[str, list[float]] | None = None, + tolerance_s: float = 1e-4, + video_backend: str | None = None, + ): + super().__init__() + self.repo_id = repo_id + self.root = Path(root) if root is not None else HF_LEROBOT_HOME / repo_id + self.image_transforms = image_transforms + self.delta_timestamps = delta_timestamps + self.tolerance_s = tolerance_s + self.video_backend = video_backend or get_safe_default_codec() + self.during_training = True + self.meta = LeRobotDatasetMetadata(repo_id, root=self.root) + + selected_episodes = list(range(self.meta.total_episodes)) if episodes is None else episodes + self.episodes = sorted(int(episode_idx) for episode_idx in selected_episodes) + self.hf_dataset = self._load_hf_dataset() + self.episode_data_index = _episode_data_index(self.meta.episodes, self.episodes) + self._episode_id_to_nested_id = { + episode_id: nested_id for nested_id, episode_id in enumerate(self.episodes) + } + self.delta_indices = self._get_delta_indices(delta_timestamps) if delta_timestamps else None + + def _load_hf_dataset(self) -> datasets.Dataset: + paths = sorted((self.root / "data").glob("*/*.parquet")) + if not paths: + raise FileNotFoundError(f"No parquet files found under {self.root / 'data'}.") + features = get_hf_features_from_features(self.meta.features) + filter_expression = pa_dataset.field("episode_index").isin(self.episodes) + dataset = datasets.Dataset.from_parquet( + [str(path) for path in paths], + features=features, + columns=list(features), + filters=filter_expression, + ) + dataset.set_transform(hf_transform_to_torch) + return dataset + + def _get_delta_indices(self, delta_timestamps: dict[str, list[float]]): + return { + key: [round(timestamp * self.fps) for timestamp in timestamps] + for key, timestamps in delta_timestamps.items() + } + + @property + def fps(self) -> int: + return self.meta.fps + + @property + def num_frames(self) -> int: + return len(self.hf_dataset) + + @property + def num_episodes(self) -> int: + return len(self.episodes) + + @property + def features(self) -> dict[str, dict]: + return self.meta.features + + @property + def hf_features(self) -> datasets.Features: + return self.hf_dataset.features + + def _get_query_indices(self, idx: int, episode_index: int): + nested_id = self._episode_id_to_nested_id[episode_index] + episode_start = int(self.episode_data_index["from"][nested_id]) + episode_end = int(self.episode_data_index["to"][nested_id]) + query_indices = { + key: [max(episode_start, min(episode_end - 1, idx + delta)) for delta in deltas] + for key, deltas in self.delta_indices.items() + } + padding = { + f"{key}_is_pad": torch.BoolTensor( + [(idx + delta < episode_start) or (idx + delta >= episode_end) for delta in deltas] + ) + for key, deltas in self.delta_indices.items() + } + return query_indices, padding + + def _query_data(self, query_indices: dict[str, list[int]]): + result = {} + selected_cache = {} + for key, indices in query_indices.items(): + if key in self.meta.video_keys: + continue + cache_key = tuple(indices) + if cache_key not in selected_cache: + selected_cache[cache_key] = self.hf_dataset.select(indices) + selected = selected_cache[cache_key] + result[key] = torch.stack(list(selected[key])) + return result + + def _query_timestamps(self, current_timestamp: float, query_indices): + timestamps = {} + for key in self.meta.video_keys: + if query_indices is not None and key in query_indices: + values = self.hf_dataset.select(query_indices[key])["timestamp"] + timestamps[key] = torch.stack(list(values)).tolist() + else: + timestamps[key] = [current_timestamp] + return timestamps + + def _query_videos(self, query_timestamps, episode_index: int): + episode = self.meta.episodes[episode_index] + result = {} + for video_key, timestamps in query_timestamps.items(): + start_timestamp = episode[f"videos/{video_key}/from_timestamp"] + shifted = [start_timestamp + timestamp for timestamp in timestamps] + video_path = self.root / self.meta.get_video_file_path(episode_index, video_key) + result[video_key] = decode_video_frames( + video_path, + shifted, + self.tolerance_s, + self.video_backend, + ).squeeze(0) + return result + + def get_episode_data(self, nested_episode_index: int): + start = int(self.episode_data_index["from"][nested_episode_index]) + end = int(self.episode_data_index["to"][nested_episode_index]) + selected = self.hf_dataset.select(range(start, end)) + result = {} + for key in self.meta.features: + if key in self.meta.video_keys: + continue + values = list(selected[key]) + if values and isinstance(values[0], torch.Tensor): + result[key] = torch.stack(values) + else: + result[key] = np.asarray(values) + return result + + def __len__(self): + return self.num_frames + + def __getitem__(self, idx: int): + item = self.hf_dataset[idx] + episode_index = int(item["episode_index"]) + query_indices = None + if self.delta_indices is not None: + query_indices, padding = self._get_query_indices(idx, episode_index) + item = {**item, **padding, **self._query_data(query_indices)} + + if self.meta.video_keys and self.during_training: + timestamps = self._query_timestamps(float(item["timestamp"]), query_indices) + item = {**self._query_videos(timestamps, episode_index), **item} + + if self.image_transforms is not None: + for camera_key in self.meta.camera_keys: + item[camera_key] = self.image_transforms(item[camera_key]) + + task_index = item["task_index"] + if task_index.ndim == 0: + task_index = int(task_index) + else: + item["chunked_task_index"] = task_index + task_index = int(task_index[0]) + item["task"] = self.meta.tasks.iloc[task_index].name + item["step_is_qualified"] = True + return item + + +class MultiLeRobotDataset(torch.utils.data.Dataset): + def __init__( + self, + dataset_dirs: list[str], + episodes: dict[str, list[int]] | None = None, + image_transforms: Callable | None = None, + delta_timestamps: dict[str, list[float]] | None = None, + tolerances_s: dict[str, float] | None = None, + video_backend: str | None = None, + ): + super().__init__() + self.dataset_dirs = dataset_dirs + self.ds_names = list(dataset_dirs) + tolerances_s = tolerances_s or dict.fromkeys(dataset_dirs, 1e-4) + self._datasets = [ + LeRobotDataset( + dataset_dir, + root=Path(dataset_dir), + episodes=episodes[dataset_dir] if episodes else None, + image_transforms=image_transforms, + delta_timestamps=delta_timestamps, + tolerance_s=tolerances_s[dataset_dir], + video_backend=video_backend, + ) + for dataset_dir in dataset_dirs + ] + fps_values = {dataset.fps for dataset in self._datasets} + if len(fps_values) != 1: + raise ValueError(f"All LeRobot datasets must share one FPS, got {sorted(fps_values)}.") + + common_features = set(self._datasets[0].features) + for dataset in self._datasets[1:]: + common_features.intersection_update(dataset.features) + if not common_features: + raise ValueError("LeRobot datasets have no common features.") + self.disabled_features = set().union( + *(set(dataset.features) - common_features for dataset in self._datasets) + ) + + def set_during_training(self, during_training: bool): + for dataset in self._datasets: + dataset.during_training = during_training + + @property + def fps(self) -> int: + return self._datasets[0].fps + + @property + def num_frames(self) -> int: + return sum(dataset.num_frames for dataset in self._datasets) + + @property + def num_episodes(self) -> int: + return sum(dataset.num_episodes for dataset in self._datasets) + + @property + def video(self) -> bool: + return bool(self._datasets[0].meta.video_keys) + + @property + def features(self): + return { + key: feature + for dataset in self._datasets + for key, feature in dataset.hf_features.items() + if key not in self.disabled_features + } + + @property + def camera_keys(self) -> list[str]: + return [ + key + for key, feature in self.features.items() + if isinstance(feature, (datasets.Image, VideoFrame)) + ] + + def get_episode_data(self, episode_index: int): + for dataset in self._datasets: + if episode_index < dataset.num_episodes: + return dataset.get_episode_data(episode_index) + episode_index -= dataset.num_episodes + raise IndexError("Episode index out of bounds.") + + def __len__(self): + return self.num_frames + + def __getitem__(self, idx: int): + offset = 0 + for dataset_index, dataset in enumerate(self._datasets): + if idx < offset + dataset.num_frames: + item = dataset[idx - offset] + item["dataset_index"] = torch.tensor(dataset_index) + for key in self.disabled_features: + item.pop(key, None) + return item + offset += dataset.num_frames + raise IndexError(f"Index {idx} out of bounds for {self.num_frames} frames.") diff --git a/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot3/robot_video_dataset.py b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot3/robot_video_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..0261b906c58b8923a48ae8679844adc45bc74ab5 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/datasets/lerobot3/robot_video_dataset.py @@ -0,0 +1,9 @@ +"""FastWAM robot-video dataset backed by the LeRobot v3.0 reader.""" + +from fastwam.datasets.lerobot.robot_video_dataset import RobotVideoDataset as _RobotVideoDataset + +from .base_lerobot_dataset import BaseLerobotDataset + + +class RobotVideoDataset(_RobotVideoDataset): + base_dataset_cls = BaseLerobotDataset diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/__init__.py b/baseline/fastwam_zipper/source/src/fastwam/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f45b7d25cd81883bbc40b37fd6366225d1e84105 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/__init__.py @@ -0,0 +1 @@ +"""Model namespace for fastwam.""" diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/__init__.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/action_dit.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/action_dit.py new file mode 100644 index 0000000000000000000000000000000000000000..09ab921233e2c6032d5e860b9de41307df93e50c --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/action_dit.py @@ -0,0 +1,362 @@ +import os +import torch +import torch.nn as nn +from typing import Any, Dict, Optional + +from fastwam.utils.logging_config import get_logger + +from .wan_video_dit import ( + DiTBlock, + sinusoidal_embedding_1d, + precompute_freqs_cis, +) + +logger = get_logger(__name__) + + +class ActionHead(nn.Module): + def __init__(self, hidden_dim: int, out_dim: int, eps: float): + super().__init__() + self.norm = nn.LayerNorm(hidden_dim, eps=eps, elementwise_affine=False) + self.proj = nn.Linear(hidden_dim, out_dim) + self.modulation = nn.Parameter(torch.randn(1, 2, hidden_dim) / hidden_dim**0.5) + + def forward(self, x: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + shift, scale = (self.modulation.to(dtype=t.dtype, device=t.device) + t.unsqueeze(1)).chunk(2, dim=1) + shift = shift.squeeze(1) + scale = scale.squeeze(1) + return self.proj(self.norm(x) * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)) + + +class ActionDiT(nn.Module): + ACTION_BACKBONE_SKIP_PREFIXES = ("action_encoder.", "head.") + ACTION_BACKBONE_META_KEYS = ( + "hidden_dim", + "ffn_dim", + "num_layers", + "num_heads", + "attn_head_dim", + "text_dim", + "freq_dim", + "eps", + ) + + def __init__( + self, + hidden_dim: int, + action_dim: int, + ffn_dim: int, + text_dim: int, + freq_dim: int, + eps: float, + num_heads: int, + attn_head_dim: int, + num_layers: int, + use_gradient_checkpointing: bool = False, + ): + super().__init__() + self.hidden_dim = hidden_dim + self.action_dim = action_dim + self.ffn_dim = ffn_dim + self.text_dim = text_dim + self.freq_dim = freq_dim + self.num_heads = num_heads + self.attn_head_dim = attn_head_dim + + if num_heads <= 0: + raise ValueError(f"`num_heads` must be > 0, got {num_heads}") + if attn_head_dim <= 0: + raise ValueError(f"`attn_head_dim` must be > 0, got {attn_head_dim}") + if attn_head_dim % 2 != 0: + raise ValueError(f"`attn_head_dim` must be even for RoPE, got {attn_head_dim}") + + self.action_encoder = nn.Linear(action_dim, hidden_dim) + self.text_embedding = nn.Sequential( + nn.Linear(text_dim, hidden_dim), + nn.GELU(approximate="tanh"), + nn.Linear(hidden_dim, hidden_dim), + ) + self.time_embedding = nn.Sequential( + nn.Linear(freq_dim, hidden_dim), + nn.SiLU(), + nn.Linear(hidden_dim, hidden_dim), + ) + self.time_projection = nn.Sequential(nn.SiLU(), nn.Linear(hidden_dim, hidden_dim * 6)) + self.blocks = nn.ModuleList( + [ + DiTBlock( + hidden_dim=hidden_dim, + attn_head_dim=attn_head_dim, + num_heads=num_heads, + ffn_dim=ffn_dim, + eps=eps, + ) + for _ in range(num_layers) + ] + ) + self.head = nn.Linear(hidden_dim, action_dim) + self.freqs = precompute_freqs_cis(attn_head_dim, end=1024) + + self.use_gradient_checkpointing = use_gradient_checkpointing + + def _apply(self, fn): + result = super()._apply(fn) + device = next(self.parameters()).device + self.freqs = self.freqs.to(device=device) + return result + + def get_freqs(self, seq_len: int) -> torch.Tensor: + return self.freqs[:seq_len].view(seq_len, 1, -1) + + @classmethod + def backbone_key_set(cls, keys) -> set[str]: + return { + key + for key in keys + if not any(key.startswith(prefix) for prefix in cls.ACTION_BACKBONE_SKIP_PREFIXES) + } + + @classmethod + def from_pretrained( + cls, + action_dit_config: dict[str, Any], + action_dit_pretrained_path: str | None = None, + skip_dit_load_from_pretrain: bool = False, + device: str = "cuda", + torch_dtype: torch.dtype = torch.bfloat16, + ) -> "ActionDiT": + if action_dit_config is None: + raise ValueError("`action_dit_config` is required for ActionDiT.from_pretrained().") + if skip_dit_load_from_pretrain: + logger.info( + "Skipping ActionDiT pretrained load (`skip_dit_load_from_pretrain=True`); " + "initializing action expert randomly and expecting checkpoint override." + ) + return cls(**action_dit_config).to(device=device, dtype=torch_dtype) + if not action_dit_pretrained_path: + logger.info("No `action_dit_pretrained_path` provided, initializing ActionDiT with random weights.") + return cls(**action_dit_config).to(device=device, dtype=torch_dtype) + from pathlib import Path + p = Path(action_dit_pretrained_path) + if not p.is_absolute(): + p = Path(__file__).resolve().parents[4] / p + action_dit_pretrained_path = str(p) + if not os.path.isfile(action_dit_pretrained_path): + raise FileNotFoundError( + f"`action_dit_pretrained_path` does not exist: {action_dit_pretrained_path}" + ) + + action_cfg = dict(action_dit_config) + action_expert = cls(**action_cfg).to(device=device, dtype=torch_dtype) + action_state = action_expert.state_dict() + expected_backbone_keys = cls.backbone_key_set(action_state.keys()) + + payload = torch.load(action_dit_pretrained_path, map_location="cpu") + if not isinstance(payload, dict): + raise ValueError( + f"Invalid action backbone payload type from {action_dit_pretrained_path}: {type(payload)}" + ) + + policy = payload.get("policy", {}) + if policy: + logger.info(f"ActionDiT backbone payload policy: {policy}") + + meta = payload.get("meta") + expected_meta = { + "hidden_dim": int(action_cfg["hidden_dim"]), + "ffn_dim": int(action_cfg["ffn_dim"]), + "num_layers": int(action_cfg["num_layers"]), + "num_heads": int(action_cfg["num_heads"]), + "attn_head_dim": int(action_cfg["attn_head_dim"]), + "text_dim": int(action_cfg["text_dim"]), + "freq_dim": int(action_cfg["freq_dim"]), + "eps": float(action_cfg["eps"]), + } + for key in cls.ACTION_BACKBONE_META_KEYS: + if key not in meta: + raise ValueError(f"`meta.{key}` missing in {action_dit_pretrained_path}") + expected_value = expected_meta[key] + got_value = meta[key] + if key == "eps": + if abs(float(got_value) - float(expected_value)) > 1e-12: + raise ValueError( + f"`meta.{key}` mismatch in {action_dit_pretrained_path}: " + f"expected {expected_value}, got {got_value}" + ) + elif int(got_value) != int(expected_value): + raise ValueError( + f"`meta.{key}` mismatch in {action_dit_pretrained_path}: " + f"expected {expected_value}, got {got_value}" + ) + + backbone_state_dict = payload.get("backbone_state_dict") + if not isinstance(backbone_state_dict, dict): + raise ValueError( + f"`backbone_state_dict` must be a dict in {action_dit_pretrained_path}, " + f"got {type(backbone_state_dict)}" + ) + + provided_keys = set(backbone_state_dict.keys()) + missing_keys = sorted(expected_backbone_keys - provided_keys) + unexpected_keys = sorted(provided_keys - expected_backbone_keys) + if missing_keys or unexpected_keys: + raise ValueError( + "Action backbone key mismatch in preprocessed payload. " + f"missing={missing_keys[:10]}{'...' if len(missing_keys) > 10 else ''}, " + f"unexpected={unexpected_keys[:10]}{'...' if len(unexpected_keys) > 10 else ''}" + ) + + merged_state = dict(action_state) + for key in expected_backbone_keys: + value = backbone_state_dict[key] + if not isinstance(value, torch.Tensor): + raise ValueError( + f"`backbone_state_dict[{key}]` must be torch.Tensor in {action_dit_pretrained_path}, " + f"got {type(value)}" + ) + target = merged_state[key] + if tuple(value.shape) != tuple(target.shape): + raise ValueError( + f"Shape mismatch for `{key}` in {action_dit_pretrained_path}: " + f"expected {tuple(target.shape)}, got {tuple(value.shape)}" + ) + merged_state[key] = value.to(device=target.device, dtype=target.dtype) + + action_expert.load_state_dict(merged_state, strict=True) + logger.info( + "Loaded ActionDiT backbone from %s (keys=%d; random_kept_prefixes=%s).", + action_dit_pretrained_path, + len(expected_backbone_keys), + list(cls.ACTION_BACKBONE_SKIP_PREFIXES), + ) + return action_expert.to(device=device, dtype=torch_dtype) + + def pre_dit( + self, + action_tokens: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: Optional[torch.Tensor] = None, + ) -> Dict[str, Any]: + if action_tokens.ndim != 3: + raise ValueError( + f"`action_tokens` must be 3D [B, T, action_dim], got shape {tuple(action_tokens.shape)}" + ) + if action_tokens.shape[2] != self.action_dim: + raise ValueError( + f"`action_tokens` last dim must be {self.action_dim}, got {action_tokens.shape[2]}" + ) + if timestep.ndim != 1: + raise ValueError(f"`timestep` must be 1D [B] or [1], got shape {tuple(timestep.shape)}") + if context.ndim != 3: + raise ValueError( + f"`context` must be 3D [B, L, D], got shape {tuple(context.shape)}" + ) + + batch_size = action_tokens.shape[0] + if context.shape[0] != batch_size: + raise ValueError( + f"Batch mismatch between action tokens and text context: {batch_size} vs {context.shape[0]}" + ) + if timestep.shape[0] not in (1, batch_size): + raise ValueError( + f"`timestep` length must be 1 or batch_size({batch_size}), got {timestep.shape[0]}" + ) + if timestep.shape[0] == 1 and batch_size > 1: + if self.training: + raise ValueError("During training, action timestep length must match batch_size.") + timestep = timestep.expand(batch_size) + + if context_mask is None: + context_mask = torch.ones( + (batch_size, context.shape[1]), dtype=torch.bool, device=context.device + ) + else: + if context_mask.ndim != 2: + raise ValueError(f"`context_mask` must be 2D [B, L], got shape {tuple(context_mask.shape)}") + if context_mask.shape[0] != batch_size or context_mask.shape[1] != context.shape[1]: + raise ValueError( + f"`context_mask` shape must match `context` shape [B, L], got {tuple(context_mask.shape)} vs {tuple(context.shape)}" + ) + + seq_len = action_tokens.shape[1] + if seq_len > self.freqs.shape[0]: + raise ValueError( + f"Action token length {seq_len} exceeds RoPE cache {self.freqs.shape[0]}." + ) + + t = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timestep)) + t_mod = self.time_projection(t).unflatten(1, (6, self.hidden_dim)) + + tokens = self.action_encoder(action_tokens) + context_emb = self.text_embedding(context) + context_attn_mask = context_mask.unsqueeze(1).expand(-1, seq_len, -1) + freqs = self.get_freqs(seq_len) + + return { + "tokens": tokens, + "freqs": freqs, + "t": t, + "t_mod": t_mod, + "context": context_emb, + "context_mask": context_attn_mask, + "meta": { + "batch_size": batch_size, + "seq_len": seq_len, + }, + } + + def post_dit(self, tokens: torch.Tensor, pre_state: Dict[str, Any]) -> torch.Tensor: + return self.head(tokens) + + def prepare( + self, + action_tokens: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Prepare tensor inputs for the compile-friendly DiT/MoT core.""" + seq_len = action_tokens.shape[1] + t = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timestep)) + t_mod = self.time_projection(t).unflatten(1, (6, self.hidden_dim)) + tokens = self.action_encoder(action_tokens) + context_emb = self.text_embedding(context) + context_attn_mask = context_mask.unsqueeze(1).expand(-1, seq_len, -1) + freqs = self.get_freqs(seq_len) + return tokens, t, t_mod, context_emb, context_attn_mask, freqs + + def post(self, tokens: torch.Tensor) -> torch.Tensor: + """Project action tokens produced by the tensor core.""" + return self.head(tokens) + + def forward( + self, + action_tokens: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if context_mask is None: + context_mask = torch.ones( + (action_tokens.shape[0], context.shape[1]), + dtype=torch.bool, + device=context.device, + ) + x, _t, t_mod, context_emb, context_attn_mask, freqs = self.prepare( + action_tokens=action_tokens, + timestep=timestep, + context=context, + context_mask=context_mask, + ) + + for block in self.blocks: + x = block( + x, + context_emb, + t_mod, + freqs, + context_mask=context_attn_mask, + ) + + return self.post(x) diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/fastwam.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/fastwam.py new file mode 100644 index 0000000000000000000000000000000000000000..f312ea04a5ec0cff9286263328668752a3b61b58 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/fastwam.py @@ -0,0 +1,1233 @@ +from typing import Any, Optional, Sequence, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from PIL import Image + +from fastwam.utils.logging_config import get_logger + +from .action_dit import ActionDiT +from .helpers.loader import load_wan22_ti2v_5b_components +from .mot import MoT +from .schedulers.scheduler_continuous import WanContinuousFlowMatchScheduler + +logger = get_logger(__name__) + + +class FastWAM(torch.nn.Module): + """MoT world model with video/action experts.""" + + def __init__( + self, + video_expert, + action_expert: ActionDiT, + mot: MoT, + vae, + text_encoder=None, + tokenizer=None, + text_dim: Optional[int] = None, + proprio_dim: Optional[int] = None, + device: str = "cpu", + torch_dtype: torch.dtype = torch.float32, + video_train_shift: float = 5.0, + video_infer_shift: float = 5.0, + video_num_train_timesteps: int = 1000, + action_train_shift: float = 5.0, + action_infer_shift: float = 5.0, + action_num_train_timesteps: int = 1000, + loss_lambda_video: float = 1.0, + loss_lambda_action: float = 1.0, + compile_training_denoise: bool = False, + ): + super().__init__() + self.video_expert = video_expert + self.action_expert = action_expert + self.mot = mot + # Keep trainer compatibility: optimizer and freeze logic use `model.dit`. + self.dit = self.mot + + self.vae = vae + self.text_encoder = text_encoder + self.tokenizer = tokenizer + if text_dim is None: + if self.text_encoder is None: + raise ValueError("`text_dim` is required when `text_encoder` is not loaded.") + text_dim = int(self.text_encoder.dim) + self.text_dim = int(text_dim) + self.proprio_dim = None if proprio_dim is None else int(proprio_dim) + if self.proprio_dim is not None: + self.proprio_encoder = nn.Linear(self.proprio_dim, self.text_dim).to(torch_dtype) + else: + self.proprio_encoder = None + + self.train_video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=video_num_train_timesteps, + shift=video_train_shift, + ) + self.infer_video_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=video_num_train_timesteps, + shift=video_infer_shift, + ) + self.train_action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=action_num_train_timesteps, + shift=action_train_shift, + ) + self.infer_action_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=action_num_train_timesteps, + shift=action_infer_shift, + ) + # Optional aliases for consistency with Wan22Core naming. + self.train_scheduler = self.train_video_scheduler + self.infer_scheduler = self.infer_video_scheduler + + self.device = torch.device(device) + self.torch_dtype = torch_dtype + self.loss_lambda_video = float(loss_lambda_video) + self.loss_lambda_action = float(loss_lambda_action) + self.compile_training_denoise = bool(compile_training_denoise) + self.mot.compile_training_layers = self.compile_training_denoise + + self.to(self.device) + + @classmethod + def from_wan22_pretrained( + cls, + device: str = "cuda", + torch_dtype: torch.dtype = torch.bfloat16, + model_id: str = "Wan-AI/Wan2.2-TI2V-5B", + tokenizer_model_id: str = "Wan-AI/Wan2.1-T2V-1.3B", + tokenizer_max_len: int = 512, + load_text_encoder: bool = True, + proprio_dim: Optional[int] = None, + redirect_common_files: bool = True, + video_dit_config: dict[str, Any] | None = None, + action_dit_config: dict[str, Any] | None = None, + action_dit_pretrained_path: str | None = None, + skip_dit_load_from_pretrain: bool = False, + mot_checkpoint_mixed_attn: bool = False, + video_train_shift: float = 5.0, + video_infer_shift: float = 5.0, + video_num_train_timesteps: int = 1000, + action_train_shift: float = 5.0, + action_infer_shift: float = 5.0, + action_num_train_timesteps: int = 1000, + loss_lambda_video: float = 1.0, + loss_lambda_action: float = 1.0, + compile_training_denoise: bool = False, + ): + if video_dit_config is None: + raise ValueError("`video_dit_config` is required for FastWAM.from_wan22_pretrained().") + if "text_dim" not in video_dit_config: + raise ValueError("`video_dit_config['text_dim']` is required for FastWAM.") + + components = load_wan22_ti2v_5b_components( + device=device, + torch_dtype=torch_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + tokenizer_max_len=tokenizer_max_len, + redirect_common_files=redirect_common_files, + dit_config=video_dit_config, + skip_dit_load_from_pretrain=skip_dit_load_from_pretrain, + load_text_encoder=load_text_encoder, + ) + + video_expert = components.dit + action_expert = ActionDiT.from_pretrained( + action_dit_config=action_dit_config, + action_dit_pretrained_path=action_dit_pretrained_path, + skip_dit_load_from_pretrain=skip_dit_load_from_pretrain, + device=device, + torch_dtype=torch_dtype, + ) + if int(action_expert.num_heads) != int(video_expert.num_heads): + raise ValueError("ActionDiT `num_heads` must match video expert for MoT mixed attention.") + if int(action_expert.attn_head_dim) != int(video_expert.attn_head_dim): + raise ValueError("ActionDiT `attn_head_dim` must match video expert for MoT mixed attention.") + if int(len(action_expert.blocks)) != int(len(video_expert.blocks)): + raise ValueError("ActionDiT `num_layers` must match video expert.") + + mot = MoT( + mixtures={"video": video_expert, "action": action_expert}, + mot_checkpoint_mixed_attn=mot_checkpoint_mixed_attn, + ) + + model = cls( + video_expert=video_expert, + action_expert=action_expert, + mot=mot, + vae=components.vae, + text_encoder=components.text_encoder, + tokenizer=components.tokenizer, + text_dim=int(video_dit_config["text_dim"]), + proprio_dim=proprio_dim, + device=device, + torch_dtype=torch_dtype, + video_train_shift=video_train_shift, + video_infer_shift=video_infer_shift, + video_num_train_timesteps=video_num_train_timesteps, + action_train_shift=action_train_shift, + action_infer_shift=action_infer_shift, + action_num_train_timesteps=action_num_train_timesteps, + loss_lambda_video=loss_lambda_video, + loss_lambda_action=loss_lambda_action, + compile_training_denoise=compile_training_denoise, + ) + model.model_paths = { + "video_dit": components.dit_path, + "vae": components.vae_path, + "text_encoder": components.text_encoder_path, + "tokenizer": components.tokenizer_path, + "action_dit_backbone": ( + "SKIPPED_PRETRAIN" if skip_dit_load_from_pretrain else action_dit_pretrained_path + ), + } + return model + + def to(self, *args, **kwargs): + super().to(*args, **kwargs) + self.mot.to(*args, **kwargs) + if self.text_encoder is not None: + self.text_encoder.to(*args, **kwargs) + self.vae.to(*args, **kwargs) + return self + + @staticmethod + def _check_resize_height_width(height, width, num_frames): + if height % 16 != 0: + height = (height + 15) // 16 * 16 + if width % 16 != 0: + width = (width + 15) // 16 * 16 + if num_frames % 4 != 1: + num_frames = (num_frames + 3) // 4 * 4 + 1 + return height, width, num_frames + + @torch.no_grad() + def encode_prompt(self, prompt: Union[str, Sequence[str]]): + if self.text_encoder is None or self.tokenizer is None: + raise ValueError( + "Prompt encoding requires loaded text encoder/tokenizer. " + "Set `load_text_encoder=true` or provide precomputed `context/context_mask`." + ) + ids, mask = self.tokenizer(prompt, return_mask=True, add_special_tokens=True) + ids = ids.to(self.device) + mask = mask.to(self.device, dtype=torch.bool) + prompt_emb = self.text_encoder(ids, mask) + # FIXME: original implementation's zero padding is visible in cross-attn. + seq_lens = mask.gt(0).sum(dim=1).long() + for i, v in enumerate(seq_lens): + prompt_emb[i, v:] = 0 + mask = torch.ones_like(mask) + return prompt_emb.to(device=self.device), mask + + def _append_proprio_to_context( + self, + context: torch.Tensor, + context_mask: torch.Tensor, + proprio: Optional[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor]: + if self.proprio_encoder is None or proprio is None: + return context, context_mask + if proprio.ndim != 2: + raise ValueError(f"`proprio` must be 2D [B, D], got shape {tuple(proprio.shape)}") + if self.proprio_dim is None or proprio.shape[1] != self.proprio_dim: + raise ValueError( + f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}" + ) + proprio_token = self.proprio_encoder( + proprio.to(device=self.device, dtype=context.dtype).unsqueeze(1) + ).to(dtype=context.dtype) # [B, 1, D] + proprio_mask = torch.ones((context_mask.shape[0], 1), dtype=torch.bool, device=context_mask.device) + return ( + torch.cat([context, proprio_token], dim=1), + torch.cat([context_mask, proprio_mask], dim=1), + ) + + @torch.no_grad() + def _encode_video_latents(self, video_tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)): + if tiled: + raise NotImplementedError("Batched VAE encoding does not support tiled encoding.") + if not hasattr(self, "_vae_encode_compiled"): + self._vae_encode_compiled = torch.compile( + self.vae.model.encode, + backend="cudagraphs", + fullgraph=True, + ) + return self._vae_encode_compiled( + video_tensor.to(self.device), + self.vae.scale, + ).clone() + + @torch.no_grad() + def _encode_input_image_latents_tensor(self, input_image: torch.Tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)): + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + if tiled: + raise NotImplementedError("Batched VAE image encoding does not support tiled encoding.") + image = input_image.to(device=self.device)[0].unsqueeze(1) + return self.vae.model.encode(image.unsqueeze(0), self.vae.scale) + + def _decode_latents(self, latents, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)): + video_tensor = self.vae.decode(latents, device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride) + video_tensor = video_tensor.squeeze(0).detach().float().clamp(-1, 1) + video_tensor = ((video_tensor + 1.0) * 127.5).to(torch.uint8).cpu() + frames = [] + for t in range(video_tensor.shape[1]): + frame = video_tensor[:, t].permute(1, 2, 0).numpy() + frames.append(Image.fromarray(frame)) + return frames + + def build_inputs(self, sample, tiled: bool = False): + video = sample["video"] + proprio = sample.get("proprio", None) + if video.ndim != 5: + raise ValueError(f"`sample['video']` must be 5D [B, 3, T, H, W], got shape {tuple(video.shape)}") + if video.shape[1] != 3: + raise ValueError(f"`sample['video']` channel dimension must be 3, got shape {tuple(video.shape)}") + + batch_size, _, num_frames, height, width = video.shape + if height % 16 != 0 or width % 16 != 0: + raise ValueError( + f"Video spatial dims must be multiples of 16, got H={height}, W={width}" + ) + if num_frames % 4 != 1: + raise ValueError(f"Video T must satisfy T % 4 == 1, got T={num_frames}") + if num_frames <= 1: + raise ValueError(f"Video T must be > 1 for action-conditioned training, got T={num_frames}") + + if "action" not in sample: + raise ValueError("`sample['action']` is required for FastWAM training.") + + action = sample["action"] + if action.ndim != 3: + raise ValueError(f"`sample['action']` must be 3D [B, T, a_dim], got shape {tuple(action.shape)}") + action_horizon = int(action.shape[1]) + if action_horizon % (num_frames - 1) != 0: + raise ValueError( + f"`sample['action']` temporal dimension must be divisible by video transitions ({num_frames - 1}), got {action_horizon}" + ) + + action_is_pad = sample.get("action_is_pad", None) + if action_is_pad is not None: + if action_is_pad.ndim != 2: + raise ValueError( + f"`sample['action_is_pad']` must be 2D [B, T], got shape {tuple(action_is_pad.shape)}" + ) + if action_is_pad.shape[0] != batch_size or action_is_pad.shape[1] != action_horizon: + raise ValueError( + "`sample['action_is_pad']` shape mismatch: " + f"got {tuple(action_is_pad.shape)} vs expected ({batch_size}, {action_horizon})" + ) + + image_is_pad = sample.get("image_is_pad", None) + if image_is_pad is not None: + if image_is_pad.ndim != 2: + raise ValueError( + f"`sample['image_is_pad']` must be 2D [B, T], got shape {tuple(image_is_pad.shape)}" + ) + if image_is_pad.shape[0] != batch_size or image_is_pad.shape[1] != num_frames: + raise ValueError( + "`sample['image_is_pad']` shape mismatch: " + f"got {tuple(image_is_pad.shape)} vs expected ({batch_size}, {num_frames})" + ) + + input_video = video.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + input_latents = self._encode_video_latents(input_video, tiled=tiled) + context = sample.get("context") + context_mask = sample.get("context_mask") + if context is None and context_mask is None: + prompt = sample.get("prompt") + if prompt is None: + raise ValueError("FastWAM training requires `context/context_mask` or `prompt`.") + context, context_mask = self.encode_prompt(prompt) + elif context is None or context_mask is None: + raise ValueError("`context` and `context_mask` must both exist when either is provided.") + + first_frame_latents = None + fuse_flag = False + if getattr(self.video_expert, "fuse_vae_embedding_in_latents", False): + first_frame_latents = input_latents[:, :, 0:1] + fuse_flag = True + + if context.ndim != 3 or context_mask.ndim != 2: + raise ValueError( + f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}" + ) + context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True) + if self.proprio_encoder is not None: + if proprio is None: + raise ValueError("`sample['proprio']` is required when `proprio_dim` is enabled.") + if proprio.ndim != 3: + raise ValueError(f"`sample['proprio']` must be 3D [B, T, d], got shape {tuple(proprio.shape)}") + if proprio.shape[2] != self.proprio_dim: + raise ValueError( + f"`sample['proprio']` last dim must be {self.proprio_dim}, got {proprio.shape[2]}" + ) + proprio = proprio[:, 0, :] # [B, D] + context, context_mask = self._append_proprio_to_context( + context=context, + context_mask=context_mask, + proprio=proprio.to(device=self.device, dtype=self.torch_dtype), + ) + action = action.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + + if action_is_pad is not None: + action_is_pad = action_is_pad.to(device=self.device, dtype=torch.bool, non_blocking=True) + if image_is_pad is not None: + image_is_pad = image_is_pad.to(device=self.device, dtype=torch.bool, non_blocking=True) + + return { + "context": context, + "context_mask": context_mask, + "input_latents": input_latents, + "first_frame_latents": first_frame_latents, + "fuse_vae_embedding_in_latents": fuse_flag, + "action": action, + "action_is_pad": action_is_pad, + "image_is_pad": image_is_pad, + } + + @torch.no_grad() + def _build_mot_attention_mask( + self, + video_seq_len: int, + action_seq_len: int, + video_tokens_per_frame: int, + device: torch.device, + ) -> torch.Tensor: + total_seq_len = video_seq_len + action_seq_len + mask = torch.zeros((total_seq_len, total_seq_len), dtype=torch.bool, device=device) + + # video -> video + mask[:video_seq_len, :video_seq_len] = self.video_expert.build_video_to_video_mask( + video_seq_len=video_seq_len, + video_tokens_per_frame=video_tokens_per_frame, + device=device, + ) + # action -> action + mask[video_seq_len:, video_seq_len:] = True + # action -> first-frame video only + first_frame_tokens = min(video_tokens_per_frame, video_seq_len) + mask[video_seq_len:, :first_frame_tokens] = True + return mask + + def _compute_video_loss_per_sample( + self, + pred_video: torch.Tensor, + target_video: torch.Tensor, + image_is_pad: Optional[torch.Tensor], + include_initial_video_step: bool, + ) -> torch.Tensor: + video_loss_token = F.mse_loss(pred_video.float(), target_video.float(), reduction="none").mean(dim=(1, 3, 4)) + if image_is_pad is None: + return video_loss_token.mean(dim=1) + + temporal_factor = int(self.vae.temporal_downsample_factor) + if temporal_factor <= 0: + raise ValueError(f"`vae.temporal_downsample_factor` must be positive, got {temporal_factor}.") + if image_is_pad.shape[1] < 1: + raise ValueError("`image_is_pad` must contain at least one frame.") + if (image_is_pad.shape[1] - 1) % temporal_factor != 0: + raise ValueError( + "Cannot align `image_is_pad` with video latent steps: " + f"num_frames={image_is_pad.shape[1]}, temporal_downsample_factor={temporal_factor}." + ) + + tail_is_pad = image_is_pad[:, 1:] + latent_tail_is_pad = tail_is_pad.view(image_is_pad.shape[0], -1, temporal_factor).all(dim=2) + if include_initial_video_step: + video_is_pad = torch.cat([image_is_pad[:, :1], latent_tail_is_pad], dim=1) + else: + video_is_pad = latent_tail_is_pad + + if video_is_pad.shape[1] != video_loss_token.shape[1]: + raise ValueError( + "Video-loss mask shape mismatch: " + f"mask steps={video_is_pad.shape[1]}, loss steps={video_loss_token.shape[1]}." + ) + + valid = (~video_is_pad).to(device=video_loss_token.device, dtype=video_loss_token.dtype) + valid_sum = valid.sum(dim=1).clamp(min=1.0) + return (video_loss_token * valid).sum(dim=1) / valid_sum + + def _joint_denoise_core( + self, + latents_video: torch.Tensor, + latents_action: torch.Tensor, + timestep_video: torch.Tensor, + timestep_action: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + attention_mask: torch.Tensor, + fuse_vae_embedding_in_latents: bool, + action_condition: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Run the tensor-only video/action core shared by training and inference.""" + ( + video_tokens, + t_video, + t_mod_video, + context_video, + context_mask_video, + freqs_video, + f_video, + h_video, + w_video, + _tokens_per_frame, + ) = self.video_expert.prepare( + x=latents_video, + timestep=timestep_video, + context=context, + context_mask=context_mask, + action=action_condition, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + ( + action_tokens, + _t_action, + t_mod_action, + context_action, + context_mask_action, + freqs_action, + ) = self.action_expert.prepare( + action_tokens=latents_action, + timestep=timestep_action, + context=context, + context_mask=context_mask, + ) + video_tokens, action_tokens = self.mot.forward_joint_core( + video_tokens=video_tokens, + action_tokens=action_tokens, + video_freqs=freqs_video, + action_freqs=freqs_action, + video_t_mod=t_mod_video, + action_t_mod=t_mod_action, + video_context=context_video, + video_context_mask=context_mask_video, + action_context=context_action, + action_context_mask=context_mask_action, + attention_mask=attention_mask, + ) + return ( + self.video_expert.post(video_tokens, t_video, f_video, h_video, w_video), + self.action_expert.post(action_tokens), + ) + + def training_loss(self, sample, tiled: bool = False): + inputs = self.build_inputs(sample, tiled=tiled) + input_latents = inputs["input_latents"] + batch_size = input_latents.shape[0] + context = inputs["context"] + context_mask = inputs["context_mask"] + action = inputs["action"] + action_is_pad = inputs["action_is_pad"] + image_is_pad = inputs["image_is_pad"] + + noise_video = torch.randn_like(input_latents) + timestep_video = self.train_video_scheduler.sample_training_t( + batch_size=batch_size, + device=self.device, + dtype=input_latents.dtype, + ) + latents = self.train_video_scheduler.add_noise(input_latents, noise_video, timestep_video) + target_video = self.train_video_scheduler.training_target(input_latents, noise_video, timestep_video) + + if inputs["first_frame_latents"] is not None: + latents[:, :, 0:1] = inputs["first_frame_latents"] + + noise_action = torch.randn_like(action) + timestep_action = self.train_action_scheduler.sample_training_t( + batch_size=batch_size, + device=self.device, + dtype=action.dtype, + ) + noisy_action = self.train_action_scheduler.add_noise(action, noise_action, timestep_action) + target_action = self.train_action_scheduler.training_target(action, noise_action, timestep_action) + + patch_t, patch_h, patch_w = (int(size) for size in self.video_expert.patch_size) + latent_t, latent_h, latent_w = latents.shape[-3:] + tokens_per_frame = (latent_h // patch_h) * (latent_w // patch_w) + attention_mask = self._build_mot_attention_mask( + video_seq_len=(latent_t // patch_t) * tokens_per_frame, + action_seq_len=noisy_action.shape[1], + video_tokens_per_frame=tokens_per_frame, + device=latents.device, + ) + pred_video, pred_action = self._joint_denoise_core( + latents_video=latents, + latents_action=noisy_action, + timestep_video=timestep_video, + timestep_action=timestep_action, + context=context, + context_mask=context_mask, + attention_mask=attention_mask, + fuse_vae_embedding_in_latents=inputs["fuse_vae_embedding_in_latents"], + action_condition=action, + ) + + include_initial_video_step = inputs["first_frame_latents"] is None + if inputs["first_frame_latents"] is not None: + pred_video = pred_video[:, :, 1:] + target_video = target_video[:, :, 1:] + + loss_video_per_sample = self._compute_video_loss_per_sample( + pred_video=pred_video, + target_video=target_video, + image_is_pad=image_is_pad, + include_initial_video_step=include_initial_video_step, + ) + video_weight = self.train_video_scheduler.training_weight(timestep_video).to( + loss_video_per_sample.device, dtype=loss_video_per_sample.dtype + ) + loss_video = (loss_video_per_sample * video_weight).mean() + + action_loss_token = F.mse_loss(pred_action.float(), target_action.float(), reduction="none").mean(dim=2) # [B, T] + if action_is_pad is not None: + valid = (~action_is_pad).to(device=action_loss_token.device, dtype=action_loss_token.dtype) + valid_sum = valid.sum(dim=1).clamp(min=1.0) + action_loss_per_sample = (action_loss_token * valid).sum(dim=1) / valid_sum + else: + action_loss_per_sample = action_loss_token.mean(dim=1) + + action_weight = self.train_action_scheduler.training_weight(timestep_action).to( + action_loss_per_sample.device, dtype=action_loss_per_sample.dtype + ) + loss_action = (action_loss_per_sample * action_weight).mean() + + loss_total = self.loss_lambda_video * loss_video + self.loss_lambda_action * loss_action + loss_dict = { + "loss_video": self.loss_lambda_video * float(loss_video.detach().item()), + "loss_action": self.loss_lambda_action * float(loss_action.detach().item()), + } + return loss_total, loss_dict + + @torch.no_grad() + def _predict_joint_noise( + self, + latents_video: torch.Tensor, + latents_action: torch.Tensor, + timestep_video: torch.Tensor, + timestep_action: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + fuse_vae_embedding_in_latents: bool, + gt_action: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + patch_t, patch_h, patch_w = (int(size) for size in self.video_expert.patch_size) + latent_t, latent_h, latent_w = latents_video.shape[-3:] + tokens_per_frame = (latent_h // patch_h) * (latent_w // patch_w) + attention_mask = self._build_mot_attention_mask( + video_seq_len=(latent_t // patch_t) * tokens_per_frame, + action_seq_len=latents_action.shape[1], + video_tokens_per_frame=tokens_per_frame, + device=latents_video.device, + ) + return self._joint_denoise_core( + latents_video=latents_video, + latents_action=latents_action, + timestep_video=timestep_video, + timestep_action=timestep_action, + context=context, + context_mask=context_mask, + attention_mask=attention_mask, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + action_condition=gt_action, + ) + + @torch.no_grad() + def _predict_action_noise( + self, + first_frame_latents: torch.Tensor, + latents_action: torch.Tensor, + timestep_action: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + fuse_vae_embedding_in_latents: bool, + ) -> torch.Tensor: + timestep_video = torch.zeros_like(timestep_action, dtype=first_frame_latents.dtype, device=self.device) + video_pre = self.video_expert.pre_dit( + x=first_frame_latents, + timestep=timestep_video, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + action_pre = self.action_expert.pre_dit( + action_tokens=latents_action, + timestep=timestep_action, + context=context, + context_mask=context_mask, + ) + + attention_mask = self._build_mot_attention_mask( + video_seq_len=video_pre["tokens"].shape[1], + action_seq_len=action_pre["tokens"].shape[1], + video_tokens_per_frame=int(video_pre["meta"]["tokens_per_frame"]), + device=video_pre["tokens"].device, + ) + tokens_out = self.mot( + embeds_all={ + "video": video_pre["tokens"], + "action": action_pre["tokens"], + }, + attention_mask=attention_mask, + freqs_all={ + "video": video_pre["freqs"], + "action": action_pre["freqs"], + }, + context_all={ + "video": { + "context": video_pre["context"], + "mask": video_pre["context_mask"], + }, + "action": { + "context": action_pre["context"], + "mask": action_pre["context_mask"], + }, + }, + t_mod_all={ + "video": video_pre["t_mod"], + "action": action_pre["t_mod"], + }, + ) + pred_action = self.action_expert.post_dit(tokens_out["action"], action_pre) + return pred_action + + def _denoise_action_with_video_cache( + self, + latents_action: torch.Tensor, + timestep_action: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + video_cache_k: list[torch.Tensor], + video_cache_v: list[torch.Tensor], + action_attention_mask: torch.Tensor, + ) -> torch.Tensor: + ( + action_tokens, + _t, + action_t_mod, + action_context, + action_context_mask, + action_freqs, + ) = self.action_expert.prepare( + action_tokens=latents_action, + timestep=timestep_action, + context=context, + context_mask=context_mask, + ) + action_tokens = self.mot.forward_action_with_video_cache_tensor( + action_tokens=action_tokens, + action_freqs=action_freqs, + action_t_mod=action_t_mod, + action_context=action_context, + action_context_mask=action_context_mask, + video_cache_k=video_cache_k, + video_cache_v=video_cache_v, + action_attention_mask=action_attention_mask, + ) + return self.action_expert.post(action_tokens) + + @torch.no_grad() + def _predict_action_noise_with_cache( + self, + latents_action: torch.Tensor, + timestep_action: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + video_kv_cache: list[dict[str, torch.Tensor]], + attention_mask: torch.Tensor, + video_seq_len: int, + ) -> torch.Tensor: + """Legacy dictionary-cache path retained for the optional IDM variant.""" + action_pre = self.action_expert.pre_dit( + action_tokens=latents_action, + timestep=timestep_action, + context=context, + context_mask=context_mask, + ) + action_tokens = self.mot.forward_action_with_video_cache( + action_tokens=action_pre["tokens"], + action_freqs=action_pre["freqs"], + action_t_mod=action_pre["t_mod"], + action_context_payload={ + "context": action_pre["context"], + "mask": action_pre["context_mask"], + }, + video_kv_cache=video_kv_cache, + attention_mask=attention_mask, + video_seq_len=video_seq_len, + ) + return self.action_expert.post_dit(action_tokens, action_pre) + + @torch.no_grad() + def infer_joint( + self, + prompt: Optional[str], + input_image: torch.Tensor, + num_video_frames: int, + action_horizon: int, + action: Optional[torch.Tensor] = None, # NOTE: this is gt action for conditioning videos, not for action expert + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + test_action_with_infer_action: bool = True, + compile_action_infer: bool = False, + ) -> dict[str, Any]: + self.eval() + if test_action_with_infer_action: + if seed is None: + raise ValueError("`test_action_with_infer_action=True` requires non-null `seed`.") + action_only_out = self.infer_action( + prompt=prompt, + input_image=input_image.clone(), + action_horizon=action_horizon, + context=context.clone() if context is not None else None, + context_mask=context_mask.clone() if context_mask is not None else None, + num_inference_steps=num_inference_steps, + sigma_shift=sigma_shift, + seed=seed, + rand_device=rand_device, + tiled=tiled, + proprio=proprio.clone() if proprio is not None else None, + compile_action_infer=compile_action_infer, + )["action"] + + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + _, _, height, width = input_image.shape + checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_video_frames) + if (checked_h, checked_w) != (height, width): + raise ValueError( + f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})" + ) + if checked_t != num_video_frames: + raise ValueError( + f"`num_video_frames` must satisfy T % 4 == 1, got {num_video_frames}" + ) + if action is not None: + if action.ndim == 2: + action = action.unsqueeze(0) + if action.ndim != 3 or action.shape[0] != 1 or action.shape[1] != action_horizon: + # NOTE: This enforces action condition to have the same shape as action horizon to predict, which may be unnecessary + raise ValueError( + f"`action` must have shape [1, T, a_dim] or [T, a_dim], got {tuple(action.shape)} with action_horizon={action_horizon}" + ) + action = action.to(device=self.device, dtype=self.torch_dtype) + if proprio is not None: + if self.proprio_dim is None: + raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.") + if proprio.ndim == 1: + proprio = proprio.unsqueeze(0) + elif proprio.ndim == 2 and proprio.shape[0] == 1: + pass + else: + raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}") + if proprio.shape[1] != self.proprio_dim: + raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}") + proprio = proprio.to(device=self.device, dtype=self.torch_dtype) + + latent_t = (num_video_frames - 1) // self.vae.temporal_downsample_factor + 1 + latent_h = height // self.vae.upsampling_factor + latent_w = width // self.vae.upsampling_factor + + video_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + action_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + latents_video = torch.randn( + (1, self.vae.model.z_dim, latent_t, latent_h, latent_w), + generator=video_generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + latents_action = torch.randn( + (1, action_horizon, self.action_expert.action_dim), + generator=action_generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + + input_image = input_image.to(device=self.device, dtype=self.torch_dtype) + first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled) + latents_video[:, :, 0:1] = first_frame_latents.clone() + fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False)) + + use_prompt = prompt is not None + use_context = context is not None or context_mask is not None + if use_prompt and use_context: + raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.") + if not use_prompt and not use_context: + raise ValueError("Either `prompt` or both `context/context_mask` must be provided.") + + if use_prompt: + context, context_mask = self.encode_prompt(prompt) + else: + if context is None or context_mask is None: + raise ValueError("`context` and `context_mask` must be both provided together.") + if context.ndim == 2: + context = context.unsqueeze(0) + if context_mask.ndim == 1: + context_mask = context_mask.unsqueeze(0) + if context.ndim != 3 or context_mask.ndim != 2: + raise ValueError( + f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}" + ) + context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True) + if proprio is not None: + context, context_mask = self._append_proprio_to_context( + context=context, + context_mask=context_mask, + proprio=proprio, + ) + + patch_t, patch_h, patch_w = (int(size) for size in self.video_expert.patch_size) + tokens_per_frame = (latent_h // patch_h) * (latent_w // patch_w) + joint_attention_mask = self._build_mot_attention_mask( + video_seq_len=(latent_t // patch_t) * tokens_per_frame, + action_seq_len=latents_action.shape[1], + video_tokens_per_frame=tokens_per_frame, + device=self.device, + ) + if compile_action_infer: + if action is not None: + raise ValueError( + "`compile_action_infer=True` does not support action conditioning in `infer_joint`." + ) + if not hasattr(self, "_joint_denoise_core_compiled_inference"): + self._joint_denoise_core_compiled_inference = torch.compile( + self._joint_denoise_core, + mode="reduce-overhead", + fullgraph=True, + ) + joint_denoise_core = self._joint_denoise_core_compiled_inference + else: + joint_denoise_core = self._joint_denoise_core + + infer_timesteps_video, infer_deltas_video = self.infer_video_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_video.dtype, + shift_override=sigma_shift, + ) + infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_action.dtype, + shift_override=sigma_shift, + ) + for step_t_video, step_delta_video, step_t_action, step_delta_action in zip( + infer_timesteps_video, + infer_deltas_video, + infer_timesteps_action, + infer_deltas_action, + ): + if compile_action_infer: + torch.compiler.cudagraph_mark_step_begin() + timestep_video = step_t_video.unsqueeze(0).to(dtype=latents_video.dtype, device=self.device) + timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device) + + pred_video_posi, pred_action_posi = joint_denoise_core( + latents_video=latents_video, + latents_action=latents_action, + timestep_video=timestep_video, + timestep_action=timestep_action, + context=context, + context_mask=context_mask, + attention_mask=joint_attention_mask, + fuse_vae_embedding_in_latents=fuse_flag, + action_condition=action, + ) + pred_video = pred_video_posi + pred_action = pred_action_posi + + latents_video = self.infer_video_scheduler.step(pred_video, step_delta_video, latents_video) + latents_action = self.infer_action_scheduler.step(pred_action, step_delta_action, latents_action) + latents_video[:, :, 0:1] = first_frame_latents.clone() + + action_out = latents_action[0].detach().to(device="cpu", dtype=torch.float32) + if test_action_with_infer_action: + if not torch.allclose(action_out, action_only_out, atol=1e-2, rtol=1e-2): + max_abs_diff = (action_out - action_only_out).abs().max().item() + logger.warning( + f"Action from infer_joint and infer_action differ with max abs diff {max_abs_diff:.6f}. " + ) + + return { + "video": self._decode_latents(latents_video, tiled=tiled), + "action": action_out, + } + + @torch.no_grad() + def infer_action( + self, + prompt: Optional[str], + input_image: torch.Tensor, + action_horizon: int, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + compile_action_infer: bool = False, + ) -> dict[str, Any]: + self.eval() + if str(getattr(self.video_expert, "video_attention_mask_mode", "")) != "first_frame_causal": + raise ValueError( + "`infer_action` requires `video_attention_mask_mode='first_frame_causal'`." + ) + + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + _, _, height, width = input_image.shape + if height % 16 != 0 or width % 16 != 0: + raise ValueError( + f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})" + ) + if proprio is not None: + if self.proprio_dim is None: + raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.") + if proprio.ndim == 1: + proprio = proprio.unsqueeze(0) + elif proprio.ndim == 2 and proprio.shape[0] == 1: + pass + else: + raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}") + if proprio.shape[1] != self.proprio_dim: + raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}") + proprio = proprio.to(device=self.device, dtype=self.torch_dtype) + + generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + latents_action = torch.randn( + (1, action_horizon, self.action_expert.action_dim), + generator=generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + + input_image = input_image.to(device=self.device, dtype=self.torch_dtype) + first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled) + fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False)) + + use_prompt = prompt is not None + use_context = context is not None or context_mask is not None + if use_prompt and use_context: + raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.") + if not use_prompt and not use_context: + raise ValueError("Either `prompt` or both `context/context_mask` must be provided.") + + if use_prompt: + context, context_mask = self.encode_prompt(prompt) + else: + if context is None or context_mask is None: + raise ValueError("`context` and `context_mask` must be both provided together.") + if context.ndim == 2: + context = context.unsqueeze(0) + if context_mask.ndim == 1: + context_mask = context_mask.unsqueeze(0) + if context.ndim != 3 or context_mask.ndim != 2: + raise ValueError( + f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}" + ) + context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True) + if proprio is not None: + context, context_mask = self._append_proprio_to_context( + context=context, + context_mask=context_mask, + proprio=proprio, + ) + + timestep_video = torch.zeros( + (first_frame_latents.shape[0],), + dtype=first_frame_latents.dtype, + device=self.device, + ) + ( + video_tokens, + _t_video, + video_t_mod, + video_context, + video_context_mask, + video_freqs, + _f_video, + _h_video, + _w_video, + tokens_per_frame, + ) = self.video_expert.prepare( + x=first_frame_latents, + timestep=timestep_video, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_flag, + ) + video_seq_len = int(video_tokens.shape[1]) + attention_mask = self._build_mot_attention_mask( + video_seq_len=video_seq_len, + action_seq_len=latents_action.shape[1], + video_tokens_per_frame=tokens_per_frame, + device=video_tokens.device, + ) + video_attention_mask = attention_mask[:video_seq_len, :video_seq_len] + action_attention_mask = attention_mask[video_seq_len:, :] + if compile_action_infer: + if not hasattr(self, "_prefill_video_cache_compiled"): + self._prefill_video_cache_compiled = torch.compile( + self.mot.prefill_video_cache_tensor, + mode="reduce-overhead", + fullgraph=True, + ) + if not hasattr(self, "_denoise_action_with_video_cache_compiled"): + self._denoise_action_with_video_cache_compiled = torch.compile( + self._denoise_action_with_video_cache, + mode="reduce-overhead", + fullgraph=True, + ) + prefill_video_cache = self._prefill_video_cache_compiled + denoise_action_with_video_cache = self._denoise_action_with_video_cache_compiled + else: + prefill_video_cache = self.mot.prefill_video_cache_tensor + denoise_action_with_video_cache = self._denoise_action_with_video_cache + if compile_action_infer: + torch.compiler.cudagraph_mark_step_begin() + video_cache_k, video_cache_v = prefill_video_cache( + video_tokens=video_tokens, + video_freqs=video_freqs, + video_t_mod=video_t_mod, + video_context=video_context, + video_context_mask=video_context_mask, + video_attention_mask=video_attention_mask, + ) + if compile_action_infer: + # Inductor reduce-overhead may return graph-owned buffers that are overwritten on replay. + video_cache_k = [cache.clone() for cache in video_cache_k] + video_cache_v = [cache.clone() for cache in video_cache_v] + + infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_action.dtype, + shift_override=sigma_shift, + ) + for step_t_action, step_delta_action in zip(infer_timesteps_action, infer_deltas_action): + if compile_action_infer: + torch.compiler.cudagraph_mark_step_begin() + timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device) + + pred_action_posi = denoise_action_with_video_cache( + latents_action=latents_action, + timestep_action=timestep_action, + context=context, + context_mask=context_mask, + video_cache_k=video_cache_k, + video_cache_v=video_cache_v, + action_attention_mask=action_attention_mask, + ) + pred_action = pred_action_posi + + latents_action = self.infer_action_scheduler.step(pred_action, step_delta_action, latents_action) + + return { + "action": latents_action[0].detach().to(device="cpu", dtype=torch.float32), + } + + @torch.no_grad() + def infer( + self, + prompt: Optional[str], + input_image: torch.Tensor, + num_frames: int, + action: Optional[torch.Tensor] = None, + action_horizon: Optional[int] = None, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 5.0, + action_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + ): + return self.infer_joint( + prompt=prompt, + input_image=input_image, + num_video_frames=num_frames, + action_horizon=action_horizon, + action=action, + proprio=proprio, + context=context, + context_mask=context_mask, + negative_prompt=negative_prompt, + text_cfg_scale=text_cfg_scale, + num_inference_steps=num_inference_steps, + sigma_shift=sigma_shift, + seed=seed, + rand_device=rand_device, + tiled=tiled, + ) + + def save_checkpoint(self, path, optimizer=None, step=None): + payload = { + "mot": self.mot.state_dict(), + "step": step, + "torch_dtype": str(self.torch_dtype), + } + if self.proprio_encoder is not None: + payload["proprio_encoder"] = self.proprio_encoder.state_dict() + if optimizer is not None: + payload["optimizer"] = optimizer.state_dict() + torch.save(payload, path) + + def load_checkpoint(self, path, optimizer=None): + payload = torch.load(path, map_location="cpu") + if "mot" in payload: + self.mot.load_state_dict(payload["mot"], strict=False) + elif "dit" in payload: + logger.warning("Loading legacy `dit` checkpoint into video expert only.") + self.video_expert.load_state_dict(payload["dit"], strict=False) + else: + raise ValueError(f"Checkpoint missing both `mot` and `dit` keys: {path}") + if self.proprio_encoder is not None: + if "proprio_encoder" in payload: + self.proprio_encoder.load_state_dict(payload["proprio_encoder"], strict=True) + else: + logger.warning("Checkpoint has no `proprio_encoder` weights; keeping current `proprio_encoder` params.") + elif "proprio_encoder" in payload: + logger.warning("Checkpoint contains `proprio_encoder` weights but current model has `proprio_dim=None`; ignoring.") + + if optimizer is not None and "optimizer" in payload: + optimizer.load_state_dict(payload["optimizer"]) + return payload + + def forward(self, *args, **kwargs): + return self.training_loss(*args, **kwargs) diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/fastwam_idm.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/fastwam_idm.py new file mode 100644 index 0000000000000000000000000000000000000000..bddb3850ef5b76a48d93d8227eb408674f21dd32 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/fastwam_idm.py @@ -0,0 +1,589 @@ +from typing import Any, Optional + +import torch +import torch.nn.functional as F + +from fastwam.utils.logging_config import get_logger + +from .fastwam_joint import FastWAMJoint + +logger = get_logger(__name__) + + +class FastWAMIDM(FastWAMJoint): + """IDM variant with teacher-forcing video conditioning for action denoising.""" + + video_cond_noise_prob: float + + @classmethod + def from_wan22_pretrained(cls, *, video_cond_noise_prob: float = 0.5, **kwargs): + prob = float(video_cond_noise_prob) + if not 0.0 <= prob <= 1.0: + raise ValueError(f"`video_cond_noise_prob` must be in [0, 1], got {prob}.") + model = super().from_wan22_pretrained(**kwargs) + model.video_cond_noise_prob = prob + return model + + @torch.no_grad() + def _build_teacher_forcing_attention_mask( + self, + noisy_video_seq_len: int, + cond_video_seq_len: int, + action_seq_len: int, + noisy_video_tokens_per_frame: int, + cond_video_tokens_per_frame: int, + batch_size: int, + device: torch.device, + ) -> torch.Tensor: + del batch_size + if noisy_video_tokens_per_frame != cond_video_tokens_per_frame: + raise ValueError( + "Teacher-forcing requires identical `tokens_per_frame` for noisy and cond video branches, " + f"got {noisy_video_tokens_per_frame} and {cond_video_tokens_per_frame}." + ) + + noisy_end = noisy_video_seq_len + cond_end = noisy_video_seq_len + cond_video_seq_len + total_seq_len = cond_end + action_seq_len + mask = torch.zeros((total_seq_len, total_seq_len), dtype=torch.bool, device=device) + + # noisy_video -> noisy_video + mask[:noisy_end, :noisy_end] = self.video_expert.build_video_to_video_mask( + video_seq_len=noisy_video_seq_len, + video_tokens_per_frame=noisy_video_tokens_per_frame, + device=device, + ) + # cond_video -> cond_video + mask[noisy_end:cond_end, noisy_end:cond_end] = self.video_expert.build_video_to_video_mask( + video_seq_len=cond_video_seq_len, + video_tokens_per_frame=cond_video_tokens_per_frame, + device=device, + ) + # action -> action + mask[cond_end:, cond_end:] = True + # action -> cond_video only + mask[cond_end:, noisy_end:cond_end] = True + return mask + + def _teacher_forcing_training_denoise_core( + self, + latents_noisy: torch.Tensor, + latents_cond: torch.Tensor, + noisy_action: torch.Tensor, + timestep_video: torch.Tensor, + timestep_video_cond: torch.Tensor, + timestep_action: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + fuse_vae_embedding_in_latents: bool, + ) -> tuple[torch.Tensor, torch.Tensor]: + ( + noisy_video_tokens, + t_video_noisy, + t_mod_video_noisy, + context_video_noisy, + context_mask_video_noisy, + freqs_video_noisy, + f_noisy, + h_noisy, + w_noisy, + noisy_video_tokens_per_frame, + ) = self.video_expert.prepare( + x=latents_noisy, + timestep=timestep_video, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + ( + cond_video_tokens, + _t_video_cond, + t_mod_video_cond, + _context_video_cond, + context_mask_video_cond, + freqs_video_cond, + _f_cond, + _h_cond, + _w_cond, + cond_video_tokens_per_frame, + ) = self.video_expert.prepare( + x=latents_cond, + timestep=timestep_video_cond, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + ( + action_tokens, + _t_action, + t_mod_action, + context_action, + context_mask_action, + freqs_action, + ) = self.action_expert.prepare( + action_tokens=noisy_action, + timestep=timestep_action, + context=context, + context_mask=context_mask, + ) + + noisy_video_seq_len = int(noisy_video_tokens.shape[1]) + cond_video_seq_len = int(cond_video_tokens.shape[1]) + merged_video_tokens = torch.cat([noisy_video_tokens, cond_video_tokens], dim=1) + merged_video_freqs = torch.cat([freqs_video_noisy, freqs_video_cond], dim=0) + merged_video_t_mod = torch.cat([t_mod_video_noisy, t_mod_video_cond], dim=1) + merged_video_context_mask = torch.cat( + [context_mask_video_noisy, context_mask_video_cond], dim=1 + ) + attention_mask = self._build_teacher_forcing_attention_mask( + noisy_video_seq_len=noisy_video_seq_len, + cond_video_seq_len=cond_video_seq_len, + action_seq_len=action_tokens.shape[1], + noisy_video_tokens_per_frame=noisy_video_tokens_per_frame, + cond_video_tokens_per_frame=cond_video_tokens_per_frame, + batch_size=action_tokens.shape[0], + device=merged_video_tokens.device, + ) + + merged_video_out, action_out = self.mot.forward_joint_core( + video_tokens=merged_video_tokens, + action_tokens=action_tokens, + video_freqs=merged_video_freqs, + action_freqs=freqs_action, + video_t_mod=merged_video_t_mod, + action_t_mod=t_mod_action, + video_context=context_video_noisy, + video_context_mask=merged_video_context_mask, + action_context=context_action, + action_context_mask=context_mask_action, + attention_mask=attention_mask, + ) + + pred_video_tokens = merged_video_out[:, :noisy_video_seq_len] + pred_video = self.video_expert.post( + pred_video_tokens, + t_video_noisy, + f_noisy, + h_noisy, + w_noisy, + ) + pred_action = self.action_expert.post(action_out) + return pred_video, pred_action + + def training_loss(self, sample, tiled: bool = False): + inputs = self.build_inputs(sample, tiled=tiled) + input_latents = inputs["input_latents"] + batch_size = input_latents.shape[0] + context = inputs["context"] + context_mask = inputs["context_mask"] + action = inputs["action"] + action_is_pad = inputs["action_is_pad"] + image_is_pad = inputs["image_is_pad"] + fuse_flag = inputs["fuse_vae_embedding_in_latents"] + if not bool(getattr(self.video_expert, "seperated_timestep", False)) or not fuse_flag: + raise ValueError( + "Teacher-forcing requires token-wise `t_mod`; " + "ensure `seperated_timestep=true` and `fuse_vae_embedding_in_latents=true`." + ) + + # Branch A: noisy video (for video denoising target). + noise_video = torch.randn_like(input_latents) + timestep_video = self.train_video_scheduler.sample_training_t( + batch_size=batch_size, + device=self.device, + dtype=input_latents.dtype, + ) + latents_noisy = self.train_video_scheduler.add_noise(input_latents, noise_video, timestep_video) + target_video = self.train_video_scheduler.training_target(input_latents, noise_video, timestep_video) + if inputs["first_frame_latents"] is not None: + latents_noisy[:, :, 0:1] = inputs["first_frame_latents"] + + # Branch B: noisy action. + noise_action = torch.randn_like(action) + timestep_action = self.train_action_scheduler.sample_training_t( + batch_size=batch_size, + device=self.device, + dtype=action.dtype, + ) + noisy_action = self.train_action_scheduler.add_noise(action, noise_action, timestep_action) + target_action = self.train_action_scheduler.training_target(action, noise_action, timestep_action) + + # Branch C: teacher-forcing cond-video. + # Each sample is independently noised with probability `video_cond_noise_prob`. + cond_noise_mask = torch.rand((batch_size,), device=self.device) < float(self.video_cond_noise_prob) + timestep_video_cond = torch.zeros_like(timestep_video, dtype=input_latents.dtype, device=self.device) + latents_cond = input_latents + if bool(cond_noise_mask.any()): + timestep_video_cond_sampled = self.train_video_scheduler.sample_training_t( + batch_size=batch_size, + device=self.device, + dtype=input_latents.dtype, + ) + timestep_video_cond = torch.where(cond_noise_mask, timestep_video_cond_sampled, timestep_video_cond) + noise_video_cond = torch.randn_like(input_latents) + latents_cond_noisy = self.train_video_scheduler.add_noise( + input_latents, noise_video_cond, timestep_video_cond_sampled + ) + cond_noise_selector = cond_noise_mask.view(batch_size, 1, 1, 1, 1) + latents_cond = torch.where(cond_noise_selector, latents_cond_noisy, input_latents) + if inputs["first_frame_latents"] is not None: + latents_cond = latents_cond.clone() + latents_cond[:, :, 0:1] = inputs["first_frame_latents"] + + pred_video, pred_action = self._teacher_forcing_training_denoise_core( + latents_noisy=latents_noisy, + latents_cond=latents_cond, + noisy_action=noisy_action, + timestep_video=timestep_video, + timestep_video_cond=timestep_video_cond, + timestep_action=timestep_action, + context=context, + context_mask=context_mask, + fuse_vae_embedding_in_latents=fuse_flag, + ) + + include_initial_video_step = inputs["first_frame_latents"] is None + if inputs["first_frame_latents"] is not None: + pred_video = pred_video[:, :, 1:] + target_video = target_video[:, :, 1:] + + loss_video_per_sample = self._compute_video_loss_per_sample( + pred_video=pred_video, + target_video=target_video, + image_is_pad=image_is_pad, + include_initial_video_step=include_initial_video_step, + ) + video_weight = self.train_video_scheduler.training_weight(timestep_video).to( + loss_video_per_sample.device, dtype=loss_video_per_sample.dtype + ) + loss_video = (loss_video_per_sample * video_weight).mean() + + action_loss_token = F.mse_loss(pred_action.float(), target_action.float(), reduction="none").mean(dim=2) + if action_is_pad is not None: + valid = (~action_is_pad).to(device=action_loss_token.device, dtype=action_loss_token.dtype) + valid_sum = valid.sum(dim=1).clamp(min=1.0) + action_loss_per_sample = (action_loss_token * valid).sum(dim=1) / valid_sum + else: + action_loss_per_sample = action_loss_token.mean(dim=1) + + action_weight = self.train_action_scheduler.training_weight(timestep_action).to( + action_loss_per_sample.device, dtype=action_loss_per_sample.dtype + ) + loss_action = (action_loss_per_sample * action_weight).mean() + + loss_total = self.loss_lambda_video * loss_video + self.loss_lambda_action * loss_action + loss_dict = { + "loss_video": self.loss_lambda_video * float(loss_video.detach().item()), + "loss_action": self.loss_lambda_action * float(loss_action.detach().item()), + } + return loss_total, loss_dict + + def _denoise_video( + self, + latents_video: torch.Tensor, + timestep_video: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + video_self_attn_mask: Optional[torch.Tensor], + fuse_vae_embedding_in_latents: bool, + ) -> torch.Tensor: + x_tokens, t, t_mod, context_emb, context_attn_mask, freqs, f, h, w, _ = ( + self.video_expert.prepare( + x=latents_video, + timestep=timestep_video, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + ) + for block in self.video_expert.blocks: + x_tokens = block( + x_tokens, + context_emb, + t_mod, + freqs, + context_mask=context_attn_mask, + self_attn_mask=video_self_attn_mask, + ) + x = self.video_expert.head(x_tokens, t) + return self.video_expert.unpatchify(x, (f, h, w)) + + @torch.no_grad() + def infer_joint( + self, + prompt: Optional[str], + input_image: torch.Tensor, + num_video_frames: int, + action_horizon: int, + action: Optional[torch.Tensor] = None, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + test_action_with_infer_action: bool = True, + compile_action_infer: bool = False, + ) -> dict[str, Any]: + del test_action_with_infer_action + if action is not None: + logger.warning( + "`FastWAMIDM.infer_joint` ignores `action` input; " + "video is denoised in a standalone first stage." + ) + + out = self.infer_action( + prompt=prompt, + input_image=input_image, + num_video_frames=num_video_frames, + action_horizon=action_horizon, + proprio=proprio, + context=context, + context_mask=context_mask, + negative_prompt=negative_prompt, + text_cfg_scale=text_cfg_scale, + num_inference_steps=num_inference_steps, + sigma_shift=sigma_shift, + seed=seed, + rand_device=rand_device, + tiled=tiled, + compile_action_infer=compile_action_infer, + ) + return { + "video": self._decode_latents(out["video_latents"], tiled=tiled), + "action": out["action"], + } + + @torch.no_grad() + def infer_action( + self, + prompt: Optional[str], + input_image: torch.Tensor, + action_horizon: int, + num_video_frames: int, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + compile_action_infer: bool = False, + ) -> dict[str, Any]: + del negative_prompt, text_cfg_scale + self.eval() + + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + _, _, height, width = input_image.shape + checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_video_frames) + if (checked_h, checked_w) != (height, width): + raise ValueError( + f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})" + ) + if checked_t != num_video_frames: + raise ValueError( + f"`num_video_frames` must satisfy T % 4 == 1, got {num_video_frames}" + ) + + if proprio is not None: + if self.proprio_dim is None: + raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.") + if proprio.ndim == 1: + proprio = proprio.unsqueeze(0) + elif proprio.ndim == 2 and proprio.shape[0] == 1: + pass + else: + raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}") + if proprio.shape[1] != self.proprio_dim: + raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}") + proprio = proprio.to(device=self.device, dtype=self.torch_dtype) + + latent_t = (num_video_frames - 1) // self.vae.temporal_downsample_factor + 1 + latent_h = height // self.vae.upsampling_factor + latent_w = width // self.vae.upsampling_factor + + video_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + action_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + latents_video = torch.randn( + (1, self.vae.model.z_dim, latent_t, latent_h, latent_w), + generator=video_generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + latents_action = torch.randn( + (1, action_horizon, self.action_expert.action_dim), + generator=action_generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + + input_image = input_image.to(device=self.device, dtype=self.torch_dtype) + first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled) + latents_video[:, :, 0:1] = first_frame_latents.clone() + fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False)) + + use_prompt = prompt is not None + use_context = context is not None or context_mask is not None + if use_prompt and use_context: + raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.") + if not use_prompt and not use_context: + raise ValueError("Either `prompt` or both `context/context_mask` must be provided.") + + if use_prompt: + context, context_mask = self.encode_prompt(prompt) + else: + if context is None or context_mask is None: + raise ValueError("`context` and `context_mask` must be both provided together.") + if context.ndim == 2: + context = context.unsqueeze(0) + if context_mask.ndim == 1: + context_mask = context_mask.unsqueeze(0) + if context.ndim != 3 or context_mask.ndim != 2: + raise ValueError( + f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}" + ) + context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True) + if proprio is not None: + context, context_mask = self._append_proprio_to_context( + context=context, + context_mask=context_mask, + proprio=proprio, + ) + + patch_t, patch_h, patch_w = (int(value) for value in self.video_expert.patch_size) + video_tokens_per_frame = (latent_h // patch_h) * (latent_w // patch_w) + video_seq_len = (latent_t // patch_t) * video_tokens_per_frame + video_self_attn_mask = self.video_expert.build_video_to_video_mask( + video_seq_len=video_seq_len, + video_tokens_per_frame=video_tokens_per_frame, + device=self.device, + ) + if compile_action_infer: + if not hasattr(self, "_denoise_video_compiled"): + self._denoise_video_compiled = torch.compile( + self._denoise_video, + fullgraph=True, + ) + denoise_video = self._denoise_video_compiled + else: + denoise_video = self._denoise_video + + # Stage 1: denoise video only. + infer_timesteps_video, infer_deltas_video = self.infer_video_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_video.dtype, + shift_override=sigma_shift, + ) + for step_t_video, step_delta_video in zip(infer_timesteps_video, infer_deltas_video): + timestep_video = step_t_video.unsqueeze(0).to(dtype=latents_video.dtype, device=self.device) + pred_video = denoise_video( + latents_video=latents_video, + timestep_video=timestep_video, + context=context, + context_mask=context_mask, + video_self_attn_mask=video_self_attn_mask, + fuse_vae_embedding_in_latents=fuse_flag, + ) + latents_video = self.infer_video_scheduler.step(pred_video, step_delta_video, latents_video) + latents_video[:, :, 0:1] = first_frame_latents.clone() + + # Stage 2: freeze denoised video as cond and denoise action via video K/V cache. + timestep_video_cond = torch.zeros( + (latents_video.shape[0],), dtype=latents_video.dtype, device=self.device + ) + ( + video_cond_tokens, + _t_video_cond, + video_cond_t_mod, + video_cond_context, + video_cond_context_mask, + video_cond_freqs, + _f_cond, + _h_cond, + _w_cond, + video_tokens_per_frame, + ) = self.video_expert.prepare( + x=latents_video, + timestep=timestep_video_cond, + context=context, + context_mask=context_mask, + action=None, + fuse_vae_embedding_in_latents=fuse_flag, + ) + video_seq_len = int(video_cond_tokens.shape[1]) + attention_mask = self._build_mot_attention_mask( + video_seq_len=video_seq_len, + action_seq_len=latents_action.shape[1], + video_tokens_per_frame=video_tokens_per_frame, + device=video_cond_tokens.device, + ) + video_attention_mask = attention_mask[:video_seq_len, :video_seq_len] + action_attention_mask = attention_mask[video_seq_len:, :] + if compile_action_infer: + if not hasattr(self, "_prefill_video_cache_compiled"): + self._prefill_video_cache_compiled = torch.compile( + self.mot.prefill_video_cache_tensor, + fullgraph=True, + ) + if not hasattr(self, "_denoise_action_with_video_cache_compiled"): + self._denoise_action_with_video_cache_compiled = torch.compile( + self._denoise_action_with_video_cache, + fullgraph=True, + ) + prefill_video_cache = self._prefill_video_cache_compiled + denoise_action_with_video_cache = self._denoise_action_with_video_cache_compiled + else: + prefill_video_cache = self.mot.prefill_video_cache_tensor + denoise_action_with_video_cache = self._denoise_action_with_video_cache + video_cache_k, video_cache_v = prefill_video_cache( + video_tokens=video_cond_tokens, + video_freqs=video_cond_freqs, + video_t_mod=video_cond_t_mod, + video_context=video_cond_context, + video_context_mask=video_cond_context_mask, + video_attention_mask=video_attention_mask, + ) + if compile_action_infer: + video_cache_k = [cache.clone() for cache in video_cache_k] + video_cache_v = [cache.clone() for cache in video_cache_v] + + infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_action.dtype, + shift_override=sigma_shift, + ) + for step_t_action, step_delta_action in zip(infer_timesteps_action, infer_deltas_action): + timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device) + pred_action = denoise_action_with_video_cache( + latents_action=latents_action, + timestep_action=timestep_action, + context=context, + context_mask=context_mask, + video_cache_k=video_cache_k, + video_cache_v=video_cache_v, + action_attention_mask=action_attention_mask, + ) + latents_action = self.infer_action_scheduler.step(pred_action, step_delta_action, latents_action) + + return { + "video_latents": latents_video, + "action": latents_action[0].detach().to(device="cpu", dtype=torch.float32), + } diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/fastwam_joint.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/fastwam_joint.py new file mode 100644 index 0000000000000000000000000000000000000000..66d2d59f9555a0c4f60b43c8034413f80d1f7dc5 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/fastwam_joint.py @@ -0,0 +1,240 @@ +from typing import Any, Optional + +import torch + +from fastwam.utils.logging_config import get_logger + +from .fastwam import FastWAM + +logger = get_logger(__name__) + + +class FastWAMJoint(FastWAM): + """FastWAM variant where action attends to all video latent tokens.""" + + @classmethod + def from_wan22_pretrained(cls, **kwargs): + video_dit_config = kwargs.get("video_dit_config", None) + if not isinstance(video_dit_config, dict): + raise ValueError( + "`video_dit_config` must be provided as dict for FastWAMJoint." + ) + if bool(video_dit_config.get("action_conditioned", False)): + raise ValueError( + "FastWAMJoint requires `video_dit_config['action_conditioned']=false`." + ) + return super().from_wan22_pretrained(**kwargs) + + @torch.no_grad() + def _build_mot_attention_mask( + self, + video_seq_len: int, + action_seq_len: int, + video_tokens_per_frame: int, + device: torch.device, + ) -> torch.Tensor: + total_seq_len = video_seq_len + action_seq_len + mask = torch.zeros((total_seq_len, total_seq_len), dtype=torch.bool, device=device) + + # video -> video + mask[:video_seq_len, :video_seq_len] = self.video_expert.build_video_to_video_mask( + video_seq_len=video_seq_len, + video_tokens_per_frame=video_tokens_per_frame, + device=device, + ) + # action -> action + mask[video_seq_len:, video_seq_len:] = True + # action -> full video + mask[video_seq_len:, :video_seq_len] = True + return mask + + @torch.no_grad() + def infer_joint( + self, + prompt: Optional[str], + input_image: torch.Tensor, + num_video_frames: int, + action_horizon: int, + action: Optional[torch.Tensor] = None, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + test_action_with_infer_action: bool = True, + compile_action_infer: bool = False, + ) -> dict[str, Any]: + del compile_action_infer + if test_action_with_infer_action: + logger.warning( + "`FastWAMJoint.infer_joint` ignores `test_action_with_infer_action=True` " + "and always runs with `test_action_with_infer_action=False`." + ) + return super().infer_joint( + prompt=prompt, + input_image=input_image, + num_video_frames=num_video_frames, + action_horizon=action_horizon, + action=action, + proprio=proprio, + context=context, + context_mask=context_mask, + negative_prompt=negative_prompt, + text_cfg_scale=text_cfg_scale, + num_inference_steps=num_inference_steps, + sigma_shift=sigma_shift, + seed=seed, + rand_device=rand_device, + tiled=tiled, + test_action_with_infer_action=False, + ) + + @torch.no_grad() + def infer_action( + self, + prompt: Optional[str], + input_image: torch.Tensor, + action_horizon: int, + num_video_frames: int, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + compile_action_infer: bool = False, + ) -> dict[str, Any]: + del compile_action_infer + self.eval() + + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + _, _, height, width = input_image.shape + checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_video_frames) + if (checked_h, checked_w) != (height, width): + raise ValueError( + f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})" + ) + if checked_t != num_video_frames: + raise ValueError( + f"`num_video_frames` must satisfy T % 4 == 1, got {num_video_frames}" + ) + + if proprio is not None: + if self.proprio_dim is None: + raise ValueError("`proprio` was provided but `proprio_dim=None` so `proprio_encoder` is disabled.") + if proprio.ndim == 1: + proprio = proprio.unsqueeze(0) + elif proprio.ndim == 2 and proprio.shape[0] == 1: + pass + else: + raise ValueError(f"`proprio` must be [D] or [1,D], got shape {tuple(proprio.shape)}") + if proprio.shape[1] != self.proprio_dim: + raise ValueError(f"`proprio` last dim must be {self.proprio_dim}, got {proprio.shape[1]}") + proprio = proprio.to(device=self.device, dtype=self.torch_dtype) + + latent_t = (num_video_frames - 1) // self.vae.temporal_downsample_factor + 1 + latent_h = height // self.vae.upsampling_factor + latent_w = width // self.vae.upsampling_factor + + video_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + action_generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + latents_video = torch.randn( + (1, self.vae.model.z_dim, latent_t, latent_h, latent_w), + generator=video_generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + latents_action = torch.randn( + (1, action_horizon, self.action_expert.action_dim), + generator=action_generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + + input_image = input_image.to(device=self.device, dtype=self.torch_dtype) + first_frame_latents = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled) + latents_video[:, :, 0:1] = first_frame_latents.clone() + fuse_flag = bool(getattr(self.video_expert, "fuse_vae_embedding_in_latents", False)) + + use_prompt = prompt is not None + use_context = context is not None or context_mask is not None + if use_prompt and use_context: + raise ValueError("`prompt` and `context/context_mask` are mutually exclusive.") + if not use_prompt and not use_context: + raise ValueError("Either `prompt` or both `context/context_mask` must be provided.") + + if use_prompt: + context, context_mask = self.encode_prompt(prompt) + else: + if context is None or context_mask is None: + raise ValueError("`context` and `context_mask` must be both provided together.") + if context.ndim == 2: + context = context.unsqueeze(0) + if context_mask.ndim == 1: + context_mask = context_mask.unsqueeze(0) + if context.ndim != 3 or context_mask.ndim != 2: + raise ValueError( + f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}" + ) + context = context.to(device=self.device, dtype=self.torch_dtype, non_blocking=True) + context_mask = context_mask.to(device=self.device, dtype=torch.bool, non_blocking=True) + if proprio is not None: + context, context_mask = self._append_proprio_to_context( + context=context, + context_mask=context_mask, + proprio=proprio, + ) + + infer_timesteps_video, infer_deltas_video = self.infer_video_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_video.dtype, + shift_override=sigma_shift, + ) + infer_timesteps_action, infer_deltas_action = self.infer_action_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents_action.dtype, + shift_override=sigma_shift, + ) + for step_t_video, step_delta_video, step_t_action, step_delta_action in zip( + infer_timesteps_video, + infer_deltas_video, + infer_timesteps_action, + infer_deltas_action, + ): + timestep_video = step_t_video.unsqueeze(0).to(dtype=latents_video.dtype, device=self.device) + timestep_action = step_t_action.unsqueeze(0).to(dtype=latents_action.dtype, device=self.device) + + pred_video_posi, pred_action_posi = self._predict_joint_noise( + latents_video=latents_video, + latents_action=latents_action, + timestep_video=timestep_video, + timestep_action=timestep_action, + context=context, + context_mask=context_mask, + fuse_vae_embedding_in_latents=fuse_flag, + gt_action=None, + ) + + latents_video = self.infer_video_scheduler.step(pred_video_posi, step_delta_video, latents_video) + latents_action = self.infer_action_scheduler.step(pred_action_posi, step_delta_action, latents_action) + latents_video[:, :, 0:1] = first_frame_latents.clone() + + return { + "action": latents_action[0].detach().to(device="cpu", dtype=torch.float32), + } diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/fastwam_optional_idm.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/fastwam_optional_idm.py new file mode 100644 index 0000000000000000000000000000000000000000..62a16576512e4a67f714f83145fc71172f10d6c1 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/fastwam_optional_idm.py @@ -0,0 +1,120 @@ +from typing import Any, Optional + +import torch + +from .fastwam import FastWAM +from .fastwam_idm import FastWAMIDM + + +class FastWAMOptionalIDM(FastWAMIDM): + """FastWAM variant where full-video IDM conditioning is optional.""" + + action_idm_prob: float + + @classmethod + def from_wan22_pretrained(cls, *, action_idm_prob: float, **kwargs): + prob = float(action_idm_prob) + if not 0.0 <= prob <= 1.0: + raise ValueError(f"`action_idm_prob` must be in [0, 1], got {prob}.") + model = super().from_wan22_pretrained(**kwargs) + model.action_idm_prob = prob + return model + + @torch.no_grad() + def _build_teacher_forcing_attention_mask( + self, + noisy_video_seq_len: int, + cond_video_seq_len: int, + action_seq_len: int, + noisy_video_tokens_per_frame: int, + cond_video_tokens_per_frame: int, + batch_size: int, + device: torch.device, + ) -> torch.Tensor: + full_cond_mask = super()._build_teacher_forcing_attention_mask( + noisy_video_seq_len=noisy_video_seq_len, + cond_video_seq_len=cond_video_seq_len, + action_seq_len=action_seq_len, + noisy_video_tokens_per_frame=noisy_video_tokens_per_frame, + cond_video_tokens_per_frame=cond_video_tokens_per_frame, + batch_size=batch_size, + device=device, + ) + + noisy_end = noisy_video_seq_len + cond_end = noisy_video_seq_len + cond_video_seq_len + first_frame_tokens = min(cond_video_tokens_per_frame, cond_video_seq_len) + first_frame_cond_mask = full_cond_mask.clone() + first_frame_cond_mask[cond_end:, noisy_end:cond_end] = False + first_frame_cond_mask[ + cond_end:, + noisy_end : noisy_end + first_frame_tokens, + ] = True + + use_idm_mask = ( + torch.rand((batch_size, 1, 1, 1), device=device) < self.action_idm_prob + ) + return torch.where(use_idm_mask, full_cond_mask, first_frame_cond_mask) + + @torch.no_grad() + def infer_action( + self, + prompt: Optional[str], + input_image: torch.Tensor, + action_horizon: int, + num_video_frames: Optional[int] = None, + proprio: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + compile_action_infer: bool = False, + action_infer_mode: str = "idm", + ) -> dict[str, Any]: + if action_infer_mode == "idm": + if num_video_frames is None: + raise ValueError("`num_video_frames` is required for `action_infer_mode='idm'`.") + return FastWAMIDM.infer_action( + self, + prompt=prompt, + input_image=input_image, + action_horizon=action_horizon, + num_video_frames=num_video_frames, + proprio=proprio, + context=context, + context_mask=context_mask, + negative_prompt=negative_prompt, + text_cfg_scale=text_cfg_scale, + num_inference_steps=num_inference_steps, + sigma_shift=sigma_shift, + seed=seed, + rand_device=rand_device, + tiled=tiled, + compile_action_infer=compile_action_infer, + ) + + if action_infer_mode == "first_frame": + return FastWAM.infer_action( + self, + prompt=prompt, + input_image=input_image, + action_horizon=action_horizon, + proprio=proprio, + context=context, + context_mask=context_mask, + negative_prompt=negative_prompt, + text_cfg_scale=text_cfg_scale, + num_inference_steps=num_inference_steps, + sigma_shift=sigma_shift, + seed=seed, + rand_device=rand_device, + tiled=tiled, + compile_action_infer=compile_action_infer, + ) + + raise ValueError("`action_infer_mode` must be one of: idm, first_frame.") diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/__init__.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..35bac73c6b329a83d34b02e39a12012305879347 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/__init__.py @@ -0,0 +1,15 @@ +from .io import ModelConfig, hash_model_file, load_state_dict +from .state_dict_converters import ( + wan_video_dit_from_diffusers, + wan_video_dit_state_dict_converter, + wan_video_vae_state_dict_converter, +) + +__all__ = [ + "ModelConfig", + "hash_model_file", + "load_state_dict", + "wan_video_dit_from_diffusers", + "wan_video_dit_state_dict_converter", + "wan_video_vae_state_dict_converter", +] diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/gradient.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/gradient.py new file mode 100644 index 0000000000000000000000000000000000000000..636ef5c2e29a96553bfc6d21e47c70506a3831f4 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/gradient.py @@ -0,0 +1,25 @@ +import torch + + +def create_custom_forward(module): + def custom_forward(*inputs, **kwargs): + return module(*inputs, **kwargs) + return custom_forward + + +def gradient_checkpoint_forward( + model, + use_gradient_checkpointing, + *args, + **kwargs, +): + if use_gradient_checkpointing: + model_output = torch.utils.checkpoint.checkpoint( + create_custom_forward(model), + *args, + **kwargs, + use_reentrant=False, + ) + else: + model_output = model(*args, **kwargs) + return model_output diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/io.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/io.py new file mode 100644 index 0000000000000000000000000000000000000000..26d65d08cfa52a40c6e41d4e6b1d9fd4b65843b6 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/io.py @@ -0,0 +1,207 @@ +import glob +import hashlib +import os +from dataclasses import dataclass +from typing import Dict, Optional, Union + +import torch +from safetensors import safe_open + + +@dataclass +class ModelConfig: + path: Union[str, list[str], None] = None + model_id: Optional[str] = None + origin_file_pattern: Union[str, list[str], None] = None + download_source: Optional[str] = None + local_model_path: Optional[str] = None + skip_download: Optional[bool] = None + state_dict: Optional[Dict[str, torch.Tensor]] = None + + def check_input(self): + if self.path is None and self.model_id is None: + raise ValueError("ModelConfig requires either `path` or (`model_id`, `origin_file_pattern`).") + + def parse_original_file_pattern(self): + if self.origin_file_pattern in [None, "", "./"]: + return "*" + if isinstance(self.origin_file_pattern, list): + return self.origin_file_pattern + if self.origin_file_pattern.endswith("/"): + return self.origin_file_pattern + "*" + return self.origin_file_pattern + + def parse_download_source(self): + if self.download_source is not None: + return self.download_source + env = os.environ.get("DIFFSYNTH_DOWNLOAD_SOURCE") + return env if env is not None else "modelscope" + + def parse_skip_download(self): + if self.skip_download is not None: + return self.skip_download + env = os.environ.get("DIFFSYNTH_SKIP_DOWNLOAD") + if env is None: + return False + return env.lower() == "true" + + def reset_local_model_path(self): + if os.environ.get("DIFFSYNTH_MODEL_BASE_PATH") is not None: + self.local_model_path = os.environ.get("DIFFSYNTH_MODEL_BASE_PATH") + elif self.local_model_path is None: + self.local_model_path = "./checkpoints/" + + def require_downloading(self): + if self.path is not None: + return False + origin_file_pattern = self.parse_original_file_pattern() + local_root = os.path.join(self.local_model_path, self.model_id) + if isinstance(origin_file_pattern, list): + all_exist = True + for pattern in origin_file_pattern: + matches = glob.glob(os.path.join(local_root, pattern)) + if len(matches) == 0: + all_exist = False + break + if all_exist: + return False + else: + if len(glob.glob(os.path.join(local_root, origin_file_pattern))) > 0: + return False + return not self.parse_skip_download() + + def download(self): + origin_file_pattern = self.parse_original_file_pattern() + root = os.path.join(self.local_model_path, self.model_id) + downloaded_files = glob.glob(origin_file_pattern, root_dir=root) + download_source = self.parse_download_source().lower() + if download_source == "modelscope": + from modelscope import snapshot_download + + snapshot_download( + self.model_id, + local_dir=root, + allow_file_pattern=origin_file_pattern, + ignore_file_pattern=downloaded_files, + local_files_only=False, + ) + elif download_source == "huggingface": + from huggingface_hub import snapshot_download as hf_snapshot_download + + hf_snapshot_download( + self.model_id, + local_dir=root, + allow_patterns=origin_file_pattern, + ignore_patterns=downloaded_files, + local_files_only=False, + ) + else: + raise ValueError("`download_source` should be `modelscope` or `huggingface`.") + + def download_if_necessary(self): + self.check_input() + self.reset_local_model_path() + if self.require_downloading(): + self.download() + if self.path is None: + if self.origin_file_pattern in [None, "", "./"]: + self.path = os.path.join(self.local_model_path, self.model_id) + else: + matches = glob.glob(os.path.join(self.local_model_path, self.model_id, self.origin_file_pattern)) + matches.sort() + self.path = matches + if isinstance(self.path, list) and len(self.path) == 1: + self.path = self.path[0] + + +def load_state_dict(file_path, torch_dtype=None, device="cpu"): + if isinstance(file_path, list): + state_dict = {} + for file_path_ in file_path: + state_dict.update(load_state_dict(file_path_, torch_dtype=torch_dtype, device=device)) + return state_dict + if file_path.endswith(".safetensors"): + return load_state_dict_from_safetensors(file_path, torch_dtype=torch_dtype, device=device) + return load_state_dict_from_bin(file_path, torch_dtype=torch_dtype, device=device) + + +def load_state_dict_from_safetensors(file_path, torch_dtype=None, device="cpu"): + state_dict = {} + with safe_open(file_path, framework="pt", device=str(device)) as f: + for key in f.keys(): + value = f.get_tensor(key) + if torch_dtype is not None: + value = value.to(torch_dtype) + state_dict[key] = value + return state_dict + + +def load_state_dict_from_bin(file_path, torch_dtype=None, device="cpu"): + state_dict = torch.load(file_path, map_location=device, weights_only=True) + if len(state_dict) == 1: + if "state_dict" in state_dict: + state_dict = state_dict["state_dict"] + elif "module" in state_dict: + state_dict = state_dict["module"] + elif "model_state" in state_dict: + state_dict = state_dict["model_state"] + if torch_dtype is not None: + for key in state_dict: + if isinstance(state_dict[key], torch.Tensor): + state_dict[key] = state_dict[key].to(torch_dtype) + return state_dict + + +def _load_keys_dict_from_safetensors(file_path): + keys_dict = {} + with safe_open(file_path, framework="pt", device="cpu") as f: + for key in f.keys(): + keys_dict[key] = f.get_slice(key).get_shape() + return keys_dict + + +def _convert_state_dict_to_keys_dict(state_dict): + keys_dict = {} + for key, value in state_dict.items(): + if isinstance(value, torch.Tensor): + keys_dict[key] = list(value.shape) + else: + keys_dict[key] = _convert_state_dict_to_keys_dict(value) + return keys_dict + + +def _load_keys_dict_from_bin(file_path): + state_dict = load_state_dict_from_bin(file_path) + return _convert_state_dict_to_keys_dict(state_dict) + + +def _load_keys_dict(file_path): + if isinstance(file_path, list): + merged = {} + for path in file_path: + merged.update(_load_keys_dict(path)) + return merged + if file_path.endswith(".safetensors"): + return _load_keys_dict_from_safetensors(file_path) + return _load_keys_dict_from_bin(file_path) + + +def _convert_keys_dict_to_single_str(keys_dict, with_shape=True): + keys = [] + for key, value in keys_dict.items(): + if isinstance(key, str): + if isinstance(value, dict): + keys.append(key + "|" + _convert_keys_dict_to_single_str(value, with_shape=with_shape)) + else: + if with_shape: + shape = "_".join(map(str, list(value))) + keys.append(key + ":" + shape) + keys.append(key) + keys.sort() + return ",".join(keys) + + +def hash_model_file(path, with_shape=True): + keys_dict = _load_keys_dict(path) + keys_str = _convert_keys_dict_to_single_str(keys_dict, with_shape=with_shape).encode("UTF-8") + return hashlib.md5(keys_str).hexdigest() diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/loader.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/loader.py new file mode 100644 index 0000000000000000000000000000000000000000..99d92ff409fb104c6ebf8cc29905baa12d80e28f --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/loader.py @@ -0,0 +1,221 @@ +from dataclasses import dataclass +import inspect +from typing import Any + +import torch +import time + +from .io import ModelConfig, hash_model_file, load_state_dict +from .state_dict_converters import ( + wan_video_vae_state_dict_converter, +) +from ..wan_video_dit import WanVideoDiT +from ..wan_video_text_encoder import HuggingfaceTokenizer, WanTextEncoder +from ..wan_video_vae import WanVideoVAE38 +from fastwam.utils.logging_config import get_logger + +logger = get_logger(__name__) +SKIPPED_PRETRAIN_SENTINEL = "SKIPPED_PRETRAIN" + + +@dataclass +class Wan22LoadedComponents: + dit: WanVideoDiT + vae: WanVideoVAE38 + text_encoder: WanTextEncoder | None + tokenizer: HuggingfaceTokenizer | None + dit_path: str + vae_path: str + text_encoder_path: str | None + tokenizer_path: str | None + + +WAN22_MODEL_REGISTRY = [ + { + # Example: ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth") + "model_hash": "9c8818c2cbea55eca56c7b447df170da", + "model_name": "wan_video_text_encoder", + "model_class": WanTextEncoder, + }, + { + # Example: ModelConfig(model_id="Wan-AI/Wan2.2-TI2V-5B", origin_file_pattern="diffusion_pytorch_model*.safetensors") + "model_hash": "1f5ab7703c6fc803fdded85ff040c316", + "model_name": "wan_video_dit", + "model_class": WanVideoDiT, + }, + { + # Example: ModelConfig(model_id="Wan-AI/Wan2.2-TI2V-5B", origin_file_pattern="Wan2.2_VAE.pth") + "model_hash": "e1de6c02cdac79f8b739f4d3698cd216", + "model_name": "wan_video_vae", + "model_class": WanVideoVAE38, + "state_dict_converter": wan_video_vae_state_dict_converter, + }, +] + + +def _validate_dit_config(dit_config: dict[str, Any]) -> dict[str, Any]: + if not isinstance(dit_config, dict): + raise ValueError(f"`dit_config` must be a dict, got {type(dit_config)}") + + validated = dict(dit_config) + + signature = inspect.signature(WanVideoDiT.__init__) + allowed_keys = set() + required_keys = set() + for name, param in signature.parameters.items(): + if name == "self": + continue + allowed_keys.add(name) + if param.default is inspect.Signature.empty: + required_keys.add(name) + + unknown_keys = sorted(set(validated) - allowed_keys) + if unknown_keys: + raise ValueError( + f"Unknown keys in `dit_config`: {unknown_keys}. " + f"Allowed keys: {sorted(allowed_keys)}" + ) + + missing_keys = sorted(required_keys - set(validated)) + if missing_keys: + raise ValueError( + f"Missing required keys in `dit_config`: {missing_keys}. " + "Please specify all required WanVideoDiT constructor args." + ) + + return validated + + +def _load_registered_model( + path, + model_name: str, + torch_dtype: torch.dtype, + device: str, + model_kwargs_override: dict[str, Any] | None = None, +): + model_hash = hash_model_file(path) + + matched_config = None + for config in WAN22_MODEL_REGISTRY: + if config["model_hash"] == model_hash and config["model_name"] == model_name: + matched_config = config + break + if matched_config is None: + raise ValueError( + f"Cannot detect model type for {model_name}. File: {path}. " + f"Model hash: {model_hash}. This standalone package follows DiffSynth hash-based loading." + ) + + model_class = matched_config["model_class"] + model_kwargs = dict(matched_config.get("extra_kwargs", {})) + if model_kwargs_override is not None: + model_kwargs.update(model_kwargs_override) + state_dict_converter = matched_config.get("state_dict_converter") + + model = model_class(**model_kwargs) + state_dict = load_state_dict(path, torch_dtype=torch_dtype, device="cpu") + if state_dict_converter is not None: + state_dict = state_dict_converter(state_dict) + + model.load_state_dict(state_dict, strict=False) + model = model.to(device=device, dtype=torch_dtype) + return model + + +def _resolve_configs(model_id: str, tokenizer_model_id: str, redirect_common_files: bool = True): + dit_config = ModelConfig(model_id=model_id, origin_file_pattern="diffusion_pytorch_model*.safetensors") + text_config = ModelConfig(model_id=model_id, origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth") + vae_config = ModelConfig(model_id=model_id, origin_file_pattern="Wan2.2_VAE.pth") + tokenizer_config = ModelConfig(model_id=tokenizer_model_id, origin_file_pattern="google/umt5-xxl/") + + if redirect_common_files: + redirect_dict = { + "models_t5_umt5-xxl-enc-bf16.pth": ("DiffSynth-Studio/Wan-Series-Converted-Safetensors", "models_t5_umt5-xxl-enc-bf16.safetensors"), + "Wan2.2_VAE.pth": ("DiffSynth-Studio/Wan-Series-Converted-Safetensors", "Wan2.2_VAE.safetensors"), + } + text_config.model_id, text_config.origin_file_pattern = redirect_dict[text_config.origin_file_pattern] + vae_config.model_id, vae_config.origin_file_pattern = redirect_dict[vae_config.origin_file_pattern] + return dit_config, text_config, vae_config, tokenizer_config + + +def load_wan22_ti2v_5b_components( + device: str = "cuda", + torch_dtype: torch.dtype = torch.bfloat16, + model_id: str = "Wan-AI/Wan2.2-TI2V-5B", + tokenizer_model_id: str = "Wan-AI/Wan2.1-T2V-1.3B", + tokenizer_max_len: int = 512, + redirect_common_files: bool = True, + dit_config: dict[str, Any] | None = None, + skip_dit_load_from_pretrain: bool = False, + load_text_encoder: bool = True, +): + logger.info("Loading Wan2.2-TI2V-5B components...") + start = time.time() + + if dit_config is None: + raise ValueError("`dit_config` is required for Wan2.2-TI2V-5B loading.") + validated_dit_config = _validate_dit_config(dit_config) + + dit_model_config, text_config, vae_config, tokenizer_config = _resolve_configs( + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + redirect_common_files=redirect_common_files, + ) + + vae_config.download_if_necessary() + if load_text_encoder: + text_config.download_if_necessary() + tokenizer_config.download_if_necessary() + + if skip_dit_load_from_pretrain: + logger.info( + "Skipping pretrained video DiT load (`skip_dit_load_from_pretrain=True`); " + "initializing video expert randomly and expecting checkpoint override." + ) + dit: WanVideoDiT = WanVideoDiT(**validated_dit_config).to(device=device, dtype=torch_dtype) + dit_path = SKIPPED_PRETRAIN_SENTINEL + else: + dit_model_config.download_if_necessary() + dit = _load_registered_model( + dit_model_config.path, + "wan_video_dit", + torch_dtype=torch_dtype, + device=device, + model_kwargs_override=validated_dit_config, + ) + dit_path = str(dit_model_config.path) + text_encoder: WanTextEncoder | None = None + tokenizer: HuggingfaceTokenizer | None = None + text_encoder_path: str | None = None + tokenizer_path: str | None = None + if load_text_encoder: + text_encoder = _load_registered_model( + text_config.path, + "wan_video_text_encoder", + torch_dtype=torch_dtype, + device=device, + ) + tokenizer = HuggingfaceTokenizer( + name=tokenizer_config.path, + seq_len=int(tokenizer_max_len), + clean="whitespace", + ) + text_encoder_path = str(text_config.path) + tokenizer_path = str(tokenizer_config.path) + else: + logger.info( + "Skipping pretrained text encoder/tokenizer load (`load_text_encoder=False`); " + "training must provide cached `context/context_mask`." + ) + vae: WanVideoVAE38 = _load_registered_model(vae_config.path, "wan_video_vae", torch_dtype=torch_dtype, device=device) + logger.info("Finished loading Wan2.2-TI2V-5B components in %.2f seconds.", time.time() - start) + return Wan22LoadedComponents( + dit=dit, + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + dit_path=dit_path, + vae_path=str(vae_config.path), + text_encoder_path=text_encoder_path, + tokenizer_path=tokenizer_path, + ) diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/state_dict_converters.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/state_dict_converters.py new file mode 100644 index 0000000000000000000000000000000000000000..3a83b5c554d79278c3b73859eea092a78f3ec528 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/helpers/state_dict_converters.py @@ -0,0 +1,90 @@ +def wan_video_vae_state_dict_converter(state_dict): + converted = {} + if "model_state" in state_dict: + state_dict = state_dict["model_state"] + for name, value in state_dict.items(): + converted[f"model.{name}"] = value + return converted + + +def wan_video_dit_from_diffusers(state_dict): + rename_dict = { + "blocks.0.attn1.norm_k.weight": "blocks.0.self_attn.norm_k.weight", + "blocks.0.attn1.norm_q.weight": "blocks.0.self_attn.norm_q.weight", + "blocks.0.attn1.to_k.bias": "blocks.0.self_attn.k.bias", + "blocks.0.attn1.to_k.weight": "blocks.0.self_attn.k.weight", + "blocks.0.attn1.to_out.0.bias": "blocks.0.self_attn.o.bias", + "blocks.0.attn1.to_out.0.weight": "blocks.0.self_attn.o.weight", + "blocks.0.attn1.to_q.bias": "blocks.0.self_attn.q.bias", + "blocks.0.attn1.to_q.weight": "blocks.0.self_attn.q.weight", + "blocks.0.attn1.to_v.bias": "blocks.0.self_attn.v.bias", + "blocks.0.attn1.to_v.weight": "blocks.0.self_attn.v.weight", + "blocks.0.attn2.norm_k.weight": "blocks.0.cross_attn.norm_k.weight", + "blocks.0.attn2.norm_q.weight": "blocks.0.cross_attn.norm_q.weight", + "blocks.0.attn2.to_k.bias": "blocks.0.cross_attn.k.bias", + "blocks.0.attn2.to_k.weight": "blocks.0.cross_attn.k.weight", + "blocks.0.attn2.to_out.0.bias": "blocks.0.cross_attn.o.bias", + "blocks.0.attn2.to_out.0.weight": "blocks.0.cross_attn.o.weight", + "blocks.0.attn2.to_q.bias": "blocks.0.cross_attn.q.bias", + "blocks.0.attn2.to_q.weight": "blocks.0.cross_attn.q.weight", + "blocks.0.attn2.to_v.bias": "blocks.0.cross_attn.v.bias", + "blocks.0.attn2.to_v.weight": "blocks.0.cross_attn.v.weight", + "blocks.0.attn2.add_k_proj.bias": "blocks.0.cross_attn.k_img.bias", + "blocks.0.attn2.add_k_proj.weight": "blocks.0.cross_attn.k_img.weight", + "blocks.0.attn2.add_v_proj.bias": "blocks.0.cross_attn.v_img.bias", + "blocks.0.attn2.add_v_proj.weight": "blocks.0.cross_attn.v_img.weight", + "blocks.0.attn2.norm_added_k.weight": "blocks.0.cross_attn.norm_k_img.weight", + "blocks.0.ffn.net.0.proj.bias": "blocks.0.ffn.0.bias", + "blocks.0.ffn.net.0.proj.weight": "blocks.0.ffn.0.weight", + "blocks.0.ffn.net.2.bias": "blocks.0.ffn.2.bias", + "blocks.0.ffn.net.2.weight": "blocks.0.ffn.2.weight", + "blocks.0.norm2.bias": "blocks.0.norm3.bias", + "blocks.0.norm2.weight": "blocks.0.norm3.weight", + "blocks.0.scale_shift_table": "blocks.0.modulation", + "condition_embedder.text_embedder.linear_1.bias": "text_embedding.0.bias", + "condition_embedder.text_embedder.linear_1.weight": "text_embedding.0.weight", + "condition_embedder.text_embedder.linear_2.bias": "text_embedding.2.bias", + "condition_embedder.text_embedder.linear_2.weight": "text_embedding.2.weight", + "condition_embedder.time_embedder.linear_1.bias": "time_embedding.0.bias", + "condition_embedder.time_embedder.linear_1.weight": "time_embedding.0.weight", + "condition_embedder.time_embedder.linear_2.bias": "time_embedding.2.bias", + "condition_embedder.time_embedder.linear_2.weight": "time_embedding.2.weight", + "condition_embedder.time_proj.bias": "time_projection.1.bias", + "condition_embedder.time_proj.weight": "time_projection.1.weight", + "condition_embedder.image_embedder.ff.net.0.proj.bias": "img_emb.proj.1.bias", + "condition_embedder.image_embedder.ff.net.0.proj.weight": "img_emb.proj.1.weight", + "condition_embedder.image_embedder.ff.net.2.bias": "img_emb.proj.3.bias", + "condition_embedder.image_embedder.ff.net.2.weight": "img_emb.proj.3.weight", + "condition_embedder.image_embedder.norm1.bias": "img_emb.proj.0.bias", + "condition_embedder.image_embedder.norm1.weight": "img_emb.proj.0.weight", + "condition_embedder.image_embedder.norm2.bias": "img_emb.proj.4.bias", + "condition_embedder.image_embedder.norm2.weight": "img_emb.proj.4.weight", + "patch_embedding.bias": "patch_embedding.bias", + "patch_embedding.weight": "patch_embedding.weight", + "scale_shift_table": "head.modulation", + "proj_out.bias": "head.head.bias", + "proj_out.weight": "head.head.weight", + } + converted = {} + for name, value in state_dict.items(): + if name in rename_dict: + converted[rename_dict[name]] = value + else: + probe_name = ".".join(name.split(".")[:1] + ["0"] + name.split(".")[2:]) + if probe_name in rename_dict: + mapped = rename_dict[probe_name] + mapped = ".".join(mapped.split(".")[:1] + [name.split(".")[1]] + mapped.split(".")[2:]) + converted[mapped] = value + return converted + + +def wan_video_dit_state_dict_converter(state_dict): + converted = {} + for name, value in state_dict.items(): + if name.startswith("vace"): + continue + if name.split(".")[0] in ["pose_patch_embedding", "face_adapter", "face_encoder", "motion_encoder"]: + continue + key = name[6:] if name.startswith("model.") else name + converted[key] = value + return converted diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/mot.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/mot.py new file mode 100644 index 0000000000000000000000000000000000000000..faa745ba159d2f9f1f4eb0c1c1cab3bd93827a01 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/mot.py @@ -0,0 +1,819 @@ +from __future__ import annotations + +from typing import Dict, Optional + +import torch +import torch.nn as nn + +from .wan_video_dit import flash_attention, modulate, rope_apply +from fastwam.utils.logging_config import get_logger + +logger = get_logger(__name__) + + +class MoT(nn.Module): + def __init__( + self, + mixtures: Dict[str, nn.Module], + mot_checkpoint_mixed_attn: bool = False, + ): + super().__init__() + if not mixtures: + raise ValueError("`mixtures` cannot be empty.") + if "video" not in mixtures or "action" not in mixtures: + raise ValueError("`mixtures` must include both 'video' and 'action' experts.") + + self.mixtures = nn.ModuleDict(mixtures) + self.expert_order = list(self.mixtures.keys()) + if mot_checkpoint_mixed_attn: + raise ValueError("Wan MoT gradient checkpointing is not supported by the compiled tensor core.") + self.mot_checkpoint_mixed_attn = False + self.compile_training_layers = False + + first_expert = self.mixtures[self.expert_order[0]] + self.num_layers = len(first_expert.blocks) + self.num_heads = first_expert.num_heads + self.attn_head_dim = first_expert.attn_head_dim + + for name in self.expert_order[1:]: + expert = self.mixtures[name] + if len(expert.blocks) != self.num_layers: + raise ValueError( + f"All experts must have same number of layers; got {self.num_layers} and {len(expert.blocks)}" + ) + if expert.num_heads != self.num_heads: + raise ValueError( + f"All experts must have same num_heads; got {self.num_heads} and {expert.num_heads}" + ) + if expert.attn_head_dim != self.attn_head_dim: + raise ValueError( + "All experts must have same attn_head_dim; " + f"got {self.attn_head_dim} and {expert.attn_head_dim}" + ) + checkpointed_experts = [ + name + for name in self.expert_order + if bool(getattr(self.mixtures[name], "use_gradient_checkpointing", False)) + ] + if checkpointed_experts: + raise ValueError( + "Wan expert gradient checkpointing is not supported by the compiled tensor core: " + f"{checkpointed_experts}." + ) + + logger.info(f"Initialized MoT with experts: {self.expert_order}, num_layers={self.num_layers}") + for name in self.expert_order: + expert = self.mixtures[name] + logger.info(f" Expert '{name}': num_params={sum(p.numel() for p in expert.parameters()) / 1e9:.2f} B") + + @staticmethod + def _split_modulation(block, t_mod: torch.Tensor): + has_seq = len(t_mod.shape) == 4 + chunk_dim = 2 if has_seq else 1 + + base_mod = block.modulation.to(dtype=t_mod.dtype, device=t_mod.device) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (base_mod + t_mod).chunk(6, dim=chunk_dim) + if has_seq: + # means t_mod has separate modulation for each token, otherwise same modulation for all tokens in the block + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + shift_msa.squeeze(2), + scale_msa.squeeze(2), + gate_msa.squeeze(2), + shift_mlp.squeeze(2), + scale_mlp.squeeze(2), + gate_mlp.squeeze(2), + ) + return shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp + + def _mixed_attention( + self, + q_cat: torch.Tensor, + k_cat: torch.Tensor, + v_cat: torch.Tensor, + attention_mask: torch.Tensor, + ) -> torch.Tensor: + attn_mask = attention_mask.to(device=q_cat.device) + + return flash_attention( + q=q_cat, + k=k_cat, + v=v_cat, + num_heads=self.num_heads, + ctx_mask=attn_mask, + ) + + @staticmethod + def _apply_expert_post_block( + block, + residual_x: torch.Tensor, + mixed_attn_out: torch.Tensor, + gate_msa: torch.Tensor, + shift_mlp: torch.Tensor, + scale_mlp: torch.Tensor, + gate_mlp: torch.Tensor, + context_payload: Optional[dict], + ) -> torch.Tensor: + x = block.gate(residual_x, gate_msa, block.self_attn.o(mixed_attn_out)) + + if context_payload is not None: + context = context_payload.get("context") + if context is not None: + context_mask = context_payload.get("mask") + if context_mask is not None and context_mask.dim() == 3: + context_mask = context_mask.unsqueeze(1) + x = x + block.cross_attn(block.norm3(x), context, ctx_mask=context_mask) + + mlp_input = modulate(block.norm2(x), shift_mlp, scale_mlp) + x = block.gate(x, gate_mlp, block.ffn(mlp_input)) + return x + + @staticmethod + def _apply_expert_post_block_tensor( + block, + residual_x: torch.Tensor, + mixed_attn_out: torch.Tensor, + gate_msa: torch.Tensor, + shift_mlp: torch.Tensor, + scale_mlp: torch.Tensor, + gate_mlp: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + ) -> torch.Tensor: + x = block.gate(residual_x, gate_msa, block.self_attn.o(mixed_attn_out)) + x = x + block.cross_attn( + block.norm3(x), + context, + ctx_mask=context_mask.unsqueeze(1), + ) + mlp_input = modulate(block.norm2(x), shift_mlp, scale_mlp) + return block.gate(x, gate_mlp, block.ffn(mlp_input)) + + def _build_expert_attention_io( + self, + expert, + block, + x: torch.Tensor, + freqs: torch.Tensor, + t_mod: torch.Tensor, + ) -> tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + bool, + ]: + """Build per-expert attention tensors and post-block states. + + Args: + expert: Expert module that owns this `block`; only used to read + `use_gradient_checkpointing`. + block: Transformer block for current layer (`expert.blocks[layer_idx]`). + x: Current expert tokens, shape [B, S, D]. + freqs: RoPE frequencies aligned with token sequence, shape [S, 1, rope_dim]. + t_mod: Time modulation tensor for this expert/layer. + + Returns: + q: Query after q-proj, RMSNorm, and RoPE, shape [B, S, H*Dh]. + k: Key after k-proj, RMSNorm, and RoPE, shape [B, S, H*Dh]. + v: Value after v-proj, shape [B, S, H*Dh]. + residual_x: Original input `x` for residual path in post block. + gate_msa: Gating tensor for self-attention residual branch. + shift_mlp: Shift tensor for MLP modulation. + scale_mlp: Scale tensor for MLP modulation. + gate_mlp: Gating tensor for MLP residual branch. + use_gradient_checkpointing: Whether this expert enables checkpointing. + """ + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self._split_modulation(block, t_mod) + attn_input = modulate(block.norm1(x), shift_msa, scale_msa) + + q = block.self_attn.norm_q(block.self_attn.q(attn_input)) + k = block.self_attn.norm_k(block.self_attn.k(attn_input)) + v = block.self_attn.v(attn_input) + + q = rope_apply(q, freqs, block.num_heads) + k = rope_apply(k, freqs, block.num_heads) + + use_gradient_checkpointing = bool(getattr(expert, "use_gradient_checkpointing", False)) + return ( + q, + k, + v, + x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + use_gradient_checkpointing, + ) + + def _apply_post_with_optional_checkpoint( + self, + block, + residual_x: torch.Tensor, + gate_msa: torch.Tensor, + shift_mlp: torch.Tensor, + scale_mlp: torch.Tensor, + gate_mlp: torch.Tensor, + use_gradient_checkpointing: bool, + mixed_slice: torch.Tensor, + context_payload: Optional[dict], + ) -> torch.Tensor: + """Apply post-attention computations, with optional checkpointing. + + Args: + block: Transformer block for current layer. + residual_x: Residual input tokens before attention update, shape [B, S, D]. + gate_msa: Gating tensor used after mixed self-attention. + shift_mlp: Shift tensor for MLP input modulation. + scale_mlp: Scale tensor for MLP input modulation. + gate_mlp: Gating tensor used after MLP. + use_gradient_checkpointing: If True and training, checkpoint this post block. + mixed_slice: Mixed-attention output for this expert, shape [B, S, H*Dh]. + context_payload: Optional dict for cross-attention. + - `context`: encoder states [B, L, D] + - `mask`: attention mask [B, S, L] or [B, 1, S, L] + + Returns: + Updated expert tokens after self-attn residual, optional cross-attn, and MLP. + """ + def _post_fn( + _mixed_slice: torch.Tensor, + _x: torch.Tensor, + _gate_msa: torch.Tensor, + _shift_mlp: torch.Tensor, + _scale_mlp: torch.Tensor, + _gate_mlp: torch.Tensor, + _block=block, + _context_payload=context_payload, + ) -> torch.Tensor: + return self._apply_expert_post_block( + block=_block, + residual_x=_x, + mixed_attn_out=_mixed_slice, + gate_msa=_gate_msa, + shift_mlp=_shift_mlp, + scale_mlp=_scale_mlp, + gate_mlp=_gate_mlp, + context_payload=_context_payload, + ) + + if use_gradient_checkpointing and self.training: + return torch.utils.checkpoint.checkpoint( + _post_fn, + mixed_slice, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + use_reentrant=False, + ) + return _post_fn( + mixed_slice, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + ) + + def prefill_video_cache( + self, + video_tokens: torch.Tensor, + video_freqs: torch.Tensor, + video_t_mod: torch.Tensor, + video_context_payload: Optional[dict], + video_attention_mask: torch.Tensor, + ) -> list[dict[str, torch.Tensor]]: + """Prefill video branch once and cache per-layer K/V for action denoising. + + Args: + video_tokens: Video tokens before layer 0, shape [B, Sv, D]. + video_freqs: Video RoPE frequencies, shape [Sv, 1, rope_dim]. + video_t_mod: Video time modulation tensor. + video_context_payload: Optional dict for video cross-attention. + - `context`: encoder states [B, L, D] + - `mask`: attention mask [B, Sv, L] or [B, 1, Sv, L] + video_attention_mask: Video self-attention mask, shape [Sv, Sv]. + + Returns: + Layer-wise cache list with length `num_layers`. + Each entry contains: + - `k`: video key tensor [B, Sv, H*Dh] + - `v`: video value tensor [B, Sv, H*Dh] + """ + if "video" not in self.mixtures: + raise ValueError("MoT requires `video` expert for `prefill_video_cache`.") + if video_attention_mask.ndim != 2: + raise ValueError( + f"`video_attention_mask` must be 2D [S,S], got shape {tuple(video_attention_mask.shape)}" + ) + if video_attention_mask.shape[0] != video_attention_mask.shape[1]: + raise ValueError( + f"`video_attention_mask` must be square, got shape {tuple(video_attention_mask.shape)}" + ) + if video_attention_mask.shape[0] != video_tokens.shape[1]: + raise ValueError( + "`video_attention_mask` seq length mismatch: " + f"mask={video_attention_mask.shape[0]} vs tokens={video_tokens.shape[1]}" + ) + + expert = self.mixtures["video"] + x = video_tokens + kv_cache: list[dict[str, torch.Tensor]] = [] + for layer_idx in range(self.num_layers): + block = expert.blocks[layer_idx] + # Build video Q/K/V from current layer input tokens. + ( + q, + k, + v, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + use_gradient_checkpointing, + ) = self._build_expert_attention_io( + expert=expert, + block=block, + x=x, + freqs=video_freqs, + t_mod=video_t_mod, + ) + # Video prefill uses only video self-attention mask. + mixed = self._mixed_attention( + q_cat=q, + k_cat=k, + v_cat=v, + attention_mask=video_attention_mask, + ) + # Update video tokens for the next layer and persist current layer K/V. + x = self._apply_post_with_optional_checkpoint( + block=block, + residual_x=residual_x, + gate_msa=gate_msa, + shift_mlp=shift_mlp, + scale_mlp=scale_mlp, + gate_mlp=gate_mlp, + use_gradient_checkpointing=use_gradient_checkpointing, + mixed_slice=mixed, + context_payload=video_context_payload, + ) + kv_cache.append({"k": k, "v": v}) + return kv_cache + + def forward_action_with_video_cache( + self, + action_tokens: torch.Tensor, + action_freqs: torch.Tensor, + action_t_mod: torch.Tensor, + action_context_payload: Optional[dict], + video_kv_cache: list[dict[str, torch.Tensor]], + attention_mask: torch.Tensor, + video_seq_len: int, + ) -> torch.Tensor: + """Run action branch with cached video K/V instead of recomputing video tokens. + + Args: + action_tokens: Action tokens before layer 0, shape [B, Sa, D]. + action_freqs: Action RoPE frequencies, shape [Sa, 1, rope_dim]. + action_t_mod: Action time modulation tensor. + action_context_payload: Optional dict for action cross-attention. + - `context`: encoder states [B, L, D] + - `mask`: attention mask [B, Sa, L] or [B, 1, Sa, L] + video_kv_cache: Layer-wise cached video K/V from `prefill_video_cache`. + attention_mask: Joint [video+action] mask, shape [Sv+Sa, Sv+Sa]. + video_seq_len: Video token count `Sv` in the joint sequence prefix. + + Returns: + Updated action tokens after all layers, shape [B, Sa, D]. + """ + if "action" not in self.mixtures: + raise ValueError("MoT requires `action` expert for `forward_action_with_video_cache`.") + if len(video_kv_cache) != self.num_layers: + raise ValueError( + f"`video_kv_cache` must contain {self.num_layers} layers, got {len(video_kv_cache)}." + ) + if attention_mask.ndim != 2: + raise ValueError(f"`attention_mask` must be 2D [S,S], got shape {tuple(attention_mask.shape)}") + if attention_mask.shape[0] != attention_mask.shape[1]: + raise ValueError(f"`attention_mask` must be square, got shape {tuple(attention_mask.shape)}") + + action_seq_len = int(action_tokens.shape[1]) + total_seq_len = int(video_seq_len) + action_seq_len + if attention_mask.shape[0] != total_seq_len: + raise ValueError( + "`attention_mask` seq length mismatch: " + f"mask={attention_mask.shape[0]} vs expected_total={total_seq_len}" + ) + # Use the action query rows from the joint [video+action] mask. + action_attention_mask = attention_mask[video_seq_len:total_seq_len, :total_seq_len] + + expert = self.mixtures["action"] + x = action_tokens + for layer_idx in range(self.num_layers): + block = expert.blocks[layer_idx] + # Action query/key/value are still step-dependent and must be recomputed each step. + ( + q_action, + k_action, + v_action, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + use_gradient_checkpointing, + ) = self._build_expert_attention_io( + expert=expert, + block=block, + x=x, + freqs=action_freqs, + t_mod=action_t_mod, + ) + layer_cache = video_kv_cache[layer_idx] + if "k" not in layer_cache or "v" not in layer_cache: + raise ValueError( + f"`video_kv_cache[{layer_idx}]` must contain `k` and `v`." + ) + + k_video = layer_cache["k"] + v_video = layer_cache["v"] + if k_video.shape[1] != video_seq_len or v_video.shape[1] != video_seq_len: + raise ValueError( + f"`video_kv_cache[{layer_idx}]` seq len mismatch, expected {video_seq_len}." + ) + + # Mixed attention: action queries attend to cached video K/V plus current action K/V. + k_cat = torch.cat([k_video, k_action], dim=1) + v_cat = torch.cat([v_video, v_action], dim=1) + mixed = self._mixed_attention( + q_cat=q_action, + k_cat=k_cat, + v_cat=v_cat, + attention_mask=action_attention_mask, + ) + x = self._apply_post_with_optional_checkpoint( + block=block, + residual_x=residual_x, + gate_msa=gate_msa, + shift_mlp=shift_mlp, + scale_mlp=scale_mlp, + gate_mlp=gate_mlp, + use_gradient_checkpointing=use_gradient_checkpointing, + mixed_slice=mixed, + context_payload=action_context_payload, + ) + return x + + def prefill_video_cache_tensor( + self, + video_tokens: torch.Tensor, + video_freqs: torch.Tensor, + video_t_mod: torch.Tensor, + video_context: torch.Tensor, + video_context_mask: torch.Tensor, + video_attention_mask: torch.Tensor, + ) -> tuple[list[torch.Tensor], list[torch.Tensor]]: + expert = self.mixtures["video"] + x = video_tokens + cache_k_list: list[torch.Tensor] = [] + cache_v_list: list[torch.Tensor] = [] + for layer_idx in range(self.num_layers): + block = expert.blocks[layer_idx] + ( + q, + k, + v, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + _use_gradient_checkpointing, + ) = self._build_expert_attention_io( + expert=expert, + block=block, + x=x, + freqs=video_freqs, + t_mod=video_t_mod, + ) + mixed = flash_attention( + q=q, + k=k, + v=v, + num_heads=self.num_heads, + ctx_mask=video_attention_mask.to(device=q.device), + ) + x = self._apply_expert_post_block_tensor( + block=block, + residual_x=residual_x, + mixed_attn_out=mixed, + gate_msa=gate_msa, + shift_mlp=shift_mlp, + scale_mlp=scale_mlp, + gate_mlp=gate_mlp, + context=video_context, + context_mask=video_context_mask, + ) + cache_k_list.append(k) + cache_v_list.append(v) + return cache_k_list, cache_v_list + + def forward_action_with_video_cache_tensor( + self, + action_tokens: torch.Tensor, + action_freqs: torch.Tensor, + action_t_mod: torch.Tensor, + action_context: torch.Tensor, + action_context_mask: torch.Tensor, + video_cache_k: list[torch.Tensor], + video_cache_v: list[torch.Tensor], + action_attention_mask: torch.Tensor, + ) -> torch.Tensor: + expert = self.mixtures["action"] + x = action_tokens + for layer_idx in range(self.num_layers): + block = expert.blocks[layer_idx] + ( + q_action, + k_action, + v_action, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + _use_gradient_checkpointing, + ) = self._build_expert_attention_io( + expert=expert, + block=block, + x=x, + freqs=action_freqs, + t_mod=action_t_mod, + ) + mixed = flash_attention( + q=q_action, + k=torch.cat([video_cache_k[layer_idx], k_action], dim=1), + v=torch.cat([video_cache_v[layer_idx], v_action], dim=1), + num_heads=self.num_heads, + ctx_mask=action_attention_mask.to(device=q_action.device), + ) + x = self._apply_expert_post_block_tensor( + block=block, + residual_x=residual_x, + mixed_attn_out=mixed, + gate_msa=gate_msa, + shift_mlp=shift_mlp, + scale_mlp=scale_mlp, + gate_mlp=gate_mlp, + context=action_context, + context_mask=action_context_mask, + ) + return x + + def _forward_joint_layer( + self, + video_block: nn.Module, + action_block: nn.Module, + video_tokens: torch.Tensor, + action_tokens: torch.Tensor, + video_freqs: torch.Tensor, + action_freqs: torch.Tensor, + video_t_mod: torch.Tensor, + action_t_mod: torch.Tensor, + video_context: torch.Tensor, + video_context_mask: torch.Tensor, + action_context: torch.Tensor, + action_context_mask: torch.Tensor, + attention_mask: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + video_expert = self.mixtures["video"] + action_expert = self.mixtures["action"] + ( + q_video, + k_video, + v_video, + residual_video, + gate_msa_video, + shift_mlp_video, + scale_mlp_video, + gate_mlp_video, + _video_checkpointing, + ) = self._build_expert_attention_io( + expert=video_expert, + block=video_block, + x=video_tokens, + freqs=video_freqs, + t_mod=video_t_mod, + ) + ( + q_action, + k_action, + v_action, + residual_action, + gate_msa_action, + shift_mlp_action, + scale_mlp_action, + gate_mlp_action, + _action_checkpointing, + ) = self._build_expert_attention_io( + expert=action_expert, + block=action_block, + x=action_tokens, + freqs=action_freqs, + t_mod=action_t_mod, + ) + mixed = flash_attention( + q=torch.cat([q_video, q_action], dim=1), + k=torch.cat([k_video, k_action], dim=1), + v=torch.cat([v_video, v_action], dim=1), + num_heads=self.num_heads, + ctx_mask=attention_mask.to(device=q_video.device), + ) + video_seq_len = video_tokens.shape[1] + return ( + self._apply_expert_post_block_tensor( + block=video_block, + residual_x=residual_video, + mixed_attn_out=mixed[:, :video_seq_len], + gate_msa=gate_msa_video, + shift_mlp=shift_mlp_video, + scale_mlp=scale_mlp_video, + gate_mlp=gate_mlp_video, + context=video_context, + context_mask=video_context_mask, + ), + self._apply_expert_post_block_tensor( + block=action_block, + residual_x=residual_action, + mixed_attn_out=mixed[:, video_seq_len:], + gate_msa=gate_msa_action, + shift_mlp=shift_mlp_action, + scale_mlp=scale_mlp_action, + gate_mlp=gate_mlp_action, + context=action_context, + context_mask=action_context_mask, + ), + ) + + def forward_joint_core( + self, + video_tokens: torch.Tensor, + action_tokens: torch.Tensor, + video_freqs: torch.Tensor, + action_freqs: torch.Tensor, + video_t_mod: torch.Tensor, + action_t_mod: torch.Tensor, + video_context: torch.Tensor, + video_context_mask: torch.Tensor, + action_context: torch.Tensor, + action_context_mask: torch.Tensor, + attention_mask: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + if self.training and self.compile_training_layers and not hasattr(self, "_compiled_joint_layer"): + self._compiled_joint_layer = torch.compile( + self._forward_joint_layer, + fullgraph=True, + ) + + x_video = video_tokens + x_action = action_tokens + for layer_idx in range(self.num_layers): + layer = ( + self._compiled_joint_layer + if self.training and self.compile_training_layers + else self._forward_joint_layer + ) + x_video, x_action = layer( + video_block=self.mixtures["video"].blocks[layer_idx], + action_block=self.mixtures["action"].blocks[layer_idx], + video_tokens=x_video, + action_tokens=x_action, + video_freqs=video_freqs, + action_freqs=action_freqs, + video_t_mod=video_t_mod, + action_t_mod=action_t_mod, + video_context=video_context, + video_context_mask=video_context_mask, + action_context=action_context, + action_context_mask=action_context_mask, + attention_mask=attention_mask, + ) + return x_video, x_action + + def forward( + self, + embeds_all: Dict[str, torch.Tensor], + attention_mask: torch.Tensor, + freqs_all: Dict[str, torch.Tensor], + context_all: Dict[str, Optional[dict]], + t_mod_all: Dict[str, torch.Tensor], + ): + missing = [k for k in self.expert_order if k not in embeds_all] + if missing: + raise ValueError(f"Missing expert tokens for {missing}") + missing = [k for k in self.expert_order if k not in freqs_all] + if missing: + raise ValueError(f"Missing expert freqs for {missing}") + missing = [k for k in self.expert_order if k not in t_mod_all] + if missing: + raise ValueError(f"Missing expert t_mod for {missing}") + + if attention_mask.ndim != 2: + raise ValueError(f"`attention_mask` must be 2D [S, S], got shape {tuple(attention_mask.shape)}") + if attention_mask.shape[0] != attention_mask.shape[1]: + raise ValueError(f"`attention_mask` must be square, got shape {tuple(attention_mask.shape)}") + + tokens_all = {k: v for k, v in embeds_all.items()} + + for layer_idx in range(self.num_layers): + q_chunks = [] + k_chunks = [] + v_chunks = [] + cached = {} + seq_lens = [] + + for name in self.expert_order: + expert = self.mixtures[name] + block = expert.blocks[layer_idx] + x = tokens_all[name] + freqs = freqs_all[name] + t_mod = t_mod_all[name] + + ( + q, + k, + v, + residual_x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + use_gradient_checkpointing, + ) = self._build_expert_attention_io( + expert=expert, + block=block, + x=x, + freqs=freqs, + t_mod=t_mod, + ) + + q_chunks.append(q) + k_chunks.append(k) + v_chunks.append(v) + seq_lens.append(x.shape[1]) + cached[name] = { + "block": block, + "residual_x": residual_x, + "gate_msa": gate_msa, + "shift_mlp": shift_mlp, + "scale_mlp": scale_mlp, + "gate_mlp": gate_mlp, + "use_gradient_checkpointing": use_gradient_checkpointing, + } + + # 3. concat all tokens for mixed attention + q_cat = torch.cat(q_chunks, dim=1) + k_cat = torch.cat(k_chunks, dim=1) + v_cat = torch.cat(v_chunks, dim=1) + + total_seq = q_cat.shape[1] + if attention_mask.shape[0] != total_seq: + raise ValueError( + "Attention mask seq length mismatch: " + f"mask={attention_mask.shape[0]} vs tokens={total_seq}" + ) + + mixed = self._mixed_attention(q_cat=q_cat, k_cat=k_cat, v_cat=v_cat, attention_mask=attention_mask) + + start = 0 + for name, seq_len in zip(self.expert_order, seq_lens): + # 4. split mixed attention output and apply post-attention blocks for each expert + end = start + seq_len + mixed_slice = mixed[:, start:end, :] + cached_expert = cached[name] + block = cached_expert["block"] + context_payload = context_all.get(name) + + updated_tokens = self._apply_post_with_optional_checkpoint( + block=block, + residual_x=cached_expert["residual_x"], + gate_msa=cached_expert["gate_msa"], + shift_mlp=cached_expert["shift_mlp"], + scale_mlp=cached_expert["scale_mlp"], + gate_mlp=cached_expert["gate_mlp"], + use_gradient_checkpointing=cached_expert["use_gradient_checkpointing"], + mixed_slice=mixed_slice, + context_payload=context_payload, + ) + + tokens_all[name] = updated_tokens + start = end + + return tokens_all diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/schedulers/scheduler_continuous.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/schedulers/scheduler_continuous.py new file mode 100644 index 0000000000000000000000000000000000000000..9028f6c86e2e35bc9efd75dda26d9cf88ff898bb --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/schedulers/scheduler_continuous.py @@ -0,0 +1,88 @@ +import torch + + +class WanContinuousFlowMatchScheduler: + """Continuous-time Flow-Matching scheduler with shift-based sampling.""" + + def __init__(self, num_train_timesteps: int = 1000, shift: float = 5.0, eps: float = 1e-10): + if num_train_timesteps <= 0: + raise ValueError(f"`num_train_timesteps` must be positive, got {num_train_timesteps}") + if shift <= 0: + raise ValueError(f"`shift` must be positive, got {shift}") + self.num_train_timesteps = int(num_train_timesteps) + self.shift = float(shift) + self.eps = float(eps) + self._y_min, self._weight_norm_const = self._precompute_training_weight_stats() + + @staticmethod + def _phi(u: torch.Tensor, shift: float) -> torch.Tensor: + return shift * u / (1.0 + (shift - 1.0) * u) + + def _precompute_training_weight_stats(self) -> tuple[float, float]: + steps = self.num_train_timesteps + u_grid = torch.linspace(1.0, 0.0, steps + 1, dtype=torch.float64)[:-1] + t_grid = self._phi(u_grid, self.shift) * float(steps) + y_grid = torch.exp(-2.0 * ((t_grid - (steps / 2.0)) / steps) ** 2) + y_min = float(y_grid.min().item()) + y_shifted_grid = y_grid - y_min + norm_const = float(y_shifted_grid.mean().item()) + return y_min, norm_const + + def sample_training_t(self, batch_size: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor: + if batch_size <= 0: + raise ValueError(f"`batch_size` must be positive, got {batch_size}") + u = torch.rand((batch_size,), device=device, dtype=torch.float32) + sigma = self._phi(u, self.shift) + timestep = sigma * float(self.num_train_timesteps) + return timestep.to(dtype=dtype) + + def training_weight(self, timestep: torch.Tensor) -> torch.Tensor: + t = timestep.to(dtype=torch.float32) + steps = float(self.num_train_timesteps) + y = torch.exp(-2.0 * ((t - (steps / 2.0)) / steps) ** 2) + y_shifted = y - self._y_min + weight = y_shifted / (self._weight_norm_const + self.eps) + if weight.numel() == 1: + return weight.reshape(()) + return weight + + def add_noise(self, original_samples: torch.Tensor, noise: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor: + sigma = (timestep / float(self.num_train_timesteps)).to( + original_samples.device, dtype=original_samples.dtype + ) + if sigma.ndim == 0: + return (1 - sigma) * original_samples + sigma * noise + sigma = sigma.view(-1, *([1] * (original_samples.ndim - 1))) + return (1 - sigma) * original_samples + sigma * noise + + @staticmethod + def training_target(sample: torch.Tensor, noise: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor: + del timestep + return noise - sample + + def build_inference_schedule( + self, + num_inference_steps: int, + device: torch.device, + dtype: torch.dtype, + shift_override: float | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if num_inference_steps <= 0: + raise ValueError(f"`num_inference_steps` must be positive, got {num_inference_steps}") + shift = self.shift if shift_override is None else float(shift_override) + if shift <= 0: + raise ValueError(f"`shift` must be positive, got {shift}") + + u_steps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + sigma_steps = self._phi(u_steps, shift) + timesteps = sigma_steps[:-1] * float(self.num_train_timesteps) + deltas = sigma_steps[1:] - sigma_steps[:-1] + return timesteps.to(dtype=dtype), deltas.to(dtype=dtype) + + @staticmethod + def step(model_output: torch.Tensor, delta: torch.Tensor, sample: torch.Tensor) -> torch.Tensor: + delta = delta.to(sample.device, dtype=sample.dtype) + if delta.ndim == 0: + return sample + model_output * delta + delta = delta.view(-1, *([1] * (sample.ndim - 1))) + return sample + model_output * delta diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/wan22.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/wan22.py new file mode 100644 index 0000000000000000000000000000000000000000..e50978ec3f44fc343abb81bf990d76ee6417c7a3 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/wan22.py @@ -0,0 +1,399 @@ +import numpy as np +import os +import torch +import torch.nn.functional as F +from PIL import Image +from typing import Any, Optional, Sequence, Union + +from .helpers.loader import load_wan22_ti2v_5b_components +from .schedulers.scheduler_continuous import WanContinuousFlowMatchScheduler +from .wan_video_dit import WanVideoDiT + + +class Wan22Core(torch.nn.Module): + """Standalone Wan2.2-TI2V-5B core without pipeline unit graph.""" + + def __init__( + self, + dit: WanVideoDiT, + vae, + text_encoder, + tokenizer, + device="cpu", + torch_dtype=torch.float32, + train_shift: float = 5.0, + infer_shift: float = 5.0, + num_train_timesteps: int = 1000, + ): + super().__init__() + self.dit = dit + self.vae = vae + self.text_encoder = text_encoder + self.tokenizer = tokenizer + self.train_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=num_train_timesteps, + shift=train_shift, + ) + self.infer_scheduler = WanContinuousFlowMatchScheduler( + num_train_timesteps=num_train_timesteps, + shift=infer_shift, + ) + self.device = torch.device(device) + self.torch_dtype = torch_dtype + self.to(self.device) + + @classmethod + def from_wan22_pretrained( + cls, + device="cuda", + torch_dtype=torch.bfloat16, + model_id="Wan-AI/Wan2.2-TI2V-5B", + tokenizer_model_id="Wan-AI/Wan2.1-T2V-1.3B", + tokenizer_max_len: int = 512, + redirect_common_files=True, + dit_config: dict[str, Any] | None = None, + train_shift: float = 5.0, + infer_shift: float = 5.0, + num_train_timesteps: int = 1000, + ): + if dit_config is None: + raise ValueError("`dit_config` is required for Wan22Core.from_wan22_pretrained().") + components = load_wan22_ti2v_5b_components( + device=device, + torch_dtype=torch_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + tokenizer_max_len=tokenizer_max_len, + redirect_common_files=redirect_common_files, + dit_config=dit_config, + ) + model = cls( + dit=components.dit, + vae=components.vae, + text_encoder=components.text_encoder, + tokenizer=components.tokenizer, + device=device, + torch_dtype=torch_dtype, + train_shift=train_shift, + infer_shift=infer_shift, + num_train_timesteps=num_train_timesteps, + ) + model.model_paths = { + "dit": components.dit_path, + "vae": components.vae_path, + "text_encoder": components.text_encoder_path, + "tokenizer": components.tokenizer_path, + } + return model + + def to(self, *args, **kwargs): + super().to(*args, **kwargs) + self.dit.to(*args, **kwargs) + self.text_encoder.to(*args, **kwargs) + self.vae.to(*args, **kwargs) + return self + + @staticmethod + def _check_resize_height_width(height, width, num_frames): + if height % 16 != 0: + height = (height + 15) // 16 * 16 + if width % 16 != 0: + width = (width + 15) // 16 * 16 + if num_frames % 4 != 1: + num_frames = (num_frames + 3) // 4 * 4 + 1 + return height, width, num_frames + + def encode_prompt(self, prompt: Union[str, Sequence[str]]): + ids, mask = self.tokenizer(prompt, return_mask=True, add_special_tokens=True) + ids = ids.to(self.device) + mask = mask.to(self.device, dtype=torch.bool) + prompt_emb = self.text_encoder(ids, mask) + return prompt_emb.to(device=self.device), mask + + def _encode_video_latents(self, video_tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)): + if tiled: + raise NotImplementedError("Batched VAE encoding does not support tiled encoding.") + return self.vae.model.encode(video_tensor.to(self.device), self.vae.scale) + + def _encode_input_image_latents_tensor(self, input_image: torch.Tensor, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)): + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + if tiled: + raise NotImplementedError("Batched VAE image encoding does not support tiled encoding.") + image = input_image.to(device=self.device)[0].unsqueeze(1) + return self.vae.model.encode(image.unsqueeze(0), self.vae.scale) + + def _decode_latents(self, latents, tiled=False, tile_size=(30, 52), tile_stride=(15, 26)): + video_tensor = self.vae.decode(latents, device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride) + video_tensor = video_tensor.squeeze(0).detach().float().clamp(-1, 1) + video_tensor = ((video_tensor + 1.0) * 127.5).to(torch.uint8).cpu() + frames = [] + for t in range(video_tensor.shape[1]): + frame = video_tensor[:, t].permute(1, 2, 0).numpy() + frames.append(Image.fromarray(frame)) + return frames + + def _model_fn(self, + latents: torch.Tensor, # [B, C, T, H, W] + timestep: torch.Tensor, # [B] or [1] (inference mode) + context: torch.Tensor, # [B, L, D] + context_mask: Optional[torch.Tensor] = None, # [B, L] + action: Optional[torch.Tensor] = None, # [B, T-1, a_dim] + fuse_vae_embedding_in_latents=False): + return self.dit( + x=latents, + timestep=timestep, + context=context, + context_mask=context_mask, + action=action, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + + def build_inputs(self, sample, tiled=False): + video = sample["video"] + prompt = sample["prompt"] + if not isinstance(video, torch.Tensor): + raise TypeError( + f"`sample['video']` must be a torch.Tensor with shape [B, 3, T, H, W], got {type(video)}" + ) + if video.ndim != 5: + raise ValueError(f"`sample['video']` must be 5D [B, 3, T, H, W], got shape {tuple(video.shape)}") + if video.shape[1] != 3: + raise ValueError(f"`sample['video']` channel dimension must be 3, got shape {tuple(video.shape)}") + + if isinstance(prompt, str): + prompt_list = [prompt] + elif isinstance(prompt, Sequence): + prompt_list = list(prompt) + else: + raise TypeError(f"`sample['prompt']` must be str or list[str], got {type(prompt)}") + + batch_size, _, num_frames, height, width = video.shape + if len(prompt_list) != batch_size: + raise ValueError( + f"Prompt batch mismatch: got len(prompt)={len(prompt_list)} and video batch={batch_size}" + ) + if height % 16 != 0 or width % 16 != 0: + raise ValueError( + f"Video spatial dims must be multiples of 16, got H={height}, W={width}" + ) + if num_frames % 4 != 1: + raise ValueError(f"Video T must satisfy T % 4 == 1, got T={num_frames}") + + input_video = video.to(device=self.device, dtype=self.torch_dtype) + input_latents = self._encode_video_latents(input_video, tiled=tiled) # [B, C, Latent_T, H', W'] + + first_frame_latents = None + fuse_flag = False + if getattr(self.dit, "fuse_vae_embedding_in_latents", False): + first_frame_latents = input_latents[:, :, 0:1] + fuse_flag = True + context, context_mask = self.encode_prompt(prompt_list) + + action = None + if "action" in sample: + action = sample["action"] + if not isinstance(action, torch.Tensor): + raise TypeError( + f"`sample['action']` must be a torch.Tensor with shape [B, T, a_dim], got {type(action)}" + ) + if action.ndim != 3: + raise ValueError(f"`sample['action']` must be 3D [B, T, a_dim], got shape {tuple(action.shape)}") + if action.shape[1] <= 0: + raise ValueError(f"`sample['action']` temporal dimension must be positive, got {action.shape[1]}") + if action.shape[1] % (num_frames - 1) != 0: + raise ValueError( + "`sample['action']` temporal dimension must be divisible by video transitions " + f"({num_frames - 1}), got {action.shape[1]}" + ) + action = action.to(device=self.device, dtype=self.torch_dtype) + + return { + "context": context, + "context_mask": context_mask, + "input_latents": input_latents, + "first_frame_latents": first_frame_latents, + "fuse_vae_embedding_in_latents": fuse_flag, + "action": action, + } + + def training_loss(self, sample, tiled=False): + inputs = self.build_inputs(sample, tiled=tiled) + input_latents = inputs["input_latents"] + batch_size = input_latents.shape[0] + action = inputs["action"] + context = inputs["context"] + context_mask = inputs["context_mask"] + + # 1. Continuous timestep sampling and noise injection. + noise = torch.randn_like(input_latents) + timestep = self.train_scheduler.sample_training_t( + batch_size=batch_size, + device=self.device, + dtype=input_latents.dtype, + ) + latents = self.train_scheduler.add_noise(input_latents, noise, timestep) + target = self.train_scheduler.training_target(input_latents, noise, timestep) + + # 2. fix first latent + if inputs["first_frame_latents"] is not None: + latents[:, :, 0: 1] = inputs["first_frame_latents"] + + pred = self._model_fn( + latents=latents, # [B, C, Latent_T, H', W'] + timestep=timestep, + context=context, + context_mask=context_mask, + action=action, + fuse_vae_embedding_in_latents=inputs["fuse_vae_embedding_in_latents"], + ) + if inputs["first_frame_latents"] is not None: + pred = pred[:, :, 1:] + target = target[:, :, 1:] + loss_per_sample = F.mse_loss(pred.float(), target.float(), reduction="none").mean(dim=(1, 2, 3, 4)) + sample_weight = self.train_scheduler.training_weight(timestep).to( + loss_per_sample.device, dtype=loss_per_sample.dtype + ) + loss_total = (loss_per_sample * sample_weight).mean() + loss_dict = { + "loss_video": float(loss_total.detach().item()), + } + return loss_total, loss_dict + + @torch.no_grad() + def infer( + self, + prompt: str, + input_image: torch.Tensor, + num_frames: int, + action: Optional[torch.Tensor] = None, + negative_prompt: Optional[str] = None, + text_cfg_scale: float = 5.0, + action_cfg_scale: float = 1.0, + num_inference_steps: int = 20, + sigma_shift: Optional[float] = None, + seed: Optional[int] = None, + rand_device: str = "cpu", + tiled: bool = False, + **kwargs + ): + self.eval() + if input_image.ndim == 3: + input_image = input_image.unsqueeze(0) + if input_image.ndim != 4 or input_image.shape[0] != 1 or input_image.shape[1] != 3: + raise ValueError( + f"`input_image` must have shape [1,3,H,W] or [3,H,W], got {tuple(input_image.shape)}" + ) + _, _, height, width = input_image.shape + checked_h, checked_w, checked_t = self._check_resize_height_width(height, width, num_frames) + if (checked_h, checked_w) != (height, width): + raise ValueError( + f"`input_image` must be resized before infer, expected multiples of 16 but got HxW=({height},{width})" + ) + if checked_t != num_frames: + raise ValueError( + f"`num_frames` must satisfy T % 4 == 1, got {num_frames}" + ) + + latent_t = (num_frames - 1) // self.vae.temporal_downsample_factor + 1 + latent_h = height // self.vae.upsampling_factor + latent_w = width // self.vae.upsampling_factor + + generator = None if seed is None else torch.Generator(device=rand_device).manual_seed(seed) + latents = torch.randn( + (1, self.vae.model.z_dim, latent_t, latent_h, latent_w), + generator=generator, + device=rand_device, + dtype=torch.float32, + ).to(device=self.device, dtype=self.torch_dtype) + input_image = input_image.to(device=self.device, dtype=self.torch_dtype) + if action is not None: + action = action.to(device=self.device, dtype=latents.dtype) + if action.ndim != 2: + raise ValueError(f"`action` must be 2D [T, a_dim], got shape {tuple(action.shape)}") + action = action.unsqueeze(0) # [1, T, a_dim] + if action.shape[1] % (num_frames - 1) != 0: + raise ValueError( + "`action` temporal dimension must be divisible by `num_frames - 1`, " + f"got {action.shape[1]} vs {num_frames - 1}" + ) + if action_cfg_scale != 1.0 and action is None: + raise ValueError("`action_cfg_scale` != 1.0 requires non-null `action` input.") + + z = self._encode_input_image_latents_tensor(input_image=input_image, tiled=tiled) + latents[:, :, 0:1] = z + first_frame_latents = z + fuse_flag = True + + context_posi, context_posi_mask = self.encode_prompt(prompt) + context_nega = None + context_nega_mask = None + if text_cfg_scale != 1.0: + context_nega, context_nega_mask = self.encode_prompt("" if negative_prompt is None else negative_prompt) + action_nega = torch.zeros_like(action) if (action is not None and action_cfg_scale != 1.0) else None + + infer_timesteps, infer_deltas = self.infer_scheduler.build_inference_schedule( + num_inference_steps=num_inference_steps, + device=self.device, + dtype=latents.dtype, + shift_override=sigma_shift, + ) + for step_t, step_delta in zip(infer_timesteps, infer_deltas): + timestep = step_t.unsqueeze(0).to(dtype=latents.dtype, device=self.device) + noise_pred_posi = self._model_fn( + latents=latents, + timestep=timestep, + context=context_posi, + context_mask=context_posi_mask, + action=action, + fuse_vae_embedding_in_latents=fuse_flag, + ) + noise_pred = noise_pred_posi + if context_nega is not None: + noise_pred_text_nega = self._model_fn( + latents=latents, + timestep=timestep, + context=context_nega, + context_mask=context_nega_mask, + action=action, + fuse_vae_embedding_in_latents=fuse_flag, + ) + noise_pred = noise_pred + (text_cfg_scale - 1.0) * (noise_pred_posi - noise_pred_text_nega) + if action_nega is not None: + noise_pred_action_nega = self._model_fn( + latents=latents, + timestep=timestep, + context=context_posi, + context_mask=context_posi_mask, + action=action_nega, + fuse_vae_embedding_in_latents=fuse_flag, + ) + noise_pred = noise_pred + (action_cfg_scale - 1.0) * (noise_pred_posi - noise_pred_action_nega) + latents = self.infer_scheduler.step(noise_pred, step_delta, latents) + latents[:, :, 0:1] = first_frame_latents + + return {"video": self._decode_latents(latents, tiled=tiled)} + + def save_checkpoint(self, path, optimizer=None, step=None): + payload = { + "dit": self.dit.state_dict(), + "step": step, + "torch_dtype": str(self.torch_dtype), + } + if optimizer is not None: + payload["optimizer"] = optimizer.state_dict() + torch.save(payload, path) + + def load_checkpoint(self, path, optimizer=None): + payload = torch.load(path, map_location="cpu") + self.dit.load_state_dict(payload["dit"], strict=False) + if optimizer is not None and "optimizer" in payload: + optimizer.load_state_dict(payload["optimizer"]) + return payload + + def forward(self, *args, **kwargs): + return self.training_loss(*args, **kwargs) diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/wan_video_dit.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/wan_video_dit.py new file mode 100644 index 0000000000000000000000000000000000000000..c1ab84700f72ab3de8916cfc4d30d9b531def171 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/wan_video_dit.py @@ -0,0 +1,772 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import math +from typing import Any, Dict, Tuple, Optional +from einops import rearrange + +from fastwam.utils.logging_config import get_logger + +logger = get_logger(__name__) + + +def flash_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, num_heads: int, ctx_mask: Optional[torch.Tensor] = None, compatibility_mode=True): + if compatibility_mode: + q = rearrange(q, "b s (n d) -> b n s d", n=num_heads) + k = rearrange(k, "b s (n d) -> b n s d", n=num_heads) + v = rearrange(v, "b s (n d) -> b n s d", n=num_heads) + x = F.scaled_dot_product_attention(q, k, v, attn_mask=ctx_mask) + x = rearrange(x, "b n s d -> b s (n d)", n=num_heads) + return x + else: + raise NotImplementedError("Only compatibility mode is implemented for flash attention. Please set compatibility_mode=True.") + + + +def modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor): + return (x * (1 + scale) + shift) + + +def sinusoidal_embedding_1d(dim, position): + sinusoid = torch.outer(position.type(torch.float64), torch.pow( + 10000, -torch.arange(dim//2, dtype=torch.float64, device=position.device).div(dim//2))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + return x.to(position.dtype) + + +def precompute_freqs_cis_3d(dim: int, end: int = 1024, theta: float = 10000.0): + # 3d rope precompute + f_freqs_cis = precompute_freqs_cis(dim - 2 * (dim // 3), end, theta) + h_freqs_cis = precompute_freqs_cis(dim // 3, end, theta) + w_freqs_cis = precompute_freqs_cis(dim // 3, end, theta) + return f_freqs_cis, h_freqs_cis, w_freqs_cis + + +def precompute_freqs_cis(dim: int, end: int = 1024, theta: float = 10000.0): + # 1d rope precompute + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2) + [: (dim // 2)].double() / dim)) + freqs = torch.outer(torch.arange(end, device=freqs.device), freqs) + freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex64 + return freqs_cis + + +def rope_apply(x, freqs, num_heads): + batch_size, seq_len, _ = x.shape + x_heads = x.view(batch_size, seq_len, num_heads, -1) + x_out = torch.view_as_complex( + x_heads.to(torch.float64).reshape(batch_size, seq_len, num_heads, -1, 2) + ) + return torch.view_as_real(x_out * freqs).flatten(2).to(x.dtype) + + +def create_group_causal_attn_mask( + num_temporal_groups: int, num_query_per_group: int, num_key_per_group: int, mode: str = "causal" +) -> torch.Tensor: + """ + Creates a group-based attention mask for scaled dot-product attention with two modes: + 'causal' and 'group_diagonal'. + + Parameters: + - num_temporal_groups (int): The number of temporal groups (e.g., frames in a video sequence). + - num_query_per_group (int): The number of query tokens per temporal group. (e.g., latent tokens in a frame, H x W). + - num_key_per_group (int): The number of key tokens per temporal group. (e.g., action tokens per frame). + - mode (str): The mode of the attention mask. Options are: + - 'causal': Query tokens can attend to key tokens from the same or previous temporal groups. + - 'group_diagonal': Query tokens can attend only to key tokens from the same temporal group. + + Returns: + - attn_mask (torch.Tensor): A boolean tensor of shape (L, S), where: + - L = num_temporal_groups * num_query_per_group (total number of query tokens) + - S = num_temporal_groups * num_key_per_group (total number of key tokens) + The mask indicates where attention is allowed (True) and disallowed (False). + + Example: + Input: + num_temporal_groups = 3 + num_query_per_group = 4 + num_key_per_group = 2 + Output: + Causal Mask Shape: torch.Size([12, 6]) + Group Diagonal Mask Shape: torch.Size([12, 6]) + if mode='causal': + tensor([[ True, True, False, False, False, False], + [ True, True, False, False, False, False], + [ True, True, False, False, False, False], + [ True, True, False, False, False, False], + [ True, True, True, True, False, False], + [ True, True, True, True, False, False], + [ True, True, True, True, False, False], + [ True, True, True, True, False, False], + [ True, True, True, True, True, True], + [ True, True, True, True, True, True], + [ True, True, True, True, True, True], + [ True, True, True, True, True, True]]) + + if mode='group_diagonal': + tensor([[ True, True, False, False, False, False], + [ True, True, False, False, False, False], + [ True, True, False, False, False, False], + [ True, True, False, False, False, False], + [False, False, True, True, False, False], + [False, False, True, True, False, False], + [False, False, True, True, False, False], + [False, False, True, True, False, False], + [False, False, False, False, True, True], + [False, False, False, False, True, True], + [False, False, False, False, True, True], + [False, False, False, False, True, True]]) + + """ + assert mode in ["causal", "group_diagonal"], f"Mode {mode} must be 'causal' or 'group_diagonal'" + + # Total number of query and key tokens + total_num_query_tokens = num_temporal_groups * num_query_per_group # Total number of query tokens (L) + total_num_key_tokens = num_temporal_groups * num_key_per_group # Total number of key tokens (S) + + # Generate time indices for query and key tokens (shape: [L] and [S]) + query_time_indices = torch.arange(num_temporal_groups).repeat_interleave(num_query_per_group) # Shape: [L] + key_time_indices = torch.arange(num_temporal_groups).repeat_interleave(num_key_per_group) # Shape: [S] + + # Expand dimensions to compute outer comparison + query_time_indices = query_time_indices.unsqueeze(1) # Shape: [L, 1] + key_time_indices = key_time_indices.unsqueeze(0) # Shape: [1, S] + + if mode == "causal": + # Causal Mode: Query can attend to keys where key_time <= query_time + attn_mask = query_time_indices >= key_time_indices # Shape: [L, S] + elif mode == "group_diagonal": + # Group Diagonal Mode: Query can attend only to keys where key_time == query_time + attn_mask = query_time_indices == key_time_indices # Shape: [L, S] + + assert attn_mask.shape == (total_num_query_tokens, total_num_key_tokens), "Attention mask shape mismatch" + return attn_mask + + +class RMSNorm(nn.Module): + def __init__(self, dim, eps=1e-5): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps) + + def forward(self, x): + dtype = x.dtype + return self.norm(x.float()).to(dtype) * self.weight + + +class AttentionModule(nn.Module): + def __init__(self, num_heads): + super().__init__() + self.num_heads = num_heads + + def forward(self, q, k, v, ctx_mask=None): + x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=ctx_mask) + return x + + +class SelfAttention(nn.Module): + def __init__(self, hidden_dim: int, attn_head_dim: int, num_heads: int, eps: float = 1e-6): + super().__init__() + self.hidden_dim = hidden_dim + self.num_heads = num_heads + self.attn_head_dim = attn_head_dim + self.attn_hidden_dim = self.num_heads * self.attn_head_dim + + self.q = nn.Linear(hidden_dim, self.attn_hidden_dim) + self.k = nn.Linear(hidden_dim, self.attn_hidden_dim) + self.v = nn.Linear(hidden_dim, self.attn_hidden_dim) + self.o = nn.Linear(self.attn_hidden_dim, hidden_dim) + self.norm_q = RMSNorm(self.attn_hidden_dim, eps=eps) + self.norm_k = RMSNorm(self.attn_hidden_dim, eps=eps) + + # self.attn = AttentionModule(self.num_heads) + + def forward(self, x, freqs, self_attn_mask: Optional[torch.Tensor] = None): + q = self.norm_q(self.q(x)) + k = self.norm_k(self.k(x)) + v = self.v(x) + q = rope_apply(q, freqs, self.num_heads) + k = rope_apply(k, freqs, self.num_heads) + x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=self_attn_mask) + return self.o(x) + + +class CrossAttention(nn.Module): + def __init__(self, hidden_dim: int, attn_head_dim: int, num_heads: int, eps: float = 1e-6,): + super().__init__() + self.hidden_dim = hidden_dim + self.num_heads = num_heads + self.attn_head_dim = attn_head_dim + self.attn_hidden_dim = self.num_heads * self.attn_head_dim + + self.q = nn.Linear(hidden_dim, self.attn_hidden_dim) + self.k = nn.Linear(hidden_dim, self.attn_hidden_dim) + self.v = nn.Linear(hidden_dim, self.attn_hidden_dim) + self.o = nn.Linear(self.attn_hidden_dim, hidden_dim) + self.norm_q = RMSNorm(self.attn_hidden_dim, eps=eps) + self.norm_k = RMSNorm(self.attn_hidden_dim, eps=eps) + + # self.attn = AttentionModule(self.num_heads) + + def forward(self, x: torch.Tensor, ctx: torch.Tensor, ctx_mask: Optional[torch.Tensor] = None): + q = self.norm_q(self.q(x)) + k = self.norm_k(self.k(ctx)) + v = self.v(ctx) + x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, ctx_mask=ctx_mask) + return self.o(x) + + +class GateModule(nn.Module): + def __init__(self,): + super().__init__() + + def forward(self, x, gate, residual): + return x + gate * residual + +class DiTBlock(nn.Module): + def __init__(self, hidden_dim: int, attn_head_dim: int, num_heads: int, ffn_dim: int, eps: float = 1e-6): + super().__init__() + self.hidden_dim = hidden_dim + self.attn_head_dim = attn_head_dim + self.num_heads = num_heads + self.ffn_dim = ffn_dim + + self.self_attn = SelfAttention(hidden_dim, attn_head_dim, num_heads, eps) + self.cross_attn = CrossAttention( + hidden_dim, attn_head_dim, num_heads, eps) + self.norm1 = nn.LayerNorm(hidden_dim, eps=eps, elementwise_affine=False) + self.norm2 = nn.LayerNorm(hidden_dim, eps=eps, elementwise_affine=False) + self.norm3 = nn.LayerNorm(hidden_dim, eps=eps) + self.ffn = nn.Sequential(nn.Linear(hidden_dim, ffn_dim), nn.GELU( + approximate='tanh'), nn.Linear(ffn_dim, hidden_dim)) + self.modulation = nn.Parameter(torch.randn(1, 6, hidden_dim) / hidden_dim**0.5) + self.gate = GateModule() + + def forward(self, x, context, t_mod, freqs, context_mask=None, self_attn_mask: Optional[torch.Tensor] = None): + if context_mask is not None and context_mask.dim() == 3: + context_mask = context_mask.unsqueeze(1) # (B, 1, seq_len, context_len), 1 for heads + has_seq = len(t_mod.shape) == 4 + chunk_dim = 2 if has_seq else 1 + # msa: multi-head self-attention mlp: multi-layer perceptron + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + self.modulation.to(dtype=t_mod.dtype, device=t_mod.device) + t_mod).chunk(6, dim=chunk_dim) + if has_seq: + # means t_mod has separate modulation for each token, otherwise same modulation for all tokens in the block + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + shift_msa.squeeze(2), scale_msa.squeeze(2), gate_msa.squeeze(2), + shift_mlp.squeeze(2), scale_mlp.squeeze(2), gate_mlp.squeeze(2), + ) + input_x = modulate(self.norm1(x), shift_msa, scale_msa) + x = self.gate(x, gate_msa, self.self_attn(input_x, freqs, self_attn_mask=self_attn_mask)) + x = x + self.cross_attn(self.norm3(x), context, ctx_mask=context_mask) + input_x = modulate(self.norm2(x), shift_mlp, scale_mlp) + x = self.gate(x, gate_mlp, self.ffn(input_x)) + return x + + +class MLP(torch.nn.Module): + def __init__(self, in_dim, out_dim, has_pos_emb=False): + super().__init__() + self.proj = torch.nn.Sequential( + nn.LayerNorm(in_dim), + nn.Linear(in_dim, in_dim), + nn.GELU(), + nn.Linear(in_dim, out_dim), + nn.LayerNorm(out_dim) + ) + self.has_pos_emb = has_pos_emb + if has_pos_emb: + self.emb_pos = torch.nn.Parameter(torch.zeros((1, 514, 1280))) + + def forward(self, x): + if self.has_pos_emb: + x = x + self.emb_pos.to(dtype=x.dtype, device=x.device) + return self.proj(x) + + +class Head(nn.Module): + def __init__(self, dim: int, out_dim: int, patch_size: Tuple[int, int, int], eps: float): + super().__init__() + self.dim = dim + self.patch_size = patch_size + self.norm = nn.LayerNorm(dim, eps=eps, elementwise_affine=False) + self.head = nn.Linear(dim, out_dim * math.prod(patch_size)) + self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5) + + def forward(self, x, t_mod): + if len(t_mod.shape) == 3: + shift, scale = (self.modulation.unsqueeze(0).to(dtype=t_mod.dtype, device=t_mod.device) + t_mod.unsqueeze(2)).chunk(2, dim=2) + x = (self.head(self.norm(x) * (1 + scale.squeeze(2)) + shift.squeeze(2))) + else: + shift, scale = (self.modulation.to(dtype=t_mod.dtype, device=t_mod.device) + t_mod).chunk(2, dim=1) + x = (self.head(self.norm(x) * (1 + scale) + shift)) + return x + + +class WanVideoDiT(torch.nn.Module): + def __init__( + self, + hidden_dim: int, + in_dim: int, + ffn_dim: int, + out_dim: int, + text_dim: int, + freq_dim: int, + eps: float, + patch_size: Tuple[int, int, int], + num_heads: int, + attn_head_dim: int, + num_layers: int, + has_image_input: bool, + has_image_pos_emb: bool = False, + has_ref_conv: bool = False, + add_control_adapter: bool = False, + in_dim_control_adapter: int = 24, + seperated_timestep: bool = False, + require_vae_embedding: bool = False, + require_clip_embedding: bool = False, + fuse_vae_embedding_in_latents: bool = True, + action_conditioned: bool = False, + action_dim: int = 7, + action_group_causal_mask_mode = "causal", + video_attention_mask_mode: str = "bidirectional", + use_gradient_checkpointing: bool = False, + ): + super().__init__() + self.hidden_dim = hidden_dim + self.in_dim = in_dim + self.freq_dim = freq_dim + self.patch_size = patch_size + self.num_heads = num_heads + self.attn_head_dim = attn_head_dim + self.seperated_timestep = seperated_timestep + self.require_vae_embedding = require_vae_embedding + self.require_clip_embedding = require_clip_embedding + self.fuse_vae_embedding_in_latents = fuse_vae_embedding_in_latents + self.video_attention_mask_mode = str(video_attention_mask_mode) + + if num_heads <= 0: + raise ValueError(f"`num_heads` must be > 0, got {num_heads}") + if attn_head_dim <= 0: + raise ValueError(f"`attn_head_dim` must be > 0, got {attn_head_dim}") + if attn_head_dim % 2 != 0: + raise ValueError( + f"`attn_head_dim` must be even for RoPE, got {attn_head_dim}" + ) + + self.action_conditioned = action_conditioned + self.action_dim = action_dim + assert has_image_input == False + assert require_clip_embedding == False + assert require_vae_embedding == False and fuse_vae_embedding_in_latents == True, "Only support fusing vae embedding in latents" + + self.patch_embedding = nn.Conv3d( + in_dim, hidden_dim, kernel_size=patch_size, stride=patch_size) + self.text_embedding = nn.Sequential( + nn.Linear(text_dim, hidden_dim), + nn.GELU(approximate='tanh'), + nn.Linear(hidden_dim, hidden_dim) + ) + self.time_embedding = nn.Sequential( + nn.Linear(freq_dim, hidden_dim), + nn.SiLU(), + nn.Linear(hidden_dim, hidden_dim) + ) + self.time_projection = nn.Sequential( + nn.SiLU(), nn.Linear(hidden_dim, hidden_dim * 6)) + self.blocks = nn.ModuleList([ + DiTBlock(hidden_dim, attn_head_dim, num_heads, ffn_dim, eps) + for _ in range(num_layers) + ]) + self.head = Head(hidden_dim, out_dim, patch_size, eps) + self.freqs = precompute_freqs_cis_3d(attn_head_dim) + if has_ref_conv: + self.ref_conv = nn.Conv2d(16, hidden_dim, kernel_size=(2, 2), stride=(2, 2)) + self.has_image_pos_emb = has_image_pos_emb + self.has_ref_conv = has_ref_conv + self.control_adapter = None + + if self.action_conditioned: + self.action_embedding = nn.Linear(action_dim, hidden_dim) + self.action_group_causal_mask_mode = action_group_causal_mask_mode + + self.use_gradient_checkpointing = use_gradient_checkpointing + if self.use_gradient_checkpointing: + logger.info("Using gradient checkpointing for DiT blocks. This will save memory but use more computation.") + + def _apply(self, fn): + result = super()._apply(fn) + device = next(self.parameters()).device + self.freqs = tuple(freqs.to(device=device) for freqs in self.freqs) + return result + + def get_freqs(self, f: int, h: int, w: int) -> torch.Tensor: + freq_f, freq_h, freq_w = self.freqs + return torch.cat( + [ + freq_f[:f].view(f, 1, 1, -1).expand(f, h, w, -1), + freq_h[:h].view(1, h, 1, -1).expand(f, h, w, -1), + freq_w[:w].view(1, 1, w, -1).expand(f, h, w, -1), + ], + dim=-1, + ).reshape(f * h * w, 1, -1) + + + def patchify(self, x: torch.Tensor, control_camera_latents_input: Optional[torch.Tensor] = None): + x = self.patch_embedding(x) + if self.control_adapter is not None and control_camera_latents_input is not None: + y_camera = self.control_adapter(control_camera_latents_input) + x = [u + v for u, v in zip(x, y_camera)] + x = x[0].unsqueeze(0) + return x + + def unpatchify(self, x: torch.Tensor, grid_size: torch.Tensor): + return rearrange( + x, 'b (f h w) (x y z c) -> b c (f x) (h y) (w z)', + f=grid_size[0], h=grid_size[1], w=grid_size[2], + x=self.patch_size[0], y=self.patch_size[1], z=self.patch_size[2] + ) + + def _validate_forward_inputs( + self, + x: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: Optional[torch.Tensor], + action: Optional[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if x.ndim != 5: + raise ValueError(f"`latents` must be 5D [B, C, T, H, W], got shape {tuple(x.shape)}") + num_latent_frames = x.shape[2] + if context.ndim != 3: + raise ValueError(f"`context` must be 3D [B, L, D], got shape {tuple(context.shape)}") + if timestep.ndim != 1: + raise ValueError(f"`timestep` must be 1D [B] or [1], got shape {tuple(timestep.shape)}") + if self.action_conditioned: + allow_text_only_single_frame = (num_latent_frames == 1 and action is None) + if not allow_text_only_single_frame: + assert action is not None, "Action input is required for action-conditioned model." + if action.ndim != 3: + raise ValueError(f"`action` must be 3D [B, action_horizon, action_dim], got shape {tuple(action.shape)}") + if action.shape[2] != self.action_dim: + raise ValueError(f"`action` last dimension must be {self.action_dim}, got {action.shape[2]}") + if num_latent_frames <= 1: + raise ValueError(f"video length must be > 1 for action-conditioned model, got {num_latent_frames}") + if action.shape[1] % (num_latent_frames - 1) != 0: + raise ValueError( + f"action horizon must be divisible by (num_latent_frames - 1), got action_horizon={action.shape[1]}" + ) + if context_mask is None: + context_mask = torch.ones((context.shape[0], context.shape[1]), dtype=torch.bool, device=context.device) + else: + if context_mask.ndim != 2: + raise ValueError(f"`context_mask` must be 2D [B, L], got shape {tuple(context_mask.shape)}") + if context_mask.shape[0] != context.shape[0] or context_mask.shape[1] != context.shape[1]: + raise ValueError(f"`context_mask` shape must match `context` shape [B, L], got {tuple(context_mask.shape)} vs {tuple(context.shape)}") + + batch_size = x.shape[0] + if batch_size != context.shape[0]: + if not self.training and batch_size == 1: + x = x.expand(context.shape[0], -1, -1, -1, -1) + batch_size = context.shape[0] + else: + raise ValueError( + f"Batch mismatch between latents and context: {batch_size} vs {context.shape[0]}." + ) + + if timestep.shape[0] not in (1, batch_size): + raise ValueError( + f"`timestep` length must be 1 or batch_size({batch_size}), got {timestep.shape[0]}" + ) + if timestep.shape[0] == 1 and batch_size > 1: + assert not self.training, "During training, timestep length must match batch_size." + timestep = timestep.expand(batch_size) + return x, timestep, context_mask + + def build_video_to_video_mask( + self, + video_seq_len: int, + video_tokens_per_frame: int, + device: torch.device, + ) -> torch.Tensor: + if video_seq_len <= 0: + raise ValueError(f"`video_seq_len` must be positive, got {video_seq_len}") + if video_tokens_per_frame <= 0: + raise ValueError(f"`video_tokens_per_frame` must be positive, got {video_tokens_per_frame}") + + if self.video_attention_mask_mode == "bidirectional": + return torch.ones((video_seq_len, video_seq_len), dtype=torch.bool, device=device) + + if self.video_attention_mask_mode == "per_frame_causal": + if video_seq_len % video_tokens_per_frame != 0: + raise ValueError( + "`video_seq_len` must be divisible by `video_tokens_per_frame` in `per_frame_causal` mode, " + f"got {video_seq_len} and {video_tokens_per_frame}" + ) + num_video_frames = video_seq_len // video_tokens_per_frame + frame_causal = torch.tril( + torch.ones((num_video_frames, num_video_frames), dtype=torch.bool, device=device) + ) + return frame_causal.repeat_interleave(video_tokens_per_frame, dim=0).repeat_interleave( + video_tokens_per_frame, dim=1 + ) + + if self.video_attention_mask_mode == "first_frame_causal": + video_mask = torch.ones((video_seq_len, video_seq_len), dtype=torch.bool, device=device) + first_frame_tokens = min(video_tokens_per_frame, video_seq_len) + video_mask[:first_frame_tokens, first_frame_tokens:] = False + return video_mask + + raise ValueError(f"Unsupported video attention mask mode: {self.video_attention_mask_mode}") + + def pre_dit( + self, + x: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: Optional[torch.Tensor] = None, + action: Optional[torch.Tensor] = None, + fuse_vae_embedding_in_latents: bool = False, + control_camera_latents_input: Optional[torch.Tensor] = None, + ) -> Dict[str, Any]: + x, timestep, context_mask = self._validate_forward_inputs( + x=x, + timestep=timestep, + context=context, + context_mask=context_mask, + action=action, + ) + + batch_size = x.shape[0] + patch_h = int(self.patch_size[1]) + patch_w = int(self.patch_size[2]) + if x.shape[3] % patch_h != 0 or x.shape[4] % patch_w != 0: + raise ValueError( + "Latent spatial shape must be divisible by DiT patch size, " + f"got HxW=({x.shape[3]}, {x.shape[4]}), patch=({patch_h}, {patch_w})" + ) + tokens_per_frame = (x.shape[3] // patch_h) * (x.shape[4] // patch_w) + + if self.seperated_timestep and fuse_vae_embedding_in_latents: + if not hasattr(self, "patch_size") or len(self.patch_size) < 3: + raise ValueError(f"Invalid dit.patch_size: {getattr(self, 'patch_size', None)}") + + token_timesteps = torch.ones( + (batch_size, x.shape[2], tokens_per_frame), + dtype=timestep.dtype, + device=timestep.device, + ) * timestep.view(batch_size, 1, 1) + token_timesteps[:, 0, :] = 0 + token_timesteps = token_timesteps.reshape(batch_size, -1) + token_t_emb = sinusoidal_embedding_1d(self.freq_dim, token_timesteps.reshape(-1)) + t = self.time_embedding(token_t_emb).reshape(batch_size, -1, self.hidden_dim) + t_mod = self.time_projection(t).unflatten(2, (6, self.hidden_dim)) + else: + raise NotImplementedError("Only support seperated_timestep with fuse_vae_embedding_in_latents for now.") + t = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timestep)) + t_mod = self.time_projection(t).unflatten(1, (6, self.hidden_dim)) + x = self.patchify(x, control_camera_latents_input=control_camera_latents_input) + f, h, w = x.shape[2:] + + context = self.text_embedding(context) # (B, L, dim) + context_len = context.shape[1] + if self.action_conditioned and action is not None: + action_len = action.shape[1] + action_emb = self.action_embedding(action) # (B, action_len, dim) + action_pos_embed = sinusoidal_embedding_1d(self.hidden_dim, + torch.arange(action_len, device=action_emb.device)) # (action_len, dim) + action_emb = action_emb + action_pos_embed.unsqueeze(0) # (B, action_len, dim) + context = torch.cat([context, action_emb], dim=1) # (B, context_len + action_len, dim) + + # new mask + num_temporal_groups = f - 1 # first latent frame do not attend to actions + if num_temporal_groups <= 0: + raise ValueError( + "Action-conditioned context mask requires at least 2 latent frames when `action` is provided." + ) + assert action_emb.shape[1] % num_temporal_groups == 0, \ + f"Action embedding length {action_emb.shape[1]} must be divisible by number of temporal groups {num_temporal_groups}" + # Each latent frame (from the 2nd one) attends to the corresponding group of action tokens + action_group_mask = create_group_causal_attn_mask( + num_temporal_groups=num_temporal_groups, + num_query_per_group=tokens_per_frame, + num_key_per_group=action_len // num_temporal_groups, + mode=self.action_group_causal_mask_mode, + ).to(context.device) # ((f-1)*tokens_per_frame, action_len) + + seq_len = f * h * w # query length + final_context_mask = torch.zeros((batch_size, seq_len, context.shape[1]), dtype=torch.bool, device=context.device) # (B, seq_len, L + action_len) + # all latent frames attend to text tokens + final_context_mask[:, :, :context_len] = context_mask.unsqueeze(1).expand(-1, seq_len, -1) # (B, seq_len, L) + # latent frames from the 2nd one attend to action tokens + final_context_mask[:, tokens_per_frame:, context_len:] = action_group_mask.unsqueeze(0).expand(batch_size, -1, -1) # (B, seq_len, action_len) + context_mask = final_context_mask + elif self.action_conditioned and action is None: + if f != 1: + raise ValueError( + "Action-conditioned model requires `action` unless running single-frame text-only mode with num_latent_frames=1." + ) + context_mask = context_mask.unsqueeze(1).expand(-1, f * h * w, -1) # (B, seq_len, L) + else: + context_mask = context_mask.unsqueeze(1).expand(-1, f * h * w, -1) # (B, seq_len, L) + + x_tokens = rearrange(x, "b c f h w -> b (f h w) c").contiguous() + + freqs = self.get_freqs(f, h, w) + + return { + "tokens": x_tokens, + "freqs": freqs, + "t": t, + "t_mod": t_mod, + "context": context, + "context_mask": context_mask, + "meta": { + "grid_size": (f, h, w), + "tokens_per_frame": tokens_per_frame, + "batch_size": batch_size, + }, + } + + def post_dit(self, x_tokens: torch.Tensor, pre_state: Dict[str, Any]) -> torch.Tensor: + f, h, w = pre_state["meta"]["grid_size"] + x = self.head(x_tokens, pre_state["t"]) + x = self.unpatchify(x, (f, h, w)) + return x + + def prepare( + self, + x: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: torch.Tensor, + action: Optional[torch.Tensor] = None, + fuse_vae_embedding_in_latents: bool = False, + control_camera_latents_input: Optional[torch.Tensor] = None, + ) -> tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + int, + int, + int, + int, + ]: + """Prepare tensor inputs for the compile-friendly DiT/MoT core.""" + batch_size = x.shape[0] + patch_h = int(self.patch_size[1]) + patch_w = int(self.patch_size[2]) + tokens_per_frame = (x.shape[3] // patch_h) * (x.shape[4] // patch_w) + + if not self.seperated_timestep or not fuse_vae_embedding_in_latents: + raise NotImplementedError( + "Only support seperated_timestep with fuse_vae_embedding_in_latents for now." + ) + token_timesteps = torch.ones( + (batch_size, x.shape[2], tokens_per_frame), + dtype=timestep.dtype, + device=timestep.device, + ) * timestep.view(batch_size, 1, 1) + token_timesteps[:, 0, :] = 0 + token_timesteps = token_timesteps.reshape(batch_size, -1) + token_t_emb = sinusoidal_embedding_1d(self.freq_dim, token_timesteps.reshape(-1)) + t = self.time_embedding(token_t_emb).reshape(batch_size, -1, self.hidden_dim) + t_mod = self.time_projection(t).unflatten(2, (6, self.hidden_dim)) + + x = self.patchify(x, control_camera_latents_input=control_camera_latents_input) + f, h, w = x.shape[2:] + context = self.text_embedding(context) + context_len = context.shape[1] + if self.action_conditioned and action is not None: + action_len = action.shape[1] + action_emb = self.action_embedding(action) + action_pos_embed = sinusoidal_embedding_1d( + self.hidden_dim, + torch.arange(action_len, device=action_emb.device), + ) + action_emb = action_emb + action_pos_embed.unsqueeze(0) + context = torch.cat([context, action_emb], dim=1) + num_temporal_groups = f - 1 + action_group_mask = create_group_causal_attn_mask( + num_temporal_groups=num_temporal_groups, + num_query_per_group=tokens_per_frame, + num_key_per_group=action_len // num_temporal_groups, + mode=self.action_group_causal_mask_mode, + ).to(context.device) + seq_len = f * h * w + final_context_mask = torch.zeros( + (batch_size, seq_len, context.shape[1]), + dtype=torch.bool, + device=context.device, + ) + final_context_mask[:, :, :context_len] = context_mask.unsqueeze(1).expand( + -1, seq_len, -1 + ) + final_context_mask[:, tokens_per_frame:, context_len:] = action_group_mask.unsqueeze( + 0 + ).expand(batch_size, -1, -1) + context_mask = final_context_mask + else: + context_mask = context_mask.unsqueeze(1).expand(-1, f * h * w, -1) + + x_tokens = rearrange(x, "b c f h w -> b (f h w) c").contiguous() + freqs = self.get_freqs(f, h, w) + return x_tokens, t, t_mod, context, context_mask, freqs, f, h, w, tokens_per_frame + + def post( + self, + x_tokens: torch.Tensor, + t: torch.Tensor, + f: int, + h: int, + w: int, + ) -> torch.Tensor: + """Convert tensor-core video tokens back into latent predictions.""" + return self.unpatchify(self.head(x_tokens, t), (f, h, w)) + + def forward( + self, + x: torch.Tensor, + timestep: torch.Tensor, + context: torch.Tensor, + context_mask: Optional[torch.Tensor] = None, + action: Optional[torch.Tensor] = None, + fuse_vae_embedding_in_latents: bool = False, + ): + if context_mask is None: + context_mask = torch.ones( + (x.shape[0], context.shape[1]), + dtype=torch.bool, + device=context.device, + ) + x_tokens, t, t_mod, context_emb, context_attn_mask, freqs, f, h, w, tokens_per_frame = self.prepare( + x=x, + timestep=timestep, + context=context, + context_mask=context_mask, + action=action, + fuse_vae_embedding_in_latents=fuse_vae_embedding_in_latents, + ) + self_attn_mask = self.build_video_to_video_mask( + video_seq_len=x_tokens.shape[1], + video_tokens_per_frame=tokens_per_frame, + device=x_tokens.device, + ) if self.video_attention_mask_mode != "bidirectional" else None # special rule for faster speed + + for block in self.blocks: + x_tokens = block( + x_tokens, + context_emb, + t_mod, + freqs, + context_mask=context_attn_mask, + self_attn_mask=self_attn_mask, + ) + + return self.post(x_tokens, t, f, h, w) diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/wan_video_text_encoder.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/wan_video_text_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..24f89f49c971d3798a2eed8e20faa98ec01cba20 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/wan_video_text_encoder.py @@ -0,0 +1,342 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import AutoTokenizer +import html +import string +try: + import ftfy +except ModuleNotFoundError: + class _FTFY: + @staticmethod + def fix_text(text): + return text + + ftfy = _FTFY() +try: + import regex as re +except ModuleNotFoundError: + import re + +def fp16_clamp(x): + if x.dtype == torch.float16 and torch.isinf(x).any(): + clamp = torch.finfo(x.dtype).max - 1000 + x = torch.clamp(x, min=-clamp, max=clamp) + return x + + +class GELU(nn.Module): + + def forward(self, x): + return 0.5 * x * (1.0 + torch.tanh( + math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0)))) + + +class T5LayerNorm(nn.Module): + + def __init__(self, dim, eps=1e-6): + super(T5LayerNorm, self).__init__() + self.dim = dim + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def forward(self, x): + x = x * torch.rsqrt(x.float().pow(2).mean(dim=-1, keepdim=True) + + self.eps) + if self.weight.dtype in [torch.float16, torch.bfloat16]: + x = x.type_as(self.weight) + return self.weight * x + + +class T5Attention(nn.Module): + + def __init__(self, dim, dim_attn, num_heads, dropout=0.1): + assert dim_attn % num_heads == 0 + super(T5Attention, self).__init__() + self.dim = dim + self.dim_attn = dim_attn + self.num_heads = num_heads + self.head_dim = dim_attn // num_heads + + # layers + self.q = nn.Linear(dim, dim_attn, bias=False) + self.k = nn.Linear(dim, dim_attn, bias=False) + self.v = nn.Linear(dim, dim_attn, bias=False) + self.o = nn.Linear(dim_attn, dim, bias=False) + self.dropout = nn.Dropout(dropout) + + def forward(self, x, context=None, mask=None, pos_bias=None): + """ + x: [B, L1, C]. + context: [B, L2, C] or None. + mask: [B, L2] or [B, L1, L2] or None. + """ + # check inputs + context = x if context is None else context + b, n, c = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.q(x).view(b, -1, n, c) + k = self.k(context).view(b, -1, n, c) + v = self.v(context).view(b, -1, n, c) + + # attention bias + attn_bias = x.new_zeros(b, n, q.size(1), k.size(1)) + if pos_bias is not None: + attn_bias += pos_bias + if mask is not None: + assert mask.ndim in [2, 3] + mask = mask.view(b, 1, 1, + -1) if mask.ndim == 2 else mask.unsqueeze(1) + attn_bias.masked_fill_(mask == 0, torch.finfo(x.dtype).min) + + # compute attention (T5 does not use scaling) + attn = torch.einsum('binc,bjnc->bnij', q, k) + attn_bias + attn = F.softmax(attn.float(), dim=-1).type_as(attn) + x = torch.einsum('bnij,bjnc->binc', attn, v) + + # output + x = x.reshape(b, -1, n * c) + x = self.o(x) + x = self.dropout(x) + return x + + +class T5FeedForward(nn.Module): + + def __init__(self, dim, dim_ffn, dropout=0.1): + super(T5FeedForward, self).__init__() + self.dim = dim + self.dim_ffn = dim_ffn + + # layers + self.gate = nn.Sequential(nn.Linear(dim, dim_ffn, bias=False), GELU()) + self.fc1 = nn.Linear(dim, dim_ffn, bias=False) + self.fc2 = nn.Linear(dim_ffn, dim, bias=False) + self.dropout = nn.Dropout(dropout) + + def forward(self, x): + x = self.fc1(x) * self.gate(x) + x = self.dropout(x) + x = self.fc2(x) + x = self.dropout(x) + return x + + +class T5SelfAttention(nn.Module): + + def __init__(self, + dim, + dim_attn, + dim_ffn, + num_heads, + num_buckets, + shared_pos=True, + dropout=0.1): + super(T5SelfAttention, self).__init__() + self.dim = dim + self.dim_attn = dim_attn + self.dim_ffn = dim_ffn + self.num_heads = num_heads + self.num_buckets = num_buckets + self.shared_pos = shared_pos + + # layers + self.norm1 = T5LayerNorm(dim) + self.attn = T5Attention(dim, dim_attn, num_heads, dropout) + self.norm2 = T5LayerNorm(dim) + self.ffn = T5FeedForward(dim, dim_ffn, dropout) + self.pos_embedding = None if shared_pos else T5RelativeEmbedding( + num_buckets, num_heads, bidirectional=True) + + def forward(self, x, mask=None, pos_bias=None): + e = pos_bias if self.shared_pos else self.pos_embedding( + x.size(1), x.size(1)) + x = fp16_clamp(x + self.attn(self.norm1(x), mask=mask, pos_bias=e)) + x = fp16_clamp(x + self.ffn(self.norm2(x))) + return x + + +class T5RelativeEmbedding(nn.Module): + + def __init__(self, num_buckets, num_heads, bidirectional, max_dist=128): + super(T5RelativeEmbedding, self).__init__() + self.num_buckets = num_buckets + self.num_heads = num_heads + self.bidirectional = bidirectional + self.max_dist = max_dist + + # layers + self.embedding = nn.Embedding(num_buckets, num_heads) + + def forward(self, lq, lk): + device = self.embedding.weight.device + # rel_pos = torch.arange(lk).unsqueeze(0).to(device) - \ + # torch.arange(lq).unsqueeze(1).to(device) + rel_pos = torch.arange(lk, device=device).unsqueeze(0) - \ + torch.arange(lq, device=device).unsqueeze(1) + rel_pos = self._relative_position_bucket(rel_pos) + rel_pos_embeds = self.embedding(rel_pos) + rel_pos_embeds = rel_pos_embeds.permute(2, 0, 1).unsqueeze( + 0) # [1, N, Lq, Lk] + return rel_pos_embeds.contiguous() + + def _relative_position_bucket(self, rel_pos): + # preprocess + if self.bidirectional: + num_buckets = self.num_buckets // 2 + rel_buckets = (rel_pos > 0).long() * num_buckets + rel_pos = torch.abs(rel_pos) + else: + num_buckets = self.num_buckets + rel_buckets = 0 + rel_pos = -torch.min(rel_pos, torch.zeros_like(rel_pos)) + + # embeddings for small and large positions + max_exact = num_buckets // 2 + rel_pos_large = max_exact + (torch.log(rel_pos.float() / max_exact) / + math.log(self.max_dist / max_exact) * + (num_buckets - max_exact)).long() + rel_pos_large = torch.min( + rel_pos_large, torch.full_like(rel_pos_large, num_buckets - 1)) + rel_buckets += torch.where(rel_pos < max_exact, rel_pos, rel_pos_large) + return rel_buckets + +def init_weights(m): + if isinstance(m, T5LayerNorm): + nn.init.ones_(m.weight) + elif isinstance(m, T5FeedForward): + nn.init.normal_(m.gate[0].weight, std=m.dim**-0.5) + nn.init.normal_(m.fc1.weight, std=m.dim**-0.5) + nn.init.normal_(m.fc2.weight, std=m.dim_ffn**-0.5) + elif isinstance(m, T5Attention): + nn.init.normal_(m.q.weight, std=(m.dim * m.dim_attn)**-0.5) + nn.init.normal_(m.k.weight, std=m.dim**-0.5) + nn.init.normal_(m.v.weight, std=m.dim**-0.5) + nn.init.normal_(m.o.weight, std=(m.num_heads * m.dim_attn)**-0.5) + elif isinstance(m, T5RelativeEmbedding): + nn.init.normal_( + m.embedding.weight, std=(2 * m.num_buckets * m.num_heads)**-0.5) + + +class WanTextEncoder(torch.nn.Module): + + def __init__(self, + vocab=256384, + dim=4096, + dim_attn=4096, + dim_ffn=10240, + num_heads=64, + num_layers=24, + num_buckets=32, + shared_pos=False, + dropout=0.1): + super(WanTextEncoder, self).__init__() + self.dim = dim + self.dim_attn = dim_attn + self.dim_ffn = dim_ffn + self.num_heads = num_heads + self.num_layers = num_layers + self.num_buckets = num_buckets + self.shared_pos = shared_pos + + # layers + self.token_embedding = vocab if isinstance(vocab, nn.Embedding) \ + else nn.Embedding(vocab, dim) + self.pos_embedding = T5RelativeEmbedding( + num_buckets, num_heads, bidirectional=True) if shared_pos else None + self.dropout = nn.Dropout(dropout) + self.blocks = nn.ModuleList([ + T5SelfAttention(dim, dim_attn, dim_ffn, num_heads, num_buckets, + shared_pos, dropout) for _ in range(num_layers) + ]) + self.norm = T5LayerNorm(dim) + + # Skip costly random init when loading pretrained checkpoints. + # Verified: checkpoint fully covers this module (missing/unexpected keys are both 0). + # self.apply(init_weights) + + def forward(self, ids, mask=None): + x = self.token_embedding(ids) + x = self.dropout(x) + e = self.pos_embedding(x.size(1), + x.size(1)) if self.shared_pos else None + for block in self.blocks: + x = block(x, mask, pos_bias=e) + x = self.norm(x) + x = self.dropout(x) + return x + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +def canonicalize(text, keep_punctuation_exact_string=None): + text = text.replace('_', ' ') + if keep_punctuation_exact_string: + text = keep_punctuation_exact_string.join( + part.translate(str.maketrans('', '', string.punctuation)) + for part in text.split(keep_punctuation_exact_string)) + else: + text = text.translate(str.maketrans('', '', string.punctuation)) + text = text.lower() + text = re.sub(r'\s+', ' ', text) + return text.strip() + + +class HuggingfaceTokenizer: + + def __init__(self, name, seq_len=None, clean=None, **kwargs): + assert clean in (None, 'whitespace', 'lower', 'canonicalize') + self.name = name + self.seq_len = seq_len + self.clean = clean + + # init tokenizer + self.tokenizer = AutoTokenizer.from_pretrained(name, **kwargs) + self.vocab_size = self.tokenizer.vocab_size + + def __call__(self, sequence, **kwargs): + return_mask = kwargs.pop('return_mask', False) + + # arguments + _kwargs = {'return_tensors': 'pt'} + if self.seq_len is not None: + _kwargs.update({ + 'padding': 'max_length', + 'truncation': True, + 'max_length': self.seq_len + }) + _kwargs.update(**kwargs) + + # tokenization + if isinstance(sequence, str): + sequence = [sequence] + if self.clean: + sequence = [self._clean(u) for u in sequence] + ids = self.tokenizer(sequence, **_kwargs) + + # output + if return_mask: + return ids.input_ids, ids.attention_mask + else: + return ids.input_ids + + def _clean(self, text): + if self.clean == 'whitespace': + text = whitespace_clean(basic_clean(text)) + elif self.clean == 'lower': + text = whitespace_clean(basic_clean(text)).lower() + elif self.clean == 'canonicalize': + text = canonicalize(basic_clean(text)) + return text diff --git a/baseline/fastwam_zipper/source/src/fastwam/models/wan22/wan_video_vae.py b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/wan_video_vae.py new file mode 100644 index 0000000000000000000000000000000000000000..b2e99cb802dc082eb055bd3ac349897112faa536 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/models/wan22/wan_video_vae.py @@ -0,0 +1,1375 @@ +from einops import rearrange, repeat + +import torch +import torch.nn as nn +import torch.nn.functional as F +from tqdm import tqdm + +CACHE_T = 2 + + +def check_is_instance(model, module_class): + if isinstance(model, module_class): + return True + if hasattr(model, "module") and isinstance(model.module, module_class): + return True + return False + + +def block_causal_mask(x, block_size): + # params + b, n, s, _, device = *x.size(), x.device + assert s % block_size == 0 + num_blocks = s // block_size + + # build mask + mask = torch.zeros(b, n, s, s, dtype=torch.bool, device=device) + for i in range(num_blocks): + mask[:, :, + i * block_size:(i + 1) * block_size, :(i + 1) * block_size] = 1 + return mask + + +class CausalConv3d(nn.Conv3d): + """ + Causal 3d convolusion. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._padding = (self.padding[2], self.padding[2], self.padding[1], + self.padding[1], 2 * self.padding[0], 0) + self.padding = (0, 0, 0) + + def forward(self, x, cache_x=None): + padding = list(self._padding) + if cache_x is not None and self._padding[4] > 0: + cache_x = cache_x.to(x.device) + x = torch.cat([cache_x, x], dim=2) + padding[4] -= cache_x.shape[2] + x = F.pad(x, padding) + + return super().forward(x) + + +class RMS_norm(nn.Module): + + def __init__(self, dim, channel_first=True, images=True, bias=False): + super().__init__() + broadcastable_dims = (1, 1, 1) if not images else (1, 1) + shape = (dim, *broadcastable_dims) if channel_first else (dim,) + + self.channel_first = channel_first + self.scale = dim**0.5 + self.gamma = nn.Parameter(torch.ones(shape)) + self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0. + + def forward(self, x): + return F.normalize( + x, dim=(1 if self.channel_first else + -1)) * self.scale * self.gamma + self.bias + + +class Upsample(nn.Upsample): + + def forward(self, x): + """ + Fix bfloat16 support for nearest neighbor interpolation. + """ + return super().forward(x.float()).type_as(x) + + +class Resample(nn.Module): + + def __init__(self, dim, mode): + assert mode in ('none', 'upsample2d', 'upsample3d', 'downsample2d', + 'downsample3d') + super().__init__() + self.dim = dim + self.mode = mode + + # layers + if mode == 'upsample2d': + self.resample = nn.Sequential( + Upsample(scale_factor=(2., 2.), mode='nearest-exact'), + nn.Conv2d(dim, dim // 2, 3, padding=1)) + elif mode == 'upsample3d': + self.resample = nn.Sequential( + Upsample(scale_factor=(2., 2.), mode='nearest-exact'), + nn.Conv2d(dim, dim // 2, 3, padding=1)) + self.time_conv = CausalConv3d(dim, + dim * 2, (3, 1, 1), + padding=(1, 0, 0)) + + elif mode == 'downsample2d': + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), + nn.Conv2d(dim, dim, 3, stride=(2, 2))) + elif mode == 'downsample3d': + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), + nn.Conv2d(dim, dim, 3, stride=(2, 2))) + self.time_conv = CausalConv3d(dim, + dim, (3, 1, 1), + stride=(2, 1, 1), + padding=(0, 0, 0)) + + else: + self.resample = nn.Identity() + + def forward(self, x, feat_cache=None, feat_idx=[0]): + b, c, t, h, w = x.size() + if self.mode == 'upsample3d': + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = 'Rep' + feat_idx[0] += 1 + else: + + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[ + idx] is not None and feat_cache[idx] != 'Rep': + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + if cache_x.shape[2] < 2 and feat_cache[ + idx] is not None and feat_cache[idx] == 'Rep': + cache_x = torch.cat([ + torch.zeros_like(cache_x).to(cache_x.device), + cache_x + ], + dim=2) + if feat_cache[idx] == 'Rep': + x = self.time_conv(x) + else: + x = self.time_conv(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + + x = x.reshape(b, 2, c, t, h, w) + x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]), + 3) + x = x.reshape(b, c, t * 2, h, w) + t = x.shape[2] + x = rearrange(x, 'b c t h w -> (b t) c h w') + x = self.resample(x) + x = rearrange(x, '(b t) c h w -> b c t h w', t=t) + + if self.mode == 'downsample3d': + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = x.clone() + feat_idx[0] += 1 + else: + cache_x = x[:, :, -1:, :, :].clone() + x = self.time_conv( + torch.cat([feat_cache[idx][:, :, -1:, :, :], x], 2)) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + return x, feat_cache, feat_idx + + def init_weight(self, conv): + conv_weight = conv.weight + nn.init.zeros_(conv_weight) + c1, c2, t, h, w = conv_weight.size() + one_matrix = torch.eye(c1, c2) + init_matrix = one_matrix + nn.init.zeros_(conv_weight) + conv_weight.data[:, :, 1, 0, 0] = init_matrix + conv.weight.data.copy_(conv_weight) + nn.init.zeros_(conv.bias.data) + + def init_weight2(self, conv): + conv_weight = conv.weight.data + nn.init.zeros_(conv_weight) + c1, c2, t, h, w = conv_weight.size() + init_matrix = torch.eye(c1 // 2, c2) + conv_weight[:c1 // 2, :, -1, 0, 0] = init_matrix + conv_weight[c1 // 2:, :, -1, 0, 0] = init_matrix + conv.weight.data.copy_(conv_weight) + nn.init.zeros_(conv.bias.data) + + + +def patchify(x, patch_size): + if patch_size == 1: + return x + if x.dim() == 4: + x = rearrange(x, "b c (h q) (w r) -> b (c r q) h w", q=patch_size, r=patch_size) + elif x.dim() == 5: + x = rearrange(x, + "b c f (h q) (w r) -> b (c r q) f h w", + q=patch_size, + r=patch_size) + else: + raise ValueError(f"Invalid input shape: {x.shape}") + return x + + +def unpatchify(x, patch_size): + if patch_size == 1: + return x + if x.dim() == 4: + x = rearrange(x, "b (c r q) h w -> b c (h q) (w r)", q=patch_size, r=patch_size) + elif x.dim() == 5: + x = rearrange(x, + "b (c r q) f h w -> b c f (h q) (w r)", + q=patch_size, + r=patch_size) + return x + + +class Resample38(Resample): + + def __init__(self, dim, mode): + assert mode in ( + "none", + "upsample2d", + "upsample3d", + "downsample2d", + "downsample3d", + ) + super(Resample, self).__init__() + self.dim = dim + self.mode = mode + + # layers + if mode == "upsample2d": + self.resample = nn.Sequential( + Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, dim, 3, padding=1), + ) + elif mode == "upsample3d": + self.resample = nn.Sequential( + Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, dim, 3, padding=1), + ) + self.time_conv = CausalConv3d(dim, dim * 2, (3, 1, 1), padding=(1, 0, 0)) + elif mode == "downsample2d": + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2)) + ) + elif mode == "downsample3d": + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2)) + ) + self.time_conv = CausalConv3d( + dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0) + ) + else: + self.resample = nn.Identity() + +class ResidualBlock(nn.Module): + + def __init__(self, in_dim, out_dim, dropout=0.0): + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + + # layers + self.residual = nn.Sequential( + RMS_norm(in_dim, images=False), nn.SiLU(), + CausalConv3d(in_dim, out_dim, 3, padding=1), + RMS_norm(out_dim, images=False), nn.SiLU(), nn.Dropout(dropout), + CausalConv3d(out_dim, out_dim, 3, padding=1)) + self.shortcut = CausalConv3d(in_dim, out_dim, 1) \ + if in_dim != out_dim else nn.Identity() + + def forward(self, x, feat_cache=None, feat_idx=[0]): + h = self.shortcut(x) + for layer in self.residual: + if check_is_instance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x + h, feat_cache, feat_idx + + +class AttentionBlock(nn.Module): + """ + Causal self-attention with a single head. + """ + + def __init__(self, dim): + super().__init__() + self.dim = dim + + # layers + self.norm = RMS_norm(dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x): + identity = x + b, c, t, h, w = x.size() + x = rearrange(x, 'b c t h w -> (b t) c h w') + x = self.norm(x) + # compute query, key, value + q, k, v = self.to_qkv(x).reshape(b * t, 1, c * 3, -1).permute( + 0, 1, 3, 2).contiguous().chunk(3, dim=-1) + + # apply attention + x = F.scaled_dot_product_attention( + q, + k, + v, + #attn_mask=block_causal_mask(q, block_size=h * w) + ) + x = x.squeeze(1).permute(0, 2, 1).reshape(b * t, c, h, w) + + # output + x = self.proj(x) + x = rearrange(x, '(b t) c h w-> b c t h w', t=t) + return x + identity + + +class AvgDown3D(nn.Module): + def __init__( + self, + in_channels, + out_channels, + factor_t, + factor_s=1, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.factor_t = factor_t + self.factor_s = factor_s + self.factor = self.factor_t * self.factor_s * self.factor_s + + assert in_channels * self.factor % out_channels == 0 + self.group_size = in_channels * self.factor // out_channels + + def forward(self, x: torch.Tensor) -> torch.Tensor: + pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t + pad = (0, 0, 0, 0, pad_t, 0) + x = F.pad(x, pad) + B, C, T, H, W = x.shape + x = x.view( + B, + C, + T // self.factor_t, + self.factor_t, + H // self.factor_s, + self.factor_s, + W // self.factor_s, + self.factor_s, + ) + x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous() + x = x.view( + B, + C * self.factor, + T // self.factor_t, + H // self.factor_s, + W // self.factor_s, + ) + x = x.view( + B, + self.out_channels, + self.group_size, + T // self.factor_t, + H // self.factor_s, + W // self.factor_s, + ) + x = x.mean(dim=2) + return x + + +class DupUp3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + factor_t, + factor_s=1, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + + self.factor_t = factor_t + self.factor_s = factor_s + self.factor = self.factor_t * self.factor_s * self.factor_s + + assert out_channels * self.factor % in_channels == 0 + self.repeats = out_channels * self.factor // in_channels + + def forward(self, x: torch.Tensor, first_chunk=False) -> torch.Tensor: + x = x.repeat_interleave(self.repeats, dim=1) + x = x.view( + x.size(0), + self.out_channels, + self.factor_t, + self.factor_s, + self.factor_s, + x.size(2), + x.size(3), + x.size(4), + ) + x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous() + x = x.view( + x.size(0), + self.out_channels, + x.size(2) * self.factor_t, + x.size(4) * self.factor_s, + x.size(6) * self.factor_s, + ) + if first_chunk: + x = x[:, :, self.factor_t - 1 :, :, :] + return x + + +class Down_ResidualBlock(nn.Module): + def __init__( + self, in_dim, out_dim, dropout, mult, temperal_downsample=False, down_flag=False + ): + super().__init__() + + # Shortcut path with downsample + self.avg_shortcut = AvgDown3D( + in_dim, + out_dim, + factor_t=2 if temperal_downsample else 1, + factor_s=2 if down_flag else 1, + ) + + # Main path with residual blocks and downsample + downsamples = [] + for _ in range(mult): + downsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + in_dim = out_dim + + # Add the final downsample block + if down_flag: + mode = "downsample3d" if temperal_downsample else "downsample2d" + downsamples.append(Resample38(out_dim, mode=mode)) + + self.downsamples = nn.Sequential(*downsamples) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + x_copy = x.clone() + for module in self.downsamples: + x, feat_cache, feat_idx = module(x, feat_cache, feat_idx) + + return x + self.avg_shortcut(x_copy), feat_cache, feat_idx + + +class Up_ResidualBlock(nn.Module): + def __init__( + self, in_dim, out_dim, dropout, mult, temperal_upsample=False, up_flag=False + ): + super().__init__() + # Shortcut path with upsample + if up_flag: + self.avg_shortcut = DupUp3D( + in_dim, + out_dim, + factor_t=2 if temperal_upsample else 1, + factor_s=2 if up_flag else 1, + ) + else: + self.avg_shortcut = None + + # Main path with residual blocks and upsample + upsamples = [] + for _ in range(mult): + upsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + in_dim = out_dim + + # Add the final upsample block + if up_flag: + mode = "upsample3d" if temperal_upsample else "upsample2d" + upsamples.append(Resample38(out_dim, mode=mode)) + + self.upsamples = nn.Sequential(*upsamples) + + def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False): + x_main = x.clone() + for module in self.upsamples: + x_main, feat_cache, feat_idx = module(x_main, feat_cache, feat_idx) + if self.avg_shortcut is not None: + x_shortcut = self.avg_shortcut(x, first_chunk) + x_main = x_main + x_shortcut + return x_main, feat_cache, feat_idx + + +class Encoder3d(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[True, True, False], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + + # dimensions + dims = [dim * u for u in [1] + dim_mult] + scale = 1.0 + + # init block + self.conv1 = CausalConv3d(3, dims[0], 3, padding=1) + + # downsample blocks + downsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + for _ in range(num_res_blocks): + downsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + if scale in attn_scales: + downsamples.append(AttentionBlock(out_dim)) + in_dim = out_dim + + # downsample block + if i != len(dim_mult) - 1: + mode = 'downsample3d' if temperal_downsample[ + i] else 'downsample2d' + downsamples.append(Resample(out_dim, mode=mode)) + scale /= 2.0 + self.downsamples = nn.Sequential(*downsamples) + + # middle blocks + self.middle = nn.Sequential(ResidualBlock(out_dim, out_dim, dropout), + AttentionBlock(out_dim), + ResidualBlock(out_dim, out_dim, dropout)) + + # output blocks + self.head = nn.Sequential(RMS_norm(out_dim, images=False), nn.SiLU(), + CausalConv3d(out_dim, z_dim, 3, padding=1)) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + ## downsamples + for layer in self.downsamples: + if feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## middle + for layer in self.middle: + if check_is_instance(layer, ResidualBlock) and feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## head + for layer in self.head: + if check_is_instance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x, feat_cache, feat_idx + + +class Encoder3d_38(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[False, True, True], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + + # dimensions + dims = [dim * u for u in [1] + dim_mult] + scale = 1.0 + + # init block + self.conv1 = CausalConv3d(12, dims[0], 3, padding=1) + + # downsample blocks + downsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + t_down_flag = ( + temperal_downsample[i] if i < len(temperal_downsample) else False + ) + downsamples.append( + Down_ResidualBlock( + in_dim=in_dim, + out_dim=out_dim, + dropout=dropout, + mult=num_res_blocks, + temperal_downsample=t_down_flag, + down_flag=i != len(dim_mult) - 1, + ) + ) + scale /= 2.0 + self.downsamples = nn.Sequential(*downsamples) + + # middle blocks + self.middle = nn.Sequential( + ResidualBlock(out_dim, out_dim, dropout), + AttentionBlock(out_dim), + ResidualBlock(out_dim, out_dim, dropout), + ) + + # # output blocks + self.head = nn.Sequential( + RMS_norm(out_dim, images=False), + nn.SiLU(), + CausalConv3d(out_dim, z_dim, 3, padding=1), + ) + + + def forward(self, x, feat_cache=None, feat_idx=[0]): + + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + ## downsamples + for layer in self.downsamples: + if feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## middle + for layer in self.middle: + if isinstance(layer, ResidualBlock) and feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## head + for layer in self.head: + if isinstance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :] + .unsqueeze(2) + .to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + + return x, feat_cache, feat_idx + + +class Decoder3d(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_upsample=[False, True, True], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_upsample = temperal_upsample + + # dimensions + dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + scale = 1.0 / 2**(len(dim_mult) - 2) + + # init block + self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1) + + # middle blocks + self.middle = nn.Sequential(ResidualBlock(dims[0], dims[0], dropout), + AttentionBlock(dims[0]), + ResidualBlock(dims[0], dims[0], dropout)) + + # upsample blocks + upsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + if i == 1 or i == 2 or i == 3: + in_dim = in_dim // 2 + for _ in range(num_res_blocks + 1): + upsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + if scale in attn_scales: + upsamples.append(AttentionBlock(out_dim)) + in_dim = out_dim + + # upsample block + if i != len(dim_mult) - 1: + mode = 'upsample3d' if temperal_upsample[i] else 'upsample2d' + upsamples.append(Resample(out_dim, mode=mode)) + scale *= 2.0 + self.upsamples = nn.Sequential(*upsamples) + + # output blocks + self.head = nn.Sequential(RMS_norm(out_dim, images=False), nn.SiLU(), + CausalConv3d(out_dim, 3, 3, padding=1)) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + ## conv1 + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + ## middle + for layer in self.middle: + if check_is_instance(layer, ResidualBlock) and feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## upsamples + for layer in self.upsamples: + if feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## head + for layer in self.head: + if check_is_instance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x, feat_cache, feat_idx + + + +class Decoder3d_38(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_upsample=[False, True, True], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_upsample = temperal_upsample + + # dimensions + dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + scale = 1.0 / 2 ** (len(dim_mult) - 2) + # init block + self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1) + + # middle blocks + self.middle = nn.Sequential(ResidualBlock(dims[0], dims[0], dropout), + AttentionBlock(dims[0]), + ResidualBlock(dims[0], dims[0], dropout)) + + # upsample blocks + upsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + t_up_flag = temperal_upsample[i] if i < len(temperal_upsample) else False + upsamples.append( + Up_ResidualBlock(in_dim=in_dim, + out_dim=out_dim, + dropout=dropout, + mult=num_res_blocks + 1, + temperal_upsample=t_up_flag, + up_flag=i != len(dim_mult) - 1)) + self.upsamples = nn.Sequential(*upsamples) + + # output blocks + self.head = nn.Sequential(RMS_norm(out_dim, images=False), nn.SiLU(), + CausalConv3d(out_dim, 12, 3, padding=1)) + + + def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False): + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + for layer in self.middle: + if check_is_instance(layer, ResidualBlock) and feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## upsamples + for layer in self.upsamples: + if feat_cache is not None: + x, feat_cache, feat_idx = layer(x, feat_cache, feat_idx, first_chunk) + else: + x = layer(x) + + ## head + for layer in self.head: + if check_is_instance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :] + .unsqueeze(2) + .to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x, feat_cache, feat_idx + + +def count_conv3d(model): + count = 0 + for m in model.modules(): + if isinstance(m, CausalConv3d): + count += 1 + return count + + +class VideoVAE_(nn.Module): + + def __init__(self, + dim=96, + z_dim=16, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[False, True, True], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + self.temperal_upsample = temperal_downsample[::-1] + + # modules + self.encoder = Encoder3d(dim, z_dim * 2, dim_mult, num_res_blocks, + attn_scales, self.temperal_downsample, dropout) + self.conv1 = CausalConv3d(z_dim * 2, z_dim * 2, 1) + self.conv2 = CausalConv3d(z_dim, z_dim, 1) + self.decoder = Decoder3d(dim, z_dim, dim_mult, num_res_blocks, + attn_scales, self.temperal_upsample, dropout) + + def forward(self, x): + mu, log_var = self.encode(x) + z = self.reparameterize(mu, log_var) + x_recon = self.decode(z) + return x_recon, mu, log_var + + def encode(self, x, scale): + self.clear_cache() + ## cache + t = x.shape[2] + iter_ = 1 + (t - 1) // 4 + + for i in range(iter_): + self._enc_conv_idx = [0] + if i == 0: + out, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, :1, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx) + else: + out_, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, 1 + 4 * (i - 1):1 + 4 * i, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx) + out = torch.cat([out, out_], 2) + mu, log_var = self.conv1(out).chunk(2, dim=1) + if isinstance(scale[0], torch.Tensor): + scale = [s.to(dtype=mu.dtype, device=mu.device) for s in scale] + mu = (mu - scale[0].view(1, self.z_dim, 1, 1, 1)) * scale[1].view( + 1, self.z_dim, 1, 1, 1) + else: + scale = scale.to(dtype=mu.dtype, device=mu.device) + mu = (mu - scale[0]) * scale[1] + return mu + + def decode(self, z, scale): + self.clear_cache() + # z: [b,c,t,h,w] + if isinstance(scale[0], torch.Tensor): + scale = [s.to(dtype=z.dtype, device=z.device) for s in scale] + z = z / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view( + 1, self.z_dim, 1, 1, 1) + else: + scale = scale.to(dtype=z.dtype, device=z.device) + z = z / scale[1] + scale[0] + iter_ = z.shape[2] + x = self.conv2(z) + for i in range(iter_): + self._conv_idx = [0] + if i == 0: + out, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx) + else: + out_, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx) + out = torch.cat([out, out_], 2) # may add tensor offload + return out + + def reparameterize(self, mu, log_var): + std = torch.exp(0.5 * log_var) + eps = torch.randn_like(std) + return eps * std + mu + + def sample(self, imgs, deterministic=False): + mu, log_var = self.encode(imgs) + if deterministic: + return mu + std = torch.exp(0.5 * log_var.clamp(-30.0, 20.0)) + return mu + std * torch.randn_like(std) + + def clear_cache(self): + self._conv_num = count_conv3d(self.decoder) + self._conv_idx = [0] + self._feat_map = [None] * self._conv_num + # cache encode + self._enc_conv_num = count_conv3d(self.encoder) + self._enc_conv_idx = [0] + self._enc_feat_map = [None] * self._enc_conv_num + + +class WanVideoVAE(nn.Module): + + def __init__(self, z_dim=16): + super().__init__() + + mean = [ + -0.7571, -0.7089, -0.9113, 0.1075, -0.1745, 0.9653, -0.1517, 1.5508, + 0.4134, -0.0715, 0.5517, -0.3632, -0.1922, -0.9497, 0.2503, -0.2921 + ] + std = [ + 2.8184, 1.4541, 2.3275, 2.6558, 1.2196, 1.7708, 2.6052, 2.0743, + 3.2687, 2.1526, 2.8652, 1.5579, 1.6382, 1.1253, 2.8251, 1.9160 + ] + self.register_buffer("mean", torch.tensor(mean), persistent=False) + self.register_buffer("inv_std", torch.tensor(std).reciprocal(), persistent=False) + + # init model + self.model = VideoVAE_(z_dim=z_dim).eval().requires_grad_(False) + self.upsampling_factor = 8 + self.temporal_downsample_factor = 4 + self.z_dim = z_dim + + @property + def scale(self): + return [self.mean, self.inv_std] + + + def build_1d_mask(self, length, left_bound, right_bound, border_width): + x = torch.ones((length,)) + if not left_bound: + x[:border_width] = (torch.arange(border_width) + 1) / border_width + if not right_bound: + x[-border_width:] = torch.flip((torch.arange(border_width) + 1) / border_width, dims=(0,)) + return x + + + def build_mask(self, data, is_bound, border_width): + _, _, _, H, W = data.shape + h = self.build_1d_mask(H, is_bound[0], is_bound[1], border_width[0]) + w = self.build_1d_mask(W, is_bound[2], is_bound[3], border_width[1]) + + h = repeat(h, "H -> H W", H=H, W=W) + w = repeat(w, "W -> H W", H=H, W=W) + + mask = torch.stack([h, w]).min(dim=0).values + mask = rearrange(mask, "H W -> 1 1 1 H W") + return mask + + + def tiled_decode(self, hidden_states, device, tile_size, tile_stride): + _, _, T, H, W = hidden_states.shape + size_h, size_w = tile_size + stride_h, stride_w = tile_stride + + # Split tasks + tasks = [] + for h in range(0, H, stride_h): + if (h-stride_h >= 0 and h-stride_h+size_h >= H): continue + for w in range(0, W, stride_w): + if (w-stride_w >= 0 and w-stride_w+size_w >= W): continue + h_, w_ = h + size_h, w + size_w + tasks.append((h, h_, w, w_)) + + data_device = "cpu" + computation_device = device + + out_T = T * 4 - 3 + weight = torch.zeros((1, 1, out_T, H * self.upsampling_factor, W * self.upsampling_factor), dtype=hidden_states.dtype, device=data_device) + values = torch.zeros((1, 3, out_T, H * self.upsampling_factor, W * self.upsampling_factor), dtype=hidden_states.dtype, device=data_device) + + for h, h_, w, w_ in tqdm(tasks, desc="VAE decoding"): + hidden_states_batch = hidden_states[:, :, :, h:h_, w:w_].to(computation_device) + hidden_states_batch = self.model.decode(hidden_states_batch, self.scale).to(data_device) + + mask = self.build_mask( + hidden_states_batch, + is_bound=(h==0, h_>=H, w==0, w_>=W), + border_width=((size_h - stride_h) * self.upsampling_factor, (size_w - stride_w) * self.upsampling_factor) + ).to(dtype=hidden_states.dtype, device=data_device) + + target_h = h * self.upsampling_factor + target_w = w * self.upsampling_factor + values[ + :, + :, + :, + target_h:target_h + hidden_states_batch.shape[3], + target_w:target_w + hidden_states_batch.shape[4], + ] += hidden_states_batch * mask + weight[ + :, + :, + :, + target_h: target_h + hidden_states_batch.shape[3], + target_w: target_w + hidden_states_batch.shape[4], + ] += mask + values = values / weight + values = values.clamp_(-1, 1) + return values + + + def tiled_encode(self, video, device, tile_size, tile_stride): + _, _, T, H, W = video.shape + size_h, size_w = tile_size + stride_h, stride_w = tile_stride + + # Split tasks + tasks = [] + for h in range(0, H, stride_h): + if (h-stride_h >= 0 and h-stride_h+size_h >= H): continue + for w in range(0, W, stride_w): + if (w-stride_w >= 0 and w-stride_w+size_w >= W): continue + h_, w_ = h + size_h, w + size_w + tasks.append((h, h_, w, w_)) + + data_device = "cpu" + computation_device = device + + out_T = (T + 3) // 4 + weight = torch.zeros((1, 1, out_T, H // self.upsampling_factor, W // self.upsampling_factor), dtype=video.dtype, device=data_device) + values = torch.zeros((1, self.z_dim, out_T, H // self.upsampling_factor, W // self.upsampling_factor), dtype=video.dtype, device=data_device) + + for h, h_, w, w_ in tqdm(tasks, desc="VAE encoding"): + hidden_states_batch = video[:, :, :, h:h_, w:w_].to(computation_device) + hidden_states_batch = self.model.encode(hidden_states_batch, self.scale).to(data_device) + + mask = self.build_mask( + hidden_states_batch, + is_bound=(h==0, h_>=H, w==0, w_>=W), + border_width=((size_h - stride_h) // self.upsampling_factor, (size_w - stride_w) // self.upsampling_factor) + ).to(dtype=video.dtype, device=data_device) + + target_h = h // self.upsampling_factor + target_w = w // self.upsampling_factor + values[ + :, + :, + :, + target_h:target_h + hidden_states_batch.shape[3], + target_w:target_w + hidden_states_batch.shape[4], + ] += hidden_states_batch * mask + weight[ + :, + :, + :, + target_h: target_h + hidden_states_batch.shape[3], + target_w: target_w + hidden_states_batch.shape[4], + ] += mask + values = values / weight + return values + + + def single_encode(self, video, device): + video = video.to(device) + x = self.model.encode(video, self.scale) + return x + + + def single_decode(self, hidden_state, device): + hidden_state = hidden_state.to(device) + video = self.model.decode(hidden_state, self.scale) + return video.clamp_(-1, 1) + + + def encode(self, videos, device, tiled=False, tile_size=(34, 34), tile_stride=(18, 16)): + raise NotImplementedError( + "WanVideoVAE.encode() legacy per-video loop is disabled. " + "Call `vae.model.encode(batched_video, vae.scale)` for fixed-shape batched encode." + ) + + + def decode(self, hidden_states, device, tiled=False, tile_size=(34, 34), tile_stride=(18, 16)): + hidden_states = [hidden_state.to("cpu") for hidden_state in hidden_states] + videos = [] + for hidden_state in hidden_states: + hidden_state = hidden_state.unsqueeze(0) + if tiled: + video = self.tiled_decode(hidden_state, device, tile_size, tile_stride) + else: + video = self.single_decode(hidden_state, device) + video = video.squeeze(0) + videos.append(video) + videos = torch.stack(videos) + return videos + + + @staticmethod + def state_dict_converter(): + return WanVideoVAEStateDictConverter() + + +class WanVideoVAEStateDictConverter: + + def __init__(self): + pass + + def from_civitai(self, state_dict): + state_dict_ = {} + if 'model_state' in state_dict: + state_dict = state_dict['model_state'] + for name in state_dict: + state_dict_['model.' + name] = state_dict[name] + return state_dict_ + + +class VideoVAE38_(VideoVAE_): + + def __init__(self, + dim=160, + z_dim=48, + dec_dim=256, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[False, True, True], + dropout=0.0): + super(VideoVAE_, self).__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + self.temperal_upsample = temperal_downsample[::-1] + + # modules + self.encoder = Encoder3d_38(dim, z_dim * 2, dim_mult, num_res_blocks, + attn_scales, self.temperal_downsample, dropout) + self.conv1 = CausalConv3d(z_dim * 2, z_dim * 2, 1) + self.conv2 = CausalConv3d(z_dim, z_dim, 1) + self.decoder = Decoder3d_38(dec_dim, z_dim, dim_mult, num_res_blocks, + attn_scales, self.temperal_upsample, dropout) + + + def encode(self, x, scale): + self.clear_cache() + x = patchify(x, patch_size=2) + t = x.shape[2] + iter_ = 1 + (t - 1) // 4 + for i in range(iter_): + self._enc_conv_idx = [0] + if i == 0: + out, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, :1, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx) + else: + out_, self._enc_feat_map, self._enc_conv_idx = self.encoder(x[:, :, 1 + 4 * (i - 1):1 + 4 * i, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx) + out = torch.cat([out, out_], 2) + mu, log_var = self.conv1(out).chunk(2, dim=1) + if isinstance(scale[0], torch.Tensor): + scale = [s.to(dtype=mu.dtype, device=mu.device) for s in scale] + mu = (mu - scale[0].view(1, self.z_dim, 1, 1, 1)) * scale[1].view( + 1, self.z_dim, 1, 1, 1) + else: + scale = scale.to(dtype=mu.dtype, device=mu.device) + mu = (mu - scale[0]) * scale[1] + self.clear_cache() + return mu + + + def decode(self, z, scale): + self.clear_cache() + if isinstance(scale[0], torch.Tensor): + scale = [s.to(dtype=z.dtype, device=z.device) for s in scale] + z = z / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view( + 1, self.z_dim, 1, 1, 1) + else: + scale = scale.to(dtype=z.dtype, device=z.device) + z = z / scale[1] + scale[0] + iter_ = z.shape[2] + x = self.conv2(z) + for i in range(iter_): + self._conv_idx = [0] + if i == 0: + out, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx, + first_chunk=True) + else: + out_, self._feat_map, self._conv_idx = self.decoder(x[:, :, i:i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx) + out = torch.cat([out, out_], 2) + out = unpatchify(out, patch_size=2) + self.clear_cache() + return out + + +class WanVideoVAE38(WanVideoVAE): + + def __init__(self, z_dim=48, dim=160): + super(WanVideoVAE, self).__init__() + + mean = [ + -0.2289, -0.0052, -0.1323, -0.2339, -0.2799, 0.0174, 0.1838, 0.1557, + -0.1382, 0.0542, 0.2813, 0.0891, 0.1570, -0.0098, 0.0375, -0.1825, + -0.2246, -0.1207, -0.0698, 0.5109, 0.2665, -0.2108, -0.2158, 0.2502, + -0.2055, -0.0322, 0.1109, 0.1567, -0.0729, 0.0899, -0.2799, -0.1230, + -0.0313, -0.1649, 0.0117, 0.0723, -0.2839, -0.2083, -0.0520, 0.3748, + 0.0152, 0.1957, 0.1433, -0.2944, 0.3573, -0.0548, -0.1681, -0.0667 + ] + std = [ + 0.4765, 1.0364, 0.4514, 1.1677, 0.5313, 0.4990, 0.4818, 0.5013, + 0.8158, 1.0344, 0.5894, 1.0901, 0.6885, 0.6165, 0.8454, 0.4978, + 0.5759, 0.3523, 0.7135, 0.6804, 0.5833, 1.4146, 0.8986, 0.5659, + 0.7069, 0.5338, 0.4889, 0.4917, 0.4069, 0.4999, 0.6866, 0.4093, + 0.5709, 0.6065, 0.6415, 0.4944, 0.5726, 1.2042, 0.5458, 1.6887, + 0.3971, 1.0600, 0.3943, 0.5537, 0.5444, 0.4089, 0.7468, 0.7744 + ] + self.register_buffer("mean", torch.tensor(mean), persistent=False) + self.register_buffer("inv_std", torch.tensor(std).reciprocal(), persistent=False) + + # init model + self.model = VideoVAE38_(z_dim=z_dim, dim=dim).eval().requires_grad_(False) + self.upsampling_factor = 16 + self.temporal_downsample_factor = 4 + self.z_dim = z_dim diff --git a/baseline/fastwam_zipper/source/src/fastwam/runtime.py b/baseline/fastwam_zipper/source/src/fastwam/runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..e521dda66377ef4e3edf0f4d1b023380438a5285 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/runtime.py @@ -0,0 +1,533 @@ +import logging +import os +import inspect +from pathlib import Path + +import torch +from hydra.utils import instantiate +from omegaconf import DictConfig +from PIL import Image +import numpy as np +from einops import repeat +from omegaconf import OmegaConf + +from .trainer import Wan22Trainer +from .utils.logging_config import get_logger, setup_logging +from .utils.video_io import save_mp4 +from .utils import misc + +logger = get_logger(__name__) + + +def _normalize_mixed_precision(mixed_precision: str) -> str: + if not isinstance(mixed_precision, str): + raise ValueError(f"`mixed_precision` must be str, got {type(mixed_precision)}") + key = mixed_precision.strip().lower() + if key not in {"no", "fp16", "bf16"}: + raise ValueError( + f"Unsupported mixed_precision: {mixed_precision}. " + "Expected one of: ['no', 'fp16', 'bf16']." + ) + return key + + +def _mixed_precision_to_model_dtype(mixed_precision: str) -> torch.dtype: + precision = _normalize_mixed_precision(mixed_precision) + if precision == "no": + return torch.float32 + if precision == "fp16": + return torch.float16 + return torch.bfloat16 + + +def create_wan22_model( + model_id: str, + tokenizer_model_id: str, + dit_config, + tokenizer_max_len: int = 512, + train_shift: float = 5.0, + infer_shift: float = 5.0, + num_train_timesteps: int = 1000, + redirect_common_files: bool = True, + model_dtype: torch.dtype = torch.bfloat16, + device: str = "cuda", +): + from .models.wan22.wan22 import Wan22Core + + if isinstance(dit_config, DictConfig): + dit_config = OmegaConf.to_container(dit_config, resolve=True) + if not isinstance(dit_config, dict): + raise ValueError(f"`dit_config` must resolve to a dict, got {type(dit_config)}") + + return Wan22Core.from_wan22_pretrained( + device=device, + torch_dtype=model_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + tokenizer_max_len=int(tokenizer_max_len), + redirect_common_files=bool(redirect_common_files), + dit_config=dit_config, + train_shift=float(train_shift), + infer_shift=float(infer_shift), + num_train_timesteps=int(num_train_timesteps), + ) + + +def create_fastwam( + model_id: str, + tokenizer_model_id: str, + video_dit_config, + tokenizer_max_len: int = 512, + load_text_encoder: bool = True, + proprio_dim: int | None = None, + action_dit_config=None, + action_dit_pretrained_path: str | None = None, + skip_dit_load_from_pretrain: bool = False, + video_scheduler=None, + action_scheduler=None, + loss=None, + mot_checkpoint_mixed_attn: bool = False, + compile_training_denoise: bool = False, + redirect_common_files: bool = True, + model_dtype: torch.dtype = torch.bfloat16, + device: str = "cuda", +): + from .models.wan22.fastwam import FastWAM + + if isinstance(video_dit_config, DictConfig): + video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True) + if not isinstance(video_dit_config, dict): + raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}") + + if isinstance(action_dit_config, DictConfig): + action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True) + if action_dit_config is None: + action_dit_config = {} + if not isinstance(action_dit_config, dict): + raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}") + + if isinstance(video_scheduler, DictConfig): + video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True) + if video_scheduler is None: + video_scheduler = {} + if not isinstance(video_scheduler, dict): + raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}") + + if isinstance(action_scheduler, DictConfig): + action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True) + if action_scheduler is None: + raise ValueError("`action_scheduler` is required for FastWAM.") + if not isinstance(action_scheduler, dict): + raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}") + required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"} + missing_keys = required_action_scheduler_keys - set(action_scheduler.keys()) + if missing_keys: + raise ValueError( + f"`action_scheduler` missing required keys: {sorted(missing_keys)}. " + "Expected keys: train_shift, infer_shift, num_train_timesteps." + ) + + if isinstance(loss, DictConfig): + loss = OmegaConf.to_container(loss, resolve=True) + if loss is None: + loss = {} + if not isinstance(loss, dict): + raise ValueError(f"`loss` must be dict-like, got {type(loss)}") + + return FastWAM.from_wan22_pretrained( + device=device, + torch_dtype=model_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + tokenizer_max_len=int(tokenizer_max_len), + load_text_encoder=bool(load_text_encoder), + proprio_dim=(None if proprio_dim is None else int(proprio_dim)), + redirect_common_files=bool(redirect_common_files), + video_dit_config=video_dit_config, + action_dit_config=action_dit_config, + action_dit_pretrained_path=action_dit_pretrained_path, + skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain), + mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn), + video_train_shift=float(video_scheduler.get("train_shift", 5.0)), + video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)), + video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)), + action_train_shift=float(action_scheduler["train_shift"]), + action_infer_shift=float(action_scheduler["infer_shift"]), + action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]), + loss_lambda_video=float(loss.get("lambda_video", 1.0)), + loss_lambda_action=float(loss.get("lambda_action", 1.0)), + compile_training_denoise=bool(compile_training_denoise), + ) + + +def create_fastwam_joint( + model_id: str, + tokenizer_model_id: str, + video_dit_config, + tokenizer_max_len: int = 512, + load_text_encoder: bool = True, + proprio_dim: int | None = None, + action_dit_config=None, + action_dit_pretrained_path: str | None = None, + skip_dit_load_from_pretrain: bool = False, + video_scheduler=None, + action_scheduler=None, + loss=None, + mot_checkpoint_mixed_attn: bool = False, + redirect_common_files: bool = True, + model_dtype: torch.dtype = torch.bfloat16, + device: str = "cuda", +): + from .models.wan22.fastwam_joint import FastWAMJoint + + if isinstance(video_dit_config, DictConfig): + video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True) + if not isinstance(video_dit_config, dict): + raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}") + + if isinstance(action_dit_config, DictConfig): + action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True) + if action_dit_config is None: + action_dit_config = {} + if not isinstance(action_dit_config, dict): + raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}") + + if isinstance(video_scheduler, DictConfig): + video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True) + if video_scheduler is None: + video_scheduler = {} + if not isinstance(video_scheduler, dict): + raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}") + + if isinstance(action_scheduler, DictConfig): + action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True) + if action_scheduler is None: + raise ValueError("`action_scheduler` is required for FastWAM.") + if not isinstance(action_scheduler, dict): + raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}") + required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"} + missing_keys = required_action_scheduler_keys - set(action_scheduler.keys()) + if missing_keys: + raise ValueError( + f"`action_scheduler` missing required keys: {sorted(missing_keys)}. " + "Expected keys: train_shift, infer_shift, num_train_timesteps." + ) + + if isinstance(loss, DictConfig): + loss = OmegaConf.to_container(loss, resolve=True) + if loss is None: + loss = {} + if not isinstance(loss, dict): + raise ValueError(f"`loss` must be dict-like, got {type(loss)}") + + return FastWAMJoint.from_wan22_pretrained( + device=device, + torch_dtype=model_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + tokenizer_max_len=int(tokenizer_max_len), + load_text_encoder=bool(load_text_encoder), + proprio_dim=(None if proprio_dim is None else int(proprio_dim)), + redirect_common_files=bool(redirect_common_files), + video_dit_config=video_dit_config, + action_dit_config=action_dit_config, + action_dit_pretrained_path=action_dit_pretrained_path, + skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain), + mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn), + video_train_shift=float(video_scheduler.get("train_shift", 5.0)), + video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)), + video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)), + action_train_shift=float(action_scheduler["train_shift"]), + action_infer_shift=float(action_scheduler["infer_shift"]), + action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]), + loss_lambda_video=float(loss.get("lambda_video", 1.0)), + loss_lambda_action=float(loss.get("lambda_action", 1.0)), + ) + + +def create_fastwam_idm( + model_id: str, + tokenizer_model_id: str, + video_dit_config, + video_cond_noise_prob: float = 0.5, + tokenizer_max_len: int = 512, + load_text_encoder: bool = True, + proprio_dim: int | None = None, + action_dit_config=None, + action_dit_pretrained_path: str | None = None, + skip_dit_load_from_pretrain: bool = False, + video_scheduler=None, + action_scheduler=None, + loss=None, + mot_checkpoint_mixed_attn: bool = False, + compile_training_denoise: bool = False, + redirect_common_files: bool = True, + model_dtype: torch.dtype = torch.bfloat16, + device: str = "cuda", +): + from .models.wan22.fastwam_idm import ( + FastWAMIDM, + ) + + if isinstance(video_dit_config, DictConfig): + video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True) + if not isinstance(video_dit_config, dict): + raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}") + + if isinstance(action_dit_config, DictConfig): + action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True) + if action_dit_config is None: + action_dit_config = {} + if not isinstance(action_dit_config, dict): + raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}") + + if isinstance(video_scheduler, DictConfig): + video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True) + if video_scheduler is None: + video_scheduler = {} + if not isinstance(video_scheduler, dict): + raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}") + + if isinstance(action_scheduler, DictConfig): + action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True) + if action_scheduler is None: + raise ValueError("`action_scheduler` is required for FastWAM.") + if not isinstance(action_scheduler, dict): + raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}") + required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"} + missing_keys = required_action_scheduler_keys - set(action_scheduler.keys()) + if missing_keys: + raise ValueError( + f"`action_scheduler` missing required keys: {sorted(missing_keys)}. " + "Expected keys: train_shift, infer_shift, num_train_timesteps." + ) + + if isinstance(loss, DictConfig): + loss = OmegaConf.to_container(loss, resolve=True) + if loss is None: + loss = {} + if not isinstance(loss, dict): + raise ValueError(f"`loss` must be dict-like, got {type(loss)}") + + return FastWAMIDM.from_wan22_pretrained( + device=device, + torch_dtype=model_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + video_cond_noise_prob=float(video_cond_noise_prob), + tokenizer_max_len=int(tokenizer_max_len), + load_text_encoder=bool(load_text_encoder), + proprio_dim=(None if proprio_dim is None else int(proprio_dim)), + redirect_common_files=bool(redirect_common_files), + video_dit_config=video_dit_config, + action_dit_config=action_dit_config, + action_dit_pretrained_path=action_dit_pretrained_path, + skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain), + mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn), + video_train_shift=float(video_scheduler.get("train_shift", 5.0)), + video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)), + video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)), + action_train_shift=float(action_scheduler["train_shift"]), + action_infer_shift=float(action_scheduler["infer_shift"]), + action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]), + loss_lambda_video=float(loss.get("lambda_video", 1.0)), + loss_lambda_action=float(loss.get("lambda_action", 1.0)), + compile_training_denoise=bool(compile_training_denoise), + ) + + +def create_fastwam_optional_idm( + model_id: str, + tokenizer_model_id: str, + video_dit_config, + action_idm_prob: float, + video_cond_noise_prob: float = 0.5, + tokenizer_max_len: int = 512, + load_text_encoder: bool = True, + proprio_dim: int | None = None, + action_dit_config=None, + action_dit_pretrained_path: str | None = None, + skip_dit_load_from_pretrain: bool = False, + video_scheduler=None, + action_scheduler=None, + loss=None, + mot_checkpoint_mixed_attn: bool = False, + compile_training_denoise: bool = False, + redirect_common_files: bool = True, + model_dtype: torch.dtype = torch.bfloat16, + device: str = "cuda", +): + from .models.wan22.fastwam_optional_idm import FastWAMOptionalIDM + + if isinstance(video_dit_config, DictConfig): + video_dit_config = OmegaConf.to_container(video_dit_config, resolve=True) + if not isinstance(video_dit_config, dict): + raise ValueError(f"`video_dit_config` must resolve to a dict, got {type(video_dit_config)}") + + if isinstance(action_dit_config, DictConfig): + action_dit_config = OmegaConf.to_container(action_dit_config, resolve=True) + if action_dit_config is None: + action_dit_config = {} + if not isinstance(action_dit_config, dict): + raise ValueError(f"`action_dit_config` must resolve to a dict, got {type(action_dit_config)}") + + if isinstance(video_scheduler, DictConfig): + video_scheduler = OmegaConf.to_container(video_scheduler, resolve=True) + if video_scheduler is None: + video_scheduler = {} + if not isinstance(video_scheduler, dict): + raise ValueError(f"`video_scheduler` must be dict-like, got {type(video_scheduler)}") + + if isinstance(action_scheduler, DictConfig): + action_scheduler = OmegaConf.to_container(action_scheduler, resolve=True) + if action_scheduler is None: + raise ValueError("`action_scheduler` is required for FastWAMOptionalIDM.") + if not isinstance(action_scheduler, dict): + raise ValueError(f"`action_scheduler` must be dict-like, got {type(action_scheduler)}") + required_action_scheduler_keys = {"train_shift", "infer_shift", "num_train_timesteps"} + missing_keys = required_action_scheduler_keys - set(action_scheduler.keys()) + if missing_keys: + raise ValueError( + f"`action_scheduler` missing required keys: {sorted(missing_keys)}. " + "Expected keys: train_shift, infer_shift, num_train_timesteps." + ) + + if isinstance(loss, DictConfig): + loss = OmegaConf.to_container(loss, resolve=True) + if loss is None: + loss = {} + if not isinstance(loss, dict): + raise ValueError(f"`loss` must be dict-like, got {type(loss)}") + + return FastWAMOptionalIDM.from_wan22_pretrained( + device=device, + torch_dtype=model_dtype, + model_id=model_id, + tokenizer_model_id=tokenizer_model_id, + action_idm_prob=float(action_idm_prob), + video_cond_noise_prob=float(video_cond_noise_prob), + tokenizer_max_len=int(tokenizer_max_len), + load_text_encoder=bool(load_text_encoder), + proprio_dim=(None if proprio_dim is None else int(proprio_dim)), + redirect_common_files=bool(redirect_common_files), + video_dit_config=video_dit_config, + action_dit_config=action_dit_config, + action_dit_pretrained_path=action_dit_pretrained_path, + skip_dit_load_from_pretrain=bool(skip_dit_load_from_pretrain), + mot_checkpoint_mixed_attn=bool(mot_checkpoint_mixed_attn), + video_train_shift=float(video_scheduler.get("train_shift", 5.0)), + video_infer_shift=float(video_scheduler.get("infer_shift", 5.0)), + video_num_train_timesteps=int(video_scheduler.get("num_train_timesteps", 1000)), + action_train_shift=float(action_scheduler["train_shift"]), + action_infer_shift=float(action_scheduler["infer_shift"]), + action_num_train_timesteps=int(action_scheduler["num_train_timesteps"]), + loss_lambda_video=float(loss.get("lambda_video", 1.0)), + loss_lambda_action=float(loss.get("lambda_action", 1.0)), + compile_training_denoise=bool(compile_training_denoise), + ) + + +def build_datasets(data_cfg: DictConfig): + train_ds = instantiate(data_cfg.train) + if data_cfg.get("val") is None: + val_ds = train_ds + else: + train_stats_path = data_cfg.train.get("pretrained_norm_stats") + default_stats_path = os.path.join(misc.get_work_dir(), "dataset_stats.json") + val_stats_path = data_cfg.val.get("pretrained_norm_stats") + pretrained_norm_stats = val_stats_path or train_stats_path or default_stats_path + logger.info("Building val dataset with pretrained_norm_stats: %s", pretrained_norm_stats) + val_ds = instantiate(data_cfg.val, pretrained_norm_stats=pretrained_norm_stats) + return train_ds, val_ds + + +def _resolve_train_device() -> str: + if not torch.cuda.is_available(): + return "cpu" + device_count = torch.cuda.device_count() + if device_count <= 1: + return "cuda:0" + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + if local_rank < 0 or local_rank >= device_count: + return "cuda:0" + return f"cuda:{local_rank}" + + +def run_training(cfg: DictConfig): + setup_logging( + log_level=logging.INFO, + is_main_process=torch.distributed.get_rank() == 0 if torch.distributed.is_initialized() else True, + ) + misc.register_work_dir(cfg.output_dir) + config_payload = OmegaConf.to_container(cfg, resolve=True) + with open(Path(cfg.output_dir) / "config.yaml", "w") as f: + OmegaConf.save(config_payload, f) + + model_device = _resolve_train_device() + mixed_precision = _normalize_mixed_precision(cfg.mixed_precision) + model_dtype = _mixed_precision_to_model_dtype(mixed_precision) + model = instantiate(cfg.model, model_dtype=model_dtype, device=model_device) + train_ds, val_ds = build_datasets(cfg.data) + + trainer = Wan22Trainer( + cfg=cfg, + model=model, + train_dataset=train_ds, + val_dataset=val_ds, + ) + trainer.train() + +def run_inference(cfg: DictConfig): + setup_logging(log_level=logging.INFO) + inference_cfg = cfg.inference + mixed_precision = _normalize_mixed_precision(cfg.mixed_precision) + model_dtype = _mixed_precision_to_model_dtype(mixed_precision) + + model = instantiate(cfg.model, model_dtype=model_dtype, device=str(inference_cfg.device)) + checkpoint_path = inference_cfg.get("checkpoint_path") + if checkpoint_path: + ckpt = Path(checkpoint_path) + if ckpt.exists(): + logger.info("Loading finetuned checkpoint: %s", checkpoint_path) + model.load_checkpoint(checkpoint_path) + else: + logger.warning("Checkpoint not found, skipping load: %s", checkpoint_path) + model.eval() + + def center_crop_resize(img: Image, width: int, height: int) -> Image.Image: + src_w, src_h = img.size + scale = max(width / src_w, height / src_h) + resized = img.resize((round(src_w * scale), round(src_h * scale)), resample=Image.BILINEAR) + rw, rh = resized.size + left = max((rw - width) // 2, 0) + top = max((rh - height) // 2, 0) + return resized.crop((left, top, left + width, top + height)) + + input_image = Image.open(str(inference_cfg.input_image_path)).convert("RGB") + input_image = center_crop_resize(input_image, width=inference_cfg.width, height=inference_cfg.height) + arr = np.array(input_image, dtype=np.float32) + x = torch.from_numpy(arr) + x = x.to(device=model.device, dtype=model.torch_dtype) + x = x * (2.0 / 255.0) - 1.0 + x = repeat(x, "H W C -> B C H W", B=1) + output_mp4 = str(inference_cfg.output_mp4) + + infer_kwargs = { + "prompt": str(inference_cfg.prompt), + "negative_prompt": str(inference_cfg.negative_prompt), + "text_cfg_scale": float(inference_cfg.text_cfg_scale), + "action_cfg_scale": float(inference_cfg.action_cfg_scale), + "input_image": x, + "num_frames": int(inference_cfg.num_frames), + "num_inference_steps": int(inference_cfg.num_inference_steps), + "sigma_shift": None if inference_cfg.get("sigma_shift") is None else float(inference_cfg.sigma_shift), + "seed": int(inference_cfg.seed), + "rand_device": str(inference_cfg.rand_device), + "tiled": bool(inference_cfg.tiled), + } + + infer_out = model.infer(**infer_kwargs) + video = infer_out["video"] + save_mp4(video, output_mp4, fps=15) + logger.info("Saved inference video to %s", output_mp4) + return output_mp4 diff --git a/baseline/fastwam_zipper/source/src/fastwam/trainer.py b/baseline/fastwam_zipper/source/src/fastwam/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..fee9203cf442eec5475acccf45c30a9025c2c7f4 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/trainer.py @@ -0,0 +1,832 @@ +import logging +import json +import inspect +import os +import re +from math import ceil +from pathlib import Path +import time + +import numpy as np +import torch +from accelerate import Accelerator +from omegaconf import DictConfig +from PIL import Image +from torch.optim.lr_scheduler import ConstantLR, CosineAnnealingLR, LinearLR, SequentialLR +from torch.utils.data import DataLoader + +from .utils.fs import ensure_dir +from .utils.logging_config import get_logger, setup_logging +from .utils.pytorch_utils import set_global_seed +from .utils.samplers import ResumableEpochSampler +from .utils.video_io import save_mp4 +from .utils.video_metrics import pil_frames_to_video_tensor, video_psnr, video_ssim + +logger = get_logger(__name__) + + +class Wan22Trainer: + def __init__(self, model, train_dataset, val_dataset=None, *, cfg: DictConfig): + self.model = model + self.train_dataset = train_dataset + self.val_dataset = val_dataset + self.cfg = cfg + self.output_dir = str(cfg.output_dir) + self.learning_rate = float(cfg.learning_rate) + self.weight_decay = float(cfg.weight_decay) + self.batch_size = int(cfg.batch_size) + self.num_workers = int(cfg.num_workers) + self.num_epochs = int(cfg.num_epochs) + max_steps = cfg.max_steps + self.max_steps = int(max_steps) if max_steps is not None else None + self.log_every = int(cfg.log_every) + self.save_every = int(cfg.save_every) + self.eval_every = int(cfg.eval_every) + self.eval_num_inference_steps = int(cfg.eval_num_inference_steps) + self.gradient_accumulation_steps = int(cfg.gradient_accumulation_steps) + self.max_grad_norm = float(cfg.max_grad_norm) + self.seed = int(cfg.seed) + + self.resume = cfg.resume + self.mixed_precision = str(cfg.mixed_precision).strip().lower() + if self.mixed_precision not in {"no", "fp16", "bf16"}: + raise ValueError( + f"Unsupported mixed_precision: {cfg.mixed_precision}. " + "Expected one of: ['no', 'fp16', 'bf16']." + ) + self.wandb_enabled = bool(cfg.wandb.enabled) + + self.accelerator = Accelerator( + gradient_accumulation_steps=self.gradient_accumulation_steps, + mixed_precision=self.mixed_precision, + step_scheduler_with_optimizer=False, + ) + + logger.info( + "Accelerate training: distributed_type=%s zero_stage=%s world_size=%d process_index=%d cfg_mixed_precision=%s accelerator_mixed_precision=%s grad_accum=%d grad_clip=%.4f", + self.accelerator.distributed_type, + self.accelerator.state.deepspeed_plugin.deepspeed_config.get("zero_optimization", {}).get("stage", "unknown"), + self.accelerator.num_processes, + self.accelerator.process_index, + self.mixed_precision, + self.accelerator.mixed_precision, + self.gradient_accumulation_steps, + self.max_grad_norm, + ) + logger.info("using accelerator.device=%s", self.accelerator.device) + worker_init_fn = set_global_seed(self.seed, get_worker_init_fn=True) + self._assert_dataset_length_consistent(self.train_dataset, "train_dataset") + if self.val_dataset is not None: + self._assert_dataset_length_consistent(self.val_dataset, "val_dataset") + + # Freeze non-trainable modules before optimizer/deepspeed initialization. + # This keeps DiT (+ optional proprio encoder) as trainable when ZeRO builds optimizer state. + self._apply_dit_only_train_mode(self.model) + trainable_params = list(self.model.dit.parameters()) + proprio_encoder = getattr(self.model, "proprio_encoder", None) + if proprio_encoder is not None: + trainable_params.extend(list(proprio_encoder.parameters())) + self.optimizer = torch.optim.AdamW( + trainable_params, + lr=self.learning_rate, + weight_decay=self.weight_decay, + betas=(0.9, 0.95), + ) + + self.train_loader = self._build_loader(self.train_dataset, worker_init_fn=worker_init_fn) + total_train_steps = self._estimate_total_train_steps() + self.max_steps = total_train_steps + warmup_steps = int(total_train_steps * 0.05) + self.scheduler = self._build_scheduler( + scheduler_type=cfg.lr_scheduler_type, + total_train_steps=total_train_steps, + warmup_steps=warmup_steps, + ) + self.global_step = 0 + self.epoch = 0 + self.batch_in_epoch = 0 + + self.checkpoint_root = os.path.join(self.output_dir, "checkpoints") + self.weights_dir = os.path.join(self.checkpoint_root, "weights") + self.state_dir = os.path.join(self.checkpoint_root, "state") + self.eval_dir = os.path.join(self.output_dir, "eval") + + ensure_dir(self.output_dir) + ensure_dir(self.checkpoint_root) + ensure_dir(self.weights_dir) + ensure_dir(self.state_dir) + ensure_dir(self.eval_dir) + + if bool(getattr(self.model, "compile_training_denoise", False)): + logger.info("Compiling training denoise forward/backward before DeepSpeed initialization.") + compile_start = time.perf_counter() + warmup_loader = DataLoader( + self.train_dataset, + batch_size=self.batch_size, + shuffle=False, + sampler=self.train_sampler, + num_workers=0, + ) + warmup_sample = next(iter(warmup_loader)) + with self.accelerator.autocast(): + warmup_loss, _ = self.model.training_loss(warmup_sample) + warmup_loss.backward() + self.optimizer.zero_grad(set_to_none=True) + if torch.cuda.is_available(): + torch.cuda.synchronize(self.accelerator.device) + logger.info( + "Finished training denoise compile warmup in %.2f seconds.", + time.perf_counter() - compile_start, + ) + set_global_seed(self.seed) + + self.model, self.optimizer, self.train_loader, self.scheduler = self.accelerator.prepare( + self.model, self.optimizer, self.train_loader, self.scheduler + ) + self.optimizer.zero_grad(set_to_none=True) + self.wandb_run = None + self._init_wandb() + self._resume_or_load_checkpoint() + + val_size = len(self.val_dataset) if self.val_dataset is not None else len(self.train_dataset) + logger.info("Train/val dataset size: %d/%d", len(self.train_dataset), val_size) + + def _init_wandb(self): + if not self.wandb_enabled or not self.accelerator.is_main_process: + return + try: + import wandb + except ImportError as e: + raise ImportError( + "wandb logging is enabled in config (`wandb.enabled=true`) but wandb is not installed." + ) from e + + self.wandb_run = wandb.init( + entity=self.cfg.wandb.workspace, + project=self.cfg.wandb.project, + name=self.cfg.wandb.name, + group=None if self.cfg.wandb.group in (None, "null", "") else str(self.cfg.wandb.group), + mode=self.cfg.wandb.mode, + dir=self.output_dir, + ) + logger.info( + "Initialized wandb run: workspace=%s project=%s name=%s", + self.cfg.wandb.workspace, + self.cfg.wandb.project, + self.cfg.wandb.name, + ) + + def _wandb_log(self, payload: dict): + if self.wandb_run is None: + return + self.wandb_run.log(payload, step=self.global_step) + + def _finish_wandb(self): + if self.wandb_run is None: + return + self.wandb_run.finish() + self.wandb_run = None + + def _build_loader(self, dataset, worker_init_fn=None): + self.train_sampler = ResumableEpochSampler( + dataset=dataset, + seed=self.seed, + batch_size=self.batch_size, + num_processes=self.accelerator.num_processes, + ) + return DataLoader( + dataset, + batch_size=self.batch_size, + shuffle=False, + sampler=self.train_sampler, + num_workers=self.num_workers, + pin_memory=torch.cuda.is_available(), + worker_init_fn=worker_init_fn, + ) + + def _assert_dataset_length_consistent(self, dataset, dataset_name: str): + if not hasattr(dataset, "__len__"): + raise TypeError(f"`{dataset_name}` must implement __len__ for rank consistency checks.") + + local_length = len(dataset) + gathered_lengths = self.accelerator.gather( + torch.tensor([local_length], device=self.accelerator.device, dtype=torch.int64) + ).reshape(-1) + if torch.all(gathered_lengths == gathered_lengths[0]): + return + + if self.accelerator.is_main_process: + print(f"[dataset-check] {dataset_name} length mismatch across ranks after initialization:") + for rank, rank_length in enumerate(gathered_lengths.cpu().tolist()): + print(f"rank {rank}: {rank_length}") + self.accelerator.wait_for_everyone() + raise RuntimeError( + f"{dataset_name} length mismatch across ranks: {gathered_lengths.cpu().tolist()}" + ) + + def _estimate_total_train_steps(self) -> int: + if self.max_steps is not None: + return max(int(self.max_steps), 1) + + if not hasattr(self.train_dataset, "__len__"): + raise TypeError("`train_dataset` must implement __len__ when `max_steps` is None.") + + num_processes = max(int(self.accelerator.num_processes), 1) + global_batch_size = max(self.batch_size * num_processes, 1) + micro_steps_per_epoch = max(ceil(len(self.train_dataset) / global_batch_size), 1) + opt_steps_per_epoch = max( + ceil(micro_steps_per_epoch / self.gradient_accumulation_steps), + 1, + ) + return max(opt_steps_per_epoch * self.num_epochs, 1) + + def _build_scheduler(self, scheduler_type, total_train_steps: int, warmup_steps: int = 0): + scheduler_type = str(scheduler_type).strip().lower() + total_train_steps = max(int(total_train_steps), 1) + warmup_steps = min(max(int(warmup_steps), 0), total_train_steps - 1) + + remaining_steps = max(total_train_steps - warmup_steps, 1) + if scheduler_type == "cosine": + main_scheduler = CosineAnnealingLR( + self.optimizer, + T_max=remaining_steps, + eta_min=self.learning_rate * 0.01, + ) + elif scheduler_type == "constant": + main_scheduler = ConstantLR(self.optimizer, factor=1.0, total_iters=remaining_steps) + else: + raise ValueError( + f"Unsupported lr_scheduler_type: {scheduler_type}. " + "Expected one of: ['cosine', 'constant']." + ) + + if warmup_steps <= 0: + return main_scheduler + + warmup_scheduler = LinearLR( + self.optimizer, + start_factor=1.0 / warmup_steps, + end_factor=1.0, + total_iters=warmup_steps, + ) + return SequentialLR( + self.optimizer, + schedulers=[warmup_scheduler, main_scheduler], + milestones=[warmup_steps], + ) + + def _estimate_eta(self): + elapsed = max(time.perf_counter() - self.run_start_time, 1e-6) + done_steps = max(self.global_step - self.run_start_step, 1) + steps_per_sec = done_steps / elapsed + remaining_steps = max(self.max_steps - self.global_step, 0) + eta_seconds = int(remaining_steps / max(steps_per_sec, 1e-9)) + eta_h, eta_rem = divmod(eta_seconds, 3600) + eta_m, eta_s = divmod(eta_rem, 60) + return f"{eta_h:02d}:{eta_m:02d}:{eta_s:02d}", steps_per_sec + + def _resume_or_load_checkpoint(self): + resume = self.resume + if not resume: + return + resume_path = Path(str(resume)) + if resume_path.is_dir(): + logger.info("Resuming full training state from directory: %s", resume) + self.load_training_state(str(resume_path)) + return + if not resume_path.exists(): + raise FileNotFoundError(f"Resume checkpoint not found: {resume}") + logger.info("Loading weight checkpoint only: %s", resume) + self.accelerator.unwrap_model(self.model).load_checkpoint(str(resume_path), optimizer=None) + logger.warning("Loaded .pt weights only; optimizer/scheduler/step were not restored under ZeRO2.") + + def _set_dit_only_train_mode(self): + # Match DiffSynth's freeze_except("dit"): only DiT stays trainable/in-train-mode. + logger.info("Setting DiT to train mode and freezing other model components.") + model = self.accelerator.unwrap_model(self.model) + self._apply_dit_only_train_mode(model) + + @staticmethod + def _apply_dit_only_train_mode(model): + model.eval() + model.requires_grad_(False) + model.dit.train() + model.dit.requires_grad_(True) + proprio_encoder = getattr(model, "proprio_encoder", None) + if proprio_encoder is not None: + proprio_encoder.train() + proprio_encoder.requires_grad_(True) + + @staticmethod + def _to_batched_eval_sample(sample): + video = sample["video"] + prompt = sample["prompt"] + action = sample.get("action", None) + proprio = sample.get("proprio", None) + context = sample.get("context", None) + context_mask = sample.get("context_mask", None) + + if not isinstance(video, torch.Tensor): + raise TypeError( + f"Expected tensor video for evaluation, got {type(video)}. " + "Evaluation now expects `video` with shape [3,T,H,W] or [B,3,T,H,W]." + ) + if video.ndim == 4: + video = video.unsqueeze(0) + if video.ndim != 5: + raise ValueError(f"Expected video shape [3,T,H,W] or [B,3,T,H,W], got {tuple(video.shape)}") + num_video_frames = video.shape[2] + if num_video_frames <= 1: + raise ValueError(f"`sample['video']` must have at least 2 frames for action evaluation, got {num_video_frames}") + + if isinstance(prompt, str): + prompt = [prompt] + elif isinstance(prompt, tuple): + prompt = list(prompt) + elif not isinstance(prompt, list): + raise TypeError(f"Expected prompt type str/list[str], got {type(prompt)}") + if len(prompt) != video.shape[0]: + raise ValueError(f"Prompt batch mismatch: len(prompt)={len(prompt)} vs video batch={video.shape[0]}") + + action_horizon = None + action = None + if "action" in sample: + action = sample["action"] + if not isinstance(action, torch.Tensor): + raise TypeError( + f"`sample['action']` must be a torch.Tensor, got {type(action)}" + ) + if action.ndim == 2: + action = action.unsqueeze(0) + if action.ndim != 3: + raise ValueError(f"`sample['action']` must be 3D [B, T, a_dim], got shape {tuple(action.shape)}") + if action.shape[1] % (num_video_frames - 1) != 0: + raise ValueError(f"`sample['action']` temporal dimension must be divisible by video frames-1={num_video_frames - 1}, got {action.shape[1]}") + action_horizon = int(action.shape[1]) + + proprio = None + if "proprio" in sample: + proprio = sample["proprio"] + if not isinstance(proprio, torch.Tensor): + raise TypeError(f"`sample['proprio']` must be a torch.Tensor, got {type(proprio)}") + if proprio.ndim == 2: + proprio = proprio.unsqueeze(0) + if proprio.ndim != 3: + raise ValueError(f"`sample['proprio']` must be 3D [B, T, d], got shape {tuple(proprio.shape)}") + + if context is not None or context_mask is not None: + if context is None or context_mask is None: + raise ValueError("`context` and `context_mask` must both exist in eval sample.") + if context.ndim == 2: + context = context.unsqueeze(0) + if context_mask.ndim == 1: + context_mask = context_mask.unsqueeze(0) + if context.ndim != 3 or context_mask.ndim != 2: + raise ValueError( + f"`context/context_mask` must be [B,L,D]/[B,L], got {tuple(context.shape)} and {tuple(context_mask.shape)}" + ) + + return { + "video": video, + "prompt": prompt, + "action": action, + "proprio": proprio, + "context": context, + "context_mask": context_mask, + "action_horizon": action_horizon, + } + + @torch.no_grad() + def evaluate(self): + if self.val_dataset is None: + return None + + model = self.accelerator.unwrap_model(self.model) + was_dit_training = model.dit.training + model.eval() + + # eval_index = (self.global_step + self.accelerator.process_index) % len(self.val_dataset) + rng = torch.Generator(device="cpu").manual_seed(self.global_step + self.accelerator.process_index) + eval_index = torch.randint(0, len(self.val_dataset), (1,), generator=rng).item() + sample = self._to_batched_eval_sample(self.val_dataset[eval_index]) + + # 1. training loss + with self.accelerator.autocast(): + val_loss, _ = model.training_loss(sample) + val_loss = val_loss.float().item() + + prompt = sample["prompt"][0] + video0 = sample["video"][0] # Tensor [3, T, H, W] in (-1, 1) + action = sample["action"][0] if "action" in sample and sample["action"] is not None else None + proprio = sample["proprio"][0, 0] if "proprio" in sample and sample["proprio"] is not None else None # from [1, T, d] to [d] + input_image = video0[:, 0].unsqueeze(0) + _, num_frames, _, _ = video0.shape + + # 2. inference and video saving + infer_kwargs = { + "input_image": input_image, + "num_frames": num_frames, + "action": action, + "action_horizon": sample['action_horizon'], + "proprio": proprio, + "text_cfg_scale": 1.0, + "action_cfg_scale": 1.0, + "num_inference_steps": self.eval_num_inference_steps, + "seed": 42, + "tiled": False, + } + if sample["context"] is not None: + infer_kwargs["prompt"] = None + infer_kwargs["context"] = sample["context"][0] + infer_kwargs["context_mask"] = sample["context_mask"][0] + else: + infer_kwargs["prompt"] = prompt + + pred = model.infer( + **infer_kwargs, + ) + + pred_video = pred["video"] + pred_action = pred.get("action", None) + + # 3. inference metrics against GT video + pred_video_tensor = pil_frames_to_video_tensor(pred_video) + gt_video_tensor = ((video0.detach().float().cpu().clamp(-1.0, 1.0) + 1.0) * 0.5).contiguous() + + assert pred_video_tensor.shape == gt_video_tensor.shape, ( + "Eval infer prediction/GT shape mismatch: " + f"pred={tuple(pred_video_tensor.shape)} vs gt={tuple(gt_video_tensor.shape)}" + ) + + psnr_rollout_vs_gt = video_psnr(pred=pred_video_tensor, target=gt_video_tensor) + ssim_rollout_vs_gt = video_ssim(pred=pred_video_tensor, target=gt_video_tensor) + + action_l1 = None + action_l2 = None + if action is not None and pred_action is not None: + if sample["proprio"] is None: + raise ValueError("Eval sample must contain `proprio` for action denormalization.") + proprio = sample["proprio"].detach().to(device="cpu", dtype=torch.float32) + + processor = self.val_dataset.lerobot_dataset.processor + + denorm_actions = {} + action_meta = processor.shape_meta["action"] + state_meta = processor.shape_meta["state"] + for action_name, raw_action in (("pred", pred_action), ("gt", action)): + if not isinstance(raw_action, torch.Tensor): + raise TypeError(f"{action_name} action must be a torch.Tensor, got {type(raw_action)}") + if raw_action.ndim == 2: + action_btd = raw_action.unsqueeze(0) + elif raw_action.ndim == 3 and raw_action.shape[0] == 1: + action_btd = raw_action + else: + raise ValueError( + f"{action_name} action must have shape [T, D] or [1, T, D], got {tuple(raw_action.shape)}" + ) + action_btd = action_btd.detach().to(device="cpu", dtype=torch.float32) + + batch = { + "action": action_btd, + "state": proprio, + } + batch = processor.action_state_merger.backward(batch) + batch = processor.normalizer.backward(batch) + merged_batch = { + "action": {meta["key"]: batch["action"][meta["key"]].squeeze(0) for meta in action_meta}, + "state": {meta["key"]: batch["state"][meta["key"]].squeeze(0) for meta in state_meta}, + } + merged_batch = processor.action_state_merger.forward(merged_batch) + denorm_action = merged_batch["action"].unsqueeze(0) + if denorm_action.ndim != 3 or denorm_action.shape[0] != 1: + raise ValueError( + f"Denormalized {action_name} action must have shape [1, T, D], got {tuple(denorm_action.shape)}" + ) + denorm_actions[action_name] = denorm_action + + pred_action_denorm = denorm_actions["pred"] + gt_action_denorm = denorm_actions["gt"] + + if pred_action_denorm.shape != gt_action_denorm.shape: + raise ValueError( + "Predicted action/GT action shape mismatch after denormalization: " + f"pred={tuple(pred_action_denorm.shape)} vs gt={tuple(gt_action_denorm.shape)}" + ) + action_diff = pred_action_denorm - gt_action_denorm + action_l1 = action_diff.abs().mean().item() + action_l2 = action_diff.pow(2).mean().item() + + # 4. VAE reconstruction metrics against GT video + gt_video_batch = video0.unsqueeze(0).to(device=model.device, dtype=model.torch_dtype) + vae_latents = model._encode_video_latents(gt_video_batch, tiled=False) + vae_recon_video = model._decode_latents(vae_latents, tiled=False) + vae_video_tensor = pil_frames_to_video_tensor(vae_recon_video) + + assert vae_video_tensor.shape == gt_video_tensor.shape, ( + "Eval VAE reconstruction/GT shape mismatch: " + f"vae={tuple(vae_video_tensor.shape)} vs gt={tuple(gt_video_tensor.shape)}" + ) + + psnr_decode_vs_gt = video_psnr(pred=vae_video_tensor, target=gt_video_tensor) + ssim_decode_vs_gt = video_ssim(pred=vae_video_tensor, target=gt_video_tensor) + + psnr_rollout_vs_decode = video_psnr(pred=pred_video_tensor, target=vae_video_tensor) + ssim_rollout_vs_decode = video_ssim(pred=pred_video_tensor, target=vae_video_tensor) + + stitched_video_tensor = torch.cat( + [pred_video_tensor, vae_video_tensor, gt_video_tensor], + dim=2, + ).contiguous() + stitched_frames = [] + for t in range(stitched_video_tensor.shape[1]): + frame = (stitched_video_tensor[:, t].permute(1, 2, 0).clamp(0.0, 1.0).numpy() * 255.0).astype(np.uint8) + stitched_frames.append(Image.fromarray(frame)) + + video_path = os.path.join( + self.eval_dir, + f"step_{self.global_step:06d}_rank_{self.accelerator.process_index:03d}.mp4", + ) + save_mp4(stitched_frames, video_path, fps=8) + + local_metrics = torch.tensor( + [ + float(val_loss), + float(psnr_rollout_vs_gt), + float(ssim_rollout_vs_gt), + float(psnr_rollout_vs_decode), + float(ssim_rollout_vs_decode), + float(psnr_decode_vs_gt), + float(ssim_decode_vs_gt), + float(action_l2) if action_l2 is not None else -1.0, + float(action_l1) if action_l1 is not None else -1.0, + ], + device=self.accelerator.device, + dtype=torch.float32, + ).unsqueeze(0) + gathered_metrics = self.accelerator.gather_for_metrics(local_metrics) + mean_metrics = gathered_metrics[:, :7].mean(dim=0) + action_l2_mean = gathered_metrics[:, 7].mean().item() if action_l2 is not None else None + action_l1_mean = gathered_metrics[:, 8].mean().item() if action_l1 is not None else None + + if was_dit_training: + self._set_dit_only_train_mode() + + result = { + "val_loss": float(mean_metrics[0].item()), + "psnr_rg": float(mean_metrics[1].item()), + "ssim_rg": float(mean_metrics[2].item()), + "psnr_rd": float(mean_metrics[3].item()), + "ssim_rd": float(mean_metrics[4].item()), + "psnr_dg": float(mean_metrics[5].item()), + "ssim_dg": float(mean_metrics[6].item()), + "video_path": video_path, + } + if action_l2_mean is not None: + result["action_l2"] = float(action_l2_mean) + if action_l1_mean is not None: + result["action_l1"] = float(action_l1_mean) + return result + + def _save_weights_checkpoint(self, step_tag: str): + model = self.accelerator.unwrap_model(self.model) + ckpt_path = os.path.join(self.weights_dir, f"{step_tag}.pt") + model.save_checkpoint(ckpt_path, optimizer=None, step=self.global_step) + return ckpt_path + + def _save_trainer_state(self, state_path: str): + state_file = os.path.join(state_path, "trainer_state.json") + payload = { + "global_step": int(self.global_step), + "epoch": int(self.epoch), + "batch_in_epoch": int(self.batch_in_epoch), + } + with open(state_file, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=True, indent=2) + + def save_checkpoint(self): + step_tag = f"step_{self.global_step:06d}" + + self.accelerator.wait_for_everyone() + ckpt_path = None + if self.accelerator.is_main_process: + ckpt_path = self._save_weights_checkpoint(step_tag=step_tag) + self.accelerator.wait_for_everyone() + + state_path = os.path.join(self.state_dir, step_tag) + ensure_dir(state_path) + self.accelerator.save_state(output_dir=state_path) + if self.accelerator.is_main_process: + self._save_trainer_state(state_path) + self.accelerator.wait_for_everyone() + + return {"weights_path": ckpt_path, "state_path": state_path} + + def load_training_state(self, state_dir: str): + self.accelerator.load_state(input_dir=state_dir) + state_file = Path(state_dir) / "trainer_state.json" + if state_file.exists(): + with open(state_file, "r", encoding="utf-8") as f: + payload = json.load(f) + self.global_step = int(payload["global_step"]) + + if "epoch" in payload and "batch_in_epoch" in payload: + self.epoch = int(payload["epoch"]) + self.batch_in_epoch = int(payload["batch_in_epoch"]) + self.train_sampler.set_epoch_offset(self.epoch) + self.train_sampler.set_resume_batch_offset(self.batch_in_epoch) + logger.info( + "Restored dataloader progress: epoch=%d batch_in_epoch=%d sample_offset=%d", + self.epoch, + self.batch_in_epoch, + self.batch_in_epoch * self.batch_size * self.accelerator.num_processes, + ) + else: + self.epoch = 0 + self.batch_in_epoch = 0 + self.train_sampler.clear_resume_batch_offset() + logger.warning( + "State file does not contain `epoch`/`batch_in_epoch`; " + "optimizer/scheduler were restored, but dataloader progress resume is skipped." + ) + self.accelerator.wait_for_everyone() + return + + match = re.search(r"step[_-](\d+)$", str(state_dir).rstrip("/")) + if match: + self.global_step = int(match.group(1)) + else: + self.global_step = 0 + self.epoch = 0 + self.batch_in_epoch = 0 + self.train_sampler.clear_resume_batch_offset() + self.accelerator.wait_for_everyone() + logger.info("Loaded accelerate training state from %s at step=%d", state_dir, self.global_step) + logger.warning( + "State file `%s` is missing; dataloader progress resume is skipped.", + state_file, + ) + + def train(self): + self._set_dit_only_train_mode() + + unwrapped_model = self.accelerator.unwrap_model(self.model) + + if self.max_steps is None: + raise ValueError("`max_steps` must be set before entering the while-step training loop.") + + logger.info("Starting training with max_steps=%d.", self.max_steps) + data_iter = iter(self.train_loader) + self.run_start_step = self.global_step + self.run_start_time = time.perf_counter() + + while self.global_step < self.max_steps: + try: + sample = next(data_iter) + self.batch_in_epoch += 1 + except StopIteration: + self.epoch += 1 + self.batch_in_epoch = 0 + self.train_sampler.clear_resume_batch_offset() + data_iter = iter(self.train_loader) + continue + + with self.accelerator.accumulate(self.model): + train_model = self.model if hasattr(self.model, "training_loss") else self.accelerator.unwrap_model(self.model) + + with self.accelerator.autocast(): + loss, loss_dict = train_model.training_loss(sample) + self.accelerator.backward(loss) + + if self.accelerator.sync_gradients: + grad_norm = self.accelerator.clip_grad_norm_(self.model.parameters(), self.max_grad_norm) + self.optimizer.step() + if not self.accelerator.optimizer_step_was_skipped: + self.scheduler.step() + self.optimizer.zero_grad(set_to_none=True) + self.global_step += 1 + global_loss = float( + self.accelerator.gather(loss.detach().float().reshape(1)).mean().item() + ) + global_loss_metrics = {} + for key, value in loss_dict.items(): + metric_tensor = torch.tensor(float(value), device=loss.device, dtype=torch.float32).reshape(1) + global_loss_metrics[key] = float( + self.accelerator.gather(metric_tensor).mean().item() + ) + grad_norm_tensor = torch.tensor(grad_norm, device=loss.device, dtype=torch.float32) + global_grad_norm = float(self.accelerator.gather(grad_norm_tensor).mean().item()) + + current_lr = float(self.optimizer.param_groups[0]["lr"]) + + if self.log_every > 0 and self.global_step % self.log_every == 0 and self.accelerator.is_main_process: + eta_str, steps_per_sec = self._estimate_eta() + description = "[train] epoch=%d step=%d/%d loss=%.4f " % ( + self.epoch, + self.global_step, + self.max_steps, + global_loss, + ) + if global_loss_metrics: + detail_str = " ".join([f"{k}={v:.4f}" for k, v in sorted(global_loss_metrics.items())]) + description += detail_str + " " + description += "lr=%.2e speed=%.2f step/s, %.2f samples/s eta=%s" % ( + current_lr, + steps_per_sec, + steps_per_sec * self.batch_size * self.accelerator.num_processes, + eta_str, + ) + logger.info(description) + + wandb_payload = { + "train/loss": global_loss, + "train/grad_norm": global_grad_norm, + "train/lr": current_lr, + "performance/steps_per_sec": steps_per_sec, + "performance/samples_per_sec": steps_per_sec * self.batch_size * self.accelerator.num_processes, + } + for key, value in global_loss_metrics.items(): + wandb_payload[f"train/{key}"] = value + self._wandb_log(wandb_payload) + + if ( + self.eval_every > 0 + and self.val_dataset is not None + and self.global_step % self.eval_every == 0 + ): + metrics = self.evaluate() + self.accelerator.wait_for_everyone() + if metrics is not None and self.accelerator.is_main_process: + description = "[eval] step=%d val_loss=%.4f infer_psnr=%.4f infer_ssim=%.4f" % ( + self.global_step, + metrics["val_loss"], + metrics["psnr_rd"], + metrics["ssim_rd"], + ) + if "action_l2" in metrics: + description += " action_l2=%.4f" % metrics["action_l2"] + if "action_l1" in metrics: + description += " action_l1=%.4f" % metrics["action_l1"] + logger.info(description) + eval_payload = { + "eval/val_loss": float(metrics["val_loss"]), + "eval/psnr_rg": float(metrics["psnr_rg"]), + "eval/ssim_rg": float(metrics["ssim_rg"]), + "eval/psnr_rd": float(metrics["psnr_rd"]), + "eval/ssim_rd": float(metrics["ssim_rd"]), + "eval/psnr_dg": float(metrics["psnr_dg"]), + "eval/ssim_dg": float(metrics["ssim_dg"]), + } + if "action_l2" in metrics: + eval_payload["eval/action_l2"] = float(metrics["action_l2"]) + if "action_l1" in metrics: + eval_payload["eval/action_l1"] = float(metrics["action_l1"]) + self._wandb_log(eval_payload) + + is_final_step = self.global_step >= self.max_steps + smoke_skip_final_save = ( + is_final_step and os.environ.get("FASTWAM_SMOKE_NO_SAVE") == "1" + ) + checkpoint_at_current_step = ( + self.save_every > 0 + and self.global_step % self.save_every == 0 + and not smoke_skip_final_save + ) + ckpt_info = None + if checkpoint_at_current_step: + ckpt_info = self.save_checkpoint() + if self.accelerator.is_main_process: + logger.info( + "[ckpt] step=%d weights=%s state=%s", + self.global_step, + ckpt_info["weights_path"], + ckpt_info["state_path"], + ) + + if is_final_step: + if smoke_skip_final_save: + if self.accelerator.is_main_process: + logger.info( + "[done] max_steps reached step=%d; smoke mode skipped checkpoint save", + self.global_step, + ) + return + if ckpt_info is None: + ckpt_info = self.save_checkpoint() + if self.accelerator.is_main_process: + logger.info( + "[done] max_steps reached step=%d weights=%s state=%s", + self.global_step, + ckpt_info["weights_path"], + ckpt_info["state_path"], + ) + return + + ckpt_info = self.save_checkpoint() + if self.accelerator.is_main_process: + logger.info( + "[done] training finished step=%d weights=%s state=%s", + self.global_step, + ckpt_info["weights_path"], + ckpt_info["state_path"], + ) + diff --git a/baseline/fastwam_zipper/source/src/fastwam/utils/__init__.py b/baseline/fastwam_zipper/source/src/fastwam/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3d0eb927e25fab27c8dae7daa3d52a31c2d532b2 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/utils/__init__.py @@ -0,0 +1,4 @@ +from .fs import ensure_dir +from .video_io import save_mp4 + +__all__ = ["ensure_dir", "save_mp4"] diff --git a/baseline/fastwam_zipper/source/src/fastwam/utils/config_resolvers.py b/baseline/fastwam_zipper/source/src/fastwam/utils/config_resolvers.py new file mode 100644 index 0000000000000000000000000000000000000000..4b1273b96b716ffa1028dfce42474376930a19f1 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/utils/config_resolvers.py @@ -0,0 +1,70 @@ +import math +from pathlib import Path +from typing import Any, List, Callable, Optional +from omegaconf import OmegaConf + + +def _register(name: str, func: Callable) -> None: + """Idempotently register a resolver, replacing any existing one.""" + OmegaConf.register_new_resolver(name, func, replace=True) + +def _oc_load(path: str, key: Optional[str] = None) -> Any: + """ + Load a YAML/JSON config and optionally select a key. + Uses Hydra's to_absolute_path to honor original working dir. + """ + try: + from hydra.utils import to_absolute_path + except ImportError: + to_absolute_path = None # should not happen in normal Hydra runs + load_path = Path(path) + if not load_path.is_absolute() and to_absolute_path is not None: + load_path = Path(to_absolute_path(path)) + cfg = OmegaConf.load(load_path) + if key is None or key == "": + return cfg + return OmegaConf.select(cfg, key) + +def sum_shapes(shape_meta_list): + if not shape_meta_list: + return 0 + total = sum(int(item['shape']) for item in shape_meta_list if item["key"] is not None) + return total + +def max_action_dim(embodiment_datasets_cfg): + max_dim = 0 + for dataset_name, dataset_cfg in embodiment_datasets_cfg.items(): + if "shape_meta" in dataset_cfg: + action_cfg = dataset_cfg.shape_meta.action + current_dim = sum_shapes(action_cfg) + + if current_dim > max_dim: + max_dim = current_dim + + return max_dim + +def max_state_dim(embodiment_datasets_cfg): + max_dim = 0 + for dataset_name, dataset_cfg in embodiment_datasets_cfg.items(): + if "shape_meta" in dataset_cfg: + state_cfg = dataset_cfg.shape_meta.state + current_dim = sum_shapes(state_cfg) + if current_dim > max_dim: + max_dim = current_dim + + return max_dim + +def register_default_resolvers() -> None: + """ + Register all resolvers commonly used across entrypoints. + Safe to call multiple times. + """ + _register("oc.load", _oc_load) + _register("eval", eval) # allows arbitrary python code execution in configs using the ${eval:''} resolver + _register("split", lambda s, idx: s.split('/')[int(idx)]) # split string + _register("max", lambda x: max(x)) + _register("round_up", math.ceil) + _register("round_down", math.floor) + _register("sum_shapes", sum_shapes) + _register("max_action_dim", max_action_dim) + _register("max_state_dim", max_state_dim) diff --git a/baseline/fastwam_zipper/source/src/fastwam/utils/fs.py b/baseline/fastwam_zipper/source/src/fastwam/utils/fs.py new file mode 100644 index 0000000000000000000000000000000000000000..b210bde5a81689be2969db6059ab71833958f43a --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/utils/fs.py @@ -0,0 +1,5 @@ +import os + + +def ensure_dir(path: str): + os.makedirs(path, exist_ok=True) diff --git a/baseline/fastwam_zipper/source/src/fastwam/utils/logging_config.py b/baseline/fastwam_zipper/source/src/fastwam/utils/logging_config.py new file mode 100644 index 0000000000000000000000000000000000000000..eba39b2fcff02c4d8488cf099d414eca19fbbc47 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/utils/logging_config.py @@ -0,0 +1,140 @@ +import logging +from typing import Optional +import os + +import torch.distributed as dist + +from rich.logging import RichHandler + + +def setup_logging( + log_level: int = logging.INFO, + is_main_process: Optional[bool] = None, + rich_handler_kwargs: Optional[dict] = None, + formatter_kwargs: Optional[dict] = None, + preserve_hydra_handlers: bool = True +) -> None: + """ + Configure the logging system for the entire codebase with Rich formatting. + + In distributed training, only the main process outputs logs while other processes are silenced. + This function configures the root logger so all child loggers inherit the same configuration. + + Args: + log_level: Logging level (default INFO), only applies to main process + is_main_process: Whether this is the main process. If None, infer automatically. + rich_handler_kwargs: Additional kwargs to pass to RichHandler + formatter_kwargs: Additional kwargs to pass to Formatter (fmt and datefmt) + preserve_hydra_handlers: Keep existing FileHandlers from Hydra (default True) + + Example: + ```python + # In a single-machine script + from fastwam.utils.logging_config import setup_logging + setup_logging() + + # In a distributed training script + from accelerate import PartialState + from fastwam.utils.logging_config import setup_logging + + distributed_state = PartialState() + setup_logging( + log_level=logging.INFO, + is_main_process=distributed_state.is_main_process + ) + ``` + """ + if is_main_process is None: + is_main_process = _is_main_process() + + root_logger = logging.getLogger() + + if is_main_process: + # Save existing FileHandlers (e.g., from Hydra) if requested + existing_file_handlers = [] + if preserve_hydra_handlers: + existing_file_handlers = [ + h for h in root_logger.handlers + if isinstance(h, logging.FileHandler) + ] + + # Clear all default handlers on the root logger + root_logger.handlers.clear() + + # Configure RichHandler parameters + default_rich_kwargs = { + "markup": True, + "rich_tracebacks": True, + "show_level": True, + "show_path": True, + "show_time": True, + } + if rich_handler_kwargs: + default_rich_kwargs.update(rich_handler_kwargs) + + # Create RichHandler + rich_handler = RichHandler(**default_rich_kwargs) + + # Configure Formatter + default_formatter_kwargs = { + "fmt": "| >> %(message)s", + "datefmt": "%m/%d [%H:%M:%S]", + } + if formatter_kwargs: + default_formatter_kwargs.update(formatter_kwargs) + + formatter = logging.Formatter(**default_formatter_kwargs) + rich_handler.setFormatter(formatter) + + # Add handler and set logging level + root_logger.addHandler(rich_handler) + + # Restore existing FileHandlers from Hydra + for handler in existing_file_handlers: + root_logger.addHandler(handler) + + root_logger.setLevel(log_level) + + else: + # In non-main processes, set root logger level to ERROR to silence all logs + root_logger.setLevel(logging.ERROR) + + +def _is_main_process() -> bool: + """ + Best-effort check for main process without any synchronization. + """ + # Prefer torch.distributed state if initialized. + if dist is not None and dist.is_available() and dist.is_initialized(): + return dist.get_rank() == 0 + + # Fallback to environment variables commonly set by launchers. + for key in ("RANK", "SLURM_PROCID", "LOCAL_RANK"): + if key in os.environ: + return os.environ.get(key, "0") in ("0", "0\n", "") + + return True + +def get_logger(name: str = __name__, level: int = logging.INFO) -> logging.Logger: + """ + Drop-in replacement for accelerate.logging.get_logger: + - No implicit barriers. + - Only the main process emits log records by default. + """ + logger = logging.getLogger(name) + logger.setLevel(level) + + # if not logger.handlers and _is_main_process(): + # handler = logging.StreamHandler() + # formatter = logging.Formatter( + # fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + # datefmt="%Y-%m-%d %H:%M:%S", + # ) + # handler.setFormatter(formatter) + # logger.addHandler(handler) + + if not _is_main_process(): + logger.propagate = False + logger.disabled = True + + return logger diff --git a/baseline/fastwam_zipper/source/src/fastwam/utils/misc.py b/baseline/fastwam_zipper/source/src/fastwam/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..9981aa8913c2b565c3b6cf5e317eed12d6a7222c --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/utils/misc.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +from __future__ import annotations + +import collections +import collections.abc +import functools +import json +import os +import random +import time +from contextlib import ContextDecorator +from pathlib import Path +from typing import Any, Callable, List, Optional, Tuple, TypeVar +from urllib.parse import urlparse + +import boto3 +import numpy as np +import termcolor +import torch +from torch import nn +from torch.distributed._functional_collectives import AsyncCollectiveTensor +from torch.distributed._tensor.api import DTensor + +_WORK_DIR: str | None = None +_DEFAULT_WORK_DIR = "./runs/" + + +def register_work_dir(path: str | os.PathLike | None) -> None: + global _WORK_DIR + _WORK_DIR = str(path) if path is not None else None + os.makedirs(path, exist_ok=True) + + +def get_work_dir() -> str | None: + return _WORK_DIR if _WORK_DIR is not None else _DEFAULT_WORK_DIR diff --git a/baseline/fastwam_zipper/source/src/fastwam/utils/pytorch_utils.py b/baseline/fastwam_zipper/source/src/fastwam/utils/pytorch_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0d31fcd1cd41c2a9b5361a4200e02479cf794c5e --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/utils/pytorch_utils.py @@ -0,0 +1,137 @@ +from typing import Dict, List, Callable, Optional +import os +import random +import collections + +import torch +import torch.distributed as dist +import numpy as np + + +def _resolve_global_rank() -> int: + if torch.distributed.is_available() and torch.distributed.is_initialized(): + return int(torch.distributed.get_rank()) + return int(os.environ.get("RANK", os.environ.get("SLURM_PROCID", os.environ.get("LOCAL_RANK", "0")))) + + +def set_global_seed(seed: int, get_worker_init_fn: bool = False) -> Optional[Callable[[int], None]]: + """Sets seed for all randomness libraries (mostly random, numpy, torch) and produces a `worker_init_fn`""" + assert np.iinfo(np.uint32).min <= seed < np.iinfo(np.uint32).max, "Seed outside the np.uint32 bounds!" + + # Set Seed as an Environment Variable + os.environ["EXPERIMENT_GLOBAL_SEED"] = str(seed) + + # Process-specific seeding: offset by global rank so each process gets a different seed + global_rank = _resolve_global_rank() + process_seed = seed + global_rank + + random.seed(process_seed) + np.random.seed(process_seed) + torch.manual_seed(process_seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(process_seed) + + return worker_init_function if get_worker_init_fn else None + + +def worker_init_function(worker_id: int) -> None: + """ + Borrowed directly from PyTorch-Lightning; inspired by this issue comment in the PyTorch repo: + > Ref: https://github.com/pytorch/pytorch/issues/5059#issuecomment-817392562 + + Intuition: You can think of the seed sequence spawn function as a "janky" torch.Generator() or jax.PRNGKey that + you can run iterative splitting on to get new (predictable) randomness. + + :param worker_id: Identifier for the given worker [0, num_workers) for the Dataloader in question. + """ + # Get current global `rank` (if running distributed) and `process_seed` + process_seed = torch.initial_seed() + global_rank = _resolve_global_rank() + + # Back out the "base" (original) seed - the per-worker seed is set in PyTorch: + # > https://pytorch.org/docs/stable/data.html#data-loading-randomness + base_seed = process_seed - worker_id + + # "Magic" code --> basically creates a seed sequence that mixes different "sources" and seeds every library... + seed_seq = np.random.SeedSequence([base_seed, worker_id, global_rank]) + + # Use 128 bits (4 x 32-bit words) to represent seed --> generate_state(k) produces a `k` element array! + np.random.seed(seed_seq.generate_state(4)) + + # Spawn distinct child sequences for PyTorch (reseed) and stdlib random + torch_seed_seq, random_seed_seq, tf_seed_seq = seed_seq.spawn(3) + + # Torch Manual seed takes 64 bits (so just specify a dtype of uint64 + torch.manual_seed(torch_seed_seq.generate_state(1, dtype=np.uint64)[0]) + + # Use 128 Bits for `random`, but express as integer instead of as an array + random_seed = (random_seed_seq.generate_state(2, dtype=np.uint64).astype(list) * [1 << 64, 1]).sum() + random.seed(random_seed) + + +def dict_apply( + x: Dict[str, torch.Tensor], + func: Callable[[torch.Tensor], torch.Tensor] + ) -> Dict[str, torch.Tensor]: + result = dict() + for key, value in x.items(): + if isinstance(value, dict): + result[key] = dict_apply(value, func) + else: + result[key] = func(value) + return result + + +def dict_to_array(x): + data = np.concatenate([item for _, item in x.items()], axis=-1) + return data + + +def pad_remaining_dims(x, target): + assert x.shape == target.shape[:len(x.shape)] + return x.reshape(x.shape + (1,)*(len(target.shape) - len(x.shape))) + + +def dict_apply_split( + x: Dict[str, torch.Tensor], + split_func: Callable[[torch.Tensor], Dict[str, torch.Tensor]] + ) -> Dict[str, torch.Tensor]: + results = collections.defaultdict(dict) + for key, value in x.items(): + result = split_func(value) + for k, v in result.items(): + results[k][key] = v + return results + + +def dict_apply_reduce( + x: List[Dict[str, torch.Tensor]], + reduce_func: Callable[[List[torch.Tensor]], torch.Tensor] + ) -> Dict[str, torch.Tensor]: + result = dict() + for key in x[0].keys(): + result[key] = reduce_func([x_[key] for x_ in x]) + return result + + +def optimizer_to(optimizer, device): + for state in optimizer.state.values(): + for k, v in state.items(): + if isinstance(v, torch.Tensor): + state[k] = v.to(device=device) + return optimizer + +def is_rank0() -> bool: + """ + Best-effort check for main process without any synchronization. + """ + # Prefer torch.distributed state if initialized. + if dist is not None and dist.is_available() and dist.is_initialized(): + return dist.get_rank() == 0 + + # Fallback to environment variables commonly set by launchers. + for key in ("RANK", "SLURM_PROCID", "LOCAL_RANK"): + if key in os.environ: + return os.environ.get(key, "0") in ("0", "0\n", "") + + return True diff --git a/baseline/fastwam_zipper/source/src/fastwam/utils/samplers.py b/baseline/fastwam_zipper/source/src/fastwam/utils/samplers.py new file mode 100644 index 0000000000000000000000000000000000000000..8b4f52a26a749fadc78a5c4ae7fa9068570be000 --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/utils/samplers.py @@ -0,0 +1,39 @@ +from typing import Iterator, Sized + +import torch +from torch.utils.data import Sampler + + +class ResumableEpochSampler(Sampler[int]): + def __init__(self, dataset: Sized, seed: int, batch_size: int, num_processes: int): + self.dataset = dataset + self.seed = int(seed) + self.batch_size = int(batch_size) + self.num_processes = int(num_processes) + self.epoch = 0 + self.epoch_offset = 0 + self.resume_batch_offset = 0 + + def set_epoch(self, epoch: int): + self.epoch = int(epoch) + + def set_epoch_offset(self, epoch_offset: int): + self.epoch_offset = int(epoch_offset) + + def set_resume_batch_offset(self, batch_in_epoch: int): + self.resume_batch_offset = int(batch_in_epoch) + + def clear_resume_batch_offset(self): + self.resume_batch_offset = 0 + + def __iter__(self) -> Iterator[int]: + g = torch.Generator(device="cpu") + g.manual_seed(self.seed + self.epoch + self.epoch_offset) + indices = torch.randperm(len(self.dataset), generator=g).tolist() + if self.epoch == 0 and self.resume_batch_offset > 0: + sample_offset = self.resume_batch_offset * self.batch_size * self.num_processes + indices = indices[sample_offset:] + return iter(indices) + + def __len__(self) -> int: + return len(self.dataset) diff --git a/baseline/fastwam_zipper/source/src/fastwam/utils/video_io.py b/baseline/fastwam_zipper/source/src/fastwam/utils/video_io.py new file mode 100644 index 0000000000000000000000000000000000000000..74898b9c02a79c6b34b36af215a32743a8541d5d --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/utils/video_io.py @@ -0,0 +1,34 @@ +import os +from typing import Iterable + +import imageio +import numpy as np +from PIL import Image + +from .fs import ensure_dir + + +def _to_even_frame(frame: np.ndarray) -> np.ndarray: + h, w = frame.shape[:2] + pad_h = h % 2 + pad_w = w % 2 + if pad_h == 0 and pad_w == 0: + return frame + return np.pad(frame, ((0, pad_h), (0, pad_w), (0, 0)), mode="edge") + + +def save_mp4(frames: Iterable[Image.Image], path: str, fps: int = 8): + ensure_dir(os.path.dirname(path) or ".") + writer = imageio.get_writer( + path, + fps=max(fps, 1), + codec="libx264", + format="FFMPEG", + pixelformat="yuv420p", + ) + try: + for frame in frames: + arr = np.array(frame.convert("RGB")) + writer.append_data(_to_even_frame(arr)) + finally: + writer.close() diff --git a/baseline/fastwam_zipper/source/src/fastwam/utils/video_metrics.py b/baseline/fastwam_zipper/source/src/fastwam/utils/video_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..6b2837fa1d7b3c7c2532e99121ee46d68eb6be6e --- /dev/null +++ b/baseline/fastwam_zipper/source/src/fastwam/utils/video_metrics.py @@ -0,0 +1,95 @@ +from typing import Sequence + +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image + + +def pil_frames_to_video_tensor(frames: Sequence[Image.Image]) -> torch.Tensor: + if len(frames) == 0: + raise ValueError("`frames` must be non-empty.") + + frame_tensors = [] + for frame in frames: + arr = np.array(frame.convert("RGB"), dtype=np.float32) / 255.0 + x = torch.from_numpy(arr).permute(2, 0, 1).contiguous() # [3, H, W] + frame_tensors.append(x) + return torch.stack(frame_tensors, dim=1) # [3, T, H, W] + + +def _gaussian_kernel_2d(kernel_size: int, sigma: float, channels: int, device: torch.device, dtype: torch.dtype): + coords = torch.arange(kernel_size, device=device, dtype=dtype) - (kernel_size - 1) / 2.0 + g = torch.exp(-(coords**2) / (2.0 * sigma * sigma)) + g = g / g.sum() + kernel_2d = torch.outer(g, g) + kernel_2d = kernel_2d / kernel_2d.sum() + kernel_2d = kernel_2d.view(1, 1, kernel_size, kernel_size) + return kernel_2d.repeat(channels, 1, 1, 1) + + +def video_psnr(pred: torch.Tensor, target: torch.Tensor, data_range: float = 1.0, eps: float = 1e-8) -> float: + """ + Compute average PSNR over all frames. + Expects `pred` and `target` in shape [3, T, H, W] with values in [0, 1]. + """ + if pred.shape != target.shape: + raise ValueError(f"Shape mismatch: pred={tuple(pred.shape)} target={tuple(target.shape)}") + + pred = pred.float() + target = target.float() + mse = (pred - target).pow(2).mean(dim=(0, 2, 3)) # [T] + psnr = 10.0 * torch.log10((data_range * data_range) / (mse + eps)) + return float(psnr.mean().item()) + + +def video_ssim( + pred: torch.Tensor, + target: torch.Tensor, + data_range: float = 1.0, + kernel_size: int = 11, + sigma: float = 1.5, + k1: float = 0.01, + k2: float = 0.03, +) -> float: + """ + Compute average SSIM over all frames. + Expects `pred` and `target` in shape [3, T, H, W] with values in [0, 1]. + """ + if pred.shape != target.shape: + raise ValueError(f"Shape mismatch: pred={tuple(pred.shape)} target={tuple(target.shape)}") + if pred.ndim != 4 or pred.shape[0] != 3: + raise ValueError(f"Expected [3, T, H, W], got {tuple(pred.shape)}") + if kernel_size % 2 == 0: + raise ValueError("`kernel_size` must be odd.") + + pred = pred.float().permute(1, 0, 2, 3).contiguous() # [T, 3, H, W] + target = target.float().permute(1, 0, 2, 3).contiguous() # [T, 3, H, W] + channels = pred.shape[1] + kernel = _gaussian_kernel_2d( + kernel_size=kernel_size, + sigma=sigma, + channels=channels, + device=pred.device, + dtype=pred.dtype, + ) + + pad = kernel_size // 2 + mu_x = F.conv2d(pred, kernel, padding=pad, groups=channels) + mu_y = F.conv2d(target, kernel, padding=pad, groups=channels) + + mu_x2 = mu_x * mu_x + mu_y2 = mu_y * mu_y + mu_xy = mu_x * mu_y + + sigma_x2 = F.conv2d(pred * pred, kernel, padding=pad, groups=channels) - mu_x2 + sigma_y2 = F.conv2d(target * target, kernel, padding=pad, groups=channels) - mu_y2 + sigma_xy = F.conv2d(pred * target, kernel, padding=pad, groups=channels) - mu_xy + + c1 = (k1 * data_range) ** 2 + c2 = (k2 * data_range) ** 2 + + numerator = (2.0 * mu_xy + c1) * (2.0 * sigma_xy + c2) + denominator = (mu_x2 + mu_y2 + c1) * (sigma_x2 + sigma_y2 + c2) + ssim_map = numerator / (denominator + 1e-12) + return float(ssim_map.mean().item()) diff --git a/baseline/fastwam_zipper/step_016380.pt b/baseline/fastwam_zipper/step_016380.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3742d2871c43fa8ed8b312a54ba25deaf732d5f --- /dev/null +++ b/baseline/fastwam_zipper/step_016380.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e574c8cc7bb5747852cd815cfed9896d8d0f1db83eee28a367bd21beb25f348 +size 12041813369 diff --git a/baseline/fastwam_zipper/validation.json b/baseline/fastwam_zipper/validation.json new file mode 100644 index 0000000000000000000000000000000000000000..c2988abff2a5b6ab103c338f241584aeabb14507 --- /dev/null +++ b/baseline/fastwam_zipper/validation.json @@ -0,0 +1,25 @@ +{ + "status": "PASS_CPU_PACKAGING_VALIDATION", + "task": "zipper_pulling", + "step": 16380, + "strict_meta_state_dict": true, + "training_image_preprocessing_parity": true, + "training_normalization_parity": true, + "inverse_normalization": true, + "cached_context_valid": true, + "original_source_unchanged": true, + "offline_vae_resolution": true, + "wrapper_signature_and_conversion": true, + "new_GPU_forward": "NOT_RUN", + "new_robot_rollout": "NOT_RUN", + "CPU_validation_environment": { + "torch": "2.5.1", + "torchvision": "0.20.1", + "numpy": "1.24.4", + "pillow": "11.3.0", + "transformers": "4.49.0", + "einops": "0.8.1", + "safetensors": "0.4.5", + "rich": "15.0.0" + } +}