5-task joint config

spatial_grafting_5task_p2/config.yaml

@@ -0,0 +1,420 @@
+output_dir: /shared_work/maya/fastwam/runs/spatial_grafting_5task_p2_2026-08-23_21-48-59
+batch_size: 2
+num_workers: 8
+lr_scheduler_type: cosine
+learning_rate: 0.0001
+num_epochs: 5
+max_steps: 10869
+log_every: 10
+save_every: 500
+eval_every: 0
+eval_num_inference_steps: 10
+gradient_accumulation_steps: 8
+mixed_precision: bf16
+distributed_timeout_minutes: 60
+seed: 42
+max_grad_norm: 1.0
+weight_decay: 0.01
+resume: /shared_work/maya/fastwam/runs/spatial_grafting_5task_p1_2026-08-23_21-48-59/checkpoints/weights/step_002718.pt
+allow_unverified_full_state_resume: false
+freeze_video_backbone: false
+spatial_warmup_steps: 0
+spatial_warmup_epochs: 0
+freeze_fastwam_during_spatial_warmup: false
+fastwam_inference_only: false
+spatial_lr: 0.0001
+fastwam_lr: 1.0e-05
+node_ram_guard:
+  enabled: true
+  min_available_percent: 20.0
+hf_upload:
+  enabled: false
+  dry_run: false
+  fail_on_error: false
+  script_path: huggingface/upload_to_hf.sh
+  bucket_run_name: null
+wandb:
+  enabled: false
+  workspace: Faster-WAM
+  project: fast-wam
+  name: spatial_grafting_5task_p2_2026-08-23_21-48-59
+  group: null
+  mode: online
+data:
+  train:
+    _target_: fastwam.datasets.lerobot.robot_video_dataset.RobotVideoDataset
+    dataset_dirs:
+    - /shared_work/datasets/robotwin2.0_spatial_full
+    shape_meta:
+      images:
+      - key: cam_high
+        raw_shape:
+        - 3
+        - 480
+        - 640
+        shape:
+        - 3
+        - 240
+        - 320
+      - key: cam_left_wrist
+        raw_shape:
+        - 3
+        - 480
+        - 640
+        shape:
+        - 3
+        - 240
+        - 320
+      - key: cam_right_wrist
+        raw_shape:
+        - 3
+        - 480
+        - 640
+        shape:
+        - 3
+        - 240
+        - 320
+      action:
+      - key: default
+        raw_shape: 14
+        shape: 14
+      state:
+      - key: default
+        raw_shape: 14
+        shape: 14
+    num_frames: 33
+    global_sample_stride: 1
+    action_video_freq_ratio: 4
+    video_size:
+    - 384
+    - 320
+    camera_key: null
+    val_set_proportion: 0.01
+    is_training_set: true
+    episode_indices_path: /shared_work/maya/fastwam/splits/first5_tasks_episodes.json
+    pretrained_norm_stats: ./data/robotwin2.0/dataset_stats.json
+    skip_padding_as_possible: false
+    concat_multi_camera: robotwin
+    spatial_field_map:
+      spatial_depth:
+        key: observation.spatial_depth
+        shape:
+        - 3
+        - 1
+        - 18
+        - 24
+      spatial_intrinsics:
+        key: observation.spatial_intrinsics
+        shape:
+        - 3
+        - 3
+        - 3
+      spatial_extrinsics_w2c:
+        key: observation.spatial_extrinsics_w2c
+        shape:
+        - 3
+        - 4
+        - 4
+    processor:
+      _target_: fastwam.datasets.lerobot.processors.fastwam_processor.FastWAMProcessor
+      shape_meta:
+        images:
+        - key: cam_high
+          raw_shape:
+          - 3
+          - 480
+          - 640
+          shape:
+          - 3
+          - 240
+          - 320
+        - key: cam_left_wrist
+          raw_shape:
+          - 3
+          - 480
+          - 640
+          shape:
+          - 3
+          - 240
+          - 320
+        - key: cam_right_wrist
+          raw_shape:
+          - 3
+          - 480
+          - 640
+          shape:
+          - 3
+          - 240
+          - 320
+        action:
+        - key: default
+          raw_shape: 14
+          shape: 14
+        state:
+        - key: default
+          raw_shape: 14
+          shape: 14
+      num_obs_steps: 33
+      num_output_cameras: 3
+      action_output_dim: 14
+      proprio_output_dim: 14
+      action_state_transforms: null
+      use_stepwise_action_norm: false
+      norm_default_mode: z-score
+      norm_exception_mode: null
+      action_state_merger:
+        _target_: fastwam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign
+      train_transforms:
+      - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor
+      - _target_: torchvision.transforms.Resize
+        size:
+        - 240
+        - 320
+      val_transforms:
+      - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor
+      - _target_: torchvision.transforms.Resize
+        size:
+        - 240
+        - 320
+    text_embedding_cache_dir: /shared_work/datasets/text_embeds_cache/robotwin
+    context_len: 128
+    video_latent_cache_dir: null
+  val:
+    _target_: fastwam.datasets.lerobot.robot_video_dataset.RobotVideoDataset
+    dataset_dirs:
+    - /shared_work/datasets/robotwin2.0_spatial_full
+    shape_meta:
+      images:
+      - key: cam_high
+        raw_shape:
+        - 3
+        - 480
+        - 640
+        shape:
+        - 3
+        - 240
+        - 320
+      - key: cam_left_wrist
+        raw_shape:
+        - 3
+        - 480
+        - 640
+        shape:
+        - 3
+        - 240
+        - 320
+      - key: cam_right_wrist
+        raw_shape:
+        - 3
+        - 480
+        - 640
+        shape:
+        - 3
+        - 240
+        - 320
+      action:
+      - key: default
+        raw_shape: 14
+        shape: 14
+      state:
+      - key: default
+        raw_shape: 14
+        shape: 14
+    num_frames: 33
+    global_sample_stride: 1
+    action_video_freq_ratio: 4
+    video_size:
+    - 384
+    - 320
+    camera_key: null
+    val_set_proportion: 0.01
+    is_training_set: false
+    episode_indices_path: /shared_work/maya/fastwam/splits/first5_tasks_episodes.json
+    pretrained_norm_stats: ./data/robotwin2.0/dataset_stats.json
+    skip_padding_as_possible: false
+    concat_multi_camera: robotwin
+    spatial_field_map:
+      spatial_depth:
+        key: observation.spatial_depth
+        shape:
+        - 3
+        - 1
+        - 18
+        - 24
+      spatial_intrinsics:
+        key: observation.spatial_intrinsics
+        shape:
+        - 3
+        - 3
+        - 3
+      spatial_extrinsics_w2c:
+        key: observation.spatial_extrinsics_w2c
+        shape:
+        - 3
+        - 4
+        - 4
+    processor:
+      _target_: fastwam.datasets.lerobot.processors.fastwam_processor.FastWAMProcessor
+      shape_meta:
+        images:
+        - key: cam_high
+          raw_shape:
+          - 3
+          - 480
+          - 640
+          shape:
+          - 3
+          - 240
+          - 320
+        - key: cam_left_wrist
+          raw_shape:
+          - 3
+          - 480
+          - 640
+          shape:
+          - 3
+          - 240
+          - 320
+        - key: cam_right_wrist
+          raw_shape:
+          - 3
+          - 480
+          - 640
+          shape:
+          - 3
+          - 240
+          - 320
+        action:
+        - key: default
+          raw_shape: 14
+          shape: 14
+        state:
+        - key: default
+          raw_shape: 14
+          shape: 14
+      num_obs_steps: 33
+      num_output_cameras: 3
+      action_output_dim: 14
+      proprio_output_dim: 14
+      action_state_transforms: null
+      use_stepwise_action_norm: false
+      norm_default_mode: z-score
+      norm_exception_mode: null
+      action_state_merger:
+        _target_: fastwam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign
+      train_transforms:
+      - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor
+      - _target_: torchvision.transforms.Resize
+        size:
+        - 240
+        - 320
+      val_transforms:
+      - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor
+      - _target_: torchvision.transforms.Resize
+        size:
+        - 240
+        - 320
+    text_embedding_cache_dir: /shared_work/datasets/text_embeds_cache/robotwin
+    context_len: 128
+    video_latent_cache_dir: null
+model:
+  _target_: fastwam.runtime.create_fastwam
+  model_id: Wan-AI/Wan2.2-TI2V-5B
+  tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B
+  tokenizer_max_len: 128
+  load_text_encoder: false
+  proprio_dim: 14
+  redirect_common_files: true
+  mot_checkpoint_mixed_attn: false
+  action_dit_pretrained_path: null
+  skip_dit_load_from_pretrain: true
+  freeze_video_backbone: false
+  use_spatial_grafting: true
+  spatial_grafting:
+    da3_model_id: depth-anything/DA3-GIANT-1.1
+    da3_feature_layers:
+    - 19
+    - 27
+    - 33
+    - 39
+    da3_channels: 1536
+    da3_input_hw:
+    - 252
+    - 336
+    da3_patch_size: 14
+    inject_last_n: 10
+    num_heads: 8
+    spatial_scale: 1.0
+    init_std: 0.01
+    logit_gain_init: 3.0
+    logit_gain_max: 8.0
+    fourier_bands: 10
+    point_centre:
+    - -0.0222
+    - 0.0967
+    - 0.7403
+    point_centre_ee_l:
+    - 0.811
+    - 0.278
+    - 0.332
+    point_centre_ee_r:
+    - 0.268
+    - 0.895
+    - 0.154
+    point_scale: 0.5198
+    point_max_depth: 5.0
+    s1: true
+    no_bias: true
+    sample_token_center: true
+    tap_token_center: true
+    center_fused: true
+    tap_gain: true
+    film_init_std: 0.01
+    inject_dropout: 0.1
+  video_dit_config:
+    has_image_input: false
+    patch_size:
+    - 1
+    - 2
+    - 2
+    in_dim: 48
+    hidden_dim: 3072
+    ffn_dim: 14336
+    freq_dim: 256
+    text_dim: 4096
+    out_dim: 48
+    num_heads: 24
+    attn_head_dim: 128
+    num_layers: 30
+    eps: 1.0e-06
+    seperated_timestep: true
+    require_clip_embedding: false
+    require_vae_embedding: false
+    fuse_vae_embedding_in_latents: true
+    use_gradient_checkpointing: false
+    video_attention_mask_mode: first_frame_causal
+    action_conditioned: false
+    action_dim: 14
+    action_group_causal_mask_mode: group_diagonal
+  action_dit_config:
+    action_dim: 14
+    hidden_dim: 1024
+    ffn_dim: 4096
+    num_heads: 24
+    attn_head_dim: 128
+    num_layers: 30
+    text_dim: 4096
+    freq_dim: 256
+    eps: 1.0e-06
+    use_gradient_checkpointing: false
+    action_text_cross_attn: true
+  video_scheduler:
+    train_shift: 5.0
+    infer_shift: 5.0
+    num_train_timesteps: 1000
+  action_scheduler:
+    train_shift: 5.0
+    infer_shift: 5.0
+    num_train_timesteps: 1000
+  loss:
+    lambda_action: 1.0