update_config

checkpoint.json

@@ -42,12 +42,12 @@
       "temporal_modality_margin": 15000
     },
     "schema_version": 1,
-    "sink_frames": 1,
+    "sink_frames": 3,
     "temporal_compression_factor": 4,
     "text_cache_max_len": 512,
     "vae_spatial_compression_factor": 16,
     "video_temporal_causal": true,
-    "window_frames": 51
+    "window_frames": 30
   },
   "source_weights": "ema",
   "student_only": true

config.json

@@ -25,14 +25,14 @@
     },
     "latent_patch_size": 2,
     "schema_version": 1,
-    "sink_frames": 1,
+    "sink_frames": 3,
     "temporal_compression_factor": 4,
     "temporal_modality_margin": 15000,
     "text_cache_max_len": 512,
     "unified_3d_mrope_reset_spatial_ids": true,
     "vae_spatial_compression_factor": 16,
     "video_temporal_causal": true,
-    "window_frames": 51
+    "window_frames": 30
   },
   "image_token_id": 151655,
   "model": {
@@ -63,7 +63,7 @@
       "ar_distilled_sample_type": null,
       "ar_static_und_cache_max_len": 512,
       "attention_io_layout": "sequence_sharded",
-      "attention_sink_size": 1,
+      "attention_sink_size": 3,
       "causal_training_strategy": "none",
       "clamp_empty_varlen_kv": true,
       "compile": {
@@ -121,22 +121,29 @@
       "flex_attention": {
         "_type": "causal_flex_attention_config",
         "backend": "auto",
-        "deduplicate_cross_view": false,
         "geometry_position_encoding": "baseline",
         "mask": {
           "_type": "causal_flex_attention_mask_config",
           "attention_scope": "all_views",
           "control_attends_sensor": false,
-          "control_to_control_window": null,
-          "control_to_sensor_window": null,
-          "decomposed_temporal_window_includes_first_frame": false,
-          "decomposed_temporal_window_seconds": null,
+          "cross_view_temporal_window": {
+            "_type": "cross_view_temporal_window_config",
+            "bounds": null,
+            "includes_first_frame": false
+          },
           "lidar_attends_captions": true,
           "radar_attends_captions": true,
-          "sensor_to_control_window": null,
-          "sensor_to_sensor_window": null
+          "same_view_temporal_window": {
+            "_type": "same_view_temporal_window_config",
+            "control_to_control_bounds": null,
+            "control_to_sensor_bounds": null,
+            "sensor_to_control_bounds": null,
+            "sensor_to_sensor_bounds": null
+          }
         },
-        "maskless_cross_view_band_keys": "all_views",
+        "maskless_cross_view_action_attends_action": false,
+        "maskless_cross_view_attends_action": false,
+        "maskless_cross_view_band_keys": "other_views",
         "maskless_cross_view_band_radius": 0,
         "maskless_cross_view_include_frame_zero": false,
         "prope_intrinsics": false,
@@ -159,7 +166,7 @@
       "input_video_key": "video",
       "joint_attn_implementation": "three_way",
       "kv_cache_dtype": null,
-      "kv_cache_inference_size": 51,
+      "kv_cache_inference_size": 30,
       "kv_cache_kernel_impl": "triton",
       "latent_downsample_factor": 16,
       "lbl": {
@@ -184,22 +191,29 @@
       "multiview_attention": {
         "_type": "flex_attention_config",
         "backend": "auto",
-        "deduplicate_cross_view": false,
         "geometry_position_encoding": "baseline",
         "mask": {
           "_type": "flex_attention_mask_config",
           "attention_scope": "all_views",
           "control_attends_sensor": false,
-          "control_to_control_window": null,
-          "control_to_sensor_window": null,
-          "decomposed_temporal_window_includes_first_frame": false,
-          "decomposed_temporal_window_seconds": null,
+          "cross_view_temporal_window": {
+            "_type": "cross_view_temporal_window_config",
+            "bounds": null,
+            "includes_first_frame": false
+          },
           "lidar_attends_captions": true,
           "radar_attends_captions": true,
-          "sensor_to_control_window": null,
-          "sensor_to_sensor_window": null
+          "same_view_temporal_window": {
+            "_type": "same_view_temporal_window_config",
+            "control_to_control_bounds": null,
+            "control_to_sensor_bounds": null,
+            "sensor_to_control_bounds": null,
+            "sensor_to_sensor_bounds": null
+          }
         },
-        "maskless_cross_view_band_keys": "all_views",
+        "maskless_cross_view_action_attends_action": false,
+        "maskless_cross_view_attends_action": false,
+        "maskless_cross_view_band_keys": "other_views",
         "maskless_cross_view_band_radius": 0,
         "maskless_cross_view_include_frame_zero": false,
         "prope_intrinsics": false,
@@ -279,6 +293,7 @@
       "rolling_prompt_context_after": 0,
       "rolling_prompt_context_before": 0,
       "rolling_prompt_policy": null,
+      "rolling_static_caption_max_tokens": 1024,
       "rolling_text_position_mode": "fixed",
       "rolling_text_sink_tokens": 4,
       "sigma_diffusion_forcing": 0.02,
@@ -303,6 +318,7 @@
       },
       "teacher_forcing_target_only_no_text_pass2": false,
       "teacher_forcing_transfer_control_dropout_rate": 0.0,
+      "timestep_embedding_chunk_size": 128000,
       "tokenizer": {
         "_target": "wan2pt2_vae_interface",
         "bucket_name": "bucket",

cosmos3_nano_sim_bimanual_artifact.json

@@ -5385,14 +5385,14 @@
       "temporal_modality_margin": 15000
     },
     "schema_version": 1,
-    "sink_frames": 1,
+    "sink_frames": 3,
     "temporal_compression_factor": 4,
     "text_cache_max_len": 512,
     "vae_spatial_compression_factor": 16,
     "video_temporal_causal": true,
-    "window_frames": 51
+    "window_frames": 30
   },
-  "manifest_sha256": "7ab55d41bc5ec447c681336b2d0858c7fcb851c250dce2cabcf7cd49fabc516d",
+  "manifest_sha256": "56afbbc0079d79617172006c36b8fa0243a5a0ca3d1ab3645e9d5162bf13be6f",
   "tensor_inventory": {
     "embed_tokens.weight": {
       "dtype": "BF16",

cosmos_dreams_manifest.json

@@ -1,49 +0,0 @@
-{
-  "attention_mode": "three_way",
-  "checkpoint_hash": "f72aa96e1b639ddc3db45caac76ff23810ffd63c47fb6bb108cd7c46f0b8f936",
-  "checkpoint_id": "causal_8b_sf_dmd_transfer_4modality_480p_ga_v2_midtrain_causal_control_with_rgb_history_text_dropout_0@iter_000001750",
-  "checkpoint_iteration": 1750,
-  "chunk_partition": [
-    1,
-    4
-  ],
-  "chunk_size": 4,
-  "conditioning": {
-    "emphasize_control_in_prompt": true,
-    "hints": [
-      "edge",
-      "blur",
-      "depth",
-      "seg"
-    ],
-    "mode": "control_video",
-    "no_eviction": true,
-    "share_vision_temporal_positions": true,
-    "system_prompt_id": "cosmos3_transfer_v1",
-    "transfer_control_attention_mode": "causal_control_with_rgb_history"
-  },
-  "fixed_step_sampler_config": {
-    "num_train_timesteps": 1000,
-    "sample_type": "sde",
-    "t_list": [
-      1.0,
-      0.9375,
-      0.8333333333333334,
-      0.625
-    ]
-  },
-  "latent_patch_size": 2,
-  "mrope": {
-    "base_fps": 24.0,
-    "enable_fps_modulation": true,
-    "reset_spatial_ids": true,
-    "temporal_modality_margin": 15000
-  },
-  "schema_version": 1,
-  "sink_frames": 0,
-  "temporal_compression_factor": 4,
-  "text_cache_max_len": 512,
-  "vae_spatial_compression_factor": 16,
-  "video_temporal_causal": true,
-  "window_frames": 96
-}

model_index.json

@@ -20,17 +20,17 @@
     },
     "latent_patch_size": 2,
     "schema_version": 1,
-    "sink_frames": 1,
+    "sink_frames": 3,
     "temporal_compression_factor": 4,
     "temporal_modality_margin": 15000,
     "text_cache_max_len": 512,
     "unified_3d_mrope_reset_spatial_ids": true,
     "vae_spatial_compression_factor": 16,
     "video_temporal_causal": true,
-    "window_frames": 51
+    "window_frames": 30
   },
-  "_diffusers_version": "0.39.0",
-  "default_use_system_prompt": true,
+  "_diffusers_version": "0.40.0",
+  "default_use_system_prompt": false,
   "enable_safety_checker": false,
   "safety_checker": [
     null,
@@ -46,7 +46,7 @@
   ],
   "text_tokenizer": [
     "transformers",
-    "Qwen2TokenizerFast"
+    "Qwen2Tokenizer"
   ],
   "transformer": [
     "diffusers",

modular_model_index.json

@@ -21,28 +21,30 @@
     },
     "latent_patch_size": 2,
     "schema_version": 1,
-    "sink_frames": 1,
+    "sink_frames": 3,
     "temporal_compression_factor": 4,
     "temporal_modality_margin": 15000,
     "text_cache_max_len": 512,
     "unified_3d_mrope_reset_spatial_ids": true,
     "vae_spatial_compression_factor": 16,
     "video_temporal_causal": true,
-    "window_frames": 51
+    "window_frames": 30
   },
-  "_diffusers_version": "0.39.0",
+  "_diffusers_version": "0.40.0",
+  "default_use_system_prompt": false,
   "distilled_sigmas": [
     1.0,
     0.9375,
     0.8333333333333334,
     0.625
   ],
+  "enable_safety_checker": false,
   "is_distilled": true,
   "scheduler": [
     "diffusers",
     "FlowMatchEulerDiscreteScheduler",
     {
-      "pretrained_model_name_or_path": "diffusers_format",
+      "pretrained_model_name_or_path": "diffusers_format7",
       "subfolder": "scheduler",
       "type_hint": [
         "diffusers",
@@ -53,13 +55,13 @@
   ],
   "text_tokenizer": [
     "transformers",
-    "Qwen2TokenizerFast",
+    "Qwen2Tokenizer",
     {
-      "pretrained_model_name_or_path": "diffusers_format",
+      "pretrained_model_name_or_path": "diffusers_format7",
       "subfolder": "text_tokenizer",
       "type_hint": [
         "transformers",
-        "Qwen2TokenizerFast"
+        "Qwen2Tokenizer"
       ],
       "variant": null
     }
@@ -68,7 +70,7 @@
     "diffusers",
     "Cosmos3OmniTransformer",
     {
-      "pretrained_model_name_or_path": "diffusers_format",
+      "pretrained_model_name_or_path": "diffusers_format7",
       "subfolder": "transformer",
       "type_hint": [
         "diffusers",
@@ -77,11 +79,12 @@
       "variant": null
     }
   ],
+  "use_native_flow_schedule": false,
   "vae": [
     "diffusers",
     "AutoencoderKLWan",
     {
-      "pretrained_model_name_or_path": "diffusers_format",
+      "pretrained_model_name_or_path": "diffusers_format7",
       "subfolder": "vae",
       "type_hint": [
         "diffusers",

scheduler/scheduler_config.json

@@ -1,6 +1,6 @@
 {
   "_class_name": "FlowMatchEulerDiscreteScheduler",
-  "_diffusers_version": "0.39.0",
+  "_diffusers_version": "0.40.0",
   "base_image_seq_len": 256,
   "base_shift": 0.5,
   "fixed_step_requires_explicit_sigmas": true,

text_tokenizer/tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
-size 11422654
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

text_tokenizer/tokenizer_config.json

@@ -1,217 +1,11 @@
 {
-  "add_bos_token": false,
   "add_prefix_space": false,
-  "added_tokens_decoder": {
-    "151643": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151644": {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151645": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151646": {
-      "content": "<|object_ref_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151647": {
-      "content": "<|object_ref_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151648": {
-      "content": "<|box_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151649": {
-      "content": "<|box_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151650": {
-      "content": "<|quad_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151651": {
-      "content": "<|quad_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151652": {
-      "content": "<|vision_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151653": {
-      "content": "<|vision_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151654": {
-      "content": "<|vision_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151655": {
-      "content": "<|image_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151656": {
-      "content": "<|video_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151657": {
-      "content": "<tool_call>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151658": {
-      "content": "</tool_call>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151659": {
-      "content": "<|fim_prefix|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151660": {
-      "content": "<|fim_middle|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151661": {
-      "content": "<|fim_suffix|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151662": {
-      "content": "<|fim_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151663": {
-      "content": "<|repo_name|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151664": {
-      "content": "<|file_sep|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151665": {
-      "content": "<tool_response>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151666": {
-      "content": "</tool_response>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151667": {
-      "content": "<think>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151668": {
-      "content": "</think>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    }
-  },
-  "additional_special_tokens": [
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
     "<|object_ref_start|>",
@@ -226,11 +20,8 @@
     "<|image_pad|>",
     "<|video_pad|>"
   ],
-  "bos_token": null,
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
-  "errors": "replace",
-  "extra_special_tokens": {},
+  "is_local": false,
+  "local_files_only": false,
   "model_max_length": 262144,
   "pad_token": "<|endoftext|>",
   "split_special_tokens": false,

transformer/config.json

@@ -1,6 +1,6 @@
 {
   "_class_name": "Cosmos3OmniTransformer",
-  "_diffusers_version": "0.39.0",
+  "_diffusers_version": "0.40.0",
   "action_dim": 32,
   "action_gen": false,
   "attention_bias": false,
@@ -40,14 +40,14 @@
     },
     "latent_patch_size": 2,
     "schema_version": 1,
-    "sink_frames": 1,
+    "sink_frames": 3,
     "temporal_compression_factor": 4,
     "temporal_modality_margin": 15000,
     "text_cache_max_len": 512,
     "unified_3d_mrope_reset_spatial_ids": true,
     "vae_spatial_compression_factor": 16,
     "video_temporal_causal": true,
-    "window_frames": 51
+    "window_frames": 30
   },
   "enable_fps_modulation": true,
   "head_dim": 128,
@@ -76,6 +76,7 @@
       20,
       20
     ],
+    "rope_theta": 5000000,
     "rope_type": "default"
   },
   "rope_theta": 5000000,

vae/config.json

@@ -1,6 +1,6 @@
 {
   "_class_name": "AutoencoderKLWan",
-  "_diffusers_version": "0.39.0",
+  "_diffusers_version": "0.40.0",
   "_name_or_path": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
   "attn_scales": [],
   "base_dim": 160,