Hugging Face's logo

limxdynamics
/
FluxVLAEngine

Safetensors
Model card Files
xet
 Community
6

Add Cosmos3 Nano RoboCasa pure-vision checkpoint

#6
by RJK - opened
base: refs/heads/main
←
from: refs/pr/6
Discussion Files changed
+154460
-0
Files changed (13) hide show
  1. .gitattributes +1 -0
  2. cosmos3_nano_robocasa_full_data_full_finetune_bs2048/checkpoints/step-010000-epoch-03-loss=1.1285.safetensors +3 -0
  3. cosmos3_nano_robocasa_full_data_full_finetune_bs2048/config.json +1155 -0
  4. cosmos3_nano_robocasa_full_data_full_finetune_bs2048/config.yaml +995 -0
  5. cosmos3_nano_robocasa_full_data_full_finetune_bs2048/dataset_statistics.json +396 -0
  6. cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/added_tokens.json +28 -0
  7. cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/chat_template.jinja +120 -0
  8. cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/merges.txt +0 -0
  9. cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/special_tokens_map.json +31 -0
  10. cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/tokenizer.json +3 -0
  11. cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/tokenizer_config.json +239 -0
  12. cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/vocab.json +0 -0
  13. cosmos3_nano_robocasa_full_data_full_finetune_bs2048/vlm_backbone_config.json +100 -0
.gitattributes CHANGED
@@ -63,3 +63,4 @@ gr00t_qwen3vl_0.6b_robocasa_full_data_full_finetune_bs128/tokenizer/tokenizer.js
63
  Qwen3-VL-0.6B/tokenizer.json filter=lfs diff=lfs merge=lfs -text
64
  dit4dit_robocasa_full_data_full_finetune_bs64/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
65
  dit4dit_robocasa_full_data_full_finetune_bs64/dit4dit_robocasa_full_data_full_finetune_2026_08_29_12_40_23.jsonl filter=lfs diff=lfs merge=lfs -text
 
 
63
  Qwen3-VL-0.6B/tokenizer.json filter=lfs diff=lfs merge=lfs -text
64
  dit4dit_robocasa_full_data_full_finetune_bs64/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
65
  dit4dit_robocasa_full_data_full_finetune_bs64/dit4dit_robocasa_full_data_full_finetune_2026_08_29_12_40_23.jsonl filter=lfs diff=lfs merge=lfs -text
66
+ cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
cosmos3_nano_robocasa_full_data_full_finetune_bs2048/checkpoints/step-010000-epoch-03-loss=1.1285.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f98de25f84d96aeb6e98c5cd93b9af39bd336dc22029521ebf1b6901af3c6623
3
+ size 63511447112
cosmos3_nano_robocasa_full_data_full_finetune_bs2048/config.json ADDED
@@ -0,0 +1,1155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_ACTION_DIM": 29,
3
+ "_ACTION_HORIZON": 16,
4
+ "_ACTION_LR": 0.00025,
5
+ "_BASE_LR": 5e-05,
6
+ "_CFG_DROPOUT_RATE": 0.1,
7
+ "_CKPT_ROOT": "./checkpoints",
8
+ "_CONDITIONING_FPS": 20.0,
9
+ "_COSMOS3_NANO_CKPT": "./checkpoints/Cosmos3-Nano",
10
+ "_COSMOS3_NANO_NAME_MAPPING": {
11
+ ".self_attn.k_norm.": ".self_attn.norm_k.",
12
+ ".self_attn.k_norm_moe_gen.": ".self_attn.norm_added_k.",
13
+ ".self_attn.k_proj.": ".self_attn.to_k.",
14
+ ".self_attn.k_proj_moe_gen.": ".self_attn.add_k_proj.",
15
+ ".self_attn.o_proj.": ".self_attn.to_out.",
16
+ ".self_attn.o_proj_moe_gen.": ".self_attn.to_add_out.",
17
+ ".self_attn.q_norm.": ".self_attn.norm_q.",
18
+ ".self_attn.q_norm_moe_gen.": ".self_attn.norm_added_q.",
19
+ ".self_attn.q_proj.": ".self_attn.to_q.",
20
+ ".self_attn.q_proj_moe_gen.": ".self_attn.add_q_proj.",
21
+ ".self_attn.v_proj.": ".self_attn.to_v.",
22
+ ".self_attn.v_proj_moe_gen.": ".self_attn.add_v_proj.",
23
+ "action_in_proj.": "action_proj_in.",
24
+ "action_modality_embed": "action_modality_embed",
25
+ "action_out_proj.": "action_proj_out.",
26
+ "time_embedder.mlp.0.": "time_embedder.linear_1.",
27
+ "time_embedder.mlp.2.": "time_embedder.linear_2.",
28
+ "vision_in_proj.projector.": "proj_in.",
29
+ "vision_out_proj.projector.": "proj_out.",
30
+ "vlm_backbone.lm_head.weight": "lm_head.weight",
31
+ "vlm_backbone.model.language_model.embed_tokens.weight": "embed_tokens.weight",
32
+ "vlm_backbone.model.language_model.layers.": "layers.",
33
+ "vlm_backbone.model.language_model.norm.weight": "norm.weight",
34
+ "vlm_backbone.model.language_model.norm_moe_gen.weight": "norm_moe_gen.weight",
35
+ "vlm_backbone.model.visual.blocks.": "blocks.",
36
+ "vlm_backbone.model.visual.deepstack_merger_list.": "deepstack_merger_list.",
37
+ "vlm_backbone.model.visual.merger.": "merger.",
38
+ "vlm_backbone.model.visual.patch_embed.": "patch_embed.",
39
+ "vlm_backbone.model.visual.pos_embed.": "pos_embed."
40
+ },
41
+ "_COSMOS3_NANO_SPECIAL_TOKENS": {
42
+ "end_of_generation": 151653,
43
+ "eos_token_id": 151645,
44
+ "start_of_generation": 151652
45
+ },
46
+ "_COSMOS3_NANO_TOKENIZER": {
47
+ "model_max_length": 4096,
48
+ "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
49
+ "padding_side": "right",
50
+ "trust_remote_code": true,
51
+ "type": "PretrainedTokenizer"
52
+ },
53
+ "_COSMOS3_NANO_TRANSFORMER": "./checkpoints/Cosmos3-Nano/transformer",
54
+ "_COSMOS3_NANO_VISION_ENCODER": "./checkpoints/Cosmos3-Nano/vision_encoder",
55
+ "_COSMOS3_NANO_VLM_CONFIG": {
56
+ "image_token_id": 151655,
57
+ "model_type": "qwen3_vl",
58
+ "text_config": {
59
+ "attention_bias": false,
60
+ "attention_dropout": 0.0,
61
+ "bos_token_id": 151643,
62
+ "eos_token_id": 151645,
63
+ "head_dim": 128,
64
+ "hidden_act": "silu",
65
+ "hidden_size": 4096,
66
+ "intermediate_size": 12288,
67
+ "layer_types": [
68
+ "full_attention",
69
+ "full_attention",
70
+ "full_attention",
71
+ "full_attention",
72
+ "full_attention",
73
+ "full_attention",
74
+ "full_attention",
75
+ "full_attention",
76
+ "full_attention",
77
+ "full_attention",
78
+ "full_attention",
79
+ "full_attention",
80
+ "full_attention",
81
+ "full_attention",
82
+ "full_attention",
83
+ "full_attention",
84
+ "full_attention",
85
+ "full_attention",
86
+ "full_attention",
87
+ "full_attention",
88
+ "full_attention",
89
+ "full_attention",
90
+ "full_attention",
91
+ "full_attention",
92
+ "full_attention",
93
+ "full_attention",
94
+ "full_attention",
95
+ "full_attention",
96
+ "full_attention",
97
+ "full_attention",
98
+ "full_attention",
99
+ "full_attention",
100
+ "full_attention",
101
+ "full_attention",
102
+ "full_attention",
103
+ "full_attention"
104
+ ],
105
+ "max_position_embeddings": 262144,
106
+ "model_type": "qwen3_vl_text",
107
+ "num_attention_heads": 32,
108
+ "num_hidden_layers": 36,
109
+ "num_key_value_heads": 8,
110
+ "pad_token_id": 0,
111
+ "rms_norm_eps": 1e-06,
112
+ "rope_scaling": {
113
+ "mrope_interleaved": true,
114
+ "mrope_section": [
115
+ 24,
116
+ 20,
117
+ 20
118
+ ],
119
+ "rope_type": "default"
120
+ },
121
+ "rope_theta": 5000000,
122
+ "tie_word_embeddings": false,
123
+ "vocab_size": 151936
124
+ },
125
+ "tie_word_embeddings": false,
126
+ "video_token_id": 151656,
127
+ "vision_config": {
128
+ "deepstack_visual_indexes": [
129
+ 8,
130
+ 16,
131
+ 24
132
+ ],
133
+ "depth": 27,
134
+ "hidden_act": "gelu_pytorch_tanh",
135
+ "hidden_size": 1152,
136
+ "in_channels": 3,
137
+ "initializer_range": 0.02,
138
+ "intermediate_size": 4304,
139
+ "model_type": "qwen3_vl",
140
+ "num_heads": 16,
141
+ "num_position_embeddings": 2304,
142
+ "out_hidden_size": 4096,
143
+ "patch_size": 16,
144
+ "spatial_merge_size": 2,
145
+ "temporal_patch_size": 2
146
+ },
147
+ "vision_end_token_id": 151653,
148
+ "vision_start_token_id": 151652,
149
+ "vocab_size": 151936
150
+ },
151
+ "_FRAME_WINDOW_SIZE": 17,
152
+ "_GRAD_ACCUMULATION_STEPS": 8,
153
+ "_IMAGE_HEIGHT": 256,
154
+ "_IMAGE_WIDTH": 256,
155
+ "_MAX_ACTION_DIM": 64,
156
+ "_MAX_STATE_DIM": 64,
157
+ "_MAX_STEPS": 10000,
158
+ "_OFFICIAL_GR1_STATS_PATH": "./datasets/robocasa_gr1_24tasks_first30ep/official_groot_gr1_dataset_statistics.json",
159
+ "_PER_DEVICE_BATCH_SIZE": 16,
160
+ "_PREPEND_STATE_TO_ACTION": true,
161
+ "_ROBOCASA_DATA_ROOT": "./datasets/robocasa_lerobot_V2.1",
162
+ "_ROBOCASA_EMBODIMENT_ID": 24,
163
+ "_ROBOCASA_ENV_SUFFIX": "_GR1ArmsAndWaistFourierHands_Env",
164
+ "_ROBOCASA_TASK_DIRS": [
165
+ "PnPBottleToCabinetClose",
166
+ "PnPCanToDrawerClose",
167
+ "PnPCupToDrawerClose",
168
+ "PnPMilkToMicrowaveClose",
169
+ "PnPPotatoToMicrowaveClose",
170
+ "PnPWineToCabinetClose",
171
+ "PosttrainPnPNovelFromCuttingboardToBasketSplitA",
172
+ "PosttrainPnPNovelFromCuttingboardToCardboardboxSplitA",
173
+ "PosttrainPnPNovelFromCuttingboardToPanSplitA",
174
+ "PosttrainPnPNovelFromCuttingboardToPotSplitA",
175
+ "PosttrainPnPNovelFromCuttingboardToTieredbasketSplitA",
176
+ "PosttrainPnPNovelFromPlacematToBasketSplitA",
177
+ "PosttrainPnPNovelFromPlacematToBowlSplitA",
178
+ "PosttrainPnPNovelFromPlacematToPlateSplitA",
179
+ "PosttrainPnPNovelFromPlacematToTieredshelfSplitA",
180
+ "PosttrainPnPNovelFromPlateToBowlSplitA",
181
+ "PosttrainPnPNovelFromPlateToCardboardboxSplitA",
182
+ "PosttrainPnPNovelFromPlateToPanSplitA",
183
+ "PosttrainPnPNovelFromPlateToPlateSplitA",
184
+ "PosttrainPnPNovelFromTrayToCardboardboxSplitA",
185
+ "PosttrainPnPNovelFromTrayToPlateSplitA",
186
+ "PosttrainPnPNovelFromTrayToPotSplitA",
187
+ "PosttrainPnPNovelFromTrayToTieredbasketSplitA",
188
+ "PosttrainPnPNovelFromTrayToTieredshelfSplitA"
189
+ ],
190
+ "_ROBOCASA_TASK_PREFIX": "gr1_unified",
191
+ "_SAVE_ITER_INTERVAL": 1000,
192
+ "_STATISTIC_NAME": "robocasa_gr1_24tasks_30ep",
193
+ "_TRANSFORMS": [
194
+ {
195
+ "embodiment_id": 24,
196
+ "name_mappings": {
197
+ "actions": [
198
+ "actions"
199
+ ],
200
+ "observation.state": [
201
+ "states"
202
+ ]
203
+ },
204
+ "parquet_keys": [
205
+ "observation.state",
206
+ "timestamp",
207
+ "actions",
208
+ "info",
209
+ "stats",
210
+ "action_masks"
211
+ ],
212
+ "type": "ProcessParquetInputs",
213
+ "video_keys": [
214
+ "observation.images.ego_view"
215
+ ]
216
+ },
217
+ {
218
+ "action_metadata": {
219
+ "append_viewpoint": false,
220
+ "conditioning_fps": 20.0,
221
+ "frame_window_size": 17,
222
+ "video_height": 256,
223
+ "video_width": 256,
224
+ "viewpoint": "ego_view"
225
+ },
226
+ "cfg_dropout_rate": 0.1,
227
+ "format_prompt_as_json": true,
228
+ "max_len": 512,
229
+ "tokenizer": {
230
+ "model_max_length": 4096,
231
+ "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
232
+ "padding_side": "right",
233
+ "trust_remote_code": true,
234
+ "type": "PretrainedTokenizer"
235
+ },
236
+ "type": "ProcessCosmos3Prompt"
237
+ },
238
+ {
239
+ "scale": 0.95,
240
+ "type": "RandomCropImages"
241
+ },
242
+ {
243
+ "height": 256,
244
+ "type": "ResizeImages",
245
+ "width": 256
246
+ },
247
+ {
248
+ "brightness": 0.3,
249
+ "contrast": 0.4,
250
+ "hue": 0.08,
251
+ "saturation": 0.5,
252
+ "type": "ColorJitterImages"
253
+ },
254
+ {
255
+ "type": "SimpleNormalizeImages"
256
+ },
257
+ {
258
+ "action_dim": 64,
259
+ "action_key": "action",
260
+ "action_norm_type": "quantile",
261
+ "state_dim": 64,
262
+ "state_key": "action",
263
+ "state_norm_type": "quantile",
264
+ "type": "NormalizeStatesAndActions"
265
+ },
266
+ {
267
+ "conditioning_fps": 20.0,
268
+ "frame_window_size": 17,
269
+ "mode": "wam",
270
+ "prepend_state_to_action": true,
271
+ "raw_action_dim": 29,
272
+ "type": "BuildCosmos3Sequence"
273
+ },
274
+ {
275
+ "frame_window_size": 17,
276
+ "num_views": 1,
277
+ "tile_direction": "horizontal",
278
+ "type": "PrepareVideo"
279
+ },
280
+ {
281
+ "height": 256,
282
+ "type": "ResizeAndReflectPad",
283
+ "width": 256
284
+ }
285
+ ],
286
+ "_VIDEO_HEIGHT": 256,
287
+ "_VIDEO_WIDTH": 256,
288
+ "_WAN22_VAE_PATH": "./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
289
+ "eval": {
290
+ "action_order": "fluxvla",
291
+ "benchmark": "robocasa",
292
+ "dataset": {
293
+ "extra_tensor_keys": [
294
+ "conditioning_fps",
295
+ "prepend_state_to_action"
296
+ ],
297
+ "transforms": [
298
+ {
299
+ "center_crop_scale": 0.95,
300
+ "embodiment_id": 24,
301
+ "img_key": "video.ego_view_bg_crop_pad_res256_freq20",
302
+ "normalize": true,
303
+ "resize_size": 256,
304
+ "type": "ProcessRobocasaEvalInputs",
305
+ "value_range": "tanh"
306
+ },
307
+ {
308
+ "conditioning_fps": 20.0,
309
+ "prepend_state_to_action": false,
310
+ "type": "SetCosmos3ActionMetadata"
311
+ },
312
+ {
313
+ "action_key": null,
314
+ "state_dim": 64,
315
+ "state_key": "action",
316
+ "state_norm_type": "quantile",
317
+ "type": "NormalizeStatesAndActions"
318
+ },
319
+ {
320
+ "action_metadata": {
321
+ "append_viewpoint": false,
322
+ "conditioning_fps": 20.0,
323
+ "frame_window_size": 17,
324
+ "video_height": 256,
325
+ "video_width": 256,
326
+ "viewpoint": "ego_view"
327
+ },
328
+ "cfg_dropout_rate": 0.0,
329
+ "format_prompt_as_json": true,
330
+ "max_len": 512,
331
+ "output_attention_mask_key": "lang_masks",
332
+ "output_key": "lang_tokens",
333
+ "tokenizer": {
334
+ "model_max_length": 4096,
335
+ "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
336
+ "padding_side": "right",
337
+ "trust_remote_code": true,
338
+ "type": "PretrainedTokenizer"
339
+ },
340
+ "type": "ProcessCosmos3Prompt"
341
+ },
342
+ {
343
+ "frame_window_size": 1,
344
+ "num_views": 1,
345
+ "tile_direction": "horizontal",
346
+ "type": "PrepareVideo"
347
+ },
348
+ {
349
+ "height": 256,
350
+ "type": "ResizeAndReflectPad",
351
+ "width": 256
352
+ }
353
+ ],
354
+ "type": "RobocasaEvalDataset",
355
+ "unnorm_key": "robocasa_gr1_24tasks_30ep"
356
+ },
357
+ "denormalize_action": {
358
+ "action_dim": 29,
359
+ "clip_actions": false,
360
+ "norm_type": "quantile",
361
+ "stats_order": "native",
362
+ "type": "DenormalizeRobocasaAction"
363
+ },
364
+ "enable_mixed_precision_training": true,
365
+ "eval_chunk_size": 8,
366
+ "max_episode_steps": 720,
367
+ "mixed_precision_dtype": "bf16",
368
+ "model_family": "cosmos3",
369
+ "num_trials_per_task": 50,
370
+ "seed": 7,
371
+ "task_list": [
372
+ "gr1_unified/PnPBottleToCabinetClose_GR1ArmsAndWaistFourierHands_Env",
373
+ "gr1_unified/PnPCanToDrawerClose_GR1ArmsAndWaistFourierHands_Env",
374
+ "gr1_unified/PnPCupToDrawerClose_GR1ArmsAndWaistFourierHands_Env",
375
+ "gr1_unified/PnPMilkToMicrowaveClose_GR1ArmsAndWaistFourierHands_Env",
376
+ "gr1_unified/PnPPotatoToMicrowaveClose_GR1ArmsAndWaistFourierHands_Env",
377
+ "gr1_unified/PnPWineToCabinetClose_GR1ArmsAndWaistFourierHands_Env",
378
+ "gr1_unified/PosttrainPnPNovelFromCuttingboardToBasketSplitA_GR1ArmsAndWaistFourierHands_Env",
379
+ "gr1_unified/PosttrainPnPNovelFromCuttingboardToCardboardboxSplitA_GR1ArmsAndWaistFourierHands_Env",
380
+ "gr1_unified/PosttrainPnPNovelFromCuttingboardToPanSplitA_GR1ArmsAndWaistFourierHands_Env",
381
+ "gr1_unified/PosttrainPnPNovelFromCuttingboardToPotSplitA_GR1ArmsAndWaistFourierHands_Env",
382
+ "gr1_unified/PosttrainPnPNovelFromCuttingboardToTieredbasketSplitA_GR1ArmsAndWaistFourierHands_Env",
383
+ "gr1_unified/PosttrainPnPNovelFromPlacematToBasketSplitA_GR1ArmsAndWaistFourierHands_Env",
384
+ "gr1_unified/PosttrainPnPNovelFromPlacematToBowlSplitA_GR1ArmsAndWaistFourierHands_Env",
385
+ "gr1_unified/PosttrainPnPNovelFromPlacematToPlateSplitA_GR1ArmsAndWaistFourierHands_Env",
386
+ "gr1_unified/PosttrainPnPNovelFromPlacematToTieredshelfSplitA_GR1ArmsAndWaistFourierHands_Env",
387
+ "gr1_unified/PosttrainPnPNovelFromPlateToBowlSplitA_GR1ArmsAndWaistFourierHands_Env",
388
+ "gr1_unified/PosttrainPnPNovelFromPlateToCardboardboxSplitA_GR1ArmsAndWaistFourierHands_Env",
389
+ "gr1_unified/PosttrainPnPNovelFromPlateToPanSplitA_GR1ArmsAndWaistFourierHands_Env",
390
+ "gr1_unified/PosttrainPnPNovelFromPlateToPlateSplitA_GR1ArmsAndWaistFourierHands_Env",
391
+ "gr1_unified/PosttrainPnPNovelFromTrayToCardboardboxSplitA_GR1ArmsAndWaistFourierHands_Env",
392
+ "gr1_unified/PosttrainPnPNovelFromTrayToPlateSplitA_GR1ArmsAndWaistFourierHands_Env",
393
+ "gr1_unified/PosttrainPnPNovelFromTrayToPotSplitA_GR1ArmsAndWaistFourierHands_Env",
394
+ "gr1_unified/PosttrainPnPNovelFromTrayToTieredbasketSplitA_GR1ArmsAndWaistFourierHands_Env",
395
+ "gr1_unified/PosttrainPnPNovelFromTrayToTieredshelfSplitA_GR1ArmsAndWaistFourierHands_Env"
396
+ ],
397
+ "task_suite_name": "robocasa",
398
+ "total_tasks": 24,
399
+ "type": "RobocasaEvalRunner",
400
+ "unnorm_key": "robocasa_gr1_24tasks_30ep"
401
+ },
402
+ "inference_model": {
403
+ "action_horizon": 16,
404
+ "action_in_proj": {
405
+ "input_size": 64,
406
+ "num_domains": 32,
407
+ "output_size": 4096,
408
+ "type": "DomainAwareLinear"
409
+ },
410
+ "action_out_proj": {
411
+ "input_size": 4096,
412
+ "num_domains": 32,
413
+ "output_size": 64,
414
+ "type": "DomainAwareLinear"
415
+ },
416
+ "base_fps": 24.0,
417
+ "enable_fps_modulation": true,
418
+ "enable_vision_loss": true,
419
+ "freeze_non_moe_vlm_backbone": true,
420
+ "freeze_vlm_backbone": false,
421
+ "latent_patch_size": 2,
422
+ "max_action_dim": 64,
423
+ "name_mapping": {
424
+ ".self_attn.k_norm.": ".self_attn.norm_k.",
425
+ ".self_attn.k_norm_moe_gen.": ".self_attn.norm_added_k.",
426
+ ".self_attn.k_proj.": ".self_attn.to_k.",
427
+ ".self_attn.k_proj_moe_gen.": ".self_attn.add_k_proj.",
428
+ ".self_attn.o_proj.": ".self_attn.to_out.",
429
+ ".self_attn.o_proj_moe_gen.": ".self_attn.to_add_out.",
430
+ ".self_attn.q_norm.": ".self_attn.norm_q.",
431
+ ".self_attn.q_norm_moe_gen.": ".self_attn.norm_added_q.",
432
+ ".self_attn.q_proj.": ".self_attn.to_q.",
433
+ ".self_attn.q_proj_moe_gen.": ".self_attn.add_q_proj.",
434
+ ".self_attn.v_proj.": ".self_attn.to_v.",
435
+ ".self_attn.v_proj_moe_gen.": ".self_attn.add_v_proj.",
436
+ "action_in_proj.": "action_proj_in.",
437
+ "action_modality_embed": "action_modality_embed",
438
+ "action_out_proj.": "action_proj_out.",
439
+ "time_embedder.mlp.0.": "time_embedder.linear_1.",
440
+ "time_embedder.mlp.2.": "time_embedder.linear_2.",
441
+ "vision_in_proj.projector.": "proj_in.",
442
+ "vision_out_proj.projector.": "proj_out.",
443
+ "vlm_backbone.lm_head.weight": "lm_head.weight",
444
+ "vlm_backbone.model.language_model.embed_tokens.weight": "embed_tokens.weight",
445
+ "vlm_backbone.model.language_model.layers.": "layers.",
446
+ "vlm_backbone.model.language_model.norm.weight": "norm.weight",
447
+ "vlm_backbone.model.language_model.norm_moe_gen.weight": "norm_moe_gen.weight",
448
+ "vlm_backbone.model.visual.blocks.": "blocks.",
449
+ "vlm_backbone.model.visual.deepstack_merger_list.": "deepstack_merger_list.",
450
+ "vlm_backbone.model.visual.merger.": "merger.",
451
+ "vlm_backbone.model.visual.patch_embed.": "patch_embed.",
452
+ "vlm_backbone.model.visual.pos_embed.": "pos_embed."
453
+ },
454
+ "num_embodiment_domains": 32,
455
+ "ori_action_dim": 29,
456
+ "packed_attention_backend": "flash2",
457
+ "position_embedding_type": "unified_3d_mrope",
458
+ "pretrained_name_or_path": "./checkpoints/Cosmos3-Nano/transformer",
459
+ "rectified_flow_inference_config": {
460
+ "num_steps": 30,
461
+ "num_train_timesteps": 1000,
462
+ "scheduler_type": "unipc",
463
+ "shift": 10.0,
464
+ "use_dynamic_shifting": false,
465
+ "use_karras_sigmas": false
466
+ },
467
+ "rectified_flow_training_config": {
468
+ "action_loss_weight": 10.0,
469
+ "high_sigma_ratio": 0.05,
470
+ "high_sigma_timesteps_max": 1000,
471
+ "high_sigma_timesteps_min": 995,
472
+ "independent_action_schedule": false,
473
+ "normalize_loss_by_active": false,
474
+ "shift": {
475
+ "256": 3,
476
+ "480": 5,
477
+ "720": 10
478
+ },
479
+ "shift_action": null,
480
+ "train_time_action_distribution": "logitnormal",
481
+ "train_time_image_distribution": "logitnormal",
482
+ "train_time_video_distribution": "waver",
483
+ "train_time_weight": "uniform",
484
+ "use_discrete_rf": false,
485
+ "use_dynamic_shift": false,
486
+ "use_high_sigma_strategy": false,
487
+ "use_high_sigma_strategy_action": false,
488
+ "vision_loss_weight": 10.0
489
+ },
490
+ "special_tokens": {
491
+ "end_of_generation": 151653,
492
+ "eos_token_id": 151645,
493
+ "start_of_generation": 151652
494
+ },
495
+ "timestep_scale": 0.001,
496
+ "type": "Cosmos3FlowMatching",
497
+ "unified_3d_mrope_reset_spatial_ids": true,
498
+ "unified_3d_mrope_temporal_modality_margin": 15000,
499
+ "vision_in_proj": {
500
+ "in_dim": 192,
501
+ "out_dim": 4096,
502
+ "type": "LinearProjector"
503
+ },
504
+ "vision_latent_dim": 48,
505
+ "vision_out_proj": {
506
+ "in_dim": 4096,
507
+ "out_dim": 192,
508
+ "type": "LinearProjector"
509
+ },
510
+ "vision_vae": {
511
+ "encode_exact_durations": [
512
+ 17
513
+ ],
514
+ "pretrained_name_or_path": null,
515
+ "type": "Cosmos3Wan22VAE"
516
+ },
517
+ "vlm_backbone": {
518
+ "include_visual": false,
519
+ "skip_init_weights": true,
520
+ "type": "Cosmos3MoTBackbone",
521
+ "vision_encoder_path": "./checkpoints/Cosmos3-Nano/vision_encoder",
522
+ "vlm_config": {
523
+ "image_token_id": 151655,
524
+ "model_type": "qwen3_vl",
525
+ "text_config": {
526
+ "attention_bias": false,
527
+ "attention_dropout": 0.0,
528
+ "bos_token_id": 151643,
529
+ "eos_token_id": 151645,
530
+ "head_dim": 128,
531
+ "hidden_act": "silu",
532
+ "hidden_size": 4096,
533
+ "intermediate_size": 12288,
534
+ "layer_types": [
535
+ "full_attention",
536
+ "full_attention",
537
+ "full_attention",
538
+ "full_attention",
539
+ "full_attention",
540
+ "full_attention",
541
+ "full_attention",
542
+ "full_attention",
543
+ "full_attention",
544
+ "full_attention",
545
+ "full_attention",
546
+ "full_attention",
547
+ "full_attention",
548
+ "full_attention",
549
+ "full_attention",
550
+ "full_attention",
551
+ "full_attention",
552
+ "full_attention",
553
+ "full_attention",
554
+ "full_attention",
555
+ "full_attention",
556
+ "full_attention",
557
+ "full_attention",
558
+ "full_attention",
559
+ "full_attention",
560
+ "full_attention",
561
+ "full_attention",
562
+ "full_attention",
563
+ "full_attention",
564
+ "full_attention",
565
+ "full_attention",
566
+ "full_attention",
567
+ "full_attention",
568
+ "full_attention",
569
+ "full_attention",
570
+ "full_attention"
571
+ ],
572
+ "max_position_embeddings": 262144,
573
+ "model_type": "qwen3_vl_text",
574
+ "num_attention_heads": 32,
575
+ "num_hidden_layers": 36,
576
+ "num_key_value_heads": 8,
577
+ "pad_token_id": 0,
578
+ "rms_norm_eps": 1e-06,
579
+ "rope_scaling": {
580
+ "mrope_interleaved": true,
581
+ "mrope_section": [
582
+ 24,
583
+ 20,
584
+ 20
585
+ ],
586
+ "rope_type": "default"
587
+ },
588
+ "rope_theta": 5000000,
589
+ "tie_word_embeddings": false,
590
+ "vocab_size": 151936
591
+ },
592
+ "tie_word_embeddings": false,
593
+ "video_token_id": 151656,
594
+ "vision_config": {
595
+ "deepstack_visual_indexes": [
596
+ 8,
597
+ 16,
598
+ 24
599
+ ],
600
+ "depth": 27,
601
+ "hidden_act": "gelu_pytorch_tanh",
602
+ "hidden_size": 1152,
603
+ "in_channels": 3,
604
+ "initializer_range": 0.02,
605
+ "intermediate_size": 4304,
606
+ "model_type": "qwen3_vl",
607
+ "num_heads": 16,
608
+ "num_position_embeddings": 2304,
609
+ "out_hidden_size": 4096,
610
+ "patch_size": 16,
611
+ "spatial_merge_size": 2,
612
+ "temporal_patch_size": 2
613
+ },
614
+ "vision_end_token_id": 151653,
615
+ "vision_start_token_id": 151652,
616
+ "vocab_size": 151936
617
+ }
618
+ }
619
+ },
620
+ "model": {
621
+ "action_horizon": 16,
622
+ "action_in_proj": {
623
+ "input_size": 64,
624
+ "num_domains": 32,
625
+ "output_size": 4096,
626
+ "type": "DomainAwareLinear"
627
+ },
628
+ "action_out_proj": {
629
+ "input_size": 4096,
630
+ "num_domains": 32,
631
+ "output_size": 64,
632
+ "type": "DomainAwareLinear"
633
+ },
634
+ "base_fps": 24.0,
635
+ "enable_fps_modulation": true,
636
+ "enable_vision_loss": true,
637
+ "freeze_non_moe_vlm_backbone": true,
638
+ "freeze_vlm_backbone": false,
639
+ "latent_patch_size": 2,
640
+ "max_action_dim": 64,
641
+ "name_mapping": {
642
+ ".self_attn.k_norm.": ".self_attn.norm_k.",
643
+ ".self_attn.k_norm_moe_gen.": ".self_attn.norm_added_k.",
644
+ ".self_attn.k_proj.": ".self_attn.to_k.",
645
+ ".self_attn.k_proj_moe_gen.": ".self_attn.add_k_proj.",
646
+ ".self_attn.o_proj.": ".self_attn.to_out.",
647
+ ".self_attn.o_proj_moe_gen.": ".self_attn.to_add_out.",
648
+ ".self_attn.q_norm.": ".self_attn.norm_q.",
649
+ ".self_attn.q_norm_moe_gen.": ".self_attn.norm_added_q.",
650
+ ".self_attn.q_proj.": ".self_attn.to_q.",
651
+ ".self_attn.q_proj_moe_gen.": ".self_attn.add_q_proj.",
652
+ ".self_attn.v_proj.": ".self_attn.to_v.",
653
+ ".self_attn.v_proj_moe_gen.": ".self_attn.add_v_proj.",
654
+ "action_in_proj.": "action_proj_in.",
655
+ "action_modality_embed": "action_modality_embed",
656
+ "action_out_proj.": "action_proj_out.",
657
+ "time_embedder.mlp.0.": "time_embedder.linear_1.",
658
+ "time_embedder.mlp.2.": "time_embedder.linear_2.",
659
+ "vision_in_proj.projector.": "proj_in.",
660
+ "vision_out_proj.projector.": "proj_out.",
661
+ "vlm_backbone.lm_head.weight": "lm_head.weight",
662
+ "vlm_backbone.model.language_model.embed_tokens.weight": "embed_tokens.weight",
663
+ "vlm_backbone.model.language_model.layers.": "layers.",
664
+ "vlm_backbone.model.language_model.norm.weight": "norm.weight",
665
+ "vlm_backbone.model.language_model.norm_moe_gen.weight": "norm_moe_gen.weight",
666
+ "vlm_backbone.model.visual.blocks.": "blocks.",
667
+ "vlm_backbone.model.visual.deepstack_merger_list.": "deepstack_merger_list.",
668
+ "vlm_backbone.model.visual.merger.": "merger.",
669
+ "vlm_backbone.model.visual.patch_embed.": "patch_embed.",
670
+ "vlm_backbone.model.visual.pos_embed.": "pos_embed."
671
+ },
672
+ "num_embodiment_domains": 32,
673
+ "ori_action_dim": 29,
674
+ "packed_attention_backend": "flash2",
675
+ "position_embedding_type": "unified_3d_mrope",
676
+ "pretrained_name_or_path": "./checkpoints/Cosmos3-Nano/transformer",
677
+ "rectified_flow_inference_config": {
678
+ "num_steps": 30,
679
+ "num_train_timesteps": 1000,
680
+ "scheduler_type": "unipc",
681
+ "shift": 10.0,
682
+ "use_dynamic_shifting": false,
683
+ "use_karras_sigmas": false
684
+ },
685
+ "rectified_flow_training_config": {
686
+ "action_loss_weight": 10.0,
687
+ "high_sigma_ratio": 0.05,
688
+ "high_sigma_timesteps_max": 1000,
689
+ "high_sigma_timesteps_min": 995,
690
+ "independent_action_schedule": false,
691
+ "normalize_loss_by_active": false,
692
+ "shift": {
693
+ "256": 3,
694
+ "480": 5,
695
+ "720": 10
696
+ },
697
+ "shift_action": null,
698
+ "train_time_action_distribution": "logitnormal",
699
+ "train_time_image_distribution": "logitnormal",
700
+ "train_time_video_distribution": "waver",
701
+ "train_time_weight": "uniform",
702
+ "use_discrete_rf": false,
703
+ "use_dynamic_shift": false,
704
+ "use_high_sigma_strategy": false,
705
+ "use_high_sigma_strategy_action": false,
706
+ "vision_loss_weight": 10.0
707
+ },
708
+ "special_tokens": {
709
+ "end_of_generation": 151653,
710
+ "eos_token_id": 151645,
711
+ "start_of_generation": 151652
712
+ },
713
+ "timestep_scale": 0.001,
714
+ "type": "Cosmos3FlowMatching",
715
+ "unified_3d_mrope_reset_spatial_ids": true,
716
+ "unified_3d_mrope_temporal_modality_margin": 15000,
717
+ "vision_in_proj": {
718
+ "in_dim": 192,
719
+ "out_dim": 4096,
720
+ "type": "LinearProjector"
721
+ },
722
+ "vision_latent_dim": 48,
723
+ "vision_out_proj": {
724
+ "in_dim": 4096,
725
+ "out_dim": 192,
726
+ "type": "LinearProjector"
727
+ },
728
+ "vision_vae": {
729
+ "encode_exact_durations": [
730
+ 17
731
+ ],
732
+ "pretrained_name_or_path": "./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
733
+ "type": "Cosmos3Wan22VAE"
734
+ },
735
+ "vlm_backbone": {
736
+ "include_visual": false,
737
+ "skip_init_weights": true,
738
+ "type": "Cosmos3MoTBackbone",
739
+ "vision_encoder_path": "./checkpoints/Cosmos3-Nano/vision_encoder",
740
+ "vlm_config": {
741
+ "image_token_id": 151655,
742
+ "model_type": "qwen3_vl",
743
+ "text_config": {
744
+ "attention_bias": false,
745
+ "attention_dropout": 0.0,
746
+ "bos_token_id": 151643,
747
+ "eos_token_id": 151645,
748
+ "head_dim": 128,
749
+ "hidden_act": "silu",
750
+ "hidden_size": 4096,
751
+ "intermediate_size": 12288,
752
+ "layer_types": [
753
+ "full_attention",
754
+ "full_attention",
755
+ "full_attention",
756
+ "full_attention",
757
+ "full_attention",
758
+ "full_attention",
759
+ "full_attention",
760
+ "full_attention",
761
+ "full_attention",
762
+ "full_attention",
763
+ "full_attention",
764
+ "full_attention",
765
+ "full_attention",
766
+ "full_attention",
767
+ "full_attention",
768
+ "full_attention",
769
+ "full_attention",
770
+ "full_attention",
771
+ "full_attention",
772
+ "full_attention",
773
+ "full_attention",
774
+ "full_attention",
775
+ "full_attention",
776
+ "full_attention",
777
+ "full_attention",
778
+ "full_attention",
779
+ "full_attention",
780
+ "full_attention",
781
+ "full_attention",
782
+ "full_attention",
783
+ "full_attention",
784
+ "full_attention",
785
+ "full_attention",
786
+ "full_attention",
787
+ "full_attention",
788
+ "full_attention"
789
+ ],
790
+ "max_position_embeddings": 262144,
791
+ "model_type": "qwen3_vl_text",
792
+ "num_attention_heads": 32,
793
+ "num_hidden_layers": 36,
794
+ "num_key_value_heads": 8,
795
+ "pad_token_id": 0,
796
+ "rms_norm_eps": 1e-06,
797
+ "rope_scaling": {
798
+ "mrope_interleaved": true,
799
+ "mrope_section": [
800
+ 24,
801
+ 20,
802
+ 20
803
+ ],
804
+ "rope_type": "default"
805
+ },
806
+ "rope_theta": 5000000,
807
+ "tie_word_embeddings": false,
808
+ "vocab_size": 151936
809
+ },
810
+ "tie_word_embeddings": false,
811
+ "video_token_id": 151656,
812
+ "vision_config": {
813
+ "deepstack_visual_indexes": [
814
+ 8,
815
+ 16,
816
+ 24
817
+ ],
818
+ "depth": 27,
819
+ "hidden_act": "gelu_pytorch_tanh",
820
+ "hidden_size": 1152,
821
+ "in_channels": 3,
822
+ "initializer_range": 0.02,
823
+ "intermediate_size": 4304,
824
+ "model_type": "qwen3_vl",
825
+ "num_heads": 16,
826
+ "num_position_embeddings": 2304,
827
+ "out_hidden_size": 4096,
828
+ "patch_size": 16,
829
+ "spatial_merge_size": 2,
830
+ "temporal_patch_size": 2
831
+ },
832
+ "vision_end_token_id": 151653,
833
+ "vision_start_token_id": 151652,
834
+ "vocab_size": 151936
835
+ }
836
+ }
837
+ },
838
+ "runner": {
839
+ "change_key_name": false,
840
+ "collator": {
841
+ "keys": [
842
+ "images",
843
+ "states",
844
+ "actions",
845
+ "action_masks",
846
+ "img_masks",
847
+ "frame_masks",
848
+ "embodiment_ids",
849
+ "raw_action_dim",
850
+ "conditioning_fps",
851
+ "action_fps"
852
+ ],
853
+ "meta_keys": [
854
+ "text_token_ids",
855
+ "sequence_plan",
856
+ "task_description",
857
+ "stats",
858
+ "info",
859
+ "timestamp",
860
+ "viewpoint"
861
+ ],
862
+ "type": "DictCollator"
863
+ },
864
+ "enable_gradient_checkpointing": true,
865
+ "enable_mixed_precision_training": true,
866
+ "grad_accumulation_steps": 8,
867
+ "lr_scheduler": {
868
+ "cycle_length": 100000,
869
+ "type": "linear-warmup+linear-decay",
870
+ "warmup_steps": 500
871
+ },
872
+ "max_grad_norm": 1.0,
873
+ "max_keep_ckpts": 2,
874
+ "max_steps": 10000,
875
+ "metric": {
876
+ "active_trackers": [
877
+ "jsonl",
878
+ "wandb"
879
+ ],
880
+ "grad_accumulation_steps": 8,
881
+ "run_dir": "work_dirs",
882
+ "type": "VLAMetric",
883
+ "window_size": 1
884
+ },
885
+ "mixed_precision_dtype": "bf16",
886
+ "optimizer": {
887
+ "betas": [
888
+ 0.9,
889
+ 0.99
890
+ ],
891
+ "eps": 1e-08,
892
+ "exclude_1d_from_weight_decay": false,
893
+ "fused": true,
894
+ "lr": 5e-05,
895
+ "paramwise_learning_rate": {
896
+ "action_in_proj.": 0.00025,
897
+ "action_modality_embed": 0.00025,
898
+ "action_out_proj.": 0.00025
899
+ },
900
+ "type": "AdamW",
901
+ "weight_decay": 0.05
902
+ },
903
+ "sampler": null,
904
+ "save_iter_interval": 1000,
905
+ "sharding_strategy": "full-shard",
906
+ "tokenizer": {
907
+ "model_max_length": 4096,
908
+ "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
909
+ "padding_side": "right",
910
+ "trust_remote_code": true,
911
+ "type": "PretrainedTokenizer"
912
+ },
913
+ "type": "FSDPTrainRunner"
914
+ },
915
+ "themis": {
916
+ "ros_server": {
917
+ "dataset_section": "eval",
918
+ "denormalize_context": {},
919
+ "denormalize_per_action": true,
920
+ "device": "cuda:0",
921
+ "enable_mixed_precision": true,
922
+ "evaluation_reporting": {
923
+ "report_kind": "robocasa",
924
+ "result_output_dir": "work_dirs/fluxthemis"
925
+ },
926
+ "forward_seed": false,
927
+ "mixed_precision_dtype": "bf16",
928
+ "model_outputs_environment_actions": false,
929
+ "ros_version": 1,
930
+ "workers": {
931
+ "lease_timeout_s": 900.0,
932
+ "request_timeout_s": 120.0,
933
+ "startup_timeout_s": 900.0
934
+ }
935
+ },
936
+ "runner": {
937
+ "environment": {
938
+ "action_order": "fluxvla",
939
+ "deterministic_env": true,
940
+ "prompt_key": "annotation.human.coarse_action",
941
+ "render_key": "video.ego_view_pad_res256_freq20",
942
+ "task_list": [
943
+ "gr1_unified/PnPBottleToCabinetClose_GR1ArmsAndWaistFourierHands_Env",
944
+ "gr1_unified/PnPCanToDrawerClose_GR1ArmsAndWaistFourierHands_Env",
945
+ "gr1_unified/PnPCupToDrawerClose_GR1ArmsAndWaistFourierHands_Env",
946
+ "gr1_unified/PnPMilkToMicrowaveClose_GR1ArmsAndWaistFourierHands_Env",
947
+ "gr1_unified/PnPPotatoToMicrowaveClose_GR1ArmsAndWaistFourierHands_Env",
948
+ "gr1_unified/PnPWineToCabinetClose_GR1ArmsAndWaistFourierHands_Env",
949
+ "gr1_unified/PosttrainPnPNovelFromCuttingboardToBasketSplitA_GR1ArmsAndWaistFourierHands_Env",
950
+ "gr1_unified/PosttrainPnPNovelFromCuttingboardToCardboardboxSplitA_GR1ArmsAndWaistFourierHands_Env",
951
+ "gr1_unified/PosttrainPnPNovelFromCuttingboardToPanSplitA_GR1ArmsAndWaistFourierHands_Env",
952
+ "gr1_unified/PosttrainPnPNovelFromCuttingboardToPotSplitA_GR1ArmsAndWaistFourierHands_Env",
953
+ "gr1_unified/PosttrainPnPNovelFromCuttingboardToTieredbasketSplitA_GR1ArmsAndWaistFourierHands_Env",
954
+ "gr1_unified/PosttrainPnPNovelFromPlacematToBasketSplitA_GR1ArmsAndWaistFourierHands_Env",
955
+ "gr1_unified/PosttrainPnPNovelFromPlacematToBowlSplitA_GR1ArmsAndWaistFourierHands_Env",
956
+ "gr1_unified/PosttrainPnPNovelFromPlacematToPlateSplitA_GR1ArmsAndWaistFourierHands_Env",
957
+ "gr1_unified/PosttrainPnPNovelFromPlacematToTieredshelfSplitA_GR1ArmsAndWaistFourierHands_Env",
958
+ "gr1_unified/PosttrainPnPNovelFromPlateToBowlSplitA_GR1ArmsAndWaistFourierHands_Env",
959
+ "gr1_unified/PosttrainPnPNovelFromPlateToCardboardboxSplitA_GR1ArmsAndWaistFourierHands_Env",
960
+ "gr1_unified/PosttrainPnPNovelFromPlateToPanSplitA_GR1ArmsAndWaistFourierHands_Env",
961
+ "gr1_unified/PosttrainPnPNovelFromPlateToPlateSplitA_GR1ArmsAndWaistFourierHands_Env",
962
+ "gr1_unified/PosttrainPnPNovelFromTrayToCardboardboxSplitA_GR1ArmsAndWaistFourierHands_Env",
963
+ "gr1_unified/PosttrainPnPNovelFromTrayToPlateSplitA_GR1ArmsAndWaistFourierHands_Env",
964
+ "gr1_unified/PosttrainPnPNovelFromTrayToPotSplitA_GR1ArmsAndWaistFourierHands_Env",
965
+ "gr1_unified/PosttrainPnPNovelFromTrayToTieredbasketSplitA_GR1ArmsAndWaistFourierHands_Env",
966
+ "gr1_unified/PosttrainPnPNovelFromTrayToTieredshelfSplitA_GR1ArmsAndWaistFourierHands_Env"
967
+ ],
968
+ "type": "RoboCasaEnvironment"
969
+ },
970
+ "episodes_per_task": 50,
971
+ "evaluator": {
972
+ "type": "SuccessRateEvaluator"
973
+ },
974
+ "execute_horizon": 8,
975
+ "max_episode_steps": 720,
976
+ "model_client": {
977
+ "type": "FluxVLAROSModelClient"
978
+ },
979
+ "parallel_workers": 1,
980
+ "seed": 7,
981
+ "simulator_gpu_ids": null,
982
+ "stop_on_success": true,
983
+ "type": "EvalRunner",
984
+ "work_dir": "work_dirs/fluxthemis"
985
+ },
986
+ "transport": {
987
+ "image_encoding": "rgb8",
988
+ "image_keys": [
989
+ "video.ego_view_bg_crop_pad_res256_freq20"
990
+ ],
991
+ "report_service_name": "/fluxvla/report_evaluation",
992
+ "service_name": "/fluxvla/predict_action",
993
+ "state_keys": [
994
+ "state.left_arm",
995
+ "state.left_hand",
996
+ "state.right_arm",
997
+ "state.right_hand",
998
+ "state.waist"
999
+ ],
1000
+ "timeout_s": 30.0,
1001
+ "unnorm_key": "robocasa_gr1_24tasks_30ep"
1002
+ }
1003
+ },
1004
+ "train_dataloader": {
1005
+ "dataset": {
1006
+ "dataset_statistics_path": "./datasets/robocasa_gr1_24tasks_first30ep/official_groot_gr1_dataset_statistics.json",
1007
+ "datasets": {
1008
+ "action_key": "action",
1009
+ "action_window_size": 16,
1010
+ "data_root_path": [
1011
+ "./datasets/robocasa_lerobot_V2.1/PnPBottleToCabinetClose",
1012
+ "./datasets/robocasa_lerobot_V2.1/PnPCanToDrawerClose",
1013
+ "./datasets/robocasa_lerobot_V2.1/PnPCupToDrawerClose",
1014
+ "./datasets/robocasa_lerobot_V2.1/PnPMilkToMicrowaveClose",
1015
+ "./datasets/robocasa_lerobot_V2.1/PnPPotatoToMicrowaveClose",
1016
+ "./datasets/robocasa_lerobot_V2.1/PnPWineToCabinetClose",
1017
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromCuttingboardToBasketSplitA",
1018
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromCuttingboardToCardboardboxSplitA",
1019
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromCuttingboardToPanSplitA",
1020
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromCuttingboardToPotSplitA",
1021
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromCuttingboardToTieredbasketSplitA",
1022
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlacematToBasketSplitA",
1023
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlacematToBowlSplitA",
1024
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlacematToPlateSplitA",
1025
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlacematToTieredshelfSplitA",
1026
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlateToBowlSplitA",
1027
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlateToCardboardboxSplitA",
1028
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlateToPanSplitA",
1029
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlateToPlateSplitA",
1030
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromTrayToCardboardboxSplitA",
1031
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromTrayToPlateSplitA",
1032
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromTrayToPotSplitA",
1033
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromTrayToTieredbasketSplitA",
1034
+ "./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromTrayToTieredshelfSplitA"
1035
+ ],
1036
+ "frame_window_size": 17,
1037
+ "require_full_window": true,
1038
+ "statistic_name": "robocasa_gr1_24tasks_30ep",
1039
+ "transforms": [
1040
+ {
1041
+ "embodiment_id": 24,
1042
+ "name_mappings": {
1043
+ "actions": [
1044
+ "actions"
1045
+ ],
1046
+ "observation.state": [
1047
+ "states"
1048
+ ]
1049
+ },
1050
+ "parquet_keys": [
1051
+ "observation.state",
1052
+ "timestamp",
1053
+ "actions",
1054
+ "info",
1055
+ "stats",
1056
+ "action_masks"
1057
+ ],
1058
+ "type": "ProcessParquetInputs",
1059
+ "video_keys": [
1060
+ "observation.images.ego_view"
1061
+ ]
1062
+ },
1063
+ {
1064
+ "action_metadata": {
1065
+ "append_viewpoint": false,
1066
+ "conditioning_fps": 20.0,
1067
+ "frame_window_size": 17,
1068
+ "video_height": 256,
1069
+ "video_width": 256,
1070
+ "viewpoint": "ego_view"
1071
+ },
1072
+ "cfg_dropout_rate": 0.1,
1073
+ "format_prompt_as_json": true,
1074
+ "max_len": 512,
1075
+ "tokenizer": {
1076
+ "model_max_length": 4096,
1077
+ "model_path": "./checkpoints/Cosmos3-Nano/text_tokenizer",
1078
+ "padding_side": "right",
1079
+ "trust_remote_code": true,
1080
+ "type": "PretrainedTokenizer"
1081
+ },
1082
+ "type": "ProcessCosmos3Prompt"
1083
+ },
1084
+ {
1085
+ "scale": 0.95,
1086
+ "type": "RandomCropImages"
1087
+ },
1088
+ {
1089
+ "height": 256,
1090
+ "type": "ResizeImages",
1091
+ "width": 256
1092
+ },
1093
+ {
1094
+ "brightness": 0.3,
1095
+ "contrast": 0.4,
1096
+ "hue": 0.08,
1097
+ "saturation": 0.5,
1098
+ "type": "ColorJitterImages"
1099
+ },
1100
+ {
1101
+ "type": "SimpleNormalizeImages"
1102
+ },
1103
+ {
1104
+ "action_dim": 64,
1105
+ "action_key": "action",
1106
+ "action_norm_type": "quantile",
1107
+ "state_dim": 64,
1108
+ "state_key": "action",
1109
+ "state_norm_type": "quantile",
1110
+ "type": "NormalizeStatesAndActions"
1111
+ },
1112
+ {
1113
+ "conditioning_fps": 20.0,
1114
+ "frame_window_size": 17,
1115
+ "mode": "wam",
1116
+ "prepend_state_to_action": false,
1117
+ "raw_action_dim": 29,
1118
+ "type": "BuildCosmos3Sequence"
1119
+ },
1120
+ {
1121
+ "frame_window_size": 17,
1122
+ "num_views": 1,
1123
+ "tile_direction": "horizontal",
1124
+ "type": "PrepareVideo"
1125
+ },
1126
+ {
1127
+ "height": 256,
1128
+ "type": "ResizeAndReflectPad",
1129
+ "width": 256
1130
+ }
1131
+ ],
1132
+ "type": "ParquetDataset",
1133
+ "use_delta": false,
1134
+ "window_start_idx": 0
1135
+ },
1136
+ "name_mappings": {
1137
+ "action": [
1138
+ "action"
1139
+ ],
1140
+ "observation.state": [
1141
+ "proprio"
1142
+ ]
1143
+ },
1144
+ "statistic_keys": [
1145
+ "observation.state",
1146
+ "timestamp",
1147
+ "action"
1148
+ ],
1149
+ "statistic_name": "robocasa_gr1_24tasks_30ep",
1150
+ "type": "DistributedRepeatingDataset"
1151
+ },
1152
+ "per_device_batch_size": 16,
1153
+ "per_device_num_workers": 4
1154
+ }
1155
+ }
cosmos3_nano_robocasa_full_data_full_finetune_bs2048/config.yaml ADDED
@@ -0,0 +1,995 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _ACTION_DIM: 29
2
+ _ACTION_HORIZON: 16
3
+ _ACTION_LR: 0.00025
4
+ _BASE_LR: 5.0e-05
5
+ _CFG_DROPOUT_RATE: 0.1
6
+ _CKPT_ROOT: ./checkpoints
7
+ _CONDITIONING_FPS: 20.0
8
+ _COSMOS3_NANO_CKPT: ./checkpoints/Cosmos3-Nano
9
+ _COSMOS3_NANO_NAME_MAPPING:
10
+ .self_attn.k_norm.: .self_attn.norm_k.
11
+ .self_attn.k_norm_moe_gen.: .self_attn.norm_added_k.
12
+ .self_attn.k_proj.: .self_attn.to_k.
13
+ .self_attn.k_proj_moe_gen.: .self_attn.add_k_proj.
14
+ .self_attn.o_proj.: .self_attn.to_out.
15
+ .self_attn.o_proj_moe_gen.: .self_attn.to_add_out.
16
+ .self_attn.q_norm.: .self_attn.norm_q.
17
+ .self_attn.q_norm_moe_gen.: .self_attn.norm_added_q.
18
+ .self_attn.q_proj.: .self_attn.to_q.
19
+ .self_attn.q_proj_moe_gen.: .self_attn.add_q_proj.
20
+ .self_attn.v_proj.: .self_attn.to_v.
21
+ .self_attn.v_proj_moe_gen.: .self_attn.add_v_proj.
22
+ action_in_proj.: action_proj_in.
23
+ action_modality_embed: action_modality_embed
24
+ action_out_proj.: action_proj_out.
25
+ time_embedder.mlp.0.: time_embedder.linear_1.
26
+ time_embedder.mlp.2.: time_embedder.linear_2.
27
+ vision_in_proj.projector.: proj_in.
28
+ vision_out_proj.projector.: proj_out.
29
+ vlm_backbone.lm_head.weight: lm_head.weight
30
+ vlm_backbone.model.language_model.embed_tokens.weight: embed_tokens.weight
31
+ vlm_backbone.model.language_model.layers.: layers.
32
+ vlm_backbone.model.language_model.norm.weight: norm.weight
33
+ vlm_backbone.model.language_model.norm_moe_gen.weight: norm_moe_gen.weight
34
+ vlm_backbone.model.visual.blocks.: blocks.
35
+ vlm_backbone.model.visual.deepstack_merger_list.: deepstack_merger_list.
36
+ vlm_backbone.model.visual.merger.: merger.
37
+ vlm_backbone.model.visual.patch_embed.: patch_embed.
38
+ vlm_backbone.model.visual.pos_embed.: pos_embed.
39
+ _COSMOS3_NANO_SPECIAL_TOKENS:
40
+ end_of_generation: 151653
41
+ eos_token_id: 151645
42
+ start_of_generation: 151652
43
+ _COSMOS3_NANO_TOKENIZER:
44
+ model_max_length: 4096
45
+ model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
46
+ padding_side: right
47
+ trust_remote_code: true
48
+ type: PretrainedTokenizer
49
+ _COSMOS3_NANO_TRANSFORMER: ./checkpoints/Cosmos3-Nano/transformer
50
+ _COSMOS3_NANO_VISION_ENCODER: ./checkpoints/Cosmos3-Nano/vision_encoder
51
+ _COSMOS3_NANO_VLM_CONFIG:
52
+ image_token_id: 151655
53
+ model_type: qwen3_vl
54
+ text_config:
55
+ attention_bias: false
56
+ attention_dropout: 0.0
57
+ bos_token_id: 151643
58
+ eos_token_id: 151645
59
+ head_dim: 128
60
+ hidden_act: silu
61
+ hidden_size: 4096
62
+ intermediate_size: 12288
63
+ layer_types:
64
+ - full_attention
65
+ - full_attention
66
+ - full_attention
67
+ - full_attention
68
+ - full_attention
69
+ - full_attention
70
+ - full_attention
71
+ - full_attention
72
+ - full_attention
73
+ - full_attention
74
+ - full_attention
75
+ - full_attention
76
+ - full_attention
77
+ - full_attention
78
+ - full_attention
79
+ - full_attention
80
+ - full_attention
81
+ - full_attention
82
+ - full_attention
83
+ - full_attention
84
+ - full_attention
85
+ - full_attention
86
+ - full_attention
87
+ - full_attention
88
+ - full_attention
89
+ - full_attention
90
+ - full_attention
91
+ - full_attention
92
+ - full_attention
93
+ - full_attention
94
+ - full_attention
95
+ - full_attention
96
+ - full_attention
97
+ - full_attention
98
+ - full_attention
99
+ - full_attention
100
+ max_position_embeddings: 262144
101
+ model_type: qwen3_vl_text
102
+ num_attention_heads: 32
103
+ num_hidden_layers: 36
104
+ num_key_value_heads: 8
105
+ pad_token_id: 0
106
+ rms_norm_eps: 1.0e-06
107
+ rope_scaling:
108
+ mrope_interleaved: true
109
+ mrope_section:
110
+ - 24
111
+ - 20
112
+ - 20
113
+ rope_type: default
114
+ rope_theta: 5000000
115
+ tie_word_embeddings: false
116
+ vocab_size: 151936
117
+ tie_word_embeddings: false
118
+ video_token_id: 151656
119
+ vision_config:
120
+ deepstack_visual_indexes:
121
+ - 8
122
+ - 16
123
+ - 24
124
+ depth: 27
125
+ hidden_act: gelu_pytorch_tanh
126
+ hidden_size: 1152
127
+ in_channels: 3
128
+ initializer_range: 0.02
129
+ intermediate_size: 4304
130
+ model_type: qwen3_vl
131
+ num_heads: 16
132
+ num_position_embeddings: 2304
133
+ out_hidden_size: 4096
134
+ patch_size: 16
135
+ spatial_merge_size: 2
136
+ temporal_patch_size: 2
137
+ vision_end_token_id: 151653
138
+ vision_start_token_id: 151652
139
+ vocab_size: 151936
140
+ _FRAME_WINDOW_SIZE: 17
141
+ _GRAD_ACCUMULATION_STEPS: 8
142
+ _IMAGE_HEIGHT: 256
143
+ _IMAGE_WIDTH: 256
144
+ _MAX_ACTION_DIM: 64
145
+ _MAX_STATE_DIM: 64
146
+ _MAX_STEPS: 10000
147
+ _OFFICIAL_GR1_STATS_PATH: ./datasets/robocasa_gr1_24tasks_first30ep/official_groot_gr1_dataset_statistics.json
148
+ _PER_DEVICE_BATCH_SIZE: 16
149
+ _PREPEND_STATE_TO_ACTION: true
150
+ _ROBOCASA_DATA_ROOT: ./datasets/robocasa_lerobot_V2.1
151
+ _ROBOCASA_EMBODIMENT_ID: 24
152
+ _ROBOCASA_ENV_SUFFIX: _GR1ArmsAndWaistFourierHands_Env
153
+ _ROBOCASA_TASK_DIRS:
154
+ - PnPBottleToCabinetClose
155
+ - PnPCanToDrawerClose
156
+ - PnPCupToDrawerClose
157
+ - PnPMilkToMicrowaveClose
158
+ - PnPPotatoToMicrowaveClose
159
+ - PnPWineToCabinetClose
160
+ - PosttrainPnPNovelFromCuttingboardToBasketSplitA
161
+ - PosttrainPnPNovelFromCuttingboardToCardboardboxSplitA
162
+ - PosttrainPnPNovelFromCuttingboardToPanSplitA
163
+ - PosttrainPnPNovelFromCuttingboardToPotSplitA
164
+ - PosttrainPnPNovelFromCuttingboardToTieredbasketSplitA
165
+ - PosttrainPnPNovelFromPlacematToBasketSplitA
166
+ - PosttrainPnPNovelFromPlacematToBowlSplitA
167
+ - PosttrainPnPNovelFromPlacematToPlateSplitA
168
+ - PosttrainPnPNovelFromPlacematToTieredshelfSplitA
169
+ - PosttrainPnPNovelFromPlateToBowlSplitA
170
+ - PosttrainPnPNovelFromPlateToCardboardboxSplitA
171
+ - PosttrainPnPNovelFromPlateToPanSplitA
172
+ - PosttrainPnPNovelFromPlateToPlateSplitA
173
+ - PosttrainPnPNovelFromTrayToCardboardboxSplitA
174
+ - PosttrainPnPNovelFromTrayToPlateSplitA
175
+ - PosttrainPnPNovelFromTrayToPotSplitA
176
+ - PosttrainPnPNovelFromTrayToTieredbasketSplitA
177
+ - PosttrainPnPNovelFromTrayToTieredshelfSplitA
178
+ _ROBOCASA_TASK_PREFIX: gr1_unified
179
+ _SAVE_ITER_INTERVAL: 1000
180
+ _STATISTIC_NAME: robocasa_gr1_24tasks_30ep
181
+ _TRANSFORMS:
182
+ - embodiment_id: 24
183
+ name_mappings:
184
+ actions:
185
+ - actions
186
+ observation.state:
187
+ - states
188
+ parquet_keys:
189
+ - observation.state
190
+ - timestamp
191
+ - actions
192
+ - info
193
+ - stats
194
+ - action_masks
195
+ type: ProcessParquetInputs
196
+ video_keys:
197
+ - observation.images.ego_view
198
+ - action_metadata:
199
+ append_viewpoint: false
200
+ conditioning_fps: 20.0
201
+ frame_window_size: 17
202
+ video_height: 256
203
+ video_width: 256
204
+ viewpoint: ego_view
205
+ cfg_dropout_rate: 0.1
206
+ format_prompt_as_json: true
207
+ max_len: 512
208
+ tokenizer:
209
+ model_max_length: 4096
210
+ model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
211
+ padding_side: right
212
+ trust_remote_code: true
213
+ type: PretrainedTokenizer
214
+ type: ProcessCosmos3Prompt
215
+ - scale: 0.95
216
+ type: RandomCropImages
217
+ - height: 256
218
+ type: ResizeImages
219
+ width: 256
220
+ - brightness: 0.3
221
+ contrast: 0.4
222
+ hue: 0.08
223
+ saturation: 0.5
224
+ type: ColorJitterImages
225
+ - type: SimpleNormalizeImages
226
+ - action_dim: 64
227
+ action_key: action
228
+ action_norm_type: quantile
229
+ state_dim: 64
230
+ state_key: action
231
+ state_norm_type: quantile
232
+ type: NormalizeStatesAndActions
233
+ - conditioning_fps: 20.0
234
+ frame_window_size: 17
235
+ mode: wam
236
+ prepend_state_to_action: true
237
+ raw_action_dim: 29
238
+ type: BuildCosmos3Sequence
239
+ - frame_window_size: 17
240
+ num_views: 1
241
+ tile_direction: horizontal
242
+ type: PrepareVideo
243
+ - height: 256
244
+ type: ResizeAndReflectPad
245
+ width: 256
246
+ _VIDEO_HEIGHT: 256
247
+ _VIDEO_WIDTH: 256
248
+ _WAN22_VAE_PATH: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
249
+ eval:
250
+ action_order: fluxvla
251
+ benchmark: robocasa
252
+ dataset:
253
+ extra_tensor_keys:
254
+ - conditioning_fps
255
+ - prepend_state_to_action
256
+ transforms:
257
+ - center_crop_scale: 0.95
258
+ embodiment_id: 24
259
+ img_key: video.ego_view_bg_crop_pad_res256_freq20
260
+ normalize: true
261
+ resize_size: 256
262
+ type: ProcessRobocasaEvalInputs
263
+ value_range: tanh
264
+ - conditioning_fps: 20.0
265
+ prepend_state_to_action: false
266
+ type: SetCosmos3ActionMetadata
267
+ - action_key: null
268
+ state_dim: 64
269
+ state_key: action
270
+ state_norm_type: quantile
271
+ type: NormalizeStatesAndActions
272
+ - action_metadata:
273
+ append_viewpoint: false
274
+ conditioning_fps: 20.0
275
+ frame_window_size: 17
276
+ video_height: 256
277
+ video_width: 256
278
+ viewpoint: ego_view
279
+ cfg_dropout_rate: 0.0
280
+ format_prompt_as_json: true
281
+ max_len: 512
282
+ output_attention_mask_key: lang_masks
283
+ output_key: lang_tokens
284
+ tokenizer:
285
+ model_max_length: 4096
286
+ model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
287
+ padding_side: right
288
+ trust_remote_code: true
289
+ type: PretrainedTokenizer
290
+ type: ProcessCosmos3Prompt
291
+ - frame_window_size: 1
292
+ num_views: 1
293
+ tile_direction: horizontal
294
+ type: PrepareVideo
295
+ - height: 256
296
+ type: ResizeAndReflectPad
297
+ width: 256
298
+ type: RobocasaEvalDataset
299
+ unnorm_key: robocasa_gr1_24tasks_30ep
300
+ denormalize_action:
301
+ action_dim: 29
302
+ clip_actions: false
303
+ norm_type: quantile
304
+ stats_order: native
305
+ type: DenormalizeRobocasaAction
306
+ enable_mixed_precision_training: true
307
+ eval_chunk_size: 8
308
+ max_episode_steps: 720
309
+ mixed_precision_dtype: bf16
310
+ model_family: cosmos3
311
+ num_trials_per_task: 50
312
+ seed: 7
313
+ task_list:
314
+ - gr1_unified/PnPBottleToCabinetClose_GR1ArmsAndWaistFourierHands_Env
315
+ - gr1_unified/PnPCanToDrawerClose_GR1ArmsAndWaistFourierHands_Env
316
+ - gr1_unified/PnPCupToDrawerClose_GR1ArmsAndWaistFourierHands_Env
317
+ - gr1_unified/PnPMilkToMicrowaveClose_GR1ArmsAndWaistFourierHands_Env
318
+ - gr1_unified/PnPPotatoToMicrowaveClose_GR1ArmsAndWaistFourierHands_Env
319
+ - gr1_unified/PnPWineToCabinetClose_GR1ArmsAndWaistFourierHands_Env
320
+ - gr1_unified/PosttrainPnPNovelFromCuttingboardToBasketSplitA_GR1ArmsAndWaistFourierHands_Env
321
+ - gr1_unified/PosttrainPnPNovelFromCuttingboardToCardboardboxSplitA_GR1ArmsAndWaistFourierHands_Env
322
+ - gr1_unified/PosttrainPnPNovelFromCuttingboardToPanSplitA_GR1ArmsAndWaistFourierHands_Env
323
+ - gr1_unified/PosttrainPnPNovelFromCuttingboardToPotSplitA_GR1ArmsAndWaistFourierHands_Env
324
+ - gr1_unified/PosttrainPnPNovelFromCuttingboardToTieredbasketSplitA_GR1ArmsAndWaistFourierHands_Env
325
+ - gr1_unified/PosttrainPnPNovelFromPlacematToBasketSplitA_GR1ArmsAndWaistFourierHands_Env
326
+ - gr1_unified/PosttrainPnPNovelFromPlacematToBowlSplitA_GR1ArmsAndWaistFourierHands_Env
327
+ - gr1_unified/PosttrainPnPNovelFromPlacematToPlateSplitA_GR1ArmsAndWaistFourierHands_Env
328
+ - gr1_unified/PosttrainPnPNovelFromPlacematToTieredshelfSplitA_GR1ArmsAndWaistFourierHands_Env
329
+ - gr1_unified/PosttrainPnPNovelFromPlateToBowlSplitA_GR1ArmsAndWaistFourierHands_Env
330
+ - gr1_unified/PosttrainPnPNovelFromPlateToCardboardboxSplitA_GR1ArmsAndWaistFourierHands_Env
331
+ - gr1_unified/PosttrainPnPNovelFromPlateToPanSplitA_GR1ArmsAndWaistFourierHands_Env
332
+ - gr1_unified/PosttrainPnPNovelFromPlateToPlateSplitA_GR1ArmsAndWaistFourierHands_Env
333
+ - gr1_unified/PosttrainPnPNovelFromTrayToCardboardboxSplitA_GR1ArmsAndWaistFourierHands_Env
334
+ - gr1_unified/PosttrainPnPNovelFromTrayToPlateSplitA_GR1ArmsAndWaistFourierHands_Env
335
+ - gr1_unified/PosttrainPnPNovelFromTrayToPotSplitA_GR1ArmsAndWaistFourierHands_Env
336
+ - gr1_unified/PosttrainPnPNovelFromTrayToTieredbasketSplitA_GR1ArmsAndWaistFourierHands_Env
337
+ - gr1_unified/PosttrainPnPNovelFromTrayToTieredshelfSplitA_GR1ArmsAndWaistFourierHands_Env
338
+ task_suite_name: robocasa
339
+ total_tasks: 24
340
+ type: RobocasaEvalRunner
341
+ unnorm_key: robocasa_gr1_24tasks_30ep
342
+ inference_model:
343
+ action_horizon: 16
344
+ action_in_proj:
345
+ input_size: 64
346
+ num_domains: 32
347
+ output_size: 4096
348
+ type: DomainAwareLinear
349
+ action_out_proj:
350
+ input_size: 4096
351
+ num_domains: 32
352
+ output_size: 64
353
+ type: DomainAwareLinear
354
+ base_fps: 24.0
355
+ enable_fps_modulation: true
356
+ enable_vision_loss: true
357
+ freeze_non_moe_vlm_backbone: true
358
+ freeze_vlm_backbone: false
359
+ latent_patch_size: 2
360
+ max_action_dim: 64
361
+ name_mapping:
362
+ .self_attn.k_norm.: .self_attn.norm_k.
363
+ .self_attn.k_norm_moe_gen.: .self_attn.norm_added_k.
364
+ .self_attn.k_proj.: .self_attn.to_k.
365
+ .self_attn.k_proj_moe_gen.: .self_attn.add_k_proj.
366
+ .self_attn.o_proj.: .self_attn.to_out.
367
+ .self_attn.o_proj_moe_gen.: .self_attn.to_add_out.
368
+ .self_attn.q_norm.: .self_attn.norm_q.
369
+ .self_attn.q_norm_moe_gen.: .self_attn.norm_added_q.
370
+ .self_attn.q_proj.: .self_attn.to_q.
371
+ .self_attn.q_proj_moe_gen.: .self_attn.add_q_proj.
372
+ .self_attn.v_proj.: .self_attn.to_v.
373
+ .self_attn.v_proj_moe_gen.: .self_attn.add_v_proj.
374
+ action_in_proj.: action_proj_in.
375
+ action_modality_embed: action_modality_embed
376
+ action_out_proj.: action_proj_out.
377
+ time_embedder.mlp.0.: time_embedder.linear_1.
378
+ time_embedder.mlp.2.: time_embedder.linear_2.
379
+ vision_in_proj.projector.: proj_in.
380
+ vision_out_proj.projector.: proj_out.
381
+ vlm_backbone.lm_head.weight: lm_head.weight
382
+ vlm_backbone.model.language_model.embed_tokens.weight: embed_tokens.weight
383
+ vlm_backbone.model.language_model.layers.: layers.
384
+ vlm_backbone.model.language_model.norm.weight: norm.weight
385
+ vlm_backbone.model.language_model.norm_moe_gen.weight: norm_moe_gen.weight
386
+ vlm_backbone.model.visual.blocks.: blocks.
387
+ vlm_backbone.model.visual.deepstack_merger_list.: deepstack_merger_list.
388
+ vlm_backbone.model.visual.merger.: merger.
389
+ vlm_backbone.model.visual.patch_embed.: patch_embed.
390
+ vlm_backbone.model.visual.pos_embed.: pos_embed.
391
+ num_embodiment_domains: 32
392
+ ori_action_dim: 29
393
+ packed_attention_backend: flash2
394
+ position_embedding_type: unified_3d_mrope
395
+ pretrained_name_or_path: ./checkpoints/Cosmos3-Nano/transformer
396
+ rectified_flow_inference_config:
397
+ num_steps: 30
398
+ num_train_timesteps: 1000
399
+ scheduler_type: unipc
400
+ shift: 10.0
401
+ use_dynamic_shifting: false
402
+ use_karras_sigmas: false
403
+ rectified_flow_training_config:
404
+ action_loss_weight: 10.0
405
+ high_sigma_ratio: 0.05
406
+ high_sigma_timesteps_max: 1000
407
+ high_sigma_timesteps_min: 995
408
+ independent_action_schedule: false
409
+ normalize_loss_by_active: false
410
+ shift:
411
+ '256': 3
412
+ '480': 5
413
+ '720': 10
414
+ shift_action: null
415
+ train_time_action_distribution: logitnormal
416
+ train_time_image_distribution: logitnormal
417
+ train_time_video_distribution: waver
418
+ train_time_weight: uniform
419
+ use_discrete_rf: false
420
+ use_dynamic_shift: false
421
+ use_high_sigma_strategy: false
422
+ use_high_sigma_strategy_action: false
423
+ vision_loss_weight: 10.0
424
+ special_tokens:
425
+ end_of_generation: 151653
426
+ eos_token_id: 151645
427
+ start_of_generation: 151652
428
+ timestep_scale: 0.001
429
+ type: Cosmos3FlowMatching
430
+ unified_3d_mrope_reset_spatial_ids: true
431
+ unified_3d_mrope_temporal_modality_margin: 15000
432
+ vision_in_proj:
433
+ in_dim: 192
434
+ out_dim: 4096
435
+ type: LinearProjector
436
+ vision_latent_dim: 48
437
+ vision_out_proj:
438
+ in_dim: 4096
439
+ out_dim: 192
440
+ type: LinearProjector
441
+ vision_vae:
442
+ encode_exact_durations:
443
+ - 17
444
+ pretrained_name_or_path: null
445
+ type: Cosmos3Wan22VAE
446
+ vlm_backbone:
447
+ include_visual: false
448
+ skip_init_weights: true
449
+ type: Cosmos3MoTBackbone
450
+ vision_encoder_path: ./checkpoints/Cosmos3-Nano/vision_encoder
451
+ vlm_config:
452
+ image_token_id: 151655
453
+ model_type: qwen3_vl
454
+ text_config:
455
+ attention_bias: false
456
+ attention_dropout: 0.0
457
+ bos_token_id: 151643
458
+ eos_token_id: 151645
459
+ head_dim: 128
460
+ hidden_act: silu
461
+ hidden_size: 4096
462
+ intermediate_size: 12288
463
+ layer_types:
464
+ - full_attention
465
+ - full_attention
466
+ - full_attention
467
+ - full_attention
468
+ - full_attention
469
+ - full_attention
470
+ - full_attention
471
+ - full_attention
472
+ - full_attention
473
+ - full_attention
474
+ - full_attention
475
+ - full_attention
476
+ - full_attention
477
+ - full_attention
478
+ - full_attention
479
+ - full_attention
480
+ - full_attention
481
+ - full_attention
482
+ - full_attention
483
+ - full_attention
484
+ - full_attention
485
+ - full_attention
486
+ - full_attention
487
+ - full_attention
488
+ - full_attention
489
+ - full_attention
490
+ - full_attention
491
+ - full_attention
492
+ - full_attention
493
+ - full_attention
494
+ - full_attention
495
+ - full_attention
496
+ - full_attention
497
+ - full_attention
498
+ - full_attention
499
+ - full_attention
500
+ max_position_embeddings: 262144
501
+ model_type: qwen3_vl_text
502
+ num_attention_heads: 32
503
+ num_hidden_layers: 36
504
+ num_key_value_heads: 8
505
+ pad_token_id: 0
506
+ rms_norm_eps: 1.0e-06
507
+ rope_scaling:
508
+ mrope_interleaved: true
509
+ mrope_section:
510
+ - 24
511
+ - 20
512
+ - 20
513
+ rope_type: default
514
+ rope_theta: 5000000
515
+ tie_word_embeddings: false
516
+ vocab_size: 151936
517
+ tie_word_embeddings: false
518
+ video_token_id: 151656
519
+ vision_config:
520
+ deepstack_visual_indexes:
521
+ - 8
522
+ - 16
523
+ - 24
524
+ depth: 27
525
+ hidden_act: gelu_pytorch_tanh
526
+ hidden_size: 1152
527
+ in_channels: 3
528
+ initializer_range: 0.02
529
+ intermediate_size: 4304
530
+ model_type: qwen3_vl
531
+ num_heads: 16
532
+ num_position_embeddings: 2304
533
+ out_hidden_size: 4096
534
+ patch_size: 16
535
+ spatial_merge_size: 2
536
+ temporal_patch_size: 2
537
+ vision_end_token_id: 151653
538
+ vision_start_token_id: 151652
539
+ vocab_size: 151936
540
+ model:
541
+ action_horizon: 16
542
+ action_in_proj:
543
+ input_size: 64
544
+ num_domains: 32
545
+ output_size: 4096
546
+ type: DomainAwareLinear
547
+ action_out_proj:
548
+ input_size: 4096
549
+ num_domains: 32
550
+ output_size: 64
551
+ type: DomainAwareLinear
552
+ base_fps: 24.0
553
+ enable_fps_modulation: true
554
+ enable_vision_loss: true
555
+ freeze_non_moe_vlm_backbone: true
556
+ freeze_vlm_backbone: false
557
+ latent_patch_size: 2
558
+ max_action_dim: 64
559
+ name_mapping:
560
+ .self_attn.k_norm.: .self_attn.norm_k.
561
+ .self_attn.k_norm_moe_gen.: .self_attn.norm_added_k.
562
+ .self_attn.k_proj.: .self_attn.to_k.
563
+ .self_attn.k_proj_moe_gen.: .self_attn.add_k_proj.
564
+ .self_attn.o_proj.: .self_attn.to_out.
565
+ .self_attn.o_proj_moe_gen.: .self_attn.to_add_out.
566
+ .self_attn.q_norm.: .self_attn.norm_q.
567
+ .self_attn.q_norm_moe_gen.: .self_attn.norm_added_q.
568
+ .self_attn.q_proj.: .self_attn.to_q.
569
+ .self_attn.q_proj_moe_gen.: .self_attn.add_q_proj.
570
+ .self_attn.v_proj.: .self_attn.to_v.
571
+ .self_attn.v_proj_moe_gen.: .self_attn.add_v_proj.
572
+ action_in_proj.: action_proj_in.
573
+ action_modality_embed: action_modality_embed
574
+ action_out_proj.: action_proj_out.
575
+ time_embedder.mlp.0.: time_embedder.linear_1.
576
+ time_embedder.mlp.2.: time_embedder.linear_2.
577
+ vision_in_proj.projector.: proj_in.
578
+ vision_out_proj.projector.: proj_out.
579
+ vlm_backbone.lm_head.weight: lm_head.weight
580
+ vlm_backbone.model.language_model.embed_tokens.weight: embed_tokens.weight
581
+ vlm_backbone.model.language_model.layers.: layers.
582
+ vlm_backbone.model.language_model.norm.weight: norm.weight
583
+ vlm_backbone.model.language_model.norm_moe_gen.weight: norm_moe_gen.weight
584
+ vlm_backbone.model.visual.blocks.: blocks.
585
+ vlm_backbone.model.visual.deepstack_merger_list.: deepstack_merger_list.
586
+ vlm_backbone.model.visual.merger.: merger.
587
+ vlm_backbone.model.visual.patch_embed.: patch_embed.
588
+ vlm_backbone.model.visual.pos_embed.: pos_embed.
589
+ num_embodiment_domains: 32
590
+ ori_action_dim: 29
591
+ packed_attention_backend: flash2
592
+ position_embedding_type: unified_3d_mrope
593
+ pretrained_name_or_path: ./checkpoints/Cosmos3-Nano/transformer
594
+ rectified_flow_inference_config:
595
+ num_steps: 30
596
+ num_train_timesteps: 1000
597
+ scheduler_type: unipc
598
+ shift: 10.0
599
+ use_dynamic_shifting: false
600
+ use_karras_sigmas: false
601
+ rectified_flow_training_config:
602
+ action_loss_weight: 10.0
603
+ high_sigma_ratio: 0.05
604
+ high_sigma_timesteps_max: 1000
605
+ high_sigma_timesteps_min: 995
606
+ independent_action_schedule: false
607
+ normalize_loss_by_active: false
608
+ shift:
609
+ '256': 3
610
+ '480': 5
611
+ '720': 10
612
+ shift_action: null
613
+ train_time_action_distribution: logitnormal
614
+ train_time_image_distribution: logitnormal
615
+ train_time_video_distribution: waver
616
+ train_time_weight: uniform
617
+ use_discrete_rf: false
618
+ use_dynamic_shift: false
619
+ use_high_sigma_strategy: false
620
+ use_high_sigma_strategy_action: false
621
+ vision_loss_weight: 10.0
622
+ special_tokens:
623
+ end_of_generation: 151653
624
+ eos_token_id: 151645
625
+ start_of_generation: 151652
626
+ timestep_scale: 0.001
627
+ type: Cosmos3FlowMatching
628
+ unified_3d_mrope_reset_spatial_ids: true
629
+ unified_3d_mrope_temporal_modality_margin: 15000
630
+ vision_in_proj:
631
+ in_dim: 192
632
+ out_dim: 4096
633
+ type: LinearProjector
634
+ vision_latent_dim: 48
635
+ vision_out_proj:
636
+ in_dim: 4096
637
+ out_dim: 192
638
+ type: LinearProjector
639
+ vision_vae:
640
+ encode_exact_durations:
641
+ - 17
642
+ pretrained_name_or_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
643
+ type: Cosmos3Wan22VAE
644
+ vlm_backbone:
645
+ include_visual: false
646
+ skip_init_weights: true
647
+ type: Cosmos3MoTBackbone
648
+ vision_encoder_path: ./checkpoints/Cosmos3-Nano/vision_encoder
649
+ vlm_config:
650
+ image_token_id: 151655
651
+ model_type: qwen3_vl
652
+ text_config:
653
+ attention_bias: false
654
+ attention_dropout: 0.0
655
+ bos_token_id: 151643
656
+ eos_token_id: 151645
657
+ head_dim: 128
658
+ hidden_act: silu
659
+ hidden_size: 4096
660
+ intermediate_size: 12288
661
+ layer_types:
662
+ - full_attention
663
+ - full_attention
664
+ - full_attention
665
+ - full_attention
666
+ - full_attention
667
+ - full_attention
668
+ - full_attention
669
+ - full_attention
670
+ - full_attention
671
+ - full_attention
672
+ - full_attention
673
+ - full_attention
674
+ - full_attention
675
+ - full_attention
676
+ - full_attention
677
+ - full_attention
678
+ - full_attention
679
+ - full_attention
680
+ - full_attention
681
+ - full_attention
682
+ - full_attention
683
+ - full_attention
684
+ - full_attention
685
+ - full_attention
686
+ - full_attention
687
+ - full_attention
688
+ - full_attention
689
+ - full_attention
690
+ - full_attention
691
+ - full_attention
692
+ - full_attention
693
+ - full_attention
694
+ - full_attention
695
+ - full_attention
696
+ - full_attention
697
+ - full_attention
698
+ max_position_embeddings: 262144
699
+ model_type: qwen3_vl_text
700
+ num_attention_heads: 32
701
+ num_hidden_layers: 36
702
+ num_key_value_heads: 8
703
+ pad_token_id: 0
704
+ rms_norm_eps: 1.0e-06
705
+ rope_scaling:
706
+ mrope_interleaved: true
707
+ mrope_section:
708
+ - 24
709
+ - 20
710
+ - 20
711
+ rope_type: default
712
+ rope_theta: 5000000
713
+ tie_word_embeddings: false
714
+ vocab_size: 151936
715
+ tie_word_embeddings: false
716
+ video_token_id: 151656
717
+ vision_config:
718
+ deepstack_visual_indexes:
719
+ - 8
720
+ - 16
721
+ - 24
722
+ depth: 27
723
+ hidden_act: gelu_pytorch_tanh
724
+ hidden_size: 1152
725
+ in_channels: 3
726
+ initializer_range: 0.02
727
+ intermediate_size: 4304
728
+ model_type: qwen3_vl
729
+ num_heads: 16
730
+ num_position_embeddings: 2304
731
+ out_hidden_size: 4096
732
+ patch_size: 16
733
+ spatial_merge_size: 2
734
+ temporal_patch_size: 2
735
+ vision_end_token_id: 151653
736
+ vision_start_token_id: 151652
737
+ vocab_size: 151936
738
+ runner:
739
+ change_key_name: false
740
+ collator:
741
+ keys:
742
+ - images
743
+ - states
744
+ - actions
745
+ - action_masks
746
+ - img_masks
747
+ - frame_masks
748
+ - embodiment_ids
749
+ - raw_action_dim
750
+ - conditioning_fps
751
+ - action_fps
752
+ meta_keys:
753
+ - text_token_ids
754
+ - sequence_plan
755
+ - task_description
756
+ - stats
757
+ - info
758
+ - timestamp
759
+ - viewpoint
760
+ type: DictCollator
761
+ enable_gradient_checkpointing: true
762
+ enable_mixed_precision_training: true
763
+ grad_accumulation_steps: 8
764
+ lr_scheduler:
765
+ cycle_length: 100000
766
+ type: linear-warmup+linear-decay
767
+ warmup_steps: 500
768
+ max_grad_norm: 1.0
769
+ max_keep_ckpts: 2
770
+ max_steps: 10000
771
+ metric:
772
+ active_trackers:
773
+ - jsonl
774
+ - wandb
775
+ grad_accumulation_steps: 8
776
+ run_dir: work_dirs
777
+ type: VLAMetric
778
+ window_size: 1
779
+ mixed_precision_dtype: bf16
780
+ optimizer:
781
+ betas:
782
+ - 0.9
783
+ - 0.99
784
+ eps: 1.0e-08
785
+ exclude_1d_from_weight_decay: false
786
+ fused: true
787
+ lr: 5.0e-05
788
+ paramwise_learning_rate:
789
+ action_in_proj.: 0.00025
790
+ action_modality_embed: 0.00025
791
+ action_out_proj.: 0.00025
792
+ type: AdamW
793
+ weight_decay: 0.05
794
+ sampler: null
795
+ save_iter_interval: 1000
796
+ sharding_strategy: full-shard
797
+ tokenizer:
798
+ model_max_length: 4096
799
+ model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
800
+ padding_side: right
801
+ trust_remote_code: true
802
+ type: PretrainedTokenizer
803
+ type: FSDPTrainRunner
804
+ themis:
805
+ ros_server:
806
+ dataset_section: eval
807
+ denormalize_context: {}
808
+ denormalize_per_action: true
809
+ device: cuda:0
810
+ enable_mixed_precision: true
811
+ evaluation_reporting:
812
+ report_kind: robocasa
813
+ result_output_dir: work_dirs/fluxthemis
814
+ forward_seed: false
815
+ mixed_precision_dtype: bf16
816
+ model_outputs_environment_actions: false
817
+ ros_version: 1
818
+ workers:
819
+ lease_timeout_s: 900.0
820
+ request_timeout_s: 120.0
821
+ startup_timeout_s: 900.0
822
+ runner:
823
+ environment:
824
+ action_order: fluxvla
825
+ deterministic_env: true
826
+ prompt_key: annotation.human.coarse_action
827
+ render_key: video.ego_view_pad_res256_freq20
828
+ task_list:
829
+ - gr1_unified/PnPBottleToCabinetClose_GR1ArmsAndWaistFourierHands_Env
830
+ - gr1_unified/PnPCanToDrawerClose_GR1ArmsAndWaistFourierHands_Env
831
+ - gr1_unified/PnPCupToDrawerClose_GR1ArmsAndWaistFourierHands_Env
832
+ - gr1_unified/PnPMilkToMicrowaveClose_GR1ArmsAndWaistFourierHands_Env
833
+ - gr1_unified/PnPPotatoToMicrowaveClose_GR1ArmsAndWaistFourierHands_Env
834
+ - gr1_unified/PnPWineToCabinetClose_GR1ArmsAndWaistFourierHands_Env
835
+ - gr1_unified/PosttrainPnPNovelFromCuttingboardToBasketSplitA_GR1ArmsAndWaistFourierHands_Env
836
+ - gr1_unified/PosttrainPnPNovelFromCuttingboardToCardboardboxSplitA_GR1ArmsAndWaistFourierHands_Env
837
+ - gr1_unified/PosttrainPnPNovelFromCuttingboardToPanSplitA_GR1ArmsAndWaistFourierHands_Env
838
+ - gr1_unified/PosttrainPnPNovelFromCuttingboardToPotSplitA_GR1ArmsAndWaistFourierHands_Env
839
+ - gr1_unified/PosttrainPnPNovelFromCuttingboardToTieredbasketSplitA_GR1ArmsAndWaistFourierHands_Env
840
+ - gr1_unified/PosttrainPnPNovelFromPlacematToBasketSplitA_GR1ArmsAndWaistFourierHands_Env
841
+ - gr1_unified/PosttrainPnPNovelFromPlacematToBowlSplitA_GR1ArmsAndWaistFourierHands_Env
842
+ - gr1_unified/PosttrainPnPNovelFromPlacematToPlateSplitA_GR1ArmsAndWaistFourierHands_Env
843
+ - gr1_unified/PosttrainPnPNovelFromPlacematToTieredshelfSplitA_GR1ArmsAndWaistFourierHands_Env
844
+ - gr1_unified/PosttrainPnPNovelFromPlateToBowlSplitA_GR1ArmsAndWaistFourierHands_Env
845
+ - gr1_unified/PosttrainPnPNovelFromPlateToCardboardboxSplitA_GR1ArmsAndWaistFourierHands_Env
846
+ - gr1_unified/PosttrainPnPNovelFromPlateToPanSplitA_GR1ArmsAndWaistFourierHands_Env
847
+ - gr1_unified/PosttrainPnPNovelFromPlateToPlateSplitA_GR1ArmsAndWaistFourierHands_Env
848
+ - gr1_unified/PosttrainPnPNovelFromTrayToCardboardboxSplitA_GR1ArmsAndWaistFourierHands_Env
849
+ - gr1_unified/PosttrainPnPNovelFromTrayToPlateSplitA_GR1ArmsAndWaistFourierHands_Env
850
+ - gr1_unified/PosttrainPnPNovelFromTrayToPotSplitA_GR1ArmsAndWaistFourierHands_Env
851
+ - gr1_unified/PosttrainPnPNovelFromTrayToTieredbasketSplitA_GR1ArmsAndWaistFourierHands_Env
852
+ - gr1_unified/PosttrainPnPNovelFromTrayToTieredshelfSplitA_GR1ArmsAndWaistFourierHands_Env
853
+ type: RoboCasaEnvironment
854
+ episodes_per_task: 50
855
+ evaluator:
856
+ type: SuccessRateEvaluator
857
+ execute_horizon: 8
858
+ max_episode_steps: 720
859
+ model_client:
860
+ type: FluxVLAROSModelClient
861
+ parallel_workers: 1
862
+ seed: 7
863
+ simulator_gpu_ids: null
864
+ stop_on_success: true
865
+ type: EvalRunner
866
+ work_dir: work_dirs/fluxthemis
867
+ transport:
868
+ image_encoding: rgb8
869
+ image_keys:
870
+ - video.ego_view_bg_crop_pad_res256_freq20
871
+ report_service_name: /fluxvla/report_evaluation
872
+ service_name: /fluxvla/predict_action
873
+ state_keys:
874
+ - state.left_arm
875
+ - state.left_hand
876
+ - state.right_arm
877
+ - state.right_hand
878
+ - state.waist
879
+ timeout_s: 30.0
880
+ unnorm_key: robocasa_gr1_24tasks_30ep
881
+ train_dataloader:
882
+ dataset:
883
+ dataset_statistics_path: ./datasets/robocasa_gr1_24tasks_first30ep/official_groot_gr1_dataset_statistics.json
884
+ datasets:
885
+ action_key: action
886
+ action_window_size: 16
887
+ data_root_path:
888
+ - ./datasets/robocasa_lerobot_V2.1/PnPBottleToCabinetClose
889
+ - ./datasets/robocasa_lerobot_V2.1/PnPCanToDrawerClose
890
+ - ./datasets/robocasa_lerobot_V2.1/PnPCupToDrawerClose
891
+ - ./datasets/robocasa_lerobot_V2.1/PnPMilkToMicrowaveClose
892
+ - ./datasets/robocasa_lerobot_V2.1/PnPPotatoToMicrowaveClose
893
+ - ./datasets/robocasa_lerobot_V2.1/PnPWineToCabinetClose
894
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromCuttingboardToBasketSplitA
895
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromCuttingboardToCardboardboxSplitA
896
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromCuttingboardToPanSplitA
897
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromCuttingboardToPotSplitA
898
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromCuttingboardToTieredbasketSplitA
899
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlacematToBasketSplitA
900
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlacematToBowlSplitA
901
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlacematToPlateSplitA
902
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlacematToTieredshelfSplitA
903
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlateToBowlSplitA
904
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlateToCardboardboxSplitA
905
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlateToPanSplitA
906
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromPlateToPlateSplitA
907
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromTrayToCardboardboxSplitA
908
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromTrayToPlateSplitA
909
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromTrayToPotSplitA
910
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromTrayToTieredbasketSplitA
911
+ - ./datasets/robocasa_lerobot_V2.1/PosttrainPnPNovelFromTrayToTieredshelfSplitA
912
+ frame_window_size: 17
913
+ require_full_window: true
914
+ statistic_name: robocasa_gr1_24tasks_30ep
915
+ transforms:
916
+ - embodiment_id: 24
917
+ name_mappings:
918
+ actions:
919
+ - actions
920
+ observation.state:
921
+ - states
922
+ parquet_keys:
923
+ - observation.state
924
+ - timestamp
925
+ - actions
926
+ - info
927
+ - stats
928
+ - action_masks
929
+ type: ProcessParquetInputs
930
+ video_keys:
931
+ - observation.images.ego_view
932
+ - action_metadata:
933
+ append_viewpoint: false
934
+ conditioning_fps: 20.0
935
+ frame_window_size: 17
936
+ video_height: 256
937
+ video_width: 256
938
+ viewpoint: ego_view
939
+ cfg_dropout_rate: 0.1
940
+ format_prompt_as_json: true
941
+ max_len: 512
942
+ tokenizer:
943
+ model_max_length: 4096
944
+ model_path: ./checkpoints/Cosmos3-Nano/text_tokenizer
945
+ padding_side: right
946
+ trust_remote_code: true
947
+ type: PretrainedTokenizer
948
+ type: ProcessCosmos3Prompt
949
+ - scale: 0.95
950
+ type: RandomCropImages
951
+ - height: 256
952
+ type: ResizeImages
953
+ width: 256
954
+ - brightness: 0.3
955
+ contrast: 0.4
956
+ hue: 0.08
957
+ saturation: 0.5
958
+ type: ColorJitterImages
959
+ - type: SimpleNormalizeImages
960
+ - action_dim: 64
961
+ action_key: action
962
+ action_norm_type: quantile
963
+ state_dim: 64
964
+ state_key: action
965
+ state_norm_type: quantile
966
+ type: NormalizeStatesAndActions
967
+ - conditioning_fps: 20.0
968
+ frame_window_size: 17
969
+ mode: wam
970
+ prepend_state_to_action: false
971
+ raw_action_dim: 29
972
+ type: BuildCosmos3Sequence
973
+ - frame_window_size: 17
974
+ num_views: 1
975
+ tile_direction: horizontal
976
+ type: PrepareVideo
977
+ - height: 256
978
+ type: ResizeAndReflectPad
979
+ width: 256
980
+ type: ParquetDataset
981
+ use_delta: false
982
+ window_start_idx: 0
983
+ name_mappings:
984
+ action:
985
+ - action
986
+ observation.state:
987
+ - proprio
988
+ statistic_keys:
989
+ - observation.state
990
+ - timestamp
991
+ - action
992
+ statistic_name: robocasa_gr1_24tasks_30ep
993
+ type: DistributedRepeatingDataset
994
+ per_device_batch_size: 16
995
+ per_device_num_workers: 4
cosmos3_nano_robocasa_full_data_full_finetune_bs2048/dataset_statistics.json ADDED
@@ -0,0 +1,396 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "robocasa_gr1_24tasks_30ep": {
3
+ "proprio": {
4
+ "mean": [
5
+ -0.1721954196691514,
6
+ 0.18329788744449615,
7
+ -0.1360361576080323,
8
+ -1.0807313919067383,
9
+ 0.14450088143348713,
10
+ 0.13195440173149092,
11
+ -0.08306947350502007,
12
+ 0.21940384805202476,
13
+ 0.21819256246089933,
14
+ 0.20426052808761605,
15
+ 0.22833923995494854,
16
+ 0.06841608136892328,
17
+ 0.9091449975967411,
18
+ -0.1590277999639513,
19
+ -0.2511773109436038,
20
+ 0.08599518239498137,
21
+ -1.1420482397079474,
22
+ 0.21259814500808735,
23
+ 0.04237671568989762,
24
+ 0.03427042067050932,
25
+ 0.34993353486061096,
26
+ 0.3412855267524723,
27
+ 0.3271332383155817,
28
+ 0.3575766980648041,
29
+ 0.09723818302154554,
30
+ 1.0641368627548227,
31
+ 0.014510802924633035,
32
+ 0.014398206025362044,
33
+ -0.00020241182937752458
34
+ ],
35
+ "std": [
36
+ 0.4068214595317843,
37
+ 0.18259239196777355,
38
+ 0.2513278424739838,
39
+ 0.5786943435668955,
40
+ 0.33050763607025135,
41
+ 0.28609949350357067,
42
+ 0.3013432323932649,
43
+ 0.42784574627876293,
44
+ 0.4130409955978396,
45
+ 0.3710329830646513,
46
+ 0.4229725301265714,
47
+ 0.18603332340717335,
48
+ 0.7297042012214684,
49
+ 0.5631562471389767,
50
+ 0.24450603127479503,
51
+ 0.3282549381256103,
52
+ 0.7310536503791792,
53
+ 0.48139148950576843,
54
+ 0.35611921548843384,
55
+ 0.5090531110763546,
56
+ 0.49147421121597257,
57
+ 0.46545258164405845,
58
+ 0.44516301155090443,
59
+ 0.4866269826889039,
60
+ 0.1814946085214615,
61
+ 0.5735033154487607,
62
+ 0.1227026358246804,
63
+ 0.03244836255908008,
64
+ 0.007282953243702651
65
+ ],
66
+ "min": [
67
+ -1.7593636512756348,
68
+ -0.08899051696062088,
69
+ -1.6571800708770752,
70
+ -2.516343355178833,
71
+ -2.307133436203003,
72
+ -1.4132264852523804,
73
+ -1.4408576488494873,
74
+ -3.262805700302124,
75
+ -2.7771592140197754,
76
+ -3.4712820053100586,
77
+ -2.7828404903411865,
78
+ -2.197380781173706,
79
+ -2.636387825012207,
80
+ -2.2743337154388428,
81
+ -2.999122142791748,
82
+ -1.7424250841140747,
83
+ -2.536545753479004,
84
+ -2.6034910678863525,
85
+ -1.5032434463500977,
86
+ -1.4913568496704102,
87
+ -1.7595213651657104,
88
+ -1.8982670307159424,
89
+ -1.8622381687164307,
90
+ -1.8314869403839111,
91
+ -1.0045679807662964,
92
+ -0.31411829590797424,
93
+ -0.8131351470947266,
94
+ -0.4901888370513916,
95
+ -0.7304351925849915
96
+ ],
97
+ "max": [
98
+ 1.3570959568023682,
99
+ 1.5781080722808838,
100
+ 1.7425614595413208,
101
+ 0.04977008327841759,
102
+ 2.521491289138794,
103
+ 1.5104753971099854,
104
+ 1.3451268672943115,
105
+ 2.8765757083892822,
106
+ 2.6975045204162598,
107
+ 3.3291828632354736,
108
+ 2.7148845195770264,
109
+ 3.664421558380127,
110
+ 3.5466625690460205,
111
+ 1.7399436235427856,
112
+ 0.07876992970705032,
113
+ 2.5426626205444336,
114
+ 0.048908114433288574,
115
+ 3.000918388366699,
116
+ 1.4513440132141113,
117
+ 1.5000383853912354,
118
+ 2.397510290145874,
119
+ 2.4422738552093506,
120
+ 3.0158462524414062,
121
+ 2.457378625869751,
122
+ 1.4605257511138916,
123
+ 2.168534278869629,
124
+ 1.0323854684829712,
125
+ 0.7102982997894287,
126
+ 0.43762317299842834
127
+ ],
128
+ "q01": [
129
+ -1.6011584222316746,
130
+ -0.00048318122862838164,
131
+ -0.9488833963871,
132
+ -2.4362342357635502,
133
+ -0.9403592050075524,
134
+ -0.6489510446786882,
135
+ -0.9641011685132993,
136
+ -0.17130047902464876,
137
+ -0.0025680272374302166,
138
+ -0.0029985445667989555,
139
+ -0.002490740863140675,
140
+ -0.14894675761461273,
141
+ -0.00027310136792948466,
142
+ -1.4233640372753156,
143
+ -1.0436977982521036,
144
+ -0.7169023394584662,
145
+ -2.4936094284057644,
146
+ -0.7377233326435084,
147
+ -0.8036172896623616,
148
+ -0.9991738587617871,
149
+ -0.004218762856908145,
150
+ -0.004575904295779761,
151
+ -0.005576773989014321,
152
+ -0.0047070790780708175,
153
+ -0.18967307358980165,
154
+ 0.03200087323784828,
155
+ -0.3365072563290599,
156
+ -0.02904987111687659,
157
+ -0.02606033049523827
158
+ ],
159
+ "q99": [
160
+ 0.6305126994848259,
161
+ 0.7966297268867486,
162
+ 0.3873166218399999,
163
+ -0.07399308793246723,
164
+ 1.0455405175685892,
165
+ 0.8574009954929348,
166
+ 0.4838866069912913,
167
+ 1.5014824867248553,
168
+ 1.501653671264647,
169
+ 1.4043268024921445,
170
+ 1.5127789616584784,
171
+ 0.781709930300714,
172
+ 1.872669064998624,
173
+ 1.167767405509949,
174
+ 0.0006303858070168673,
175
+ 0.8771779745817198,
176
+ -0.0582109544426202,
177
+ 1.470731711387636,
178
+ 0.8474899172782918,
179
+ 1.2271583080291764,
180
+ 1.4994217157363876,
181
+ 1.5029035806655904,
182
+ 1.6614551365375507,
183
+ 1.5188306391239186,
184
+ 0.6953653633594535,
185
+ 1.8182492077350652,
186
+ 0.48663658648729347,
187
+ 0.12612193301320107,
188
+ 0.022280203178524964
189
+ ]
190
+ },
191
+ "timestamp": {
192
+ "mean": [
193
+ 0.0
194
+ ],
195
+ "std": [
196
+ 1.0
197
+ ],
198
+ "min": [
199
+ 0.0
200
+ ],
201
+ "max": [
202
+ 0.0
203
+ ],
204
+ "q01": null,
205
+ "q99": null
206
+ },
207
+ "action": {
208
+ "mean": [
209
+ -0.17771673202514657,
210
+ 0.18351773917675007,
211
+ -0.13720993697643286,
212
+ -1.0842658281326296,
213
+ 0.14675945043563873,
214
+ 0.12921456992626204,
215
+ -0.09504522383213035,
216
+ -0.22721463441848752,
217
+ -0.21956320106983204,
218
+ -0.2144497036933901,
219
+ -0.21446840465068817,
220
+ -0.48631641268730147,
221
+ 1.6838233470916744,
222
+ -0.16953834891319267,
223
+ -0.2535267770290373,
224
+ 0.082216002047062,
225
+ -1.1492931842803968,
226
+ 0.2176141738891601,
227
+ 0.04310995712876314,
228
+ 0.04133718833327294,
229
+ -0.543570637702943,
230
+ -0.5304322242736823,
231
+ -0.5179098844528184,
232
+ -0.4968528747558592,
233
+ -1.1343770027160642,
234
+ 2.128209590911863,
235
+ 0.014836843125522158,
236
+ 0.010428135283291354,
237
+ -0.00014937532250769447
238
+ ],
239
+ "std": [
240
+ 0.4174487590789793,
241
+ 0.18478454649448395,
242
+ 0.25687918066978443,
243
+ 0.5807053446769717,
244
+ 0.33657354116439797,
245
+ 0.29842707514762895,
246
+ 0.31536528468132013,
247
+ 1.0050891637802128,
248
+ 1.0124161243438718,
249
+ 1.0138051509857173,
250
+ 1.0200439691543575,
251
+ 2.066762685775758,
252
+ 1.4588384628295925,
253
+ 0.5752111673355106,
254
+ 0.25846862792968733,
255
+ 0.34115546941757197,
256
+ 0.7394758462905866,
257
+ 0.4870710670948032,
258
+ 0.3721199631690978,
259
+ 0.5423213839530941,
260
+ 1.1248024702072141,
261
+ 1.1392076015472405,
262
+ 1.1426663398742682,
263
+ 1.301862597465516,
264
+ 2.199875354766848,
265
+ 0.9186902642250119,
266
+ 0.12568794190883642,
267
+ 0.030695030465722098,
268
+ 0.004574332851916548
269
+ ],
270
+ "min": [
271
+ -2.291214942932129,
272
+ -0.001746351015754044,
273
+ -2.4999990463256836,
274
+ -2.814260482788086,
275
+ -2.3611011505126953,
276
+ -1.4937989711761475,
277
+ -1.4969758987426758,
278
+ -1.9597030878067017,
279
+ -1.8635213375091553,
280
+ -1.9709523916244507,
281
+ -1.5,
282
+ -3.0,
283
+ 0.0,
284
+ -2.318650960922241,
285
+ -2.9999561309814453,
286
+ -1.9257696866989136,
287
+ -2.8759055137634277,
288
+ -2.620600461959839,
289
+ -1.5001521110534668,
290
+ -1.4994292259216309,
291
+ -1.5,
292
+ -1.5,
293
+ -1.5,
294
+ -1.5,
295
+ -3.0,
296
+ 4.470348358154297e-08,
297
+ -0.8292319774627686,
298
+ -0.5185094475746155,
299
+ -0.37811079621315
300
+ ],
301
+ "max": [
302
+ 1.4113223552703857,
303
+ 1.8901419639587402,
304
+ 1.7754145860671997,
305
+ 7.309383363462985e-05,
306
+ 2.563594102859497,
307
+ 1.5000243186950684,
308
+ 1.4908421039581299,
309
+ 1.5707963705062866,
310
+ 1.646651268005371,
311
+ 1.709236979484558,
312
+ 2.0620639324188232,
313
+ 3.0,
314
+ 3.0,
315
+ 1.7834906578063965,
316
+ 0.0002448999439366162,
317
+ 2.549729585647583,
318
+ 7.45560391806066e-05,
319
+ 3.0000460147857666,
320
+ 1.4975632429122925,
321
+ 1.4998434782028198,
322
+ 1.5707963705062866,
323
+ 1.7183797359466553,
324
+ 3.979951858520508,
325
+ 3.9879753589630127,
326
+ 3.0,
327
+ 3.0,
328
+ 1.049119234085083,
329
+ 0.6198405623435974,
330
+ 0.45177245140075684
331
+ ],
332
+ "q01": [
333
+ -1.6055770933628097,
334
+ -7.429541994952163e-07,
335
+ -0.9837155520915981,
336
+ -2.480029511451721,
337
+ -0.9457575023174288,
338
+ -0.7192997813224794,
339
+ -1.008323037624359,
340
+ -1.4999999999999987,
341
+ -1.4999999999999987,
342
+ -1.4999999999999987,
343
+ -1.4999999999999987,
344
+ -2.9999999999999973,
345
+ 0.0,
346
+ -1.4375487387180332,
347
+ -1.0682010412216194,
348
+ -0.7535711079835895,
349
+ -2.6383612155914276,
350
+ -0.7471688866615306,
351
+ -0.8604200631380092,
352
+ -1.0900398135185252,
353
+ -1.4999999999999987,
354
+ -1.4999999999999987,
355
+ -1.4999999999999987,
356
+ -1.4999999999999987,
357
+ -2.9999999999999973,
358
+ 0.04336242545396091,
359
+ -0.3402548208832736,
360
+ -0.025130789913237084,
361
+ -0.016220059804618367
362
+ ],
363
+ "q99": [
364
+ 0.6335921049118061,
365
+ 0.8018405169248585,
366
+ 0.3930345878005033,
367
+ -0.06366106234490865,
368
+ 1.0508334636688252,
369
+ 0.8918659299612054,
370
+ 0.5138001590967192,
371
+ 1.4999999999999987,
372
+ 1.4999999999999987,
373
+ 1.4999999999999987,
374
+ 1.4999999999999987,
375
+ 2.9999999999999973,
376
+ 2.9999999999999973,
377
+ 1.1753968834876984,
378
+ -9.999999974752431e-07,
379
+ 0.8976036489009875,
380
+ -0.0579521823674439,
381
+ 1.4876463234424584,
382
+ 0.8774339169263857,
383
+ 1.4641386926174185,
384
+ 1.4999999999999987,
385
+ 1.4999999999999987,
386
+ 1.4999999999999987,
387
+ 1.4999999999999987,
388
+ 2.9999999999999973,
389
+ 2.9999999999999973,
390
+ 0.49598969072103627,
391
+ 0.12137954644858844,
392
+ 0.010726323025301106
393
+ ]
394
+ }
395
+ }
396
+ }
cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/added_tokens.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</think>": 151668,
3
+ "</tool_call>": 151658,
4
+ "</tool_response>": 151666,
5
+ "<think>": 151667,
6
+ "<tool_call>": 151657,
7
+ "<tool_response>": 151665,
8
+ "<|box_end|>": 151649,
9
+ "<|box_start|>": 151648,
10
+ "<|endoftext|>": 151643,
11
+ "<|file_sep|>": 151664,
12
+ "<|fim_middle|>": 151660,
13
+ "<|fim_pad|>": 151662,
14
+ "<|fim_prefix|>": 151659,
15
+ "<|fim_suffix|>": 151661,
16
+ "<|im_end|>": 151645,
17
+ "<|im_start|>": 151644,
18
+ "<|image_pad|>": 151655,
19
+ "<|object_ref_end|>": 151647,
20
+ "<|object_ref_start|>": 151646,
21
+ "<|quad_end|>": 151651,
22
+ "<|quad_start|>": 151650,
23
+ "<|repo_name|>": 151663,
24
+ "<|video_pad|>": 151656,
25
+ "<|vision_end|>": 151653,
26
+ "<|vision_pad|>": 151654,
27
+ "<|vision_start|>": 151652
28
+ }
cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/chat_template.jinja ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0].role == 'system' %}
4
+ {%- if messages[0].content is string %}
5
+ {{- messages[0].content }}
6
+ {%- else %}
7
+ {%- for content in messages[0].content %}
8
+ {%- if 'text' in content %}
9
+ {{- content.text }}
10
+ {%- endif %}
11
+ {%- endfor %}
12
+ {%- endif %}
13
+ {{- '\n\n' }}
14
+ {%- endif %}
15
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
16
+ {%- for tool in tools %}
17
+ {{- "\n" }}
18
+ {{- tool | tojson }}
19
+ {%- endfor %}
20
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
21
+ {%- else %}
22
+ {%- if messages[0].role == 'system' %}
23
+ {{- '<|im_start|>system\n' }}
24
+ {%- if messages[0].content is string %}
25
+ {{- messages[0].content }}
26
+ {%- else %}
27
+ {%- for content in messages[0].content %}
28
+ {%- if 'text' in content %}
29
+ {{- content.text }}
30
+ {%- endif %}
31
+ {%- endfor %}
32
+ {%- endif %}
33
+ {{- '<|im_end|>\n' }}
34
+ {%- endif %}
35
+ {%- endif %}
36
+ {%- set image_count = namespace(value=0) %}
37
+ {%- set video_count = namespace(value=0) %}
38
+ {%- for message in messages %}
39
+ {%- if message.role == "user" %}
40
+ {{- '<|im_start|>' + message.role + '\n' }}
41
+ {%- if message.content is string %}
42
+ {{- message.content }}
43
+ {%- else %}
44
+ {%- for content in message.content %}
45
+ {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
46
+ {%- set image_count.value = image_count.value + 1 %}
47
+ {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
48
+ <|vision_start|><|image_pad|><|vision_end|>
49
+ {%- elif content.type == 'video' or 'video' in content %}
50
+ {%- set video_count.value = video_count.value + 1 %}
51
+ {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
52
+ <|vision_start|><|video_pad|><|vision_end|>
53
+ {%- elif 'text' in content %}
54
+ {{- content.text }}
55
+ {%- endif %}
56
+ {%- endfor %}
57
+ {%- endif %}
58
+ {{- '<|im_end|>\n' }}
59
+ {%- elif message.role == "assistant" %}
60
+ {{- '<|im_start|>' + message.role + '\n' }}
61
+ {%- if message.content is string %}
62
+ {{- message.content }}
63
+ {%- else %}
64
+ {%- for content_item in message.content %}
65
+ {%- if 'text' in content_item %}
66
+ {{- content_item.text }}
67
+ {%- endif %}
68
+ {%- endfor %}
69
+ {%- endif %}
70
+ {%- if message.tool_calls %}
71
+ {%- for tool_call in message.tool_calls %}
72
+ {%- if (loop.first and message.content) or (not loop.first) %}
73
+ {{- '\n' }}
74
+ {%- endif %}
75
+ {%- if tool_call.function %}
76
+ {%- set tool_call = tool_call.function %}
77
+ {%- endif %}
78
+ {{- '<tool_call>\n{"name": "' }}
79
+ {{- tool_call.name }}
80
+ {{- '", "arguments": ' }}
81
+ {%- if tool_call.arguments is string %}
82
+ {{- tool_call.arguments }}
83
+ {%- else %}
84
+ {{- tool_call.arguments | tojson }}
85
+ {%- endif %}
86
+ {{- '}\n</tool_call>' }}
87
+ {%- endfor %}
88
+ {%- endif %}
89
+ {{- '<|im_end|>\n' }}
90
+ {%- elif message.role == "tool" %}
91
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
92
+ {{- '<|im_start|>user' }}
93
+ {%- endif %}
94
+ {{- '\n<tool_response>\n' }}
95
+ {%- if message.content is string %}
96
+ {{- message.content }}
97
+ {%- else %}
98
+ {%- for content in message.content %}
99
+ {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
100
+ {%- set image_count.value = image_count.value + 1 %}
101
+ {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
102
+ <|vision_start|><|image_pad|><|vision_end|>
103
+ {%- elif content.type == 'video' or 'video' in content %}
104
+ {%- set video_count.value = video_count.value + 1 %}
105
+ {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
106
+ <|vision_start|><|video_pad|><|vision_end|>
107
+ {%- elif 'text' in content %}
108
+ {{- content.text }}
109
+ {%- endif %}
110
+ {%- endfor %}
111
+ {%- endif %}
112
+ {{- '\n</tool_response>' }}
113
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
114
+ {{- '<|im_end|>\n' }}
115
+ {%- endif %}
116
+ {%- endif %}
117
+ {%- endfor %}
118
+ {%- if add_generation_prompt %}
119
+ {{- '<|im_start|>assistant\n' }}
120
+ {%- endif %}
cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/special_tokens_map.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|object_ref_start|>",
6
+ "<|object_ref_end|>",
7
+ "<|box_start|>",
8
+ "<|box_end|>",
9
+ "<|quad_start|>",
10
+ "<|quad_end|>",
11
+ "<|vision_start|>",
12
+ "<|vision_end|>",
13
+ "<|vision_pad|>",
14
+ "<|image_pad|>",
15
+ "<|video_pad|>"
16
+ ],
17
+ "eos_token": {
18
+ "content": "<|im_end|>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<|endoftext|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ }
31
+ }
cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
3
+ size 11422654
cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,239 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "151643": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "151644": {
14
+ "content": "<|im_start|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "151645": {
22
+ "content": "<|im_end|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "151646": {
30
+ "content": "<|object_ref_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "151647": {
38
+ "content": "<|object_ref_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "151648": {
46
+ "content": "<|box_start|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "151649": {
54
+ "content": "<|box_end|>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "151650": {
62
+ "content": "<|quad_start|>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "151651": {
70
+ "content": "<|quad_end|>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "151652": {
78
+ "content": "<|vision_start|>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "151653": {
86
+ "content": "<|vision_end|>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ },
93
+ "151654": {
94
+ "content": "<|vision_pad|>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ },
101
+ "151655": {
102
+ "content": "<|image_pad|>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false,
107
+ "special": true
108
+ },
109
+ "151656": {
110
+ "content": "<|video_pad|>",
111
+ "lstrip": false,
112
+ "normalized": false,
113
+ "rstrip": false,
114
+ "single_word": false,
115
+ "special": true
116
+ },
117
+ "151657": {
118
+ "content": "<tool_call>",
119
+ "lstrip": false,
120
+ "normalized": false,
121
+ "rstrip": false,
122
+ "single_word": false,
123
+ "special": false
124
+ },
125
+ "151658": {
126
+ "content": "</tool_call>",
127
+ "lstrip": false,
128
+ "normalized": false,
129
+ "rstrip": false,
130
+ "single_word": false,
131
+ "special": false
132
+ },
133
+ "151659": {
134
+ "content": "<|fim_prefix|>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false,
139
+ "special": false
140
+ },
141
+ "151660": {
142
+ "content": "<|fim_middle|>",
143
+ "lstrip": false,
144
+ "normalized": false,
145
+ "rstrip": false,
146
+ "single_word": false,
147
+ "special": false
148
+ },
149
+ "151661": {
150
+ "content": "<|fim_suffix|>",
151
+ "lstrip": false,
152
+ "normalized": false,
153
+ "rstrip": false,
154
+ "single_word": false,
155
+ "special": false
156
+ },
157
+ "151662": {
158
+ "content": "<|fim_pad|>",
159
+ "lstrip": false,
160
+ "normalized": false,
161
+ "rstrip": false,
162
+ "single_word": false,
163
+ "special": false
164
+ },
165
+ "151663": {
166
+ "content": "<|repo_name|>",
167
+ "lstrip": false,
168
+ "normalized": false,
169
+ "rstrip": false,
170
+ "single_word": false,
171
+ "special": false
172
+ },
173
+ "151664": {
174
+ "content": "<|file_sep|>",
175
+ "lstrip": false,
176
+ "normalized": false,
177
+ "rstrip": false,
178
+ "single_word": false,
179
+ "special": false
180
+ },
181
+ "151665": {
182
+ "content": "<tool_response>",
183
+ "lstrip": false,
184
+ "normalized": false,
185
+ "rstrip": false,
186
+ "single_word": false,
187
+ "special": false
188
+ },
189
+ "151666": {
190
+ "content": "</tool_response>",
191
+ "lstrip": false,
192
+ "normalized": false,
193
+ "rstrip": false,
194
+ "single_word": false,
195
+ "special": false
196
+ },
197
+ "151667": {
198
+ "content": "<think>",
199
+ "lstrip": false,
200
+ "normalized": false,
201
+ "rstrip": false,
202
+ "single_word": false,
203
+ "special": false
204
+ },
205
+ "151668": {
206
+ "content": "</think>",
207
+ "lstrip": false,
208
+ "normalized": false,
209
+ "rstrip": false,
210
+ "single_word": false,
211
+ "special": false
212
+ }
213
+ },
214
+ "additional_special_tokens": [
215
+ "<|im_start|>",
216
+ "<|im_end|>",
217
+ "<|object_ref_start|>",
218
+ "<|object_ref_end|>",
219
+ "<|box_start|>",
220
+ "<|box_end|>",
221
+ "<|quad_start|>",
222
+ "<|quad_end|>",
223
+ "<|vision_start|>",
224
+ "<|vision_end|>",
225
+ "<|vision_pad|>",
226
+ "<|image_pad|>",
227
+ "<|video_pad|>"
228
+ ],
229
+ "bos_token": null,
230
+ "clean_up_tokenization_spaces": false,
231
+ "eos_token": "<|im_end|>",
232
+ "errors": "replace",
233
+ "extra_special_tokens": {},
234
+ "model_max_length": 262144,
235
+ "pad_token": "<|endoftext|>",
236
+ "split_special_tokens": false,
237
+ "tokenizer_class": "Qwen2Tokenizer",
238
+ "unk_token": null
239
+ }
cosmos3_nano_robocasa_full_data_full_finetune_bs2048/tokenizer/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
cosmos3_nano_robocasa_full_data_full_finetune_bs2048/vlm_backbone_config.json ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "image_token_id": 151655,
3
+ "model_type": "qwen3_vl",
4
+ "text_config": {
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 151643,
8
+ "eos_token_id": 151645,
9
+ "head_dim": 128,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 4096,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 12288,
14
+ "layer_types": [
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention",
38
+ "full_attention",
39
+ "full_attention",
40
+ "full_attention",
41
+ "full_attention",
42
+ "full_attention",
43
+ "full_attention",
44
+ "full_attention",
45
+ "full_attention",
46
+ "full_attention",
47
+ "full_attention",
48
+ "full_attention",
49
+ "full_attention",
50
+ "full_attention"
51
+ ],
52
+ "max_position_embeddings": 262144,
53
+ "model_type": "qwen3_vl_text",
54
+ "num_attention_heads": 32,
55
+ "num_hidden_layers": 36,
56
+ "num_key_value_heads": 8,
57
+ "packed_attention_backend": "flash2",
58
+ "pad_token_id": 0,
59
+ "rms_norm_eps": 1e-06,
60
+ "rope_parameters": {
61
+ "mrope_interleaved": true,
62
+ "mrope_section": [
63
+ 24,
64
+ 20,
65
+ 20
66
+ ],
67
+ "rope_theta": 5000000,
68
+ "rope_type": "default"
69
+ },
70
+ "tie_word_embeddings": false,
71
+ "use_cache": true,
72
+ "vocab_size": 151936
73
+ },
74
+ "tie_word_embeddings": false,
75
+ "transformers_version": "5.3.0",
76
+ "video_token_id": 151656,
77
+ "vision_config": {
78
+ "deepstack_visual_indexes": [
79
+ 8,
80
+ 16,
81
+ 24
82
+ ],
83
+ "depth": 27,
84
+ "hidden_act": "gelu_pytorch_tanh",
85
+ "hidden_size": 1152,
86
+ "in_channels": 3,
87
+ "initializer_range": 0.02,
88
+ "intermediate_size": 4304,
89
+ "model_type": "qwen3_vl",
90
+ "num_heads": 16,
91
+ "num_position_embeddings": 2304,
92
+ "out_hidden_size": 4096,
93
+ "patch_size": 16,
94
+ "spatial_merge_size": 2,
95
+ "temporal_patch_size": 2
96
+ },
97
+ "vision_end_token_id": 151653,
98
+ "vision_start_token_id": 151652,
99
+ "vocab_size": 151936
100
+ }