Hugging Face's logo

yslan
/
o-intermediate-model

Model card Files
xet
 Community
yslan commited on
Commit
02050c3
·
verified ·
1 Parent(s): 9261d7a

Back up checkpoint lineage metadata cosmos_runjia_update_007400_20261107

Browse files
Files changed (17) hide show
  1. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/backup_context.json +21 -0
  2. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/backup_manifest.json +109 -0
  3. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/before_log_tail.txt +439 -0
  4. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/before_step.txt +13 -0
  5. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/before_stop_receipt.json +14 -0
  6. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/code_commit.txt +1 -0
  7. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/direct_launch_authorization.json +8 -0
  8. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/evaluation_lineage.json +24 -0
  9. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/experiment_module.py +177 -0
  10. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/formal_launch.py +3 -0
  11. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/launch_slurm_step.txt +1 -0
  12. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/launch_source_commit.txt +1 -0
  13. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/launch_source_sha.txt +1 -0
  14. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/resolved_config.yaml +41 -0
  15. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/source_dataset_config.yaml +27 -0
  16. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/stop_command_receipt.json +19 -0
  17. cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/training_launcher_source.sh +65 -0
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/backup_context.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base_model": "nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92",
3
+ "checkpoint_bytes": 2997116456,
4
+ "checkpoint_path": "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_runjia_robotwin_action_video_posttrain_20261105/attempt01/trainable_overlays/step_007400.safetensors",
5
+ "checkpoint_role": "trainable_checkpoint_only",
6
+ "checkpoint_sha256": "bb505efecfb22eddcb769d1a51f6b35ad1d4e56bcd3a81ab336889f201c322ca",
7
+ "checkpoint_sidecar_available": false,
8
+ "checkpoint_update": 7400,
9
+ "frozen_base_included": false,
10
+ "launch_source_evidence": "formal_attempt01/launch.py and direct_launch_authorization.json; later branch HEADs are not substituted.",
11
+ "launch_source_recorded_commit": "6c89d006257fba23c06dbd9a3d7bc6c04683976d",
12
+ "optimizer_state_included": false,
13
+ "owner_stop": "Slurm73011.103 cancelled by owner at 2026-11-07T15:57:31Z; last logged update 7489; durable saved overlay 7400; no step7500",
14
+ "resolved_config_provenance": "No original resolved Hydra dump or step_007400 sidecar was present in the run directory. resolved_config.yaml is a derived recipe summary. Exact training commit, experiment module, dataset YAML, direct launch authorization, launch.py, source archive identity and owner-stop evidence are included.",
15
+ "resolved_hydra_dump_available": false,
16
+ "run_id": "cosmos_runjia_robotwin_action_video_posttrain_update_007400_20261107",
17
+ "schema": "cosmos_trainable_overlay_hf_backup_context_v1",
18
+ "source_code_commit": "6c89d006257fba23c06dbd9a3d7bc6c04683976d",
19
+ "warm_start_sha256": "b46067d50dc854c225c1570da2c25691bfad1d98dd97436af6f2c5cf1ec02b9b",
20
+ "warm_start_update": 7700
21
+ }
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/backup_manifest.json ADDED
@@ -0,0 +1,109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_bytes": 2997116456,
3
+ "checkpoint_path": "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_runjia_robotwin_action_video_posttrain_20261105/attempt01/trainable_overlays/step_007400.safetensors",
4
+ "checkpoint_role": "trainable_checkpoint_only",
5
+ "checkpoint_sha256": "bb505efecfb22eddcb769d1a51f6b35ad1d4e56bcd3a81ab336889f201c322ca",
6
+ "frozen_base_included": false,
7
+ "metadata_files": [
8
+ {
9
+ "bytes": 1647,
10
+ "path": "backup_context.json",
11
+ "role": "lineage_metadata",
12
+ "sha256": "6512dadbf457cc2bbe37c477d6164b9fcfff0a84138909d0295764da28f179b3"
13
+ },
14
+ {
15
+ "bytes": 64000,
16
+ "path": "before_log_tail.txt",
17
+ "role": "lineage_metadata",
18
+ "sha256": "7be81638ef6e0ddaaba1440637a9696cf25f80aa5e4afca7b8d88a51eba117c8"
19
+ },
20
+ {
21
+ "bytes": 608,
22
+ "path": "before_step.txt",
23
+ "role": "lineage_metadata",
24
+ "sha256": "8a4a3042e430a4f5fe8363321e69059120567a36aa7f56c64b08662cb426cf8f"
25
+ },
26
+ {
27
+ "bytes": 546,
28
+ "path": "before_stop_receipt.json",
29
+ "role": "lineage_metadata",
30
+ "sha256": "6e4afaa4d20f85d2ad049193ad4d796df28ea616fc246a2e1a370c625872b023"
31
+ },
32
+ {
33
+ "bytes": 41,
34
+ "path": "code_commit.txt",
35
+ "role": "lineage_metadata",
36
+ "sha256": "e6d65dc66fe7d4a73603b034a781bbc03c2508617a04a421dce87b7175e4f905"
37
+ },
38
+ {
39
+ "bytes": 312,
40
+ "path": "direct_launch_authorization.json",
41
+ "role": "lineage_metadata",
42
+ "sha256": "1e4cdcdfee3ea264876a3f730d620ee161d9edd364b1f4fa3117c44e56fd7578"
43
+ },
44
+ {
45
+ "bytes": 1264,
46
+ "path": "evaluation_lineage.json",
47
+ "role": "lineage_metadata",
48
+ "sha256": "40b16553e3742e9acf7bff2f857c50753d8dafd1491e3635ef7a4e53b5b77618"
49
+ },
50
+ {
51
+ "bytes": 8471,
52
+ "path": "experiment_module.py",
53
+ "role": "lineage_metadata",
54
+ "sha256": "1670f7f31741bb9a80d668e420e73a8672f5efec56af033448134339d8bcccb8"
55
+ },
56
+ {
57
+ "bytes": 617,
58
+ "path": "formal_launch.py",
59
+ "role": "lineage_metadata",
60
+ "sha256": "d08b8182d6d0ac887fd51fbbd0d4cc329c5386c44d8d4f72c0eab6fa0bea2087"
61
+ },
62
+ {
63
+ "bytes": 10,
64
+ "path": "launch_slurm_step.txt",
65
+ "role": "lineage_metadata",
66
+ "sha256": "90ac2dce54f6e8691e9d44c21db08fb4a99ead613a1ce1e01c5186d866d91fed"
67
+ },
68
+ {
69
+ "bytes": 41,
70
+ "path": "launch_source_commit.txt",
71
+ "role": "lineage_metadata",
72
+ "sha256": "e6d65dc66fe7d4a73603b034a781bbc03c2508617a04a421dce87b7175e4f905"
73
+ },
74
+ {
75
+ "bytes": 41,
76
+ "path": "launch_source_sha.txt",
77
+ "role": "lineage_metadata",
78
+ "sha256": "e6d65dc66fe7d4a73603b034a781bbc03c2508617a04a421dce87b7175e4f905"
79
+ },
80
+ {
81
+ "bytes": 1794,
82
+ "path": "resolved_config.yaml",
83
+ "role": "lineage_metadata",
84
+ "sha256": "9acf76bb48a626275085301477eae23cc23f7eb6a7825df80c4da52ab6d84dcc"
85
+ },
86
+ {
87
+ "bytes": 1440,
88
+ "path": "source_dataset_config.yaml",
89
+ "role": "lineage_metadata",
90
+ "sha256": "8af4041899ffaca9cd6ed185e073861cfe7f04158ff2bf784654fcda0b68a280"
91
+ },
92
+ {
93
+ "bytes": 682,
94
+ "path": "stop_command_receipt.json",
95
+ "role": "lineage_metadata",
96
+ "sha256": "20e2c8756843c9e9258fbc2bb18195d110cf30764ce52ff98eea830f4dd38267"
97
+ },
98
+ {
99
+ "bytes": 3709,
100
+ "path": "training_launcher_source.sh",
101
+ "role": "lineage_metadata",
102
+ "sha256": "bbfa406419af292f8c8781ab5b5625cbc2b858e5d20276048f4ab86083047466"
103
+ }
104
+ ],
105
+ "optimizer_state_included": false,
106
+ "remote_prefix": "cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400",
107
+ "repository": "yslan/o-intermediate-model",
108
+ "schema": "vgg_trainable_checkpoint_backup_manifest_v1"
109
+ }
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/before_log_tail.txt ADDED
@@ -0,0 +1,439 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _runjia_robotwin_action_video_posttrain_20261105/attempt01/trainable_overlays/step_007100.safetensors', 'update': 7100, 'parameter_tensors': 534, 'tensor_bytes': 2997001984, 'base_identity': 'nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92', 'config_identity': 'cosmos3_nano_omni4d_runjia_robotwin_action_video_posttrain_33frame_global64', 'exact_training_resume': False}
2
+ [11-07 13:44:52|INFO|cosmos_framework/data/omni4d_vgg_training.py:339:__iter__] Omni4D input batch 14200: raw_wait=0.0003s main_collate_wall=0.4371s windows=16 cpu_workers=4 prefetch_factor=2
3
+ [11-07 13:45:12|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7101 : iter_speed 23.62 seconds per iteration | Loss: 0.1677
4
+ [11-07 13:45:33|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7102 : iter_speed 20.45 seconds per iteration | Loss: 0.1514
5
+ [11-07 13:45:53|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7103 : iter_speed 20.31 seconds per iteration | Loss: 0.1234
6
+ [11-07 13:46:14|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7104 : iter_speed 20.70 seconds per iteration | Loss: 0.2023
7
+ [11-07 13:46:34|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7105 : iter_speed 20.41 seconds per iteration | Loss: 0.1687
8
+ [11-07 13:46:54|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7106 : iter_speed 20.46 seconds per iteration | Loss: 0.1331
9
+ [11-07 13:47:15|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7107 : iter_speed 20.46 seconds per iteration | Loss: 0.1487
10
+ [11-07 13:47:35|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7108 : iter_speed 20.36 seconds per iteration | Loss: 0.1588
11
+ [11-07 13:47:56|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7109 : iter_speed 20.35 seconds per iteration | Loss: 0.1827
12
+ [11-07 13:48:16|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7110 : iter_speed 20.42 seconds per iteration | Loss: 0.1567
13
+ [11-07 13:48:16|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.10882 (iteration 7110)
14
+ [11-07 13:48:36|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7111 : iter_speed 20.39 seconds per iteration | Loss: 0.1368
15
+ [11-07 13:48:57|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7112 : iter_speed 20.38 seconds per iteration | Loss: 0.1650
16
+ [11-07 13:49:17|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7113 : iter_speed 20.50 seconds per iteration | Loss: 0.1680
17
+ [11-07 13:49:38|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7114 : iter_speed 20.32 seconds per iteration | Loss: 0.1219
18
+ [11-07 13:49:58|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7115 : iter_speed 20.41 seconds per iteration | Loss: 0.1594
19
+ [11-07 13:50:18|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7116 : iter_speed 20.41 seconds per iteration | Loss: 0.1525
20
+ [11-07 13:50:39|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7117 : iter_speed 20.33 seconds per iteration | Loss: 0.1512
21
+ [11-07 13:50:59|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7118 : iter_speed 20.45 seconds per iteration | Loss: 0.1421
22
+ [11-07 13:51:19|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7119 : iter_speed 20.31 seconds per iteration | Loss: 0.1169
23
+ [11-07 13:51:40|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7120 : iter_speed 20.33 seconds per iteration | Loss: 0.1247
24
+ [11-07 13:51:40|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11843 (iteration 7120)
25
+ [11-07 13:52:00|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7121 : iter_speed 20.37 seconds per iteration | Loss: 0.1258
26
+ [11-07 13:52:21|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7122 : iter_speed 20.42 seconds per iteration | Loss: 0.1424
27
+ [11-07 13:52:41|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7123 : iter_speed 20.33 seconds per iteration | Loss: 0.1952
28
+ [11-07 13:53:01|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7124 : iter_speed 20.42 seconds per iteration | Loss: 0.1673
29
+ [11-07 13:53:22|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7125 : iter_speed 20.44 seconds per iteration | Loss: 0.1572
30
+ [11-07 13:53:42|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7126 : iter_speed 20.38 seconds per iteration | Loss: 0.1362
31
+ [11-07 13:54:03|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7127 : iter_speed 20.35 seconds per iteration | Loss: 0.1747
32
+ [11-07 13:54:23|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7128 : iter_speed 20.39 seconds per iteration | Loss: 0.1227
33
+ [11-07 13:54:43|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7129 : iter_speed 20.43 seconds per iteration | Loss: 0.1206
34
+ [11-07 13:55:04|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7130 : iter_speed 20.40 seconds per iteration | Loss: 0.2041
35
+ [11-07 13:55:04|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11742 (iteration 7130)
36
+ [11-07 13:55:24|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7131 : iter_speed 20.53 seconds per iteration | Loss: 0.1174
37
+ [11-07 13:55:45|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7132 : iter_speed 20.45 seconds per iteration | Loss: 0.1698
38
+ [11-07 13:56:05|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7133 : iter_speed 20.49 seconds per iteration | Loss: 0.1287
39
+ [11-07 13:56:26|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7134 : iter_speed 20.40 seconds per iteration | Loss: 0.1537
40
+ [11-07 13:56:46|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7135 : iter_speed 20.40 seconds per iteration | Loss: 0.1526
41
+ [11-07 13:57:06|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7136 : iter_speed 20.40 seconds per iteration | Loss: 0.1609
42
+ [11-07 13:57:27|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7137 : iter_speed 20.52 seconds per iteration | Loss: 0.1630
43
+ [11-07 13:57:47|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7138 : iter_speed 20.33 seconds per iteration | Loss: 0.1893
44
+ [11-07 13:58:08|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7139 : iter_speed 20.40 seconds per iteration | Loss: 0.1389
45
+ [11-07 13:58:28|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7140 : iter_speed 20.47 seconds per iteration | Loss: 0.1961
46
+ [11-07 13:58:28|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.14611 (iteration 7140)
47
+ [11-07 13:58:49|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7141 : iter_speed 20.49 seconds per iteration | Loss: 0.1593
48
+ [11-07 13:59:09|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7142 : iter_speed 20.39 seconds per iteration | Loss: 0.1566
49
+ [11-07 13:59:29|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7143 : iter_speed 20.47 seconds per iteration | Loss: 0.1216
50
+ [11-07 13:59:50|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7144 : iter_speed 20.35 seconds per iteration | Loss: 0.1376
51
+ [11-07 14:00:10|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7145 : iter_speed 20.47 seconds per iteration | Loss: 0.1424
52
+ [11-07 14:00:31|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7146 : iter_speed 20.36 seconds per iteration | Loss: 0.1409
53
+ [11-07 14:00:51|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7147 : iter_speed 20.42 seconds per iteration | Loss: 0.1156
54
+ [11-07 14:01:11|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7148 : iter_speed 20.16 seconds per iteration | Loss: 0.1120
55
+ [11-07 14:01:32|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7149 : iter_speed 20.48 seconds per iteration | Loss: 0.1105
56
+ [11-07 14:01:52|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7150 : iter_speed 20.42 seconds per iteration | Loss: 0.1325
57
+ [11-07 14:01:52|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.10854 (iteration 7150)
58
+ [11-07 14:01:53|INFO|cosmos_framework/data/omni4d_vgg_training.py:339:__iter__] Omni4D input batch 14300: raw_wait=0.0001s main_collate_wall=0.5293s windows=16 cpu_workers=4 prefetch_factor=2
59
+ [11-07 14:02:13|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7151 : iter_speed 20.42 seconds per iteration | Loss: 0.1493
60
+ [11-07 14:02:33|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7152 : iter_speed 20.31 seconds per iteration | Loss: 0.2090
61
+ [11-07 14:02:53|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7153 : iter_speed 20.45 seconds per iteration | Loss: 0.1588
62
+ [11-07 14:03:14|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7154 : iter_speed 20.44 seconds per iteration | Loss: 0.1283
63
+ [11-07 14:03:34|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7155 : iter_speed 20.65 seconds per iteration | Loss: 0.1157
64
+ [11-07 14:03:55|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7156 : iter_speed 20.30 seconds per iteration | Loss: 0.1410
65
+ [11-07 14:04:15|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7157 : iter_speed 20.45 seconds per iteration | Loss: 0.1151
66
+ [11-07 14:04:36|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7158 : iter_speed 20.45 seconds per iteration | Loss: 0.1402
67
+ [11-07 14:04:56|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7159 : iter_speed 20.45 seconds per iteration | Loss: 0.0978
68
+ [11-07 14:05:16|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7160 : iter_speed 20.34 seconds per iteration | Loss: 0.2041
69
+ [11-07 14:05:16|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11614 (iteration 7160)
70
+ [11-07 14:05:37|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7161 : iter_speed 20.39 seconds per iteration | Loss: 0.1373
71
+ [11-07 14:05:57|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7162 : iter_speed 20.43 seconds per iteration | Loss: 0.1262
72
+ [11-07 14:06:18|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7163 : iter_speed 20.34 seconds per iteration | Loss: 0.1306
73
+ [11-07 14:06:38|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7164 : iter_speed 20.45 seconds per iteration | Loss: 0.1152
74
+ [11-07 14:06:58|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7165 : iter_speed 20.40 seconds per iteration | Loss: 0.1673
75
+ [11-07 14:07:19|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7166 : iter_speed 20.43 seconds per iteration | Loss: 0.1993
76
+ [11-07 14:07:39|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7167 : iter_speed 20.40 seconds per iteration | Loss: 0.1468
77
+ [11-07 14:08:00|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7168 : iter_speed 20.44 seconds per iteration | Loss: 0.1451
78
+ [11-07 14:08:20|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7169 : iter_speed 20.36 seconds per iteration | Loss: 0.1875
79
+ [11-07 14:08:40|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7170 : iter_speed 20.44 seconds per iteration | Loss: 0.1229
80
+ [11-07 14:08:40|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11707 (iteration 7170)
81
+ [11-07 14:09:01|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7171 : iter_speed 20.46 seconds per iteration | Loss: 0.1225
82
+ [11-07 14:09:21|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7172 : iter_speed 20.41 seconds per iteration | Loss: 0.1615
83
+ [11-07 14:09:42|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7173 : iter_speed 20.46 seconds per iteration | Loss: 0.1521
84
+ [11-07 14:10:02|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7174 : iter_speed 20.46 seconds per iteration | Loss: 0.1745
85
+ [11-07 14:10:23|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7175 : iter_speed 20.39 seconds per iteration | Loss: 0.1222
86
+ [11-07 14:10:43|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7176 : iter_speed 20.44 seconds per iteration | Loss: 0.1610
87
+ [11-07 14:11:03|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7177 : iter_speed 20.26 seconds per iteration | Loss: 0.1498
88
+ [11-07 14:11:24|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7178 : iter_speed 20.42 seconds per iteration | Loss: 0.1485
89
+ [11-07 14:11:45|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7179 : iter_speed 20.78 seconds per iteration | Loss: 0.1490
90
+ [11-07 14:12:05|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7180 : iter_speed 20.32 seconds per iteration | Loss: 0.1267
91
+ [11-07 14:12:05|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.10825 (iteration 7180)
92
+ [11-07 14:12:25|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7181 : iter_speed 20.36 seconds per iteration | Loss: 0.1367
93
+ [11-07 14:12:46|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7182 : iter_speed 20.44 seconds per iteration | Loss: 0.2203
94
+ [11-07 14:13:06|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7183 : iter_speed 20.36 seconds per iteration | Loss: 0.1257
95
+ [11-07 14:13:26|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7184 : iter_speed 20.42 seconds per iteration | Loss: 0.1478
96
+ [11-07 14:13:47|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7185 : iter_speed 20.74 seconds per iteration | Loss: 0.1747
97
+ [11-07 14:14:08|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7186 : iter_speed 20.37 seconds per iteration | Loss: 0.1942
98
+ [11-07 14:14:28|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7187 : iter_speed 20.30 seconds per iteration | Loss: 0.1866
99
+ [11-07 14:14:48|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7188 : iter_speed 20.40 seconds per iteration | Loss: 0.1435
100
+ [11-07 14:15:09|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7189 : iter_speed 20.42 seconds per iteration | Loss: 0.1738
101
+ [11-07 14:15:29|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7190 : iter_speed 20.42 seconds per iteration | Loss: 0.1943
102
+ [11-07 14:15:29|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11444 (iteration 7190)
103
+ [11-07 14:15:50|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7191 : iter_speed 20.41 seconds per iteration | Loss: 0.1281
104
+ [11-07 14:16:10|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7192 : iter_speed 20.43 seconds per iteration | Loss: 0.1124
105
+ [11-07 14:16:30|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7193 : iter_speed 20.36 seconds per iteration | Loss: 0.1639
106
+ [11-07 14:16:51|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7194 : iter_speed 20.39 seconds per iteration | Loss: 0.1343
107
+ [11-07 14:17:11|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7195 : iter_speed 20.42 seconds per iteration | Loss: 0.1112
108
+ [11-07 14:17:32|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7196 : iter_speed 20.43 seconds per iteration | Loss: 0.1601
109
+ [11-07 14:17:52|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7197 : iter_speed 20.68 seconds per iteration | Loss: 0.1319
110
+ [11-07 14:18:13|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7198 : iter_speed 20.45 seconds per iteration | Loss: 0.2566
111
+ [11-07 14:18:33|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7199 : iter_speed 20.37 seconds per iteration | Loss: 0.1665
112
+ [11-07 14:18:54|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7200 : iter_speed 20.47 seconds per iteration | Loss: 0.1708
113
+ [11-07 14:18:54|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11206 (iteration 7200)
114
+ [11-07 14:18:57|INFO|cosmos_framework/callbacks/omni4d_trainable_overlay.py:73:on_training_step_end] Published trainable safetensors at update 7200: {'status': 'complete', 'path': '/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_runjia_robotwin_action_video_posttrain_20261105/attempt01/trainable_overlays/step_007200.safetensors', 'update': 7200, 'parameter_tensors': 534, 'tensor_bytes': 2997001984, 'base_identity': 'nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92', 'config_identity': 'cosmos3_nano_omni4d_runjia_robotwin_action_video_posttrain_33frame_global64', 'exact_training_resume': False}
115
+ [11-07 14:18:58|INFO|cosmos_framework/data/omni4d_vgg_training.py:339:__iter__] Omni4D input batch 14400: raw_wait=0.0002s main_collate_wall=0.5317s windows=16 cpu_workers=4 prefetch_factor=2
116
+ [11-07 14:19:17|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7201 : iter_speed 23.83 seconds per iteration | Loss: 0.1707
117
+ [11-07 14:19:38|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7202 : iter_speed 20.40 seconds per iteration | Loss: 0.1268
118
+ [11-07 14:19:58|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7203 : iter_speed 20.60 seconds per iteration | Loss: 0.1642
119
+ [11-07 14:20:19|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7204 : iter_speed 20.31 seconds per iteration | Loss: 0.1730
120
+ [11-07 14:20:39|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7205 : iter_speed 20.66 seconds per iteration | Loss: 0.1528
121
+ [11-07 14:21:00|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7206 : iter_speed 20.47 seconds per iteration | Loss: 0.1805
122
+ [11-07 14:21:20|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7207 : iter_speed 20.50 seconds per iteration | Loss: 0.1396
123
+ [11-07 14:21:41|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7208 : iter_speed 20.30 seconds per iteration | Loss: 0.1249
124
+ [11-07 14:22:01|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7209 : iter_speed 20.46 seconds per iteration | Loss: 0.1266
125
+ [11-07 14:22:22|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7210 : iter_speed 20.49 seconds per iteration | Loss: 0.1439
126
+ [11-07 14:22:22|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.15334 (iteration 7210)
127
+ [11-07 14:22:42|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7211 : iter_speed 20.34 seconds per iteration | Loss: 0.1226
128
+ [11-07 14:23:02|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7212 : iter_speed 20.42 seconds per iteration | Loss: 0.2159
129
+ [11-07 14:23:23|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7213 : iter_speed 20.43 seconds per iteration | Loss: 0.1325
130
+ [11-07 14:23:43|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7214 : iter_speed 20.30 seconds per iteration | Loss: 0.1248
131
+ [11-07 14:24:04|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7215 : iter_speed 20.49 seconds per iteration | Loss: 0.1514
132
+ [11-07 14:24:24|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7216 : iter_speed 20.36 seconds per iteration | Loss: 0.1754
133
+ [11-07 14:24:44|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7217 : iter_speed 20.45 seconds per iteration | Loss: 0.1407
134
+ [11-07 14:25:05|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7218 : iter_speed 20.46 seconds per iteration | Loss: 0.1339
135
+ [11-07 14:25:25|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7219 : iter_speed 20.40 seconds per iteration | Loss: 0.1777
136
+ [11-07 14:25:45|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7220 : iter_speed 20.31 seconds per iteration | Loss: 0.1527
137
+ [11-07 14:25:45|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11171 (iteration 7220)
138
+ [11-07 14:26:06|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7221 : iter_speed 20.46 seconds per iteration | Loss: 0.1357
139
+ [11-07 14:26:26|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7222 : iter_speed 20.26 seconds per iteration | Loss: 0.1707
140
+ [11-07 14:26:47|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7223 : iter_speed 20.44 seconds per iteration | Loss: 0.1270
141
+ [11-07 14:27:07|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7224 : iter_speed 20.42 seconds per iteration | Loss: 0.1382
142
+ [11-07 14:27:28|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7225 : iter_speed 20.47 seconds per iteration | Loss: 0.2267
143
+ [11-07 14:27:48|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7226 : iter_speed 20.26 seconds per iteration | Loss: 0.1200
144
+ [11-07 14:28:09|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7227 : iter_speed 20.73 seconds per iteration | Loss: 0.1315
145
+ [11-07 14:28:29|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7228 : iter_speed 20.33 seconds per iteration | Loss: 0.1136
146
+ [11-07 14:28:49|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7229 : iter_speed 20.41 seconds per iteration | Loss: 0.1506
147
+ [11-07 14:29:10|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7230 : iter_speed 20.41 seconds per iteration | Loss: 0.1494
148
+ [11-07 14:29:10|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11746 (iteration 7230)
149
+ [11-07 14:29:30|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7231 : iter_speed 20.49 seconds per iteration | Loss: 0.1489
150
+ [11-07 14:29:51|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7232 : iter_speed 20.42 seconds per iteration | Loss: 0.1114
151
+ [11-07 14:30:11|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7233 : iter_speed 20.64 seconds per iteration | Loss: 0.1607
152
+ [11-07 14:30:31|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7234 : iter_speed 20.28 seconds per iteration | Loss: 0.1067
153
+ [11-07 14:30:52|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7235 : iter_speed 20.39 seconds per iteration | Loss: 0.1764
154
+ [11-07 14:31:12|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7236 : iter_speed 20.37 seconds per iteration | Loss: 0.1313
155
+ [11-07 14:31:33|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7237 : iter_speed 20.45 seconds per iteration | Loss: 0.1395
156
+ [11-07 14:31:53|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7238 : iter_speed 20.30 seconds per iteration | Loss: 0.1388
157
+ [11-07 14:32:13|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7239 : iter_speed 20.47 seconds per iteration | Loss: 0.1366
158
+ [11-07 14:32:34|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7240 : iter_speed 20.40 seconds per iteration | Loss: 0.1454
159
+ [11-07 14:32:34|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.09880 (iteration 7240)
160
+ [11-07 14:32:54|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7241 : iter_speed 20.45 seconds per iteration | Loss: 0.1381
161
+ [11-07 14:33:15|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7242 : iter_speed 20.41 seconds per iteration | Loss: 0.1252
162
+ [11-07 14:33:35|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7243 : iter_speed 20.34 seconds per iteration | Loss: 0.1974
163
+ [11-07 14:33:55|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7244 : iter_speed 20.40 seconds per iteration | Loss: 0.1492
164
+ [11-07 14:34:16|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7245 : iter_speed 20.43 seconds per iteration | Loss: 0.1547
165
+ [11-07 14:34:36|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7246 : iter_speed 20.39 seconds per iteration | Loss: 0.1658
166
+ [11-07 14:34:57|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7247 : iter_speed 20.37 seconds per iteration | Loss: 0.1696
167
+ [11-07 14:35:17|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7248 : iter_speed 20.41 seconds per iteration | Loss: 0.1211
168
+ [11-07 14:35:37|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7249 : iter_speed 20.29 seconds per iteration | Loss: 0.1479
169
+ [11-07 14:35:58|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7250 : iter_speed 20.36 seconds per iteration | Loss: 0.1746
170
+ [11-07 14:35:58|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.12215 (iteration 7250)
171
+ [11-07 14:35:58|INFO|cosmos_framework/data/omni4d_vgg_training.py:339:__iter__] Omni4D input batch 14500: raw_wait=0.0001s main_collate_wall=0.4539s windows=16 cpu_workers=4 prefetch_factor=2
172
+ [11-07 14:36:18|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7251 : iter_speed 20.49 seconds per iteration | Loss: 0.1177
173
+ [11-07 14:36:39|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7252 : iter_speed 20.41 seconds per iteration | Loss: 0.1722
174
+ [11-07 14:36:59|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7253 : iter_speed 20.31 seconds per iteration | Loss: 0.1418
175
+ [11-07 14:37:19|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7254 : iter_speed 20.47 seconds per iteration | Loss: 0.1600
176
+ [11-07 14:37:40|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7255 : iter_speed 20.48 seconds per iteration | Loss: 0.1378
177
+ [11-07 14:38:00|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7256 : iter_speed 20.40 seconds per iteration | Loss: 0.2112
178
+ [11-07 14:38:21|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7257 : iter_speed 20.66 seconds per iteration | Loss: 0.1328
179
+ [11-07 14:38:41|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7258 : iter_speed 20.46 seconds per iteration | Loss: 0.1378
180
+ [11-07 14:39:02|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7259 : iter_speed 20.42 seconds per iteration | Loss: 0.1390
181
+ [11-07 14:39:22|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7260 : iter_speed 20.45 seconds per iteration | Loss: 0.1240
182
+ [11-07 14:39:22|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11643 (iteration 7260)
183
+ [11-07 14:39:43|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7261 : iter_speed 20.49 seconds per iteration | Loss: 0.1322
184
+ [11-07 14:40:03|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7262 : iter_speed 20.31 seconds per iteration | Loss: 0.1382
185
+ [11-07 14:40:24|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7263 : iter_speed 20.46 seconds per iteration | Loss: 0.1354
186
+ [11-07 14:40:44|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7264 : iter_speed 20.33 seconds per iteration | Loss: 0.1341
187
+ [11-07 14:41:04|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7265 : iter_speed 20.44 seconds per iteration | Loss: 0.1175
188
+ [11-07 14:41:25|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7266 : iter_speed 20.33 seconds per iteration | Loss: 0.1204
189
+ [11-07 14:41:45|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7267 : iter_speed 20.47 seconds per iteration | Loss: 0.1410
190
+ [11-07 14:42:05|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7268 : iter_speed 20.34 seconds per iteration | Loss: 0.1555
191
+ [11-07 14:42:26|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7269 : iter_speed 20.45 seconds per iteration | Loss: 0.1741
192
+ [11-07 14:42:46|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7270 : iter_speed 20.37 seconds per iteration | Loss: 0.1466
193
+ [11-07 14:42:46|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.10837 (iteration 7270)
194
+ [11-07 14:43:07|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7271 : iter_speed 20.65 seconds per iteration | Loss: 0.2345
195
+ [11-07 14:43:27|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7272 : iter_speed 20.30 seconds per iteration | Loss: 0.1337
196
+ [11-07 14:43:48|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7273 : iter_speed 20.44 seconds per iteration | Loss: 0.2188
197
+ [11-07 14:44:08|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7274 : iter_speed 20.32 seconds per iteration | Loss: 0.1475
198
+ [11-07 14:44:28|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7275 : iter_speed 20.42 seconds per iteration | Loss: 0.1470
199
+ [11-07 14:44:49|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7276 : iter_speed 20.30 seconds per iteration | Loss: 0.1808
200
+ [11-07 14:45:09|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7277 : iter_speed 20.47 seconds per iteration | Loss: 0.1845
201
+ [11-07 14:45:29|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7278 : iter_speed 20.27 seconds per iteration | Loss: 0.1644
202
+ [11-07 14:45:50|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7279 : iter_speed 20.43 seconds per iteration | Loss: 0.1537
203
+ [11-07 14:46:10|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7280 : iter_speed 20.32 seconds per iteration | Loss: 0.2024
204
+ [11-07 14:46:10|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.10536 (iteration 7280)
205
+ [11-07 14:46:31|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7281 : iter_speed 20.35 seconds per iteration | Loss: 0.1175
206
+ [11-07 14:46:51|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7282 : iter_speed 20.30 seconds per iteration | Loss: 0.1185
207
+ [11-07 14:47:11|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7283 : iter_speed 20.35 seconds per iteration | Loss: 0.1410
208
+ [11-07 14:47:31|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7284 : iter_speed 20.28 seconds per iteration | Loss: 0.2026
209
+ [11-07 14:47:52|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7285 : iter_speed 20.46 seconds per iteration | Loss: 0.1725
210
+ [11-07 14:48:12|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7286 : iter_speed 20.36 seconds per iteration | Loss: 0.2014
211
+ [11-07 14:48:33|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7287 : iter_speed 20.44 seconds per iteration | Loss: 0.1090
212
+ [11-07 14:48:53|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7288 : iter_speed 20.34 seconds per iteration | Loss: 0.1339
213
+ [11-07 14:49:13|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7289 : iter_speed 20.38 seconds per iteration | Loss: 0.1022
214
+ [11-07 14:49:34|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7290 : iter_speed 20.46 seconds per iteration | Loss: 0.1747
215
+ [11-07 14:49:34|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.12141 (iteration 7290)
216
+ [11-07 14:49:54|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7291 : iter_speed 20.46 seconds per iteration | Loss: 0.1857
217
+ [11-07 14:50:15|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7292 : iter_speed 20.32 seconds per iteration | Loss: 0.1445
218
+ [11-07 14:50:35|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7293 : iter_speed 20.44 seconds per iteration | Loss: 0.1487
219
+ [11-07 14:50:55|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7294 : iter_speed 20.29 seconds per iteration | Loss: 0.1540
220
+ [11-07 14:51:16|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7295 : iter_speed 20.46 seconds per iteration | Loss: 0.1530
221
+ [11-07 14:51:36|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7296 : iter_speed 20.30 seconds per iteration | Loss: 0.1133
222
+ [11-07 14:51:57|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7297 : iter_speed 20.45 seconds per iteration | Loss: 0.1436
223
+ [11-07 14:52:17|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7298 : iter_speed 20.38 seconds per iteration | Loss: 0.1966
224
+ [11-07 14:52:37|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7299 : iter_speed 20.42 seconds per iteration | Loss: 0.1406
225
+ [11-07 14:52:58|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7300 : iter_speed 20.32 seconds per iteration | Loss: 0.1389
226
+ [11-07 14:52:58|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.10666 (iteration 7300)
227
+ [11-07 14:53:01|INFO|cosmos_framework/callbacks/omni4d_trainable_overlay.py:73:on_training_step_end] Published trainable safetensors at update 7300: {'status': 'complete', 'path': '/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_runjia_robotwin_action_video_posttrain_20261105/attempt01/trainable_overlays/step_007300.safetensors', 'update': 7300, 'parameter_tensors': 534, 'tensor_bytes': 2997001984, 'base_identity': 'nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92', 'config_identity': 'cosmos3_nano_omni4d_runjia_robotwin_action_video_posttrain_33frame_global64', 'exact_training_resume': False}
228
+ [11-07 14:53:02|INFO|cosmos_framework/data/omni4d_vgg_training.py:339:__iter__] Omni4D input batch 14600: raw_wait=0.0002s main_collate_wall=0.4596s windows=16 cpu_workers=4 prefetch_factor=2
229
+ [11-07 14:53:21|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7301 : iter_speed 23.37 seconds per iteration | Loss: 0.1476
230
+ [11-07 14:53:41|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7302 : iter_speed 20.36 seconds per iteration | Loss: 0.1193
231
+ [11-07 14:54:02|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7303 : iter_speed 20.31 seconds per iteration | Loss: 0.1414
232
+ [11-07 14:54:22|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7304 : iter_speed 20.38 seconds per iteration | Loss: 0.1189
233
+ [11-07 14:54:42|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7305 : iter_speed 20.30 seconds per iteration | Loss: 0.1180
234
+ [11-07 14:55:03|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7306 : iter_speed 20.36 seconds per iteration | Loss: 0.1742
235
+ [11-07 14:55:23|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7307 : iter_speed 20.42 seconds per iteration | Loss: 0.2316
236
+ [11-07 14:55:44|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7308 : iter_speed 20.24 seconds per iteration | Loss: 0.1704
237
+ [11-07 14:56:04|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7309 : iter_speed 20.44 seconds per iteration | Loss: 0.1205
238
+ [11-07 14:56:24|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7310 : iter_speed 20.42 seconds per iteration | Loss: 0.1334
239
+ [11-07 14:56:24|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.10908 (iteration 7310)
240
+ [11-07 14:56:45|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7311 : iter_speed 20.36 seconds per iteration | Loss: 0.1623
241
+ [11-07 14:57:05|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7312 : iter_speed 20.36 seconds per iteration | Loss: 0.1422
242
+ [11-07 14:57:25|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7313 : iter_speed 20.30 seconds per iteration | Loss: 0.1372
243
+ [11-07 14:57:46|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7314 : iter_speed 20.24 seconds per iteration | Loss: 0.1508
244
+ [11-07 14:58:06|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7315 : iter_speed 20.24 seconds per iteration | Loss: 0.1571
245
+ [11-07 14:58:26|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7316 : iter_speed 20.36 seconds per iteration | Loss: 0.1439
246
+ [11-07 14:58:47|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7317 : iter_speed 20.37 seconds per iteration | Loss: 0.1717
247
+ [11-07 14:59:07|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7318 : iter_speed 20.24 seconds per iteration | Loss: 0.1188
248
+ [11-07 14:59:27|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7319 : iter_speed 20.35 seconds per iteration | Loss: 0.1483
249
+ [11-07 14:59:48|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7320 : iter_speed 20.46 seconds per iteration | Loss: 0.1287
250
+ [11-07 14:59:48|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.10491 (iteration 7320)
251
+ [11-07 15:00:08|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7321 : iter_speed 20.38 seconds per iteration | Loss: 0.1899
252
+ [11-07 15:00:28|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7322 : iter_speed 20.35 seconds per iteration | Loss: 0.1107
253
+ [11-07 15:00:49|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7323 : iter_speed 20.35 seconds per iteration | Loss: 0.1491
254
+ [11-07 15:01:09|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7324 : iter_speed 20.44 seconds per iteration | Loss: 0.1135
255
+ [11-07 15:01:30|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7325 : iter_speed 20.36 seconds per iteration | Loss: 0.1216
256
+ [11-07 15:01:50|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7326 : iter_speed 20.40 seconds per iteration | Loss: 0.1399
257
+ [11-07 15:02:10|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7327 : iter_speed 20.31 seconds per iteration | Loss: 0.1617
258
+ [11-07 15:02:31|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7328 : iter_speed 20.44 seconds per iteration | Loss: 0.1543
259
+ [11-07 15:02:51|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7329 : iter_speed 20.39 seconds per iteration | Loss: 0.1805
260
+ [11-07 15:03:11|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7330 : iter_speed 20.40 seconds per iteration | Loss: 0.1371
261
+ [11-07 15:03:11|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.10764 (iteration 7330)
262
+ [11-07 15:03:32|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7331 : iter_speed 20.30 seconds per iteration | Loss: 0.1878
263
+ [11-07 15:03:52|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7332 : iter_speed 20.41 seconds per iteration | Loss: 0.1300
264
+ [11-07 15:04:13|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7333 : iter_speed 20.36 seconds per iteration | Loss: 0.1416
265
+ [11-07 15:04:33|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7334 : iter_speed 20.42 seconds per iteration | Loss: 0.1388
266
+ [11-07 15:04:53|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7335 : iter_speed 20.35 seconds per iteration | Loss: 0.2151
267
+ [11-07 15:05:14|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7336 : iter_speed 20.43 seconds per iteration | Loss: 0.1343
268
+ [11-07 15:05:34|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7337 : iter_speed 20.39 seconds per iteration | Loss: 0.1389
269
+ [11-07 15:05:55|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7338 : iter_speed 20.40 seconds per iteration | Loss: 0.1322
270
+ [11-07 15:06:15|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7339 : iter_speed 20.35 seconds per iteration | Loss: 0.1851
271
+ [11-07 15:06:35|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7340 : iter_speed 20.32 seconds per iteration | Loss: 0.1787
272
+ [11-07 15:06:35|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11341 (iteration 7340)
273
+ [11-07 15:06:56|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7341 : iter_speed 20.39 seconds per iteration | Loss: 0.1349
274
+ [11-07 15:07:16|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7342 : iter_speed 20.42 seconds per iteration | Loss: 0.1275
275
+ [11-07 15:07:36|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7343 : iter_speed 20.32 seconds per iteration | Loss: 0.1765
276
+ [11-07 15:07:57|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7344 : iter_speed 20.44 seconds per iteration | Loss: 0.1586
277
+ [11-07 15:08:17|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7345 : iter_speed 20.32 seconds per iteration | Loss: 0.1463
278
+ [11-07 15:08:38|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7346 : iter_speed 20.43 seconds per iteration | Loss: 0.1714
279
+ [11-07 15:08:58|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7347 : iter_speed 20.33 seconds per iteration | Loss: 0.1178
280
+ [11-07 15:09:18|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7348 : iter_speed 20.42 seconds per iteration | Loss: 0.1500
281
+ [11-07 15:09:39|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7349 : iter_speed 20.26 seconds per iteration | Loss: 0.1198
282
+ [11-07 15:09:59|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7350 : iter_speed 20.35 seconds per iteration | Loss: 0.1440
283
+ [11-07 15:09:59|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11212 (iteration 7350)
284
+ [11-07 15:09:59|INFO|cosmos_framework/data/omni4d_vgg_training.py:339:__iter__] Omni4D input batch 14700: raw_wait=0.0001s main_collate_wall=0.4432s windows=16 cpu_workers=4 prefetch_factor=2
285
+ [11-07 15:10:19|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7351 : iter_speed 20.35 seconds per iteration | Loss: 0.1366
286
+ [11-07 15:10:40|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7352 : iter_speed 20.32 seconds per iteration | Loss: 0.1444
287
+ [11-07 15:11:00|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7353 : iter_speed 20.30 seconds per iteration | Loss: 0.1765
288
+ [11-07 15:11:20|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7354 : iter_speed 20.42 seconds per iteration | Loss: 0.1189
289
+ [11-07 15:11:41|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7355 : iter_speed 20.35 seconds per iteration | Loss: 0.1803
290
+ [11-07 15:12:01|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7356 : iter_speed 20.37 seconds per iteration | Loss: 0.1261
291
+ [11-07 15:12:21|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7357 : iter_speed 20.27 seconds per iteration | Loss: 0.1350
292
+ [11-07 15:12:42|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7358 : iter_speed 20.43 seconds per iteration | Loss: 0.1413
293
+ [11-07 15:13:02|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7359 : iter_speed 20.28 seconds per iteration | Loss: 0.1419
294
+ [11-07 15:13:22|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7360 : iter_speed 20.46 seconds per iteration | Loss: 0.1574
295
+ [11-07 15:13:22|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.12210 (iteration 7360)
296
+ [11-07 15:13:43|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7361 : iter_speed 20.24 seconds per iteration | Loss: 0.1398
297
+ [11-07 15:14:03|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7362 : iter_speed 20.45 seconds per iteration | Loss: 0.1221
298
+ [11-07 15:14:23|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7363 : iter_speed 20.27 seconds per iteration | Loss: 0.1461
299
+ [11-07 15:14:44|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7364 : iter_speed 20.44 seconds per iteration | Loss: 0.1641
300
+ [11-07 15:15:04|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7365 : iter_speed 20.38 seconds per iteration | Loss: 0.1823
301
+ [11-07 15:15:25|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7366 : iter_speed 20.42 seconds per iteration | Loss: 0.1319
302
+ [11-07 15:15:45|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7367 : iter_speed 20.28 seconds per iteration | Loss: 0.1111
303
+ [11-07 15:16:05|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7368 : iter_speed 20.44 seconds per iteration | Loss: 0.1424
304
+ [11-07 15:16:26|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7369 : iter_speed 20.37 seconds per iteration | Loss: 0.1372
305
+ [11-07 15:16:46|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7370 : iter_speed 20.36 seconds per iteration | Loss: 0.1915
306
+ [11-07 15:16:46|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.12904 (iteration 7370)
307
+ [11-07 15:17:06|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7371 : iter_speed 20.29 seconds per iteration | Loss: 0.1319
308
+ [11-07 15:17:27|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7372 : iter_speed 20.37 seconds per iteration | Loss: 0.1446
309
+ [11-07 15:17:47|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7373 : iter_speed 20.31 seconds per iteration | Loss: 0.1524
310
+ [11-07 15:18:07|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7374 : iter_speed 20.42 seconds per iteration | Loss: 0.1374
311
+ [11-07 15:18:28|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7375 : iter_speed 20.37 seconds per iteration | Loss: 0.1589
312
+ [11-07 15:18:48|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7376 : iter_speed 20.35 seconds per iteration | Loss: 0.1864
313
+ [11-07 15:19:09|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7377 : iter_speed 20.41 seconds per iteration | Loss: 0.1166
314
+ [11-07 15:19:29|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7378 : iter_speed 20.22 seconds per iteration | Loss: 0.1546
315
+ [11-07 15:19:49|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7379 : iter_speed 20.44 seconds per iteration | Loss: 0.1344
316
+ [11-07 15:20:10|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7380 : iter_speed 20.36 seconds per iteration | Loss: 0.1626
317
+ [11-07 15:20:10|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11252 (iteration 7380)
318
+ [11-07 15:20:30|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7381 : iter_speed 20.64 seconds per iteration | Loss: 0.1722
319
+ [11-07 15:20:51|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7382 : iter_speed 20.36 seconds per iteration | Loss: 0.2106
320
+ [11-07 15:21:11|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7383 : iter_speed 20.40 seconds per iteration | Loss: 0.1434
321
+ [11-07 15:21:31|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7384 : iter_speed 20.39 seconds per iteration | Loss: 0.1821
322
+ [11-07 15:21:52|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7385 : iter_speed 20.41 seconds per iteration | Loss: 0.1370
323
+ [11-07 15:22:12|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7386 : iter_speed 20.30 seconds per iteration | Loss: 0.1186
324
+ [11-07 15:22:32|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7387 : iter_speed 20.36 seconds per iteration | Loss: 0.1381
325
+ [11-07 15:22:53|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7388 : iter_speed 20.41 seconds per iteration | Loss: 0.1502
326
+ [11-07 15:23:13|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7389 : iter_speed 20.38 seconds per iteration | Loss: 0.1466
327
+ [11-07 15:23:34|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7390 : iter_speed 20.40 seconds per iteration | Loss: 0.1752
328
+ [11-07 15:23:34|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.10641 (iteration 7390)
329
+ [11-07 15:23:54|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7391 : iter_speed 20.42 seconds per iteration | Loss: 0.1923
330
+ [11-07 15:24:14|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7392 : iter_speed 20.30 seconds per iteration | Loss: 0.1444
331
+ [11-07 15:24:35|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7393 : iter_speed 20.43 seconds per iteration | Loss: 0.1327
332
+ [11-07 15:24:55|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7394 : iter_speed 20.49 seconds per iteration | Loss: 0.1614
333
+ [11-07 15:25:16|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7395 : iter_speed 20.38 seconds per iteration | Loss: 0.1713
334
+ [11-07 15:25:36|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7396 : iter_speed 20.37 seconds per iteration | Loss: 0.1128
335
+ [11-07 15:25:56|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7397 : iter_speed 20.35 seconds per iteration | Loss: 0.1328
336
+ [11-07 15:26:17|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7398 : iter_speed 20.42 seconds per iteration | Loss: 0.1131
337
+ [11-07 15:26:37|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7399 : iter_speed 20.35 seconds per iteration | Loss: 0.1396
338
+ [11-07 15:26:58|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7400 : iter_speed 20.39 seconds per iteration | Loss: 0.1474
339
+ [11-07 15:26:58|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11555 (iteration 7400)
340
+ [11-07 15:27:01|INFO|cosmos_framework/callbacks/omni4d_trainable_overlay.py:73:on_training_step_end] Published trainable safetensors at update 7400: {'status': 'complete', 'path': '/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_runjia_robotwin_action_video_posttrain_20261105/attempt01/trainable_overlays/step_007400.safetensors', 'update': 7400, 'parameter_tensors': 534, 'tensor_bytes': 2997001984, 'base_identity': 'nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92', 'config_identity': 'cosmos3_nano_omni4d_runjia_robotwin_action_video_posttrain_33frame_global64', 'exact_training_resume': False}
341
+ [11-07 15:27:02|INFO|cosmos_framework/data/omni4d_vgg_training.py:339:__iter__] Omni4D input batch 14800: raw_wait=0.0004s main_collate_wall=0.4382s windows=16 cpu_workers=4 prefetch_factor=2
342
+ [11-07 15:27:22|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7401 : iter_speed 24.15 seconds per iteration | Loss: 0.1269
343
+ [11-07 15:27:42|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7402 : iter_speed 20.43 seconds per iteration | Loss: 0.1857
344
+ [11-07 15:28:03|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7403 : iter_speed 20.41 seconds per iteration | Loss: 0.1110
345
+ [11-07 15:28:23|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7404 : iter_speed 20.39 seconds per iteration | Loss: 0.1557
346
+ [11-07 15:28:43|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7405 : iter_speed 20.27 seconds per iteration | Loss: 0.1573
347
+ [11-07 15:29:04|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7406 : iter_speed 20.37 seconds per iteration | Loss: 0.1361
348
+ [11-07 15:29:24|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7407 : iter_speed 20.46 seconds per iteration | Loss: 0.1459
349
+ [11-07 15:29:44|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7408 : iter_speed 20.40 seconds per iteration | Loss: 0.1458
350
+ [11-07 15:30:05|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7409 : iter_speed 20.34 seconds per iteration | Loss: 0.1473
351
+ [11-07 15:30:25|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7410 : iter_speed 20.40 seconds per iteration | Loss: 0.1000
352
+ [11-07 15:30:25|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.13275 (iteration 7410)
353
+ [11-07 15:30:46|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7411 : iter_speed 20.42 seconds per iteration | Loss: 0.1341
354
+ [11-07 15:31:06|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7412 : iter_speed 20.38 seconds per iteration | Loss: 0.1552
355
+ [11-07 15:31:26|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7413 : iter_speed 20.37 seconds per iteration | Loss: 0.1323
356
+ [11-07 15:31:47|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7414 : iter_speed 20.30 seconds per iteration | Loss: 0.1236
357
+ [11-07 15:32:07|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7415 : iter_speed 20.42 seconds per iteration | Loss: 0.1444
358
+ [11-07 15:32:27|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7416 : iter_speed 20.36 seconds per iteration | Loss: 0.1371
359
+ [11-07 15:32:48|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7417 : iter_speed 20.30 seconds per iteration | Loss: 0.1648
360
+ [11-07 15:33:08|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7418 : iter_speed 20.29 seconds per iteration | Loss: 0.1368
361
+ [11-07 15:33:28|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7419 : iter_speed 20.41 seconds per iteration | Loss: 0.1426
362
+ [11-07 15:33:49|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7420 : iter_speed 20.34 seconds per iteration | Loss: 0.1388
363
+ [11-07 15:33:49|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11507 (iteration 7420)
364
+ [11-07 15:34:09|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7421 : iter_speed 20.44 seconds per iteration | Loss: 0.1229
365
+ [11-07 15:34:30|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7422 : iter_speed 20.36 seconds per iteration | Loss: 0.1389
366
+ [11-07 15:34:50|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7423 : iter_speed 20.31 seconds per iteration | Loss: 0.1231
367
+ [11-07 15:35:10|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7424 : iter_speed 20.35 seconds per iteration | Loss: 0.1662
368
+ [11-07 15:35:31|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7425 : iter_speed 20.44 seconds per iteration | Loss: 0.1607
369
+ [11-07 15:35:51|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7426 : iter_speed 20.24 seconds per iteration | Loss: 0.1436
370
+ [11-07 15:36:11|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7427 : iter_speed 20.44 seconds per iteration | Loss: 0.1353
371
+ [11-07 15:36:32|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7428 : iter_speed 20.40 seconds per iteration | Loss: 0.1239
372
+ [11-07 15:36:52|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7429 : iter_speed 20.36 seconds per iteration | Loss: 0.1034
373
+ [11-07 15:37:13|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7430 : iter_speed 20.44 seconds per iteration | Loss: 0.1240
374
+ [11-07 15:37:13|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.10162 (iteration 7430)
375
+ [11-07 15:37:33|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7431 : iter_speed 20.42 seconds per iteration | Loss: 0.1252
376
+ [11-07 15:37:53|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7432 : iter_speed 20.35 seconds per iteration | Loss: 0.1570
377
+ [11-07 15:38:14|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7433 : iter_speed 20.45 seconds per iteration | Loss: 0.1565
378
+ [11-07 15:38:34|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7434 : iter_speed 20.47 seconds per iteration | Loss: 0.1648
379
+ [11-07 15:38:55|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7435 : iter_speed 20.31 seconds per iteration | Loss: 0.1261
380
+ [11-07 15:39:15|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7436 : iter_speed 20.43 seconds per iteration | Loss: 0.1417
381
+ [11-07 15:39:35|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7437 : iter_speed 20.28 seconds per iteration | Loss: 0.1370
382
+ [11-07 15:39:56|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7438 : iter_speed 20.48 seconds per iteration | Loss: 0.1311
383
+ [11-07 15:40:16|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7439 : iter_speed 20.33 seconds per iteration | Loss: 0.1469
384
+ [11-07 15:40:37|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7440 : iter_speed 20.43 seconds per iteration | Loss: 0.1457
385
+ [11-07 15:40:37|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.10775 (iteration 7440)
386
+ [11-07 15:40:57|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7441 : iter_speed 20.33 seconds per iteration | Loss: 0.1748
387
+ [11-07 15:41:17|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7442 : iter_speed 20.45 seconds per iteration | Loss: 0.1856
388
+ [11-07 15:41:38|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7443 : iter_speed 20.33 seconds per iteration | Loss: 0.1458
389
+ [11-07 15:41:58|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7444 : iter_speed 20.47 seconds per iteration | Loss: 0.2180
390
+ [11-07 15:42:18|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7445 : iter_speed 20.34 seconds per iteration | Loss: 0.1791
391
+ [11-07 15:42:39|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7446 : iter_speed 20.43 seconds per iteration | Loss: 0.1740
392
+ [11-07 15:42:59|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7447 : iter_speed 20.20 seconds per iteration | Loss: 0.1714
393
+ [11-07 15:43:19|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7448 : iter_speed 20.41 seconds per iteration | Loss: 0.1660
394
+ [11-07 15:43:40|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7449 : iter_speed 20.32 seconds per iteration | Loss: 0.1232
395
+ [11-07 15:44:00|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7450 : iter_speed 20.42 seconds per iteration | Loss: 0.1388
396
+ [11-07 15:44:00|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.12937 (iteration 7450)
397
+ [11-07 15:44:01|INFO|cosmos_framework/data/omni4d_vgg_training.py:339:__iter__] Omni4D input batch 14900: raw_wait=0.0001s main_collate_wall=0.6172s windows=16 cpu_workers=4 prefetch_factor=2
398
+ [11-07 15:44:21|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7451 : iter_speed 20.31 seconds per iteration | Loss: 0.1336
399
+ [11-07 15:44:41|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7452 : iter_speed 20.42 seconds per iteration | Loss: 0.1364
400
+ [11-07 15:45:01|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7453 : iter_speed 20.27 seconds per iteration | Loss: 0.1420
401
+ [11-07 15:45:22|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7454 : iter_speed 20.38 seconds per iteration | Loss: 0.1340
402
+ [11-07 15:45:42|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7455 : iter_speed 20.32 seconds per iteration | Loss: 0.1443
403
+ [11-07 15:46:02|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7456 : iter_speed 20.42 seconds per iteration | Loss: 0.1376
404
+ [11-07 15:46:23|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7457 : iter_speed 20.35 seconds per iteration | Loss: 0.1736
405
+ [11-07 15:46:43|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7458 : iter_speed 20.41 seconds per iteration | Loss: 0.1518
406
+ [11-07 15:47:03|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7459 : iter_speed 20.39 seconds per iteration | Loss: 0.1567
407
+ [11-07 15:47:24|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7460 : iter_speed 20.38 seconds per iteration | Loss: 0.1366
408
+ [11-07 15:47:24|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11054 (iteration 7460)
409
+ [11-07 15:47:44|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7461 : iter_speed 20.33 seconds per iteration | Loss: 0.2329
410
+ [11-07 15:48:05|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7462 : iter_speed 20.43 seconds per iteration | Loss: 0.1080
411
+ [11-07 15:48:25|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7463 : iter_speed 20.27 seconds per iteration | Loss: 0.1286
412
+ [11-07 15:48:45|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7464 : iter_speed 20.42 seconds per iteration | Loss: 0.1294
413
+ [11-07 15:49:06|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7465 : iter_speed 20.31 seconds per iteration | Loss: 0.1649
414
+ [11-07 15:49:26|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7466 : iter_speed 20.40 seconds per iteration | Loss: 0.1145
415
+ [11-07 15:49:46|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7467 : iter_speed 20.26 seconds per iteration | Loss: 0.1342
416
+ [11-07 15:50:07|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7468 : iter_speed 20.44 seconds per iteration | Loss: 0.1468
417
+ [11-07 15:50:27|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7469 : iter_speed 20.33 seconds per iteration | Loss: 0.1376
418
+ [11-07 15:50:47|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7470 : iter_speed 20.41 seconds per iteration | Loss: 0.1351
419
+ [11-07 15:50:47|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.09567 (iteration 7470)
420
+ [11-07 15:51:08|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7471 : iter_speed 20.26 seconds per iteration | Loss: 0.1806
421
+ [11-07 15:51:28|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7472 : iter_speed 20.41 seconds per iteration | Loss: 0.1619
422
+ [11-07 15:51:49|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7473 : iter_speed 20.36 seconds per iteration | Loss: 0.1287
423
+ [11-07 15:52:09|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7474 : iter_speed 20.36 seconds per iteration | Loss: 0.1387
424
+ [11-07 15:52:29|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7475 : iter_speed 20.26 seconds per iteration | Loss: 0.1605
425
+ [11-07 15:52:50|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7476 : iter_speed 20.38 seconds per iteration | Loss: 0.1213
426
+ [11-07 15:53:10|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7477 : iter_speed 20.30 seconds per iteration | Loss: 0.1364
427
+ [11-07 15:53:30|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7478 : iter_speed 20.44 seconds per iteration | Loss: 0.1062
428
+ [11-07 15:53:51|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7479 : iter_speed 20.37 seconds per iteration | Loss: 0.1651
429
+ [11-07 15:54:11|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7480 : iter_speed 20.42 seconds per iteration | Loss: 0.1232
430
+ [11-07 15:54:11|INFO|cosmos_framework/callbacks/grad_clip.py:436:on_training_step_end] clip_grad_norm/video/global: 0.11009 (iteration 7480)
431
+ [11-07 15:54:31|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7481 : iter_speed 20.26 seconds per iteration | Loss: 0.2515
432
+ [11-07 15:54:52|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7482 : iter_speed 20.42 seconds per iteration | Loss: 0.1719
433
+ [11-07 15:55:12|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7483 : iter_speed 20.42 seconds per iteration | Loss: 0.1217
434
+ [11-07 15:55:32|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7484 : iter_speed 20.32 seconds per iteration | Loss: 0.1192
435
+ [11-07 15:55:53|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7485 : iter_speed 20.24 seconds per iteration | Loss: 0.1413
436
+ [11-07 15:56:13|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7486 : iter_speed 20.44 seconds per iteration | Loss: 0.1248
437
+ [11-07 15:56:34|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7487 : iter_speed 20.39 seconds per iteration | Loss: 0.1615
438
+ [11-07 15:56:54|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7488 : iter_speed 20.37 seconds per iteration | Loss: 0.2003
439
+ [11-07 15:57:14|INFO|cosmos_framework/callbacks/iter_speed.py:769:every_n_impl] 7489 : iter_speed 20.32 seconds per iteration | Loss: 0.2096
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/before_step.txt ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ StepId=73011.103 UserId=1876463298 StartTime=2026-11-05T21:14:17 TimeLimit=UNLIMITED
2
+ State=RUNNING Partition=b200 NodeList=civo-b200-1
3
+ Nodes=1 CPUs=47 Tasks=1 Name=bash Network=(null)
4
+ TRES=cpu=47,gres/gpu:nvidia_b200=2,gres/gpu=2,mem=578000M,node=1
5
+ ResvPorts=(null)
6
+ CPUFreqReq=Default Dist=Cyclic
7
+ SrunHost:Pid=civo-loginnode:3651637
8
+ TresBind=gres/gpu:none
9
+ TresPerStep=cpu:47
10
+ TresPerNode=gres/gpu:2
11
+ TresPerTask=cpu=47,cpu:47
12
+ StdOut=/scratch/shared/engs2907/logs/vgg_forecast_4d/slurm/train/cosmos_runjia_robotwin_action_video_posttrain_20261105/attempt01/slurm-73011-103.out
13
+
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/before_stop_receipt.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "time_utc": "2026-11-07T15:57:30.111691+00:00",
3
+ "authorization": "Owner consolidated instructions 7Nov: may stop B200 Nano posttrain now; no training continuation; A100 evaluation after Stream3R",
4
+ "target": "73011.103",
5
+ "allocation_cancelled": false,
6
+ "latest_durable_overlay": {
7
+ "path": "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_runjia_robotwin_action_video_posttrain_20261105/attempt01/trainable_overlays/step_007400.safetensors",
8
+ "bytes": 2997116456
9
+ },
10
+ "command": [
11
+ "scancel",
12
+ "73011.103"
13
+ ]
14
+ }
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/code_commit.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 6c89d006257fba23c06dbd9a3d7bc6c04683976d
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/direct_launch_authorization.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "decision": "owner requested direct formal training without extra20-update stop or promotion",
3
+ "source": "6c89d006257fba23c06dbd9a3d7bc6c04683976d",
4
+ "admission03_cancelled": "73011.102",
5
+ "formal_start_overlay": 7700,
6
+ "model_updates_before_formal": 0,
7
+ "optional_timing_and_stall_callbacks": false
8
+ }
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/evaluation_lineage.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_update": 7400,
3
+ "clean_train_split_sha256": "6511ed1a88a1047d74fb80dd11b1304f6d77abce6cbe3f56df4f5cf7b9f9ee7c",
4
+ "evaluations": [
5
+ {
6
+ "name": "fixed-four six-stream inference",
7
+ "path": "/scratch/shared/engs2907/logs/vgg_forecast_4d/eval/cosmos_runjia_six_stream_fixed_four_20261107/step_007400_attempt01",
8
+ "scope": "four windows; plumbing only; no quality score"
9
+ },
10
+ {
11
+ "name": "fixed-four visual export",
12
+ "path": "/scratch/shared/engs2907/logs/vgg_forecast_4d/eval/cosmos_runjia_six_stream_fixed_four_20261107/step_007400_visual_export_attempt01",
13
+ "receipt": "visual_export_manifest.json",
14
+ "scope": "12 fully decoded H.264 files; visual only"
15
+ }
16
+ ],
17
+ "quality_claim": "none in this backup manifest",
18
+ "robo_twin_split_manifest_sha256": "10467f314d57acf8deba64b94d0842b0dedcc256acf5f055df92203f7bf10f4e",
19
+ "schema": "cosmos_checkpoint_evaluation_lineage_v1",
20
+ "source_dataset_config_sha256": "8af4041899ffaca9cd6ed185e073861cfe7f04158ff2bf784654fcda0b68a280",
21
+ "training_run": "cosmos_runjia_robotwin_action_video_posttrain_20261105/attempt01",
22
+ "training_source_commit": "6c89d006257fba23c06dbd9a3d7bc6c04683976d",
23
+ "vgg_data_pin_commit": "b9e2059143b4db90a57a4bac7d5dca1a7ee03d48"
24
+ }
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/experiment_module.py ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: OpenMDW-1.1
3
+
4
+ """Continuous RoboTwin post-training with the five geometry streams and Runjia's action-image stream."""
5
+
6
+ from copy import deepcopy
7
+ from pathlib import Path
8
+
9
+ from cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_formal_5k import (
10
+ DINO_CHECKPOINT,
11
+ DINO_CHECKPOINT_SHA256,
12
+ FLEXPI_SOURCE,
13
+ FLEXPI_SOURCE_COMMIT,
14
+ )
15
+ from cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_21_frames_temporal_forcing_plain_ddp_from_8200 import (
16
+ START_OVERLAY_CONFIG_IDENTITY,
17
+ WORLD_SIZE,
18
+ recipe as five_stream_recipe,
19
+ )
20
+ from cosmos_framework.data.omni4d_mixed_sources import build_runjia_robotwin_posttrain_dataset
21
+ from cosmos_framework.data.omni4d_vgg_training import VggStageOneOnlineDinoCollator, VggStageOneOnlineDinoDataLoader
22
+ from cosmos_framework.model.generator.omni4d_native_hf_training import build_pretrained_omni4d_training_model
23
+ from cosmos_framework.utils.config import DDPConfig
24
+ from cosmos_framework.utils.lazy_config import LazyCall as L
25
+ from hydra.core.config_store import ConfigStore
26
+ from omegaconf import MISSING
27
+
28
+
29
+ START_OVERLAY_UPDATE = 7700
30
+ START_OVERLAY = "/scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_full20_b16_restart_20261105/attempt03/input_overlay_step_007700.safetensors"
31
+ START_OVERLAY_SHA256 = "b46067d50dc854c225c1570da2c25691bfad1d98dd97436af6f2c5cf1ec02b9b"
32
+ START_OVERLAY_CONFIG_IDENTITY = (
33
+ "cosmos3_nano_omni4d_stage_one_lora_r256_plain_ddp_21_frames_temporal_forcing_full19_from7300_global32"
34
+ )
35
+ BASE_IDENTITY = "nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92"
36
+ WORLD_SIZE = 2
37
+ MICROBATCH_PER_RANK = 16
38
+ GRAD_ACCUM_ITER = 2
39
+ MAX_UPDATES = 500_000
40
+ WRITER_CONFIG_IDENTITY = "cosmos3_nano_omni4d_runjia_robotwin_action_video_posttrain_33frame_global64"
41
+ OUTPUT = "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_runjia_robotwin_action_video_posttrain_20261105/attempt01"
42
+ SOURCE_CONFIG = Path(__file__).resolve().parents[4] / "configs/omni4d_runjia_action_video_robotwin_posttrain.yaml"
43
+ SOURCE_CONFIG_SHA256 = "8af4041899ffaca9cd6ed185e073861cfe7f04158ff2bf784654fcda0b68a280"
44
+ VGG_REPOSITORY = "/scratch/shared/engs2907/vgg_pins/vgg_forecast_4d_cosmos_pinned_source_20261105_caption_windows"
45
+ VGG_COMMIT = "b9e2059143b4db90a57a4bac7d5dca1a7ee03d48"
46
+
47
+ recipe = deepcopy(five_stream_recipe)
48
+ recipe.job.name = "cosmos3_nano_runjia_robotwin_action_video_posttrain_20261105"
49
+ recipe.model.model_overrides.tokenizer.encode_exact_durations = sorted(
50
+ {*recipe.model.model_overrides.tokenizer.encode_exact_durations, 33}
51
+ )
52
+ recipe.model.model_overrides.omni4d_training_recipe.source_frames = 9
53
+ recipe.model.model_overrides.omni4d_training_recipe.source_policies.trained_streams["action_video"] = True
54
+ recipe.model.model_overrides.omni4d_training_recipe.stream_weights["action_video"] = 1.0
55
+ recipe.model.model_overrides.omni4d_training_recipe.prediction_types["action_video"] = "velocity"
56
+ _modality_forcing = recipe.model.model_overrides.omni4d_training_recipe.forcing_config.modality_forcing
57
+ _modality_forcing.training_mode_shares = {
58
+ "flexible_joint": 1.0, "v2d": 0.0, "v2c": 0.0, "c2v_from_first_frame": 0.0,
59
+ "c2vd_from_first_frame": 0.0, "c2v": 0.0, "c2vd": 0.0, "vd2c": 0.0, "vc2d": 0.0,
60
+ "v2dc": 0.0, "dc2v_from_first_frame": 0.0, "d2vc_from_first_frame": 0.0,
61
+ "d2v_from_first_frame": 0.0, "dc2v": 0.0, "d2vc": 0.0, "d2v": 0.0, "d2c": 0.0,
62
+ "c2d": 0.0, "v2d_dino_generated": 0.0, "v2c_dino_generated": 0.0, "v2dc_dino_generated": 0.0,
63
+ }
64
+ _modality_forcing.draw_rules = {
65
+ "first_frame_anchor": False,
66
+ "absent": {name: 0.0 for name in ("video", "dino", "pointmap", "camera")},
67
+ "whole_given": {name: 0.0 for name in ("video", "dino", "pointmap", "camera")},
68
+ }
69
+ _modality_forcing.video.p_present = 1.0
70
+ _modality_forcing.video.p_joint = 1.0
71
+ _modality_forcing.video.cross_modal = True
72
+ for _name in ("dino", "pointmap", "camera"):
73
+ _modality_forcing[_name].p_present = 0.0
74
+ _modality_forcing[_name].p_joint = 1.0
75
+ _modality_forcing[_name].cross_modal = True
76
+ _tracking_forcing = recipe.model.model_overrides.omni4d_training_recipe.forcing_config.tracking_video
77
+ _tracking_forcing.p_present = 1.0
78
+ _tracking_forcing.p_joint = 1.0
79
+ _tracking_forcing.cross_modal = True
80
+ _tracking_forcing.absent = 0.0
81
+ _tracking_forcing.whole_given = 0.0
82
+ _tracking_forcing.conditional_mode_role = "generated"
83
+ recipe.model.model_overrides.omni4d_training_recipe.forcing_config.temporal_forcing = {
84
+ "whole_window_probability": 1.0,
85
+ "prefix_probability": 0.0,
86
+ "first_last_probability": 0.0,
87
+ "independent_probability": 0.0,
88
+ "structured_geometry_follows_pattern_probability": 0.0,
89
+ "per_latent_frame": True,
90
+ }
91
+ recipe.model.model_overrides.omni4d_stream_input_dims["action_video"] = 192
92
+ recipe.model.model_overrides.omni4d_stream_output_dims["action_video"] = 192
93
+
94
+ recipe.model.config.parallelism.data_parallel_backend = "ddp"
95
+ recipe.model.config.parallelism.data_parallel_shard_degree = 1
96
+ recipe.model.config.parallelism.data_parallel_replicate_degree = WORLD_SIZE
97
+ recipe.model.config.parallelism.context_parallel_shard_degree = 1
98
+ recipe.model.config.parallelism.cfg_parallel_shard_degree = 1
99
+ recipe.trainer.distributed_parallelism = "ddp"
100
+ recipe.trainer.ddp = DDPConfig(
101
+ find_unused_parameters=True,
102
+ static_graph=False,
103
+ broadcast_buffers=False,
104
+ gradient_as_bucket_view=True,
105
+ )
106
+ recipe.trainer.max_iter = MAX_UPDATES
107
+ recipe.trainer.logging_iter = 10
108
+ recipe.trainer.grad_accum_iter = GRAD_ACCUM_ITER
109
+ recipe.optimizer.lr = 1.0e-4
110
+ # Preserve Wan's original 8K warm-up/cosine segment and then keep training at
111
+ # its registered 1e-6 floor. The first 8K updates are unchanged by the longer
112
+ # operational run horizon.
113
+ recipe.scheduler.cycle_lengths = [8000]
114
+ recipe.scheduler.warm_up_steps = [500]
115
+ recipe.scheduler.f_min = [0.01]
116
+ recipe.trainer.callbacks.omni4d_trainable_overlay.output_dir = f"{OUTPUT}/trainable_overlays"
117
+ recipe.trainer.callbacks.omni4d_trainable_overlay.config_identity = WRITER_CONFIG_IDENTITY
118
+ recipe.trainer.callbacks.omni4d_trainable_overlay.save_updates = sorted(
119
+ {8000, *range(100, MAX_UPDATES + 1, 100)}
120
+ )
121
+ recipe.trainer.callbacks.omni4d_trainable_overlay.keep_last = 3
122
+ recipe.trainer.callbacks.omni4d_trainable_overlay.milestone_interval = 1000
123
+ recipe.checkpoint.load_training_state = False
124
+ recipe.checkpoint.only_load_scheduler_state = False
125
+ recipe.checkpoint.model_only_overlay_clock_handoff = False
126
+ recipe.checkpoint.model_only_overlay_start_iteration = None
127
+ recipe.checkpoint.save_iter = MAX_UPDATES + 1
128
+ recipe.model.trainable_overlay_path = MISSING
129
+ recipe.model.trainable_overlay_base_identity = BASE_IDENTITY
130
+ recipe.model.trainable_overlay_config_identity = START_OVERLAY_CONFIG_IDENTITY
131
+ recipe.trainer.profiling.enable_profiling = False
132
+
133
+ recipe.dataloader_train = L(VggStageOneOnlineDinoDataLoader)(
134
+ dataset=L(build_runjia_robotwin_posttrain_dataset)(
135
+ source_config_path=str(SOURCE_CONFIG),
136
+ expected_source_config_sha256=SOURCE_CONFIG_SHA256,
137
+ vgg_repository=VGG_REPOSITORY,
138
+ expected_vgg_commit=VGG_COMMIT,
139
+ flexpi_source=FLEXPI_SOURCE,
140
+ expected_flexpi_commit=FLEXPI_SOURCE_COMMIT,
141
+ rank="${oc.env:RANK}",
142
+ world_size="${oc.env:WORLD_SIZE}",
143
+ batch_size=MICROBATCH_PER_RANK,
144
+ loader_workers=4,
145
+ stream_seed=42,
146
+ ),
147
+ batch_size=MICROBATCH_PER_RANK,
148
+ num_workers=4,
149
+ prefetch_factor=2,
150
+ pin_memory=True,
151
+ drop_last=True,
152
+ main_process_collator=L(VggStageOneOnlineDinoCollator)(
153
+ flexpi_source=FLEXPI_SOURCE,
154
+ expected_flexpi_commit=FLEXPI_SOURCE_COMMIT,
155
+ dino_checkpoint=DINO_CHECKPOINT,
156
+ expected_dino_checkpoint_sha256=DINO_CHECKPOINT_SHA256,
157
+ local_rank="${oc.env:LOCAL_RANK}",
158
+ autocast_dtype="bfloat16",
159
+ ),
160
+ )
161
+
162
+
163
+ def require_launch_ready() -> None:
164
+ import hashlib
165
+
166
+ if hashlib.sha256(SOURCE_CONFIG.read_bytes()).hexdigest() != SOURCE_CONFIG_SHA256:
167
+ raise RuntimeError("the pinned RoboTwin post-training source spec changed")
168
+ if recipe.dataloader_train.batch_size * WORLD_SIZE * GRAD_ACCUM_ITER != 64:
169
+ raise RuntimeError("RoboTwin action-image training must retain Wan's effective batch size 64")
170
+
171
+
172
+ ConfigStore.instance().store(
173
+ group="experiment",
174
+ package="_global_",
175
+ name="omni4d_runjia_action_video_robotwin_posttrain",
176
+ node=recipe,
177
+ )
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/formal_launch.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ import os,subprocess
2
+ clean={k:v for k,v in os.environ.items() if not k.startswith("SLURM_")}
3
+ raise SystemExit(subprocess.run(['srun', '--jobid=73011', '--overlap', '--exact', '--ntasks=1', '--cpus-per-task=47', '--mem=578000M', '--gres=gpu:2', '--gpu-bind=none', '--cpu-bind=none', '--kill-on-bad-exit=1', '--output=/scratch/shared/engs2907/logs/vgg_forecast_4d/slurm/train/cosmos_runjia_robotwin_action_video_posttrain_20261105/attempt01/slurm-%j-%s.out', 'bash', '/scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_robotwin_six_stream_admission_20261105/formal_attempt01/compute.sh'],env=clean).returncode)
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/launch_slurm_step.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 73011.103
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/launch_source_commit.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 6c89d006257fba23c06dbd9a3d7bc6c04683976d
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/launch_source_sha.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 6c89d006257fba23c06dbd9a3d7bc6c04683976d
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/resolved_config.yaml ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run_id: cosmos_runjia_robotwin_action_video_posttrain_update_007400_20261107
2
+ checkpoint: step_007400.safetensors
3
+ config_identity: cosmos3_nano_omni4d_runjia_robotwin_action_video_posttrain_33frame_global64
4
+ source:
5
+ cosmos_training_commit: 6c89d006257fba23c06dbd9a3d7bc6c04683976d
6
+ experiment_module: cosmos_framework.configs.base.experiment.sft.omni4d_runjia_action_video_robotwin_posttrain
7
+ dataset_yaml: source_dataset_config.yaml
8
+ dataset_yaml_sha256: 8af4041899ffaca9cd6ed185e073861cfe7f04158ff2bf784654fcda0b68a280
9
+ vgg_data_pin_commit: b9e2059143b4db90a57a4bac7d5dca1a7ee03d48
10
+ base_model: nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92
11
+ training:
12
+ dataset: RoboTwin clean train split
13
+ source_split_manifest_sha256: 10467f314d57acf8deba64b94d0842b0dedcc256acf5f055df92203f7bf10f4e
14
+ clean_train_split_sha256: 6511ed1a88a1047d74fb80dd11b1304f6d77abce6cbe3f56df4f5cf7b9f9ee7c
15
+ source_frames: 9
16
+ rgb_geometry_stride: 4
17
+ action_video_frames: 33
18
+ streams: [video, pointmap, camera, dino, track_video, action_video]
19
+ precision: bf16
20
+ distributed: ddp
21
+ world_size: 2
22
+ per_rank_batch_size: 16
23
+ gradient_accumulation_steps: 2
24
+ global_batch_size: 64
25
+ learning_rate: 0.0001
26
+ max_updates: 500000
27
+ scheduler: {cycle_lengths: [8000], warm_up_steps: [500], f_min: [0.01]}
28
+ warm_start:
29
+ type: trainable_weights_only
30
+ prior_checkpoint_update: 7700
31
+ prior_checkpoint_sha256: b46067d50dc854c225c1570da2c25691bfad1d98dd97436af6f2c5cf1ec02b9b
32
+ optimizer_state_restored: false
33
+ rng_state_restored: false
34
+ exact_training_resume: false
35
+ checkpoint:
36
+ contains: trainable_parameters_only
37
+ includes_frozen_pretrained_base: false
38
+ includes_optimizer_state: false
39
+ saved_overlay_step: 7400
40
+ last_logged_update_before_owner_stop: 7489
41
+ no_step_7500_artifact: true
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/source_dataset_config.yaml ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Dedicated clean RoboTwin train split for joint geometry-track and Runjia action-image training.
2
+ robotwin:
3
+ name: robotwin
4
+ frames: 9
5
+ stride: 4
6
+ condition_only_streams: []
7
+ reader:
8
+ source_split_manifest: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/flexpi_robotwin_robot_arm_track_video_20261023/split_manifest.json
9
+ source_split_sha256: 10467f314d57acf8deba64b94d0842b0dedcc256acf5f055df92203f7bf10f4e
10
+ pretrained_norm_stats: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/robotwin_action_image_clean_train_filtered_20261102/native_stats/dataset_stats.json
11
+ action_video:
12
+ resolution: 0.5
13
+ stripe_scale: 2.0
14
+ embodiment: /scratch/shared/shug6902/logs/vgg_forecast_4d/cache/robotwin_embodiments_20261021/embodiments/aloha-agilex
15
+ clean_train_split:
16
+ manifest: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/robotwin_action_image_clean_train_filtered_20261102/split_manifest.json
17
+ sha256: 6511ed1a88a1047d74fb80dd11b1304f6d77abce6cbe3f56df4f5cf7b9f9ee7c
18
+ train_episodes: 26288
19
+ held_out_episodes: 150
20
+ excluded_clamped_episodes: 1062
21
+ geometry:
22
+ simulator_ground_truth: true
23
+ log_factor: 0.4
24
+ raymap_camera: {origin_fraction: 0.3333333333333333, origin_range: 1.0, decode_margin: 16, origin_box_grid: 1}
25
+ gauge: frame0_median_depth
26
+ release_root: /datasets/engs2907/FlexPi/robotwin_3d
27
+ calibration_root: /datasets/engs2907/FlexPi/robotwin_3d/calibration
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/stop_command_receipt.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "time_utc": "2026-11-07T15:57:30.111691+00:00",
3
+ "authorization": "Owner consolidated instructions 7Nov: may stop B200 Nano posttrain now; no training continuation; A100 evaluation after Stream3R",
4
+ "target": "73011.103",
5
+ "allocation_cancelled": false,
6
+ "latest_durable_overlay": {
7
+ "path": "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_runjia_robotwin_action_video_posttrain_20261105/attempt01/trainable_overlays/step_007400.safetensors",
8
+ "bytes": 2997116456
9
+ },
10
+ "command": [
11
+ "scancel",
12
+ "73011.103"
13
+ ],
14
+ "scancel_returncode": 0,
15
+ "stdout": "",
16
+ "stderr": "",
17
+ "scientific_success": false,
18
+ "final_state": "pending terminal snapshot"
19
+ }
cosmos_runjia_action_video_posttrain_update_007400_20261107/attempt01/update_007400/training_launcher_source.sh ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ if [[ $# < 2 ]]; then
5
+ echo "usage: $0 <fresh-attempt-root> <two-visible-gpu-uuids> [Hydra override ...]" >&2
6
+ exit 2
7
+ fi
8
+
9
+ attempt_root=$(realpath -m "$1")
10
+ expected_gpus=$2
11
+ shift 2
12
+ repo_root=${COSMOS_SOURCE_ROOT:-$(git rev-parse --show-toplevel)}
13
+ python_bin=/scratch/shared/engs2907/envs/cosmos3-nano-py313-cu128-001fe183/bin/python
14
+ module=cosmos_framework.configs.base.experiment.sft.omni4d_runjia_action_video_robotwin_posttrain
15
+ overlay=/scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_full20_b16_restart_20261105/attempt03/input_overlay_step_007700.safetensors
16
+ overlay_sha256=b46067d50dc854c225c1570da2c25691bfad1d98dd97436af6f2c5cf1ec02b9b
17
+ source_yaml="$repo_root/cosmos_framework/configs/omni4d_runjia_action_video_robotwin_posttrain.yaml"
18
+ source_sha256=8af4041899ffaca9cd6ed185e073861cfe7f04158ff2bf784654fcda0b68a280
19
+
20
+ [[ "$attempt_root" == /scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_runjia_robotwin_action_video_posttrain_20261105/* ]] || {
21
+ echo "attempt root must stay under the dedicated RoboTwin post-training output directory" >&2; exit 10;
22
+ }
23
+ [[ ! -e "$attempt_root/trainable_overlays" ]] || { echo "trainable overlay output already exists: $attempt_root" >&2; exit 11; }
24
+ [[ -x "$python_bin" ]] || { echo "canonical Cosmos Python is unavailable: $python_bin" >&2; exit 12; }
25
+ [[ "$(sha256sum "$overlay" | awk '{print $1}')" == "$overlay_sha256" ]] || {
26
+ echo "warm-start overlay checksum differs from the reviewed 7.7K artifact" >&2; exit 13;
27
+ }
28
+ [[ "$(sha256sum "$source_yaml" | awk '{print $1}')" == "$source_sha256" ]] || {
29
+ echo "RoboTwin source spec differs from the reviewed source" >&2; exit 14;
30
+ }
31
+
32
+ export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:?Slurm must bind exactly the two allocated GPUs}
33
+ [[ "$CUDA_VISIBLE_DEVICES" == "$expected_gpus" ]] || {
34
+ echo "bound GPU UUIDs differ from the reviewed allocation: $CUDA_VISIBLE_DEVICES != $expected_gpus" >&2; exit 15;
35
+ }
36
+ [[ "$CUDA_VISIBLE_DEVICES" == GPU-*,GPU-* ]] || { echo "two GPU UUIDs are required" >&2; exit 16; }
37
+ gpu0=${CUDA_VISIBLE_DEVICES%%,*}
38
+ gpu1=${CUDA_VISIBLE_DEVICES#*,}
39
+ [[ "$gpu0" != "$gpu1" ]] || { echo "GPU UUIDs must be distinct" >&2; exit 17; }
40
+ printf 'host=%s rank_world=2 gpu_uuid_0=%s gpu_uuid_1=%s\n' "$(hostname -s)" "$gpu0" "$gpu1"
41
+
42
+ export COSMOS_TRAINING=1 WANDB_MODE=offline WANDB_DIR="$attempt_root/wandb"
43
+ export PYTHONNOUSERSITE=1 PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
44
+ export HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 TOKENIZERS_PARALLELISM=false
45
+ export IMAGINAIRE_OUTPUT_ROOT=/scratch/shared/engs2907/logs/vgg_forecast_4d/train
46
+ export OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 SYN4D_NPZ_CACHE_SIZE=64
47
+ export PYTHONPATH="$repo_root"
48
+ export PATH="$(dirname "$python_bin"):$PATH"
49
+ torchcodec_library_path=$("$python_bin" "$repo_root/tools/omni4d/cosmos3_nano/prepare_torchcodec_runtime.py" --print-library-path)
50
+ export LD_LIBRARY_PATH="$torchcodec_library_path${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
51
+ ulimit -n 65536
52
+
53
+ cd "$repo_root"
54
+ "$python_bin" -m torch.distributed.run --standalone --nnodes=1 --nproc-per-node=2 --max-restarts=0 \
55
+ -m cosmos_framework.scripts.train_experiment \
56
+ --config=cosmos_framework/configs/omni4d_native_config.py \
57
+ --experiment-module="$module" \
58
+ -- \
59
+ experiment=omni4d_runjia_action_video_robotwin_posttrain \
60
+ model.trainable_overlay_path="$overlay" \
61
+ dataloader_train.dataset.source_config_path="$source_yaml" \
62
+ dataloader_train.dataset.expected_source_config_sha256="$source_sha256" \
63
+ trainer.callbacks.omni4d_trainable_overlay.output_dir="$attempt_root/trainable_overlays" \
64
+ job.name="cosmos3_nano_runjia_robotwin_action_video_posttrain_$(basename "$attempt_root")" \
65
+ "$@"