yslan commited on
Commit
331cd5d
·
verified ·
1 Parent(s): 01f5216

Add immutable checkpoint backup cosmos3_nano_full20_21frame_update_010000_20261106 training_config_source_module

Browse files
cosmos3_nano_stage_one_lora_r256_full20_21frame_ddp_update_010000_20261106/attempt01/update_010000/source_config_module.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: OpenMDW-1.1
3
+
4
+ """Full 500K 21-frame DDP run, with an update-7,320 admission checkpoint."""
5
+
6
+ from copy import deepcopy
7
+ import os
8
+ from pathlib import Path
9
+
10
+ from hydra.core.config_store import ConfigStore
11
+ from omegaconf import MISSING
12
+
13
+ from cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_21_frames_temporal_forcing_plain_ddp_from_8200 import (
14
+ GRAD_ACCUM_ITER,
15
+ MICROBATCH_PER_RANK,
16
+ START_OVERLAY_CONFIG_IDENTITY,
17
+ recipe as formal_recipe,
18
+ )
19
+
20
+
21
+ START_UPDATE = 7300
22
+ ADMISSION_UPDATE = START_UPDATE + 20
23
+ END_UPDATE = 500_000
24
+ START_OVERLAY = (
25
+ "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/"
26
+ "cosmos3_nano_five_stream_stage_one_lora_r256_replicate_only_500k_20261103/attempt01/"
27
+ "trainable_overlays/step_007300.safetensors"
28
+ )
29
+ START_OVERLAY_SHA256 = "9cbbeb3c9e01ca0f04f7ac35d1fd09b1c6f4dd1196bd764f1e75254fc88846a5"
30
+ WRITER_CONFIG_IDENTITY = (
31
+ "cosmos3_nano_omni4d_stage_one_lora_r256_plain_ddp_21_frames_temporal_forcing_"
32
+ "full20_from_overlay_global32_lr2p5e5"
33
+ )
34
+ WINDOWS_YAML = Path(
35
+ os.environ.get(
36
+ "COSMOS_VGG_WINDOWS_CONFIG",
37
+ str(Path(__file__).resolve().parents[4] / "configs/omni4d_vgg_stage_one_next_version_all_sources_21_frames_ddp_global32.yaml"),
38
+ )
39
+ )
40
+ OUTPUT = "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_next_version_full19_21_frames_ddp_from7300_20261105/attempt01"
41
+
42
+ recipe = deepcopy(formal_recipe)
43
+ recipe.job.name = "cosmos3_nano_full19_21_frames_plain_ddp_from7300_20261105"
44
+ recipe.trainer.max_iter = END_UPDATE
45
+ recipe.trainer.callbacks.omni4d_trainable_overlay.output_dir = f"{OUTPUT}/trainable_overlays"
46
+ recipe.trainer.callbacks.omni4d_trainable_overlay.config_identity = WRITER_CONFIG_IDENTITY
47
+ recipe.optimizer.lr = 2.5e-5
48
+ recipe.trainer.callbacks.omni4d_trainable_overlay.save_updates = sorted(
49
+ {ADMISSION_UPDATE, *range(START_UPDATE + 100, END_UPDATE + 1, 100)}
50
+ )
51
+ recipe.trainer.callbacks.omni4d_trainable_overlay.keep_last = 3
52
+ recipe.trainer.callbacks.omni4d_trainable_overlay.milestone_interval = 1000
53
+ recipe.dataloader_train.batch_size = MICROBATCH_PER_RANK
54
+ recipe.dataloader_train.num_workers = 12
55
+ recipe.trainer.grad_accum_iter = GRAD_ACCUM_ITER
56
+ recipe.dataloader_train.dataset.windows_config_path = MISSING
57
+ recipe.dataloader_train.dataset.expected_windows_config_sha256 = MISSING
58
+ recipe.dataloader_train.dataset.expected_vgg_commit = "b9e2059143b4db90a57a4bac7d5dca1a7ee03d48"
59
+ recipe.dataloader_train.dataset.expected_world_size = 2
60
+ recipe.model.trainable_overlay_path = MISSING
61
+ recipe.model.trainable_overlay_config_identity = START_OVERLAY_CONFIG_IDENTITY
62
+ recipe.checkpoint.model_only_overlay_start_iteration = MISSING
63
+ recipe.trainer.profiling.enable_profiling = False
64
+
65
+
66
+ def require_launch_ready() -> None:
67
+ import yaml
68
+
69
+ from cosmos_framework.data.omni4d_next_version_readiness import launch_blockers
70
+
71
+ payload = yaml.safe_load(WINDOWS_YAML.read_text(encoding="utf-8"))
72
+ blockers = launch_blockers(payload)
73
+ if blockers:
74
+ raise RuntimeError("the full 19-source 21-frame smoke is blocked:\n" + "\n".join(f"- {item}" for item in blockers))
75
+
76
+
77
+ ConfigStore.instance().store(
78
+ group="experiment",
79
+ package="_global_",
80
+ name="omni4d_stage_one_lora_r256_next_version_all_sources_21_frames_plain_ddp_from_7300",
81
+ node=recipe,
82
+ )