Add immutable checkpoint backup cosmos3_nano_full20_21frame_update_010000_20261106 training_config_source_module
Browse files
cosmos3_nano_stage_one_lora_r256_full20_21frame_ddp_update_010000_20261106/attempt01/update_010000/source_config_module.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
| 2 |
+
# SPDX-License-Identifier: OpenMDW-1.1
|
| 3 |
+
|
| 4 |
+
"""Full 500K 21-frame DDP run, with an update-7,320 admission checkpoint."""
|
| 5 |
+
|
| 6 |
+
from copy import deepcopy
|
| 7 |
+
import os
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
from hydra.core.config_store import ConfigStore
|
| 11 |
+
from omegaconf import MISSING
|
| 12 |
+
|
| 13 |
+
from cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_21_frames_temporal_forcing_plain_ddp_from_8200 import (
|
| 14 |
+
GRAD_ACCUM_ITER,
|
| 15 |
+
MICROBATCH_PER_RANK,
|
| 16 |
+
START_OVERLAY_CONFIG_IDENTITY,
|
| 17 |
+
recipe as formal_recipe,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
START_UPDATE = 7300
|
| 22 |
+
ADMISSION_UPDATE = START_UPDATE + 20
|
| 23 |
+
END_UPDATE = 500_000
|
| 24 |
+
START_OVERLAY = (
|
| 25 |
+
"/scratch/shared/engs2907/logs/vgg_forecast_4d/train/"
|
| 26 |
+
"cosmos3_nano_five_stream_stage_one_lora_r256_replicate_only_500k_20261103/attempt01/"
|
| 27 |
+
"trainable_overlays/step_007300.safetensors"
|
| 28 |
+
)
|
| 29 |
+
START_OVERLAY_SHA256 = "9cbbeb3c9e01ca0f04f7ac35d1fd09b1c6f4dd1196bd764f1e75254fc88846a5"
|
| 30 |
+
WRITER_CONFIG_IDENTITY = (
|
| 31 |
+
"cosmos3_nano_omni4d_stage_one_lora_r256_plain_ddp_21_frames_temporal_forcing_"
|
| 32 |
+
"full20_from_overlay_global32_lr2p5e5"
|
| 33 |
+
)
|
| 34 |
+
WINDOWS_YAML = Path(
|
| 35 |
+
os.environ.get(
|
| 36 |
+
"COSMOS_VGG_WINDOWS_CONFIG",
|
| 37 |
+
str(Path(__file__).resolve().parents[4] / "configs/omni4d_vgg_stage_one_next_version_all_sources_21_frames_ddp_global32.yaml"),
|
| 38 |
+
)
|
| 39 |
+
)
|
| 40 |
+
OUTPUT = "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_next_version_full19_21_frames_ddp_from7300_20261105/attempt01"
|
| 41 |
+
|
| 42 |
+
recipe = deepcopy(formal_recipe)
|
| 43 |
+
recipe.job.name = "cosmos3_nano_full19_21_frames_plain_ddp_from7300_20261105"
|
| 44 |
+
recipe.trainer.max_iter = END_UPDATE
|
| 45 |
+
recipe.trainer.callbacks.omni4d_trainable_overlay.output_dir = f"{OUTPUT}/trainable_overlays"
|
| 46 |
+
recipe.trainer.callbacks.omni4d_trainable_overlay.config_identity = WRITER_CONFIG_IDENTITY
|
| 47 |
+
recipe.optimizer.lr = 2.5e-5
|
| 48 |
+
recipe.trainer.callbacks.omni4d_trainable_overlay.save_updates = sorted(
|
| 49 |
+
{ADMISSION_UPDATE, *range(START_UPDATE + 100, END_UPDATE + 1, 100)}
|
| 50 |
+
)
|
| 51 |
+
recipe.trainer.callbacks.omni4d_trainable_overlay.keep_last = 3
|
| 52 |
+
recipe.trainer.callbacks.omni4d_trainable_overlay.milestone_interval = 1000
|
| 53 |
+
recipe.dataloader_train.batch_size = MICROBATCH_PER_RANK
|
| 54 |
+
recipe.dataloader_train.num_workers = 12
|
| 55 |
+
recipe.trainer.grad_accum_iter = GRAD_ACCUM_ITER
|
| 56 |
+
recipe.dataloader_train.dataset.windows_config_path = MISSING
|
| 57 |
+
recipe.dataloader_train.dataset.expected_windows_config_sha256 = MISSING
|
| 58 |
+
recipe.dataloader_train.dataset.expected_vgg_commit = "b9e2059143b4db90a57a4bac7d5dca1a7ee03d48"
|
| 59 |
+
recipe.dataloader_train.dataset.expected_world_size = 2
|
| 60 |
+
recipe.model.trainable_overlay_path = MISSING
|
| 61 |
+
recipe.model.trainable_overlay_config_identity = START_OVERLAY_CONFIG_IDENTITY
|
| 62 |
+
recipe.checkpoint.model_only_overlay_start_iteration = MISSING
|
| 63 |
+
recipe.trainer.profiling.enable_profiling = False
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def require_launch_ready() -> None:
|
| 67 |
+
import yaml
|
| 68 |
+
|
| 69 |
+
from cosmos_framework.data.omni4d_next_version_readiness import launch_blockers
|
| 70 |
+
|
| 71 |
+
payload = yaml.safe_load(WINDOWS_YAML.read_text(encoding="utf-8"))
|
| 72 |
+
blockers = launch_blockers(payload)
|
| 73 |
+
if blockers:
|
| 74 |
+
raise RuntimeError("the full 19-source 21-frame smoke is blocked:\n" + "\n".join(f"- {item}" for item in blockers))
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
ConfigStore.instance().store(
|
| 78 |
+
group="experiment",
|
| 79 |
+
package="_global_",
|
| 80 |
+
name="omni4d_stage_one_lora_r256_next_version_all_sources_21_frames_plain_ddp_from_7300",
|
| 81 |
+
node=recipe,
|
| 82 |
+
)
|