Back up checkpoint lineage metadata cosmos_nano_update_014700_20261107

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/backup_context.json

@@ -0,0 +1,17 @@
+{
+  "base_model": "nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92",
+  "checkpoint_bytes": 2990807208,
+  "checkpoint_path": "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_next_version_cityscapes_molmo_22_sources_main_process_loader_20261106/attempt02/trainable_overlays/step_014700.safetensors",
+  "checkpoint_role": "trainable_checkpoint_only",
+  "checkpoint_sha256": "ae92b4b891df0a6f44e22027139f48f8f6537a43f78a78f92c54a589853209fb",
+  "checkpoint_update": 14700,
+  "frozen_base_included": false,
+  "optimizer_state_included": false,
+  "resolved_config_provenance": "No original resolved Hydra dump was present in the run artifacts. resolved_config.yaml is a derived recipe summary. Exact source module, launcher, restart.env, restart plan, source windows, 11500 parent receipt and evaluation lineage are included.",
+  "resolved_hydra_dump_available": false,
+  "run_id": "cosmos_nano_full20_21frame_update_014700_20261107",
+  "schema": "cosmos_trainable_overlay_hf_backup_context_v1",
+  "source_code_commit": "97f885005227cf23bbfb33156f87751db9f14a17",
+  "warm_start_sha256": "989dca07ce763dece176e60a6b55f960a69e73ea1791d3cb01869612f1e23727",
+  "warm_start_update": 11500
+}

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/backup_manifest.json

@@ -0,0 +1,91 @@
+{
+  "checkpoint_bytes": 2990807208,
+  "checkpoint_path": "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_next_version_cityscapes_molmo_22_sources_main_process_loader_20261106/attempt02/trainable_overlays/step_014700.safetensors",
+  "checkpoint_role": "trainable_checkpoint_only",
+  "checkpoint_sha256": "ae92b4b891df0a6f44e22027139f48f8f6537a43f78a78f92c54a589853209fb",
+  "frozen_base_included": false,
+  "metadata_files": [
+    {
+      "bytes": 1204,
+      "path": "backup_context.json",
+      "role": "lineage_metadata",
+      "sha256": "55fe93ca5fa5b9b71196c08899669aad15af8c4b3245371d9d4073462e958e57"
+    },
+    {
+      "bytes": 41,
+      "path": "code_commit.txt",
+      "role": "lineage_metadata",
+      "sha256": "cc953883e6b6b896556689d3d84b55f14af180fbbb66fdc92ccbeec9e8de628b"
+    },
+    {
+      "bytes": 1324,
+      "path": "evaluation_lineage.json",
+      "role": "lineage_metadata",
+      "sha256": "01b40226216646055627cadc4a7b30ae27c9ed97f06c237ae5675823987c7b1e"
+    },
+    {
+      "bytes": 4440,
+      "path": "experiment_module.py",
+      "role": "lineage_metadata",
+      "sha256": "3664abc86b0ba16e7969cd80f339e3abf8f351fd6570288c11ac237ea43e1c79"
+    },
+    {
+      "bytes": 38946,
+      "path": "input_step_011500_receipt.json",
+      "role": "lineage_metadata",
+      "sha256": "a520d263321c55cc2b88230c695c4a696a53cbcfdc2629c84b70c13f52b0cca5"
+    },
+    {
+      "bytes": 1085,
+      "path": "launch.sh",
+      "role": "lineage_metadata",
+      "sha256": "602cdee1bc6a69551ab76b1d4cd2d5c15e4f08465420cc28a80365fff1f17000"
+    },
+    {
+      "bytes": 1490,
+      "path": "recipe_summary.yaml",
+      "role": "lineage_metadata",
+      "sha256": "0aeab55d544ad81d2f859e6f529586c0802d618f99fd6669562b790bb0e6718a"
+    },
+    {
+      "bytes": 1419,
+      "path": "restart.env",
+      "role": "lineage_metadata",
+      "sha256": "8f679cbb9e46da510fb8b9934666d346702e884dc7d7f9e301f45e251936c619"
+    },
+    {
+      "bytes": 1105,
+      "path": "restart_plan.json",
+      "role": "lineage_metadata",
+      "sha256": "d48f0cc440904c2422f6b7909dc363d6b4fe1a21916dd47db4307c6513c72745"
+    },
+    {
+      "bytes": 41,
+      "path": "run_source_sha.txt",
+      "role": "lineage_metadata",
+      "sha256": "cc953883e6b6b896556689d3d84b55f14af180fbbb66fdc92ccbeec9e8de628b"
+    },
+    {
+      "bytes": 42099,
+      "path": "source_windows.yaml",
+      "role": "lineage_metadata",
+      "sha256": "73c90b72adc27e1589f97bff64cb35ace5d03acf7fdac43a3e5a5c86055178d5"
+    },
+    {
+      "bytes": 42099,
+      "path": "source_windows_as_recorded.yaml",
+      "role": "lineage_metadata",
+      "sha256": "73c90b72adc27e1589f97bff64cb35ace5d03acf7fdac43a3e5a5c86055178d5"
+    },
+    {
+      "bytes": 32978,
+      "path": "training_launcher_source.sh",
+      "role": "lineage_metadata",
+      "sha256": "abc48951e0a04d1c5ab48d5d8c32d2b0f54a10df01a4289885e14c3f95280e9e"
+    }
+  ],
+  "optimizer_state_included": false,
+  "remote_prefix": "cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700",
+  "repository": "yslan/o-intermediate-model",
+  "schema": "vgg_trainable_checkpoint_backup_manifest_v1"
+}

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/code_commit.txt

@@ -0,0 +1 @@
+97f885005227cf23bbfb33156f87751db9f14a17

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/evaluation_lineage.json

@@ -0,0 +1,23 @@
+{
+  "checkpoint_is_trainable_overlay_only": true,
+  "checkpoint_update": 14700,
+  "dataset_windows_path": "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_next_version_cityscapes_molmo_22_sources_main_process_loader_20261106/attempt02/source_windows.yaml",
+  "dataset_windows_sha256": "73c90b72adc27e1589f97bff64cb35ace5d03acf7fdac43a3e5a5c86055178d5",
+  "evaluations": [
+    {
+      "name": "22-source static full cohort update14700",
+      "path": "/scratch/shared/engs2907/logs/vgg_forecast_4d/eval/cosmos_nano_static_full_cohort_update14700_20261107/attempt02",
+      "scope": "static depth/geometry; see source result files for exact row/status counts"
+    },
+    {
+      "name": "saved-layout temporal and 9-frame evaluations",
+      "path": "/scratch/shared/engs2907/logs/vgg_forecast_4d/eval/cosmos_22source_every_overlay_20261106/step_014700",
+      "scope": "distinct legacy-caption/empty-caption protocols; do not merge their results"
+    }
+  ],
+  "quality_claim": "none in this backup manifest",
+  "schema": "cosmos_checkpoint_evaluation_lineage_v1",
+  "training_run": "cosmos_next_version_cityscapes_molmo_22_sources_main_process_loader_20261106/attempt02",
+  "training_source_commit": "97f885005227cf23bbfb33156f87751db9f14a17",
+  "vgg_data_pin_commit": "f6992c4b67441dc9778642a252447a2687856116"
+}

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/experiment_module.py

@@ -0,0 +1,80 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Opt-in B16 continuation of the 21-frame mixture with Cityscapes and eligible MolmoMotion data."""
+
+import hashlib
+import os
+from copy import deepcopy
+from pathlib import Path
+
+from hydra.core.config_store import ConfigStore
+
+from cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_21_frames_plain_ddp_from_7300 import (
+    END_UPDATE,
+    OUTPUT as RUNNING_OUTPUT,
+    WRITER_CONFIG_IDENTITY as RUNNING_WRITER_CONFIG_IDENTITY,
+    recipe as running_recipe,
+)
+from cosmos_framework.inference.omni4d_overlay_config_identities import (
+    CITYSCAPES_AND_MOLMO_FROM_OVERLAY_CONFIG_IDENTITY,
+    TWENTY_ONE_FRAME_FROM_OVERLAY_LR_2P5E5_CONFIG_IDENTITY,
+)
+
+DEFAULT_WINDOWS_YAML = (
+    Path(__file__).resolve().parents[4]
+    / "configs/omni4d_vgg_stage_one_next_version_all_sources_plus_cityscapes_and_molmo_motion_21_frames_ddp_global32_b16_gas1.yaml"
+)
+WINDOWS_YAML = Path(os.environ.get("COSMOS_VGG_WINDOWS_CONFIG", str(DEFAULT_WINDOWS_YAML)))
+OUTPUT = "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_next_version_full20_plus_cityscapes_and_molmo_motion_21_frames_ddp_from_explicit_overlay_20261106/attempt01"
+WRITER_CONFIG_IDENTITY = CITYSCAPES_AND_MOLMO_FROM_OVERLAY_CONFIG_IDENTITY
+if OUTPUT == RUNNING_OUTPUT or WRITER_CONFIG_IDENTITY == RUNNING_WRITER_CONFIG_IDENTITY:
+    raise ValueError("the combined real-data variant requires its own output directory and overlay identity")
+
+recipe = deepcopy(running_recipe)
+recipe.dataloader_train.dataset.windows_config_path = str(WINDOWS_YAML)
+recipe.dataloader_train.dataset.expected_windows_config_sha256 = hashlib.sha256(
+    WINDOWS_YAML.read_bytes()
+).hexdigest()
+recipe.dataloader_train.dataset.expected_vgg_commit = "f6992c4b67441dc9778642a252447a2687856116"
+recipe.model.trainable_overlay_config_identity = TWENTY_ONE_FRAME_FROM_OVERLAY_LR_2P5E5_CONFIG_IDENTITY
+recipe.job.name = "cosmos3_nano_full20_plus_cityscapes_and_molmo_motion_21_frames_plain_ddp_from_explicit_overlay_20261106"
+recipe.dataloader_train.batch_size = 16
+recipe.trainer.grad_accum_iter = 1
+recipe.trainer.callbacks.omni4d_trainable_overlay.output_dir = f"{OUTPUT}/trainable_overlays"
+recipe.trainer.callbacks.omni4d_trainable_overlay.config_identity = WRITER_CONFIG_IDENTITY
+recipe.trainer.callbacks.omni4d_trainable_overlay.save_updates = list(range(100, END_UPDATE + 1, 100))
+
+
+def require_launch_ready() -> None:
+    """Require both new sources, their pinned readback/caption products and exact mixture arithmetic."""
+    import yaml
+
+    from cosmos_framework.data.omni4d_next_version_readiness import check_mixture_arithmetic, launch_blockers
+
+    payload = yaml.safe_load(WINDOWS_YAML.read_text(encoding="utf-8"))
+    city = [item for item in payload.get("real_video_sources", []) if item.get("name") == "cityscapes"]
+    molmo = payload.get("molmo_motion_sources", [])
+    if len(city) != 1 or float(city[0].get("weight", -1)) != 4.0 or int(city[0].get("frames", -1)) != 21:
+        raise RuntimeError("the combined recipe needs one 21-frame Cityscapes source at weight 4")
+    if len(molmo) != 1 or molmo[0].get("name") != "molmo_motion" or float(molmo[0].get("weight", -1)) != 6.0:
+        raise RuntimeError("the combined recipe needs one MolmoMotion source at weight 6")
+    reader = molmo[0].get("reader", {})
+    if set(reader.get("subsets", [])) != {"egodex", "xperience", "stereo4d"}:
+        raise RuntimeError("the combined recipe may use only eligible EgoDex, Xperience and Stereo4D records")
+    if set(reader.get("processed_release_only_subsets", [])) != {"egodex"}:
+        raise RuntimeError("EgoDex must be restricted to processed release MP4s; raw RGB is not aligned/eligible")
+    if float(payload["mixture_policy"]["total_weight"]) != 143.0:
+        raise RuntimeError("the 22-source combined recipe must have total weight 143")
+    check_mixture_arithmetic(payload)
+    blockers = launch_blockers(payload)
+    if blockers:
+        raise RuntimeError("the combined 21-frame data mix is not ready to launch:\n" + "\n".join(f"- {v}" for v in blockers))
+
+
+ConfigStore.instance().store(
+    group="experiment",
+    package="_global_",
+    name="omni4d_stage_one_lora_r256_next_version_all_sources_plus_cityscapes_and_molmo_motion_21_frames_plain_ddp_from_explicit_overlay",
+    node=recipe,
+)

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/input_step_011500_receipt.json

@@ -0,0 +1,15 @@
+{
+  "source": "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_next_version_full20_21_frames_ddp_b16_gas1_lr_2p5e5_20261105/attempt01/trainable_overlays/step_011500.safetensors",
+  "protected_path": "/scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_cityscapes_molmo_production_restart_20261106/attempt01/input_step_011500.safetensors",
+  "bytes": 2990807176,
+  "sha256": "989dca07ce763dece176e60a6b55f960a69e73ea1791d3cb01869612f1e23727",
+  "metadata": {
+    "format": "omni4d_trainable_weights_v1",
+    "base_identity": "nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92",
+    "update": "11500",
+    "config_identity": "cosmos3_nano_omni4d_stage_one_lora_r256_plain_ddp_21_frames_temporal_forcing_full20_from_overlay_global32_lr2p5e5",
+    "parameter_names": "[\"language_model.model.layers.0.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.0.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.0.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.0.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.0.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.0.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.0.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.0.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.0.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.0.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.0.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.0.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.0.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.0.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.1.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.1.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.1.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.1.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.1.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.1.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.1.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.1.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.1.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.1.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.1.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.1.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.1.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.1.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.2.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.2.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.2.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.2.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.2.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.2.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.2.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.2.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.2.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.2.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.2.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.2.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.2.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.2.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.3.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.3.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.3.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.3.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.3.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.3.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.3.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.3.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.3.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.3.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.3.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.3.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.3.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.3.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.4.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.4.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.4.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.4.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.4.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.4.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.4.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.4.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.4.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.4.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.4.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.4.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.4.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.4.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.5.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.5.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.5.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.5.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.5.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.5.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.5.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.5.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.5.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.5.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.5.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.5.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.5.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.5.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.6.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.6.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.6.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.6.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.6.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.6.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.6.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.6.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.6.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.6.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.6.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.6.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.6.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.6.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.7.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.7.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.7.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.7.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.7.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.7.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.7.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.7.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.7.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.7.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.7.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.7.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.7.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.7.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.8.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.8.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.8.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.8.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.8.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.8.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.8.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.8.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.8.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.8.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.8.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.8.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.8.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.8.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.9.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.9.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.9.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.9.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.9.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.9.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.9.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.9.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.9.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.9.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.9.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.9.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.9.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.9.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.10.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.10.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.10.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.10.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.10.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.10.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.10.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.10.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.10.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.10.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.10.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.10.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.10.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.10.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.11.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.11.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.11.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.11.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.11.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.11.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.11.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.11.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.11.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.11.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.11.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.11.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.11.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.11.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.12.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.12.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.12.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.12.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.12.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.12.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.12.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.12.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.12.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.12.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.12.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.12.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.12.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.12.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.13.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.13.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.13.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.13.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.13.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.13.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.13.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.13.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.13.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.13.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.13.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.13.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.13.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.13.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.14.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.14.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.14.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.14.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.14.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.14.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.14.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.14.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.14.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.14.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.14.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.14.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.14.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.14.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.15.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.15.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.15.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.15.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.15.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.15.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.15.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.15.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.15.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.15.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.15.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.15.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.15.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.15.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.16.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.16.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.16.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.16.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.16.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.16.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.16.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.16.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.16.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.16.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.16.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.16.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.16.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.16.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.17.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.17.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.17.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.17.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.17.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.17.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.17.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.17.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.17.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.17.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.17.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.17.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.17.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.17.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.18.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.18.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.18.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.18.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.18.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.18.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.18.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.18.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.18.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.18.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.18.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.18.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.18.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.18.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.19.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.19.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.19.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.19.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.19.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.19.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.19.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.19.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.19.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.19.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.19.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.19.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.19.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.19.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.20.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.20.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.20.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.20.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.20.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.20.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.20.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.20.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.20.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.20.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.20.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.20.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.20.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.20.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.21.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.21.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.21.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.21.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.21.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.21.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.21.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.21.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.21.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.21.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.21.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.21.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.21.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.21.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.22.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.22.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.22.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.22.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.22.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.22.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.22.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.22.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.22.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.22.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.22.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.22.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.22.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.22.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.23.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.23.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.23.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.23.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.23.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.23.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.23.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.23.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.23.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.23.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.23.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.23.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.23.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.23.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.24.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.24.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.24.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.24.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.24.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.24.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.24.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.24.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.24.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.24.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.24.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.24.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.24.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.24.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.25.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.25.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.25.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.25.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.25.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.25.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.25.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.25.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.25.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.25.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.25.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.25.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.25.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.25.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.26.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.26.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.26.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.26.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.26.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.26.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.26.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.26.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.26.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.26.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.26.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.26.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.26.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.26.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.27.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.27.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.27.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.27.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.27.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.27.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.27.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.27.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.27.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.27.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.27.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.27.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.27.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.27.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.28.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.28.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.28.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.28.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.28.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.28.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.28.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.28.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.28.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.28.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.28.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.28.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.28.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.28.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.29.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.29.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.29.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.29.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.29.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.29.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.29.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.29.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.29.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.29.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.29.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.29.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.29.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.29.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.30.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.30.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.30.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.30.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.30.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.30.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.30.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.30.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.30.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.30.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.30.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.30.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.30.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.30.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.31.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.31.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.31.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.31.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.31.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.31.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.31.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.31.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.31.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.31.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.31.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.31.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.31.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.31.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.32.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.32.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.32.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.32.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.32.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.32.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.32.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.32.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.32.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.32.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.32.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.32.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.32.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.32.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.33.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.33.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.33.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.33.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.33.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.33.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.33.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.33.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.33.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.33.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.33.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.33.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.33.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.33.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.34.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.34.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.34.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.34.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.34.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.34.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.34.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.34.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.34.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.34.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.34.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.34.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.34.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.34.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.35.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.35.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.35.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.35.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.35.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.35.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.35.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.35.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.35.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.35.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.35.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.35.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.35.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.35.mlp_moe_gen.down_proj.lora_B.weight\", \"time_embedder.mlp.0.weight\", \"time_embedder.mlp.0.bias\", \"time_embedder.mlp.2.weight\", \"time_embedder.mlp.2.bias\", \"omni4d_stream_projector.input.video.weight\", \"omni4d_stream_projector.input.video.bias\", \"omni4d_stream_projector.input.pointmap.weight\", \"omni4d_stream_projector.input.pointmap.bias\", \"omni4d_stream_projector.input.camera.weight\", \"omni4d_stream_projector.input.camera.bias\", \"omni4d_stream_projector.input.dino.weight\", \"omni4d_stream_projector.input.dino.bias\", \"omni4d_stream_projector.input.track_video.weight\", \"omni4d_stream_projector.input.track_video.bias\", \"omni4d_stream_projector.dino_feature_norm.weight\", \"omni4d_stream_projector.dino_feature_norm.bias\", \"omni4d_stream_projector.output.video.weight\", \"omni4d_stream_projector.output.video.bias\", \"omni4d_stream_projector.output.pointmap.weight\", \"omni4d_stream_projector.output.pointmap.bias\", \"omni4d_stream_projector.output.camera.weight\", \"omni4d_stream_projector.output.camera.bias\", \"omni4d_stream_projector.output.dino.weight\", \"omni4d_stream_projector.output.dino.bias\", \"omni4d_stream_projector.output.track_video.weight\", \"omni4d_stream_projector.output.track_video.bias\"]",
+    "exact_training_resume": "false"
+  },
+  "tensor_count": 530
+}

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/launch.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+stage=/scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_cityscapes_molmo_production_restart_20261106/attempt02
+source "$stage/restart.env"
+[[ "$(git -C "$COSMOS_SOURCE_ROOT" rev-parse HEAD)" == "$COSMOS_SOURCE_COMMIT" ]]
+git -C "$COSMOS_SOURCE_ROOT" diff --quiet HEAD --
+export COSMOS_ALLOCATION_ID=72961 COSMOS_EXPECTED_NODE=civo-b200-1
+export COSMOS_ALLOCATED_GPU_UUIDS=GPU-7899c40c-79a9-ead3-59d5-f5d7db271f64,GPU-41b7d2ea-0eeb-0e84-4b04-b2baddede118
+export COSMOS_MICROBATCH_PROFILE=b16_gas1 COSMOS_TWENTY_UPDATE_ADMISSION_TEST=0
+export COSMOS_PYTHON=/scratch/shared/engs2907/envs/cosmos3-nano-py313-cu128-001fe183/bin/python COSMOS_NATIVE_STALL_DIAGNOSTICS=1
+export COSMOS_COMPUTE_PYTHON="$COSMOS_PYTHON"
+export COSMOS_VGG_WORKER_STACK_DIR="$COSMOS_ATTEMPT_ROOT/worker_stacks"
+export COSMOS_VGG_WORKER_STACK_SOURCE_IDENTITY="${COSMOS_SOURCE_COMMIT}_windows_${COSMOS_VGG_WINDOWS_CONFIG_SHA256}_overlay_${COSMOS_TRAINABLE_OVERLAY_UPDATE}"
+exec bash "$COSMOS_SOURCE_ROOT/tools/omni4d/cosmos3_nano/run_formal_five_stream_5k_on_b200.sh"

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/recipe_summary.yaml

@@ -0,0 +1,34 @@
+run_id: cosmos_nano_full20_21frame_update_014700_20261107
+checkpoint: step_014700.safetensors
+config_identity: cosmos3_nano_omni4d_stage_one_lora_r256_plain_ddp_21_frames_temporal_forcing_full20_plus_cityscapes_and_molmo_motion_from_overlay_global32_lr2p5e5
+source:
+  cosmos_training_commit: 97f885005227cf23bbfb33156f87751db9f14a17
+  experiment_module: cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_plus_cityscapes_and_molmo_motion_21_frames_plain_ddp_from_explicit_overlay
+  windows_yaml: source_windows.yaml
+  windows_sha256: 73c90b72adc27e1589f97bff64cb35ace5d03acf7fdac43a3e5a5c86055178d5
+  vgg_data_pin_commit: f6992c4b67441dc9778642a252447a2687856116
+base_model: nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92
+training:
+  source_windows: 22
+  frames: 21
+  streams: [video, pointmap, camera, dino, track_video]
+  precision: bf16
+  distributed: native_ddp
+  world_size: 2
+  per_rank_batch_size: 16
+  gradient_accumulation_steps: 1
+  global_batch_size: 32
+  learning_rate: 0.000025
+  max_updates: 500000
+  warm_start:
+    type: trainable_weights_only
+    update: 11500
+    checkpoint_sha256: 989dca07ce763dece176e60a6b55f960a69e73ea1791d3cb01869612f1e23727
+    optimizer_state_restored: false
+    rng_state_restored: false
+  exact_training_resume: false
+checkpoint:
+  contains: trainable_lora_and_projector_parameters_only
+  includes_frozen_pretrained_base: false
+  includes_optimizer_state: false
+  output_update: 14700

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/restart.env

@@ -0,0 +1,11 @@
+export COSMOS_SOURCE_ROOT=/scratch/shared/engs2907/cosmos_city_molmo_production_97f885005_20261106
+export COSMOS_SOURCE_COMMIT=97f885005227cf23bbfb33156f87751db9f14a17
+export COSMOS_EXPERIMENT_MODULE=cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_plus_cityscapes_and_molmo_motion_21_frames_plain_ddp_from_explicit_overlay
+export COSMOS_ATTEMPT_ROOT=/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos_next_version_cityscapes_molmo_22_sources_main_process_loader_20261106/attempt02
+export COSMOS_VGG_WINDOWS_CONFIG=/scratch/shared/engs2907/cosmos_city_molmo_production_97f885005_20261106/cosmos_framework/configs/omni4d_vgg_stage_one_next_version_all_sources_plus_cityscapes_and_molmo_motion_21_frames_ddp_global32_b16_gas1.yaml
+export COSMOS_VGG_WINDOWS_CONFIG_SHA256=73c90b72adc27e1589f97bff64cb35ace5d03acf7fdac43a3e5a5c86055178d5
+export COSMOS_TRAINABLE_OVERLAY_PATH=/scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_cityscapes_molmo_production_restart_20261106/attempt01/input_step_011500.safetensors
+export COSMOS_TRAINABLE_OVERLAY_SHA256=989dca07ce763dece176e60a6b55f960a69e73ea1791d3cb01869612f1e23727
+export COSMOS_TRAINABLE_OVERLAY_UPDATE=11500
+export COSMOS_TRAINABLE_OVERLAY_CONFIG_IDENTITY=cosmos3_nano_omni4d_stage_one_lora_r256_plain_ddp_21_frames_temporal_forcing_full20_from_overlay_global32_lr2p5e5
+export COSMOS_TIMING_GATE_UPDATES=50

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/restart_plan.json

@@ -0,0 +1,24 @@
+{
+  "at_utc": "2026-11-06T17:39:02.174967+00:00",
+  "source": "/scratch/shared/engs2907/cosmos_city_molmo_production_97f885005_20261106",
+  "sha": "97f885005227cf23bbfb33156f87751db9f14a17",
+  "input_receipt": "/scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_cityscapes_molmo_production_restart_20261106/attempt01/input_step_011500_receipt.json",
+  "supersedes_invalid_step": "72961.15",
+  "dataset_constructor": {
+    "sample_offset_updates": 11500,
+    "batch_size": 16,
+    "gradient_accumulation_steps": 1
+  },
+  "weight_only_fresh_optimizer": true,
+  "acceptance_updates": 50,
+  "status": "submitted_initializing",
+  "test_commit": "8ec3746cf841e5d1e36a2a078152ca9b12541d40",
+  "launcher_code_commit": "b20856aba51d1ef9592637dc0747b3e7ebd0ab23",
+  "tests": {
+    "static_launcher": "1 passed,4 deselected,40.52s",
+    "executable_config_composition": "1 passed,39.64s"
+  },
+  "test_artifacts": "/scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_overlay_data_offset_alignment_20261106/attempt01",
+  "slurm_step": "72961.16",
+  "submitted_utc": "2026-11-06T17:43:06.667772+00:00"
+}

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/run_source_sha.txt

@@ -0,0 +1 @@
+97f885005227cf23bbfb33156f87751db9f14a17

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/source_windows.yaml

@@ -0,0 +1,834 @@
+# B16 continuation of the full 20-source mixture with native-resolution Cityscapes captions.
+# This is a separate model-only run from an explicit five-stream overlay: 22 sources, total weight 143.
+# Cityscapes train captions are converted from the completed native2048 product; val remains evaluation-only.
+# Next Cosmos stage-one version: the owner's weighted mixture at 21 frames (decision 2026-11-05).
+#
+# Weights, not shares. Total 143 over 22 logical sources:
+#   synthetic   79  (59.4%)  the eleven synthetic sources at the 20,000-update Wan run's weights, unchanged:
+#                            syn4d 15.2, omniworld_game 8.75, synthverse 11.05, vkitti 6, tartanair 6, dynamic_replica 5,
+#                            spring 8, midair 3, mvs_synth 3, tartanground 3, paralleldomain4d 10
+#   real video  26  (18.2%)  ScanNet 8, DL3DV 6, WildRGB-D 4, Waymo 4, Cityscapes 4 (geometry is input only)
+#   robot       26  (19.5%)  RoboTwin 10 (clean-train episodes only), DROID 10 (depth and camera as inputs only),
+#                            LIBERO 6 (official training demonstrations only); no action stream in stage one
+#   tracks       6  ( 4.2%)  PointOdyssey 3, Kubric 3
+#   human video  6  ( 4.2%)  MolmoMotion; DINO and sparse tracking video are targets, camera is input-only, depth absent
+# DINO is a generated target on the real-video sources (ScanNet, DL3DV, WildRGB-D, DROID, Waymo): it is computed from the
+# RGB, so it is exact on real data, and the future-feature forecasts the DINO-Foresight and VFMF comparisons make would
+# otherwise never train on real video. This changes the first launch's roles: the owner's rule is that geometry is
+# condition-only on real data, and that rule still holds for depth and camera, which stay inputs only here.
+# Waymo trains on the processed training split only (waymo.training.pickle.gz, SHA-256 6abd3476...: 798 sequences,
+# 158,081 frames; validation never enters training) through the pinned sparse-track reader's waymo stanza: sparse LiDAR
+# depth, the camera and the RGB; no tracking video; strides 1-4 at 21 frames.
+# No segmentation stream, no Cityscapes, no YAM.
+#
+# The VGG window block below is read by the pinned VGG checkout (vgg_commit) through VggStageOneWindowDataset: the
+# synthetic sources are the recorded 500K run's stanzas (omni4d_vgg_sparse_tracking_synthetic_continuation_sources.yaml,
+# SHA-256 ae49ec6d...) with the window length 21 and missing_caption: error. RoboTwin and LIBERO are read through
+# robot_sources (RobotEpisodeStageOneDataset) because the pinned VGG checkout has no Track4D reader for them.
+# The tracking-video query draw keeps the recorded 10-pixel grid (no query_count_range).
+#
+# Train membership of the real-video sources (official split files, 2026-11-05):
+#   DL3DV      the pinned reader ignores `split` and indexes every processed scene, 85 of which are official
+#              DL3DV-Benchmark (test) scenes. The 141 official benchmark scene hashes (Hugging Face
+#              DL3DV/DL3DV-Benchmark, dataset revision 9684e838) are in omni4d_dl3dv_official_benchmark_scene_hashes.txt;
+#              the 87 of them present in the processed 1K-7K root are added to the stanza's exclude_sequence_ids below
+#              (source_reader_overrides), leaving about 5,898 scenes.
+#   WildRGB-D  the pinned reader reads <category>/selected_seqs_train.json; those 2,244 processed train scenes are all in
+#              the official train-of-both lists (nvs_list.json and camera_eval_list.json of hongchi/wildrgbd, CUT3R's
+#              get_set_list rule) and overlap the official NVS and camera-evaluation validation scenes in none.
+#
+# launch_blockers names every evidence gap that stops the launch. A blocked source keeps its weight; composition
+# (omni4d_native_config.make_config) and the window dataset both refuse to start while the list is not empty. A robot
+# source additionally needs a passing real-window readback receipt (readback_receipt). WildRGB-D and Kubric train on
+# caption text converted by tools/omni4d/convert_stage_one_caption_texts.py (text-only products: no umT5 cache exists).
+#
+# VGG pin: the 500K run's f5a59cd51 plus the window-reader updates, processed-release EgoDex support, and the
+# native2048 Cityscapes caption-reader migration at f6992c4b. The approved raw EgoDex window and Cityscapes caption
+# product readbacks used this exact pin. City captions point to the completed 2026-11-06 converted text product below.
+# One makes the DROID and LIBERO robot window length the data node's num_frames: the pinned window class and the LIBERO
+# painter fixed the clock at 33 rows in five places, one a literal inside the painter's constructor; a project adapter
+# could only rebind module globals and fake attributes, so the smallest safe change is in the VGG source, with the
+# recorded 33-row nodes reading as before. Two DROID commits (cherry-picked from the VGG main stack, f5a59cd is their
+# ancestor) keep a window whose FK arm has no visible frame-0 query, with its source motion absent, instead of raising.
+# The fourth adds the stanza key position_windows (src/vggforecast/data/position_windows.py): each listed position range is
+# one window and the stride and start are drawn inside it. ScanNet and PointOdyssey use it (source_reader_overrides,
+# caption_windows: true) with the local caption windows of their products, so every window they draw lies inside one
+# caption window and the strict caption join never redraws. The fifth adds a waymo stanza to the stage-one real-source config
+# (no source uses it yet: every real source leaves it out). The pin is a separate checkout (a git worktree of the earlier pin
+# directory, which a running training still reads at 9cb6a626a), so the earlier pin's files never change under a run.
+vgg_repository: /scratch/shared/engs2907/vgg_pins/vgg_forecast_4d_cosmos_source_20261106_f6992c4
+vgg_commit: f6992c4b67441dc9778642a252447a2687856116
+recipe_source_config: /scratch/shared/engs2907/logs/vgg_forecast_4d/train/flexpi_four_d_pretraining_wan22_5b_separate_streams_full_parameters_video_depth_camera_dino_and_sparse_tracking_video_from_the_15000_update_four_d_run_20261101/attempt02/run/config.yaml
+recipe_source_config_sha256: 72590f4598cae1b329c31b3cb45d059bdaeca23642faa10ca3f31372f9ebc0aa
+window_block:
+  _target_: four_d_windows.FourDPretrainingWindows
+  sources:
+  - name: four_d_pretraining_relaunch_syn4d_sources
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_rendered_depth_sources_21_frames_512
+    weight: 15.2
+    left_out_sources:
+    - omniworld_game
+    - tartanair
+    - mvs_synth
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_omniworld_game_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_rendered_depth_sources_21_frames_512
+    weight: 8.75
+    left_out_sources:
+    - syn4d_v1
+    - syn4d_v2_mixed
+    - syn4d_v2_mixed_no_sim
+    - tartanair
+    - mvs_synth
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_synthverse_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_synthverse_dynamic_replica_vkitti_midair_spring_sources_without_midair_quality_masks_21_frames_512
+    weight: 11.05
+    left_out_sources:
+    - dynamic_replica
+    - vkitti
+    - midair
+    - spring
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_vkitti_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_synthverse_dynamic_replica_vkitti_midair_spring_sources_without_midair_quality_masks_21_frames_512
+    weight: 6.0
+    left_out_sources:
+    - synthverse_depth
+    - dynamic_replica
+    - midair
+    - spring
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_tartanair_sky_then_far_percentile_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 6.0
+    left_out_sources:
+    - dynamic_replica
+    - spring
+    - midair
+    - mvs_synth
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_dynamic_replica_stored_depth_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 5.0
+    left_out_sources:
+    - tartanair
+    - spring
+    - midair
+    - mvs_synth
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_spring_stored_depth_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 8.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - midair
+    - mvs_synth
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_midair_within_200_meters_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 3.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - spring
+    - mvs_synth
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_mvs_synth_1000_then_far_percentile_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 3.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - spring
+    - midair
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_tartanground_sky_then_far_percentile_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 3.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - spring
+    - midair
+    - mvs_synth
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_paralleldomain4d_sky_then_far_percentile_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 10.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - spring
+    - midair
+    - mvs_synth
+    - tartanground
+    condition_only_streams: []
+  - name: scannet
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 8.0
+    left_out_sources:
+    - dl3dv
+    - wildrgbd
+    - pointodyssey_train
+    - kubric_train
+    - droid
+    - waymo
+    condition_only_streams:
+    - pointmap
+    - camera
+  - name: dl3dv
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 6.0
+    left_out_sources:
+    - scannet
+    - wildrgbd
+    - pointodyssey_train
+    - kubric_train
+    - droid
+    - waymo
+    condition_only_streams:
+    - pointmap
+    - camera
+  - name: wildrgbd
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 4.0
+    left_out_sources:
+    - dl3dv
+    - scannet
+    - pointodyssey_train
+    - kubric_train
+    - droid
+    - waymo
+    condition_only_streams:
+    - pointmap
+    - camera
+  - name: waymo
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 4.0
+    left_out_sources:
+    - dl3dv
+    - scannet
+    - wildrgbd
+    - pointodyssey_train
+    - kubric_train
+    - droid
+    condition_only_streams:
+    - pointmap
+    - camera
+  - name: droid
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 10.0
+    left_out_sources:
+    - dl3dv
+    - scannet
+    - wildrgbd
+    - pointodyssey_train
+    - kubric_train
+    - waymo
+    condition_only_streams:
+    - pointmap
+    - camera
+  - name: pointodyssey_train
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 3.0
+    left_out_sources:
+    - dl3dv
+    - scannet
+    - wildrgbd
+    - kubric_train
+    - droid
+    - waymo
+    condition_only_streams: []
+  - name: kubric_train
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 3.0
+    left_out_sources:
+    - dl3dv
+    - scannet
+    - wildrgbd
+    - pointodyssey_train
+    - droid
+    - waymo
+    condition_only_streams:
+    - pointmap
+  frames: 21
+  raster_hw:
+  - 288
+  - 512
+  raymap_camera:
+    origin_fraction: 0.3333333333333333
+    origin_range: 1.0
+    decode_margin: 16
+    origin_box_grid: 1
+  pointmap_log_factor: 0.4
+  pointmap_hole_fill:
+    synthverse_depth: far_clip
+    syn4d: far_clip
+    omniworld_game: far_clip
+    dynamic_replica: nearest_labelled_depth_of_3x3_neighbours
+    tartanair: far_clip
+    dl3dv: far_clip
+    scannet: telea_log_depth
+    virtual_kitti2: far_clip
+    midair: far_clip
+    pointodyssey: far_clip
+    mvs_synth: far_clip
+    wildrgbd: telea_log_depth
+    hypersim: far_clip
+    spring: far_clip
+    kubric49: far_clip
+    kubric81: far_clip
+    droid_stereo_depth: telea_log_depth
+    tartanground: far_clip
+    paralleldomain4d: far_clip
+    syn4d_v2: far_clip
+    waymo: far_clip
+  pointmap_gauge: window_labelled_median_depth
+  geometry_image: genception_log_depth
+  context_len: 512
+  batch_size: 16
+  stream_seed: 42
+  epoch_windows: 32000000
+  val_set_proportion: 0.0
+  resume_state: null
+  gradient_accumulation_steps: 1
+  loader_workers: 12
+  loader_opencv_threads: 1
+  window_captions:
+    caption_dropout: 0.1
+    missing_caption: error
+    caches:
+      syn4d_v1_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v1_gemini_holo4d_local_and_global_with_repaired_globals_20261031/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v1_gemini_holo4d_local_and_global_with_repaired_globals_20261031/captions_t5_emb_meta.json
+      syn4d_v2_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v2_gemini_holo4d_20261030/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v2_gemini_holo4d_20261030/captions_t5_emb_meta.json
+      tartanair_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanair_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanair_gemini_holo4d/captions_t5_emb_meta.json
+      mvs_synth_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/mvs_synth_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/mvs_synth_gemini_holo4d/captions_t5_emb_meta.json
+      omniworld_game_gemini_local_official_text_chunks_and_gemini_global:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/omniworld_game_gemini_local_official_text_chunks_and_gemini_global_20261030b/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/omniworld_game_gemini_local_official_text_chunks_and_gemini_global_20261030b/captions_t5_emb_meta.json
+      synthverse_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/synthverse_gemini_holo4d_local96_and_global_20261031/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/synthverse_gemini_holo4d_local96_and_global_20261031/captions_t5_emb_meta.json
+      dynamic_replica_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/dynamic_replica_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/dynamic_replica_gemini_holo4d/captions_t5_emb_meta.json
+      vkitti_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/vkitti_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/vkitti_gemini_holo4d/captions_t5_emb_meta.json
+      midair_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/midair_gemini_holo4d_20261030/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/midair_gemini_holo4d_20261030/captions_t5_emb_meta.json
+      spring_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/spring_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/spring_gemini_holo4d/captions_t5_emb_meta.json
+      tartanground_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanground_gemini_holo4d_20261030/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanground_gemini_holo4d_20261030/captions_t5_emb_meta.json
+      paralleldomain4d_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/paralleldomain4d_gemini_holo4d_20261030/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/paralleldomain4d_gemini_holo4d_20261030/captions_t5_emb_meta.json
+      scannet_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/.scannet_gemini_holo4d.version-tovvjwai/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/.scannet_gemini_holo4d.version-tovvjwai/captions_t5_emb_meta.json
+      pointodyssey_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/.pointodyssey_gemini_holo4d.version-6vfyvsbi/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/.pointodyssey_gemini_holo4d.version-6vfyvsbi/captions_t5_emb_meta.json
+      droid_vp_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/.droid_vp_gemini_holo4d.version-e51r5urf/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/.droid_vp_gemini_holo4d.version-e51r5urf/captions_t5_emb_meta.json
+      dl3dv_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/dl3dv_gemini_holo4d_20261031/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/dl3dv_gemini_holo4d_20261031/captions_t5_emb_meta.json
+      wildrgbd_text_captions:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261105/wildrgbd_text_captions/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261105/wildrgbd_text_captions/captions_t5_emb_meta.json
+      kubric49_text_captions:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261105/kubric49_text_captions/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261105/kubric49_text_captions/captions_t5_emb_meta.json
+      waymo_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261105/waymo_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261105/waymo_gemini_holo4d/captions_t5_emb_meta.json
+    stanzas:
+      syn4d_v1:
+        product: syn4d_v1_gemini_holo4d
+        join: local_covering_then_global
+      syn4d_v2_mixed:
+        product: syn4d_v2_gemini_holo4d
+        join: local_covering_then_global
+      syn4d_v2_mixed_no_sim:
+        product: syn4d_v2_gemini_holo4d
+        join: local_covering_then_global
+      tartanair:
+        product: tartanair_gemini_holo4d
+        join: local_covering
+      mvs_synth:
+        product: mvs_synth_gemini_holo4d
+        join: local_covering
+      omniworld_game:
+        product: omniworld_game_gemini_local_official_text_chunks_and_gemini_global
+        join: local_covering_then_second_local_covering_then_global
+      pointodyssey_test: none
+      kubric49_validation: none
+      wildrgbd_test: none
+      vkitti_test:
+        product: vkitti_gemini_holo4d
+        join: local_covering_then_global
+      droid_held_out: none
+      synthverse_depth:
+        product: synthverse_gemini_holo4d
+        join: local_covering_then_global
+      dynamic_replica:
+        product: dynamic_replica_gemini_holo4d
+        join: global
+      vkitti:
+        product: vkitti_gemini_holo4d
+        join: local_covering_then_global
+      midair:
+        product: midair_gemini_holo4d
+        join: local_covering_then_global
+      spring:
+        product: spring_gemini_holo4d
+        join: global
+      tartanground:
+        product: tartanground_gemini_holo4d
+        join: local_covering_then_global
+      paralleldomain4d:
+        product: paralleldomain4d_gemini_holo4d
+        join: global
+      paralleldomain4d_validation:
+        product: paralleldomain4d_gemini_holo4d
+        join: global
+      scannet:
+        product: scannet_gemini_holo4d
+        join: local_containing_then_global
+      pointodyssey_train:
+        product: pointodyssey_gemini_holo4d
+        join: local_containing_then_global
+      droid:
+        product: droid_vp_gemini_holo4d
+        join: global
+      dl3dv:
+        product: dl3dv_gemini_holo4d
+        join: local_containing_then_global
+      wildrgbd:
+        product: wildrgbd_text_captions
+        join: local_containing_then_global
+      kubric_train:
+        product: kubric49_text_captions
+        join: global_required
+      waymo:
+        product: waymo_gemini_holo4d
+        join: local_containing_then_global
+  training_scene_exclusions:
+    reason: rendered depth passes through glass (raw EXR depth through glass panes at 190-260 m), owner 10-29; evidence /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/syn4d_glass_depth_check_20261029;
+      scifiroom_bald added 10-30 from /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/depth_clipping_audit_20261030
+    stanzas:
+      syn4d_v1:
+      - bigoffice_v1
+      - lab_bald
+      - office_bald
+      - scifiroom_bald
+      syn4d_v2_mixed:
+      - citynight
+      - mall
+      - metro
+      syn4d_v2_mixed_no_sim:
+      - citynight
+      - mall
+      - metro
+  tracking_video:
+    query_grid_px: 10
+    disc_radius_px: 2
+    occluded_style: ring
+    ring_radius_px: 3
+    ring_width_px: 1
+    xyz_normalization: quantile_0.02_0.98
+    colour_floor: 0.2
+    min_frame_zero_points: 16
+    surface_test:
+      relative_tolerance: 0.1
+      absolute_tolerance: 0.05
+    stanza_rules:
+      tartanair: static_scene_points
+      midair: static_scene_points
+      mvs_synth: static_scene_points
+      tartanground: static_scene_points
+      syn4d_v1: tracked_points
+      syn4d_v2_mixed: tracked_points
+      syn4d_v2_mixed_no_sim: tracked_points
+      omniworld_game: no_tracking_video
+      synthverse_depth: no_tracking_video
+      dynamic_replica: tracked_points
+      vkitti: no_tracking_video
+      spring: no_tracking_video
+      paralleldomain4d: no_tracking_video
+      scannet: static_scene_points
+      wildrgbd_test: static_scene_points
+      pointodyssey_test: tracked_points
+      kubric49_validation: no_tracking_video
+      vkitti_test: no_tracking_video
+      droid_held_out: no_tracking_video
+      paralleldomain4d_validation: no_tracking_video
+      dynamic_replica_test: tracked_points
+      pointodyssey_train: tracked_points
+      kubric_train: tracked_points
+      droid: tracked_points
+      dl3dv: static_scene_points
+      wildrgbd: static_scene_points
+      waymo: no_tracking_video
+    stanza_track_reading:
+      syn4d_v1:
+        stanza:
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+          annotation_provenance: syn4d_mesh_surface_forward_track_from_frame0
+          allow_missing_tracks: false
+        kwargs:
+          return_tracks: true
+          depth_only: false
+          metadata_root: /vgg/chenzhao/syn4d_metadata
+          wrapped_face_distance_thresh: 0.1
+          query_inconsistent_distance_thresh: 0.1
+      pointodyssey_test:
+        stanza:
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+      dynamic_replica_test:
+        stanza:
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+      syn4d_v2_mixed:
+        loader: syn4d_v2
+        stanza:
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+          annotation_provenance: syn4d_v2_mesh_surface_forward_track_from_frame0
+          allow_missing_tracks: false
+        kwargs:
+          return_tracks: true
+          depth_only: false
+          metadata_root: /vgg/chenzhao/syn4d_metadata
+          tracking_format: safetensor
+          mask_bg: false
+          trusted_background_depth_max: 80.0
+          wrapped_face_distance_thresh: 0.1
+          query_inconsistent_distance_thresh: 0.1
+      syn4d_v2_mixed_no_sim:
+        loader: syn4d_v2
+        stanza:
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+          annotation_provenance: syn4d_v2_mesh_surface_forward_track_from_frame0
+          allow_missing_tracks: false
+        kwargs:
+          return_tracks: true
+          depth_only: false
+          metadata_root: /vgg/chenzhao/syn4d_metadata
+          tracking_format: safetensor
+          mask_bg: false
+          trusted_background_depth_max: 80.0
+          wrapped_face_distance_thresh: 0.1
+          query_inconsistent_distance_thresh: 0.1
+      dynamic_replica:
+        stanza:
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+      pointodyssey_train:
+        stanza:
+          allow_missing_tracks: false
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+      kubric_train:
+        stanza:
+          allow_missing_tracks: false
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+      droid:
+        stanza:
+          allow_missing_tracks: false
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+  source_reader_overrides:
+    dl3dv:
+      stanza: dl3dv
+      exclude_sequence_ids:
+      - 1K/0a1b7c20a92c43c6b8954b1ac909fb2f0fa8b2997b80604bc8bbec80a1cb2da3
+      - 1K/14eb48a50e37df548894ab6d8cd628a21dae14bbe6c462e894616fc5962e6c49
+      - 1K/1da888bdedfc9629c0fa9f82cf3f5d96f8103baee0ff64d9311aea1224a9f2ae
+      - 1K/2991a75d1f0fb2b7a891d424fc2459bb954b61599b52e74a056a7acfb2998b77
+      - 1K/2cbfe28643b6636f9c70813cae7625aa858a352109493ac70fb429ce94dd55b3
+      - 1K/5a27c00f525f5484b7aa0d0107051253bf7bb395bde0b3a4486ff8c52fb4965e
+      - 1K/6d221625614d0515fa44d59a8797a49b64f86bc775e66be22f07a66aa0a0cb3a
+      - 1K/917e9c8985d353b0ee4c281f11fa84eb8550814562670cbb82cd0ec9c1194fe7
+      - 1K/918c8dad730c3b804306c5da8486124be4aa0612e85fb825338fd350c912e1b0
+      - 1K/a17a984ca90a9b5840fdf85b15104b0d18e25975981c1aa90fcdfd6eeeb285f3
+      - 1K/a726c1112ad026efc92f59f196c0698e67abd12a04135a7ff6349b1d31b089eb
+      - 1K/adb95f29c15b8a89e40c17d10596679be09d7719c6f08a453facd506d056ea8b
+      - 1K/d1b3a0b37acc72207295b05e8c359f8b43bf879eb882c60747cfc6d3c7d8efa3
+      - 1K/eb4cf52988f805e6fce11d1b239fa9de32eb157364cff06ebac0aa50e0a46567
+      - 2K/032dee9fb0a8bc1b90871dc5fe950080d0bcd3caf166447f44e60ca50ac04ec7
+      - 2K/0853979305f7ecb80bd8fc2c8df916410d471ef04ed5f1a64e9651baa41d7695
+      - 2K/093ef327b4e4f9d4ee52c02a354a53558a8652157fb0d58f3b4a708734afb334
+      - 2K/0f8ac521439691fe429e9efbed8d5ded1cee35bcf52d47731fb60d5a2ff661a7
+      - 2K/1d6a9ed47cce39fd1c4d18f776bcc97e507b81bde921ca596bd91b0b02b5e414
+      - 2K/26fd23358fa11fff0fb3180ef0b65591b486e20dcf753ce4a7aae49a37e370c7
+      - 2K/280abf7bd93b81b077af1db638229dbb09869052fa0b7b57c81c94d2db893829
+      - 2K/2beaca318994c25409dcbb6d0bdd96c3620f2f18aec44ea3f20edd302f18ca78
+      - 2K/341b4ff3dfd3d377d7167bd81f443bedafbff003bf04881b99760fc0aeb69510
+      - 2K/374ffd0c5ff8e544f588f1da3ce0d5b69dc39ff3af95d7ccec480fe5e533c45e
+      - 2K/41036716da7efda334c1d434c4141d15642e0e02f881a01b6c8c36f8bea64c45
+      - 2K/484c0aca40c1087f196d2caf3c8cb463864c08e60b98326ca02eb6dae8ce4984
+      - 2K/4ff8650b5c1140e98d957fbd1abbe2cfbbb416b37cb9f06f7f376956703801bb
+      - 2K/513e4ea2e8477b06f2b32417d5c243aec71d491cb0596a60e9fe304c635f20a1
+      - 2K/8b9fb9d9f10e8c64d5034be69809465753b8ba88ef12da82afd47d38ee789934
+      - 2K/91afb9910b042f7185c2b8e4b6b24b5785dae4542617b3f8005b5492f6d123f7
+      - 2K/ba55c875d20c34ee85ffc72264c4d77710852e5fb7d9ce4b9c26a8442850e98f
+      - 2K/c076929db6501cf7ebe386c70e6d77ea3af844a745e794f2ec17c981c465a69b
+      - 2K/ceb252f5d4419510655cf9ed7afbf3e8e688825f798d80414c7715dc8ace153a
+      - 2K/df29c225863d0173e060b1f3fde4e6eb5a57a4e4edabd397026ba1a028aeb39e
+      - 2K/ff592398657b3dfe94153332861985194a3e3c9d199c4a3a27a0ce4038e81ade
+      - 3K/35872363e17af5d173b6a0b09fcf5de94627ad5dc5f8a9ad4c579f3e70b4797a
+      - 3K/387eeb925ba47af5db9ce6102aaf6b07e6d3573b7e7568dda4cef22d5ab1342f
+      - 3K/3bb894d1933f3081134ad2d40e54de5f0636bd8b502b0a8561873bb63b0dce85
+      - 3K/5c3af581028068a3c402c7cbe16ecf9471ddf2897c34ab634b7b1b6cf81aba00
+      - 3K/669c36225b273e22ce08364f166dce910b708d7abd89e58463a15d2790b534d5
+      - 3K/66fd66cbed40a2d11ecd17ae6d6d60cd60d476616864ecdd351fe69e644f30a1
+      - 3K/70eac6ff18a1daae4eeccc5eb18723eaf7e029a77e79c79d9573f7bac59ba92b
+      - 3K/75fbbe467353cfdeb93bc1872144855576797be83a0c71c51155ee6b355d6d5e
+      - 3K/800cf88687a79c232700454c2f5a402e771bc7157037a17b0ce93da365b25575
+      - 3K/8cb2e97d26a639f05a571476240a8fa86988e6853f0f13cc05830d1578002aad
+      - 3K/9cbc5548643ca9cfe797f871bac58bccd02f0510c49b7cb3697b38dfd780a508
+      - 3K/a62f9a1c6362e7c8a3440261ea0921a0a4b60cf5822bd19579836694f26b3abd
+      - 3K/adf35184a12d4cfa3f4248b87aa5adb4f39f179df460d6d76136e13d37299a2a
+      - 3K/af0d7039e64af3959210c235e59425896ac064f4e0f721c437bbd2cf1d8ea910
+      - 3K/b3bf9079b442c681749f6bd95cfb0659c6d844f2228b2f4fa4b602a0e3627b9a
+      - 3K/b6d1134cb03c855d246f916beaa2d3d01290cf47937a5176011ff39b850ec615
+      - 3K/d8de66037bd03dd0d39d54f9978bac3318d912e30e22c21e1ada82a98ed48c53
+      - 3K/ddfdcfdf02d53dd89a055cd9e1ed30272ab3f57c7560ac2c3761bb01d76a7882
+      - 3K/e8ce51b6abfe05bf8dca47e29c8be6c1e6de27a8c9fece7a121400b931b2ca0f
+      - 3K/e9360e7a89bee835dc847cf8796093e634b759ff582558788dcfe8326f6e8901
+      - 3K/f7aaea9ac683bb1945cfeb1daa9abca3375249e0d19281a89a3f575040835464
+      - 4K/56452d9cd9e2e034764132a57058517ab085e1352f9664d2d1e10e98741334fc
+      - 4K/6d81c5ab0d480fd43d78b75ff372a8113ad38e2c03f1d69627c009883054d4c2
+      - 4K/9641a1ed7963ce5ca734cff3e6ccea3dfa8bcb0b0a3ff78f65d32a080de2d71e
+      - 4K/d3812aad538261e7f73c75762ff55f23b468bcc76f376d52ac86ca6cf3c44b4b
+      - 5K/0a485338bbdaf19ba9090b874bb36ef0599a9c9a475a382c22903cf5981c6ea6
+      - 5K/119fd56d3797e2d349ca64ddcc5851463cd13b5974b5b2e4566ed5cf7e02e6c1
+      - 5K/c37726ce770ac50a2cf5c0f43022f0268e26da0d777cd8e3a3418c4eed03fd94
+      - 5K/dac9796dd69e1c25277e29d7e30af4f21e3b575d62a0a208c2b3d1211e2d5d77
+      - 6K/06da796666297fe4c683c231edf56ec00148a6a52ab5bb159fe1be31f53a58df
+      - 6K/7a9f97660be8f2421b37dd413c898360532f17c6447c11515e8aded2d80eb99d
+      - 6K/7da3db99059a436adf537bc23309a6a4db9e011696d086f3a64037f7497d9df7
+      - 6K/e5684b3292bfd77db297839fc37ee4cce7fd59775af1a6a4827e3b4f59c036d3
+      - 6K/ec1e44d4dc0f8fa77610866495f9297a7f82158c43e1777668b84fd4b736c7bc
+      - 6K/ef59aac437132bfc1dd45a7e1f8e4800978e7bb28bf98c4428d26fb3e1da3e90
+      - 6K/f477ffc4b398bed8e0d921f0fba9825ca63f317381c535c84be23be991ae1d7a
+      - 7K/0569e83fdc248a51fc0ab082ce5e2baff15755c53c207f545e6d02d91f01d166
+      - 7K/073f5a9b983ced6fb28b23051260558b165f328a16b2d33fe20585b7ee4ad561
+      - 7K/183dd248f6a86e07c5adf9de8ee2d0abe45b1216331c03678e89634c2e9b1c7f
+      - 7K/1ba74c22670ad047981441581d00f26f4a148d1010bcac7468c615adf5fa4d5d
+      - 7K/389a460ca1995e0658e85fe8e6b520b4e88b370cd6710dfe728b1564bba31aee
+      - 7K/493816813d2d6d248eb3c2b0b77b63e54235266e9a06e270fd0d282f13960493
+      - 7K/4ae797d07b6d1644c9db6919c8cc8c0d28d72be45108ac7a3abf8dc21b599d83
+      - 7K/50c46cf8b8b22c8d2ffdef8964b05ddbceaef312c9a9ff331d1ecebfd223f72a
+      - 7K/565553aa894be621e8b4773cac288e60ad0c2cf7edb621be62b348c9a0f78380
+      - 7K/599ca3e04cae3ec83affc426af7d0d7ab36eb91cd8e539edbc13070a4d455792
+      - 7K/5c8dafad7d782c76ffad8c14e9e1244ce2b83aa12324c54a3cc10176964acf04
+      - 7K/63798f5c6fbfcb4eb686268248b8ecbc8d87d920b2bcce967eeaedfd3b3b6d82
+      - 7K/946f49be73928469000baa5ca04d2573137c5ee6a66362bcf8d130354dca8924
+      - 7K/9e9a89ae6fed06d6e2f4749b4b0059f35ca97f848cedc4a14345999e746f7884
+      - 7K/cd9c981eeb4a9091547af19181b382698e9d9eee0a838c7c9783a8a268af6aee
+      - 7K/d4fbeba0168af8fddb2fc695881787aedcd62f477c7dcec9ebca7b8594bbd95b
+    scannet:
+      stanza: scannet
+      caption_windows: true
+    pointodyssey_train:
+      stanza: pointodyssey_train
+      caption_windows: true
+robot_sources:
+- name: robotwin
+  weight: 10.0
+  frames: 21
+  stride: 4
+  # Simulator ground truth: depth, camera and the arm tracks are generated targets (owner decision 2026-11-05); DINO too.
+  condition_only_streams: []
+  reader:
+    source_split_manifest: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/flexpi_robotwin_robot_arm_track_video_20261023/split_manifest.json
+    source_split_sha256: 10467f314d57acf8deba64b94d0842b0dedcc256acf5f055df92203f7bf10f4e
+    pretrained_norm_stats: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/robotwin_action_image_clean_train_filtered_20261102/native_stats/dataset_stats.json
+  clean_train_split:
+    manifest: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/robotwin_action_image_clean_train_filtered_20261102/split_manifest.json
+    sha256: 6511ed1a88a1047d74fb80dd11b1304f6d77abce6cbe3f56df4f5cf7b9f9ee7c
+    train_episodes: 26288
+    held_out_episodes: 150
+    excluded_clamped_episodes: 1062
+  geometry:
+    simulator_ground_truth: true
+    log_factor: 0.4
+    raymap_camera: {origin_fraction: 0.3333333333333333, origin_range: 1.0, decode_margin: 16, origin_box_grid: 1}
+    gauge: window_labelled_median_depth
+    release_root: /datasets/engs2907/FlexPi/robotwin_3d
+    calibration_root: /datasets/engs2907/FlexPi/robotwin_3d/calibration
+  # Two real clean-train windows at 21 frames through reader, DINO and the native batch adapter, on pin f6992c4b.
+  readback_receipt:
+    path: /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_city_molmo_pin_migration_20261106/attempt05/robot_sources/robotwin_readback.json
+    sha256: 08c29011548725201118a245372365c0fd2430eafa6ee4b2844d2cf2fdca59ff
+- name: libero
+  weight: 6.0
+  frames: 21
+  stride: 4
+  condition_only_streams: []
+  reader:
+    hydra_overrides: []
+  geometry:
+    simulator_ground_truth: true
+  readback_receipt:
+    path: /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_city_molmo_pin_migration_20261106/attempt05/robot_sources/libero_readback.json
+    sha256: 5dd74fbd926bc0b7aac7e27da43ae55c1eaa9cb049b9c4656ba27eac71bdfedc
+real_video_sources:
+- name: cityscapes
+  weight: 4.0
+  frames: 21
+  stride: 1
+  condition_only_streams:
+  - pointmap
+  reader:
+    data_root: /scratch/shared/engs2907/datasets/cityscapes
+    train_manifest: /vgg/yushi/dataset/Cityscapes/prepared/cityscapes_train_driving_sequence_v1.jsonl
+    train_manifest_sha256: 9dd2abd964d52c752401b2aef6baa400ec52268143da8b8c623cb059b8f70475
+    train_snippets: 2975
+    train_frames: 89250
+    val_manifest: /vgg/yushi/dataset/Cityscapes/prepared/cityscapes_val_driving_sequence_v1.jsonl
+    val_manifest_sha256: 7b6fb5cc52bd4eac0a61874ee14b47988481a6a484ed73f8ca4c959bc1aa5adc
+  geometry:
+    hole_fill: telea_log_depth
+  captions:
+    product: cityscapes_gemini_holo4d
+    complete_marker: /vgg/yushi/dataset/caption_output/gemini_holo4d/native2048_v1/cityscapes/COMPLETE.json
+    texts: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261106/cityscapes_gemini_holo4d
+molmo_motion_sources:
+- name: molmo_motion
+  weight: 6.0
+  frames: 21
+  # The camera is an input only, like every real source's camera; there is no depth. DINO and the tracking video are targets.
+  condition_only_streams: [camera]
+  reader:
+    config: train/mot/wan_camera_omega_video_depth_vae_and_sparse_future_prediction_molmo_motion_21_frames_512
+    stanza: molmo_motion
+    release_root: /vgg/yushi/dataset/MolmoMotion-1M
+    subsets: [egodex, xperience, stereo4d]
+    # molmo-motion's sqrt-frequency mixing (p_i proportional to sqrt N_i, README "Training"), N_i the drawable records of
+    # the subset at this window length measured by the readback (census of 2026-11-05).
+    subset_weights: {egodex: 133.6, xperience: 432.4, stereo4d: 210.9}
+    temporal_stride: {egodex: 2, xperience: 1, stereo4d: 4}
+    processed_release_only_subsets: [egodex]
+    # Only the already reconstructed release MP4 subset is eligible. Never fall back to newly downloaded raw 30 fps
+    # EgoDex videos: their RGB raster has not been validated against the 15 fps track/camera annotations.
+    processed_release_only_subsets: [egodex]
+    # The pinned config leaves Xperience's `object` records out: their 3D was lifted with the 256-pixel camera matrix.
+    excluded_objects: {xperience: [object]}
+    max_window_redraws: 64
+  exclusions:
+    # The registered 120-record test cohort (the VGG held-out spec molmo_motion_test_120_records_droid_egodex_molmospaces_
+    # 49_frames_512; tools/omni4d/write_molmo_motion_test_cohort_manifest.py). The release's own test splits (DROID's too) and
+    # PointMotionBench (hot3d, worldtrack, davis) are asserted disjoint from the train records in code.
+    test_cohort_manifest:
+      path: /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_molmo_motion_20261105/cohort/molmo_motion_test_cohort_120_records.json
+      sha256: d1f8e8d8ad8f8f67c8bfd23319cdea026ea8cfcba1194b5f4594366e9f64fd76
+    point_motion_bench_root: /scratch/shared/engs2907/PointMotionBench
+  readback_receipt:
+    path: /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_city_molmo_pin_migration_20261106/attempt06/molmo_motion/molmo_motion_readback.json
+    sha256: 448aa162950993bfbb101d1933e7b8b7692b6a7cd3ddd2abf8faa10b7c6728ba
+launch_blockers: {}
+mixture_policy:
+  total_weight: 143.0
+  group_weights: {synthetic: 79.0, real_video: 26.0, robot: 26.0, tracks: 6.0, human_video: 6.0}
+  group_share_percent: {synthetic: 55.2, real_video: 18.2, robot: 18.2, tracks: 4.2, human_video: 4.2}
+  source_groups:
+    four_d_pretraining_relaunch_syn4d_sources: synthetic
+    four_d_pretraining_relaunch_omniworld_game_source: synthetic
+    four_d_pretraining_relaunch_synthverse_source: synthetic
+    four_d_pretraining_relaunch_vkitti_source: synthetic
+    four_d_pretraining_relaunch_tartanair_sky_then_far_percentile_source: synthetic
+    four_d_pretraining_relaunch_dynamic_replica_stored_depth_source: synthetic
+    four_d_pretraining_relaunch_spring_stored_depth_source: synthetic
+    four_d_pretraining_relaunch_midair_within_200_meters_source: synthetic
+    four_d_pretraining_relaunch_mvs_synth_1000_then_far_percentile_source: synthetic
+    four_d_pretraining_relaunch_tartanground_sky_then_far_percentile_source: synthetic
+    four_d_pretraining_relaunch_paralleldomain4d_sky_then_far_percentile_source: synthetic
+    scannet: real_video
+    dl3dv: real_video
+    wildrgbd: real_video
+    waymo: real_video
+    cityscapes: real_video
+    robotwin: robot
+    droid: robot
+    libero: robot
+    pointodyssey_train: tracks
+    kubric_train: tracks
+    molmo_motion: human_video

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/source_windows_as_recorded.yaml

@@ -0,0 +1,834 @@
+# B16 continuation of the full 20-source mixture with native-resolution Cityscapes captions.
+# This is a separate model-only run from an explicit five-stream overlay: 22 sources, total weight 143.
+# Cityscapes train captions are converted from the completed native2048 product; val remains evaluation-only.
+# Next Cosmos stage-one version: the owner's weighted mixture at 21 frames (decision 2026-11-05).
+#
+# Weights, not shares. Total 143 over 22 logical sources:
+#   synthetic   79  (59.4%)  the eleven synthetic sources at the 20,000-update Wan run's weights, unchanged:
+#                            syn4d 15.2, omniworld_game 8.75, synthverse 11.05, vkitti 6, tartanair 6, dynamic_replica 5,
+#                            spring 8, midair 3, mvs_synth 3, tartanground 3, paralleldomain4d 10
+#   real video  26  (18.2%)  ScanNet 8, DL3DV 6, WildRGB-D 4, Waymo 4, Cityscapes 4 (geometry is input only)
+#   robot       26  (19.5%)  RoboTwin 10 (clean-train episodes only), DROID 10 (depth and camera as inputs only),
+#                            LIBERO 6 (official training demonstrations only); no action stream in stage one
+#   tracks       6  ( 4.2%)  PointOdyssey 3, Kubric 3
+#   human video  6  ( 4.2%)  MolmoMotion; DINO and sparse tracking video are targets, camera is input-only, depth absent
+# DINO is a generated target on the real-video sources (ScanNet, DL3DV, WildRGB-D, DROID, Waymo): it is computed from the
+# RGB, so it is exact on real data, and the future-feature forecasts the DINO-Foresight and VFMF comparisons make would
+# otherwise never train on real video. This changes the first launch's roles: the owner's rule is that geometry is
+# condition-only on real data, and that rule still holds for depth and camera, which stay inputs only here.
+# Waymo trains on the processed training split only (waymo.training.pickle.gz, SHA-256 6abd3476...: 798 sequences,
+# 158,081 frames; validation never enters training) through the pinned sparse-track reader's waymo stanza: sparse LiDAR
+# depth, the camera and the RGB; no tracking video; strides 1-4 at 21 frames.
+# No segmentation stream, no Cityscapes, no YAM.
+#
+# The VGG window block below is read by the pinned VGG checkout (vgg_commit) through VggStageOneWindowDataset: the
+# synthetic sources are the recorded 500K run's stanzas (omni4d_vgg_sparse_tracking_synthetic_continuation_sources.yaml,
+# SHA-256 ae49ec6d...) with the window length 21 and missing_caption: error. RoboTwin and LIBERO are read through
+# robot_sources (RobotEpisodeStageOneDataset) because the pinned VGG checkout has no Track4D reader for them.
+# The tracking-video query draw keeps the recorded 10-pixel grid (no query_count_range).
+#
+# Train membership of the real-video sources (official split files, 2026-11-05):
+#   DL3DV      the pinned reader ignores `split` and indexes every processed scene, 85 of which are official
+#              DL3DV-Benchmark (test) scenes. The 141 official benchmark scene hashes (Hugging Face
+#              DL3DV/DL3DV-Benchmark, dataset revision 9684e838) are in omni4d_dl3dv_official_benchmark_scene_hashes.txt;
+#              the 87 of them present in the processed 1K-7K root are added to the stanza's exclude_sequence_ids below
+#              (source_reader_overrides), leaving about 5,898 scenes.
+#   WildRGB-D  the pinned reader reads <category>/selected_seqs_train.json; those 2,244 processed train scenes are all in
+#              the official train-of-both lists (nvs_list.json and camera_eval_list.json of hongchi/wildrgbd, CUT3R's
+#              get_set_list rule) and overlap the official NVS and camera-evaluation validation scenes in none.
+#
+# launch_blockers names every evidence gap that stops the launch. A blocked source keeps its weight; composition
+# (omni4d_native_config.make_config) and the window dataset both refuse to start while the list is not empty. A robot
+# source additionally needs a passing real-window readback receipt (readback_receipt). WildRGB-D and Kubric train on
+# caption text converted by tools/omni4d/convert_stage_one_caption_texts.py (text-only products: no umT5 cache exists).
+#
+# VGG pin: the 500K run's f5a59cd51 plus the window-reader updates, processed-release EgoDex support, and the
+# native2048 Cityscapes caption-reader migration at f6992c4b. The approved raw EgoDex window and Cityscapes caption
+# product readbacks used this exact pin. City captions point to the completed 2026-11-06 converted text product below.
+# One makes the DROID and LIBERO robot window length the data node's num_frames: the pinned window class and the LIBERO
+# painter fixed the clock at 33 rows in five places, one a literal inside the painter's constructor; a project adapter
+# could only rebind module globals and fake attributes, so the smallest safe change is in the VGG source, with the
+# recorded 33-row nodes reading as before. Two DROID commits (cherry-picked from the VGG main stack, f5a59cd is their
+# ancestor) keep a window whose FK arm has no visible frame-0 query, with its source motion absent, instead of raising.
+# The fourth adds the stanza key position_windows (src/vggforecast/data/position_windows.py): each listed position range is
+# one window and the stride and start are drawn inside it. ScanNet and PointOdyssey use it (source_reader_overrides,
+# caption_windows: true) with the local caption windows of their products, so every window they draw lies inside one
+# caption window and the strict caption join never redraws. The fifth adds a waymo stanza to the stage-one real-source config
+# (no source uses it yet: every real source leaves it out). The pin is a separate checkout (a git worktree of the earlier pin
+# directory, which a running training still reads at 9cb6a626a), so the earlier pin's files never change under a run.
+vgg_repository: /scratch/shared/engs2907/vgg_pins/vgg_forecast_4d_cosmos_source_20261106_f6992c4
+vgg_commit: f6992c4b67441dc9778642a252447a2687856116
+recipe_source_config: /scratch/shared/engs2907/logs/vgg_forecast_4d/train/flexpi_four_d_pretraining_wan22_5b_separate_streams_full_parameters_video_depth_camera_dino_and_sparse_tracking_video_from_the_15000_update_four_d_run_20261101/attempt02/run/config.yaml
+recipe_source_config_sha256: 72590f4598cae1b329c31b3cb45d059bdaeca23642faa10ca3f31372f9ebc0aa
+window_block:
+  _target_: four_d_windows.FourDPretrainingWindows
+  sources:
+  - name: four_d_pretraining_relaunch_syn4d_sources
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_rendered_depth_sources_21_frames_512
+    weight: 15.2
+    left_out_sources:
+    - omniworld_game
+    - tartanair
+    - mvs_synth
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_omniworld_game_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_rendered_depth_sources_21_frames_512
+    weight: 8.75
+    left_out_sources:
+    - syn4d_v1
+    - syn4d_v2_mixed
+    - syn4d_v2_mixed_no_sim
+    - tartanair
+    - mvs_synth
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_synthverse_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_synthverse_dynamic_replica_vkitti_midair_spring_sources_without_midair_quality_masks_21_frames_512
+    weight: 11.05
+    left_out_sources:
+    - dynamic_replica
+    - vkitti
+    - midair
+    - spring
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_vkitti_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_synthverse_dynamic_replica_vkitti_midair_spring_sources_without_midair_quality_masks_21_frames_512
+    weight: 6.0
+    left_out_sources:
+    - synthverse_depth
+    - dynamic_replica
+    - midair
+    - spring
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_tartanair_sky_then_far_percentile_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 6.0
+    left_out_sources:
+    - dynamic_replica
+    - spring
+    - midair
+    - mvs_synth
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_dynamic_replica_stored_depth_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 5.0
+    left_out_sources:
+    - tartanair
+    - spring
+    - midair
+    - mvs_synth
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_spring_stored_depth_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 8.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - midair
+    - mvs_synth
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_midair_within_200_meters_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 3.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - spring
+    - mvs_synth
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_mvs_synth_1000_then_far_percentile_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 3.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - spring
+    - midair
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_tartanground_sky_then_far_percentile_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 3.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - spring
+    - midair
+    - mvs_synth
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_paralleldomain4d_sky_then_far_percentile_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 10.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - spring
+    - midair
+    - mvs_synth
+    - tartanground
+    condition_only_streams: []
+  - name: scannet
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 8.0
+    left_out_sources:
+    - dl3dv
+    - wildrgbd
+    - pointodyssey_train
+    - kubric_train
+    - droid
+    - waymo
+    condition_only_streams:
+    - pointmap
+    - camera
+  - name: dl3dv
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 6.0
+    left_out_sources:
+    - scannet
+    - wildrgbd
+    - pointodyssey_train
+    - kubric_train
+    - droid
+    - waymo
+    condition_only_streams:
+    - pointmap
+    - camera
+  - name: wildrgbd
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 4.0
+    left_out_sources:
+    - dl3dv
+    - scannet
+    - pointodyssey_train
+    - kubric_train
+    - droid
+    - waymo
+    condition_only_streams:
+    - pointmap
+    - camera
+  - name: waymo
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 4.0
+    left_out_sources:
+    - dl3dv
+    - scannet
+    - wildrgbd
+    - pointodyssey_train
+    - kubric_train
+    - droid
+    condition_only_streams:
+    - pointmap
+    - camera
+  - name: droid
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 10.0
+    left_out_sources:
+    - dl3dv
+    - scannet
+    - wildrgbd
+    - pointodyssey_train
+    - kubric_train
+    - waymo
+    condition_only_streams:
+    - pointmap
+    - camera
+  - name: pointodyssey_train
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 3.0
+    left_out_sources:
+    - dl3dv
+    - scannet
+    - wildrgbd
+    - kubric_train
+    - droid
+    - waymo
+    condition_only_streams: []
+  - name: kubric_train
+    kind: track4d_source_config
+    config: train/mot/stage_one_real_and_native_track_source_windows_9_frames_288_512
+    weight: 3.0
+    left_out_sources:
+    - dl3dv
+    - scannet
+    - wildrgbd
+    - pointodyssey_train
+    - droid
+    - waymo
+    condition_only_streams:
+    - pointmap
+  frames: 21
+  raster_hw:
+  - 288
+  - 512
+  raymap_camera:
+    origin_fraction: 0.3333333333333333
+    origin_range: 1.0
+    decode_margin: 16
+    origin_box_grid: 1
+  pointmap_log_factor: 0.4
+  pointmap_hole_fill:
+    synthverse_depth: far_clip
+    syn4d: far_clip
+    omniworld_game: far_clip
+    dynamic_replica: nearest_labelled_depth_of_3x3_neighbours
+    tartanair: far_clip
+    dl3dv: far_clip
+    scannet: telea_log_depth
+    virtual_kitti2: far_clip
+    midair: far_clip
+    pointodyssey: far_clip
+    mvs_synth: far_clip
+    wildrgbd: telea_log_depth
+    hypersim: far_clip
+    spring: far_clip
+    kubric49: far_clip
+    kubric81: far_clip
+    droid_stereo_depth: telea_log_depth
+    tartanground: far_clip
+    paralleldomain4d: far_clip
+    syn4d_v2: far_clip
+    waymo: far_clip
+  pointmap_gauge: window_labelled_median_depth
+  geometry_image: genception_log_depth
+  context_len: 512
+  batch_size: 16
+  stream_seed: 42
+  epoch_windows: 32000000
+  val_set_proportion: 0.0
+  resume_state: null
+  gradient_accumulation_steps: 1
+  loader_workers: 12
+  loader_opencv_threads: 1
+  window_captions:
+    caption_dropout: 0.1
+    missing_caption: error
+    caches:
+      syn4d_v1_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v1_gemini_holo4d_local_and_global_with_repaired_globals_20261031/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v1_gemini_holo4d_local_and_global_with_repaired_globals_20261031/captions_t5_emb_meta.json
+      syn4d_v2_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v2_gemini_holo4d_20261030/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v2_gemini_holo4d_20261030/captions_t5_emb_meta.json
+      tartanair_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanair_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanair_gemini_holo4d/captions_t5_emb_meta.json
+      mvs_synth_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/mvs_synth_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/mvs_synth_gemini_holo4d/captions_t5_emb_meta.json
+      omniworld_game_gemini_local_official_text_chunks_and_gemini_global:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/omniworld_game_gemini_local_official_text_chunks_and_gemini_global_20261030b/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/omniworld_game_gemini_local_official_text_chunks_and_gemini_global_20261030b/captions_t5_emb_meta.json
+      synthverse_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/synthverse_gemini_holo4d_local96_and_global_20261031/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/synthverse_gemini_holo4d_local96_and_global_20261031/captions_t5_emb_meta.json
+      dynamic_replica_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/dynamic_replica_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/dynamic_replica_gemini_holo4d/captions_t5_emb_meta.json
+      vkitti_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/vkitti_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/vkitti_gemini_holo4d/captions_t5_emb_meta.json
+      midair_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/midair_gemini_holo4d_20261030/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/midair_gemini_holo4d_20261030/captions_t5_emb_meta.json
+      spring_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/spring_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/spring_gemini_holo4d/captions_t5_emb_meta.json
+      tartanground_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanground_gemini_holo4d_20261030/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanground_gemini_holo4d_20261030/captions_t5_emb_meta.json
+      paralleldomain4d_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/paralleldomain4d_gemini_holo4d_20261030/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/paralleldomain4d_gemini_holo4d_20261030/captions_t5_emb_meta.json
+      scannet_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/.scannet_gemini_holo4d.version-tovvjwai/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/.scannet_gemini_holo4d.version-tovvjwai/captions_t5_emb_meta.json
+      pointodyssey_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/.pointodyssey_gemini_holo4d.version-6vfyvsbi/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/.pointodyssey_gemini_holo4d.version-6vfyvsbi/captions_t5_emb_meta.json
+      droid_vp_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/.droid_vp_gemini_holo4d.version-e51r5urf/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/.droid_vp_gemini_holo4d.version-e51r5urf/captions_t5_emb_meta.json
+      dl3dv_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/dl3dv_gemini_holo4d_20261031/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/dl3dv_gemini_holo4d_20261031/captions_t5_emb_meta.json
+      wildrgbd_text_captions:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261105/wildrgbd_text_captions/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261105/wildrgbd_text_captions/captions_t5_emb_meta.json
+      kubric49_text_captions:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261105/kubric49_text_captions/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261105/kubric49_text_captions/captions_t5_emb_meta.json
+      waymo_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261105/waymo_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261105/waymo_gemini_holo4d/captions_t5_emb_meta.json
+    stanzas:
+      syn4d_v1:
+        product: syn4d_v1_gemini_holo4d
+        join: local_covering_then_global
+      syn4d_v2_mixed:
+        product: syn4d_v2_gemini_holo4d
+        join: local_covering_then_global
+      syn4d_v2_mixed_no_sim:
+        product: syn4d_v2_gemini_holo4d
+        join: local_covering_then_global
+      tartanair:
+        product: tartanair_gemini_holo4d
+        join: local_covering
+      mvs_synth:
+        product: mvs_synth_gemini_holo4d
+        join: local_covering
+      omniworld_game:
+        product: omniworld_game_gemini_local_official_text_chunks_and_gemini_global
+        join: local_covering_then_second_local_covering_then_global
+      pointodyssey_test: none
+      kubric49_validation: none
+      wildrgbd_test: none
+      vkitti_test:
+        product: vkitti_gemini_holo4d
+        join: local_covering_then_global
+      droid_held_out: none
+      synthverse_depth:
+        product: synthverse_gemini_holo4d
+        join: local_covering_then_global
+      dynamic_replica:
+        product: dynamic_replica_gemini_holo4d
+        join: global
+      vkitti:
+        product: vkitti_gemini_holo4d
+        join: local_covering_then_global
+      midair:
+        product: midair_gemini_holo4d
+        join: local_covering_then_global
+      spring:
+        product: spring_gemini_holo4d
+        join: global
+      tartanground:
+        product: tartanground_gemini_holo4d
+        join: local_covering_then_global
+      paralleldomain4d:
+        product: paralleldomain4d_gemini_holo4d
+        join: global
+      paralleldomain4d_validation:
+        product: paralleldomain4d_gemini_holo4d
+        join: global
+      scannet:
+        product: scannet_gemini_holo4d
+        join: local_containing_then_global
+      pointodyssey_train:
+        product: pointodyssey_gemini_holo4d
+        join: local_containing_then_global
+      droid:
+        product: droid_vp_gemini_holo4d
+        join: global
+      dl3dv:
+        product: dl3dv_gemini_holo4d
+        join: local_containing_then_global
+      wildrgbd:
+        product: wildrgbd_text_captions
+        join: local_containing_then_global
+      kubric_train:
+        product: kubric49_text_captions
+        join: global_required
+      waymo:
+        product: waymo_gemini_holo4d
+        join: local_containing_then_global
+  training_scene_exclusions:
+    reason: rendered depth passes through glass (raw EXR depth through glass panes at 190-260 m), owner 10-29; evidence /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/syn4d_glass_depth_check_20261029;
+      scifiroom_bald added 10-30 from /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/depth_clipping_audit_20261030
+    stanzas:
+      syn4d_v1:
+      - bigoffice_v1
+      - lab_bald
+      - office_bald
+      - scifiroom_bald
+      syn4d_v2_mixed:
+      - citynight
+      - mall
+      - metro
+      syn4d_v2_mixed_no_sim:
+      - citynight
+      - mall
+      - metro
+  tracking_video:
+    query_grid_px: 10
+    disc_radius_px: 2
+    occluded_style: ring
+    ring_radius_px: 3
+    ring_width_px: 1
+    xyz_normalization: quantile_0.02_0.98
+    colour_floor: 0.2
+    min_frame_zero_points: 16
+    surface_test:
+      relative_tolerance: 0.1
+      absolute_tolerance: 0.05
+    stanza_rules:
+      tartanair: static_scene_points
+      midair: static_scene_points
+      mvs_synth: static_scene_points
+      tartanground: static_scene_points
+      syn4d_v1: tracked_points
+      syn4d_v2_mixed: tracked_points
+      syn4d_v2_mixed_no_sim: tracked_points
+      omniworld_game: no_tracking_video
+      synthverse_depth: no_tracking_video
+      dynamic_replica: tracked_points
+      vkitti: no_tracking_video
+      spring: no_tracking_video
+      paralleldomain4d: no_tracking_video
+      scannet: static_scene_points
+      wildrgbd_test: static_scene_points
+      pointodyssey_test: tracked_points
+      kubric49_validation: no_tracking_video
+      vkitti_test: no_tracking_video
+      droid_held_out: no_tracking_video
+      paralleldomain4d_validation: no_tracking_video
+      dynamic_replica_test: tracked_points
+      pointodyssey_train: tracked_points
+      kubric_train: tracked_points
+      droid: tracked_points
+      dl3dv: static_scene_points
+      wildrgbd: static_scene_points
+      waymo: no_tracking_video
+    stanza_track_reading:
+      syn4d_v1:
+        stanza:
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+          annotation_provenance: syn4d_mesh_surface_forward_track_from_frame0
+          allow_missing_tracks: false
+        kwargs:
+          return_tracks: true
+          depth_only: false
+          metadata_root: /vgg/chenzhao/syn4d_metadata
+          wrapped_face_distance_thresh: 0.1
+          query_inconsistent_distance_thresh: 0.1
+      pointodyssey_test:
+        stanza:
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+      dynamic_replica_test:
+        stanza:
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+      syn4d_v2_mixed:
+        loader: syn4d_v2
+        stanza:
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+          annotation_provenance: syn4d_v2_mesh_surface_forward_track_from_frame0
+          allow_missing_tracks: false
+        kwargs:
+          return_tracks: true
+          depth_only: false
+          metadata_root: /vgg/chenzhao/syn4d_metadata
+          tracking_format: safetensor
+          mask_bg: false
+          trusted_background_depth_max: 80.0
+          wrapped_face_distance_thresh: 0.1
+          query_inconsistent_distance_thresh: 0.1
+      syn4d_v2_mixed_no_sim:
+        loader: syn4d_v2
+        stanza:
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+          annotation_provenance: syn4d_v2_mesh_surface_forward_track_from_frame0
+          allow_missing_tracks: false
+        kwargs:
+          return_tracks: true
+          depth_only: false
+          metadata_root: /vgg/chenzhao/syn4d_metadata
+          tracking_format: safetensor
+          mask_bg: false
+          trusted_background_depth_max: 80.0
+          wrapped_face_distance_thresh: 0.1
+          query_inconsistent_distance_thresh: 0.1
+      dynamic_replica:
+        stanza:
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+      pointodyssey_train:
+        stanza:
+          allow_missing_tracks: false
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+      kubric_train:
+        stanza:
+          allow_missing_tracks: false
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+      droid:
+        stanza:
+          allow_missing_tracks: false
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+  source_reader_overrides:
+    dl3dv:
+      stanza: dl3dv
+      exclude_sequence_ids:
+      - 1K/0a1b7c20a92c43c6b8954b1ac909fb2f0fa8b2997b80604bc8bbec80a1cb2da3
+      - 1K/14eb48a50e37df548894ab6d8cd628a21dae14bbe6c462e894616fc5962e6c49
+      - 1K/1da888bdedfc9629c0fa9f82cf3f5d96f8103baee0ff64d9311aea1224a9f2ae
+      - 1K/2991a75d1f0fb2b7a891d424fc2459bb954b61599b52e74a056a7acfb2998b77
+      - 1K/2cbfe28643b6636f9c70813cae7625aa858a352109493ac70fb429ce94dd55b3
+      - 1K/5a27c00f525f5484b7aa0d0107051253bf7bb395bde0b3a4486ff8c52fb4965e
+      - 1K/6d221625614d0515fa44d59a8797a49b64f86bc775e66be22f07a66aa0a0cb3a
+      - 1K/917e9c8985d353b0ee4c281f11fa84eb8550814562670cbb82cd0ec9c1194fe7
+      - 1K/918c8dad730c3b804306c5da8486124be4aa0612e85fb825338fd350c912e1b0
+      - 1K/a17a984ca90a9b5840fdf85b15104b0d18e25975981c1aa90fcdfd6eeeb285f3
+      - 1K/a726c1112ad026efc92f59f196c0698e67abd12a04135a7ff6349b1d31b089eb
+      - 1K/adb95f29c15b8a89e40c17d10596679be09d7719c6f08a453facd506d056ea8b
+      - 1K/d1b3a0b37acc72207295b05e8c359f8b43bf879eb882c60747cfc6d3c7d8efa3
+      - 1K/eb4cf52988f805e6fce11d1b239fa9de32eb157364cff06ebac0aa50e0a46567
+      - 2K/032dee9fb0a8bc1b90871dc5fe950080d0bcd3caf166447f44e60ca50ac04ec7
+      - 2K/0853979305f7ecb80bd8fc2c8df916410d471ef04ed5f1a64e9651baa41d7695
+      - 2K/093ef327b4e4f9d4ee52c02a354a53558a8652157fb0d58f3b4a708734afb334
+      - 2K/0f8ac521439691fe429e9efbed8d5ded1cee35bcf52d47731fb60d5a2ff661a7
+      - 2K/1d6a9ed47cce39fd1c4d18f776bcc97e507b81bde921ca596bd91b0b02b5e414
+      - 2K/26fd23358fa11fff0fb3180ef0b65591b486e20dcf753ce4a7aae49a37e370c7
+      - 2K/280abf7bd93b81b077af1db638229dbb09869052fa0b7b57c81c94d2db893829
+      - 2K/2beaca318994c25409dcbb6d0bdd96c3620f2f18aec44ea3f20edd302f18ca78
+      - 2K/341b4ff3dfd3d377d7167bd81f443bedafbff003bf04881b99760fc0aeb69510
+      - 2K/374ffd0c5ff8e544f588f1da3ce0d5b69dc39ff3af95d7ccec480fe5e533c45e
+      - 2K/41036716da7efda334c1d434c4141d15642e0e02f881a01b6c8c36f8bea64c45
+      - 2K/484c0aca40c1087f196d2caf3c8cb463864c08e60b98326ca02eb6dae8ce4984
+      - 2K/4ff8650b5c1140e98d957fbd1abbe2cfbbb416b37cb9f06f7f376956703801bb
+      - 2K/513e4ea2e8477b06f2b32417d5c243aec71d491cb0596a60e9fe304c635f20a1
+      - 2K/8b9fb9d9f10e8c64d5034be69809465753b8ba88ef12da82afd47d38ee789934
+      - 2K/91afb9910b042f7185c2b8e4b6b24b5785dae4542617b3f8005b5492f6d123f7
+      - 2K/ba55c875d20c34ee85ffc72264c4d77710852e5fb7d9ce4b9c26a8442850e98f
+      - 2K/c076929db6501cf7ebe386c70e6d77ea3af844a745e794f2ec17c981c465a69b
+      - 2K/ceb252f5d4419510655cf9ed7afbf3e8e688825f798d80414c7715dc8ace153a
+      - 2K/df29c225863d0173e060b1f3fde4e6eb5a57a4e4edabd397026ba1a028aeb39e
+      - 2K/ff592398657b3dfe94153332861985194a3e3c9d199c4a3a27a0ce4038e81ade
+      - 3K/35872363e17af5d173b6a0b09fcf5de94627ad5dc5f8a9ad4c579f3e70b4797a
+      - 3K/387eeb925ba47af5db9ce6102aaf6b07e6d3573b7e7568dda4cef22d5ab1342f
+      - 3K/3bb894d1933f3081134ad2d40e54de5f0636bd8b502b0a8561873bb63b0dce85
+      - 3K/5c3af581028068a3c402c7cbe16ecf9471ddf2897c34ab634b7b1b6cf81aba00
+      - 3K/669c36225b273e22ce08364f166dce910b708d7abd89e58463a15d2790b534d5
+      - 3K/66fd66cbed40a2d11ecd17ae6d6d60cd60d476616864ecdd351fe69e644f30a1
+      - 3K/70eac6ff18a1daae4eeccc5eb18723eaf7e029a77e79c79d9573f7bac59ba92b
+      - 3K/75fbbe467353cfdeb93bc1872144855576797be83a0c71c51155ee6b355d6d5e
+      - 3K/800cf88687a79c232700454c2f5a402e771bc7157037a17b0ce93da365b25575
+      - 3K/8cb2e97d26a639f05a571476240a8fa86988e6853f0f13cc05830d1578002aad
+      - 3K/9cbc5548643ca9cfe797f871bac58bccd02f0510c49b7cb3697b38dfd780a508
+      - 3K/a62f9a1c6362e7c8a3440261ea0921a0a4b60cf5822bd19579836694f26b3abd
+      - 3K/adf35184a12d4cfa3f4248b87aa5adb4f39f179df460d6d76136e13d37299a2a
+      - 3K/af0d7039e64af3959210c235e59425896ac064f4e0f721c437bbd2cf1d8ea910
+      - 3K/b3bf9079b442c681749f6bd95cfb0659c6d844f2228b2f4fa4b602a0e3627b9a
+      - 3K/b6d1134cb03c855d246f916beaa2d3d01290cf47937a5176011ff39b850ec615
+      - 3K/d8de66037bd03dd0d39d54f9978bac3318d912e30e22c21e1ada82a98ed48c53
+      - 3K/ddfdcfdf02d53dd89a055cd9e1ed30272ab3f57c7560ac2c3761bb01d76a7882
+      - 3K/e8ce51b6abfe05bf8dca47e29c8be6c1e6de27a8c9fece7a121400b931b2ca0f
+      - 3K/e9360e7a89bee835dc847cf8796093e634b759ff582558788dcfe8326f6e8901
+      - 3K/f7aaea9ac683bb1945cfeb1daa9abca3375249e0d19281a89a3f575040835464
+      - 4K/56452d9cd9e2e034764132a57058517ab085e1352f9664d2d1e10e98741334fc
+      - 4K/6d81c5ab0d480fd43d78b75ff372a8113ad38e2c03f1d69627c009883054d4c2
+      - 4K/9641a1ed7963ce5ca734cff3e6ccea3dfa8bcb0b0a3ff78f65d32a080de2d71e
+      - 4K/d3812aad538261e7f73c75762ff55f23b468bcc76f376d52ac86ca6cf3c44b4b
+      - 5K/0a485338bbdaf19ba9090b874bb36ef0599a9c9a475a382c22903cf5981c6ea6
+      - 5K/119fd56d3797e2d349ca64ddcc5851463cd13b5974b5b2e4566ed5cf7e02e6c1
+      - 5K/c37726ce770ac50a2cf5c0f43022f0268e26da0d777cd8e3a3418c4eed03fd94
+      - 5K/dac9796dd69e1c25277e29d7e30af4f21e3b575d62a0a208c2b3d1211e2d5d77
+      - 6K/06da796666297fe4c683c231edf56ec00148a6a52ab5bb159fe1be31f53a58df
+      - 6K/7a9f97660be8f2421b37dd413c898360532f17c6447c11515e8aded2d80eb99d
+      - 6K/7da3db99059a436adf537bc23309a6a4db9e011696d086f3a64037f7497d9df7
+      - 6K/e5684b3292bfd77db297839fc37ee4cce7fd59775af1a6a4827e3b4f59c036d3
+      - 6K/ec1e44d4dc0f8fa77610866495f9297a7f82158c43e1777668b84fd4b736c7bc
+      - 6K/ef59aac437132bfc1dd45a7e1f8e4800978e7bb28bf98c4428d26fb3e1da3e90
+      - 6K/f477ffc4b398bed8e0d921f0fba9825ca63f317381c535c84be23be991ae1d7a
+      - 7K/0569e83fdc248a51fc0ab082ce5e2baff15755c53c207f545e6d02d91f01d166
+      - 7K/073f5a9b983ced6fb28b23051260558b165f328a16b2d33fe20585b7ee4ad561
+      - 7K/183dd248f6a86e07c5adf9de8ee2d0abe45b1216331c03678e89634c2e9b1c7f
+      - 7K/1ba74c22670ad047981441581d00f26f4a148d1010bcac7468c615adf5fa4d5d
+      - 7K/389a460ca1995e0658e85fe8e6b520b4e88b370cd6710dfe728b1564bba31aee
+      - 7K/493816813d2d6d248eb3c2b0b77b63e54235266e9a06e270fd0d282f13960493
+      - 7K/4ae797d07b6d1644c9db6919c8cc8c0d28d72be45108ac7a3abf8dc21b599d83
+      - 7K/50c46cf8b8b22c8d2ffdef8964b05ddbceaef312c9a9ff331d1ecebfd223f72a
+      - 7K/565553aa894be621e8b4773cac288e60ad0c2cf7edb621be62b348c9a0f78380
+      - 7K/599ca3e04cae3ec83affc426af7d0d7ab36eb91cd8e539edbc13070a4d455792
+      - 7K/5c8dafad7d782c76ffad8c14e9e1244ce2b83aa12324c54a3cc10176964acf04
+      - 7K/63798f5c6fbfcb4eb686268248b8ecbc8d87d920b2bcce967eeaedfd3b3b6d82
+      - 7K/946f49be73928469000baa5ca04d2573137c5ee6a66362bcf8d130354dca8924
+      - 7K/9e9a89ae6fed06d6e2f4749b4b0059f35ca97f848cedc4a14345999e746f7884
+      - 7K/cd9c981eeb4a9091547af19181b382698e9d9eee0a838c7c9783a8a268af6aee
+      - 7K/d4fbeba0168af8fddb2fc695881787aedcd62f477c7dcec9ebca7b8594bbd95b
+    scannet:
+      stanza: scannet
+      caption_windows: true
+    pointodyssey_train:
+      stanza: pointodyssey_train
+      caption_windows: true
+robot_sources:
+- name: robotwin
+  weight: 10.0
+  frames: 21
+  stride: 4
+  # Simulator ground truth: depth, camera and the arm tracks are generated targets (owner decision 2026-11-05); DINO too.
+  condition_only_streams: []
+  reader:
+    source_split_manifest: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/flexpi_robotwin_robot_arm_track_video_20261023/split_manifest.json
+    source_split_sha256: 10467f314d57acf8deba64b94d0842b0dedcc256acf5f055df92203f7bf10f4e
+    pretrained_norm_stats: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/robotwin_action_image_clean_train_filtered_20261102/native_stats/dataset_stats.json
+  clean_train_split:
+    manifest: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/robotwin_action_image_clean_train_filtered_20261102/split_manifest.json
+    sha256: 6511ed1a88a1047d74fb80dd11b1304f6d77abce6cbe3f56df4f5cf7b9f9ee7c
+    train_episodes: 26288
+    held_out_episodes: 150
+    excluded_clamped_episodes: 1062
+  geometry:
+    simulator_ground_truth: true
+    log_factor: 0.4
+    raymap_camera: {origin_fraction: 0.3333333333333333, origin_range: 1.0, decode_margin: 16, origin_box_grid: 1}
+    gauge: window_labelled_median_depth
+    release_root: /datasets/engs2907/FlexPi/robotwin_3d
+    calibration_root: /datasets/engs2907/FlexPi/robotwin_3d/calibration
+  # Two real clean-train windows at 21 frames through reader, DINO and the native batch adapter, on pin f6992c4b.
+  readback_receipt:
+    path: /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_city_molmo_pin_migration_20261106/attempt05/robot_sources/robotwin_readback.json
+    sha256: 08c29011548725201118a245372365c0fd2430eafa6ee4b2844d2cf2fdca59ff
+- name: libero
+  weight: 6.0
+  frames: 21
+  stride: 4
+  condition_only_streams: []
+  reader:
+    hydra_overrides: []
+  geometry:
+    simulator_ground_truth: true
+  readback_receipt:
+    path: /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_city_molmo_pin_migration_20261106/attempt05/robot_sources/libero_readback.json
+    sha256: 5dd74fbd926bc0b7aac7e27da43ae55c1eaa9cb049b9c4656ba27eac71bdfedc
+real_video_sources:
+- name: cityscapes
+  weight: 4.0
+  frames: 21
+  stride: 1
+  condition_only_streams:
+  - pointmap
+  reader:
+    data_root: /scratch/shared/engs2907/datasets/cityscapes
+    train_manifest: /vgg/yushi/dataset/Cityscapes/prepared/cityscapes_train_driving_sequence_v1.jsonl
+    train_manifest_sha256: 9dd2abd964d52c752401b2aef6baa400ec52268143da8b8c623cb059b8f70475
+    train_snippets: 2975
+    train_frames: 89250
+    val_manifest: /vgg/yushi/dataset/Cityscapes/prepared/cityscapes_val_driving_sequence_v1.jsonl
+    val_manifest_sha256: 7b6fb5cc52bd4eac0a61874ee14b47988481a6a484ed73f8ca4c959bc1aa5adc
+  geometry:
+    hole_fill: telea_log_depth
+  captions:
+    product: cityscapes_gemini_holo4d
+    complete_marker: /vgg/yushi/dataset/caption_output/gemini_holo4d/native2048_v1/cityscapes/COMPLETE.json
+    texts: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/cosmos_caption_texts_20261106/cityscapes_gemini_holo4d
+molmo_motion_sources:
+- name: molmo_motion
+  weight: 6.0
+  frames: 21
+  # The camera is an input only, like every real source's camera; there is no depth. DINO and the tracking video are targets.
+  condition_only_streams: [camera]
+  reader:
+    config: train/mot/wan_camera_omega_video_depth_vae_and_sparse_future_prediction_molmo_motion_21_frames_512
+    stanza: molmo_motion
+    release_root: /vgg/yushi/dataset/MolmoMotion-1M
+    subsets: [egodex, xperience, stereo4d]
+    # molmo-motion's sqrt-frequency mixing (p_i proportional to sqrt N_i, README "Training"), N_i the drawable records of
+    # the subset at this window length measured by the readback (census of 2026-11-05).
+    subset_weights: {egodex: 133.6, xperience: 432.4, stereo4d: 210.9}
+    temporal_stride: {egodex: 2, xperience: 1, stereo4d: 4}
+    processed_release_only_subsets: [egodex]
+    # Only the already reconstructed release MP4 subset is eligible. Never fall back to newly downloaded raw 30 fps
+    # EgoDex videos: their RGB raster has not been validated against the 15 fps track/camera annotations.
+    processed_release_only_subsets: [egodex]
+    # The pinned config leaves Xperience's `object` records out: their 3D was lifted with the 256-pixel camera matrix.
+    excluded_objects: {xperience: [object]}
+    max_window_redraws: 64
+  exclusions:
+    # The registered 120-record test cohort (the VGG held-out spec molmo_motion_test_120_records_droid_egodex_molmospaces_
+    # 49_frames_512; tools/omni4d/write_molmo_motion_test_cohort_manifest.py). The release's own test splits (DROID's too) and
+    # PointMotionBench (hot3d, worldtrack, davis) are asserted disjoint from the train records in code.
+    test_cohort_manifest:
+      path: /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_molmo_motion_20261105/cohort/molmo_motion_test_cohort_120_records.json
+      sha256: d1f8e8d8ad8f8f67c8bfd23319cdea026ea8cfcba1194b5f4594366e9f64fd76
+    point_motion_bench_root: /scratch/shared/engs2907/PointMotionBench
+  readback_receipt:
+    path: /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_city_molmo_pin_migration_20261106/attempt06/molmo_motion/molmo_motion_readback.json
+    sha256: 448aa162950993bfbb101d1933e7b8b7692b6a7cd3ddd2abf8faa10b7c6728ba
+launch_blockers: {}
+mixture_policy:
+  total_weight: 143.0
+  group_weights: {synthetic: 79.0, real_video: 26.0, robot: 26.0, tracks: 6.0, human_video: 6.0}
+  group_share_percent: {synthetic: 55.2, real_video: 18.2, robot: 18.2, tracks: 4.2, human_video: 4.2}
+  source_groups:
+    four_d_pretraining_relaunch_syn4d_sources: synthetic
+    four_d_pretraining_relaunch_omniworld_game_source: synthetic
+    four_d_pretraining_relaunch_synthverse_source: synthetic
+    four_d_pretraining_relaunch_vkitti_source: synthetic
+    four_d_pretraining_relaunch_tartanair_sky_then_far_percentile_source: synthetic
+    four_d_pretraining_relaunch_dynamic_replica_stored_depth_source: synthetic
+    four_d_pretraining_relaunch_spring_stored_depth_source: synthetic
+    four_d_pretraining_relaunch_midair_within_200_meters_source: synthetic
+    four_d_pretraining_relaunch_mvs_synth_1000_then_far_percentile_source: synthetic
+    four_d_pretraining_relaunch_tartanground_sky_then_far_percentile_source: synthetic
+    four_d_pretraining_relaunch_paralleldomain4d_sky_then_far_percentile_source: synthetic
+    scannet: real_video
+    dl3dv: real_video
+    wildrgbd: real_video
+    waymo: real_video
+    cityscapes: real_video
+    robotwin: robot
+    droid: robot
+    libero: robot
+    pointodyssey_train: tracks
+    kubric_train: tracks
+    molmo_motion: human_video

cosmos_nano_full20_21frame_update_014700_20261107/attempt01/update_014700/training_launcher_source.sh

@@ -0,0 +1,528 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ ${COSMOS_CLEAN_SLURM_ENV:-0} != 1 ]]; then
+  export COSMOS_CLEAN_SLURM_ENV=1
+  exec python3 - "$0" "$@" <<'PY'
+import os
+import subprocess
+import sys
+
+environment = {key: value for key, value in os.environ.items() if not key.startswith("SLURM_")}
+raise SystemExit(subprocess.run(["bash", *sys.argv[1:]], env=environment).returncode)
+PY
+fi
+
+repo_root=${COSMOS_SOURCE_ROOT:-$(git rev-parse --show-toplevel)}
+launcher_root=$(cd -- "$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")/../../.." && pwd)
+allocation=${COSMOS_ALLOCATION_ID:?set COSMOS_ALLOCATION_ID to the owner-approved two-B200 allocation}
+expected_node=${COSMOS_EXPECTED_NODE:?set COSMOS_EXPECTED_NODE to the owner-approved B200 node}
+allocated_gpu_uuids=${COSMOS_ALLOCATED_GPU_UUIDS:?set COSMOS_ALLOCATED_GPU_UUIDS to the two UUIDs verified for this allocation}
+windows_config=${COSMOS_VGG_WINDOWS_CONFIG:?set COSMOS_VGG_WINDOWS_CONFIG to the pinned full-mixture YAML}
+windows_sha256=${COSMOS_VGG_WINDOWS_CONFIG_SHA256:?set COSMOS_VGG_WINDOWS_CONFIG_SHA256 to its SHA-256}
+experiment_module=${COSMOS_EXPERIMENT_MODULE:-cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_formal_5k}
+warmstart_path=${COSMOS_TRAINABLE_OVERLAY_PATH:-}
+next_version_ddp_module=cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_21_frames_temporal_forcing_plain_ddp_from_8200
+next_version_ddp_overlay=/scratch/shared/engs2907/logs/vgg_forecast_4d/tools/cosmos_twenty_one_frame_ddp_handoff_20261105/attempt01/protected_step_008200.safetensors
+next_version_full19_module=cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_21_frames_plain_ddp_from_7300
+next_version_full19_cityscapes_module=cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_plus_cityscapes_21_frames_plain_ddp_from_7300
+next_version_omega_module=cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_21_frames_plain_ddp_from_overlay_with_omega_stream
+next_version_full20_start_identity=cosmos3_nano_omni4d_stage_one_lora_r256_plain_ddp_21_frames_temporal_forcing_full19_from7300_global32
+next_version_full20_writer_identity=cosmos3_nano_omni4d_stage_one_lora_r256_plain_ddp_21_frames_temporal_forcing_full20_from_overlay_global32_lr2p5e5
+next_version_running_writer_identity=$next_version_full20_writer_identity
+next_version_molmo_motion_module=cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_plus_molmo_motion_21_frames_plain_ddp_from_explicit_overlay
+next_version_molmo_motion_windows_sha=3f5503a68154b225f04181e76edd7c6e57dfaddc20d9a754e5adc9a8e190a66d
+next_version_city_molmo_module=cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_plus_cityscapes_and_molmo_motion_21_frames_plain_ddp_from_explicit_overlay
+next_version_city_molmo_windows_sha=73c90b72adc27e1589f97bff64cb35ace5d03acf7fdac43a3e5a5c86055178d5
+next_version_city_molmo_vgg_commit=f6992c4b67441dc9778642a252447a2687856116
+next_version_city_molmo_writer_identity=cosmos3_nano_omni4d_stage_one_lora_r256_plain_ddp_21_frames_temporal_forcing_full20_plus_cityscapes_and_molmo_motion_from_overlay_global32_lr2p5e5
+omega_statistics_path=${COSMOS_OMEGA_STATISTICS_PATH:-}
+omega_statistics_sha256=${COSMOS_OMEGA_STATISTICS_SHA256:-}
+microbatch_profile=${COSMOS_MICROBATCH_PROFILE:-b8_gas2}
+case "$microbatch_profile" in
+  b8_gas2)
+    expected_batch=8
+    expected_accum=2
+    expected_windows_sha=4ae8815d1141d7a2215af094232eddde641bb3aff40963896cf2fe1fc00415e3
+    ;;
+  b16_gas1)
+    expected_batch=16
+    expected_accum=1
+    expected_windows_sha=0b35b990c82717b4d998691e616581573679ca53f30c44dc370577362b31905f
+    ;;
+  *) echo "COSMOS_MICROBATCH_PROFILE must be b8_gas2 or b16_gas1" >&2; exit 20 ;;
+esac
+if [[ "$experiment_module" == "$next_version_molmo_motion_module" ]]; then
+  [[ "$microbatch_profile" == b16_gas1 ]] || {
+    echo "the MolmoMotion variant is the B16 profile: set COSMOS_MICROBATCH_PROFILE=b16_gas1" >&2; exit 20
+  }
+  expected_windows_sha=$next_version_molmo_motion_windows_sha
+fi
+if [[ "$experiment_module" == "$next_version_city_molmo_module" ]]; then
+  [[ "$microbatch_profile" == b16_gas1 ]] || {
+    echo "the combined City and MolmoMotion recipe is the B16 profile: set COSMOS_MICROBATCH_PROFILE=b16_gas1" >&2; exit 20
+  }
+  expected_windows_sha=$next_version_city_molmo_windows_sha
+fi
+overlay_sha256=${COSMOS_TRAINABLE_OVERLAY_SHA256:-}
+overlay_update=${COSMOS_TRAINABLE_OVERLAY_UPDATE:-}
+overlay_config_identity=${COSMOS_TRAINABLE_OVERLAY_CONFIG_IDENTITY:-}
+twenty_update_test=${COSMOS_TWENTY_UPDATE_ADMISSION_TEST:-0}
+[[ "$twenty_update_test" == 0 || "$twenty_update_test" == 1 ]] || { echo "COSMOS_TWENTY_UPDATE_ADMISSION_TEST must be 0 or 1" >&2; exit 20; }
+timing_gate_updates=${COSMOS_TIMING_GATE_UPDATES:-0}
+[[ "$timing_gate_updates" =~ ^(0|[1-9][0-9]*)$ ]] || { echo "COSMOS_TIMING_GATE_UPDATES must be 0 or an integer >= 50" >&2; exit 20; }
+[[ "$timing_gate_updates" == 0 || "$timing_gate_updates" -ge 50 ]] || {
+  echo "COSMOS_TIMING_GATE_UPDATES must be at least 50 when enabled" >&2; exit 20;
+}
+if [[ "$timing_gate_updates" != 0 && "$experiment_module" != "$next_version_full19_module" \
+   && "$experiment_module" != "$next_version_city_molmo_module" ]]; then
+  echo "the 50-update timing gate is supported only for the full20 and combined City+Molmo recipes" >&2; exit 20
+fi
+[[ "$timing_gate_updates" == 0 || "$twenty_update_test" == 0 ]] || {
+  echo "select either the 20-update admission test or the 50-update timing gate" >&2; exit 20;
+}
+if [[ "$experiment_module" == "$next_version_ddp_module" ]]; then
+  warmstart_path=${warmstart_path:-$next_version_ddp_overlay}
+  [[ "$warmstart_path" == "$next_version_ddp_overlay" ]] || {
+    echo "the DDP 21-frame continuation must use the protected update-8200 overlay: $next_version_ddp_overlay" >&2
+    exit 21
+  }
+fi
+if [[ "$experiment_module" == "$next_version_full19_module" || "$experiment_module" == "$next_version_molmo_motion_module" \
+   || "$experiment_module" == "$next_version_city_molmo_module" ]]; then
+  [[ -f "$warmstart_path" ]] || { echo "the 21-frame continuation requires an explicit trainable overlay path" >&2; exit 21; }
+  [[ "$overlay_sha256" =~ ^[0-9a-f]{64}$ ]] || { echo "COSMOS_TRAINABLE_OVERLAY_SHA256 must be supplied explicitly" >&2; exit 21; }
+  [[ "$overlay_update" =~ ^[1-9][0-9]*$ ]] || { echo "COSMOS_TRAINABLE_OVERLAY_UPDATE must be supplied explicitly" >&2; exit 21; }
+  if [[ "$experiment_module" == "$next_version_full19_module" || "$experiment_module" == "$next_version_molmo_motion_module" \
+      || "$experiment_module" == "$next_version_city_molmo_module" ]]; then
+    [[ "$overlay_config_identity" == "$next_version_full20_start_identity" \
+       || "$overlay_config_identity" == "$next_version_running_writer_identity" \
+       || ("$experiment_module" == "$next_version_city_molmo_module" && "$overlay_config_identity" == "$next_version_city_molmo_writer_identity") ]] || {
+      echo "COSMOS_TRAINABLE_OVERLAY_CONFIG_IDENTITY must be the identity of an overlay of the full-mixture 21-frame run: $next_version_full20_start_identity or $next_version_running_writer_identity" >&2
+      exit 21
+    }
+  else
+    [[ "$overlay_config_identity" == "$next_version_full20_start_identity" ]] || {
+      echo "COSMOS_TRAINABLE_OVERLAY_CONFIG_IDENTITY must match the reviewed full-mixture 21-frame overlay" >&2
+      exit 21
+    }
+  fi
+fi
+if [[ "$experiment_module" == "$next_version_omega_module" ]]; then
+  # The same strict weights-only handoff as the five-stream restart, from a five-stream overlay (the one the restart began
+  # from or one the restart wrote); the six Omega tensors are added by the recipe's explicit allowance.
+  [[ -f "$warmstart_path" ]] || { echo "the Omega-stream run requires an explicit five-stream trainable overlay path" >&2; exit 21; }
+  [[ "$overlay_sha256" =~ ^[0-9a-f]{64}$ ]] || { echo "COSMOS_TRAINABLE_OVERLAY_SHA256 must be supplied explicitly" >&2; exit 21; }
+  [[ "$overlay_update" =~ ^[1-9][0-9]*$ ]] || { echo "COSMOS_TRAINABLE_OVERLAY_UPDATE must be supplied explicitly" >&2; exit 21; }
+  [[ "$overlay_config_identity" == "$next_version_full20_start_identity" || "$overlay_config_identity" == "$next_version_full20_writer_identity" ]] || {
+    echo "COSMOS_TRAINABLE_OVERLAY_CONFIG_IDENTITY must be a reviewed five-stream full-mixture 21-frame overlay identity" >&2
+    exit 21
+  }
+  [[ -f "$omega_statistics_path" ]] || { echo "COSMOS_OMEGA_STATISTICS_PATH must name the pinned Omega feature statistics file" >&2; exit 21; }
+  [[ "$omega_statistics_sha256" =~ ^[0-9a-f]{64}$ ]] || { echo "COSMOS_OMEGA_STATISTICS_SHA256 must be supplied explicitly" >&2; exit 21; }
+  actual_statistics_sha256=$(sha256sum "$omega_statistics_path" | awk '{print $1}')
+  [[ "$actual_statistics_sha256" == "$omega_statistics_sha256" ]] || {
+    echo "Omega statistics SHA-256 mismatch: expected=$omega_statistics_sha256 got=$actual_statistics_sha256" >&2; exit 22;
+  }
+elif [[ -n "$omega_statistics_path$omega_statistics_sha256" ]]; then
+  echo "the Omega statistics belong to the Omega-stream module only" >&2; exit 21
+fi
+default_attempt_root=$(bash "$repo_root/tools/omni4d/cosmos3_nano/resolve_native_formal_attempt_root.sh" \
+  "$experiment_module" "$warmstart_path")
+attempt_root=${COSMOS_ATTEMPT_ROOT:-$default_attempt_root}
+slurm_category=train
+if [[ "$twenty_update_test" == 1 ]]; then slurm_category=smoke; fi
+step_log_dir="/scratch/shared/engs2907/logs/vgg_forecast_4d/slurm/$slurm_category/$(basename "$(dirname "$attempt_root")")/$(basename "$attempt_root")"
+python_bin=${COSMOS_PYTHON:-/scratch/shared/engs2907/envs/cosmos3-nano-py313-cu128-001fe183/bin/python}
+compute_python=${COSMOS_COMPUTE_PYTHON:-$python_bin}
+[[ "$python_bin" == /scratch/shared/engs2907/envs/cosmos3-nano-py313-cu128-001fe183/bin/python && "$compute_python" == "$python_bin" ]] || { echo "Cosmos uses one persistent Conda environment" >&2; exit 20; }
+native_stall=${COSMOS_NATIVE_STALL_DIAGNOSTICS:-0}
+[[ "$native_stall" == 0 || "$native_stall" == 1 ]] || { echo "COSMOS_NATIVE_STALL_DIAGNOSTICS must be 0 or 1" >&2; exit 20; }
+if [[ "$twenty_update_test" == 1 && "$experiment_module" != "$next_version_full19_module" \
+   && "$experiment_module" != "$next_version_omega_module" \
+   && "$experiment_module" != "$next_version_molmo_motion_module" \
+   && "$experiment_module" != "$next_version_city_molmo_module" ]]; then
+  echo "the bounded admission prefix is supported only for the 21-frame full-mixture recipe" >&2; exit 20
+fi
+
+if [[ ! -x "$python_bin" || ! -f "$windows_config" ]]; then
+  echo "the pinned Cosmos Python environment and full-mixture source config must exist" >&2
+  exit 20
+fi
+if [[ ! "$windows_sha256" =~ ^[0-9a-f]{64}$ ]]; then
+  echo "invalid source-config SHA-256" >&2
+  exit 21
+fi
+actual_sha256=$(sha256sum "$windows_config" | awk '{print $1}')
+if [[ "$actual_sha256" != "$windows_sha256" ]]; then
+  echo "source-config hash mismatch: expected=$windows_sha256 got=$actual_sha256" >&2
+  exit 22
+fi
+expected_vgg_commit=b9e2059143b4db90a57a4bac7d5dca1a7ee03d48
+if [[ "$experiment_module" == "$next_version_city_molmo_module" ]]; then
+  expected_vgg_commit=$next_version_city_molmo_vgg_commit
+fi
+if [[ "$experiment_module" == "$next_version_ddp_module" || "$experiment_module" == "$next_version_full19_module" \
+   || "$experiment_module" == "$next_version_full19_cityscapes_module" || "$experiment_module" == "$next_version_omega_module" \
+   || "$experiment_module" == "$next_version_molmo_motion_module" || "$experiment_module" == "$next_version_city_molmo_module" ]]; then
+  expected_sources=20
+  expected_total_weight=133.0
+  expected_molmo_motion_sources=0
+  if [[ "$experiment_module" == "$next_version_molmo_motion_module" ]]; then
+    expected_sources=21
+    expected_total_weight=139.0
+    expected_molmo_motion_sources=1
+  elif [[ "$experiment_module" == "$next_version_full19_cityscapes_module" ]]; then
+    expected_sources=21
+    expected_total_weight=137.0
+  elif [[ "$experiment_module" == "$next_version_city_molmo_module" ]]; then
+    expected_sources=22
+    expected_total_weight=143.0
+    expected_molmo_motion_sources=1
+  fi
+  "$python_bin" - "$windows_config" "$expected_batch" "$expected_accum" "$expected_sources" "$expected_total_weight" \
+    "$expected_molmo_motion_sources" "$expected_vgg_commit" <<'PY'
+
+import sys
+import yaml
+
+with open(sys.argv[1], encoding="utf-8") as stream:
+    payload = yaml.safe_load(stream)
+window = payload["window_block"]
+checks = {
+    "frames": (window["frames"], 21),
+    "per-rank batch size": (window["batch_size"], int(sys.argv[2])),
+    "gradient accumulation": (window["gradient_accumulation_steps"], int(sys.argv[3])),
+    "loader worker count": (window["loader_workers"], 12),
+}
+if payload.get("vgg_commit") != sys.argv[7]:
+    raise SystemExit(f"the selected full-mixture YAML must pin the reviewed VGG source {sys.argv[7]}")
+molmo_motion = payload.get("molmo_motion_sources", [])
+if len(molmo_motion) != int(sys.argv[6]):
+    raise SystemExit("the selected YAML lists a different number of MolmoMotion sources than the selected recipe")
+if len(window.get("sources", [])) + len(payload.get("robot_sources", [])) + \
+        len(payload.get("real_video_sources", [])) + len(molmo_motion) != int(sys.argv[4]):
+    raise SystemExit("the selected YAML must retain all approved source stanzas")
+if payload.get("mixture_policy", {}).get("total_weight") != float(sys.argv[5]):
+    raise SystemExit("the selected YAML has a different full-mixture source weighting")
+failed = [f"{name}: expected {expected}, got {actual}" for name, (actual, expected) in checks.items()
+          if actual != expected]
+if failed:
+    raise SystemExit("DDP 21-frame windows config does not match the selected microbatch profile:\n" + "\n".join(failed))
+PY
+  [[ "$windows_sha256" == "$expected_windows_sha" ]] || {
+    echo "windows YAML SHA-256 does not match the selected microbatch profile" >&2; exit 22;
+  }
+fi
+if [[ -e "$attempt_root" ]]; then
+  echo "refusing to overwrite existing attempt: $attempt_root" >&2
+  exit 23
+fi
+mkdir -p "$step_log_dir"
+allocation_state=$(squeue -h -j "$allocation" -o '%T|%N|%b')
+allocation_state_name=${allocation_state%%|*}
+allocation_rest=${allocation_state#*|}
+allocation_nodes=${allocation_rest%%|*}
+allocation_gres=${allocation_rest#*|}
+if [[ "$allocation_state_name" != RUNNING || "$allocation_nodes" != *"$expected_node"* || "$allocation_gres" != *gpu*2* ]]; then
+  echo "approved allocation is not RUNNING with two GPUs on the expected node: $allocation_state" >&2
+  exit 24
+fi
+mkdir -p "$(dirname "$attempt_root")"
+if ! mkdir "$attempt_root"; then
+  echo "another launcher reserved this attempt concurrently: $attempt_root" >&2
+  exit 23
+fi
+git -C "$repo_root" diff --quiet HEAD -- || { echo "training source has tracked edits" >&2; exit 25; }
+source_sha=$(git -C "$repo_root" rev-parse HEAD)
+git -C "$repo_root" archive --format=tar "$source_sha" > "$attempt_root/source.tar"
+printf '%s\n' "$source_sha" > "$attempt_root/source_sha.txt"
+cp "$windows_config" "$attempt_root/source_windows.yaml"
+windows_config="$attempt_root/source_windows.yaml"
+if [[ "$timing_gate_updates" != 0 ]]; then
+  cp "$launcher_root/tools/omni4d/cosmos3_nano/sample_training_host_memory.py" \
+    "$attempt_root/sample_training_host_memory.py"
+fi
+if [[ "$experiment_module" == "$next_version_omega_module" ]]; then
+  cp "$omega_statistics_path" "$attempt_root/omega_feature_statistics.safetensors"
+  omega_statistics_path="$attempt_root/omega_feature_statistics.safetensors"
+fi
+
+srun --jobid="$allocation" --kill-on-bad-exit=1 --overlap --exact --nodes=1 --ntasks=1 --cpus-per-task=55 --mem=644000M \
+  --gres=gpu:2 --gpu-bind=none --cpu-bind=none --chdir=/tmp \
+  --output="$step_log_dir/slurm-%j-%s.out" \
+  bash -c '
+    set -euo pipefail
+    ulimit -Sn 65536
+    printf "trainer_open_file_limit=%s\n" "$(ulimit -Sn)"
+    repo_root=$1
+    expected_node=$2
+    windows_config=$3
+    windows_sha256=$4
+    attempt_root=$5
+    python_bin=$6
+    source_sha=$7
+    allocated_gpu_uuids=$8
+    experiment_module=$9
+    warmstart_path=${10}
+    native_stall=${11}
+    next_version_ddp_module=${12}
+    next_version_full19_module=${13}
+    next_version_full19_cityscapes_module=${14}
+    microbatch_profile=${15}
+    overlay_sha256=${16}
+    overlay_update=${17}
+    overlay_config_identity=${18}
+    twenty_update_test=${19}
+    next_version_omega_module=${20}
+    omega_statistics_path=${21}
+    omega_statistics_sha256=${22}
+    next_version_full20_start_identity=${23}
+    next_version_full20_writer_identity=${24}
+    next_version_molmo_motion_module=${25}
+    next_version_city_molmo_module=${26}
+    next_version_city_molmo_writer_identity=${27}
+    timing_gate_updates=${28}
+    case "$microbatch_profile" in
+      b8_gas2) expected_batch=8; expected_accum=2 ;;
+      b16_gas1) expected_batch=16; expected_accum=1 ;;
+      *) echo "invalid microbatch profile in compute step: $microbatch_profile" >&2; exit 20 ;;
+    esac
+    [[ -x "$python_bin" ]] || { echo "compute-node Python is missing: $python_bin" >&2; exit 20; }
+    printf "compute_python=%s\n" "$python_bin"
+    [[ "$allocated_gpu_uuids" == GPU-*,GPU-* && "$allocated_gpu_uuids" != *,*,* ]] || { echo "expected exactly two explicit allocated GPU UUIDs" >&2; exit 32; }
+    printf "slurm_step_gpus=%s inherited_cuda_visible=%s explicit_gpu_uuids=%s\n" "${SLURM_STEP_GPUS:-unset}" "${CUDA_VISIBLE_DEVICES:-unset}" "$allocated_gpu_uuids"
+    export CUDA_VISIBLE_DEVICES="$allocated_gpu_uuids"
+    slurm_tasks=${SLURM_NTASKS:?Slurm task count is required}
+    [[ "$slurm_tasks" == 1 ]] || { echo "expected one Slurm task owning both GPUs, got $slurm_tasks" >&2; exit 30; }
+    [[ "$(hostname -s)" == "$expected_node" ]] || { echo "unexpected compute node $(hostname -s)" >&2; exit 31; }
+    [[ -n "${CUDA_VISIBLE_DEVICES:-}" ]] || {
+      echo "the single Slurm task must see both allocated GPUs; CUDA_VISIBLE_DEVICES is unset" >&2; exit 32;
+    }
+    mapfile -t gpu_uuids < <("$python_bin" -c "import torch; n=torch.cuda.device_count(); assert n == 2, n; print(chr(10).join(str(torch.cuda.get_device_properties(i).uuid) for i in range(n)))")
+    [[ ${#gpu_uuids[@]} == 2 ]] || { echo "could not enumerate two GPUs from CVD=$CUDA_VISIBLE_DEVICES" >&2; exit 33; }
+    uuid0=${gpu_uuids[0]}; uuid1=${gpu_uuids[1]}
+    [[ "$uuid0" == GPU-* ]] || uuid0="GPU-$uuid0"
+    [[ "$uuid1" == GPU-* ]] || uuid1="GPU-$uuid1"
+    [[ "$uuid0" == GPU-* && "$uuid1" == GPU-* && "$uuid0" != "$uuid1" ]] || {
+      echo "could not resolve two distinct physical UUIDs from CVD=$CUDA_VISIBLE_DEVICES: $uuid0 $uuid1" >&2; exit 33;
+    }
+    mkdir -p "$attempt_root/preflight"
+    printf "%s\n%s\n" "$uuid0" "$uuid1" > "$attempt_root/preflight/gpu_uuids.txt.tmp"
+    mv "$attempt_root/preflight/gpu_uuids.txt.tmp" "$attempt_root/preflight/gpu_uuids.txt"
+    for uuid in "$uuid0" "$uuid1"; do
+      foreign_pids=$(nvidia-smi --id="$uuid" --query-compute-apps=pid --format=csv,noheader | grep -v -i "^No running processes found$" || true)
+      [[ -z "$foreign_pids" ]] || { echo "GPU $uuid is already occupied by: $foreign_pids" >&2; exit 36; }
+    done
+    actual_sha=$(sha256sum "$windows_config" | awk "{print \$1}")
+    [[ "$actual_sha" == "$windows_sha256" ]] || { echo "source-config changed after launch preflight" >&2; exit 37; }
+    node_source="/tmp/${USER}_cosmos3_formal_${SLURM_JOB_ID}_${SLURM_STEP_ID}_${source_sha}"
+    mkdir "$node_source"
+    tar -xf "$attempt_root/source.tar" -C "$node_source"
+    printf "%s\n" "$source_sha" > "$node_source/.source_ready"
+    [[ "$(cat "$node_source/.source_ready")" == "$source_sha" ]] || { echo "immutable node-local source is not ready" >&2; exit 38; }
+    repo_root="$node_source"
+    export COSMOS_BOUND_GPU_UUIDS="$uuid0,$uuid1"
+    export COSMOS_TRAINING=1 WANDB_MODE=offline WANDB_DIR="$attempt_root/wandb"
+    export PYTHONNOUSERSITE=1 HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1
+    export OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 SYN4D_NPZ_CACHE_SIZE=64
+    export COSMOS_NODE_LOCAL_COMPILER_CACHE_ROOT="/tmp/${USER}_cosmos3_compiler_${SLURM_JOB_ID}_${SLURM_STEP_ID}"
+    export IMAGINAIRE_OUTPUT_ROOT=/scratch/shared/engs2907/logs/vgg_forecast_4d/train
+    export PYTHONPATH="$repo_root"
+    export COSMOS_VGG_WINDOWS_CONFIG="$windows_config"
+    export PATH="$(dirname "$python_bin"):$PATH"
+    torchcodec_library_path=$("$python_bin" "$repo_root/tools/omni4d/cosmos3_nano/prepare_torchcodec_runtime.py" --print-library-path)
+    export LD_LIBRARY_PATH="$torchcodec_library_path${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+    printf "torchcodec_ld_library_path=%s\n" "$torchcodec_library_path"
+    printf "slurm_tasks=1 host=%s cuda_visible=%s gpu_uuids=%s,%s\n" \
+      "$(hostname -s)" "$CUDA_VISIBLE_DEVICES" "$uuid0" "$uuid1"
+    cd "$repo_root"
+    "$python_bin" tools/omni4d/cosmos3_nano/check_shared_runtime.py --output "$attempt_root/runtime_import_receipt.json"
+    warmstart_args=()
+    recipe_overrides=()
+    if [[ -n "$warmstart_path" ]]; then
+      warmstart_args+=("model.trainable_overlay_path=$warmstart_path")
+    fi
+    if [[ "$experiment_module" == "$next_version_ddp_module" ]]; then
+      recipe_overrides+=(
+        "dataloader_train.batch_size=8"
+        "dataloader_train.num_workers=12"
+        "trainer.grad_accum_iter=2"
+      )
+      overlay_clock=$("$python_bin" -c "
+import json
+import math
+import sys
+from safetensors import safe_open
+
+expected = {
+    \"format\": \"omni4d_trainable_weights_v1\",
+    \"exact_training_resume\": \"false\",
+    \"update\": \"8200\",
+    \"base_identity\": \"nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92\",
+    \"config_identity\": \"cosmos3_nano_omni4d_stage_one_lora_r256_native_layer_masks_fp32_rope_full_checkpoint_retained_forward_parameters_global32\",
+}
+with safe_open(sys.argv[1], framework=\"pt\", device=\"cpu\") as stream:
+    metadata = stream.metadata() or {}
+    keys = stream.keys()
+    tensor_headers = [(stream.get_slice(key).get_dtype(), stream.get_slice(key).get_shape()) for key in keys]
+if any(metadata.get(key) != value for key, value in expected.items()):
+    raise ValueError(\"the DDP 21-frame run requires the strict update-8200 native trainable overlay\")
+names = json.loads(metadata.get(\"parameter_names\", \"null\"))
+if not isinstance(names, list) or len(keys) != 530 or len(names) != 530 or set(names) != set(keys):
+    raise ValueError(f\"update-8200 overlay must contain the exact 530 trainable tensors, got {len(keys)}\")
+if any(dtype != \"F32\" for dtype, _ in tensor_headers):
+    raise ValueError(\"update-8200 trainable overlay tensors must all be FP32\")
+parameters = sum(math.prod(shape) for _, shape in tensor_headers)
+if parameters != 747673344:
+    raise ValueError(f\"update-8200 overlay parameter count is {parameters}, expected 747673344\")
+print(8200)
+" "$warmstart_path")
+      warmstart_args+=("checkpoint.model_only_overlay_start_iteration=$overlay_clock")
+      printf "model_only_start_iteration=%s optimizer_state=fresh exact_training_resume=false\n" "$overlay_clock"
+    elif [[ "$experiment_module" == "$next_version_full19_module" || "$experiment_module" == "$next_version_omega_module" \
+       || "$experiment_module" == "$next_version_molmo_motion_module" || "$experiment_module" == "$next_version_city_molmo_module" ]]; then
+      if [[ "$experiment_module" == "$next_version_omega_module" ]]; then
+        [[ "$overlay_config_identity" == "$next_version_full20_start_identity" || "$overlay_config_identity" == "$next_version_full20_writer_identity" ]] || {
+          echo "the Omega-stream run starts from a reviewed five-stream overlay identity" >&2; exit 39;
+        }
+        actual_statistics_sha=$(sha256sum "$omega_statistics_path")
+        actual_statistics_sha=${actual_statistics_sha%% *}
+        [[ "$actual_statistics_sha" == "$omega_statistics_sha256" ]] || {
+          echo "Omega statistics SHA-256 mismatch: expected=$omega_statistics_sha256 got=$actual_statistics_sha" >&2; exit 39;
+        }
+        recipe_overrides+=(
+          "dataloader_train.main_process_collator.omega.statistics=$omega_statistics_path"
+          "dataloader_train.main_process_collator.omega.expected_statistics_sha256=$omega_statistics_sha256"
+        )
+        printf "omega_statistics_sha256=%s trainable_overlay_new_tensors=6 start_overlay_tensors=530\n" "$omega_statistics_sha256"
+      fi
+      actual_overlay_sha=$(sha256sum "$warmstart_path")
+      actual_overlay_sha=${actual_overlay_sha%% *}
+      [[ "$actual_overlay_sha" == "$overlay_sha256" ]] || {
+        echo "trainable overlay SHA-256 mismatch: expected=$overlay_sha256 got=$actual_overlay_sha" >&2; exit 39;
+      }
+      overlay_clock=$("$python_bin" -c "
+import json, math, sys
+from safetensors import safe_open
+path, expected_update, expected_identity = sys.argv[1], int(sys.argv[2]), sys.argv[3]
+expected = {
+    \"format\": \"omni4d_trainable_weights_v1\",
+    \"exact_training_resume\": \"false\",
+    \"update\": str(expected_update),
+    \"base_identity\": \"nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92\",
+    \"config_identity\": expected_identity,
+}
+with safe_open(path, framework=\"pt\", device=\"cpu\") as stream:
+    metadata = stream.metadata() or {}
+    keys = stream.keys()
+    headers = [(stream.get_slice(key).get_dtype(), stream.get_slice(key).get_shape()) for key in keys]
+if any(metadata.get(key) != value for key, value in expected.items()):
+    raise ValueError(f\"trainable overlay metadata mismatch for requested update {expected_update}\")
+names = json.loads(metadata.get(\"parameter_names\", \"null\"))
+if not isinstance(names, list) or len(keys) != 530 or len(names) != 530 or set(names) != set(keys):
+    raise ValueError(f\"trainable overlay must contain the exact 530 trainable tensors, got {len(keys)}\")
+if any(dtype != \"F32\" for dtype, _ in headers):
+    raise ValueError(\"trainable overlay tensors must all be FP32\")
+if sum(math.prod(shape) for _, shape in headers) != 747673344:
+    raise ValueError(\"trainable overlay parameter count differs from the native LoRA scope\")
+print(expected_update)
+" "$warmstart_path" "$overlay_update" "$overlay_config_identity")
+      warmstart_args+=("checkpoint.model_only_overlay_start_iteration=$overlay_clock")
+      warmstart_args+=("model.trainable_overlay_config_identity=$overlay_config_identity")
+      recipe_overrides+=(
+        "dataloader_train.batch_size=$expected_batch"
+        "dataloader_train.num_workers=12"
+        "trainer.grad_accum_iter=$expected_accum"
+        "dataloader_train.dataset.sample_offset_updates=$overlay_clock"
+        "dataloader_train.dataset.batch_size=$expected_batch"
+        "dataloader_train.dataset.gradient_accumulation_steps=$expected_accum"
+        "optimizer.lr=2.5e-5"
+      )
+      timing_prefetch_factor=2
+      if [[ "$microbatch_profile" == b16_gas1 ]]; then
+        recipe_overrides+=("dataloader_train.prefetch_factor=1")
+        timing_prefetch_factor=1
+      fi
+      if [[ "$twenty_update_test" == 1 ]]; then
+        test_end_update=$((overlay_clock + 20))
+        recipe_overrides+=("trainer.max_iter=$test_end_update" "trainer.callbacks.omni4d_trainable_overlay.save_updates=[$test_end_update]")
+      fi
+      if [[ "$timing_gate_updates" != 0 ]]; then
+        recipe_overrides+=(
+          "trainer.callbacks.iter_speed.native_phase_timing.output_dir=$attempt_root/native_phase_timings"
+          "trainer.callbacks.iter_speed.native_phase_timing.start_after_updates=1"
+          "trainer.callbacks.iter_speed.native_phase_timing.num_updates=$timing_gate_updates"
+        )
+        printf "timing_observation_updates=%s workers=12 prefetch_factor=%s host_memory_sample_seconds=10 training_continues=true\n" \
+          "$timing_gate_updates" "$timing_prefetch_factor"
+      fi
+      printf "model_only_start_iteration=%s optimizer_state=fresh exact_training_resume=false overlay_sha256=%s profile=%s lr=2.5e-5\n" "$overlay_clock" "$actual_overlay_sha" "$microbatch_profile"
+      if [[ "$twenty_update_test" == 1 ]]; then
+        printf "bounded_admission_test_updates=20 checkpoint_update=%s\n" "$test_end_update"
+      fi
+    elif [[ "$experiment_module" == cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_replicate_only_500k \
+       || "$experiment_module" == cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_replicate_only_500k_temporal_forcing \
+       || "$experiment_module" == cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_plain_ddp_500k \
+       || "$experiment_module" == cosmos_framework.configs.base.experiment.sft.omni4d_stage_one_lora_r256_next_version_all_sources_21_frames_temporal_forcing_from_7300 ]]; then
+      overlay_clock=$("$python_bin" -c "
+import sys
+from safetensors import safe_open
+with safe_open(sys.argv[1], framework=\"pt\", device=\"cpu\") as stream:
+    metadata = stream.metadata()
+if metadata.get(\"format\") != \"omni4d_trainable_weights_v1\" or metadata.get(\"exact_training_resume\") != \"false\":
+    raise ValueError(\"500K handoff requires a trainable-only model-weight overlay\")
+update = int(metadata[\"update\"])
+if not 0 < update < 500000:
+    raise ValueError(\"overlay update must leave work in the configured 500K horizon\")
+print(update)
+" "$warmstart_path")
+      warmstart_args+=("checkpoint.model_only_overlay_start_iteration=$overlay_clock")
+      printf "model_only_start_iteration=%s optimizer_state=fresh exact_training_resume=false\n" "$overlay_clock"
+    fi
+    nvidia-smi --id="$uuid0,$uuid1" --query-gpu=timestamp,uuid,utilization.gpu,memory.used,power.draw --format=csv,noheader -l 1 > "$attempt_root/gpu_utilization.csv" 2> "$attempt_root/gpu_utilization.err" &
+    metrics_pid=$!
+    nvidia-smi --id="$uuid0,$uuid1" --query-compute-apps=gpu_uuid,pid,used_gpu_memory --format=csv,noheader -l 10 > "$attempt_root/gpu_processes.csv" 2> "$attempt_root/gpu_processes.err" &
+    process_metrics_pid=$!
+    host_memory_pid=""
+    if [[ "$timing_gate_updates" != 0 ]]; then
+      "$python_bin" "$attempt_root/sample_training_host_memory.py" \
+        --root-pid "$$" --output "$attempt_root/host_memory.csv" --interval-seconds 10 &
+      host_memory_pid=$!
+    fi
+    diagnostics_pid=""
+    if [[ "$native_stall" == 1 ]]; then
+      native_stall_options="output_dir:$attempt_root/native_stall,source_identity:${source_sha}_windows_${windows_sha256}_overlay_$(basename "$warmstart_path"),signal:SIGUSR2,progress_timeout_seconds:300,max_updates:500000"
+      if [[ "$twenty_update_test" == 1 ]]; then
+        native_stall_options+=",capture_batch_details_updates:20"
+      elif [[ "$timing_gate_updates" != 0 ]]; then
+        native_stall_options+=",capture_batch_details_updates:$timing_gate_updates"
+      fi
+      warmstart_args+=("trainer.callbacks.iter_speed.native_stall_diagnostics={$native_stall_options}")
+      (
+        while true; do
+          if compgen -G "$attempt_root/native_stall/rank_*_pid_*.handler.json" >/dev/null; then
+            "$python_bin" tools/omni4d/watch_native_training_stall.py "$attempt_root/native_stall" --send-signal >> "$attempt_root/native_stall_watchdog.log" 2>&1 || true
+          fi
+          sleep 30
+        done
+      ) &
+      diagnostics_pid=$!
+    fi
+    trap "kill $metrics_pid $process_metrics_pid ${diagnostics_pid:-} ${host_memory_pid:-} 2>/dev/null || true" EXIT
+    "$python_bin" -m torch.distributed.run --standalone --nnodes=1 --nproc-per-node=2 --max-restarts=0 \
+      -m cosmos_framework.scripts.train_experiment \
+      --config=cosmos_framework/configs/omni4d_native_config.py \
+      --experiment-module="$experiment_module" \
+      -- \
+      experiment="${experiment_module##*.}" \
+      job.name="$(basename "$(dirname "$attempt_root")")_$(basename "$attempt_root")" \
+      dataloader_train.dataset.windows_config_path="$windows_config" \
+      dataloader_train.dataset.expected_windows_config_sha256="$windows_sha256" \
+      trainer.callbacks.omni4d_trainable_overlay.output_dir="$attempt_root/trainable_overlays" \
+      "${warmstart_args[@]}" "${recipe_overrides[@]}"
+  ' bash "$repo_root" "$expected_node" "$windows_config" "$windows_sha256" "$attempt_root" "$python_bin" \
+    "$source_sha" "$allocated_gpu_uuids" "$experiment_module" "$warmstart_path" "$native_stall" "$next_version_ddp_module" "$next_version_full19_module" \
+    "$next_version_full19_cityscapes_module" "$microbatch_profile" "$overlay_sha256" "$overlay_update" "$overlay_config_identity" "$twenty_update_test" \
+    "$next_version_omega_module" "$omega_statistics_path" "$omega_statistics_sha256" "$next_version_full20_start_identity" "$next_version_full20_writer_identity" \
+    "$next_version_molmo_motion_module" "$next_version_city_molmo_module" "$next_version_city_molmo_writer_identity" "$timing_gate_updates"