Kuavo original action: multi_task_dit checkpoint 10K

v2/multi_task_dit/checkpoints/checkpoint-10K/README.md

@@ -0,0 +1,7 @@
+# multi_task_dit: original-action Kuavo, checkpoint 10000 of 30000 updates
+
+Original v3 data; 30Hz to 10Hz logical view; state20/action23; seed42; actual batch128; accumulation1.
+Torso targets are the original action commands. Invalid action elements are masked.
+N1.5/DiT use LeTools-pinned LeRobot; N1.7 uses bundled native model, processor, sharded loader and Trainer.
+LeTools revision: 6d2002613241ae2a32c5d8fd753f80caa1383fba. See provenance for adapter and YAML.
+N1.7 uses native checkpoint layout. Optimizer/RNG states stay on the training server.

v2/multi_task_dit/checkpoints/checkpoint-10K/checkpoint_manifest.json

@@ -0,0 +1,302 @@
+{
+  "schema": 2,
+  "policy": "multi_task_dit",
+  "step": 10000,
+  "repo_id": "RwH-Lab/Kuavo_ckpt",
+  "remote_prefix": "v2/multi_task_dit/checkpoints/checkpoint-10K",
+  "source_checkpoint": "/home/work/mntvol/RwH-Kuavo_ckpt/v3_rawaction/multi_task_dit/checkpoints/010000",
+  "source_signature": {
+    "pretrained_model/config.json": [
+      2964,
+      1793104970876621208
+    ],
+    "pretrained_model/model.safetensors": [
+      1110874532,
+      1793104972611445238
+    ],
+    "pretrained_model/policy_postprocessor.json": [
+      659,
+      1793104972670128375
+    ],
+    "pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors": [
+      2860,
+      1793104934154675628
+    ],
+    "pretrained_model/policy_preprocessor.json": [
+      1896,
+      1793104939229702410
+    ],
+    "pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors": [
+      2860,
+      1793104939226303274
+    ],
+    "pretrained_model/train_config.json": [
+      7322,
+      1793104939222364217
+    ],
+    "training_state/optimizer_param_groups.json": [
+      5342,
+      1793104936859016090
+    ],
+    "training_state/optimizer_state.safetensors": [
+      1716334020,
+      1793104977322473307
+    ],
+    "training_state/rng_state.safetensors": [
+      15708,
+      1793104974693417186
+    ],
+    "training_state/scheduler_state.json": [
+      333,
+      1793104977390581314
+    ],
+    "training_state/training_step.json": [
+      21,
+      1793104974690185169
+    ]
+  },
+  "tensor_count": 505,
+  "training_state_included": false,
+  "files": {
+    "README.md": {
+      "size": 554,
+      "sha256": "6abb178b18332426d2530aff35e55a94bb088ea56fa074414e43741ea5afc520",
+      "git_blob_sha1": "9ce80cd130087c6817b6f3ff8ba31c25fe641d40"
+    },
+    "pretrained_model/config.json": {
+      "size": 2964,
+      "sha256": "c034fc26d286314eb58b28af9c4cb4f6bb6ffb0983ac642fd50a36a3bcdd50a3",
+      "git_blob_sha1": "78836fe1300e29eb2be5395290772c72a62d7469"
+    },
+    "pretrained_model/model.safetensors": {
+      "size": 1110874532,
+      "sha256": "08faf5e4134d7620e237f894aa02d2c102b6870adb4c01f8e575b2ca27a03ef0",
+      "git_blob_sha1": "8fc556098dd6ce42e6cc5e73017539bc2a8b85b2"
+    },
+    "pretrained_model/policy_postprocessor.json": {
+      "size": 659,
+      "sha256": "f953e66e6857340ed76934257dcf9e00b18a12a865b0f4f2b332d1a285839dad",
+      "git_blob_sha1": "4c44db41a739258cef9f98302259f9b0d15a501f"
+    },
+    "pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors": {
+      "size": 2860,
+      "sha256": "e1e9ad65b2902a52587822fa5121e86fadf0c3a2469c4a014f34707b416b1bbd",
+      "git_blob_sha1": "b9afb173e9a4ee05a75acd072865d6ceef64da8f"
+    },
+    "pretrained_model/policy_preprocessor.json": {
+      "size": 1896,
+      "sha256": "30892a46fe786931cc7b131d7c3b55e227f0284f201b6ee1ce95034cb93ce725",
+      "git_blob_sha1": "0a5c7c55adc0ef1bef64d8697a261730e4dfc7de"
+    },
+    "pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors": {
+      "size": 2860,
+      "sha256": "e1e9ad65b2902a52587822fa5121e86fadf0c3a2469c4a014f34707b416b1bbd",
+      "git_blob_sha1": "b9afb173e9a4ee05a75acd072865d6ceef64da8f"
+    },
+    "pretrained_model/train_config.json": {
+      "size": 7322,
+      "sha256": "182a15faf8d65a918ac2481eb1ffebac7a0c1b45f0da76393fd2fb5c27908f2a",
+      "git_blob_sha1": "e7d64337a1286dddae7e0b8b518a20a80ca0daca"
+    },
+    "provenance/configs/Bs128_30k/data/DATA_CHANGES.md": {
+      "size": 509,
+      "sha256": "62b56e19aca04147363650a7ca23cf6bff3b9f9e780978cbc7b9ad58fb5782ea",
+      "git_blob_sha1": "09a3c536857a6ef41d7ca8af865fdf1067097232"
+    },
+    "provenance/configs/Bs128_30k/data/dataset.yaml": {
+      "size": 592,
+      "sha256": "a95789a682ec84e2e5b596b2d0c5da8a11ad3abca6d1c39ce6dd9b3942203bc6",
+      "git_blob_sha1": "24953451806778143fa67a9308fdbff31ac7b51c"
+    },
+    "provenance/configs/Bs128_30k/data/prepare.py": {
+      "size": 361,
+      "sha256": "06cef117826af42d15421c0a2c709120f00c181f38f2873f7dc5a0bc17d447de",
+      "git_blob_sha1": "176bdbae5d88c8dca293ec71161f657b5c97c237"
+    },
+    "provenance/configs/Bs128_30k/data/verify.py": {
+      "size": 365,
+      "sha256": "4df3c512a6c0d810c6d11f6e7fdb99b46407adfe43fdea94939b2bfdf5077a80",
+      "git_blob_sha1": "60f2f83e6a01bb0d9274c2d4e49054401dc7d0c3"
+    },
+    "provenance/configs/Bs128_30k/train/CONTRACT.md": {
+      "size": 1481,
+      "sha256": "6a6793237591e0ffc8178266b483308fe53e7c5a7f206626bb284a2f6cf9772a",
+      "git_blob_sha1": "630bc7d682901a222bb7f7ec09dc0b66541e6db7"
+    },
+    "provenance/configs/Bs128_30k/train/compat.py": {
+      "size": 136,
+      "sha256": "ab948478240b23d037e7623711bf3cc0053db72f43ed9af218a04412cecde6af",
+      "git_blob_sha1": "55b0ae6c9979f0d329194a99129e248cabc6ea30"
+    },
+    "provenance/configs/Bs128_30k/train/gr00t_n15.yaml": {
+      "size": 1144,
+      "sha256": "04686afff2f338dfb2067e7669a8675fd0cc354a1a2f9f7e22d794fadd4d81b5",
+      "git_blob_sha1": "f3e3cea3aee9e2532941516482b7c96a3ab3ad2a"
+    },
+    "provenance/configs/Bs128_30k/train/gr00t_n17.yaml": {
+      "size": 1382,
+      "sha256": "2620d1aa6e5328f2f6e870ba9179997c5567acb2c31b69f786dc8861551867d8",
+      "git_blob_sha1": "58529a1277c2ac7766ea775b80bc38052a1ac79a"
+    },
+    "provenance/configs/Bs128_30k/train/hf_watcher.py": {
+      "size": 346,
+      "sha256": "aef5d741bf30a567341651680ec0c89919e3be3ef67b373765ea707dd091e0a6",
+      "git_blob_sha1": "d933df64b4fd46d5bd12840ef12fca28671c5f98"
+    },
+    "provenance/configs/Bs128_30k/train/launch.sh": {
+      "size": 811,
+      "sha256": "a267aadd48e1ac00c564561e89715406a2d653a7ea4fe3dfa8c23f0c56210e10",
+      "git_blob_sha1": "e5ce544f5645597ca960084933aad619f24cba28"
+    },
+    "provenance/configs/Bs128_30k/train/load_n17.py": {
+      "size": 136,
+      "sha256": "e509c8ff64e9bfb65e8a0027b9bf6b88ab453b4882f2d7e93a41da63843ec3cd",
+      "git_blob_sha1": "85eed0b4356436f2bd2ded2f2241e4ff92fdd065"
+    },
+    "provenance/configs/Bs128_30k/train/loss_mask.py": {
+      "size": 143,
+      "sha256": "6b46d48cd5d7075721d44fb5b1fb17e942b2eacb51f0e31020248b77bd6690bb",
+      "git_blob_sha1": "5d98223ccfd34385f362ea36487a09c3ae691004"
+    },
+    "provenance/configs/Bs128_30k/train/multi_task_dit.yaml": {
+      "size": 737,
+      "sha256": "8c2bbe6729e83e387dc30d919eb156d7da6a4f3dd89720771c1eeea51e985573",
+      "git_blob_sha1": "9ed3273a427ee67a94f61a24109fde181bfaa2de"
+    },
+    "provenance/configs/Bs128_30k/train/native_assets.py": {
+      "size": 138,
+      "sha256": "acb80a4b8e58348941f0aff6684c9db8d3508665c4e920749cc77d895bc55318",
+      "git_blob_sha1": "383fc3571cb09c3fa01c5f2b4a2602e1e8616fd1"
+    },
+    "provenance/configs/Bs128_30k/train/orchestrate.py": {
+      "size": 347,
+      "sha256": "960224e8968a3181cf246c6c24c89ffac4fa6b8f1f7b00ada1dbf773d9fffbb6",
+      "git_blob_sha1": "3bdadd272cde613eeb9b679f7872b99ee26a34e8"
+    },
+    "provenance/configs/Bs128_30k/train/run.py": {
+      "size": 347,
+      "sha256": "01d3c3e053fa01937763eb548ad8ce0e4527a9c19caeb1d41ec7da496f5ef550",
+      "git_blob_sha1": "d1105742fba295f481404a112e1f162b22bc0d34"
+    },
+    "provenance/configs/Bs128_30k/train/run_n17.py": {
+      "size": 343,
+      "sha256": "4c0724e714c320fc444f9b997c09be2582dc0ce5851259db5aa4eca4a486c522",
+      "git_blob_sha1": "61ade41949124d5f2cd9b75eaa269db154d9998e"
+    },
+    "provenance/configs/Bs128_30k/train/total/gr00t_n15_total.yaml": {
+      "size": 5745,
+      "sha256": "a16c6a1b7733da06523a2f63f7eb227c84bbddf27c18343a324004db7b2f16aa",
+      "git_blob_sha1": "8cfa3cff2322fb2523570037a5790f9bf6168669"
+    },
+    "provenance/configs/Bs128_30k/train/total/multi_task_dit_total.yaml": {
+      "size": 6499,
+      "sha256": "45b92a3935b04e13f531839e414e586f195422925d33f02b15c9615dce77c310",
+      "git_blob_sha1": "a1cd3d6af83ded29518089384c7840b4ea9e4986"
+    },
+    "provenance/scripts/__init__.py": {
+      "size": 65,
+      "sha256": "d5cb3b6372ce5367b0e1c6a3aabcacf5ae0c69c1f123c3fdc768dbdab86615fd",
+      "git_blob_sha1": "b60a71c83c559cc4515886759eb03bff532ef809"
+    },
+    "provenance/scripts/adapters/__init__.py": {
+      "size": 85,
+      "sha256": "d66d70788ab53bdfd8e0235ba7150604476d3654dece06d0871aa512bbf02db0",
+      "git_blob_sha1": "ed130013a85392ebd51771dbb14d18e0f3aa164a"
+    },
+    "provenance/scripts/adapters/action_validity.py": {
+      "size": 4223,
+      "sha256": "28284a5c5725a0cf8391c740818a25adf6e946845772db82801533c8784605e8",
+      "git_blob_sha1": "96b7d695011f8f41745245ceba576754c3714f44"
+    },
+    "provenance/scripts/adapters/groot_v3.py": {
+      "size": 4834,
+      "sha256": "f23c095efaefcaa3bbeb20487ba70141520783ffd820cf39354b9ee1a93463b1",
+      "git_blob_sha1": "577fef6bfdb8f6bd13c4b8ee48316fabc5fd1809"
+    },
+    "provenance/scripts/adapters/language.py": {
+      "size": 2378,
+      "sha256": "0c2d35f55e8a43f1772bc1f92d1582dc8c8696b330a4c999c8188a440fe7adfa",
+      "git_blob_sha1": "693cbaede81dc7ab903d42a3279f01b1b33240a7"
+    },
+    "provenance/scripts/data/__init__.py": {
+      "size": 38,
+      "sha256": "d3bc18a39403772a7ab3803cb57fd46fa6b47c10140162291029a1800b62cfc0",
+      "git_blob_sha1": "263a31b0589ca2481bfddad6515eb3a86a645c12"
+    },
+    "provenance/scripts/data/prepare.py": {
+      "size": 492,
+      "sha256": "f37d9e1c00c0abc08c32851ea50931c5a810a161fb4d0b602a916ac120bd6bb6",
+      "git_blob_sha1": "343931e66cea374ec8beb6d1e94830a67c167bb3"
+    },
+    "provenance/scripts/data/validate.py": {
+      "size": 3761,
+      "sha256": "ea0a55b480136c519baaae589ad527a4953338ed8894a89fce80c33ffdd1ad93",
+      "git_blob_sha1": "564aeb727b01544a83e48f40f48dbe4026e4f75c"
+    },
+    "provenance/scripts/data/view.py": {
+      "size": 10237,
+      "sha256": "ca7f03c7fca94575b9e089bed2727981406791b8ddbf382b61059bc3bbf937e6",
+      "git_blob_sha1": "7c86c48b6815e79f303edd54a4bf4db45b89f339"
+    },
+    "provenance/scripts/tests/test_contract.py": {
+      "size": 3756,
+      "sha256": "c240afb5d8154783c0a86fd5cf26cd330ed5ec56ae4d6e236281eccf78b351c6",
+      "git_blob_sha1": "b0d4c7630e4abe15f4fb9b673456714bce7bf450"
+    },
+    "provenance/scripts/train/__init__.py": {
+      "size": 34,
+      "sha256": "5f51caf72525efcf81ce8bdfc06321746fd185d4c932f734a3b0ffa141af74ce",
+      "git_blob_sha1": "61ef7af826bb64fd194a8cabe53818989f3cbcdb"
+    },
+    "provenance/scripts/train/checkpoints.py": {
+      "size": 1766,
+      "sha256": "c5feb6d74497a28945f574fad39846c6d8e16f9798bb7fbe60f23af65c5dcb78",
+      "git_blob_sha1": "869f01e29a3cf6fd343df5589718dd1d2f82b2cf"
+    },
+    "provenance/scripts/train/hf_watcher.py": {
+      "size": 18729,
+      "sha256": "5bced15077e5d0b63c2baeb5b3a05e5f21e9e69c26fc3ed1363c4140ee0ac5dd",
+      "git_blob_sha1": "0cb4da36c3ca84ee20e4f4b9db91ea30a71dddd1"
+    },
+    "provenance/scripts/train/language_guard.py": {
+      "size": 683,
+      "sha256": "80bb7e687c84c7120c90c2a1b74ad57ba9c0c724ad42e3dbd62656c94fb321ab",
+      "git_blob_sha1": "a6307b5c104e20fadaf96340b1686e5fd8da8164"
+    },
+    "provenance/scripts/train/load_n17.py": {
+      "size": 1110,
+      "sha256": "1ce062dcbedc08396abb041d2f33b1c0f249bfcc094898757d504394fa4a089c",
+      "git_blob_sha1": "2b24ef4fbf614dca5da4f4a64807faa4440e5a7d"
+    },
+    "provenance/scripts/train/native_assets.py": {
+      "size": 2450,
+      "sha256": "01ddf2cbef9fe60131a90b0b4866014c9ad7e617f98bad4cd548065658845b3d",
+      "git_blob_sha1": "30b2c0504938f8d9e32bdde5b387687a2aba6e70"
+    },
+    "provenance/scripts/train/orchestrate.py": {
+      "size": 4554,
+      "sha256": "7c0111f58ea1031959cbfcd64139df9af34220ac242280fb5da2e883db0b9cb1",
+      "git_blob_sha1": "2aac4e118eae079cb0f6d941c6ebdda6223f12e8"
+    },
+    "provenance/scripts/train/run_lerobot.py": {
+      "size": 15482,
+      "sha256": "ad58a8ae3091483da73293370efe4234d3e74eb6b2f694a5fa773f1a15d7fb16",
+      "git_blob_sha1": "13de7a3d0f3bce212317426443ea94b1647fcda9"
+    },
+    "provenance/scripts/train/run_n17.py": {
+      "size": 9892,
+      "sha256": "f76c2e7aa09bcb381b712399b18ea1be76767ba1cc2a4e564b01c2c439cfb313",
+      "git_blob_sha1": "968d716454f9fb611686d4fa570d940fe46b7811"
+    },
+    "provenance/scripts/train/start_watchers.py": {
+      "size": 1155,
+      "sha256": "7440fea95063139b0867489e3c82d0899fe89b63b30501038563eefa7410d003",
+      "git_blob_sha1": "aeb5b7bbc727f3b338b698a64e7a82e7bca0ba75"
+    },
+    "provenance/scripts/train/validate_checkpoint.py": {
+      "size": 6027,
+      "sha256": "b3bc7aeb656654b303b65dcb48ae1e2030b6297bfd42cdad69cd364262a378a1",
+      "git_blob_sha1": "7d4b92277a4f867b0498f6610d7765819f4b7880"
+    }
+  }
+}

v2/multi_task_dit/checkpoints/checkpoint-10K/pretrained_model/config.json

@@ -0,0 +1,110 @@
+{
+    "type": "multi_task_dit",
+    "n_obs_steps": 2,
+    "input_features": {
+        "observation.images.head": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.images.left_wrist": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.images.right_wrist": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                20
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                23
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "use_peft": false,
+    "push_to_hub": false,
+    "repo_id": null,
+    "private": null,
+    "tags": null,
+    "license": null,
+    "pretrained_path": null,
+    "horizon": 32,
+    "n_action_steps": 24,
+    "objective": "diffusion",
+    "noise_scheduler_type": "DDPM",
+    "num_train_timesteps": 100,
+    "beta_schedule": "squaredcos_cap_v2",
+    "beta_start": 0.0001,
+    "beta_end": 0.02,
+    "prediction_type": "epsilon",
+    "clip_sample": true,
+    "clip_sample_range": 1.0,
+    "num_inference_steps": null,
+    "sigma_min": 0.0,
+    "num_integration_steps": 100,
+    "integration_method": "euler",
+    "timestep_sampling_strategy": "beta",
+    "timestep_sampling_s": 0.999,
+    "timestep_sampling_alpha": 1.5,
+    "timestep_sampling_beta": 1.0,
+    "hidden_dim": 512,
+    "num_layers": 6,
+    "num_heads": 8,
+    "dropout": 0.1,
+    "use_positional_encoding": false,
+    "timestep_embed_dim": 256,
+    "use_rope": true,
+    "rope_base": 10000.0,
+    "vision_encoder_name": "/home/work/.cache/huggingface/hub/models--openai--clip-vit-base-patch16/snapshots/57c216476eefef5ab752ec549e440a49ae4ae5f3",
+    "use_separate_rgb_encoder_per_camera": false,
+    "vision_encoder_lr_multiplier": 0.1,
+    "image_resize_shape": null,
+    "image_crop_shape": [
+        224,
+        224
+    ],
+    "image_crop_is_random": true,
+    "text_encoder_name": "/home/work/.cache/huggingface/hub/models--openai--clip-vit-base-patch16/snapshots/57c216476eefef5ab752ec549e440a49ae4ae5f3",
+    "tokenizer_max_length": 77,
+    "tokenizer_padding": "max_length",
+    "tokenizer_padding_side": "right",
+    "tokenizer_truncation": true,
+    "normalization_mapping": {
+        "VISUAL": "MEAN_STD",
+        "STATE": "MIN_MAX",
+        "ACTION": "MIN_MAX"
+    },
+    "optimizer_lr": 2e-05,
+    "optimizer_betas": [
+        0.95,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 0.0,
+    "scheduler_name": "cosine",
+    "scheduler_warmup_steps": 0,
+    "do_mask_loss_for_padding": false,
+    "drop_n_last_frames": 7
+}

v2/multi_task_dit/checkpoints/checkpoint-10K/pretrained_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08faf5e4134d7620e237f894aa02d2c102b6870adb4c01f8e575b2ca27a03ef0
+size 1110874532

v2/multi_task_dit/checkpoints/checkpoint-10K/pretrained_model/policy_postprocessor.json

@@ -0,0 +1,32 @@
+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "unnormalizer_processor",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              23
+            ]
+          }
+        },
+        "norm_map": {
+          "VISUAL": "MEAN_STD",
+          "STATE": "MIN_MAX",
+          "ACTION": "MIN_MAX"
+        }
+      },
+      "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors"
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

v2/multi_task_dit/checkpoints/checkpoint-10K/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1e9ad65b2902a52587822fa5121e86fadf0c3a2469c4a014f34707b416b1bbd
+size 2860

v2/multi_task_dit/checkpoints/checkpoint-10K/pretrained_model/policy_preprocessor.json

@@ -0,0 +1,83 @@
+{
+  "name": "policy_preprocessor",
+  "steps": [
+    {
+      "registry_name": "rename_observations_processor",
+      "config": {
+        "rename_map": {}
+      }
+    },
+    {
+      "registry_name": "to_batch_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "tokenizer_processor",
+      "config": {
+        "max_length": 77,
+        "task_key": "task",
+        "padding_side": "right",
+        "padding": "max_length",
+        "truncation": true,
+        "tokenizer_name": "/home/work/.cache/huggingface/hub/models--openai--clip-vit-base-patch16/snapshots/57c216476eefef5ab752ec549e440a49ae4ae5f3"
+      }
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cuda",
+        "float_dtype": null
+      }
+    },
+    {
+      "registry_name": "normalizer_processor",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "observation.images.head": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              224,
+              224
+            ]
+          },
+          "observation.images.left_wrist": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              224,
+              224
+            ]
+          },
+          "observation.images.right_wrist": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              224,
+              224
+            ]
+          },
+          "observation.state": {
+            "type": "STATE",
+            "shape": [
+              20
+            ]
+          },
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              23
+            ]
+          }
+        },
+        "norm_map": {
+          "VISUAL": "MEAN_STD",
+          "STATE": "MIN_MAX",
+          "ACTION": "MIN_MAX"
+        }
+      },
+      "state_file": "policy_preprocessor_step_4_normalizer_processor.safetensors"
+    }
+  ]
+}

v2/multi_task_dit/checkpoints/checkpoint-10K/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1e9ad65b2902a52587822fa5121e86fadf0c3a2469c4a014f34707b416b1bbd
+size 2860

v2/multi_task_dit/checkpoints/checkpoint-10K/pretrained_model/train_config.json

@@ -0,0 +1,249 @@
+{
+    "dataset": {
+        "repo_id": "RwH-Lab/rwh-auto-step",
+        "root": "/home/work/mntvol/data/RwH-Kuavo-v2/rwh-auto-step",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "affine": {
+                    "weight": 1.0,
+                    "type": "RandomAffine",
+                    "kwargs": {
+                        "degrees": [
+                            -5.0,
+                            5.0
+                        ],
+                        "translate": [
+                            0.05,
+                            0.05
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "pyav",
+        "return_uint8": false,
+        "streaming": false
+    },
+    "env": null,
+    "policy": {
+        "type": "multi_task_dit",
+        "n_obs_steps": 2,
+        "input_features": {
+            "observation.images.head": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.left_wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.right_wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    20
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    23
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "use_peft": false,
+        "push_to_hub": false,
+        "repo_id": null,
+        "private": null,
+        "tags": null,
+        "license": null,
+        "pretrained_path": null,
+        "horizon": 32,
+        "n_action_steps": 24,
+        "objective": "diffusion",
+        "noise_scheduler_type": "DDPM",
+        "num_train_timesteps": 100,
+        "beta_schedule": "squaredcos_cap_v2",
+        "beta_start": 0.0001,
+        "beta_end": 0.02,
+        "prediction_type": "epsilon",
+        "clip_sample": true,
+        "clip_sample_range": 1.0,
+        "num_inference_steps": null,
+        "sigma_min": 0.0,
+        "num_integration_steps": 100,
+        "integration_method": "euler",
+        "timestep_sampling_strategy": "beta",
+        "timestep_sampling_s": 0.999,
+        "timestep_sampling_alpha": 1.5,
+        "timestep_sampling_beta": 1.0,
+        "hidden_dim": 512,
+        "num_layers": 6,
+        "num_heads": 8,
+        "dropout": 0.1,
+        "use_positional_encoding": false,
+        "timestep_embed_dim": 256,
+        "use_rope": true,
+        "rope_base": 10000.0,
+        "vision_encoder_name": "/home/work/.cache/huggingface/hub/models--openai--clip-vit-base-patch16/snapshots/57c216476eefef5ab752ec549e440a49ae4ae5f3",
+        "use_separate_rgb_encoder_per_camera": false,
+        "vision_encoder_lr_multiplier": 0.1,
+        "image_resize_shape": null,
+        "image_crop_shape": [
+            224,
+            224
+        ],
+        "image_crop_is_random": true,
+        "text_encoder_name": "/home/work/.cache/huggingface/hub/models--openai--clip-vit-base-patch16/snapshots/57c216476eefef5ab752ec549e440a49ae4ae5f3",
+        "tokenizer_max_length": 77,
+        "tokenizer_padding": "max_length",
+        "tokenizer_padding_side": "right",
+        "tokenizer_truncation": true,
+        "normalization_mapping": {
+            "VISUAL": "MEAN_STD",
+            "STATE": "MIN_MAX",
+            "ACTION": "MIN_MAX"
+        },
+        "optimizer_lr": 2e-05,
+        "optimizer_betas": [
+            0.95,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 0.0,
+        "scheduler_name": "cosine",
+        "scheduler_warmup_steps": 0,
+        "do_mask_loss_for_padding": false,
+        "drop_n_last_frames": 7
+    },
+    "output_dir": "/home/work/mntvol/RwH-Kuavo_ckpt/v3_rawaction/multi_task_dit",
+    "job_name": "rawaction_bs128_30k_multi_task_dit",
+    "resume": false,
+    "seed": 42,
+    "cudnn_deterministic": false,
+    "num_workers": 4,
+    "batch_size": 128,
+    "prefetch_factor": 4,
+    "persistent_workers": true,
+    "steps": 30000,
+    "eval_freq": 0,
+    "log_freq": 100,
+    "tolerance_s": 0.0001,
+    "save_checkpoint": true,
+    "save_freq": 10000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adam",
+        "lr": 2e-05,
+        "weight_decay": 0.0,
+        "grad_clip_norm": 10.0,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 0,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": true
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": true,
+        "project": "kuavo",
+        "entity": "minje227_hyu-hanyang-university",
+        "notes": null,
+        "run_id": "bee4vl8k",
+        "mode": "online",
+        "add_tags": true
+    },
+    "peft": null,
+    "use_rabc": false,
+    "rabc_progress_path": null,
+    "rabc_kappa": 0.01,
+    "rabc_epsilon": 1e-06,
+    "rabc_head_mode": "sparse",
+    "rename_map": {},
+    "checkpoint_path": null
+}

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/data/DATA_CHANGES.md

@@ -0,0 +1,7 @@
+# Original v3 logical view
+
+`dataset.yaml` is shared by N1.5,N1.7,MultiTaskDiT. No data.output.root and no v3->v2 conversion.
+Rows/timestamps, task episodes and normalization statistics are prepared once per process.
+Videos stay original; each backend decodes requested timestamps and uses its native prefetch.
+Torso action and validity come from original action/validity.action. No future-state substitution.
+Reports/indices/statistics are stored under the training output _audit, not in the source dataset.

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/data/dataset.yaml

@@ -0,0 +1,31 @@
+source:
+  repo_id: RwH-Lab/rwh-auto-step
+  revision: 55e6893084d2a1dd1906e09c6561cc29edc420bf
+  root: /home/work/mntvol/data/RwH-Kuavo-v2/rwh-auto-step
+sampling:
+  source_fps: 30
+  fps: 10
+  split_at_task_change: true
+state:
+  source: observation.state
+  indices:
+  - 0
+  - 20
+  invalid_stored_zero: keep_as_input
+action:
+  source: action
+  indices:
+  - 0
+  - 23
+  validity_feature: validity.action
+  invalid: mask_loss
+statistics:
+  state: all_inputs
+  action: valid_targets
+video:
+  mode: original
+  alignment: timestamp
+adapters:
+  gr00t_n17:
+    enabled: true
+    type: letools_native_v3

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/data/prepare.py

@@ -0,0 +1,8 @@
+"""Compatibility entry point; audit the original-data view without conversion."""
+import runpy
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
+if '--config' not in sys.argv:
+    sys.argv += ['--config', str(Path(__file__).with_name('dataset.yaml'))]
+runpy.run_module('scripts.data.prepare', run_name='__main__')

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/data/verify.py

@@ -0,0 +1,8 @@
+"""Compatibility entry point for exhaustive source/view and sampled-video checks."""
+import runpy
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
+if '--config' not in sys.argv:
+    sys.argv += ['--config', str(Path(__file__).with_name('dataset.yaml'))]
+runpy.run_module('scripts.data.validate', run_name='__main__')

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/CONTRACT.md

@@ -0,0 +1,19 @@
+# Original-action training contract
+
+All three policies reference `../data/dataset.yaml`. Original v3 files stay read-only.
+30Hz -> 10Hz logical sampling, split task changes before subsampling, state20/action23.
+Torso uses original action[:4]. Invalid action elements only are excluded from loss and statistics.
+Actual batch128, accumulation1, seed42, 30000 updates, no evaluation split.
+N1.5: observation1/action16. N1.7: observation1/action40. DiT: observation2/horizon32/execution24.
+Full action/history chunks must remain within each logical task episode.
+
+N1.5/DiT call the LeTools config converter and pinned LeRobot Trainer.
+N1.7 calls LeTools bundled native pipeline, processor, shard cache, collator, and HF Trainer.
+Runtime adapters only: original-v3 metadata/timestamps, source validity masks, per-sample N1.5 language.
+N1.5/DiT letterbox images to 224px before native preprocessing; N1.7 keeps native image augmentation.
+N1.7 runs in the dedicated Python3.10/Torch2.7.1 conda environment required by LeTools. No model, loss, Trainer-class, or CUDA-layout patch is applied.
+Checkpoints: 10k,20k,30k locally. HF watcher publishes verified 10k/20k/30k per policy under v2 after the run succeeds, excluding optimizer/RNG.
+DiT starts exactly once on the first successfully completed GPU0/1.
+
+LeTools:6d2002613241ae2a32c5d8fd753f80caa1383fba; LeRobot:a07f22e22ce88cddff1f6eddced9ea008fbfc37c.
+Runtime and reusable code are in `RwH/scripts`, not a private N1.7 checkout copy.

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/compat.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+import sys
+sys.path.insert(0,str(Path(__file__).resolve().parents[3]))
+from scripts.adapters.language import *

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/gr00t_n15.yaml

@@ -0,0 +1,55 @@
+dataset:
+  config: ../data/dataset.yaml
+training:
+  resume: false
+  output_dir: /home/work/mntvol/RwH-Kuavo_ckpt/v3_rawaction/gr00t_n15
+  job_name: rawaction_bs128_30k_gr00t_n15
+  batch_size: 128
+  steps: 30000
+  save_freq: 10000
+  log_freq: 100
+  eval_freq: 0
+  num_workers: 4
+  seed: 42
+  use_policy_training_preset: false
+  optimizer:
+    type: adamw
+    lr: 0.0001
+    betas:
+    - 0.95
+    - 0.999
+    eps: 1.0e-08
+    weight_decay: 1.0e-05
+    grad_clip_norm: 10.0
+  scheduler:
+    type: cosine_decay_with_warmup
+    num_warmup_steps: 1500
+    num_decay_steps: 30000
+    peak_lr: 0.0001
+    decay_lr: 1.0e-05
+wandb:
+  enable: true
+  project: kuavo
+  entity: minje227_hyu-hanyang-university
+  mode: online
+  disable_artifact: true
+policy:
+  type: groot
+  pretrained_path: null
+  base_model_path: nvidia/GR00T-N1.5-3B
+  embodiment_tag: new_embodiment
+  use_bf16: true
+  chunk_size: 16
+  n_action_steps: 16
+  max_state_dim: 64
+  max_action_dim: 32
+  image_size:
+  - 224
+  - 224
+  tune_llm: false
+  tune_visual: false
+  tune_projector: true
+  tune_diffusion_model: true
+  optimizer_lr: 0.0001
+  warmup_ratio: 0.05
+  dataloader_num_workers: 8

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/gr00t_n17.yaml

@@ -0,0 +1,57 @@
+dataset:
+  config: ../data/dataset.yaml
+training:
+  resume: false
+  output_dir: /home/work/mntvol/RwH-Kuavo_ckpt/v3_rawaction/gr00t_n17
+  job_name: rawaction_bs128_30k_gr00t_n17
+  batch_size: 128
+  global_batch_size: 128
+  gradient_accumulation_steps: 1
+  steps: 30000
+  save_freq: 10000
+  log_freq: 100
+  num_workers: 6
+  seed: 42
+  optimizer:
+    lr: 0.0001
+    betas:
+    - 0.9
+    - 0.999
+    weight_decay: 1.0e-05
+    grad_clip_norm: 1.0
+  scheduler:
+    type: cosine
+    warmup_ratio: 0.05
+    decay_lr: 0.0
+  shard_size: 1024
+  episode_sampling_rate: 0.1
+  num_shards_per_epoch: 100000
+policy:
+  type: groot_n17
+  base_model_path: nvidia/GR00T-N1.7-3B
+  base_revision: 2fc962b973bccdd5d8ce4f67cc63b264d6886495
+  runtime_root: /home/work/workspace/RwH-Kuavo_V2/LeTools-Learning/kuavo_model/external_models/gr00tn1d7
+  embodiment_tag: new_embodiment
+  chunk_size: 40
+  n_action_steps: 40
+  max_state_dim: 132
+  max_action_dim: 132
+  tune_llm: false
+  tune_visual: false
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  use_bf16: true
+  use_relative_action: false
+  state_dropout_prob: 0.2
+loss:
+  validity_feature: validity.action
+  dimension_mask: true
+  exclude_action_padding: true
+  reduction: sum_valid_elements_div_valid_count
+wandb:
+  enable: true
+  project: kuavo
+  entity: minje227_hyu-hanyang-university
+  mode: online
+  disable_artifact: true

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/hf_watcher.py

@@ -0,0 +1,7 @@
+"""Compatibility entry point; implementation lives in RwH/scripts."""
+from pathlib import Path
+import os,runpy,sys
+root=Path(__file__).resolve().parents[3]
+sys.path.insert(0,str(root))
+os.environ['RWH_TRAIN_CONFIG_DIR']=str(Path(__file__).resolve().parent)
+if __name__=='__main__':runpy.run_module('scripts.train.hf_watcher',run_name='__main__')

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/launch.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+source /home/work/workspace/session-bootstrap.sh >/dev/null
+set -euo pipefail
+task_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+project="$(cd "$task_dir/../../.." && pwd)"
+export RWH_TRAIN_CONFIG_DIR="$task_dir"
+export PYTHONPATH="$project:$project/../LeTools-Learning/third_party/lerobot/src"
+export WANDB_API_KEY="$(cat /home/work/.local/share/wandb/kuavo_api_key)"
+export WANDB__SERVICE_WAIT=120
+export TOKENIZERS_PARALLELISM=false OMP_NUM_THREADS=8 MKL_NUM_THREADS=8 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1
+policy="$1"
+shift
+cd "$project"
+if [ "$policy" = gr00t_n17 ]; then
+  exec /home/work/.local/miniconda3/envs/rwh_n17_letools/bin/python -m scripts.train.run_n17 "$@"
+else
+  exec /home/work/.local/miniconda3/envs/rwh/bin/python -m scripts.train.run_lerobot "$policy" "$@"
+fi

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/load_n17.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+import sys
+sys.path.insert(0,str(Path(__file__).resolve().parents[3]))
+from scripts.train.load_n17 import load

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/loss_mask.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+import sys
+sys.path.insert(0,str(Path(__file__).resolve().parents[3]))
+from scripts.adapters.action_validity import *

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/multi_task_dit.yaml

@@ -0,0 +1,33 @@
+dataset:
+  config: ../data/dataset.yaml
+training:
+  resume: false
+  output_dir: /home/work/mntvol/RwH-Kuavo_ckpt/v3_rawaction/multi_task_dit
+  job_name: rawaction_bs128_30k_multi_task_dit
+  batch_size: 128
+  steps: 30000
+  save_freq: 10000
+  log_freq: 100
+  eval_freq: 0
+  num_workers: 4
+  seed: 42
+wandb:
+  enable: true
+  project: kuavo
+  entity: minje227_hyu-hanyang-university
+  mode: online
+  disable_artifact: true
+policy:
+  type: multi_task_dit
+  pretrained_path: null
+  vision_encoder_name: openai/clip-vit-base-patch16
+  text_encoder_name: openai/clip-vit-base-patch16
+  n_obs_steps: 2
+  horizon: 32
+  n_action_steps: 24
+  objective: diffusion
+  image_crop_shape:
+  - 224
+  - 224
+  optimizer_lr: 2.0e-05
+  scheduler_name: cosine

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/native_assets.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+import sys
+sys.path.insert(0,str(Path(__file__).resolve().parents[3]))
+from scripts.train.native_assets import *

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/orchestrate.py

@@ -0,0 +1,7 @@
+"""Compatibility entry point; implementation lives in RwH/scripts."""
+from pathlib import Path
+import os,runpy,sys
+root=Path(__file__).resolve().parents[3]
+sys.path.insert(0,str(root))
+os.environ['RWH_TRAIN_CONFIG_DIR']=str(Path(__file__).resolve().parent)
+if __name__=='__main__':runpy.run_module('scripts.train.orchestrate',run_name='__main__')

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/run.py

@@ -0,0 +1,7 @@
+"""Compatibility entry point; implementation lives in RwH/scripts."""
+from pathlib import Path
+import os,runpy,sys
+root=Path(__file__).resolve().parents[3]
+sys.path.insert(0,str(root))
+os.environ['RWH_TRAIN_CONFIG_DIR']=str(Path(__file__).resolve().parent)
+if __name__=='__main__':runpy.run_module('scripts.train.run_lerobot',run_name='__main__')

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/run_n17.py

@@ -0,0 +1,7 @@
+"""Compatibility entry point; implementation lives in RwH/scripts."""
+from pathlib import Path
+import os,runpy,sys
+root=Path(__file__).resolve().parents[3]
+sys.path.insert(0,str(root))
+os.environ['RWH_TRAIN_CONFIG_DIR']=str(Path(__file__).resolve().parent)
+if __name__=='__main__':runpy.run_module('scripts.train.run_n17',run_name='__main__')

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/total/gr00t_n15_total.yaml

@@ -0,0 +1,104 @@
+dataset:  # 数据集相关设置。
+  repo_id: "lerobot/your_dataset"  # 数据集名字，用来做标识。
+  root: null  # 本地数据路径。
+  episodes: null  # 只训练指定的 episode；不填就是全量。
+  image_transforms: {}  # 额外的图像预处理设置。
+  revision: null  # 远程数据集版本；本地训练一般不用改。
+  use_imagenet_stats: true  # 图像是否用 ImageNet 的均值方差做归一化。
+  video_backend: pyav  # 读视频帧用的后端。一般用 pyav。
+  return_uint8: false  # 是否保留图像为 uint8，而不是转换成 float tensor。
+  streaming: false  # 是否流式读数据。一般保持 false。
+
+training:  # 训练流程设置。
+  output_dir: "outputs/train/gr00t"  # 训练输出目录。
+  job_name: "gr00t"  # 实验名。
+  resume: false  # 是否从 output_dir 最新 checkpoint 继续训；续训改为 true，并修改 output_dir 为上阶段 run 的产物路径，如 outputs/train/gr00t_YYYYMMDD_HHMMSS。
+  seed: 1000  # 随机种子。
+  cudnn_deterministic: false  # 更稳定可复现，但会慢一点。
+  num_workers: 4  # 读数据用的进程数。
+  batch_size: 8  # 每张卡上的 batch size。
+  prefetch_factor: 4  # 每个读数据进程提前准备多少个 batch。
+  persistent_workers: true  # 是否让读数据进程常驻。
+  steps: 10000  # 总训练步数。
+  eval_freq: 1000  # 每隔多少步做一次评测。
+  log_freq: 100  # 每隔多少步打一次日志。
+  tolerance_s: 1.0e-4  # 视频帧和标注对齐时允许的时间误差。
+  save_checkpoint: true  # 是否保存 checkpoint。
+  save_freq: 1000  # 每隔多少步存一次权重。
+  use_policy_training_preset: true  # 是否用模型自带的优化器和调度器默认值。
+  optimizer: null  # 手动指定优化器；一般留空。
+  scheduler: null  # 手动指定学习率策略；一般留空。
+  use_rabc: false  # 是否启用 RABC。一般不用。
+  rabc_progress_path: null  # RABC 用的额外文件路径。一般不用。
+  rabc_kappa: 0.01  # RABC 参数。一般不用改。
+  rabc_epsilon: 1.0e-6  # RABC 的稳定性参数。一般不用改。
+  rabc_head_mode: "sparse"  # RABC 模式。一般不用改。
+  rename_map: {}  # 特征重命名表。一般不用。
+
+eval:  # 训练中评测设置。
+  n_episodes: 50  # 每次评测跑多少条。
+  batch_size: 0  # 评测 batch size。0 表示自动。
+  use_async_envs: true  # 评测环境是否并行异步跑。
+
+wandb:  # wandb 设置。
+  enable: false  # 是否启用 wandb。
+  disable_artifact: true  # 是否禁用向 wandb 上传权重文件。
+  project: "lerobot"  # wandb 项目名。
+  entity: null  # wandb 账号或团队名。
+  notes: null  # wandb 备注。
+  run_id: null  # wandb run id。
+  mode: null  # wandb 模式。
+  add_tags: true  # 是否自动加 wandb 标签。
+
+peft: null  # PEFT 设置。一般留空。
+
+policy:  # 模型相关设置。
+  type: "groot"  # 模型类型。
+  n_obs_steps: 1  # 一次预测会看几帧观测。
+  input_features: {}  # 手动指定输入特征。一般不用。
+  output_features: {}  # 手动指定输出特征。一般不用。
+  device: "cuda"  # 运行设备。
+  use_amp: false  # 是否启用混合精度。
+  use_peft: false  # 是否启用 PEFT。
+  push_to_hub: false  # 是否上传到 Hugging Face。
+  repo_id: null  # 上传时用的仓库名。
+  private: null  # 上传仓库是否私有。
+  tags: null  # 上传时的标签。
+  license: null  # 上传时的 license。
+  pretrained_path: null  # 加载 LeRobot 整包 checkpoint；从 NVIDIA 基座微调时保持 null（权重见 base_model_path）。二阶段微调请改为上一 run 的 <policy_name>_YYYYMMDD_HHMMSS/checkpoints/<step>/pretrained_model 路径。
+  chunk_size: 50  # 每次预测多长的动作序列。
+  n_action_steps: 50  # 每次真正执行多少步动作。
+  max_state_dim: 64  # 状态最大维度。
+  max_action_dim: 32  # 动作最大维度。
+  normalization_mapping:  # 不同输入的归一化方式。
+    VISUAL: "IDENTITY"  # 图像怎么归一化。
+    STATE: "MEAN_STD"  # 状态怎么归一化。
+    ACTION: "MEAN_STD"  # 动作怎么归一化。
+  image_size: [224, 224]  # 模型期望的图像尺寸。
+  base_model_path: "nvidia/GR00T-N1.5-3B"  # GR00T 基座模型。
+  tokenizer_assets_repo: "lerobot/eagle2hg-processor-groot-n1p5"  # processor 资源仓库。
+  embodiment_tag: "new_embodiment"  # 机器人形态标签。
+  tune_llm: false  # 是否训练语言部分。
+  tune_visual: false  # 是否训练视觉部分。
+  tune_projector: true  # 是否训练 projector。
+  tune_diffusion_model: true  # 是否训练动作头。
+  lora_alpha: 16  # LoRA alpha。
+  lora_dropout: 0.1  # LoRA dropout。
+  lora_full_model: false  # 是否全模型加 LoRA。
+  optimizer_lr: 1.0e-4  # 主学习率。
+  optimizer_betas: [0.95, 0.999]  # 优化器 beta 参数。
+  optimizer_eps: 1.0e-8  # 优化器 epsilon。
+  optimizer_weight_decay: 1.0e-5  # weight decay。
+  warmup_ratio: 0.05  # warmup 比例。
+  use_bf16: true  # 是否用 bf16。
+  video_backend: "decord"  # 这个模型训练时用的视频后端。
+  balance_dataset_weights: true  # 多数据集时是否平衡各数据集。
+  balance_trajectory_weights: true  # 是否平衡不同轨迹。
+  dataset_paths: null  # 多数据集路径列表。
+  output_dir: "./tmp/gr00t"  # 这个模型内部使用的输出目录。
+  save_steps: 1000  # 模型内部保存步数。
+  max_steps: 10000  # 模型内部最大训练步数。
+  batch_size: 32  # 模型内部 batch size。
+  dataloader_num_workers: 8  # 模型内部读数据进程数。
+  report_to: "wandb"  # 日志上报后端。
+  resume: false  # 模型内部是否恢复训练。

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/configs/Bs128_30k/train/total/multi_task_dit_total.yaml

@@ -0,0 +1,118 @@
+dataset:  # 数据集相关设置。
+  repo_id: "lerobot/your_dataset"  # 数据集名字，用来做标识。
+  root: null  # 本地数据路径。
+  episodes: null  # 只训练指定的 episode；不填就是全量。
+  image_transforms: {}  # 额外的图像预处理设置。
+  revision: null  # 远程数据集版本；本地训练一般不用改。
+  use_imagenet_stats: true  # 图像是否用 ImageNet 的均值方差做归一化。
+  video_backend: pyav  # 读视频帧用的后端。一般用 pyav。
+  return_uint8: false  # 是否保留图像为 uint8，而不是转换成 float tensor。
+  streaming: false  # 是否流式读数据。一般保持 false。
+
+training:  # 训练流程设置。
+  output_dir: "outputs/train/multi_task_dit"  # 训练输出目录。
+  job_name: "multi_task_dit"  # 实验名。
+  resume: false  # 是否从 output_dir 最新 checkpoint 继续训；续训改为 true，并修改 output_dir 为上阶段 run 的产物路径，如 outputs/train/multi_task_dit_YYYYMMDD_HHMMSS。
+  seed: 1000  # 随机种子。
+  cudnn_deterministic: false  # 更稳定可复现，但会慢一点。
+  num_workers: 4  # 读数据用的进程数。
+  batch_size: 8  # 每张卡上的 batch size。
+  prefetch_factor: 4  # 每个读数据进程提前准备多少个 batch。
+  persistent_workers: true  # 是否让读数据进程常驻。
+  steps: 100000  # 总训练步数。
+  eval_freq: 10000  # 每隔多少步做一次评测。
+  log_freq: 100  # 每隔多少步打一次日志。
+  tolerance_s: 1.0e-4  # 视频帧和标注对齐时允许的时间误差。
+  save_checkpoint: true  # 是否保存 checkpoint。
+  save_freq: 10000  # 每隔多少步存一次权重。
+  use_policy_training_preset: true  # 是否用模型自带的优化器和调度器默认值。
+  optimizer: null  # 手动指定优化器；一般留空。
+  scheduler: null  # 手动指定学习率策略；一般留空。
+  use_rabc: false  # 是否启用 RABC。一般不用。
+  rabc_progress_path: null  # RABC 用的额外文件路径。一般不用。
+  rabc_kappa: 0.01  # RABC 参数。一般不用改。
+  rabc_epsilon: 1.0e-6  # RABC 的稳定性参数。一般不用改。
+  rabc_head_mode: "sparse"  # RABC 模式。一般不用改。
+  rename_map: {}  # 特征重命名表。一般不用。
+
+eval:  # 训练中评测设置。
+  n_episodes: 50  # 每次评测跑多少条。
+  batch_size: 0  # 评测 batch size。0 表示自动。
+  use_async_envs: true  # 评测环境是否并行异步跑。
+
+wandb:  # wandb 设置。
+  enable: false  # 是否启用 wandb。
+  disable_artifact: true  # 是否禁用向 wandb 上传权重文件。
+  project: "lerobot"  # wandb 项目名。
+  entity: null  # wandb 账号或团队名。
+  notes: null  # wandb 备注。
+  run_id: null  # wandb run id。
+  mode: null  # wandb 模式。
+  add_tags: true  # 是否自动加 wandb 标签。
+
+peft: null  # PEFT 设置。一般留空。
+
+policy:  # 模型相关设置。
+  type: "multi_task_dit"  # 模型类型。
+  n_obs_steps: 2  # 一次预测会看几帧观测。
+  input_features: {}  # 手动指定输入特征。一般不用。
+  output_features: {}  # 手动指定输出特征。一般不用。
+  device: "cuda"  # 运行设备。
+  use_amp: false  # 是否启用混合精度。
+  use_peft: false  # 是否启用 PEFT。
+  push_to_hub: false  # 是否上传到 Hugging Face。
+  repo_id: null  # 上传时用的仓库名。
+  private: null  # 上传仓库是否私有。
+  tags: null  # 上传时的标签。
+  license: null  # 上传时的 license。
+  pretrained_path: null  # 通常从头训（填 null）或填本地/LeRobot checkpoint。二阶段微调请改为上一 run 的 <policy_name>_YYYYMMDD_HHMMSS/checkpoints/<step>/pretrained_model 路径。
+  horizon: 32  # 模型内部预测的总长度。
+  n_action_steps: 24  # 每次真正执行多少步动作。
+  objective: "diffusion"  # 训练目标类型。
+  noise_scheduler_type: "DDPM"  # 噪声调度器类型。
+  num_train_timesteps: 100  # 训练时的扩散步数。
+  beta_schedule: "squaredcos_cap_v2"  # beta 调度方式。
+  beta_start: 1.0e-4  # beta 起始值。
+  beta_end: 0.02  # beta 结束值。
+  prediction_type: "epsilon"  # 模型预测什么目标。
+  clip_sample: true  # 是否裁剪去噪结果。
+  clip_sample_range: 1.0  # 裁剪范围。
+  num_inference_steps: null  # 推理时去噪多少步；不填就用默认。
+  sigma_min: 0.0  # 最小 sigma。
+  num_integration_steps: 100  # 积分步数。
+  integration_method: "euler"  # 积分方法。
+  timestep_sampling_strategy: "beta"  # 时间步采样策略。
+  timestep_sampling_s: 0.999  # 时间步采样参数 s。
+  timestep_sampling_alpha: 1.5  # 时间步采样 alpha。
+  timestep_sampling_beta: 1.0  # 时间步采样 beta。
+  hidden_dim: 512  # 主隐藏维度。
+  num_layers: 6  # 层数。
+  num_heads: 8  # 注意力头数。
+  dropout: 0.1  # dropout 比例。
+  use_positional_encoding: false  # 是否启用位置编码。
+  timestep_embed_dim: 256  # 时间嵌入维度。
+  use_rope: true  # 是否启用 RoPE。
+  rope_base: 10000.0  # RoPE base。
+  vision_encoder_name: "openai/clip-vit-base-patch16"  # 视觉编码器模型名。
+  use_separate_rgb_encoder_per_camera: false  # 每个相机是否单独编码。
+  vision_encoder_lr_multiplier: 0.1  # 视觉编码器学习率倍率。
+  image_resize_shape: null  # 图像先 resize 到多大。
+  image_crop_shape: [224, 224]  # 图像 crop 尺寸。
+  image_crop_is_random: true  # 是否随机 crop。
+  text_encoder_name: "openai/clip-vit-base-patch16"  # 文本编码器模型名。
+  tokenizer_max_length: 77  # 文本最大长度。
+  tokenizer_padding: "max_length"  # padding 模式。
+  tokenizer_padding_side: "right"  # padding 方向。
+  tokenizer_truncation: true  # 是否截断过长文本。
+  normalization_mapping:  # 不同输入的归一化方式。
+    VISUAL: "MEAN_STD"  # 图像怎么归一化。
+    STATE: "MIN_MAX"  # 状态怎么归一化。
+    ACTION: "MIN_MAX"  # 动作怎么归一化。
+  optimizer_lr: 2.0e-5  # 主学习率。
+  optimizer_betas: [0.95, 0.999]  # 优化器 beta 参数。
+  optimizer_eps: 1.0e-8  # 优化器 epsilon。
+  optimizer_weight_decay: 0.0  # weight decay。
+  scheduler_name: "cosine"  # 学习率策略。
+  scheduler_warmup_steps: 0  # warmup 步数。
+  do_mask_loss_for_padding: false  # 算 loss 时是否忽略 padding。
+  drop_n_last_frames: null  # 训练时丢掉最后几帧，减少 padding 影响。

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/__init__.py

@@ -0,0 +1 @@
+"""Reusable RwH data views and LeTools training entry points."""

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/adapters/__init__.py

@@ -0,0 +1 @@
+"""Small compatibility adapters; model and trainer implementations stay upstream."""

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/adapters/action_validity.py

@@ -0,0 +1,98 @@
+"""Project-only dimension masks; never rewrite vendored policies."""
+import inspect
+import textwrap
+
+
+def masked_mean(error, mask):
+    mask = mask.to(device=error.device, dtype=error.dtype).expand_as(error)
+    return (error.float() * mask).sum() / mask.sum().clamp(min=1)
+
+
+class MaskBank:
+    def __init__(self, view, policy):
+        import torch
+        self.mask = torch.as_tensor(view.validity.copy(), dtype=torch.bool)
+        self.policy = policy
+        self.offsets = torch.as_tensor(view.offsets(policy))
+        self.calls = 0
+
+    def attach(self, processed, raw):
+        import torch
+        index = raw["index"].detach().cpu().long().reshape(-1)
+        mask = self.mask[index[:, None] + self.offsets[None, :]]
+        if raw.get("action_is_pad") is not None:
+            mask &= ~raw["action_is_pad"].detach().cpu().bool().unsqueeze(-1)
+        if self.policy == "gr00t_n15":
+            padded = torch.zeros_like(processed["action_mask"], dtype=torch.bool)
+            if padded.shape[:2] != mask.shape[:2]:
+                raise RuntimeError("N1.5 mask/chunk alignment mismatch")
+            padded[..., :23] = mask.to(padded.device)
+            processed["action_mask"] = processed["action_mask"].bool() & padded
+        else:
+            if processed["action"].shape != mask.shape:
+                raise RuntimeError("DiT mask/chunk alignment mismatch")
+            processed["rwh_action_validity"] = mask.to(processed["action"].device)
+        self.calls += 1
+        if self.calls == 1:
+            print("RWH_MASK_ATTACHED", self.policy, list(mask.shape), "invalid", int((~mask).sum()), flush=True)
+        return processed
+
+
+def diffusion_loss(self, model, batch, conditioning_vec):
+    import torch
+    import torch.nn.functional as F
+    actions = batch["action"]
+    noise = torch.randn_like(actions)
+    timesteps = torch.randint(0, self.noise_scheduler.config.num_train_timesteps,
+                              (actions.shape[0],), device=actions.device).long()
+    noisy = self.noise_scheduler.add_noise(actions, noise, timesteps)
+    prediction_type = self.noise_scheduler.config.prediction_type
+    if prediction_type == "epsilon":
+        target = noise
+    elif prediction_type == "sample":
+        target = actions
+    else:
+        raise ValueError(prediction_type)
+    predicted = model(noisy, timesteps, conditioning_vec=conditioning_vec)
+    mask = batch["rwh_action_validity"]
+    if "action_is_pad" in batch:
+        mask = mask & ~batch["action_is_pad"].unsqueeze(-1)
+    return masked_mean(F.mse_loss(predicted.float(), target.float(), reduction="none"), mask)
+
+
+def install(policy):
+    if policy == "multi_task_dit":
+        from lerobot.policies.multi_task_dit.modeling_multi_task_dit import DiffusionObjective
+        DiffusionObjective.compute_loss = diffusion_loss
+    elif policy == "gr00t_n15":
+        from lerobot.policies.groot.action_head.flow_matching_action_head import FlowmatchingActionHead
+        original = inspect.getsource(FlowmatchingActionHead.forward)
+        needle = "loss = loss.sum() / action_mask.sum()"
+        if original.count(needle) != 1:
+            raise RuntimeError("Unknown upstream N1.5 loss implementation")
+        source = textwrap.dedent(original).replace(needle, "loss = loss.float().sum() / action_mask.sum().clamp(min=1)")
+        namespace = {}
+        exec(compile(source, "<rwh_n15_masked_forward>", "exec"), FlowmatchingActionHead.forward.__globals__, namespace)
+        FlowmatchingActionHead.forward = namespace["forward"]
+    else:
+        raise ValueError(policy)
+
+
+def verify():
+    import torch
+    p = torch.randn(2, 3, 23, requires_grad=True)
+    target = torch.randn_like(p)
+    mask = torch.ones_like(p, dtype=torch.bool)
+    mask[0, 1, 20:23] = False
+    loss = masked_mean((p - target).square(), mask)
+    loss.backward()
+    assert torch.equal(p.grad[~mask], torch.zeros_like(p.grad[~mask]))
+    assert p.grad[mask].abs().sum() > 0
+    assert masked_mean(p.square(), torch.zeros_like(mask)).item() == 0
+    return {"dimension_gradient_zero": True, "valid_dimension_gradient_nonzero": True,
+            "all_masked_loss_zero": True, "mask_shape": list(mask.shape)}
+
+
+if __name__ == "__main__":
+    print(verify())
+

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/adapters/groot_v3.py

@@ -0,0 +1,86 @@
+"""Adapt v3 row/timestamp views to the bundled native episode interface."""
+from pathlib import Path
+import numpy as np
+import pandas as pd
+
+
+def install(view):
+    import torch
+    from PIL import Image
+    import gr00t.data.dataset.sharded_single_step_dataset as single
+    import gr00t.data.dataset.factory as factory
+    from gr00t.utils.video_utils import get_frames_by_timestamps
+
+    class EpisodeLoader:
+        def __init__(self, dataset_path, modality_configs, video_backend, video_backend_kwargs=None):
+            if Path(dataset_path).resolve()!=view.root: raise ValueError('Unexpected source dataset')
+            self.modality_configs=modality_configs
+            self.video_backend=video_backend
+            self.episode_lengths=[s['end']-s['start'] for s in view.segments]
+        def get_episode_length(self,index): return self.episode_lengths[index]
+        def __len__(self): return len(self.episode_lengths)
+        def get_initial_actions(self): return []
+        def get_dataset_statistics(self):
+            return {kind:{'body':{k:v for k,v in view.stats[feature].items() if k!='count'}}
+                    for kind,feature in [('state','observation.state'),('action','action')]}
+        def __getitem__(self,index):
+            s=view.segments[index]; rows=np.arange(s['start'],s['end'])
+            df=pd.DataFrame({'state.body':list(view.state[rows]),'action.body':list(view.action[rows]),
+                             'rwh.validity':list(view.validity[rows]),'rwh.anchor':rows,
+                             'language.task':[view.tasks[s['task']]]*len(rows)})
+            for camera in view.cameras:
+                needed=getattr(self,'requested',{}).get(index,np.arange(len(rows)))
+                locations=[view.video_location(rows[i],camera) for i in needed]
+                if not locations:
+                    df['video.'+camera]=[None]*len(rows)
+                    continue
+                assert len({str(p) for p,t in locations})==1
+                frames=get_frames_by_timestamps(str(locations[0][0]),np.asarray([t for p,t in locations]),
+                                                video_backend=self.video_backend)
+                assert len(frames)==len(needed)
+                values=[None]*len(rows)
+                for i,frame in zip(needed,frames):values[int(i)]=Image.fromarray(np.ascontiguousarray(frame))
+                df['video.'+camera]=values
+            return df
+
+    # Scoped to this process and exact source root. Upstream files remain clean.
+    single.LeRobotEpisodeLoader=EpisodeLoader
+    original_shard=single.ShardedSingleStepDataset.get_shard
+    def shard(self,index):
+        # Keep native shard scheduling, processing and prefetch. Only v3 I/O gets
+        # the exact requested frame indices so unused full episodes aren't decoded.
+        deltas=np.asarray(self.modality_configs['video'].delta_indices)
+        requested={}
+        for ep,steps in self.sharded_episodes[index]:
+            needed=(np.asarray(steps)[:,None]+deltas[None,:]).reshape(-1)
+            requested.setdefault(int(ep),[]).extend(needed.tolist())
+        self.episode_loader.requested={k:np.unique(v).astype(int) for k,v in requested.items()}
+        try:return original_shard(self,index)
+        finally:self.episode_loader.requested={}
+    single.ShardedSingleStepDataset.get_shard=shard
+    original=single.ShardedSingleStepDataset.get_datapoint
+    def datapoint(self,episode_data,step_index):
+        encoded=original(self,episode_data,step_index)
+        indices=step_index+np.asarray(self.modality_configs['action'].delta_indices)
+        validity=np.stack(episode_data['rwh.validity'].iloc[indices])
+        mask=encoded['action_mask']
+        padded=torch.zeros_like(mask,dtype=torch.bool)
+        padded[:len(indices),:23]=torch.as_tensor(validity,device=mask.device)
+        encoded['action_mask']=mask.bool() & padded
+        return encoded
+    single.ShardedSingleStepDataset.get_datapoint=datapoint
+    def validated_stats(path,*args):
+        if Path(path).resolve()!=view.root: raise ValueError('Unexpected statistics source')
+        return view.stats
+    factory.generate_stats=validated_stats
+    factory.generate_rel_stats=validated_stats
+
+
+def modality(view,horizon):
+    from gr00t.data.types import ModalityConfig, ActionConfig, ActionRepresentation, ActionType, ActionFormat
+    return {'new_embodiment':{
+        'video':ModalityConfig(delta_indices=[0],modality_keys=view.cameras),
+        'state':ModalityConfig(delta_indices=[0],modality_keys=['body']),
+        'action':ModalityConfig(delta_indices=list(range(horizon)),modality_keys=['body'],
+             action_configs=[ActionConfig(rep=ActionRepresentation.ABSOLUTE,type=ActionType.NON_EEF,format=ActionFormat.DEFAULT)]),
+        'language':ModalityConfig(delta_indices=[0],modality_keys=['task'])}}

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/adapters/language.py

@@ -0,0 +1,56 @@
+"""Runtime compatibility fixes for this pinned LeTools revision.
+
+Import/install this module before loading saved GR00T processors for inference.
+"""
+
+
+def letterbox224(image):
+    from torchvision.transforms import functional as F
+    height, width = image.shape[-2:]
+    scale = 224 / max(height, width)
+    new_h, new_w = max(1, round(height * scale)), max(1, round(width * scale))
+    image = F.resize(image, [new_h, new_w], antialias=True)
+    left, top = (224 - new_w) // 2, (224 - new_h) // 2
+    return F.pad(image, [left, top, 224 - new_w - left, 224 - new_h - top])
+
+
+def install(policy):
+    if policy not in ("gr00t", "gr00t_n15"):
+        return
+    from lerobot.policies.groot.processor_groot import GrootPackInputsStep, GrootEagleEncodeStep
+    from lerobot.types import TransitionKey as K
+    if getattr(GrootPackInputsStep, "_rwh_prompts_fixed", False):
+        return
+    original_pack = GrootPackInputsStep.__call__
+    original_encode = GrootEagleEncodeStep.__call__
+
+    def pack(self, transition):
+        prompts = transition.get(K.COMPLEMENTARY_DATA, {}).get(self.language_key)
+        result = original_pack(self, transition)
+        if isinstance(prompts, list):
+            if self.formalize_language:
+                prompts = ["".join(c for c in p.lower() if c.isalnum() or c.isspace()) for p in prompts]
+            result[K.COMPLEMENTARY_DATA]["language"] = prompts
+        return result
+
+    def encode(self, transition):
+        obs = transition.get(K.OBSERVATION, {})
+        comp = transition.get(K.COMPLEMENTARY_DATA, {})
+        prompts = comp.get("language")
+        if not isinstance(prompts, list) or "video" not in obs:
+            return original_encode(self, transition)
+        if len(prompts) != obs["video"].shape[0]:
+            raise ValueError("One task prompt is required for every GR00T batch sample")
+        contents = []
+        for i, prompt in enumerate(prompts):
+            one = {K.OBSERVATION: {"video": obs["video"][i:i + 1]},
+                   K.COMPLEMENTARY_DATA: {"language": prompt}}
+            encoded = original_encode(self, one)
+            contents.extend(encoded[K.COMPLEMENTARY_DATA]["eagle_content"])
+        comp["eagle_content"] = contents
+        return transition
+
+    GrootPackInputsStep.__call__ = pack
+    GrootPackInputsStep._rwh_prompts_fixed = True
+    GrootEagleEncodeStep.__call__ = encode
+

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/data/__init__.py

@@ -0,0 +1 @@
+"""Read-only source dataset views."""

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/data/prepare.py

@@ -0,0 +1,18 @@
+"""Read and audit the configured logical view without exporting a dataset."""
+import argparse
+import json
+from scripts.data.view import DataView
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--config', required=True)
+    parser.add_argument('--audit-dir', required=True)
+    args = parser.parse_args()
+    view = DataView(args.config)
+    view.save_audit(args.audit_dir)
+    print(json.dumps(view.manifest(), indent=2))
+
+
+if __name__ == '__main__':
+    main()

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/data/validate.py

@@ -0,0 +1,67 @@
+"""Exhaustive index/target checks plus sampled native video comparisons."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import numpy as np
+from scripts.data.view import DataView, statistics
+
+
+def source_inventory(root):
+    return {str(p.relative_to(root)):[p.stat().st_size,p.stat().st_mtime_ns] for p in root.rglob('*') if p.is_file()}
+
+
+def main():
+    p=argparse.ArgumentParser();p.add_argument('--config',required=True);p.add_argument('--output',required=True)
+    args=p.parse_args();view=DataView(args.config)
+    inventory=source_inventory(view.root)
+    assert np.array_equal(view.action,view.raw['action'][view.rows,:23])
+    assert np.array_equal(view.validity,view.raw['validity.action'][view.rows,:23])
+    assert np.array_equal(view.state,view.raw['observation.state'][view.rows,:20])
+    assert view.validity[:,:4].all()
+    anchor_report={}
+    for policy in ['gr00t_n15','gr00t_n17','multi_task_dit']:
+        anchors=view.anchors(policy);idx=anchors[:,None]+view.offsets(policy)[None,:]
+        assert (view.episode[idx]==view.episode[anchors,None]).all()
+        assert (view.task[idx]==view.task[anchors,None]).all()
+        assert (np.diff(view.rows[idx],axis=1)==view.stride).all()
+        anchor_report[policy]={'count':len(anchors),'first':int(anchors[0]),'last':int(anchors[-1])}
+    for j in range(23):
+        actual=view.action[view.validity[:,j],j]
+        assert np.isclose(view.stats['action']['mean'][j],actual.mean(dtype=np.float64))
+        assert view.stats['action']['count'][j]==len(actual)
+    # Native LeRobot video decode is compared against an independent PyAV seek.
+    import av
+    from lerobot.datasets.lerobot_dataset import LeRobotDataset
+    native=LeRobotDataset(view.config['source']['repo_id'],root=view.root,video_backend='pyav',
+                          return_uint8=True,tolerance_s=1/30+.001)
+    rng=np.random.default_rng(42)
+    chosen=set(rng.choice(len(view.rows),24,replace=False).tolist())
+    for s in view.segments[::max(1,len(view.segments)//12)]:chosen.update([s['start'],s['end']-1])
+    checks=[]
+    for i in sorted(chosen):
+        sample=native[int(view.rows[i])]
+        for camera in view.cameras:
+            path,timestamp=view.video_location(i,camera)
+            with av.open(str(path)) as container:
+                stream=container.streams.video[0];stream.thread_type='NONE';stream.codec_context.thread_count=1
+                container.seek(max(0,int(timestamp/stream.time_base)),stream=stream,backward=True)
+                decoded=[]
+                for frame in container.decode(stream):
+                    decoded.append((abs(frame.time-timestamp),frame.time,frame.to_ndarray(format='rgb24')))
+                    if frame.time>=timestamp:break
+            distance,pts,image=min(decoded,key=lambda x:x[0])
+            assert distance<1/60+.001,(i,camera,timestamp,pts)
+            other=sample[camera].permute(1,2,0).numpy()
+            assert np.array_equal(image,other),(i,camera,int(np.max(np.abs(image.astype(int)-other))))
+            checks.append({'row':i,'source_row':int(view.rows[i]),'camera':camera,
+                           'timestamp':timestamp,'decoded_pts':pts,'sha256':hashlib.sha256(image.tobytes()).hexdigest()})
+    assert inventory==source_inventory(view.root),'Source dataset was mutated'
+    out=Path(args.output);view.save_audit(out)
+    report={'status':'passed','manifest':view.manifest(),'anchors':anchor_report,'video_checks':checks,
+            'all_selected_targets_equal_source':True,'all_anchors_task_safe':True,'source_unchanged':True}
+    (out/'data_validation.json').write_text(json.dumps(report,indent=2)+'\n')
+    print(json.dumps({**report,'video_checks':len(checks)},indent=2))
+
+
+if __name__=='__main__':main()

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/data/view.py

@@ -0,0 +1,166 @@
+"""A small in-memory index over original v3 files; never exports another dataset."""
+from pathlib import Path
+import copy
+import hashlib
+import json
+import numpy as np
+import pyarrow.parquet as pq
+import yaml
+
+
+def load_config(path):
+    path = Path(path).resolve()
+    cfg = yaml.safe_load(path.read_text())
+    allowed = {'source', 'sampling', 'state', 'action', 'statistics', 'video', 'adapters'}
+    if set(cfg) - allowed:
+        raise ValueError(f'Unknown data keys: {set(cfg)-allowed}')
+    if 'output' in cfg or cfg['action']['source'] != 'action':
+        raise ValueError('Expected original actions and a read-only view')
+    if cfg['state']['indices'] != [0, 20] or cfg['action']['indices'] != [0, 23]:
+        raise ValueError('Expected state20/action23')
+    if not cfg['sampling']['split_at_task_change']:
+        raise ValueError('Task boundaries are mandatory')
+    if cfg['action']['invalid'] != 'mask_loss' or cfg['statistics'] != {'state':'all_inputs', 'action':'valid_targets'}:
+        raise ValueError('Unsupported validity/statistics contract')
+    if cfg['video'] != {'mode':'original', 'alignment':'timestamp'}:
+        raise ValueError('Videos must use original timestamps')
+    if cfg['adapters'] != {'gr00t_n17':{'enabled':True,'type':'letools_native_v3'}}:
+        raise ValueError('This v3 contract requires the enabled native N1.7 adapter')
+    return cfg
+
+
+def statistics(values, mask=None):
+    result = {k: [] for k in ['mean', 'std', 'min', 'max', 'q01', 'q99', 'count']}
+    for d in range(values.shape[1]):
+        a = values[:, d] if mask is None else values[mask[:, d], d]
+        if not len(a) or not np.isfinite(a).all():
+            raise ValueError(f'No finite valid statistics for dimension {d}')
+        for k, v in zip(result, [a.mean(dtype=np.float64), a.std(dtype=np.float64), a.min(), a.max(),
+                                 np.quantile(a, .01), np.quantile(a, .99), len(a)]):
+            result[k].append(float(v) if k != 'count' else int(v))
+    return result
+
+
+class DataView:
+    def __init__(self, config_path):
+        self.config_path = str(Path(config_path).resolve())
+        self.config = load_config(config_path)
+        self.root = Path(self.config['source']['root']).resolve()
+        self.info = json.loads((self.root/'meta/info.json').read_text())
+        assert self.info['codebase_version'].startswith('v3')
+        sf, fps = self.config['sampling']['source_fps'], self.config['sampling']['fps']
+        if self.info['fps'] != sf or sf % fps:
+            raise ValueError('This adapter requires an exact integer FPS stride')
+        self.stride, self.fps = sf // fps, fps
+        if self.stride < 1: raise ValueError('Cannot upsample')
+        files = sorted((self.root/'data').rglob('*.parquet'))
+        table = pq.read_table(files)
+        order = np.argsort(table['index'].to_numpy())
+        self.raw = {k: np.asarray(table[k].to_pylist())[order] for k in
+                    ['index', 'episode_index', 'frame_index', 'timestamp', 'task_index',
+                     self.config['state']['source'], 'action', self.config['action']['validity_feature']]}
+        assert np.array_equal(self.raw['index'], np.arange(len(order)))
+        self.episode_meta = {int(r['episode_index']): r for r in
+                             pq.read_table(sorted((self.root/'meta/episodes').rglob('*.parquet'))).to_pylist()}
+        tasks = pq.read_table(self.root/'meta/tasks.parquet').to_pandas()
+        self.tasks = {int(row.task_index): str(label) for label, row in tasks.iterrows()}
+        self.cameras = [k for k,v in self.info['features'].items() if v['dtype']=='video']
+        self.segments, selection = [], []
+        for ep in sorted(self.episode_meta):
+            rows = np.flatnonzero(self.raw['episode_index']==ep)
+            assert np.all(np.diff(rows)==1)
+            task = self.raw['task_index'][rows]
+            edges = np.r_[0, np.flatnonzero(task[1:]!=task[:-1])+1, len(rows)]
+            selected = rows[::self.stride]
+            for lo, hi in zip(edges[:-1], edges[1:]):
+                chosen = selected[(selected>=rows[lo]) & (selected<rows[hi-1]+1)]
+                if not len(chosen): continue
+                start = len(selection); selection.extend(chosen.tolist())
+                self.segments.append(dict(start=start, end=len(selection), source_episode=ep,
+                                          source_from=int(rows[lo]), source_to=int(rows[hi-1])+1,
+                                          task=int(task[lo])))
+        self.rows = np.asarray(selection, dtype=np.int64)
+        self.state = self.raw[self.config['state']['source']][self.rows, :20].astype(np.float32)
+        self.action = self.raw['action'][self.rows, :23].astype(np.float32)
+        self.validity = self.raw[self.config['action']['validity_feature']][self.rows,:23].astype(bool)
+        self.task = self.raw['task_index'][self.rows].astype(np.int64)
+        self.episode = np.empty(len(self.rows), dtype=np.int64)
+        for i,s in enumerate(self.segments): self.episode[s['start']:s['end']] = i
+        assert np.isfinite(self.state).all() and np.isfinite(self.action).all()
+        self.stats = {'observation.state': statistics(self.state), 'action': statistics(self.action,self.validity)}
+        signature = [(str(p.relative_to(self.root)),p.stat().st_size,p.stat().st_mtime_ns) for p in files]
+        self.fingerprint = hashlib.sha256(json.dumps([self.config,signature],sort_keys=True).encode()).hexdigest()
+
+    def anchors(self, policy):
+        history, future = {'gr00t_n15':(0,15),'gr00t_n17':(0,39),'multi_task_dit':(1,30)}[policy]
+        return np.concatenate([np.arange(s['start']+history,s['end']-future,dtype=np.int64)
+                               for s in self.segments if s['end']-s['start']>history+future])
+
+    def offsets(self, policy):
+        return np.arange(-1,31) if policy=='multi_task_dit' else np.arange(40 if policy=='gr00t_n17' else 16)
+
+    def video_location(self, row, camera):
+        raw = int(self.rows[row]); ep = int(self.raw['episode_index'][raw]); meta=self.episode_meta[ep]
+        prefix='videos/'+camera+'/'
+        path=self.root/self.info['video_path'].format(video_key=camera,
+              chunk_index=int(meta[prefix+'chunk_index']),file_index=int(meta[prefix+'file_index']))
+        return path, float(meta[prefix+'from_timestamp'])+float(self.raw['timestamp'][raw])
+
+    def manifest(self):
+        return dict(fingerprint=self.fingerprint, config=self.config, source_rows=len(self.raw['index']),
+                    selected_rows=len(self.rows), task_episodes=len(self.segments),
+                    eligible_anchors={p:len(self.anchors(p)) for p in ['gr00t_n15','gr00t_n17','multi_task_dit']},
+                    invalid_action_elements=int((~self.validity).sum()), original_torso=True)
+
+    def save_audit(self, directory):
+        directory=Path(directory); directory.mkdir(parents=True,exist_ok=True)
+        (directory/'data_manifest.json').write_text(json.dumps(self.manifest(),indent=2)+'\n')
+        (directory/'statistics.json').write_text(json.dumps(self.stats,indent=2)+'\n')
+        np.savez(directory/'indices.npz',source_rows=self.rows,logical_episode=self.episode)
+
+
+def lerobot_dataset(view, policy):
+    import torch
+    from lerobot.datasets.lerobot_dataset import LeRobotDataset
+    from lerobot.utils.constants import IMAGENET_STATS
+    from scripts.adapters.language import letterbox224
+    native=LeRobotDataset(view.config['source']['repo_id'],root=view.root,video_backend='pyav',
+                          return_uint8=True,tolerance_s=1/30+.001)
+    meta=copy.copy(native.meta)
+    meta.info=copy.deepcopy(meta.info)
+    meta.info.update(fps=view.fps,total_frames=len(view.rows),total_episodes=len(view.segments))
+    meta.info['features']={k:copy.deepcopy(view.info['features'][k]) for k in
+                           [*view.cameras,'observation.state','action']}
+    meta.info['features']['observation.state']['shape']=[20]
+    meta.info['features']['observation.state']['names']=view.info['features']['observation.state']['names'][:20]
+    from datasets import Dataset
+    meta.episodes=Dataset.from_list([{'episode_index':i,'length':s['end']-s['start'],
+        'dataset_from_index':s['start'],'dataset_to_index':s['end'],
+        'tasks':[view.tasks[s['task']]]} for i,s in enumerate(view.segments)])
+    for camera in view.cameras: meta.info['features'][camera]['shape']=[224,224,3]
+    meta.stats={k:{s:torch.tensor(v) for s,v in stats.items()} for k,stats in view.stats.items()}
+    for camera in view.cameras:
+        meta.stats[camera]={k:torch.tensor(v,dtype=torch.float32) for k,v in IMAGENET_STATS.items()}
+    class ViewedDataset(torch.utils.data.Dataset):
+        def __init__(self):
+            self.meta=meta; self.root=view.root; self.episodes=None
+            self.num_frames=len(view.rows); self.num_episodes=len(view.segments)
+            self.features=meta.features; self.fps=view.fps; self.repo_id=native.repo_id
+        def __len__(self): return self.num_frames
+        def __getitem__(self,index):
+            index=int(index); offsets=view.offsets(policy); seq=index+offsets
+            if seq.min()<0 or seq.max()>=len(view.rows) or np.any(view.episode[seq]!=view.episode[index]):
+                raise ValueError('Sampling crossed a task boundary')
+            obs_indices=[index-1,index] if policy=='multi_task_dit' else [index]
+            frames=[native[int(view.rows[i])] for i in obs_indices]
+            result={k:torch.stack([letterbox224(f[k]) for f in frames]) for k in view.cameras}
+            states=torch.tensor(view.state[obs_indices])
+            if policy!='multi_task_dit':
+                result={k:v[0] for k,v in result.items()}; states=states[0]
+            result.update({'observation.state':states,'action':torch.tensor(view.action[seq]),
+                'action_is_pad':torch.zeros(len(seq),dtype=torch.bool), 'index':torch.tensor(index),
+                'episode_index':torch.tensor(view.episode[index]), 'task_index':torch.tensor(view.task[index]),
+                'timestamp':torch.tensor(index/view.fps,dtype=torch.float32),
+                'task':view.tasks[int(view.task[index])]})
+            return result
+    return ViewedDataset()

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/tests/test_contract.py

@@ -0,0 +1,65 @@
+"""CPU tests for missing targets, batch masks and the first-finisher queue."""
+import importlib
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+class Contract(unittest.TestCase):
+    def test_masked_targets_and_all_masked_gradients(self):
+        import torch
+        from scripts.adapters.action_validity import masked_mean
+        p=torch.randn(3,40,132,requires_grad=True)
+        mask=torch.zeros_like(p,dtype=torch.bool);mask[...,:23]=True;mask[1,7,20:23]=False
+        a=p.square();b=a.clone();b[~mask]=1e8
+        torch.testing.assert_close(masked_mean(a,mask),masked_mean(b,mask))
+        masked_mean(b,mask).backward()
+        self.assertTrue((p.grad[~mask]==0).all());self.assertGreater(p.grad[mask].abs().sum(),0)
+        p.grad=None;loss=masked_mean(p.square(),torch.zeros_like(mask));loss.backward()
+        self.assertEqual(float(loss.detach()),0);self.assertTrue((p.grad==0).all())
+
+    def test_real_source_mask_offsets(self):
+        import numpy as np
+        import torch
+        from scripts.data.view import DataView
+        from scripts.adapters.action_validity import MaskBank
+        v=DataView(os.environ['RWH_DATA_CONFIG'])
+        for policy in ['gr00t_n15','multi_task_dit']:
+            anchors=v.anchors(policy);seq=anchors[:,None]+v.offsets(policy)[None,:]
+            bad=np.flatnonzero((~v.validity[seq]).any(axis=(1,2)))
+            self.assertGreater(len(bad),0)
+            selected=anchors[np.resize(bad,128)];expected=torch.tensor(v.raw['validity.action'][v.rows[selected[:,None]+v.offsets(policy)],:23])
+            processed={'action_mask':torch.ones(128,16,32,dtype=torch.bool)} if policy=='gr00t_n15' else {'action':torch.zeros(128,32,23)}
+            actual=MaskBank(v,policy).attach(processed,{'index':torch.tensor(selected)})
+            mask=actual['action_mask'] if policy=='gr00t_n15' else actual['rwh_action_validity']
+            self.assertTrue(torch.equal(mask[...,:23],expected))
+            self.assertFalse(mask[...,23:].any())
+
+    def test_first_finisher_starts_dit_once(self):
+        with tempfile.TemporaryDirectory() as td:
+            out=Path(td);os.environ.setdefault('RWH_TRAIN_CONFIG_DIR',td)
+            module=importlib.import_module('scripts.train.orchestrate')
+            policies=['gr00t_n15','gr00t_n17','multi_task_dit']
+            for policy in policies:
+                p=out/'_validation'/policy/'result.json';p.parent.mkdir(parents=True)
+                p.write_text(json.dumps({'status':'passed','masked_gradient_checks':3,'batch_size':128,'gradient_accumulation_steps':1}))
+                p=out/'_audit'/f'reload_{policy}.json';p.parent.mkdir(parents=True,exist_ok=True)
+                p.write_text(json.dumps({'status':'passed','separate_process_reload':True}))
+            (out/'_audit/data_validation.json').write_text('{"status":"passed"}')
+            launched=[]
+            class Child:
+                pid=999
+                def poll(self):return 0
+            def launch(command,env,**kw):
+                policy=command[2];gpu=int(command[4]);launched.append((policy,gpu))
+                p=Path(env['RWH_RESULT_DIR'])/'result.json'
+                p.write_text(json.dumps({'status':'passed','completed_at':10 if policy=='gr00t_n17' else 20}))
+                return Child()
+            with patch.object(module,'OUTPUT',out),patch.object(module.subprocess,'Popen',launch),patch.object(module.time,'sleep',lambda _:None):
+                module.main()
+            self.assertEqual(launched,[('gr00t_n15',0),('gr00t_n17',1),('multi_task_dit',1)])
+            self.assertEqual(json.loads((out/'_runs/controller_status.json').read_text())['state'],'complete')
+
+if __name__=='__main__':unittest.main()

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/train/__init__.py

@@ -0,0 +1 @@
+"""LeTools training launchers."""

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/train/checkpoints.py

@@ -0,0 +1,35 @@
+"""Validate the native Transformers checkpoint layout, including weight shards."""
+import json
+from pathlib import Path
+
+
+def weight_files(model_dir):
+    root=Path(model_dir)
+    index=root/'model.safetensors.index.json'
+    if index.exists():
+        mapping=json.loads(index.read_text())['weight_map']
+        names=sorted(set(mapping.values()))
+        if not names or any(Path(n).name!=n for n in names):raise ValueError('Unsafe/empty weight index')
+        files=[root/n for n in names]
+        if not all(p.is_file() and p.stat().st_size>0 for p in files):raise ValueError('Missing model shard')
+        from safetensors import safe_open
+        for path in files:
+            with safe_open(str(path),framework='pt',device='cpu') as sf:
+                if set(sf.keys())!={key for key,name in mapping.items() if name==path.name}:
+                    raise ValueError('Weight index and shard keys differ: '+path.name)
+    else:
+        files=[root/'model.safetensors']
+        if not files[0].is_file():raise ValueError('No safetensors weights or shard index')
+    return files
+
+
+def validate_native_checkpoint(path,step):
+    path=Path(path)
+    for name in ['config.json','processor_config.json','statistics.json','embodiment_id.json','trainer_state.json']:
+        if not (path/name).exists():
+            raise ValueError('Missing native checkpoint file: '+name)
+    processor=json.loads((path/'processor_config.json').read_text())
+    if not processor['processor_kwargs']['modality_configs']:raise ValueError('Missing saved modalities')
+    state=json.loads((path/'trainer_state.json').read_text())
+    if state['global_step']!=step:raise ValueError('Wrong native saved step')
+    return {'step':step,'weight_files':[p.name for p in weight_files(path)]}

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/train/hf_watcher.py

@@ -0,0 +1,363 @@
+"""Publish verified 10k/20k/30k checkpoints from successful Kuavo runs.
+
+CPU only. Never modifies a trainer checkpoint, last symlink, or optimizer state.
+All paths/revisions are pinned in hf_watcher.json before starting this process.
+"""
+from __future__ import annotations
+
+import argparse
+import fcntl
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import time
+
+POLICIES = ("multi_task_dit", "gr00t_n15", "gr00t_n17")
+STEPS = (10000, 20000, 30000)
+
+
+def checkpoint_plan(cfg):
+    configured = cfg.get("steps_by_policy", {p: list(STEPS) for p in POLICIES})
+    if set(configured) != set(POLICIES):
+        raise ValueError("Expected checkpoint steps for all three policies")
+    for policy, steps in configured.items():
+        if not steps or len(steps) != len(set(steps)) or any(s not in STEPS for s in steps):
+            raise ValueError("Invalid checkpoint steps: " + policy)
+    return [(policy, step) for step in STEPS for policy in POLICIES if step in configured[policy]]
+
+
+def remote_prefix(cfg, policy, step):
+    root = cfg["remote_root"]
+    if root != "v2":
+        raise ValueError("The approved HF experiment root is v2")
+    return f"{root}/{policy}/checkpoints/checkpoint-{step // 1000}K"
+
+
+class NotReady(Exception):
+    pass
+
+
+def read(path):
+    return json.loads(Path(path).read_text())
+
+
+def atomic_json(path, value):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + ".tmp")
+    temporary.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n")
+    temporary.replace(path)
+
+
+def signature(root):
+    """A stable native snapshot includes training_state, even though it is not uploaded."""
+    values = {}
+    for f in sorted(root.rglob("*")):
+        if f.is_symlink():
+            raise ValueError("Checkpoint contains an unexpected symlink")
+        if f.is_file():
+            s = f.stat()
+            values[str(f.relative_to(root))] = [s.st_size, s.st_mtime_ns]
+    return values
+
+
+def completed_source(cfg, policy, step):
+    output = Path(cfg["output_root"]) / policy
+    result = read(cfg["results"][policy])
+    if result.get("status") != "passed":
+        raise NotReady("Waiting for successful final training and inference verification")
+    if result["policy"] != policy or result["smoke"] or result["steps_requested"] != 30000:
+        raise ValueError("Unexpected production identity")
+    if Path(result["output_dir"]) != output:
+        raise ValueError("Wrong production output")
+    final = Path(result["checkpoint"])
+    expected_final = output / ("checkpoint-30000" if policy == "gr00t_n17" else "checkpoints/030000")
+    if final.resolve() != expected_final.resolve():
+        raise ValueError("Unexpected final checkpoint location")
+    checkpoint = output / (f"checkpoint-{step}" if policy == "gr00t_n17" else f"checkpoints/{step:06d}")
+    if checkpoint.resolve().parent not in [output.resolve(), (output/"checkpoints").resolve()]:
+        raise ValueError("Checkpoint outside this run")
+    if policy == "gr00t_n17":
+        if read(final/"rwh_save_complete.json")["step"] != 30000:
+            raise ValueError("Native save marker step mismatch")
+        from scripts.train.checkpoints import validate_native_checkpoint
+        validate_native_checkpoint(checkpoint, step)
+        model = checkpoint
+        required = ["config.json", "processor_config.json", "statistics.json", "embodiment_id.json"]
+    else:
+        if read(checkpoint/"training_state/training_step.json")["step"] != step:
+            raise ValueError("LeRobot training step mismatch")
+        model = checkpoint/"pretrained_model"
+        required = ["config.json", "train_config.json", "policy_preprocessor.json", "policy_postprocessor.json", "model.safetensors"]
+        train = read(model/"train_config.json")
+        for key,value in {"steps":30000,"batch_size":128,"seed":42,"save_freq":10000}.items():
+            if train[key] != value: raise ValueError("Wrong training contract: "+key)
+    for name in required:
+        if not (model/name).is_file(): raise ValueError("Missing checkpoint file: "+name)
+    from scripts.train.checkpoints import weight_files
+    weight_files(model)
+    snapshot=signature(checkpoint)
+    newest=max(mtime for size,mtime in snapshot.values())/1e9
+    if time.time()-newest < cfg["stable_seconds"]: raise NotReady("Waiting for stability interval")
+    return checkpoint,snapshot
+
+
+def file_info(path):
+    size = path.stat().st_size
+    sha256 = hashlib.sha256()
+    blob = hashlib.sha1(f"blob {size}\0".encode())
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
+            sha256.update(chunk)
+            blob.update(chunk)
+    return {"size": size, "sha256": sha256.hexdigest(), "git_blob_sha1": blob.hexdigest()}
+
+
+def validate_tensors(model):
+    import torch
+    from safetensors import safe_open
+    torch.set_num_threads(2)
+    count = 0
+    for file in sorted(model.glob("*.safetensors")):
+        with safe_open(str(file), framework="pt", device="cpu") as tensors:
+            for name in tensors.keys():
+                tensor = tensors.get_slice(name)
+                shape = tensor.get_shape()
+                if not shape:
+                    chunks = [tensors.get_tensor(name)]
+                else:
+                    rows = max(1, 8_000_000 // max(1, math.prod(shape[1:])))
+                    chunks = (tensor[i:i + rows] for i in range(0, shape[0], rows))
+                for chunk in chunks:
+                    if not torch.isfinite(chunk).all().item():
+                        raise ValueError("Nonfinite saved tensor: " + name)
+                count += 1
+    if not count:
+        raise ValueError("No saved tensors")
+    return count
+
+
+def compare_remote(api, repo, prefix, revision, files):
+    """LFS SHA256 or Git object SHA1 proves actual content, beyond size equality."""
+    from huggingface_hub.utils import EntryNotFoundError
+    try:
+        entries = list(api.list_repo_tree(repo, path_in_repo=prefix, revision=revision,
+                                          repo_type="model", recursive=True))
+    except EntryNotFoundError:
+        entries = []
+    remote = {entry.path[len(prefix) + 1:]: entry for entry in entries if hasattr(entry, "size")}
+    missing = sorted(set(files) - set(remote))
+    extra = sorted(set(remote) - set(files))
+    mismatch = []
+    for name in sorted(set(files) & set(remote)):
+        expected, actual = files[name], remote[name]
+        lfs = actual.lfs
+        remote_hash = lfs.sha256 if lfs is not None else actual.blob_id
+        expected_hash = expected["sha256"] if lfs is not None else expected["git_blob_sha1"]
+        if actual.size != expected["size"] or remote_hash != expected_hash:
+            mismatch.append(name)
+    return {"missing": missing, "extra": extra, "mismatch": mismatch,
+            "file_count": len(remote), "bytes": sum(x.size for x in remote.values())}
+
+
+def matches(check):
+    return not any(check[k] for k in ("missing", "extra", "mismatch"))
+
+
+def payload(cfg, policy, step, checkpoint, snapshot):
+    prefix = remote_prefix(cfg, policy, step)
+    work=Path(cfg["state_dir"])/"payloads"/cfg["remote_root"]/policy/str(step)
+    manifest_path=work/"checkpoint_manifest.json"
+    model=checkpoint if policy=="gr00t_n17" else checkpoint/"pretrained_model"
+    def included(f):
+        rel=f.relative_to(checkpoint)
+        return not any(x in ("training_state",) for x in rel.parts) and not f.name.startswith(("optimizer", "scheduler", "rng_state"))
+    if manifest_path.exists():
+        manifest=read(manifest_path)
+        if manifest["source_signature"] != snapshot or manifest["remote_prefix"] != prefix:
+            raise ValueError("Saved checkpoint or destination changed")
+    else:
+        count=validate_tensors(model)
+        provenance=work/"provenance";provenance.mkdir(parents=True,exist_ok=True)
+        for source,target in cfg["provenance"][policy].items():
+            dest=provenance/target;dest.parent.mkdir(parents=True,exist_ok=True)
+            dest.write_bytes(Path(source).read_bytes())
+        (work/"README.md").write_text(
+            f"# {policy}: original-action Kuavo, checkpoint {step} of 30000 updates\n\n"
+            "Original v3 data; 30Hz to 10Hz logical view; state20/action23; seed42; actual batch128; accumulation1.\n"
+            "Torso targets are the original action commands. Invalid action elements are masked.\n"
+            "N1.5/DiT use LeTools-pinned LeRobot; N1.7 uses bundled native model, processor, sharded loader and Trainer.\n"
+            "LeTools revision: 6d2002613241ae2a32c5d8fd753f80caa1383fba. See provenance for adapter and YAML.\n"
+            "N1.7 uses native checkpoint layout. Optimizer/RNG states stay on the training server.\n")
+        paths={str(f.relative_to(checkpoint)):f for f in checkpoint.rglob("*") if f.is_file() and included(f)}
+        paths.update({str(f.relative_to(work)):f for f in work.rglob("*") if f.is_file()})
+        manifest={"schema":2,"policy":policy,"step":step,"repo_id":cfg["repo_id"],
+            "remote_prefix":prefix,
+            "source_checkpoint":str(checkpoint),"source_signature":snapshot,"tensor_count":count,
+            "training_state_included":False,"files":{n:file_info(f) for n,f in sorted(paths.items())}}
+        if signature(checkpoint)!=snapshot: raise ValueError("Checkpoint changed during validation")
+        atomic_json(manifest_path,manifest)
+    paths={name:work/name if name.startswith("provenance/") or name=="README.md" else checkpoint/name
+           for name in manifest["files"]}
+    paths["checkpoint_manifest.json"]=manifest_path
+    files=dict(manifest["files"]);files["checkpoint_manifest.json"]=file_info(manifest_path)
+    return manifest,paths,files
+
+
+def publish(api, cfg, policy, step):
+    from huggingface_hub import CommitOperationAdd
+    checkpoint, snapshot = completed_source(cfg, policy, step)
+    manifest, paths, files = payload(cfg, policy, step, checkpoint, snapshot)
+    prefix, repo = manifest["remote_prefix"], cfg["repo_id"]
+    print(json.dumps({"mapping": {"local": str(checkpoint), "HF": repo + "/" + prefix},
+                      "step": step, "files": len(files)}), flush=True)
+    head = api.model_info(repo).sha
+    before = compare_remote(api, repo, prefix, head, files)
+    if before["extra"] or before["mismatch"]:
+        raise ValueError("Existing HF checkpoint conflicts with the validated local manifest")
+    uploaded = False
+    if before["missing"]:
+        if signature(checkpoint) != snapshot:
+            raise ValueError("Checkpoint changed before upload")
+        commit = api.create_commit(
+            repo, repo_type="model", parent_commit=head, num_threads=2,
+            operations=[CommitOperationAdd(path_in_repo=prefix + "/" + name,
+                                           path_or_fileobj=str(paths[name])) for name in before["missing"]],
+            commit_message=f"Kuavo original action: {policy} checkpoint {step // 1000}K")
+        head = commit.oid
+        uploaded = True
+    check = compare_remote(api, repo, prefix, head, files)
+    if not matches(check):
+        raise ValueError("Remote manifest verification failed: " + json.dumps(check))
+    if signature(checkpoint) != snapshot:
+        raise ValueError("Source changed during upload; do not mark verified")
+    return {"status": "verified", "policy": policy, "step": step, "revision": head,
+            "prefix": prefix, "uploaded_this_attempt": uploaded, "verification": check,
+            "verified_at": time.time(), "manifest": str(Path(cfg["state_dir"]) / "payloads" /
+                                                          cfg["remote_root"] / policy / str(step) / "checkpoint_manifest.json")}
+
+
+def once(api, cfg, state, selected=None, selected_step=None, audit=False):
+    plan = checkpoint_plan(cfg)
+    for step in STEPS:
+        for policy in POLICIES:
+            if (policy, step) not in plan:
+                continue
+            if selected and policy != selected or selected_step and step != selected_step:
+                continue
+            key = f"{policy}/{step}"
+            prior = state["checkpoints"].get(key, {})
+            same_destination = prior.get("prefix") == remote_prefix(cfg, policy, step)
+            if prior.get("status") == "verified" and same_destination and not audit:
+                continue
+            if prior.get("retry_after", 0) > time.time() and same_destination and not audit:
+                continue
+            state.update(current=key, heartbeat=time.time(), status="running")
+            atomic_json(Path(cfg["state_dir"]) / "status.json", state)
+            try:
+                record = publish(api, cfg, policy, step)
+            except (NotReady, FileNotFoundError, json.JSONDecodeError) as exc:
+                record = {"status": "waiting", "reason": str(exc), "checked_at": time.time()}
+            except Exception as exc:
+                # Never serialize tokens/headers or HTTP exception payloads.
+                attempts = prior.get("attempts", 0) + 1
+                record = {"status": "error", "error_type": type(exc).__name__,
+                          "reason": str(exc)[:500] if isinstance(exc, ValueError) else "See service type; retry scheduled",
+                          "attempts": attempts, "retry_after": time.time() + min(1800, 60 * 2 ** min(attempts, 5))}
+            record["prefix"] = remote_prefix(cfg, policy, step)
+            state["checkpoints"][key] = record
+            state["heartbeat"] = time.time()
+            atomic_json(Path(cfg["state_dir"]) / "status.json", state)
+            print(json.dumps({"checkpoint": key, **record}), flush=True)
+    state["current"] = None
+    records = [state["checkpoints"].get(f"{p}/{s}", {}) for p, s in plan]
+    state["expected_checkpoints"] = len(plan)
+    state["verified_count"] = sum(x.get("status") == "verified" and
+        x.get("prefix") == remote_prefix(cfg, p, s) for (p, s), x in zip(plan, records))
+    if state["verified_count"] == len(plan):
+        # Recheck the entire set against ONE current HF revision before declaring
+        # completion, including checkpoints uploaded many hours earlier.
+        try:
+            revision = api.model_info(cfg["repo_id"]).sha
+            for record in records:
+                manifest_path = Path(record["manifest"])
+                manifest = read(manifest_path)
+                files = dict(manifest["files"])
+                files["checkpoint_manifest.json"] = file_info(manifest_path)
+                check = compare_remote(api, cfg["repo_id"], record["prefix"], revision, files)
+                if not matches(check):
+                    raise ValueError("Completion audit failed: " + record["prefix"])
+            state["completion_audit"] = {"status": "verified", "revision": revision,
+                                         "checkpoints": len(plan), "missing": 0, "extra": 0, "mismatch": 0}
+        except Exception as exc:
+            state["completion_audit"] = {"status": "error", "error_type": type(exc).__name__}
+    state["status"] = "complete" if state["verified_count"] == len(plan) and state.get("completion_audit", {}).get("status") == "verified" else "watching"
+    state["attention"] = [k for k, v in state["checkpoints"].items() if v.get("status") == "error"]
+    if state.get("completion_audit", {}).get("status") == "error":
+        state["attention"].append("completion_audit_failed")
+    for policy, path in cfg["results"].items():
+        try:
+            if read(path)["status"] == "failed":
+                state["attention"].append(policy + ":training_failed")
+        except (FileNotFoundError, json.JSONDecodeError):
+            try:
+                controller = read(Path(cfg["run_dir"]) / "controller_status.json")
+                if controller.get("jobs", {}).get(policy, {}).get("state") == "failed":
+                    state["attention"].append(policy + ":training_failed")
+            except (FileNotFoundError, json.JSONDecodeError):
+                pass
+    state["heartbeat"] = time.time()
+    atomic_json(Path(cfg["state_dir"]) / "status.json", state)
+
+
+def main():
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument("--config", required=True, type=Path)
+    parser.add_argument("--once", action="store_true")
+    parser.add_argument("--policy", choices=POLICIES)
+    parser.add_argument("--step", type=int, choices=STEPS)
+    parser.add_argument("--audit-remote", action="store_true")
+    args = parser.parse_args()
+    os.environ["CUDA_VISIBLE_DEVICES"] = ""
+    os.environ["HF_HUB_OFFLINE"] = "0"
+    os.environ["TRANSFORMERS_OFFLINE"] = "0"
+    os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
+    os.environ["OMP_NUM_THREADS"] = "2"
+    os.environ["MKL_NUM_THREADS"] = "2"
+    from huggingface_hub import HfApi
+    cfg = read(args.config)
+    state_dir = Path(cfg["state_dir"])
+    state_dir.mkdir(parents=True, exist_ok=True)
+    with (state_dir / "watcher.lock").open("w") as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        token_file = cfg.get('token_file')
+        api = HfApi(token=Path(token_file).read_text().strip() if token_file else None)
+        identity = api.whoami()
+        if identity["name"] != cfg["hf_account"]:
+            raise ValueError("Unexpected HF account")
+        if not any(o["name"] == "RwH-Lab" and o.get("roleInOrg") in ("write", "admin")
+                   for o in identity.get("orgs", [])):
+            raise ValueError("Missing HF organization write permission")
+        # User authorized a new checkpoint repo; keep private on creation.
+        api.create_repo(cfg["repo_id"], repo_type="model", private=True, exist_ok=True)
+        if not api.model_info(cfg["repo_id"]).private:
+            raise ValueError("Expected a private checkpoint repository")
+        status_path = state_dir / "status.json"
+        state = read(status_path) if status_path.exists() else {"schema": 1, "checkpoints": {}}
+        if state.get("run_dir", cfg["run_dir"]) != cfg["run_dir"]:
+            raise ValueError("State belongs to a different run")
+        state.update(repo_id=cfg["repo_id"], run_dir=cfg["run_dir"], pid=os.getpid(),
+                     expected_checkpoints=len(checkpoint_plan(cfg)), remote_root=cfg["remote_root"],
+                     started_at=state.get("started_at", time.time()))
+        while True:
+            once(api, cfg, state, args.policy, args.step, args.audit_remote)
+            if args.once or state["status"] == "complete":
+                break
+            time.sleep(cfg["poll_seconds"])
+    print("HF_WATCHER_EXIT", state["status"], state["verified_count"], flush=True)
+
+
+if __name__ == "__main__":
+    main()

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/train/language_guard.py

@@ -0,0 +1,13 @@
+"""Fail before a training update if a sampled state's language is invalid."""
+
+
+def validate_training_batch(batch, task_lookup):
+    prompts = batch.get("task")
+    task_indices = batch["task_index"].tolist()
+    if not isinstance(prompts, list) or len(prompts) != 128:
+        raise RuntimeError("Every sampled state must have its own task language")
+    if any(not isinstance(p, str) or not p.strip() or p != task_lookup[int(i)]
+           for p, i in zip(prompts, task_indices, strict=True)):
+        raise RuntimeError("Empty or mismatched state/task language")
+    if batch["action_is_pad"].any():
+        raise RuntimeError("A sampled action chunk crossed its skill boundary")

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/train/load_n17.py

@@ -0,0 +1,20 @@
+"""Load a standalone native checkpoint with portable pinned backbone assets."""
+from pathlib import Path
+import sys
+
+def load(path,device='cuda'):
+    project=Path(__file__).resolve().parents[2]
+    sys.path.insert(0,str(project.parent/'LeTools-Learning/kuavo_model/external_models/gr00tn1d7'))
+    from scripts.train.native_assets import local_cosmos
+    from gr00t.configs.model.gr00t_n1d7 import Gr00tN1d7Config
+    from gr00t.model.gr00t_n1d7.gr00t_n1d7 import Gr00tN1d7
+    from gr00t.model.gr00t_n1d7.processing_gr00t_n1d7 import Gr00tN1d7Processor
+    path=Path(path)
+    if (path/'pretrained_model').exists():path=path/'pretrained_model'
+    cfg=Gr00tN1d7Config.from_pretrained(path);cfg.model_name=local_cosmos()
+    model=Gr00tN1d7.from_pretrained(path,config=cfg,
+              transformers_loading_kwargs={'local_files_only':True,'trust_remote_code':True}).to(device).eval()
+    processor=Gr00tN1d7Processor.from_pretrained(path,model_name=local_cosmos(),
+              transformers_loading_kwargs={'local_files_only':True,'trust_remote_code':True})
+    processor.eval()
+    return model,processor

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/train/native_assets.py

@@ -0,0 +1,48 @@
+"""Pin local assets while preserving the native backbone-name routing contract."""
+import os
+from pathlib import Path
+
+
+def pinned_cosmos_name():
+    from huggingface_hub import try_to_load_from_cache
+    name = "nvidia/Cosmos-Reason2-2B"
+    config = try_to_load_from_cache(name, "config.json", revision="main")
+    if not isinstance(config, str) or Path(config).parent.name != "9ce19a195e423419c349abfc86fd07178b230561":
+        raise RuntimeError("Cosmos cached main differs from the approved revision")
+    return name
+
+
+def local_cosmos():
+    from huggingface_hub import snapshot_download
+    revision = "9ce19a195e423419c349abfc86fd07178b230561"
+    snapshot = Path(snapshot_download("nvidia/Cosmos-Reason2-2B", revision=revision, local_files_only=True))
+    # Native get_backbone_cls recognizes the nvidia/Cosmos-Reason2 substring.
+    alias = Path(os.environ.get("HF_HOME", str(Path.home()/".cache/huggingface"))) / "rwh_pinned/nvidia/Cosmos-Reason2-2B" / revision
+    alias.parent.mkdir(parents=True, exist_ok=True)
+    if alias.is_symlink():
+        if alias.resolve() != snapshot.resolve():
+            raise RuntimeError("Pinned Cosmos alias points at another snapshot")
+    elif alias.exists():
+        raise RuntimeError("Pinned Cosmos alias is occupied")
+    else:
+        try: alias.symlink_to(snapshot, target_is_directory=True)
+        except FileExistsError:
+            if alias.resolve() != snapshot.resolve(): raise
+    return str(alias)
+
+
+def verify_legacy_refs(policy):
+    from huggingface_hub import try_to_load_from_cache
+    assets = {
+        "nvidia/GR00T-N1.5-3B": "869830fc749c35f34771aa5209f923ac57e4564e",
+        "openai/clip-vit-base-patch16": "57c216476eefef5ab752ec549e440a49ae4ae5f3",
+        "lerobot/eagle2hg-processor-groot-n1p5": "baf604d8a5caf26fda5cc545f141bc1814156237",
+    }
+    repos = ["openai/clip-vit-base-patch16"] if policy == "multi_task_dit" else ["nvidia/GR00T-N1.5-3B"]
+    for repo in repos:
+        current = try_to_load_from_cache(repo, "config.json", revision="main")
+        pinned = try_to_load_from_cache(repo, "config.json", revision=assets[repo])
+        # DiT receives the explicit pinned CLIP snapshot path in its config.
+        if not isinstance(pinned, str) or (policy != "multi_task_dit" and
+                (not isinstance(current, str) or Path(current).parent.name != assets[repo])):
+            raise RuntimeError("Cached main is not the approved model revision: " + repo)

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/train/orchestrate.py

@@ -0,0 +1,92 @@
+"""Start verified fresh runs; queue DiT on the first successfully completed GPU."""
+import fcntl
+import json
+import os
+from pathlib import Path
+import subprocess
+import time
+
+ROOT = Path(os.environ["RWH_TRAIN_CONFIG_DIR"]).resolve()
+OUTPUT = Path("/home/work/mntvol/RwH-Kuavo_ckpt/v3_rawaction")
+POLICIES = ("gr00t_n15", "gr00t_n17", "multi_task_dit")
+
+
+def write(path, value):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text(json.dumps(value, indent=2) + "\n")
+    tmp.replace(path)
+
+
+def main():
+    OUTPUT.mkdir(parents=True, exist_ok=True)
+    lock = (OUTPUT / "launch.lock").open("w")
+    fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    for policy in POLICIES:
+        result = json.loads((OUTPUT / "_validation" / policy / "result.json").read_text())
+        if result["status"] != "passed" or result.get("masked_gradient_checks", 0) < 3:
+            raise RuntimeError("Full-batch/mask/inference validation not passed: " + policy)
+        if result["batch_size"] != 128 or result["gradient_accumulation_steps"] != 1:
+            raise RuntimeError("Validation used another batch contract")
+        reload = json.loads((OUTPUT / '_audit' / f'reload_{policy}.json').read_text())
+        if reload.get('status') != 'passed' or not reload.get('separate_process_reload'):
+            raise RuntimeError('Separate-process checkpoint reload not passed: '+policy)
+        if (OUTPUT / policy).exists():
+            raise FileExistsError("Production output already exists: " + policy)
+    validation = json.loads((OUTPUT / "_audit/data_validation.json").read_text())
+    if validation["status"] != "passed":
+        raise RuntimeError("Data validation failed")
+    status_path = OUTPUT / "_runs/controller_status.json"
+    status = {"state": "running", "started_at": time.time(), "pid": os.getpid(),
+        "batch_size": 128, "gradient_accumulation_steps": 1,
+        "steps": 30000, "save_every": 10000, "jobs": {}}
+    children = {}
+    def start(policy, gpu):
+        logdir = OUTPUT / "_runs" / policy
+        logdir.mkdir(parents=True, exist_ok=True)
+        env = os.environ.copy()
+        env["RWH_RESULT_DIR"] = str(logdir)
+        stream = (logdir / "console.log").open("a")
+        child = subprocess.Popen(["bash", str(ROOT / "launch.sh"), policy, "--gpu", str(gpu)],
+            env=env, stdout=stream, stderr=subprocess.STDOUT)
+        stream.close()
+        children[policy] = child
+        status["jobs"][policy] = {"state": "running", "gpu": gpu, "pid": child.pid,
+            "started_at": time.time(), "result": str(logdir / "result.json")}
+        write(status_path, status)
+        print("TRAINING_STARTED", policy, gpu, child.pid, flush=True)
+    start("gr00t_n15", 0)
+    start("gr00t_n17", 1)
+    status["jobs"]["multi_task_dit"] = {"state": "queued", "reason": "start once on the first successfully completed GPU 0 or 1"}
+    write(status_path, status)
+    while True:
+        finished=[]
+        for policy,child in list(children.items()):
+            exit_code=child.poll()
+            if exit_code is None or status["jobs"][policy]["state"]!="running":continue
+            result_path=OUTPUT/"_runs"/policy/"result.json"
+            try:result=json.loads(result_path.read_text())
+            except (FileNotFoundError,json.JSONDecodeError):result={}
+            finished.append((result.get("completed_at",time.time()),policy,exit_code,result))
+        # Both jobs can finish during one poll. Honor actual validated completion time.
+        for finished_at,policy,exit_code,result in sorted(finished):
+            passed=exit_code==0 and result.get("status")=="passed"
+            status["jobs"][policy].update(state="passed" if passed else "failed",
+                exit_code=exit_code,finished_at=finished_at)
+            if passed and status["jobs"]["multi_task_dit"]["state"]=="queued":
+                start("multi_task_dit",status["jobs"][policy]["gpu"])
+            write(status_path,status)
+        running = any(j["state"] == "running" for j in status["jobs"].values())
+        if not running:
+            status["state"] = "complete" if all(j["state"] == "passed" for j in status["jobs"].values()) else "failed"
+            write(status_path, status)
+            if status["state"] != "complete": raise RuntimeError("Production training failure; inspect controller status")
+            lock.close()
+            break
+        status["heartbeat"] = time.time()
+        write(status_path, status)
+        time.sleep(15)
+
+
+if __name__ == "__main__":
+    main()

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/train/run_lerobot.py

@@ -0,0 +1,312 @@
+"""Project-local LeTools runner with policy-specific, skill-safe sampling.
+
+The upstream trainer remains untouched. Only its DataLoader reference is wrapped.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import importlib.util
+import json
+import os
+from pathlib import Path
+import statistics
+import sys
+import time
+import traceback
+
+
+ROOT = Path(os.environ["RWH_TRAIN_CONFIG_DIR"]).resolve()
+PROJECT = ROOT.parents[3]
+LETOOLS = PROJECT / "LeTools-Learning"
+OUTPUT_ROOT = Path("/home/work/mntvol/RwH-Kuavo_ckpt/v3_rawaction")
+LOG_ROOT = Path(os.environ.get("RWH_RUN_LOG_ROOT", str(OUTPUT_ROOT / "_runs")))
+
+
+def write_json(path, value):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n")
+
+
+def resolve(policy):
+    spec = importlib.util.spec_from_file_location("letools_train_config", LETOOLS / "kuavo_model/train.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    total = module._load_yaml(ROOT / "total" / f"{policy}_total.yaml")
+    simple = module._load_yaml(ROOT / f"{policy}.yaml")
+    simple["dataset"] = {"repo_id": DATA_VIEW.config["source"]["repo_id"], "root": str(DATA_VIEW.root)}
+    return module._convert_to_lerobot_train_config(module._deep_merge(total, simple))
+
+
+class Proxy:
+    def __init__(self, target, **overrides):
+        self.target, self.overrides = target, overrides
+
+    def __getattr__(self, key):
+        return self.overrides[key] if key in self.overrides else getattr(self.target, key)
+
+
+class CallableProxy(Proxy):
+    def __call__(self, *args, **kwargs):
+        return self.overrides["call"](*args, **kwargs)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("policy", choices=["multi_task_dit", "gr00t_n15"])
+    parser.add_argument("--gpu", required=True, choices=["0", "1"])
+    parser.add_argument("--smoke-steps", type=int, default=0)
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+    if int(os.environ.get("WORLD_SIZE", "1")) != 1:
+        raise RuntimeError("This contract uses one GPU per policy and actual batch 128")
+    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
+    os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+    os.environ.setdefault("OMP_NUM_THREADS", "8")
+    sys.path.insert(0, str(LETOOLS / "third_party/lerobot/src"))
+    from scripts.train.native_assets import verify_legacy_refs
+    verify_legacy_refs(args.policy)
+    from scripts.data.view import DataView, lerobot_dataset
+    import yaml
+    global DATA_VIEW
+    simple = yaml.safe_load((ROOT/f"{args.policy}.yaml").read_text())
+    DATA_VIEW = DataView(ROOT / simple["dataset"]["config"])
+    cfg = resolve(args.policy)
+    if args.policy == "multi_task_dit":
+        from huggingface_hub import snapshot_download
+        clip = snapshot_download("openai/clip-vit-base-patch16", revision="57c216476eefef5ab752ec549e440a49ae4ae5f3", allow_patterns=["*.json", "*.txt", "*.safetensors", "pytorch_model.bin"], local_files_only=True)
+        cfg["policy"]["vision_encoder_name"] = clip
+        cfg["policy"]["text_encoder_name"] = clip
+
+    stamp = time.strftime("%Y%m%d_%H%M%S")
+    tag = f"{args.policy}_{'smoke' if args.smoke_steps else 'train'}_{stamp}"
+    log_dir = Path(os.environ.get("RWH_RESULT_DIR", str(LOG_ROOT / tag)))
+    if args.smoke_steps:
+        if args.smoke_steps < 3:
+            raise ValueError("Use at least three updates for a meaningful smoke test")
+        cfg.update(steps=args.smoke_steps, save_freq=args.smoke_steps, log_freq=1,
+                   output_dir=str(OUTPUT_ROOT / "smoke" / tag))
+    if args.smoke_steps:
+        cfg["wandb"]["enable"] = False
+    cfg["dataset"]["video_backend"] = "pyav"
+    if cfg["batch_size"] != 128 or cfg["seed"] != 42 or cfg.get("resume"):
+        raise ValueError("Expected fresh actual batch=128, seed=42 contract")
+    config_path = log_dir / "resolved_train_config.json"
+    write_json(config_path, cfg)
+    print("RWH_CONFIG", config_path, flush=True)
+    import torch
+    import draccus
+    from lerobot.configs.train import TrainPipelineConfig
+    import lerobot.scripts.lerobot_train as trainer
+    from scripts.adapters import language as compat
+    from scripts.train.language_guard import validate_training_batch
+    compat.install(args.policy)
+    from scripts.adapters import action_validity as loss_mask
+    loss_mask.install(args.policy)
+    if args.policy in ("gr00t_n15", "multi_task_dit"):
+        def make_dataset(config):
+            return lerobot_dataset(DATA_VIEW, args.policy)
+
+        trainer.make_dataset = make_dataset
+    sys.argv = [sys.argv[0], f"--config_path={config_path}"]
+    # The native parser also loads registered policy configuration types.
+    parsed = draccus.parse(TrainPipelineConfig, config_path=str(config_path), args=[])
+    parsed.validate()
+    write_json(log_dir / "effective_train_config.json", parsed.to_dict())
+    if args.dry_run:
+        print("RWH_DRY_RUN_OK", args.policy, flush=True)
+        return
+
+    data_root = Path(cfg["dataset"]["root"])
+    mask_bank = loss_mask.MaskBank(DATA_VIEW, args.policy)
+    anchors = DATA_VIEW.anchors(args.policy).tolist()
+    manifest = DATA_VIEW.manifest()
+    task_lookup = DATA_VIEW.tasks
+    DATA_VIEW.save_audit(log_dir / "data")
+    report = {"status": "running", "policy": args.policy, "gpu": int(args.gpu),
+              "batch_size": 128, "gradient_accumulation_steps": 1,
+              "steps_requested": cfg["steps"], "smoke": bool(args.smoke_steps),
+              "eligible_anchors": len(anchors), "output_dir": cfg["output_dir"],
+              "sampling_rule": "full history/action chunks within logical task episodes",
+              "config_path": str(config_path), "measurements": [],
+              "source_revision": DATA_VIEW.config["source"]["revision"], "data_fingerprint": DATA_VIEW.fingerprint, "trainer_source": trainer.__file__}
+    report_path = log_dir / "result.json"
+    write_json(report_path, report)
+
+    class PolicySampler(torch.utils.data.Sampler):
+        def __init__(self):
+            self.epoch = 0
+
+        def __len__(self):
+            return len(anchors)
+
+        def __iter__(self):
+            generator = torch.Generator().manual_seed(42 + self.epoch)
+            self.epoch += 1
+            ordered = [anchors[i] for i in torch.randperm(len(anchors), generator=generator).tolist()]
+            if args.smoke_steps:
+                candidates = torch.tensor(anchors)
+                bad = (~mask_bank.mask[candidates[:, None] + mask_bank.offsets[None, :], 20:23]).any(dim=(1, 2))
+                ordered[0] = anchors[int(torch.nonzero(bad)[0])]
+            return iter(ordered)
+
+    preview = list(iter(PolicySampler()))[:128]
+    report["first_batch_anchors"] = preview
+    report["first_batch_sha256"] = hashlib.sha256(json.dumps(preview).encode()).hexdigest()
+    original_loader = torch.utils.data.DataLoader
+
+    def loader(dataset, *positional, **kwargs):
+        if Path(dataset.root).resolve() != data_root.resolve():
+            raise ValueError("Unexpected dataset; refusing to apply the training sampler")
+        kwargs.update(sampler=PolicySampler(), shuffle=False, drop_last=True)
+        return original_loader(dataset, *positional, **kwargs)
+
+    trainer.torch = Proxy(torch, utils=Proxy(torch.utils, data=Proxy(torch.utils.data, DataLoader=loader)))
+    original_update = trainer.update_policy
+    holder = {}
+
+    def first_sample(value):
+        if isinstance(value, torch.Tensor):
+            return value[:1].detach().clone() if value.ndim else value.detach().clone()
+        if isinstance(value, dict):
+            return {k: first_sample(v) for k, v in value.items()}
+        if isinstance(value, list):
+            return value[:1]
+        return value
+
+    original_processors = trainer.make_pre_post_processors
+
+    def make_processors(*positional, **kwargs):
+        pre, post = original_processors(*positional, **kwargs)
+        holder.update(preprocessor=pre, postprocessor=post)
+
+        def preprocess(batch):
+            validate_training_batch(batch, task_lookup)
+            if "raw_batch" not in holder:
+                observed = batch["index"].tolist()
+                if observed != preview:
+                    raise RuntimeError("The actual first batch differs from the policy's seeded sampler")
+                report["observed_first_batch_anchors"] = observed
+                holder["raw_batch"] = first_sample(batch)
+            result = pre(batch)
+            return mask_bank.attach(result, batch)
+
+        return CallableProxy(pre, call=preprocess), post
+
+    trainer.make_pre_post_processors = make_processors
+
+    def update(metrics, policy, batch, *positional, **kwargs):
+        if "policy" not in holder:
+            holder.update(policy=policy, accelerator=kwargs["accelerator"])
+            if cfg.get("wandb", {}).get("enable"):
+                import wandb
+                if wandb.run is not None:
+                    report.update(wandb_url=wandb.run.url, wandb_run_id=wandb.run.id)
+            report["parameters"] = sum(p.numel() for p in policy.parameters())
+            report["trainable_parameters"] = sum(p.numel() for p in policy.parameters() if p.requires_grad)
+            report["batch_shapes"] = {k: list(v.shape) for k, v in batch.items() if isinstance(v, torch.Tensor)}
+            if kwargs["accelerator"].gradient_accumulation_steps != 1:
+                raise RuntimeError("Gradient accumulation is prohibited")
+            report["optimizer_groups"] = [
+                {k: v for k, v in group.items() if k != "params"}
+                for group in positional[0].param_groups
+            ]
+            report["trainable_parameter_names"] = [n for n, p in policy.named_parameters() if p.requires_grad]
+            report["updates_started_unix"] = time.time()
+        torch.cuda.synchronize()
+        start = time.perf_counter()
+        if args.smoke_steps:
+            holder["current_mask"] = batch["action_mask"] if args.policy == "gr00t_n15" else batch["rwh_action_validity"]
+            if not holder.get("gradient_hook_installed"):
+                module = policy._groot_model.action_head.action_decoder if args.policy == "gr00t_n15" else policy.noise_predictor
+                def capture(module, inputs, output):
+                    if not output.requires_grad:
+                        return
+                    mask = holder["current_mask"].detach().bool()
+                    def check(grad):
+                        tail = grad[:, -mask.shape[1]:]
+                        if tail[~mask].abs().sum().item() != 0 or tail[mask].abs().sum().item() == 0:
+                            raise RuntimeError("Real policy dimension-mask gradient verification failed")
+                        report["masked_gradient_checks"] = report.get("masked_gradient_checks", 0) + 1
+                        return grad
+                    output.register_hook(check)
+                module.register_forward_hook(capture)
+                holder["gradient_hook_installed"] = True
+        result = original_update(metrics, policy, batch, *positional, **kwargs)
+        torch.cuda.synchronize()
+        measurement = {"step": len(report["measurements"]) + 1, "update_s": time.perf_counter() - start,
+                       "loss": float(result[0].loss.val), "grad_norm": float(result[0].grad_norm.val),
+                       "lr": float(result[0].lr.val),
+                       "dataloading_s": float(result[0].dataloading_s.val),
+                       "max_allocated_gib": torch.cuda.max_memory_allocated() / 2**30,
+                       "max_reserved_gib": torch.cuda.max_memory_reserved() / 2**30}
+        if not all(__import__("math").isfinite(measurement[k]) for k in ("loss", "grad_norm")):
+            raise RuntimeError("Nonfinite training loss or gradient norm")
+        report["measurements"].append(measurement)
+        report["last_update_finished_unix"] = time.time()
+        if args.smoke_steps or measurement["step"] == 1 or measurement["step"] % cfg["log_freq"] == 0:
+            write_json(report_path, report)
+            print("RWH_STEP", json.dumps(measurement), flush=True)
+        return result
+
+    trainer.update_policy = update
+    try:
+        started = time.perf_counter()
+        trainer.train()
+        report["training_wall_s"] = time.perf_counter() - started
+        policy = holder["accelerator"].unwrap_model(holder["policy"])
+        policy.eval()
+        raw_batch = holder["raw_batch"]
+        if args.policy == "multi_task_dit":
+            # Online inference supplies the current frame. select_action fills
+            # the two-observation history queues, repeating the first frame.
+            raw_batch = {k: v[:, -1] if k.startswith("observation.") and isinstance(v, torch.Tensor)
+                         and v.ndim >= 3 else v for k, v in raw_batch.items()}
+        inference_batch = holder["preprocessor"](raw_batch)
+        policy.reset()
+        with torch.inference_mode():
+            if args.policy == "multi_task_dit":
+                first_action = policy.select_action(inference_batch)
+                remaining = list(policy._queues["action"])
+                actions = torch.stack([first_action, *remaining], dim=1)
+            else:
+                actions = policy.predict_action_chunk(inference_batch)
+        if not torch.isfinite(actions).all() or actions.shape[-1] != 23:
+            raise RuntimeError(f"Invalid prediction shape/values: {actions.shape}")
+        report["prediction_shape"] = list(actions.shape)
+        physical_action = holder["postprocessor"](actions[:, 0])
+        if physical_action.shape != (1, 23) or not torch.isfinite(physical_action).all():
+            raise RuntimeError("Invalid postprocessed physical action")
+        report["postprocessed_action_shape"] = list(physical_action.shape)
+        checkpoint = Path(cfg["output_dir"]) / "checkpoints/last"
+        if not (checkpoint / "pretrained_model/train_config.json").is_file():
+            raise RuntimeError("Missing saved checkpoint training config")
+        from safetensors import safe_open
+        weights = sorted((checkpoint / "pretrained_model").rglob("*.safetensors"))
+        if not weights:
+            raise RuntimeError("Missing checkpoint weights")
+        tensors = 0
+        for weight in weights:
+            with safe_open(str(weight), framework="pt", device="cpu") as saved:
+                for key in saved.keys():
+                    if not torch.isfinite(saved.get_tensor(key)).all():
+                        raise RuntimeError(f"Nonfinite checkpoint tensor: {key}")
+                    tensors += 1
+        report["checkpoint_tensors_verified"] = tensors
+        report["checkpoint"] = str(checkpoint.resolve())
+        measurements = report["measurements"][2:]
+        report["median_update_s_after_warmup"] = statistics.median(x["update_s"] for x in measurements)
+        report["median_step_s_after_warmup"] = statistics.median(x["update_s"] + x["dataloading_s"] for x in measurements)
+        report["status"] = "passed"
+        report["completed_at"] = time.time()
+    except BaseException as exc:
+        report.update(status="failed", error_type=type(exc).__name__, error=str(exc)[-1800:])
+        write_json(report_path, report)
+        raise
+    finally:
+        write_json(report_path, report)
+        print("RWH_RESULT", report_path, report["status"], flush=True)
+
+
+if __name__ == "__main__":
+    main()

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/train/run_n17.py

@@ -0,0 +1,170 @@
+"""Configure and call LeTools' bundled N1.7 pipeline and HF Trainer."""
+import argparse
+import json
+import math
+import os
+from pathlib import Path
+import sys
+import time
+
+ROOT=Path(os.environ['RWH_TRAIN_CONFIG_DIR']).resolve()
+LETOOLS=ROOT.parents[3]/'LeTools-Learning'
+
+
+def write(path,value):
+    path=Path(path); path.parent.mkdir(parents=True,exist_ok=True)
+    tmp=path.with_suffix('.tmp');tmp.write_text(json.dumps(value,indent=2,ensure_ascii=False)+'\n');tmp.replace(path)
+
+
+def main():
+    parser=argparse.ArgumentParser()
+    parser.add_argument('--gpu',required=True,choices=['0','1'])
+    parser.add_argument('--smoke-steps',type=int,default=0)
+    args=parser.parse_args()
+    os.environ['CUDA_VISIBLE_DEVICES']=args.gpu
+    os.environ['NO_ALBUMENTATIONS_UPDATE']='1'
+    sys.path.insert(0,str(LETOOLS/'kuavo_model/external_models/gr00tn1d7'))
+    import yaml
+    import numpy as np
+    import torch
+    torch.set_num_threads(8)
+    if args.smoke_steps:
+        import faulthandler
+        faulthandler.dump_traceback_later(90, repeat=True)
+    from huggingface_hub import snapshot_download
+    from transformers import TrainerCallback
+    from scripts.data.view import DataView
+    from scripts.adapters import groot_v3
+    from gr00t.configs.base_config import get_default_config
+    from gr00t.configs.data.data_config import SingleDatasetConfig
+    import gr00t.experiment.experiment as experiment
+    from scripts.train.native_assets import pinned_cosmos_name
+
+    simple=yaml.safe_load((ROOT/'gr00t_n17.yaml').read_text())
+    view=DataView(ROOT/simple['dataset']['config'])
+    t,p=simple['training'],simple['policy']
+    if t['batch_size']!=128 or t['gradient_accumulation_steps']!=1 or t['resume']:
+        raise ValueError('Expected fresh actual batch128/accumulation1')
+    if p['chunk_size']!=40 or p['n_action_steps']!=40: raise ValueError('Expected 40 steps')
+    if int(os.environ.get('WORLD_SIZE','1'))!=1: raise ValueError('One GPU per model required')
+    output=Path(t['output_dir'])
+    if args.smoke_steps: output=output.parent/'_validation'/('gr00t_n17_'+time.strftime('%Y%m%d_%H%M%S'))
+    if output.exists(): raise FileExistsError(str(output))
+    log=Path(os.environ.get('RWH_RESULT_DIR',str(output.parent/'_runs/gr00t_n17')))
+    report={'status':'loading','policy':'gr00t_n17','gpu':int(args.gpu),'batch_size':128,
+            'gradient_accumulation_steps':1,'steps_requested':args.smoke_steps or t['steps'],
+            'smoke':bool(args.smoke_steps),'output_dir':str(output),'measurements':[],
+            'data_fingerprint':view.fingerprint,'source_revision':view.config['source']['revision'],
+            'runtime_root':str(LETOOLS/'kuavo_model/external_models/gr00tn1d7'),
+            'trainer_source':experiment.__file__}
+    result=log/'result.json';write(result,report);view.save_audit(log/'data')
+    base=snapshot_download(p['base_model_path'],revision=p['base_revision'],local_files_only=True)
+    cfg=get_default_config(); cfg.load_config_path=None
+    cfg.data.datasets=[SingleDatasetConfig(dataset_paths=[str(view.root)],mix_ratio=1.0,embodiment_tag='new_embodiment')]
+    cfg.data.modality_configs=groot_v3.modality(view,40)
+    cfg.data.seed=42;cfg.data.allow_padding=False;cfg.data.video_backend='torchcodec'
+    cfg.data.shard_size=t['shard_size'];cfg.data.episode_sampling_rate=t['episode_sampling_rate']
+    cfg.data.num_shards_per_epoch=t['num_shards_per_epoch'];cfg.data.download_cache=False
+    cfg.data.multiprocessing_context='fork'
+    cfg.model.model_name=pinned_cosmos_name()
+    cfg.model.action_horizon=40
+    cfg.model.tune_llm=False;cfg.model.tune_visual=False
+    cfg.model.tune_projector=True;cfg.model.tune_diffusion_model=True;cfg.model.tune_vlln=True
+    cfg.model.load_bf16=True;cfg.model.backbone_trainable_params_fp32=True
+    cfg.model.state_dropout_prob=p['state_dropout_prob']
+    cfg.model.use_relative_action=False
+    cfg.training.output_dir=str(output);cfg.training.experiment_name=None
+    cfg.training.start_from_checkpoint=base
+    cfg.training.global_batch_size=128;cfg.training.gradient_accumulation_steps=1
+    cfg.training.num_gpus=1;cfg.training.max_steps=args.smoke_steps or t['steps']
+    cfg.training.save_steps=args.smoke_steps or t['save_freq'];cfg.training.save_total_limit=3
+    cfg.training.learning_rate=t['optimizer']['lr'];cfg.training.optim='adamw_torch'
+    cfg.training.weight_decay=t['optimizer']['weight_decay']
+    cfg.training.max_grad_norm=t['optimizer']['grad_clip_norm']
+    cfg.training.warmup_ratio=t['scheduler']['warmup_ratio'];cfg.training.logging_steps=1 if args.smoke_steps else t['log_freq']
+    cfg.training.dataloader_num_workers=t['num_workers']
+    cfg.training.transformers_local_files_only=True
+    cfg.training.use_wandb=bool(simple['wandb']['enable'] and not args.smoke_steps)
+    cfg.training.wandb_project=simple['wandb']['project']
+    os.environ['WANDB_ENTITY']=simple['wandb']['entity']
+    groot_v3.install(view)
+    original=experiment.Gr00tTrainer
+    holder={}
+
+    class Progress(TrainerCallback):
+        def on_step_end(self,args,state,control,**kw):
+            loss=float(holder['last_loss'])
+            if not math.isfinite(loss): raise ValueError('Nonfinite loss')
+            report.update(status='running',last_step=state.global_step,heartbeat=time.time())
+            if state.global_step==1 or state.global_step%args.logging_steps==0 or report['smoke']:
+                row={'step':state.global_step,'loss':loss,'elapsed_s':time.time()-holder['started'],
+                     'max_allocated_gib':torch.cuda.max_memory_allocated()/2**30}
+                report['measurements'].append(row);write(result,report)
+                print('RWH_STEP',json.dumps(row),flush=True)
+        def on_log(self,args,state,control,logs=None,**kw):
+            if logs and 'grad_norm' in logs and not math.isfinite(float(logs['grad_norm'])):
+                raise ValueError('Nonfinite gradient norm')
+        def on_train_end(self,args,state,control,**kw):
+            from scripts.train.checkpoints import validate_native_checkpoint
+            path=Path(args.output_dir)/f'checkpoint-{state.global_step}'
+            validate_native_checkpoint(path,state.global_step)
+            write(path/'rwh_save_complete.json',{'step':state.global_step,'fingerprint':view.fingerprint})
+            report.update(checkpoint=str(path),saved_step=state.global_step);write(result,report)
+
+    def create_trainer(*a,**kw):
+        trainer=original(*a,**kw)
+        # The instance is the unmodified native Gr00tTrainer. Only add a standard
+        # progress callback and observational model hooks; no loss/update overrides.
+        trainer.add_callback(Progress());holder['trainer']=trainer
+        report.update(parameters=sum(p.numel() for p in trainer.model.parameters()),
+            trainable_parameters=sum(p.numel() for p in trainer.model.parameters() if p.requires_grad))
+        def before(model,inputs,kwargs):
+            b=kwargs.get('inputs',inputs[0] if inputs else None)
+            if b['action'].shape!=(128,40,132):raise ValueError(f"Invalid batch: {b['action'].shape}")
+            if b['action_mask'][...,23:].any():raise ValueError('Padding entered loss')
+            holder['mask']=b['action_mask']
+            if 'batch_shapes' not in report:
+                report['batch_shapes']={k:list(v.shape) for k,v in b.items() if torch.is_tensor(v)}
+                report['masked_target_elements']=int((~b['action_mask'][...,:23].bool()).sum())
+                write(result,report);print('RWH_NATIVE_FIRST_BATCH',report['batch_shapes'],flush=True)
+        def after(model,inputs,output):
+            holder['last_loss']=output['loss'].detach().float().cpu().item()
+        holder['before_handle']=trainer.model.register_forward_pre_hook(before,with_kwargs=True)
+        holder['after_handle']=trainer.model.register_forward_hook(after)
+        if args.smoke_steps:
+            def capture(module,inputs,output):
+                if not output.requires_grad:return
+                mask=holder['mask'].bool()
+                def check(grad):
+                    tail=grad[:,-mask.shape[1]:]
+                    if tail[~mask].abs().sum().item()!=0 or tail[mask].abs().sum().item()==0:
+                        raise ValueError('Native action-mask gradient check failed')
+                    report['masked_gradient_checks']=report.get('masked_gradient_checks',0)+1
+                    return grad
+                output.register_hook(check)
+            trainer.model.action_head.action_decoder.register_forward_hook(capture)
+        return trainer
+    experiment.Gr00tTrainer=create_trainer
+    try:
+        holder['started']=time.time();experiment.run(cfg)
+        holder['before_handle'].remove();holder['after_handle'].remove()
+        trainer=holder['trainer'];model=trainer.model;model.eval()
+        processor=trainer.train_dataset.processor;processor.eval()
+        ds=trainer.train_dataset.datasets[0]
+        i=next(i for i,s in enumerate(view.segments) if s['end']-s['start']>=40)
+        df=ds.episode_loader[i]
+        sample=ds.get_datapoint(df,0)
+        one=processor.collator([sample])['inputs'];one.pop('action',None);one.pop('action_mask',None)
+        one={k:v.to(model.device) for k,v in one.items()}
+        with torch.inference_mode(),torch.autocast('cuda',dtype=torch.bfloat16):
+            predicted=model.get_action(one)['action_pred']
+        if predicted.shape!=(1,40,132) or not torch.isfinite(predicted).all():raise ValueError('Invalid prediction')
+        report.update(status='passed',prediction_shape=list(predicted.shape),completed_at=time.time())
+    except BaseException as exc:
+        report.update(status='failed',error_type=type(exc).__name__,error=str(exc)[-2500:]);raise
+    finally:
+        write(result,report)
+        if args.smoke_steps:faulthandler.cancel_dump_traceback_later()
+
+
+if __name__=='__main__': main()

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/train/start_watchers.py

@@ -0,0 +1,26 @@
+"""Start one controller and one HF watcher, detached from the SSH session."""
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+
+def main():
+    root=Path(__file__).resolve().parents[2]
+    config=root/'configs/Bs128_30k/train'
+    out=Path('/home/work/mntvol/RwH-Kuavo_ckpt/v3_rawaction')
+    runs=out/'_runs';runs.mkdir(parents=True,exist_ok=True)
+    env=os.environ.copy();env['RWH_TRAIN_CONFIG_DIR']=str(config)
+    env['PYTHONPATH']=str(root)+':'+str(root.parent/'LeTools-Learning/third_party/lerobot/src')
+    jobs={}
+    for name,module,args in [
+        ('controller','scripts.train.orchestrate',[]),
+        ('hf_watcher','scripts.train.hf_watcher',['--config',str(config/'hf_watcher.json')])]:
+        log=(runs/(name+'.log')).open('a')
+        child=subprocess.Popen([sys.executable,'-u','-m',module,*args],env=env,cwd=root,
+                  stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
+        jobs[name]={'pid':child.pid,'log':str(runs/(name+'.log'))}
+    (runs/'watcher_processes.json').write_text(json.dumps(jobs,indent=2)+'\n')
+    print(json.dumps(jobs,indent=2))
+
+if __name__=='__main__':main()

v2/multi_task_dit/checkpoints/checkpoint-10K/provenance/scripts/train/validate_checkpoint.py

@@ -0,0 +1,95 @@
+"""Reload a real saved checkpoint in a separate process and predict physical actions."""
+import argparse
+import json
+import os
+from pathlib import Path
+import sys
+
+
+def main():
+    ap=argparse.ArgumentParser();ap.add_argument('--policy',required=True);ap.add_argument('--checkpoint',required=True)
+    ap.add_argument('--gpu',required=True);ap.add_argument('--data-config',required=True);ap.add_argument('--output',required=True)
+    args=ap.parse_args();os.environ['CUDA_VISIBLE_DEVICES']=args.gpu
+    import torch
+    torch.set_num_threads(8)
+    import numpy as np
+    from scripts.data.view import DataView,lerobot_dataset
+    view=DataView(args.data_config);path=Path(args.checkpoint)
+    if args.policy=='gr00t_n17':
+        project=Path(__file__).resolve().parents[2]
+        sys.path.insert(0,str(project.parent/'LeTools-Learning/kuavo_model/external_models/gr00tn1d7'))
+        from scripts.train.native_assets import local_cosmos
+        from scripts.adapters.groot_v3 import install
+        from gr00t.model.gr00t_n1d7.gr00t_n1d7 import Gr00tN1d7
+        from gr00t.configs.model.gr00t_n1d7 import Gr00tN1d7Config
+        from gr00t.model.gr00t_n1d7.processing_gr00t_n1d7 import Gr00tN1d7Processor
+        from gr00t.data.dataset.sharded_single_step_dataset import ShardedSingleStepDataset
+        from gr00t.data.embodiment_tags import EmbodimentTag
+        cfg=Gr00tN1d7Config.from_pretrained(path);cfg.model_name=local_cosmos()
+        model,info=Gr00tN1d7.from_pretrained(path,config=cfg,output_loading_info=True,
+            transformers_loading_kwargs={'trust_remote_code':True,'local_files_only':True})
+        assert not any(info.get(k) for k in ['missing_keys','unexpected_keys','mismatched_keys'])
+        model=model.to('cuda').eval()
+        proc=Gr00tN1d7Processor.from_pretrained(path,model_name=local_cosmos(),
+                    transformers_loading_kwargs={'trust_remote_code':True,'local_files_only':True})
+        proc.eval();install(view)
+        ds=ShardedSingleStepDataset(dataset_path=view.root,embodiment_tag=EmbodimentTag.NEW_EMBODIMENT,
+             modality_configs=proc.modality_configs['new_embodiment'],video_backend='torchcodec',shard_size=256)
+        ds.set_processor(proc)
+        i=next(i for i,s in enumerate(view.segments) if s['end']-s['start']>=40)
+        sample=ds.get_datapoint(ds.episode_loader[i],0)
+        masks_checked=0
+        anchors=view.anchors('gr00t_n17')
+        seq=anchors[:,None]+np.arange(40)[None,:]
+        bad=anchors[(~view.validity[seq]).any(axis=(1,2))]
+        for anchor in [int(anchors[0]),int(anchors[len(anchors)//2]),int(anchors[-1]),*map(int,bad[:3])]:
+            ep=int(view.episode[anchor]);segment=view.segments[ep]
+            df=ds.episode_loader[ep];encoded=ds.get_datapoint(df,anchor-segment['start'])
+            expected=view.raw['validity.action'][view.rows[anchor:anchor+40],:23]
+            assert np.array_equal(encoded['action_mask'][:,:23].numpy(),expected)
+            assert not encoded['action_mask'][:,23:].any()
+            lo=np.asarray(view.stats['action']['q01']);hi=np.asarray(view.stats['action']['q99'])
+            delta=hi-lo;good=~np.isclose(hi,lo)
+            normalized=np.zeros((40,23));normalized[:,good]=2*(view.action[anchor:anchor+40,good]-lo[good])/delta[good]-1
+            normalized=np.clip(normalized,-1,1)
+            np.testing.assert_allclose(encoded['action'][:,:23].numpy(),normalized,rtol=1e-5,atol=1e-5)
+            masks_checked+=int((~expected).sum())
+        assert masks_checked>0,'No real missing physical target was tested'
+        inp=proc.collator([sample])['inputs'];inp.pop('action');inp.pop('action_mask')
+        inp={k:v.cuda() for k,v in inp.items()}
+        with torch.inference_mode(),torch.autocast('cuda',dtype=torch.bfloat16): pred=model.get_action(inp)['action_pred']
+        assert pred.shape==(1,40,132) and torch.isfinite(pred).all()
+        # Native decoder preserves the learned physical group ordering.
+        physical=proc.decode_action(pred.float().cpu().numpy(),EmbodimentTag.NEW_EMBODIMENT,
+                                    state={'body':view.state[view.segments[i]['start']:view.segments[i]['start']+1][None]})
+        action=physical['body']
+        assert action.shape==(1,40,23) and np.isfinite(action).all()
+    else:
+        from scripts.adapters.language import install
+        install(args.policy)
+        from lerobot.policies.factory import get_policy_class,make_pre_post_processors
+        from lerobot.configs import PreTrainedConfig
+        cfg=PreTrainedConfig.from_pretrained(path)
+        model=get_policy_class(cfg.type).from_pretrained(path).to('cuda').eval()
+        pre,post=make_pre_post_processors(cfg,pretrained_path=str(path))
+        dataset=lerobot_dataset(view,args.policy)
+        sample=dataset[int(view.anchors(args.policy)[0])]
+        if args.policy=='multi_task_dit':
+            for k in list(sample):
+                if k.startswith('observation.'):sample[k]=sample[k][-1]
+        raw={k:v.unsqueeze(0) if torch.is_tensor(v) else [v] for k,v in sample.items()}
+        for k in view.cameras:raw[k]=raw[k].float()/255
+        batch=pre(raw);model.reset()
+        with torch.inference_mode():
+            if args.policy=='multi_task_dit':
+                first=model.select_action(batch);pred=torch.stack([first,*list(model._queues['action'])],dim=1)
+            else:pred=model.predict_action_chunk(batch)
+        assert torch.isfinite(pred).all() and pred.shape[-1]==23
+        action=post(pred[:,0]);assert action.shape==(1,23) and torch.isfinite(action).all()
+    report={'status':'passed','policy':args.policy,'checkpoint':str(path),'separate_process_reload':True,
+            'prediction_shape':list(pred.shape),'physical_action_shape':list(action.shape)}
+    if args.policy=='gr00t_n17':report.update(real_missing_target_elements_checked=masks_checked,native_normalization_matches_source=True)
+    Path(args.output).write_text(json.dumps(report,indent=2)+'\n');print(json.dumps(report))
+
+
+if __name__=='__main__':main()